Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 copilot-swe-agent[bot]
|
e2837c5543 |
@@ -0,0 +1,10 @@
|
||||
version = 1
|
||||
|
||||
[[analyzers]]
|
||||
name = "python"
|
||||
|
||||
[analyzers.meta]
|
||||
runtime_version = "3.x.x"
|
||||
|
||||
[[analyzers]]
|
||||
name = "javascript"
|
||||
@@ -5,137 +5,6 @@ REDIS_URL=redis://redis:6379/0
|
||||
EXTERNAL_HOSTNAME=docuelevate.example.com
|
||||
GOTENBERG_URL=http://gotenberg:3000
|
||||
ALLOW_FILE_DELETE=true # Allow deletion of file records
|
||||
COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2)
|
||||
|
||||
# **UI / Appearance**
|
||||
# Default colour scheme: system (follow OS), light, or dark
|
||||
# Individual users can always override with the navbar dark-mode toggle.
|
||||
# UI_DEFAULT_COLOR_SCHEME=system
|
||||
|
||||
# **Batch Processing Settings**
|
||||
# Control throttling behavior for the /processall endpoint to prevent overwhelming downstream APIs
|
||||
PROCESSALL_THROTTLE_THRESHOLD=20 # Number of files above which throttling is applied (default: 20)
|
||||
PROCESSALL_THROTTLE_DELAY=3 # Delay in seconds between each task submission when throttling (default: 3)
|
||||
|
||||
# **Task Retry Settings**
|
||||
# Failed tasks are automatically retried with exponential backoff and jitter.
|
||||
# TASK_RETRY_MAX_RETRIES=3 # Max retry attempts per task (default: 3)
|
||||
# TASK_RETRY_DELAYS=60,300,900 # Countdown (seconds) before each retry; 1 min, 5 min, 15 min
|
||||
# TASK_RETRY_JITTER=true # Add ±20% random jitter to prevent thundering-herd (default: true)
|
||||
|
||||
# **Client-Side Upload Throttling**
|
||||
# Controls pacing when the browser uploads files (especially large directory drops).
|
||||
# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly.
|
||||
UPLOAD_CONCURRENCY=3 # Max simultaneous uploads from the browser (default: 3)
|
||||
UPLOAD_QUEUE_DELAY_MS=500 # Delay (ms) between starting each upload slot (default: 500)
|
||||
|
||||
# **File Upload Size Limits** (Security - see SECURITY_AUDIT.md)
|
||||
# Maximum file upload size in bytes. Default: 1GB (1073741824 bytes)
|
||||
# Prevents resource exhaustion attacks. Adjust based on your server capacity.
|
||||
MAX_UPLOAD_SIZE=1073741824
|
||||
|
||||
# Maximum size for a single file chunk in bytes (optional)
|
||||
# If set and a file exceeds this size, it will be split into smaller chunks for processing
|
||||
# Default: None (no splitting). Example: 104857600 for 100MB chunks
|
||||
# MAX_SINGLE_FILE_SIZE=104857600
|
||||
|
||||
# **Request Body Size Limit** (Security - see SECURITY_AUDIT.md)
|
||||
# Maximum request body size in bytes for non-file-upload requests (JSON, form data, etc.).
|
||||
# Default: 1MB (1048576 bytes). File uploads are governed by MAX_UPLOAD_SIZE above.
|
||||
# Prevents memory exhaustion from oversized JSON/form payloads.
|
||||
# MAX_REQUEST_BODY_SIZE=1048576
|
||||
|
||||
# **Security Headers** (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
|
||||
# Disabled by default since most deployments use a reverse proxy (Traefik, Nginx, etc.)
|
||||
# that already adds these headers. Set to true only if deploying directly without a reverse proxy.
|
||||
# SECURITY_HEADERS_ENABLED=false
|
||||
|
||||
# If you enable security headers, you can also configure individual headers:
|
||||
# Strict-Transport-Security (HSTS) - Forces HTTPS connections
|
||||
# Only effective when served over HTTPS. Disable if not using HTTPS or if proxy adds this header
|
||||
# SECURITY_HEADER_HSTS_ENABLED=true
|
||||
# SECURITY_HEADER_HSTS_VALUE="max-age=31536000; includeSubDomains"
|
||||
|
||||
# Content-Security-Policy (CSP) - Controls resource loading
|
||||
# Customize based on your application's resource loading needs
|
||||
# Default allows self-hosted resources, inline scripts/styles, and external images
|
||||
# SECURITY_HEADER_CSP_ENABLED=true
|
||||
# SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
|
||||
# X-Frame-Options - Prevents clickjacking attacks
|
||||
# Options: DENY (no framing), SAMEORIGIN (same origin framing only), ALLOW-FROM uri
|
||||
# SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||||
# SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="DENY"
|
||||
|
||||
# X-Content-Type-Options - Prevents MIME sniffing
|
||||
# Always set to 'nosniff' when enabled
|
||||
# SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
|
||||
# **CORS (Cross-Origin Resource Sharing)** (see SECURITY_AUDIT.md – Infrastructure Security)
|
||||
# Disabled by default: most deployments rely on a reverse proxy (Traefik, Nginx, etc.) to inject
|
||||
# CORS headers. Set CORS_ENABLED=true only if DocuElevate is exposed directly without a proxy,
|
||||
# or if your proxy does not handle CORS. When enabled, only list the exact origins that need access.
|
||||
#
|
||||
# Rationale for reverse-proxy-first approach:
|
||||
# Traefik/Nginx already set Access-Control-Allow-Origin (and related headers) for every response,
|
||||
# so adding the middleware here would duplicate headers. When this flag is False the application
|
||||
# trusts the proxy layer to enforce CORS policy; set it to True for standalone / direct-access
|
||||
# deployments only.
|
||||
#
|
||||
# CORS_ENABLED=false
|
||||
#
|
||||
# Comma-separated list of allowed origins (use * to allow all - not recommended with credentials)
|
||||
# CORS_ALLOWED_ORIGINS=https://app.example.com,https://admin.example.com
|
||||
#
|
||||
# Allow cookies / Authorization headers in cross-origin requests
|
||||
# Must be False when CORS_ALLOWED_ORIGINS=* (browser security requirement)
|
||||
# CORS_ALLOW_CREDENTIALS=false
|
||||
#
|
||||
# Allowed HTTP methods (comma-separated)
|
||||
# CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,OPTIONS,PATCH
|
||||
#
|
||||
# Allowed request headers (use * to allow all)
|
||||
# CORS_ALLOWED_HEADERS=*
|
||||
|
||||
# **Audit Logging & SIEM Integration** (see docs/ConfigurationGuide.md#audit-logging)
|
||||
# Enable HTTP request audit logging middleware
|
||||
AUDIT_LOGGING_ENABLED=true
|
||||
# Include client IP in audit log entries (disable for GDPR-sensitive deployments)
|
||||
AUDIT_LOG_INCLUDE_CLIENT_IP=true
|
||||
|
||||
# Forward audit events to an external SIEM system (Syslog, Splunk, Logstash, Grafana, etc.)
|
||||
# AUDIT_SIEM_ENABLED=false
|
||||
# AUDIT_SIEM_TRANSPORT=syslog # syslog | http
|
||||
# AUDIT_SIEM_SYSLOG_HOST=localhost
|
||||
# AUDIT_SIEM_SYSLOG_PORT=514
|
||||
# AUDIT_SIEM_SYSLOG_PROTOCOL=udp # udp | tcp
|
||||
# AUDIT_SIEM_HTTP_URL= # e.g. https://splunk:8088/services/collector/event
|
||||
# AUDIT_SIEM_HTTP_TOKEN= # Bearer / HEC token
|
||||
# AUDIT_SIEM_HTTP_CUSTOM_HEADERS= # Comma-separated Key:Value pairs
|
||||
|
||||
# **Rate Limiting** (see SECURITY_AUDIT.md and docs/API.md)
|
||||
# Protects against DoS attacks and API abuse by limiting request rates per IP/user
|
||||
# Enabled by default - highly recommended for production
|
||||
RATE_LIMITING_ENABLED=true
|
||||
|
||||
# Default rate limit for all API endpoints (format: count/period)
|
||||
# Periods can be: second, minute, hour, day
|
||||
# Default: 100 requests per minute per IP/user
|
||||
RATE_LIMIT_DEFAULT=100/minute
|
||||
|
||||
# Rate limit for file upload endpoints
|
||||
# Allows faster uploads while still preventing abuse
|
||||
# Default: 600 uploads per minute per IP/user
|
||||
RATE_LIMIT_UPLOAD=600/minute
|
||||
|
||||
# Rate limit for authentication endpoints
|
||||
# Strict limit to prevent brute force attacks
|
||||
# Default: 10 attempts per minute per IP
|
||||
RATE_LIMIT_AUTH=10/minute
|
||||
|
||||
# Note: Processing endpoints (OCR, metadata extraction) use built-in queue throttling
|
||||
# via Celery task queue to control processing rates and prevent upstream API overloads.
|
||||
# No additional API-level rate limit is needed for processing endpoints.
|
||||
|
||||
# **Authentication**
|
||||
AUTH_ENABLED=true
|
||||
@@ -144,30 +13,6 @@ AUTH_ENABLED=true
|
||||
SESSION_SECRET=b39fd43f68d0491ca942f28a16e484b1e763fe9accf4445ca2669a5f3b179eb4
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=your_secure_password
|
||||
ADMIN_GROUP_NAME=admin
|
||||
|
||||
# **Multi-User Mode**
|
||||
# When enabled, each user has their own document space with isolated uploads,
|
||||
# search, and file management. Requires AUTH_ENABLED=true.
|
||||
MULTI_USER_ENABLED=false
|
||||
# Allow users to self-register with an email address and password.
|
||||
# Set to true to enable the /signup page. Requires MULTI_USER_ENABLED=true.
|
||||
# When SMTP is configured, a verification email is sent before the account is activated.
|
||||
# Without SMTP, accounts are activated immediately upon registration.
|
||||
# ALLOW_LOCAL_SIGNUP=false
|
||||
# Default upload limit per user per day (0 = unlimited)
|
||||
DEFAULT_DAILY_UPLOAD_LIMIT=0
|
||||
# Show unowned documents (owner_id=NULL) to all users (true) or only admins (false)
|
||||
UNOWNED_DOCS_VISIBLE_TO_ALL=true
|
||||
# Auto-assign this owner ID to documents ingested without a session (e.g. IMAP, API)
|
||||
# Leave empty/unset to keep them unowned until claimed.
|
||||
# DEFAULT_OWNER_ID=
|
||||
|
||||
# **Subscription / Quota Settings**
|
||||
# Soft-limit overage buffer in percent (0–200). Announced quota is multiplied by (1 + percent/100)
|
||||
# for actual enforcement. E.g. 20 means a 150-doc/month plan enforces at 180. 0 = enforce exactly.
|
||||
# Per-plan overage_percent set in the Plan Designer overrides this global default.
|
||||
# SUBSCRIPTION_OVERAGE_PERCENT=20
|
||||
|
||||
# **OpenID Connect/Authentik Settings**
|
||||
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
|
||||
@@ -175,148 +20,27 @@ AUTHENTIK_CLIENT_SECRET=<yourAuthentikClientSecret>
|
||||
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docuelevate/.well-known/openid-configuration>
|
||||
OAUTH_PROVIDER_NAME="Authentik SSO"
|
||||
|
||||
# **Social Login Providers**
|
||||
# Enable one or more social login providers to let users sign in with existing accounts.
|
||||
# Each provider requires separate OAuth credentials. See docs/SocialLoginSetup.md for details.
|
||||
|
||||
# Google Sign-In (https://console.cloud.google.com/apis/credentials)
|
||||
# SOCIAL_AUTH_GOOGLE_ENABLED=false
|
||||
# SOCIAL_AUTH_GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com
|
||||
# SOCIAL_AUTH_GOOGLE_CLIENT_SECRET=your-google-client-secret
|
||||
|
||||
# Microsoft Sign-In / Azure AD (https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps)
|
||||
# SOCIAL_AUTH_MICROSOFT_ENABLED=false
|
||||
# SOCIAL_AUTH_MICROSOFT_CLIENT_ID=your-microsoft-application-id
|
||||
# SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=your-microsoft-client-secret
|
||||
# SOCIAL_AUTH_MICROSOFT_TENANT=common # common | organizations | consumers | <tenant-id>
|
||||
|
||||
# Apple Sign-In (https://developer.apple.com/account/resources)
|
||||
# SOCIAL_AUTH_APPLE_ENABLED=false
|
||||
# SOCIAL_AUTH_APPLE_CLIENT_ID=com.example.docuelevate
|
||||
# SOCIAL_AUTH_APPLE_TEAM_ID=ABCDE12345
|
||||
# SOCIAL_AUTH_APPLE_KEY_ID=FGHIJ67890
|
||||
# SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
|
||||
|
||||
# Dropbox Sign-In (https://www.dropbox.com/developers/apps)
|
||||
# SOCIAL_AUTH_DROPBOX_ENABLED=false
|
||||
# SOCIAL_AUTH_DROPBOX_CLIENT_ID=your-dropbox-app-key
|
||||
# SOCIAL_AUTH_DROPBOX_CLIENT_SECRET=your-dropbox-app-secret
|
||||
|
||||
# **AI/ML Services**
|
||||
# Select your AI provider: openai | azure | anthropic | gemini | ollama | openrouter | portkey | litellm
|
||||
AI_PROVIDER=openai
|
||||
# Model override (optional – falls back to OPENAI_MODEL when not set)
|
||||
# AI_MODEL=gpt-4o-mini
|
||||
|
||||
# --- OpenAI (AI_PROVIDER=openai) ---
|
||||
# OpenAI
|
||||
OPENAI_API_KEY="<OPENAI_API_KEY>"
|
||||
OPENAI_BASE_URL=https://api.openai.com/v1
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
|
||||
# --- Anthropic Claude (AI_PROVIDER=anthropic) ---
|
||||
# ANTHROPIC_API_KEY=sk-ant-...
|
||||
# AI_MODEL=claude-3-5-sonnet-20241022
|
||||
# Azure AI
|
||||
AZURE_REGION="eastus"
|
||||
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
|
||||
AZURE_AI_KEY=<AZURE_AI_KEY>
|
||||
|
||||
# --- Google Gemini (AI_PROVIDER=gemini) ---
|
||||
# GEMINI_API_KEY=AIza...
|
||||
# AI_MODEL=gemini-1.5-pro
|
||||
|
||||
# --- Ollama local LLMs (AI_PROVIDER=ollama) ---
|
||||
# OLLAMA_BASE_URL=http://localhost:11434
|
||||
# AI_MODEL=llama3.2
|
||||
|
||||
# --- OpenRouter (AI_PROVIDER=openrouter) ---
|
||||
# OPENROUTER_API_KEY=sk-or-...
|
||||
# AI_MODEL=anthropic/claude-3.5-sonnet
|
||||
|
||||
# --- Portkey AI Gateway (AI_PROVIDER=portkey) ---
|
||||
# PORTKEY_API_KEY=pk-...
|
||||
# PORTKEY_VIRTUAL_KEY=vk-... # optional – routes to provider credentials in Portkey vault
|
||||
# PORTKEY_CONFIG=pc-... # optional – saved Config ID for fallbacks / load balancing
|
||||
|
||||
# --- Azure OpenAI (AI_PROVIDER=azure) ---
|
||||
# OPENAI_API_KEY=<azure-key>
|
||||
# OPENAI_BASE_URL=https://my-resource.openai.azure.com
|
||||
# AZURE_OPENAI_API_VERSION=2024-02-01
|
||||
# AI_MODEL=gpt-4o # deployment name in Azure
|
||||
|
||||
# Azure Document Intelligence (OCR – separate from AI provider above)
|
||||
# **Email Settings (shared SMTP – password reset, verification, and system notifications)**
|
||||
# **Email Settings**
|
||||
EMAIL_HOST=smtp.example.com
|
||||
EMAIL_PORT=587
|
||||
EMAIL_USERNAME=docuelevate@example.com
|
||||
EMAIL_PASSWORD=your_secure_email_password
|
||||
EMAIL_USE_TLS=True
|
||||
EMAIL_SENDER=DocuElevate System <docuelevate@example.com>
|
||||
# EMAIL_DEFAULT_RECIPIENT is not used for document delivery (see DEST_EMAIL_* below)
|
||||
|
||||
# **Email Destination Settings (dedicated SMTP for document delivery)**
|
||||
# These settings are intentionally separate from the shared EMAIL_* settings above.
|
||||
# Configuring EMAIL_HOST for password reset / notifications does NOT automatically
|
||||
# enable the email destination – you must set DEST_EMAIL_HOST to activate it.
|
||||
# DEST_EMAIL_ENABLED=true # Set to false to disable email delivery without removing credentials
|
||||
DEST_EMAIL_HOST=smtp.example.com
|
||||
DEST_EMAIL_PORT=587
|
||||
DEST_EMAIL_USERNAME=docuelevate@example.com
|
||||
DEST_EMAIL_PASSWORD=your_secure_email_password
|
||||
DEST_EMAIL_USE_TLS=True
|
||||
DEST_EMAIL_SENDER=DocuElevate Delivery <docuelevate@example.com>
|
||||
DEST_EMAIL_DEFAULT_RECIPIENT=recipient@example.com
|
||||
|
||||
# **Watch Folder Ingestion**
|
||||
# DocuElevate can automatically monitor directories (local, FTP, SFTP, and cloud providers) for new files.
|
||||
#
|
||||
# Local watch folders — works with any mounted path (SMB/CIFS, NFS, local disk, etc.)
|
||||
# Set WATCH_FOLDERS to a comma-separated list of absolute paths inside the container.
|
||||
WATCH_FOLDERS=
|
||||
WATCH_FOLDER_POLL_INTERVAL=1
|
||||
WATCH_FOLDER_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# FTP ingest — poll an FTP directory for new files (uses FTP connection settings above)
|
||||
FTP_INGEST_ENABLED=false
|
||||
FTP_INGEST_FOLDER=
|
||||
FTP_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# SFTP ingest — poll an SFTP directory for new files (uses SFTP connection settings above)
|
||||
SFTP_INGEST_ENABLED=false
|
||||
SFTP_INGEST_FOLDER=
|
||||
SFTP_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# Dropbox ingest — poll a Dropbox folder (uses Dropbox OAuth credentials above)
|
||||
DROPBOX_INGEST_ENABLED=false
|
||||
DROPBOX_INGEST_FOLDER=
|
||||
DROPBOX_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# Google Drive ingest — poll a Google Drive folder (uses Google Drive credentials above)
|
||||
GOOGLE_DRIVE_INGEST_ENABLED=false
|
||||
GOOGLE_DRIVE_INGEST_FOLDER_ID=
|
||||
GOOGLE_DRIVE_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# OneDrive ingest — poll a OneDrive folder (uses OneDrive MSAL credentials above)
|
||||
ONEDRIVE_INGEST_ENABLED=false
|
||||
ONEDRIVE_INGEST_FOLDER_PATH=
|
||||
ONEDRIVE_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# Nextcloud ingest — poll a Nextcloud folder (uses Nextcloud WebDAV credentials above)
|
||||
NEXTCLOUD_INGEST_ENABLED=false
|
||||
NEXTCLOUD_INGEST_FOLDER=
|
||||
NEXTCLOUD_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# Amazon S3 ingest — poll an S3 prefix (uses S3/AWS credentials above)
|
||||
S3_INGEST_ENABLED=false
|
||||
S3_INGEST_PREFIX=
|
||||
S3_INGEST_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# WebDAV ingest — poll a WebDAV folder (uses WebDAV credentials above)
|
||||
WEBDAV_INGEST_ENABLED=false
|
||||
WEBDAV_INGEST_FOLDER=
|
||||
WEBDAV_INGEST_DELETE_AFTER_PROCESS=false
|
||||
EMAIL_DEFAULT_RECIPIENT=recipient@example.com
|
||||
|
||||
# **IMAP Settings**
|
||||
# DocuElevate polls these mailboxes for new email attachments and automatically ingests them.
|
||||
# No manual forwarding required — DocuElevate acts as an IMAP *client*.
|
||||
# For HP Scanners / Scan-to-Email: configure the scanner to send to a dedicated mailbox,
|
||||
# then point DocuElevate at that mailbox using the settings below.
|
||||
IMAP1_HOST=mail.example.com
|
||||
IMAP1_PORT=993
|
||||
IMAP1_USERNAME=<IMAP1_USERNAME>
|
||||
@@ -333,21 +57,8 @@ IMAP2_SSL=true
|
||||
IMAP2_POLL_INTERVAL_MINUTES=10
|
||||
IMAP2_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# IMAP Readonly Mode (Feature Flag)
|
||||
# When true, IMAP processing will fetch and process attachments but will NOT modify
|
||||
# the mailbox state (no starring, labeling, deleting, or flag changes).
|
||||
# Use for pre-production instances that share a mailbox with production.
|
||||
IMAP_READONLY_MODE=false
|
||||
|
||||
# Controls which attachment types are ingested from IMAP emails.
|
||||
# 'documents_only' (default) – PDFs and office files only; images are skipped.
|
||||
# 'all' – all supported file types including images.
|
||||
# Per-user IMAP accounts can override this global default.
|
||||
IMAP_ATTACHMENT_FILTER=documents_only
|
||||
|
||||
# **Storage/Document Services**
|
||||
# Amazon S3
|
||||
# S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials
|
||||
AWS_REGION=us-east-1
|
||||
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
@@ -357,38 +68,22 @@ S3_STORAGE_CLASS=STANDARD
|
||||
S3_ACL=private
|
||||
|
||||
# NextCloud
|
||||
# NEXTCLOUD_ENABLED=true # Set to false to disable NextCloud uploads without removing credentials
|
||||
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
||||
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
||||
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
||||
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
||||
|
||||
# Paperless-ngx
|
||||
# PAPERLESS_ENABLED=true # Set to false to disable Paperless uploads without removing credentials
|
||||
PAPERLESS_HOST=https://paperless.example.com
|
||||
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
||||
# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
|
||||
# If set, the extracted sender information will be automatically set as a custom field in Paperless
|
||||
# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
|
||||
# PAPERLESS_CUSTOM_FIELD_ABSENDER=
|
||||
|
||||
# Optional: JSON mapping of metadata fields to Paperless custom field names
|
||||
# This allows you to map multiple extracted metadata fields to custom fields in Paperless
|
||||
# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...}
|
||||
# Available metadata fields: absender, empfaenger, correspondent, document_type, language,
|
||||
# kommunikationsart, kommunikationskategorie, reference_number, etc.
|
||||
# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
|
||||
# PAPERLESS_CUSTOM_FIELDS_MAPPING=
|
||||
|
||||
# Dropbox
|
||||
# DROPBOX_ENABLED=true # Set to false to disable Dropbox uploads without removing credentials
|
||||
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
||||
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
||||
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
||||
DROPBOX_FOLDER="/Documents/Uploads"
|
||||
|
||||
# Google Drive
|
||||
# GOOGLE_DRIVE_ENABLED=true # Set to false to disable Google Drive uploads without removing credentials
|
||||
# Service Account Method:
|
||||
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
|
||||
GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
|
||||
@@ -401,7 +96,6 @@ GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret # Required for OAuth method
|
||||
GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token # Required for OAuth method
|
||||
|
||||
# OneDrive
|
||||
# ONEDRIVE_ENABLED=true # Set to false to disable OneDrive uploads without removing credentials
|
||||
ONEDRIVE_CLIENT_ID=your-client-id
|
||||
ONEDRIVE_CLIENT_SECRET=your-client-secret
|
||||
ONEDRIVE_TENANT_ID=common
|
||||
@@ -409,7 +103,6 @@ ONEDRIVE_REFRESH_TOKEN=your-refresh-token
|
||||
ONEDRIVE_FOLDER_PATH=Documents/Uploads
|
||||
|
||||
# WebDAV
|
||||
# WEBDAV_ENABLED=true # Set to false to disable WebDAV uploads without removing credentials
|
||||
WEBDAV_URL=https://webdav.example.com/path
|
||||
WEBDAV_USERNAME=webdav_user
|
||||
WEBDAV_PASSWORD=your_secure_webdav_password
|
||||
@@ -417,9 +110,6 @@ WEBDAV_FOLDER=/Documents/Uploads
|
||||
WEBDAV_VERIFY_SSL=True
|
||||
|
||||
# FTP
|
||||
# FTP_ENABLED=true # Set to false to disable FTP uploads without removing credentials
|
||||
# Security Note: FTP_USE_TLS=True is strongly recommended for secure connections
|
||||
# Set FTP_ALLOW_PLAINTEXT=False in production to prevent unencrypted FTP
|
||||
FTP_HOST=ftp.example.com
|
||||
FTP_PORT=21
|
||||
FTP_USERNAME=ftp_user
|
||||
@@ -429,10 +119,6 @@ FTP_USE_TLS=True
|
||||
FTP_ALLOW_PLAINTEXT=True
|
||||
|
||||
# SFTP
|
||||
# SFTP_ENABLED=true # Set to false to disable SFTP uploads without removing credentials
|
||||
# Security Note: Host key verification is enabled by default (False)
|
||||
# Only set to True in development/testing environments if needed
|
||||
# When false, configure SSH known_hosts for proper host key verification
|
||||
SFTP_HOST=sftp.example.com
|
||||
SFTP_PORT=22
|
||||
SFTP_USERNAME=sftp_user
|
||||
@@ -440,21 +126,6 @@ SFTP_PASSWORD=your_secure_sftp_password
|
||||
# SFTP_PRIVATE_KEY=/path/to/private_key.pem
|
||||
# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
|
||||
SFTP_FOLDER=/Documents/Uploads
|
||||
SFTP_DISABLE_HOST_KEY_VERIFICATION=False # Default is False (secure); set to True only for testing
|
||||
|
||||
# iCloud Drive
|
||||
# ICLOUD_ENABLED=true # Set to false to disable iCloud uploads without removing credentials
|
||||
# Requires an Apple ID with iCloud Drive enabled.
|
||||
# For accounts with two-factor authentication (most accounts), generate an
|
||||
# app-specific password at https://appleid.apple.com/account/manage
|
||||
ICLOUD_USERNAME=your_apple_id@example.com
|
||||
ICLOUD_PASSWORD=your-app-specific-password
|
||||
ICLOUD_FOLDER=Documents/Uploads
|
||||
# ICLOUD_COOKIE_DIRECTORY=/path/to/cookie/dir # Optional: defaults to ~/.pyicloud
|
||||
|
||||
# **HTTP Request Settings**
|
||||
# Timeout for HTTP requests - set higher to handle large PDF files (up to 1GB)
|
||||
HTTP_REQUEST_TIMEOUT=120 # Timeout in seconds (default: 120 for large file operations)
|
||||
|
||||
# **Notification Settings**
|
||||
# Configure notification services using Apprise URL format
|
||||
@@ -475,117 +146,7 @@ NOTIFY_ON_TASK_FAILURE=True
|
||||
NOTIFY_ON_CREDENTIAL_FAILURE=True
|
||||
NOTIFY_ON_STARTUP=True
|
||||
NOTIFY_ON_SHUTDOWN=False
|
||||
NOTIFY_ON_FILE_PROCESSED=True
|
||||
|
||||
# Webhooks – Notify external systems via HTTP POST on document events.
|
||||
# Individual webhooks (URL, events, secret) are managed via /api/webhooks/.
|
||||
WEBHOOK_ENABLED=True
|
||||
|
||||
# Uptime Kuma
|
||||
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
||||
UPTIME_KUMA_PING_INTERVAL=5
|
||||
|
||||
# Backup & Restore
|
||||
# Enable automatic scheduled backups (hourly, daily, weekly)
|
||||
BACKUP_ENABLED=True
|
||||
# Directory for local backup archives (defaults to <WORKDIR>/backups)
|
||||
# BACKUP_DIR=/data/backups
|
||||
# Optional remote destination: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email
|
||||
# BACKUP_REMOTE_DESTINATION=s3
|
||||
# Sub-folder used when uploading backup archives to the remote destination
|
||||
BACKUP_REMOTE_FOLDER=backups
|
||||
# Retention: number of snapshots to keep per tier
|
||||
BACKUP_RETAIN_HOURLY=96 # 4 days of hourly snapshots
|
||||
BACKUP_RETAIN_DAILY=21 # 3 weeks of daily snapshots
|
||||
BACKUP_RETAIN_WEEKLY=13 # ~3 months of weekly snapshots
|
||||
|
||||
# **Full-Text Search (Meilisearch)**
|
||||
# URL for the Meilisearch instance.
|
||||
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
|
||||
# so container-to-container networking works without extra configuration.
|
||||
# Override to "http://localhost:7700" only when running the API process outside Docker.
|
||||
MEILISEARCH_URL=http://meilisearch:7700
|
||||
# Optional master/API key for secured Meilisearch instances
|
||||
# MEILISEARCH_API_KEY=your_master_key_here
|
||||
MEILISEARCH_INDEX_NAME=documents
|
||||
ENABLE_SEARCH=True
|
||||
|
||||
# **Duplicate Detection**
|
||||
# Exact duplicate detection (SHA-256) is always on during document processing.
|
||||
# The settings below control near-duplicate detection (same scanned content,
|
||||
# different hash) and the visibility of deduplication steps.
|
||||
ENABLE_DEDUPLICATION=True
|
||||
SHOW_DEDUPLICATION_STEP=True
|
||||
# Minimum cosine similarity score (0–1) for two documents to be flagged as
|
||||
# near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches.
|
||||
NEAR_DUPLICATE_THRESHOLD=0.85
|
||||
|
||||
# **PDF/A Archival Conversion**
|
||||
# When enabled, PDF/A copies of both the original ingested file and the processed
|
||||
# file are created and saved alongside the standard copies. This may double or
|
||||
# triple storage but provides better legal coverage with time-stamped archival copies.
|
||||
# Uses ocrmypdf with Ghostscript for the conversion.
|
||||
ENABLE_PDFA_CONVERSION=false
|
||||
# PDF/A format variant: 1 = PDF/A-1b, 2 = PDF/A-2b (default), 3 = PDF/A-3b
|
||||
PDFA_FORMAT=2
|
||||
# Upload original-file PDF/A variant to all configured storage providers
|
||||
PDFA_UPLOAD_ORIGINAL=false
|
||||
# Upload processed-file PDF/A variant to all configured storage providers
|
||||
PDFA_UPLOAD_PROCESSED=false
|
||||
# Subfolder name appended to each provider's folder for PDF/A uploads
|
||||
# e.g. if Dropbox folder is '/Documents' this puts PDF/A files into '/Documents/pdfa'
|
||||
PDFA_UPLOAD_FOLDER=pdfa
|
||||
# Google Drive folder ID for PDF/A uploads (uses folder IDs, not paths)
|
||||
# Leave empty to use the same folder as regular uploads
|
||||
GOOGLE_DRIVE_PDFA_FOLDER_ID=
|
||||
# RFC 3161 timestamping of PDF/A files (creates .tsr proof-of-existence files)
|
||||
PDFA_TIMESTAMP_ENABLED=false
|
||||
# Timestamp Authority URL (default: FreeTSA, a free RFC 3161 TSA)
|
||||
PDFA_TIMESTAMP_URL=https://freetsa.org/tsr
|
||||
# Model used to generate text embeddings for document similarity.
|
||||
# Must be supported by your OpenAI-compatible API endpoint.
|
||||
EMBEDDING_MODEL=text-embedding-3-small
|
||||
# Maximum tokens to send to the embedding model. Set below the model's
|
||||
# context window (e.g. 8000 for an 8192-token model).
|
||||
EMBEDDING_MAX_TOKENS=8000
|
||||
|
||||
# **Support / Help Center – Zammad Integration**
|
||||
# Base URL of your Zammad instance (required for chat and ticket form).
|
||||
# ZAMMAD_URL=https://zammad.example.com
|
||||
# Show a live-chat widget on the Help Center page (requires an online Zammad agent).
|
||||
# ZAMMAD_CHAT_ENABLED=false
|
||||
# Zammad chat topic ID (see Zammad → Channels → Chat → Topics).
|
||||
# ZAMMAD_CHAT_ID=1
|
||||
# Show a "Submit a Ticket" feedback form on the Help Center page.
|
||||
# ZAMMAD_FORM_ENABLED=false
|
||||
# Support e-mail address displayed on the Help Center page.
|
||||
# SUPPORT_EMAIL=support@example.com
|
||||
|
||||
# **Observability – Sentry Error & Performance Monitoring**
|
||||
# Sentry DSN – obtain from https://sentry.io (Project → Settings → Client Keys).
|
||||
# Leave commented out (or set to empty) to disable Sentry entirely.
|
||||
# SENTRY_DSN=https://<key>@o<org>.ingest.sentry.io/<project>
|
||||
#
|
||||
# Environment label shown in the Sentry dashboard (e.g. development / staging / production).
|
||||
# SENTRY_ENVIRONMENT=production
|
||||
#
|
||||
# Fraction of requests to capture for performance tracing (0.0–1.0).
|
||||
# 0.0 disables tracing; 1.0 captures every request. Default: 0.1 (10 %).
|
||||
# SENTRY_TRACES_SAMPLE_RATE=0.1
|
||||
#
|
||||
# Fraction of profiled transactions to send to Sentry (0.0–1.0).
|
||||
# Profiling is only active when SENTRY_TRACES_SAMPLE_RATE > 0. Default: 0.0 (disabled).
|
||||
# SENTRY_PROFILES_SAMPLE_RATE=0.0
|
||||
#
|
||||
# Attach PII (IP addresses, user agents) to Sentry events.
|
||||
# Disable (default) to stay GDPR/CCPA compliant.
|
||||
# SENTRY_SEND_DEFAULT_PII=false
|
||||
|
||||
# **Mobile App – Push Notifications**
|
||||
# Push notifications are delivered via Expo's push notification service
|
||||
# (https://expo.dev/notifications) which routes to APNs (iOS) and FCM (Android).
|
||||
# No additional credentials are required on the server side.
|
||||
# The mobile app registers its Expo push token via POST /api/mobile/register-device.
|
||||
#
|
||||
# To use native FCM/APNs directly (without Expo relay), replace the
|
||||
# send_expo_push_notification function in app/utils/push_notification.py.
|
||||
UPTIME_KUMA_PING_INTERVAL=5
|
||||
@@ -1,347 +0,0 @@
|
||||
# Copilot Instructions for DocuElevate
|
||||
|
||||
## Project Overview
|
||||
|
||||
DocuElevate is an intelligent document processing system that automates handling, extraction, and processing of documents. It integrates with multiple cloud storage providers (Dropbox, Google Drive, OneDrive, S3, Nextcloud) and uses AI services (OpenAI, Azure Document Intelligence) for metadata extraction and OCR.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **Backend**: FastAPI, SQLAlchemy, Celery, Redis
|
||||
- **Frontend**: Jinja2 templates, Tailwind CSS
|
||||
- **AI/ML**: OpenAI API, Azure Document Intelligence
|
||||
- **Auth**: Authentik (OAuth2), Basic Auth
|
||||
- **Infrastructure**: Docker, Docker Compose, Alembic (migrations)
|
||||
- **Testing**: Pytest, pytest-asyncio, httpx
|
||||
|
||||
## Supported Runtimes
|
||||
|
||||
- **Python**: 3.11+ (3.11 and 3.12 specified in pyproject.toml)
|
||||
- **Docker**: Production images use `python:3.14.1` / `python:3.14.1-slim`
|
||||
- **Redis**: Alpine-based (`redis:alpine`)
|
||||
- **Gotenberg**: `gotenberg/gotenberg:latest` for PDF conversion
|
||||
|
||||
## Build Commands
|
||||
|
||||
```bash
|
||||
# Install production dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Install development dependencies (includes linters, test tools)
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Run the FastAPI development server
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
# Run the Celery worker (requires Redis)
|
||||
celery -A app.celery_worker worker -B --loglevel=info -Q document_processor,default,celery
|
||||
|
||||
# Docker build and run
|
||||
docker compose up -d
|
||||
|
||||
# Database migrations
|
||||
alembic upgrade head # Apply all migrations
|
||||
alembic revision --autogenerate -m "description" # Create new migration
|
||||
```
|
||||
|
||||
## Test Commands
|
||||
|
||||
```bash
|
||||
# Run all tests with coverage (default via pyproject.toml addopts)
|
||||
pytest
|
||||
|
||||
# Run tests by marker
|
||||
pytest -m unit
|
||||
pytest -m integration
|
||||
pytest -m "not requires_external"
|
||||
|
||||
# Run a specific test file or test
|
||||
pytest tests/test_api.py -v
|
||||
pytest tests/test_api.py::test_function_name -v
|
||||
|
||||
# Coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
pytest --cov=app --cov-report=html
|
||||
```
|
||||
|
||||
## Lint / Format Commands
|
||||
|
||||
```bash
|
||||
# Format and lint with Ruff (replaces Black, isort, Flake8, Bandit — all-in-one)
|
||||
ruff format app/ tests/
|
||||
ruff check app/ tests/ --fix
|
||||
|
||||
# Type checking with mypy
|
||||
mypy app/
|
||||
|
||||
# Check for dependency vulnerabilities
|
||||
safety check
|
||||
|
||||
# Run all pre-commit hooks at once (recommended — runs ruff, mypy, secret detection, etc.)
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
## Agent Workflow (Follow for Every Task)
|
||||
|
||||
Follow these steps **in order** for every task — do not skip any:
|
||||
|
||||
1. **Understand** — read the issue/request in full before writing any code
|
||||
2. **Explore** — search the codebase for existing patterns and relevant implementations
|
||||
3. **Plan** — outline your changes as a checklist before starting
|
||||
4. **Implement** — make the smallest correct change that solves the problem
|
||||
5. **Test** — write or update tests; new code requires 100% test coverage
|
||||
6. **Document** — update all relevant docs in `docs/`; this is mandatory, not optional
|
||||
7. **Quality Gate** — run the single gate command below and fix every failure before committing:
|
||||
|
||||
```bash
|
||||
ruff format app/ tests/ && \
|
||||
ruff check app/ tests/ --fix && \
|
||||
safety check && \
|
||||
pytest --tb=short -q
|
||||
```
|
||||
|
||||
8. **Review** — re-read your own diff; confirm it is clean, secure, minimal, and well-documented
|
||||
|
||||
> All commands in the quality gate must exit with code 0. Never submit with failures.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### Code Quality
|
||||
- **Security first**: treat security as a non-negotiable requirement, not an afterthought — review [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) for every change
|
||||
- Write **clean, modern, well-documented code** — prioritize readability, maintainability, and idiomatic Python
|
||||
- Use **Ruff** for all formatting, linting, import sorting, and security scanning — `ruff format` + `ruff check --fix` (replaces Black, isort, Flake8, Bandit)
|
||||
- Line length: 120 characters (configured in `pyproject.toml`)
|
||||
- Use **type hints** for all function parameters and return values
|
||||
- Write **docstrings** for all public functions, classes, and modules
|
||||
- Maintain **100% test coverage** for new code
|
||||
|
||||
### Python Conventions
|
||||
- Use descriptive variable names (e.g., `user_document_path`, not `udp`)
|
||||
- Follow PEP 8 naming: `snake_case` for functions/variables, `PascalCase` for classes
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None` — avoid `List`, `Dict`, `Optional` from `typing`
|
||||
- Only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol`, and other constructs unavailable natively
|
||||
- Prefer `pathlib.Path` over string paths for file operations
|
||||
- Use f-strings for string formatting, not `.format()` or `%`
|
||||
- Handle exceptions explicitly - avoid bare `except:` clauses
|
||||
|
||||
### Security Best Practices
|
||||
- **Never commit secrets or credentials** to the repository
|
||||
- Use environment variables for sensitive configuration (see `.env.demo`)
|
||||
- Validate and sanitize all user inputs
|
||||
- Use parameterized queries with SQLAlchemy (never raw SQL with user input)
|
||||
- Review [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) before making security-related changes
|
||||
- Security linting is built into Ruff via `S` rules — runs automatically with `ruff check`; fix all `S`-prefixed findings
|
||||
- Run `safety check` to scan dependencies for known CVEs before submitting any PR
|
||||
|
||||
### FastAPI Patterns
|
||||
- Organize endpoints by feature in `app/api/` directory
|
||||
- Use dependency injection for database sessions and authentication
|
||||
- Return Pydantic models from endpoints for automatic validation
|
||||
- Use proper HTTP status codes (200, 201, 400, 401, 403, 404, 500)
|
||||
- Document endpoints with docstrings for OpenAPI documentation
|
||||
- Use `async def` for I/O-bound operations
|
||||
|
||||
### Database (SQLAlchemy)
|
||||
- All models are defined in `app/models.py`
|
||||
- Use Alembic for schema migrations (create migration for any model change)
|
||||
- Use declarative base for models
|
||||
- Define relationships with `relationship()` and proper `back_populates`
|
||||
- Use database sessions from `app.database.get_db()` dependency
|
||||
- Always close sessions in `finally` blocks or use context managers
|
||||
|
||||
### Celery Tasks
|
||||
- Define tasks in `app/tasks/` directory, organized by feature
|
||||
- Use descriptive task names: `module.action` (e.g., `document.process_ocr`)
|
||||
- Set appropriate retry policies and error handling
|
||||
- Log progress and errors using Python's `logging` module
|
||||
- Use `bind=True` for tasks that need access to task instance
|
||||
- Keep tasks idempotent when possible
|
||||
|
||||
### Frontend
|
||||
- Templates are in `frontend/templates/` using Jinja2
|
||||
- Static files (CSS, JS, images) in `frontend/static/`
|
||||
- Use Tailwind CSS utility classes (already configured)
|
||||
- Keep JavaScript minimal - prefer server-side rendering
|
||||
- Follow existing template structure and patterns
|
||||
|
||||
### Testing
|
||||
- Write tests in `tests/` directory, mirroring `app/` structure
|
||||
- Use pytest markers: `@pytest.mark.unit`, `@pytest.mark.integration`, etc.
|
||||
- Mock external services (OpenAI, Azure, cloud storage) in tests
|
||||
- Use `pytest.fixture` for test setup and teardown
|
||||
- Run tests with: `pytest -v`
|
||||
- Check coverage with: `pytest --cov=app --cov-report=term-missing`
|
||||
- **All tests must pass** before submitting changes — never leave failing tests
|
||||
- **All linters must pass** before submitting — run `pre-commit run --all-files`
|
||||
|
||||
### Configuration
|
||||
- All configuration is in `app/config.py` using Pydantic Settings
|
||||
- Use environment variables for configuration (12-factor app)
|
||||
- Provide sensible defaults when possible
|
||||
- Document all configuration options in `docs/ConfigurationGuide.md`
|
||||
|
||||
### Documentation
|
||||
- Keep documentation in `docs/` directory in Markdown format
|
||||
- **Always update** relevant docs when adding or changing any feature — documentation updates are mandatory, never optional
|
||||
- User-facing documentation should be clear and include examples
|
||||
- Reference existing docs: `docs/UserGuide.md`, `docs/API.md`, `docs/DeploymentGuide.md`
|
||||
- See [AGENTIC_CODING.md](../AGENTIC_CODING.md) for detailed development guide
|
||||
|
||||
### Error Handling
|
||||
- Use custom exceptions defined in application (follow existing patterns)
|
||||
- Log errors with context using Python's `logging` module
|
||||
- Return user-friendly error messages in API responses
|
||||
- Include error details in development, sanitize in production
|
||||
- In API endpoints, raise `HTTPException` with appropriate status codes (400, 401, 403, 404, 500)
|
||||
- In Celery tasks, use `self.retry(exc=e, countdown=60)` for transient errors; log and return error dict for permanent errors
|
||||
- Never use bare `except:` — always catch specific exception types
|
||||
- Wrap database operations in `try/except` with `db.rollback()` in the except block
|
||||
|
||||
### Logging Conventions
|
||||
- Use Python's built-in `logging` module: `import logging; logger = logging.getLogger(__name__)`
|
||||
- **Log levels**:
|
||||
- `logger.debug()` — detailed diagnostic information
|
||||
- `logger.info()` — general operational events (document processed, task started)
|
||||
- `logger.warning()` — recoverable issues (retrying, fallback used)
|
||||
- `logger.error()` — errors that need attention (failed operations)
|
||||
- `logger.critical()` — system-level failures requiring immediate action
|
||||
- **Always include context** in log messages: `logger.info(f"Processing document: {file_id}, user: {user_id}")`
|
||||
- **Never log sensitive data**: passwords, tokens, API keys, personal information
|
||||
- Use f-strings in log messages (consistent with project style)
|
||||
|
||||
### Architectural Boundaries
|
||||
- **`app/api/`** — REST API endpoints only; organize by feature
|
||||
- **`app/tasks/`** — Celery background tasks only; keep idempotent
|
||||
- **`app/views/`** — UI routes serving Jinja2 templates
|
||||
- **`app/utils/`** — Shared utility functions and helpers
|
||||
- **`app/routes/`** — **Deprecated**; being migrated to `app/api/` — do not add new code here
|
||||
- **`app/models.py`** — All SQLAlchemy models (single file)
|
||||
- **`app/config.py`** — All configuration via Pydantic Settings (single file)
|
||||
- **`app/database.py`** — Database engine and session setup (single file)
|
||||
- **`app/auth.py`** — Authentication logic (single file)
|
||||
- **`frontend/templates/`** — Jinja2 templates; do not mix backend logic
|
||||
- **`frontend/static/`** — CSS, JS, images; keep JavaScript minimal
|
||||
- **`tests/`** — Test files mirroring `app/` structure
|
||||
- **`migrations/`** — Alembic migration scripts; always auto-generate with `alembic revision --autogenerate`
|
||||
|
||||
### Don't Change Rules
|
||||
These files and directories are managed by automation or are critical infrastructure — **do not manually edit**:
|
||||
- **`VERSION`** — Managed by `python-semantic-release`; updated automatically on merge to main
|
||||
- **`CHANGELOG.md`** — Auto-generated from conventional commit messages by semantic-release
|
||||
- **`migrations/`** — Do not manually edit existing migration files; only create new ones via `alembic revision --autogenerate`
|
||||
- **Git tags and GitHub Releases** — Created automatically by semantic-release; never create manually
|
||||
- **`.pre-commit-config.yaml`** — Only change if adding/updating linting tools; do not remove existing hooks
|
||||
- **`pyproject.toml` `[tool.semantic_release]`** — Release configuration; do not modify without explicit approval
|
||||
- **`docker-compose.yaml` service names** — External systems depend on `api`, `worker`, `redis`, `gotenberg` names
|
||||
|
||||
### Dependencies
|
||||
- Add new dependencies to `requirements.txt` (production) or `requirements-dev.txt` (development)
|
||||
- Document any new dependencies and their licenses in README.md
|
||||
- Check for security vulnerabilities with `safety check`
|
||||
- Pin major versions, allow minor updates (e.g., `fastapi>=0.100.0,<1.0.0`)
|
||||
|
||||
### Git Workflow
|
||||
- Write clear, descriptive commit messages
|
||||
- **ALWAYS follow Conventional Commits format** (see below)
|
||||
- Keep commits focused and atomic
|
||||
- **All tests must pass** before committing — `pytest` must succeed with no failures
|
||||
- **All linters must pass** before committing — `pre-commit run --all-files` must succeed
|
||||
- Pre-commit hooks are configured (`.pre-commit-config.yaml`)
|
||||
|
||||
## Conventional Commits (REQUIRED)
|
||||
|
||||
All commit messages MUST follow the [Conventional Commits](https://www.conventionalcommits.org/) specification.
|
||||
|
||||
### Format
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
<footer>
|
||||
```
|
||||
|
||||
### Commit Types and Version Impact
|
||||
- **feat**: New feature → minor version bump (0.5.0 → 0.6.0)
|
||||
- **fix**: Bug fix → patch version bump (0.5.0 → 0.5.1)
|
||||
- **perf**: Performance improvement → patch version bump
|
||||
- **docs**: Documentation only → no version bump
|
||||
- **style**: Formatting changes → no version bump
|
||||
- **refactor**: Code refactoring → no version bump
|
||||
- **test**: Test changes → no version bump
|
||||
- **build**: Build system changes → no version bump
|
||||
- **ci**: CI/CD changes → no version bump
|
||||
- **chore**: Other changes → no version bump
|
||||
|
||||
### Breaking Changes
|
||||
For breaking changes (major version bump), add `!` after type or include `BREAKING CHANGE:` in footer:
|
||||
```
|
||||
feat(api)!: redesign authentication endpoints
|
||||
|
||||
BREAKING CHANGE: OAuth2 tokens now required instead of API keys.
|
||||
```
|
||||
Result: 0.5.0 → 1.0.0
|
||||
|
||||
### Scope Examples
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend changes
|
||||
- `auth` - Authentication
|
||||
- `storage` - Storage providers
|
||||
- `ocr` - OCR processing
|
||||
- `tasks` - Celery tasks
|
||||
- `config` - Configuration
|
||||
- `docs` - Documentation
|
||||
|
||||
### Commit Examples
|
||||
```
|
||||
feat(storage): add Amazon S3 storage provider
|
||||
fix(ocr): handle PDFs without text layer
|
||||
docs: update deployment guide with Docker setup
|
||||
refactor(tasks): consolidate duplicate code
|
||||
test: add integration tests for upload API
|
||||
chore: update dependencies for security fixes
|
||||
```
|
||||
|
||||
## Semantic Release Process
|
||||
|
||||
### Automated Versioning
|
||||
DocuElevate uses `python-semantic-release` for automated version management:
|
||||
|
||||
1. **On merge to main**: semantic-release analyzes commit messages
|
||||
2. **Automatic actions**:
|
||||
- Determines next version from commit types
|
||||
- Updates `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates Git tag with `v` prefix (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release with auto-generated notes
|
||||
- Triggers Docker image builds with version tag
|
||||
|
||||
### Agent Rules for Versioning
|
||||
- ✅ **DO**: Write conventional commit messages
|
||||
- ✅ **DO**: Use appropriate commit types for your changes
|
||||
- ✅ **DO**: Mark breaking changes explicitly
|
||||
- ❌ **DON'T**: Manually edit `VERSION` file
|
||||
- ❌ **DON'T**: Manually edit `CHANGELOG.md`
|
||||
- ❌ **DON'T**: Create version tags or GitHub Releases manually
|
||||
|
||||
These files are managed entirely by the semantic-release automation.
|
||||
|
||||
### File Organization
|
||||
- Place API endpoints in `app/api/` organized by feature
|
||||
- Background tasks go in `app/tasks/`
|
||||
- Utility functions in `app/utils/`
|
||||
- UI routes in `app/views/`
|
||||
- Database models in `app/models.py`
|
||||
- Configuration in `app/config.py`
|
||||
|
||||
### Common Patterns
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None`; only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol`
|
||||
- Import FastAPI dependencies: `from fastapi import Depends, HTTPException, status`
|
||||
- Get DB session: `db: Session = Depends(get_db)`
|
||||
- Current user: `current_user: User = Depends(get_current_user)`
|
||||
- Logger: `import logging; logger = logging.getLogger(__name__)`
|
||||
|
||||
## Resources
|
||||
- [AGENTIC_CODING.md](../AGENTIC_CODING.md) - Comprehensive development guide
|
||||
- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines
|
||||
- [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) - Security considerations
|
||||
- [README.md](../README.md) - Project overview and quickstart
|
||||
@@ -1,38 +0,0 @@
|
||||
# GitHub Copilot Workspace Configuration
|
||||
# This file configures GitHub Copilot coding agent settings for the DocuElevate repository
|
||||
|
||||
# Network allowlist for external API services
|
||||
# These domains are required for integration tests and external service connectivity
|
||||
network:
|
||||
allowlist:
|
||||
# OpenAI API - Required for AI-powered metadata extraction and GPT integration
|
||||
- api.openai.com
|
||||
|
||||
# Google OAuth2 - Required for Google Drive integration and OAuth authentication
|
||||
- oauth2.googleapis.com
|
||||
- accounts.google.com
|
||||
- www.googleapis.com
|
||||
|
||||
# Azure Cognitive Services - Required for Azure Document Intelligence and OCR
|
||||
- test.cognitiveservices.azure.com
|
||||
- "*.cognitiveservices.azure.com"
|
||||
|
||||
# Additional Azure endpoints that may be needed
|
||||
- login.microsoftonline.com
|
||||
- graph.microsoft.com
|
||||
|
||||
# AWS S3 - Required for S3 storage integration tests
|
||||
- s3.amazonaws.com
|
||||
- "*.s3.amazonaws.com"
|
||||
|
||||
# Dropbox API - Required for Dropbox storage integration
|
||||
- api.dropboxapi.com
|
||||
- content.dropboxapi.com
|
||||
|
||||
# Example/test domains - Used in test fixtures and SMTP configuration tests
|
||||
- example.com
|
||||
- smtp.example.com
|
||||
|
||||
# Package registries (if needed for dependency installation during tests)
|
||||
- pypi.org
|
||||
- files.pythonhosted.org
|
||||
@@ -10,9 +10,10 @@ updates:
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/static"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 5
|
||||
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
---
|
||||
applyTo: "docs/**/*.md"
|
||||
---
|
||||
|
||||
# Documentation Instructions
|
||||
|
||||
These instructions apply to all documentation files in the `docs/` directory.
|
||||
|
||||
## Documentation Structure
|
||||
- User-facing documentation in `docs/` directory
|
||||
- All documentation in Markdown format
|
||||
- Follow existing documentation style and structure
|
||||
|
||||
## Existing Documentation
|
||||
- `docs/UserGuide.md` - How to use DocuElevate
|
||||
- `docs/API.md` - API reference and examples
|
||||
- `docs/DeploymentGuide.md` - Deployment instructions
|
||||
- `docs/ConfigurationGuide.md` - Configuration options
|
||||
- `docs/Troubleshooting.md` - Common issues and solutions
|
||||
- `AGENTIC_CODING.md` - Development guide for AI agents
|
||||
- `CONTRIBUTING.md` - Contribution guidelines
|
||||
- `README.md` - Project overview and quickstart
|
||||
|
||||
## Markdown Style
|
||||
|
||||
### Headers
|
||||
```markdown
|
||||
# H1 - Document Title (only one per file)
|
||||
|
||||
## H2 - Major Sections
|
||||
|
||||
### H3 - Subsections
|
||||
|
||||
#### H4 - Minor subsections (use sparingly)
|
||||
```
|
||||
|
||||
### Code Blocks
|
||||
Always specify the language for syntax highlighting:
|
||||
````markdown
|
||||
```python
|
||||
def example_function():
|
||||
"""Example Python code."""
|
||||
return "Hello, World!"
|
||||
```
|
||||
|
||||
```bash
|
||||
# Shell commands
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "value"
|
||||
}
|
||||
```
|
||||
````
|
||||
|
||||
### Lists
|
||||
```markdown
|
||||
- Unordered list item 1
|
||||
- Unordered list item 2
|
||||
- Nested item
|
||||
- Another nested item
|
||||
|
||||
1. Ordered list item 1
|
||||
2. Ordered list item 2
|
||||
3. Ordered list item 3
|
||||
```
|
||||
|
||||
### Links
|
||||
```markdown
|
||||
[Link text](https://example.com)
|
||||
[Internal link](./UserGuide.md)
|
||||
[Link to section](#installation)
|
||||
```
|
||||
|
||||
### Images
|
||||
```markdown
|
||||
|
||||
|
||||
<div align="center">
|
||||
<img src="path/to/image.png" alt="Descriptive alt text" width="80%" />
|
||||
<p><em>Image caption</em></p>
|
||||
</div>
|
||||
```
|
||||
|
||||
### Tables
|
||||
```markdown
|
||||
| Column 1 | Column 2 | Column 3 |
|
||||
|----------|----------|----------|
|
||||
| Value 1 | Value 2 | Value 3 |
|
||||
| Value 4 | Value 5 | Value 6 |
|
||||
```
|
||||
|
||||
### Admonitions and Notes
|
||||
```markdown
|
||||
> **Note:** This is an important note.
|
||||
|
||||
> **Warning:** This is a warning message.
|
||||
|
||||
> **Tip:** This is a helpful tip.
|
||||
```
|
||||
|
||||
## Content Guidelines
|
||||
|
||||
### Writing Style
|
||||
- Use clear, concise language
|
||||
- Write in second person (you/your) for user-facing docs
|
||||
- Use present tense
|
||||
- Avoid jargon; explain technical terms when necessary
|
||||
- Use active voice
|
||||
- Keep sentences short and focused
|
||||
|
||||
### Documentation Types
|
||||
|
||||
#### User Documentation
|
||||
- Focus on **how to use** features, not implementation details
|
||||
- Include step-by-step instructions
|
||||
- Provide examples for common use cases
|
||||
- Add screenshots or diagrams when helpful
|
||||
- Explain what each feature does and when to use it
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
## Uploading Documents
|
||||
|
||||
To upload a document to DocuElevate:
|
||||
|
||||
1. Navigate to the Upload page
|
||||
2. Click "Choose File" and select your document
|
||||
3. Select the destination (Dropbox, Google Drive, etc.)
|
||||
4. Click "Upload"
|
||||
|
||||
The document will be automatically processed and stored in your selected destination.
|
||||
```
|
||||
|
||||
#### API Documentation
|
||||
- Document all endpoints with examples
|
||||
- Show request and response formats
|
||||
- Include authentication requirements
|
||||
- Provide example curl commands
|
||||
- Document error responses
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### POST /api/documents/upload
|
||||
|
||||
Upload a new document for processing.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST "http://localhost:8000/api/documents/upload" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \
|
||||
-F "file=@document.pdf"
|
||||
```
|
||||
|
||||
**Response (201 Created):**
|
||||
```json
|
||||
{
|
||||
"id": 123,
|
||||
"filename": "document.pdf",
|
||||
"status": "processing"
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
#### Configuration Documentation
|
||||
- List all configuration options
|
||||
- Provide default values
|
||||
- Explain what each option does
|
||||
- Include example configurations
|
||||
- Note which options are required vs. optional
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### OPENAI_API_KEY
|
||||
|
||||
**Type:** String
|
||||
**Required:** Yes
|
||||
**Default:** None
|
||||
|
||||
Your OpenAI API key for metadata extraction.
|
||||
|
||||
```bash
|
||||
OPENAI_API_KEY=sk-...
|
||||
```
|
||||
```
|
||||
|
||||
#### Troubleshooting Documentation
|
||||
- Start with the symptom/error
|
||||
- Provide clear diagnosis steps
|
||||
- Offer solutions
|
||||
- Include common causes
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### Error: "Connection refused" when starting services
|
||||
|
||||
**Cause:** Docker services are not running or ports are already in use.
|
||||
|
||||
**Solution:**
|
||||
1. Check if Docker is running: `docker ps`
|
||||
2. Check port availability: `lsof -i :8000`
|
||||
3. Restart Docker services: `docker-compose restart`
|
||||
```
|
||||
|
||||
## Code Examples
|
||||
- Always test code examples before including them
|
||||
- Use realistic examples that users can adapt
|
||||
- Include comments explaining non-obvious parts
|
||||
- Show complete examples, not just fragments
|
||||
|
||||
## Version Information
|
||||
- Update documentation when changing features
|
||||
- Note version numbers when features are added
|
||||
- Mark deprecated features clearly
|
||||
|
||||
## Cross-References
|
||||
- Link to related documentation
|
||||
- Reference other sections when appropriate
|
||||
- Keep the documentation interconnected
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
For deployment instructions, see the [Deployment Guide](./DeploymentGuide.md).
|
||||
|
||||
For API details, refer to the [API Documentation](./API.md).
|
||||
```
|
||||
|
||||
## Updating Documentation
|
||||
Documentation updates are **mandatory** — every PR that changes code must include matching documentation updates in the same PR. There are no exceptions.
|
||||
|
||||
When making code changes:
|
||||
1. **Update relevant documentation** in the same PR — never defer docs to a follow-up
|
||||
2. Check for outdated information in existing docs
|
||||
3. Add new sections for new features
|
||||
4. Update examples if behavior changes
|
||||
5. Review related documentation for consistency
|
||||
6. Update `docs/ConfigurationGuide.md` and `.env.demo` for any new or changed configuration options
|
||||
|
||||
## Screenshots and Diagrams
|
||||
- Use clear, high-quality images
|
||||
- Annotate screenshots when helpful
|
||||
- Keep diagrams simple and focused
|
||||
- Update screenshots when UI changes
|
||||
- Use consistent styling in diagrams
|
||||
|
||||
## Accessibility
|
||||
- Use descriptive alt text for images
|
||||
- Ensure proper heading hierarchy
|
||||
- Make links descriptive (avoid "click here")
|
||||
- Use semantic formatting (bold, italic, code) appropriately
|
||||
|
||||
## README.md Specific
|
||||
- Keep README concise and focused on getting started
|
||||
- Include badges for build status, version, license
|
||||
- Show the most important features first
|
||||
- Link to detailed documentation
|
||||
- Include quick start instructions
|
||||
- Add screenshots of the main interface
|
||||
|
||||
## Configuration Guide Updates
|
||||
When adding new configuration options:
|
||||
- Add to `docs/ConfigurationGuide.md`
|
||||
- Include type, default value, and description
|
||||
- Provide example usage
|
||||
- Note any dependencies on other config options
|
||||
- Update `.env.demo` with the new option
|
||||
@@ -1,197 +0,0 @@
|
||||
---
|
||||
applyTo: "frontend/**/*"
|
||||
---
|
||||
|
||||
# Frontend Instructions
|
||||
|
||||
These instructions apply to all files in the `frontend/` directory (templates, CSS, JavaScript, images).
|
||||
|
||||
## Templates (Jinja2)
|
||||
|
||||
### Location and Structure
|
||||
- All templates in `frontend/templates/`
|
||||
- Use template inheritance with `base.html`
|
||||
- Keep templates organized by feature
|
||||
|
||||
### Template Patterns
|
||||
```jinja2
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Document Upload - DocuElevate{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mx-auto px-4 py-8">
|
||||
<h1 class="text-2xl font-bold mb-4">{{ page_title }}</h1>
|
||||
|
||||
{% if error_message %}
|
||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-4">
|
||||
{{ error_message }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
<!-- Form content -->
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
```
|
||||
|
||||
### Tailwind CSS Usage
|
||||
- Use Tailwind utility classes (already configured)
|
||||
- Follow responsive design: `md:`, `lg:` breakpoints
|
||||
- Use existing color scheme from the project
|
||||
- Common patterns:
|
||||
- Containers: `container mx-auto px-4`
|
||||
- Buttons: `bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded`
|
||||
- Cards: `bg-white shadow-md rounded-lg p-6`
|
||||
- Forms: `w-full px-3 py-2 border rounded`
|
||||
|
||||
### Static Files
|
||||
- CSS files in `frontend/static/css/`
|
||||
- JavaScript in `frontend/static/js/`
|
||||
- Images in `frontend/static/images/`
|
||||
- Reference with `{{ url_for('static', path='css/style.css') }}`
|
||||
|
||||
### JavaScript
|
||||
- Keep JavaScript minimal - prefer server-side rendering
|
||||
- Use vanilla JavaScript or minimal dependencies
|
||||
- Place scripts at the end of the body
|
||||
- Use `defer` or `async` for external scripts
|
||||
```html
|
||||
<script src="{{ url_for('static', path='js/upload.js') }}" defer></script>
|
||||
```
|
||||
|
||||
### Forms
|
||||
- Use CSRF protection when needed
|
||||
- Include proper validation
|
||||
- Show clear error messages
|
||||
- Use proper `method` (GET/POST) and `enctype` for file uploads
|
||||
```html
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
<div class="mb-4">
|
||||
<label class="block text-gray-700 text-sm font-bold mb-2" for="file">
|
||||
Document File
|
||||
</label>
|
||||
<input
|
||||
type="file"
|
||||
id="file"
|
||||
name="file"
|
||||
class="w-full px-3 py-2 border rounded"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
<button type="submit" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
|
||||
Upload
|
||||
</button>
|
||||
</form>
|
||||
```
|
||||
|
||||
### Accessibility (WCAG 2.1 Level AA Required)
|
||||
|
||||
DocuElevate targets **WCAG 2.1 Level AA** compliance. Every template change **must** follow these rules.
|
||||
For the full guide with examples, see `docs/AccessibilityGuide.md`.
|
||||
|
||||
#### Semantic HTML (WCAG 1.3.1)
|
||||
- Use semantic elements: `<nav>`, `<main>`, `<article>`, `<section>`, `<header>`, `<footer>`
|
||||
- Use proper heading hierarchy: one `<h1>` per page, then `<h2>` → `<h3>` (never skip levels)
|
||||
- Use `<button>` for actions (not `<a>` or `<div>`) and `<a>` for navigation
|
||||
- Use `<table>` with `<caption>` or `aria-label`, `<thead>`/`<tbody>`, and `scope="col"`/`scope="row"` on headers
|
||||
|
||||
#### Images & Icons (WCAG 1.1.1)
|
||||
- All `<img>` elements **must** have an `alt` attribute — descriptive for content images, `alt=""` for purely decorative ones
|
||||
- Decorative Font Awesome `<i>` icons **must** have `aria-hidden="true"` when adjacent text already conveys meaning
|
||||
- Icon-only buttons **must** have `aria-label` describing the action (e.g., `aria-label="Delete file"`)
|
||||
|
||||
#### Keyboard Navigation (WCAG 2.1.1, 2.4.1, 2.4.7)
|
||||
- All interactive elements must be keyboard-reachable (native `<a>`, `<button>`, `<input>`, or add `tabindex="0"` + key handlers)
|
||||
- `base.html` provides a **skip-to-content** link (`<a href="#main-content" class="skip-link">`) — do not remove it
|
||||
- Never suppress focus indicators — the global `focus-visible` outline in `styles.css` is required
|
||||
- Custom interactive widgets (dropdowns, modals) must trap focus correctly
|
||||
|
||||
#### ARIA Attributes
|
||||
- `aria-label` — use on elements whose purpose isn't clear from visible text (icon-only buttons, unlabelled inputs)
|
||||
- `aria-hidden="true"` — use on purely decorative icons and elements that duplicate adjacent text
|
||||
- `aria-live="polite"` — add to any container whose content updates dynamically (status messages, search results, upload progress)
|
||||
- `aria-expanded` — add to buttons that toggle visibility of content (menus, accordions)
|
||||
- `aria-current="page"` — mark the current page's navigation link
|
||||
- `aria-sort` — use on sortable table column headers
|
||||
|
||||
#### Forms (WCAG 1.3.1, 3.3.2)
|
||||
- Every `<input>`, `<select>`, and `<textarea>` **must** have an associated `<label>` (via `for`/`id`) or `aria-label`
|
||||
- Error messages must be linked via `aria-describedby` or announced with `role="alert"`
|
||||
- Use `role="search"` on search form containers
|
||||
|
||||
#### Modals / Dialogs (WCAG 4.1.2)
|
||||
- Add `role="dialog"`, `aria-modal="true"`, and `aria-labelledby` pointing to the dialog title
|
||||
- Focus must move into the dialog when opened and return to the trigger when closed
|
||||
|
||||
#### Color & Contrast (WCAG 1.4.3, 1.4.1)
|
||||
- Text must meet 4.5:1 contrast ratio against its background (3:1 for large text)
|
||||
- Never rely on color alone to convey information — pair color with icons, text labels, or patterns
|
||||
- Dark-mode overrides in `styles.css` are WCAG AA-verified; maintain this when adding new colors
|
||||
|
||||
#### Touch Targets (WCAG 2.5.8)
|
||||
- All clickable/tappable elements must be at least 44×44 CSS pixels (`min-height:44px; min-width:44px`)
|
||||
|
||||
#### Automated Checks
|
||||
- The CI pipeline runs `djlint` on every PR to catch common accessibility regressions
|
||||
- Run locally: `djlint frontend/templates/ --lint`
|
||||
- Configuration is in `pyproject.toml` under `[tool.djlint]`
|
||||
|
||||
### Error Handling
|
||||
- Display user-friendly error messages
|
||||
- Use flash messages for feedback
|
||||
- Show loading states for async operations
|
||||
```jinja2
|
||||
{% with messages = get_flashed_messages(with_categories=true) %}
|
||||
{% if messages %}
|
||||
{% for category, message in messages %}
|
||||
<div class="bg-{{ category }}-100 border border-{{ category }}-400 text-{{ category }}-700 px-4 py-3 rounded mb-4">
|
||||
{{ message }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endwith %}
|
||||
```
|
||||
|
||||
### URL Generation
|
||||
- Always use `url_for()` for URLs, never hardcode
|
||||
- Examples:
|
||||
- Routes: `{{ url_for('upload_document') }}`
|
||||
- Static: `{{ url_for('static', path='css/style.css') }}`
|
||||
- API: `{{ url_for('api_document', document_id=doc.id) }}`
|
||||
|
||||
### Template Variables
|
||||
- Check if variables exist before using them
|
||||
- Use filters for formatting
|
||||
```jinja2
|
||||
{% if document %}
|
||||
<p>Uploaded: {{ document.created_at|datetime }}</p>
|
||||
<p>Size: {{ document.file_size|filesizeformat }}</p>
|
||||
{% else %}
|
||||
<p>No document found</p>
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
### Common Components
|
||||
- Follow existing patterns for headers, footers, navigation
|
||||
- Reuse template blocks and includes
|
||||
- Keep components modular
|
||||
```jinja2
|
||||
{% include 'components/navigation.html' %}
|
||||
{% include 'components/document_card.html' with document=doc %}
|
||||
```
|
||||
|
||||
## UI/UX Guidelines
|
||||
- Maintain consistent spacing using Tailwind's scale (4, 8, 16, etc.)
|
||||
- Use the existing color palette from the design
|
||||
- Ensure mobile responsiveness
|
||||
- Show loading indicators for long operations
|
||||
- Provide feedback for user actions (success/error messages)
|
||||
- Keep the interface clean and minimal
|
||||
|
||||
## Performance
|
||||
- Optimize images (compress, use appropriate formats)
|
||||
- Minimize JavaScript bundle size
|
||||
- Use lazy loading for images when appropriate
|
||||
- Cache static assets
|
||||
@@ -1,165 +0,0 @@
|
||||
---
|
||||
applyTo: "app/**/*.py"
|
||||
---
|
||||
|
||||
# Python Backend Instructions
|
||||
|
||||
These instructions apply to all Python code in the `app/` directory.
|
||||
|
||||
## Code Style
|
||||
- Use **Ruff** for all formatting, linting, import sorting, and security scanning — `ruff format app/ tests/ && ruff check app/ tests/ --fix`
|
||||
- Line length: 120 characters (configured in `pyproject.toml` `[tool.ruff]`)
|
||||
- All functions must have type hints for parameters and return values
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None`
|
||||
- Only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol` (not `Dict`, `List`, `Optional`, `Union`)
|
||||
|
||||
## Import Order (enforced by Ruff `I` rules)
|
||||
```python
|
||||
# Standard library imports
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any # Only for Any, Callable, TypeVar, Protocol
|
||||
|
||||
# Third-party imports
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# Local application imports
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import Document, User
|
||||
```
|
||||
|
||||
## Function Definitions
|
||||
```python
|
||||
def process_document(
|
||||
file_path: Path,
|
||||
user_id: int,
|
||||
metadata: dict[str, Any] | None = None
|
||||
) -> DocumentMetadata:
|
||||
"""
|
||||
Process a document and extract metadata.
|
||||
|
||||
Args:
|
||||
file_path: Path to the document file
|
||||
user_id: ID of the user uploading the document
|
||||
metadata: Optional additional metadata
|
||||
|
||||
Returns:
|
||||
DocumentMetadata object with extracted information
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
ProcessingError: If processing fails
|
||||
"""
|
||||
pass
|
||||
```
|
||||
|
||||
## FastAPI Endpoints
|
||||
- Use dependency injection for DB sessions and auth
|
||||
- Return Pydantic models for automatic validation
|
||||
- Use proper status codes from `fastapi.status`
|
||||
- Add detailed docstrings for OpenAPI docs
|
||||
```python
|
||||
from fastapi import APIRouter, Depends, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
router = APIRouter(prefix="/api/documents", tags=["documents"])
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED)
|
||||
async def create_document(
|
||||
file: UploadFile,
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
) -> DocumentResponse:
|
||||
"""Create and process a new document."""
|
||||
pass
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
- Use custom exceptions from the application
|
||||
- Log errors with context using `logging.getLogger(__name__)`
|
||||
- Return user-friendly error messages
|
||||
- Never expose internal details in production errors
|
||||
```python
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
result = process_file(file_path)
|
||||
except FileNotFoundError:
|
||||
logger.error(f"File not found: {file_path}")
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing file: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Processing failed")
|
||||
```
|
||||
|
||||
## Database Operations
|
||||
- Use SQLAlchemy ORM, never raw SQL with user input
|
||||
- Use `get_db()` dependency for sessions
|
||||
- Always commit in try/except blocks
|
||||
```python
|
||||
from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
|
||||
def create_document(db: Session, document_data: dict) -> Document:
|
||||
"""Create a new document in the database."""
|
||||
db_document = Document(**document_data)
|
||||
try:
|
||||
db.add(db_document)
|
||||
db.commit()
|
||||
db.refresh(db_document)
|
||||
return db_document
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
raise
|
||||
```
|
||||
|
||||
## Celery Tasks
|
||||
- Define in `app/tasks/` directory
|
||||
- Use descriptive names: `module.action`
|
||||
- Set retry policies
|
||||
- Log progress and errors
|
||||
```python
|
||||
from celery import shared_task
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@shared_task(bind=True, max_retries=3)
|
||||
def process_ocr(self, document_id: int) -> dict[str, Any]:
|
||||
"""Process OCR for a document."""
|
||||
try:
|
||||
# Processing logic
|
||||
logger.info(f"Processing OCR for document {document_id}")
|
||||
return {"status": "success"}
|
||||
except Exception as exc:
|
||||
logger.exception(f"OCR processing failed for {document_id}")
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
```
|
||||
|
||||
## Configuration
|
||||
- All settings in `app/config.py` using Pydantic Settings
|
||||
- Use environment variables, never hardcode values
|
||||
- Provide defaults when sensible
|
||||
```python
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class Settings(BaseSettings):
|
||||
openai_api_key: str
|
||||
max_file_size: int = 10485760 # 10MB default
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
```
|
||||
|
||||
## Security (First and Foremost)
|
||||
- **Security first**: treat every change as a potential attack surface — review `SECURITY_AUDIT.md` before making any security-related change
|
||||
- Never commit secrets, tokens, or credentials
|
||||
- Validate and sanitize all user inputs
|
||||
- Use parameterized queries — never raw SQL with user data
|
||||
- Sanitize file paths; check file permissions before access
|
||||
- Security linting is built into Ruff via `S` rules — fix all `S`-prefixed findings before committing
|
||||
- Run `safety check` before submitting any PR to catch dependency CVEs
|
||||
@@ -1,246 +0,0 @@
|
||||
---
|
||||
applyTo: "tests/**/*.py"
|
||||
---
|
||||
|
||||
# Testing Instructions
|
||||
|
||||
These instructions apply to all test files in the `tests/` directory.
|
||||
|
||||
## Test Organization
|
||||
- Mirror the structure of `app/` directory in `tests/`
|
||||
- Name test files with `test_` prefix (e.g., `test_api.py`)
|
||||
- Group related tests in classes with `Test` prefix
|
||||
- Use descriptive test function names: `test_<what>_<condition>_<expected>`
|
||||
|
||||
## Pytest Configuration
|
||||
- Configuration in `pytest.ini`
|
||||
- Run tests: `pytest -v`
|
||||
- With coverage: `pytest --cov=app --cov-report=term-missing`
|
||||
- Run specific markers: `pytest -m unit` or `pytest -m integration`
|
||||
|
||||
## Test Markers
|
||||
Use pytest markers to categorize tests:
|
||||
```python
|
||||
import pytest
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_document_validation():
|
||||
"""Test document validation logic."""
|
||||
pass
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_document_upload_api():
|
||||
"""Test document upload endpoint."""
|
||||
pass
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_large_file_processing():
|
||||
"""Test processing of large files."""
|
||||
pass
|
||||
|
||||
@pytest.mark.requires_external
|
||||
def test_openai_integration():
|
||||
"""Test OpenAI API integration."""
|
||||
pass
|
||||
```
|
||||
|
||||
Available markers:
|
||||
- `unit` - Unit tests for individual functions/methods
|
||||
- `integration` - Integration tests for API endpoints and workflows
|
||||
- `slow` - Tests that take significant time to run
|
||||
- `security` - Security-related tests
|
||||
- `requires_external` - Tests requiring external services (OpenAI, Azure, etc.)
|
||||
- `requires_db` - Tests requiring database
|
||||
- `requires_redis` - Tests requiring Redis
|
||||
|
||||
## Fixtures
|
||||
Use pytest fixtures for test setup and teardown:
|
||||
```python
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from app.database import Base
|
||||
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
"""Provide a database session for tests."""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
yield session
|
||||
|
||||
session.close()
|
||||
Base.metadata.drop_all(engine)
|
||||
|
||||
@pytest.fixture
|
||||
def sample_document():
|
||||
"""Provide a sample document for tests."""
|
||||
return {
|
||||
"filename": "test.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"size": 1024
|
||||
}
|
||||
```
|
||||
|
||||
## API Testing with FastAPI
|
||||
Use `TestClient` from FastAPI:
|
||||
```python
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
def test_upload_document():
|
||||
"""Test document upload endpoint."""
|
||||
with open("tests/fixtures/sample.pdf", "rb") as f:
|
||||
response = client.post(
|
||||
"/api/documents/upload",
|
||||
files={"file": ("test.pdf", f, "application/pdf")}
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert "id" in response.json()
|
||||
```
|
||||
|
||||
## Async Testing
|
||||
For async code, use `pytest-asyncio`:
|
||||
```python
|
||||
import pytest
|
||||
import httpx
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_document_processing():
|
||||
"""Test async document processing."""
|
||||
async with httpx.AsyncClient(app=app, base_url="http://test") as client:
|
||||
response = await client.get("/api/documents/1")
|
||||
assert response.status_code == 200
|
||||
```
|
||||
|
||||
## Mocking External Services
|
||||
Always mock external services in tests:
|
||||
```python
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_openai_metadata_extraction(mocker):
|
||||
"""Test metadata extraction with mocked OpenAI."""
|
||||
mock_response = {
|
||||
"document_type": "invoice",
|
||||
"amount": 100.00,
|
||||
"date": "2024-01-01"
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"app.utils.openai_client.extract_metadata",
|
||||
return_value=mock_response
|
||||
)
|
||||
|
||||
result = extract_document_metadata("test.pdf")
|
||||
assert result["document_type"] == "invoice"
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_azure_ocr_processing(mocker):
|
||||
"""Test OCR with mocked Azure service."""
|
||||
mock_text = "Sample extracted text"
|
||||
|
||||
mocker.patch(
|
||||
"app.utils.azure_client.extract_text",
|
||||
return_value=mock_text
|
||||
)
|
||||
|
||||
result = perform_ocr("test.pdf")
|
||||
assert result == mock_text
|
||||
```
|
||||
|
||||
## Database Testing
|
||||
```python
|
||||
@pytest.mark.requires_db
|
||||
def test_create_document(db_session):
|
||||
"""Test document creation in database."""
|
||||
from app.models import Document
|
||||
|
||||
doc = Document(
|
||||
filename="test.pdf",
|
||||
user_id=1,
|
||||
file_path="/tmp/test.pdf"
|
||||
)
|
||||
db_session.add(doc)
|
||||
db_session.commit()
|
||||
|
||||
assert doc.id is not None
|
||||
assert doc.filename == "test.pdf"
|
||||
```
|
||||
|
||||
## Test Coverage Goals
|
||||
- Achieve **100% test coverage** for all new code — use `# pragma: no cover` only for genuinely unreachable or platform-specific branches, with an inline comment explaining why
|
||||
- Enforce the threshold: `pytest --cov=app --cov-fail-under=100`
|
||||
- Focus on critical paths and error handling
|
||||
- Test both success and failure scenarios
|
||||
- Don't test third-party library code
|
||||
|
||||
## Test Structure
|
||||
Follow the Arrange-Act-Assert pattern:
|
||||
```python
|
||||
def test_document_validation():
|
||||
"""Test that invalid documents are rejected."""
|
||||
# Arrange
|
||||
invalid_document = {
|
||||
"filename": "", # Empty filename
|
||||
"size": -1 # Invalid size
|
||||
}
|
||||
|
||||
# Act
|
||||
result = validate_document(invalid_document)
|
||||
|
||||
# Assert
|
||||
assert result.is_valid is False
|
||||
assert "filename" in result.errors
|
||||
assert "size" in result.errors
|
||||
```
|
||||
|
||||
## Parameterized Tests
|
||||
Use `pytest.mark.parametrize` for multiple test cases:
|
||||
```python
|
||||
@pytest.mark.parametrize("filename,expected", [
|
||||
("document.pdf", True),
|
||||
("image.jpg", True),
|
||||
("script.exe", False),
|
||||
("", False),
|
||||
])
|
||||
def test_allowed_file_types(filename, expected):
|
||||
"""Test file type validation."""
|
||||
result = is_allowed_file(filename)
|
||||
assert result == expected
|
||||
```
|
||||
|
||||
## Test Data
|
||||
- Place test fixtures in `tests/fixtures/` directory
|
||||
- Use small sample files for testing
|
||||
- Don't commit large test files
|
||||
- Clean up test files in teardown
|
||||
|
||||
## Error Testing
|
||||
Always test error conditions:
|
||||
```python
|
||||
def test_missing_file_raises_error():
|
||||
"""Test that missing files raise appropriate error."""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
process_document("/nonexistent/file.pdf")
|
||||
|
||||
def test_invalid_api_request():
|
||||
"""Test API error handling."""
|
||||
response = client.post("/api/documents/", json={})
|
||||
assert response.status_code == 422 # Validation error
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
- Test one thing per test function
|
||||
- Use descriptive test names
|
||||
- Keep tests independent (no dependencies between tests)
|
||||
- Use fixtures for common setup
|
||||
- Mock external dependencies
|
||||
- Test edge cases and error conditions
|
||||
- Keep tests fast (use mocks for slow operations)
|
||||
- Clean up resources after tests
|
||||
@@ -1,227 +0,0 @@
|
||||
name: CI Pipeline
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
tags: ['v*', '[0-9]+.*']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
IMAGE_NAME: christianlouis/docuelevate
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 1: Static Analysis (Fast Fail Gates)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
lint:
|
||||
name: Ruff Lint & Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
- name: Install Ruff
|
||||
run: pip install ruff
|
||||
- name: Check for merge conflict markers
|
||||
run: |
|
||||
if git grep -rn -E '^(<{7} |>{7} |={7}$)' -- '.'; then
|
||||
echo "ERROR: Merge conflict markers found."
|
||||
exit 1
|
||||
fi
|
||||
- run: ruff check app/ tests/
|
||||
- run: ruff format --check app/ tests/
|
||||
|
||||
html-lint:
|
||||
name: HTML Accessibility Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
- run: pip install djlint>=1.36.0
|
||||
- run: djlint frontend/templates/ --lint
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 2: Parallel Heavy Lifters (Consolidated for Efficiency)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
mypy:
|
||||
name: Mypy Type Check
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
- name: Install Dependencies
|
||||
run: pip install -r requirements-dev.txt
|
||||
- run: mypy app/
|
||||
|
||||
dependency-scan:
|
||||
name: Dependency Scan
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
- run: pip install pip-audit>=2.7.0
|
||||
- run: pip-audit -r requirements.txt --desc on
|
||||
|
||||
run-tests:
|
||||
name: Execute All Tests (Quick + Integration)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
services:
|
||||
redis:
|
||||
image: redis:7
|
||||
ports: ["6379:6379"]
|
||||
options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
rabbitmq:
|
||||
image: rabbitmq:3-management
|
||||
ports: ["5672:5672", "15672:15672"]
|
||||
options: --health-cmd "rabbitmq-diagnostics -q ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Tests
|
||||
run: >
|
||||
pytest tests/ -v --timeout=300
|
||||
--cov=app --cov-report=xml:coverage.xml
|
||||
--junitxml=junit.xml -o junit_family=legacy
|
||||
-m "not e2e"
|
||||
|
||||
- name: Upload Unified Coverage to Codecov
|
||||
if: always()
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
fail_ci_if_error: true
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 3: Build & Push (Quality Gate)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
build:
|
||||
name: Build & Push Docker Image
|
||||
runs-on: ubuntu-latest
|
||||
needs: [run-tests, mypy, dependency-scan, html-lint]
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
- name: Generate Build Metadata
|
||||
run: |
|
||||
chmod +x scripts/generate_build_metadata.sh
|
||||
./scripts/generate_build_metadata.sh
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Extract metadata for tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.IMAGE_NAME }}
|
||||
ghcr.io/${{ github.repository_owner }}/docuelevate
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=sha,prefix={{branch}}-
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
- name: Build and Push Docker Image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
sbom: true
|
||||
provenance: mode=max
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 4: GitOps Update
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
update-k8s-manifest:
|
||||
name: Update Preprod K8s Manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
steps:
|
||||
- name: Compute image tag
|
||||
id: tag
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
echo "tag=main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "image=ghcr.io/${{ github.repository_owner }}/docuelevate:main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
- name: Checkout k8s-cluster-state
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: christianlouis/k8s-cluster-state
|
||||
token: ${{ secrets.GH_PAT }}
|
||||
path: k8s-cluster-state
|
||||
- name: Update image tag in preprod manifest
|
||||
uses: mikefarah/yq@v4.44.6
|
||||
env:
|
||||
IMAGE: ${{ steps.tag.outputs.image }}
|
||||
with:
|
||||
cmd: |
|
||||
yq -i '(.. | select(tag == "!!str") | select(test("^(ghcr\\.io/christianlouis/docuelevate|christianlouis/docuelevate):"))) = strenv(IMAGE)' \
|
||||
k8s-cluster-state/apps/docuelevate/preprod/docuelevate-stack.yaml
|
||||
- name: Commit and push
|
||||
run: |
|
||||
cd k8s-cluster-state
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add apps/docuelevate/preprod/docuelevate-stack.yaml
|
||||
if git diff --staged --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "chore(preprod): update docuelevate image to ${{ steps.tag.outputs.tag }}"
|
||||
git push
|
||||
fi
|
||||
@@ -0,0 +1,41 @@
|
||||
name: "CodeQL Security Scanning"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main", "develop" ]
|
||||
pull_request:
|
||||
branches: [ "main", "develop" ]
|
||||
schedule:
|
||||
- cron: '0 0 * * 1' # Run every Monday at midnight
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python', 'javascript' ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-and-quality
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -1,60 +0,0 @@
|
||||
name: "CodeQL Advanced"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
schedule:
|
||||
- cron: '37 1 * * 1'
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
security-events: write
|
||||
packages: read
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: actions
|
||||
build-mode: none
|
||||
- language: javascript
|
||||
build-mode: none
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
- language: python
|
||||
build-mode: none
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
|
||||
- name: Run manual build steps
|
||||
if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -0,0 +1,18 @@
|
||||
name: Deploy to Production
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Build and Push Docker Image"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Call Deployment Webhook
|
||||
run: |
|
||||
curl -X POST https://docker2.kuechenserver.org/api/stacks/webhooks/960c7d8e-97ec-4175-a8dc-73f037b02349
|
||||
@@ -0,0 +1,75 @@
|
||||
name: Build and Push Docker Image
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
tags:
|
||||
- 'v*'
|
||||
- '[0-9]+.*'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
IMAGE_NAME: christianlouis/document-processor
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Git Tag (if applicable)
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
|
||||
|
||||
- name: Extract metadata for tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.IMAGE_NAME }}
|
||||
ghcr.io/${{ github.repository_owner }}/document-processor
|
||||
|
||||
- name: Build and Push Docker Image with Provenance and SBOM
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
sbom: true
|
||||
provenance: mode=max
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
ghcr.io/${{ github.repository_owner }}/document-processor:latest
|
||||
ghcr.io/${{ github.repository_owner }}/document-processor:${{ github.sha }}
|
||||
${{ startsWith(github.ref, 'refs/tags/') && format('{0}:{1}', env.IMAGE_NAME, env.VERSION) || '' }}
|
||||
${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}/document-processor:{1}', github.repository_owner, env.VERSION) || '' }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -0,0 +1,35 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ vars.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_PAT }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
driver: cloud
|
||||
endpoint: "christianlouis/cklbuilder"
|
||||
install: true
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
tags: "${{ vars.DOCKER_USER }}/docuelevate:latest"
|
||||
outputs: ${{ github.event_name == 'pull_request' && 'type=cacheonly' || 'type=registry' }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
@@ -1,80 +0,0 @@
|
||||
name: Semantic Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
packages: write
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Semantic Release
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'christianlouis/DocuElevate'
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install python-semantic-release
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Run Semantic Release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
semantic-release version --print
|
||||
semantic-release version
|
||||
semantic-release publish
|
||||
|
||||
- name: Update changelog if no new version was released
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
if git diff --name-only HEAD~1 2>/dev/null | grep -q CHANGELOG.md; then
|
||||
echo "CHANGELOG.md was already updated by semantic-release version"
|
||||
else
|
||||
semantic-release changelog
|
||||
if ! git diff --quiet CHANGELOG.md; then
|
||||
git add CHANGELOG.md
|
||||
git commit -m "docs(changelog): update changelog [skip ci]"
|
||||
git push
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Update build metadata files if changed
|
||||
run: |
|
||||
for f in VERSION BUILD_DATE GIT_SHA RUNTIME_INFO; do
|
||||
if [ -f "$f" ]; then
|
||||
git add -f "$f"
|
||||
fi
|
||||
done
|
||||
if ! git diff --staged --quiet; then
|
||||
git commit -m "chore(release): update build metadata files [skip ci]"
|
||||
git push
|
||||
fi
|
||||
@@ -1,97 +0,0 @@
|
||||
name: Ruff Auto-Fix
|
||||
|
||||
# This workflow automatically fixes ruff formatting and linting issues
|
||||
# and commits them back to the PR branch when issues are detected.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
paths:
|
||||
- '**.py'
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
ruff-auto-fix:
|
||||
name: Auto-fix Ruff Issues
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on PRs from the same repository (not forks) for security
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
|
||||
steps:
|
||||
- name: Checkout PR branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Ruff
|
||||
run: pip install ruff
|
||||
|
||||
- name: Run Ruff Check with Auto-fix
|
||||
run: |
|
||||
echo "Running ruff check with auto-fix..."
|
||||
ruff check app/ tests/ --fix || true
|
||||
|
||||
- name: Run Ruff Format
|
||||
run: |
|
||||
echo "Running ruff format..."
|
||||
ruff format app/ tests/
|
||||
|
||||
- name: Check for changes
|
||||
id: check_changes
|
||||
run: |
|
||||
if [[ -n $(git status --porcelain) ]]; then
|
||||
echo "changes=true" >> $GITHUB_OUTPUT
|
||||
echo "Changes detected after running ruff auto-fix"
|
||||
else
|
||||
echo "changes=false" >> $GITHUB_OUTPUT
|
||||
echo "No changes needed - code is already properly formatted"
|
||||
fi
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.check_changes.outputs.changes == 'true'
|
||||
run: |
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
git add app/ tests/
|
||||
git commit -m "style: apply ruff auto-fix
|
||||
|
||||
- Auto-formatted code with ruff format
|
||||
- Applied ruff linting fixes with --fix
|
||||
|
||||
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
|
||||
git push
|
||||
|
||||
- name: Comment on PR
|
||||
if: steps.check_changes.outputs.changes == 'true'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '✨ Ruff auto-fix applied! The code has been automatically formatted and linting issues have been fixed.\n\nPlease pull the latest changes:\n```bash\ngit pull\n```'
|
||||
})
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
if [[ "${{ steps.check_changes.outputs.changes }}" == "true" ]]; then
|
||||
echo "✅ Ruff auto-fix completed and changes committed"
|
||||
else
|
||||
echo "✅ No changes needed - code is already properly formatted"
|
||||
fi
|
||||
@@ -0,0 +1,56 @@
|
||||
name: Run Tests & Linting
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Tests
|
||||
run: pytest tests/ -v --cov=app --cov-report=xml --cov-report=term
|
||||
|
||||
- name: Upload Coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Run Linter (Flake8)
|
||||
run: flake8 app/ --max-line-length=120 --extend-ignore=E203,W503
|
||||
continue-on-error: false
|
||||
|
||||
- name: Run Code Formatter (Black)
|
||||
run: black --check app/ --line-length=120
|
||||
continue-on-error: false
|
||||
|
||||
- name: Run Type Checker (Mypy)
|
||||
run: mypy app/ --ignore-missing-imports
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Linter (Pylint)
|
||||
run: pylint app/ --max-line-length=120 --disable=C0111,C0103,R0903
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Security Linter (Bandit)
|
||||
run: bandit -r app/ -ll -f json -o bandit-report.json
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Bandit Report
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: bandit-report
|
||||
path: bandit-report.json
|
||||
+196
-202
@@ -1,202 +1,196 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# Environment files - NEVER commit these!
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
*.env
|
||||
|
||||
# Secrets and credentials
|
||||
*secret*
|
||||
*credentials*.json
|
||||
!frontend/static/* # Allow static files even if they match patterns
|
||||
!docs/* # Allow documentation files
|
||||
|
||||
# Private keys
|
||||
*.pem
|
||||
*.key
|
||||
*.p12
|
||||
*.pfx
|
||||
id_rsa*
|
||||
ssh_host_*
|
||||
|
||||
# Database files - may contain sensitive data
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
database.db
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
junit.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
/docs_build
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Build metadata files - generated at build time
|
||||
GIT_SHA
|
||||
RUNTIME_INFO
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# Environment files - NEVER commit these!
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
*.env
|
||||
|
||||
# Secrets and credentials
|
||||
*secret*
|
||||
*credentials*.json
|
||||
!frontend/static/* # Allow static files even if they match patterns
|
||||
!docs/* # Allow documentation files
|
||||
|
||||
# Private keys
|
||||
*.pem
|
||||
*.key
|
||||
*.p12
|
||||
*.pfx
|
||||
id_rsa*
|
||||
ssh_host_*
|
||||
|
||||
# Database files - may contain sensitive data
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
database.db
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
+28
-14
@@ -19,13 +19,35 @@ repos:
|
||||
- id: detect-aws-credentials
|
||||
args: ['--allow-missing-credentials']
|
||||
|
||||
# Ruff - Fast Python linter and formatter (replaces Black, Flake8, isort, Bandit)
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.3.0
|
||||
# Python code formatting
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.1.1
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [ --fix ]
|
||||
- id: ruff-format
|
||||
- id: black
|
||||
args: ['--line-length=120']
|
||||
language_version: python3.11
|
||||
|
||||
# Import sorting
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
args: ['--profile=black', '--line-length=120']
|
||||
|
||||
# Linting
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 7.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
args: ['--max-line-length=120', '--extend-ignore=E203,W503']
|
||||
|
||||
# Security linting
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.7.6
|
||||
hooks:
|
||||
- id: bandit
|
||||
args: ['-ll', '-r', 'app/']
|
||||
exclude: 'tests/'
|
||||
|
||||
# Type checking
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
@@ -47,11 +69,3 @@ repos:
|
||||
.+\.json|
|
||||
.env.demo
|
||||
)$
|
||||
|
||||
# Conventional commits validation
|
||||
- repo: https://github.com/compilerla/conventional-pre-commit
|
||||
rev: v3.0.0
|
||||
hooks:
|
||||
- id: conventional-pre-commit
|
||||
stages: [commit-msg]
|
||||
args: []
|
||||
|
||||
+1
-1
@@ -19,4 +19,4 @@ mkdocs:
|
||||
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- requirements: docs/requirements.txt
|
||||
Vendored
-7
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"python.testing.pytestArgs": [
|
||||
"tests"
|
||||
],
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.pytestEnabled": true
|
||||
}
|
||||
+36
-129
@@ -1,6 +1,6 @@
|
||||
# Agentic Coding Guide for DocuElevate
|
||||
|
||||
**Version:** 1.0
|
||||
**Version:** 1.0
|
||||
**Last Updated:** 2026-02-06
|
||||
|
||||
This guide helps AI coding agents work effectively with the DocuElevate codebase. It provides context, conventions, and best practices for autonomous code contributions.
|
||||
@@ -71,39 +71,23 @@ DocuElevate/
|
||||
- Ensure you understand the Celery task flow
|
||||
- Consider impact on database schema
|
||||
|
||||
### Documentation-First Principle
|
||||
|
||||
**Documentation is as important as tests and code.** Every change must include documentation updates in the same commit/PR.
|
||||
|
||||
| Change type | What to update |
|
||||
|-------------|---------------|
|
||||
| New feature | `docs/UserGuide.md`, `docs/API.md` (if API), `docs/ConfigurationGuide.md` (if config) |
|
||||
| New config option | `docs/ConfigurationGuide.md` and `.env.demo` |
|
||||
| New API endpoint | `docs/API.md` |
|
||||
| Bug fix (user-visible) | `docs/Troubleshooting.md` |
|
||||
| Deployment change | `docs/DeploymentGuide.md` |
|
||||
| Security change | `SECURITY_AUDIT.md` |
|
||||
| Breaking change | CHANGELOG (auto-generated) + migration notes in relevant docs |
|
||||
|
||||
**Never edit `CHANGELOG.md` or `VERSION` manually.** These are managed automatically by `python-semantic-release` on every merge to `main`.
|
||||
|
||||
### Code Conventions
|
||||
|
||||
#### Python Style
|
||||
```python
|
||||
# Use Ruff formatting (line length: 120)
|
||||
# Use Black formatting (line length: 120)
|
||||
# Use type hints
|
||||
def process_document(file_path: str, metadata: Dict[str, Any]) -> DocumentMetadata:
|
||||
"""
|
||||
Process a document and extract metadata.
|
||||
|
||||
|
||||
Args:
|
||||
file_path: Absolute path to the document file
|
||||
metadata: Additional metadata to include
|
||||
|
||||
|
||||
Returns:
|
||||
DocumentMetadata object with extracted information
|
||||
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
ProcessingError: If processing fails
|
||||
@@ -241,7 +225,7 @@ logger = logging.getLogger(__name__)
|
||||
def my_background_task(self, param: str):
|
||||
"""
|
||||
Description of what this task does.
|
||||
|
||||
|
||||
Args:
|
||||
param: Description of parameter
|
||||
"""
|
||||
@@ -263,7 +247,7 @@ def my_background_task(self, param: str):
|
||||
```python
|
||||
class MyModel(Base):
|
||||
__tablename__ = "my_table"
|
||||
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -291,20 +275,20 @@ logger = logging.getLogger(__name__)
|
||||
def upload_to_my_provider(file_path: str, metadata: dict) -> str:
|
||||
"""
|
||||
Upload file to My Provider.
|
||||
|
||||
|
||||
Args:
|
||||
file_path: Local path to file
|
||||
metadata: Document metadata
|
||||
|
||||
|
||||
Returns:
|
||||
URL or ID of uploaded file
|
||||
|
||||
|
||||
Raises:
|
||||
ProviderError: If upload fails
|
||||
"""
|
||||
if not settings.my_provider_api_key:
|
||||
raise ValueError("MY_PROVIDER_API_KEY not configured")
|
||||
|
||||
|
||||
# Implementation
|
||||
pass
|
||||
```
|
||||
@@ -358,11 +342,11 @@ def validate_file_path(file_path: str, base_dir: str = "/workdir") -> Path:
|
||||
try:
|
||||
path = Path(file_path).resolve()
|
||||
base = Path(base_dir).resolve()
|
||||
|
||||
|
||||
# Ensure path is within base directory
|
||||
if not path.is_relative_to(base):
|
||||
raise ValueError("Path outside allowed directory")
|
||||
|
||||
|
||||
return path
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
@@ -483,23 +467,23 @@ def process_large_batch(self, file_ids: List[int]):
|
||||
def complex_function(param1: str, param2: int = 10) -> Dict[str, Any]:
|
||||
"""
|
||||
One-line summary of what the function does.
|
||||
|
||||
|
||||
More detailed explanation if needed. Can span multiple
|
||||
lines and include examples.
|
||||
|
||||
|
||||
Args:
|
||||
param1: Description of param1
|
||||
param2: Description of param2, defaults to 10
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary containing:
|
||||
- key1: Description
|
||||
- key2: Description
|
||||
|
||||
|
||||
Raises:
|
||||
ValueError: If param1 is empty
|
||||
FileNotFoundError: If file doesn't exist
|
||||
|
||||
|
||||
Examples:
|
||||
>>> result = complex_function("test", 5)
|
||||
>>> print(result['key1'])
|
||||
@@ -584,7 +568,7 @@ logger.info(f"Password: {password}") # BAD!
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Git Workflow & Versioning
|
||||
## 🔄 Git Workflow
|
||||
|
||||
### Branch Names
|
||||
- `feature/description` - New features
|
||||
@@ -593,103 +577,28 @@ logger.info(f"Password: {password}") # BAD!
|
||||
- `refactor/description` - Code refactoring
|
||||
- `docs/description` - Documentation updates
|
||||
|
||||
### Conventional Commits (REQUIRED)
|
||||
|
||||
**All commit messages MUST follow the Conventional Commits specification for automated versioning.**
|
||||
|
||||
#### Format
|
||||
### Commit Messages
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
type(scope): Short description (max 72 chars)
|
||||
|
||||
<body>
|
||||
Longer description if needed. Explain:
|
||||
- What changed
|
||||
- Why it changed
|
||||
- Any breaking changes
|
||||
|
||||
<footer>
|
||||
Fixes #123
|
||||
```
|
||||
|
||||
#### Commit Types and Version Bumps
|
||||
- **feat**: New feature → **minor version bump** (0.5.0 → 0.6.0)
|
||||
- **fix**: Bug fix → **patch version bump** (0.5.0 → 0.5.1)
|
||||
- **perf**: Performance improvement → **patch version bump**
|
||||
- **docs**: Documentation only → **no version bump**
|
||||
- **style**: Code style/formatting → **no version bump**
|
||||
- **refactor**: Code refactoring → **no version bump**
|
||||
- **test**: Test changes → **no version bump**
|
||||
- **build**: Build system changes → **no version bump**
|
||||
- **ci**: CI/CD changes → **no version bump**
|
||||
- **chore**: Other changes → **no version bump**
|
||||
|
||||
#### Breaking Changes
|
||||
Add `!` after type/scope or include `BREAKING CHANGE:` in footer for **major version bump**:
|
||||
```
|
||||
feat(api)!: redesign authentication endpoints
|
||||
|
||||
BREAKING CHANGE: OAuth2 tokens now required instead of API keys
|
||||
```
|
||||
Result: 0.5.0 → 1.0.0
|
||||
|
||||
#### Scope Examples
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend/UI changes
|
||||
- `auth` - Authentication
|
||||
- `storage` - Storage providers
|
||||
- `ocr` - OCR processing
|
||||
- `tasks` - Celery tasks
|
||||
- `config` - Configuration
|
||||
|
||||
#### Good Commit Examples
|
||||
```
|
||||
feat(storage): add Amazon S3 storage provider
|
||||
|
||||
Implements S3StorageProvider with upload, download, delete operations.
|
||||
Includes configuration for bucket, region, and credentials.
|
||||
|
||||
Closes #123
|
||||
```
|
||||
|
||||
```
|
||||
fix(ocr): handle PDFs without text layer
|
||||
|
||||
Previously failed silently. Now properly processes through Azure.
|
||||
|
||||
Fixes #456
|
||||
```
|
||||
|
||||
```
|
||||
docs: update deployment guide with Docker Compose
|
||||
|
||||
Added step-by-step instructions for Docker Compose deployment.
|
||||
```
|
||||
|
||||
### Semantic Release Automation
|
||||
|
||||
DocuElevate uses `python-semantic-release` for automated version management.
|
||||
|
||||
#### How It Works
|
||||
1. **PR merges to main** with conventional commits
|
||||
2. **semantic-release analyzes** commit messages
|
||||
3. **Automatic updates**:
|
||||
- Bumps `VERSION` file
|
||||
- Updates `CHANGELOG.md`
|
||||
- Creates Git tag (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release
|
||||
- Triggers Docker builds
|
||||
|
||||
#### Agent Rules
|
||||
- ✅ **DO**: Write conventional commit messages
|
||||
- ✅ **DO**: Use correct commit types
|
||||
- ✅ **DO**: Include `BREAKING CHANGE:` when applicable
|
||||
- ❌ **DON'T**: Manually edit `VERSION` file
|
||||
- ❌ **DON'T**: Manually edit `CHANGELOG.md`
|
||||
- ❌ **DON'T**: Create version tags or releases manually
|
||||
Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
|
||||
|
||||
### Pull Requests
|
||||
1. Create PR with descriptive title (conventional format if single change)
|
||||
1. Create PR with descriptive title
|
||||
2. Fill out PR template
|
||||
3. Link related issues
|
||||
4. Ensure CI passes
|
||||
5. Request reviews
|
||||
6. Address feedback
|
||||
7. Merge when approved (commits retain conventional format)
|
||||
7. Squash merge when approved
|
||||
|
||||
---
|
||||
|
||||
@@ -697,28 +606,26 @@ DocuElevate uses `python-semantic-release` for automated version management.
|
||||
|
||||
Before submitting code:
|
||||
|
||||
- [ ] Code follows style guide (Ruff formatted)
|
||||
- [ ] Commit messages use conventional commit format
|
||||
- [ ] Code follows style guide (Black formatted)
|
||||
- [ ] All tests pass (`pytest`)
|
||||
- [ ] New code has tests
|
||||
- [ ] Coverage doesn't decrease
|
||||
- [ ] Documentation updated if needed
|
||||
- [ ] No secrets or credentials in code
|
||||
- [ ] Linting passes (`ruff check`)
|
||||
- [ ] Linting passes (`flake8`, `pylint`)
|
||||
- [ ] Type hints added (`mypy` clean)
|
||||
- [ ] No manual edits to `VERSION` or `CHANGELOG.md`
|
||||
- [ ] Security scan passed (included in `ruff check`)
|
||||
- [ ] CHANGELOG.md updated (if user-facing)
|
||||
- [ ] Security scan passed (`bandit`)
|
||||
|
||||
Run full check:
|
||||
```bash
|
||||
pytest --cov=app
|
||||
ruff check app/ tests/
|
||||
ruff format --check app/ tests/
|
||||
black app/ tests/
|
||||
flake8 app/ --max-line-length=120
|
||||
mypy app/
|
||||
bandit -r app/
|
||||
```
|
||||
|
||||
**Note:** This project uses Ruff, which replaces Black, Flake8, isort, and Bandit with a single, faster tool.
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Agent Collaboration
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
# Agent Pre-PR Ruff Policy
|
||||
|
||||
This document defines the policy for Copilot/AI Agents when creating or updating pull requests in the DocuElevate repository.
|
||||
|
||||
## Required Steps Before Creating/Updating a PR
|
||||
|
||||
### 1. Run the Ruff Auto-Fix Script
|
||||
|
||||
Before creating or updating a branch or opening a PR, **ALWAYS** run:
|
||||
|
||||
```bash
|
||||
./scripts/run-ruff-and-fix.sh
|
||||
```
|
||||
|
||||
### 2. Handle Script Exit Codes
|
||||
|
||||
**If the script exits with code 0 (success):**
|
||||
- ✅ All ruff checks passed or were auto-fixed
|
||||
- Continue with PR creation process
|
||||
- Push the branch (including any auto-fix commits)
|
||||
|
||||
**If the script exits with code non-zero (failure):**
|
||||
- ❌ Stop PR creation immediately
|
||||
- Do NOT open the PR
|
||||
- Report the ruff failures to the human user
|
||||
- Include the full `ruff check` output in your response
|
||||
- Create an issue or report the failures for manual resolution
|
||||
|
||||
### 3. Handle Auto-Fix Commits
|
||||
|
||||
If the script commits fixes:
|
||||
- The script will automatically commit changes with message: `style: ruff auto-fixes`
|
||||
- Push the branch with the commit(s) before continuing PR creation
|
||||
- This ensures CI sees the fixed code when the PR is opened
|
||||
|
||||
## Example Agent Workflow
|
||||
|
||||
```
|
||||
1. Clone repository and create feature branch
|
||||
2. Make code changes to implement feature
|
||||
3. Run: ./scripts/run-ruff-and-fix.sh
|
||||
4. Check exit code:
|
||||
- If 0: Push branch and create PR
|
||||
- If non-zero: Report failures, do NOT create PR
|
||||
5. Run project tests (if ruff passed)
|
||||
6. If all checks pass: Create the PR
|
||||
7. If any check fails: Report failures instead of creating PR
|
||||
```
|
||||
|
||||
## Why This Policy Exists
|
||||
|
||||
1. **Prevent CI Failures**: Ensures PRs don't break ruff checks in CI
|
||||
2. **Auto-Fix Minor Issues**: Automatically fixes formatting and simple linting issues
|
||||
3. **Surface Manual Issues Early**: Identifies issues that need human attention before PR creation
|
||||
4. **Maintain Code Quality**: Enforces consistent code style across the repository
|
||||
|
||||
## Integration with Existing CI
|
||||
|
||||
DocuElevate has two workflows that handle ruff:
|
||||
|
||||
1. **`.github/workflows/ci.yml`** (Lint Job)
|
||||
- Runs `ruff check` (without --fix) on all pushes and PRs
|
||||
- Fails CI if issues are found
|
||||
- Runs early in the pipeline to catch style issues before tests
|
||||
|
||||
2. **`.github/workflows/ruff-auto-fix.yml`**
|
||||
- Runs on PRs when Python files change
|
||||
- Automatically applies `ruff --fix` and `ruff format`
|
||||
- Commits fixes back to the PR branch
|
||||
- Posts a comment notifying the author
|
||||
|
||||
This agent script ensures that most issues are caught and fixed **before** the PR is created, reducing the need for the auto-fix workflow to intervene.
|
||||
|
||||
## Local Development
|
||||
|
||||
Developers should also use this script or set up pre-commit hooks:
|
||||
|
||||
```bash
|
||||
# Install pre-commit hooks (recommended)
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
|
||||
# Or run manually before committing
|
||||
./scripts/run-ruff-and-fix.sh
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Script fails with "ruff: command not found"
|
||||
|
||||
The script installs ruff automatically. If this fails:
|
||||
```bash
|
||||
pip install ruff
|
||||
```
|
||||
|
||||
### Script fails with Git errors
|
||||
|
||||
Ensure you're in a Git repository with proper configuration:
|
||||
```bash
|
||||
git config user.name "Your Name"
|
||||
git config user.email "your.email@example.com"
|
||||
```
|
||||
|
||||
### Ruff issues remain after --fix
|
||||
|
||||
Some issues cannot be auto-fixed (e.g., unused imports, complex logic issues). These require manual resolution:
|
||||
1. Review the ruff output
|
||||
2. Fix the issues manually
|
||||
3. Run the script again to verify
|
||||
|
||||
## Configuration
|
||||
|
||||
Ruff configuration is in `pyproject.toml` under `[tool.ruff]` and `[tool.ruff.lint]`.
|
||||
|
||||
Default settings:
|
||||
- Line length: 120 characters
|
||||
- Target Python version: 3.11+
|
||||
- Enabled rules: Pyflakes (F), pycodestyle (E, W), isort (I), bandit (S), flake8-bugbear (B), pylint (PL)
|
||||
|
||||
## Questions?
|
||||
|
||||
See the [Contributing Guide](CONTRIBUTING.md) for more information on code quality standards and development workflow.
|
||||
@@ -1,8 +1,8 @@
|
||||
# Repository Analysis & Improvement Summary
|
||||
|
||||
**Date:** 2026-02-06
|
||||
**Repository:** christianlouis/DocuElevate
|
||||
**Current Version:** v0.5.0
|
||||
**Date:** 2026-02-06
|
||||
**Repository:** christianlouis/DocuElevate
|
||||
**Current Version:** v0.3.2
|
||||
|
||||
## Executive Summary
|
||||
|
||||
@@ -101,7 +101,7 @@ tests/
|
||||
- ✅ Added Bandit security scanning
|
||||
- ✅ Improved linting configuration (line length: 120)
|
||||
|
||||
**New codeql.yml:**
|
||||
**New codeql.yaml:**
|
||||
- ✅ Security scanning for Python and JavaScript
|
||||
- ✅ Scheduled weekly scans
|
||||
- ✅ Runs on PRs and main branch pushes
|
||||
@@ -230,7 +230,7 @@ httpx>=0.26.0
|
||||
|
||||
### Medium Priority (Next Month)
|
||||
- [ ] Fix Pydantic V1 → V2 migration warnings
|
||||
- [x] Migrate from PyPDF2 to pypdf (modern fork) - ✅ Completed 2026-02-12
|
||||
- [ ] Migrate from PyPDF2 to pypdf (modern fork)
|
||||
- [ ] Consolidate storage provider code
|
||||
- [ ] Add API pagination
|
||||
- [ ] Implement retry logic for Celery tasks
|
||||
@@ -343,6 +343,6 @@ httpx>=0.26.0
|
||||
|
||||
---
|
||||
|
||||
**Prepared by:** GitHub Copilot Agent
|
||||
**Review Status:** Ready for maintainer review
|
||||
**Prepared by:** GitHub Copilot Agent
|
||||
**Review Status:** Ready for maintainer review
|
||||
**Recommended Action:** Merge and continue with TODO.md priorities
|
||||
+1
-1
@@ -1 +1 @@
|
||||
2026-03-13T08:33:59Z
|
||||
2025-04-11
|
||||
|
||||
-2805
File diff suppressed because it is too large
Load Diff
+15
-335
@@ -31,164 +31,8 @@ We welcome feature requests! Please submit an issue with:
|
||||
1. Fork the repository
|
||||
2. Create a new branch for your changes
|
||||
3. Make your changes
|
||||
4. **Follow conventional commit format** (see below)
|
||||
5. Run the tests to ensure everything works
|
||||
6. Submit a pull request with a clear description of the changes
|
||||
|
||||
## Commit Message Format
|
||||
|
||||
DocuElevate follows the [Conventional Commits](https://www.conventionalcommits.org/) specification for commit messages. This enables automatic version bumping and changelog generation.
|
||||
|
||||
### Format
|
||||
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
<footer>
|
||||
```
|
||||
|
||||
### Type
|
||||
|
||||
Must be one of the following:
|
||||
|
||||
- **feat**: A new feature (triggers minor version bump)
|
||||
- **fix**: A bug fix (triggers patch version bump)
|
||||
- **docs**: Documentation only changes
|
||||
- **style**: Changes that don't affect code meaning (formatting, etc.)
|
||||
- **refactor**: Code change that neither fixes a bug nor adds a feature
|
||||
- **perf**: Performance improvement (triggers patch version bump)
|
||||
- **test**: Adding or updating tests
|
||||
- **build**: Changes to build system or dependencies
|
||||
- **ci**: Changes to CI configuration files and scripts
|
||||
- **chore**: Other changes that don't modify src or test files
|
||||
|
||||
### Scope (Optional)
|
||||
|
||||
The scope should be the name of the affected module or area:
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend/UI changes
|
||||
- `auth` - Authentication changes
|
||||
- `storage` - Storage provider changes
|
||||
- `ocr` - OCR processing changes
|
||||
- `tasks` - Celery task changes
|
||||
- `config` - Configuration changes
|
||||
|
||||
### Subject
|
||||
|
||||
The subject contains a succinct description of the change:
|
||||
- Use imperative, present tense: "change" not "changed" nor "changes"
|
||||
- Don't capitalize first letter
|
||||
- No period (.) at the end
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
For breaking changes, add `!` after the type/scope or include `BREAKING CHANGE:` in the footer:
|
||||
|
||||
```
|
||||
feat!: redesign authentication API
|
||||
|
||||
BREAKING CHANGE: The /api/auth endpoint now requires OAuth2 tokens instead of API keys.
|
||||
```
|
||||
|
||||
This triggers a major version bump.
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
feat(storage): add support for Amazon S3 storage provider
|
||||
|
||||
Add S3StorageProvider class with upload, download, and delete operations.
|
||||
Includes configuration options for bucket name, region, and credentials.
|
||||
|
||||
Closes #123
|
||||
```
|
||||
|
||||
```
|
||||
fix(ocr): handle PDF files without text layer
|
||||
|
||||
Previously, PDFs without existing text layers would fail silently.
|
||||
Now properly processes them through Azure Document Intelligence.
|
||||
|
||||
Fixes #456
|
||||
```
|
||||
|
||||
```
|
||||
docs: update deployment guide with Docker Compose setup
|
||||
|
||||
Added step-by-step instructions for deploying with Docker Compose,
|
||||
including environment variable configuration and service dependencies.
|
||||
```
|
||||
|
||||
```
|
||||
chore: update dependencies to fix security vulnerabilities
|
||||
|
||||
Updated authlib to 1.6.5+ and starlette to 0.49.1+
|
||||
```
|
||||
|
||||
## Versioning and Releases
|
||||
|
||||
DocuElevate uses [semantic-release](https://github.com/semantic-release/semantic-release) for automated version management and releases:
|
||||
|
||||
- **Releases are automated**: When PRs are merged to `main`, semantic-release analyzes commit messages and automatically:
|
||||
- Determines the next version number
|
||||
- Updates the `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates a Git tag with `v` prefix (e.g., `v0.6.0`)
|
||||
- Creates a GitHub Release with auto-generated notes
|
||||
- Triggers Docker image builds with the new version tag
|
||||
|
||||
- **Version Bumps**:
|
||||
- `feat:` commits → minor version bump (0.5.0 → 0.6.0)
|
||||
- `fix:` or `perf:` commits → patch version bump (0.5.0 → 0.5.1)
|
||||
- `feat!:` or `BREAKING CHANGE:` → major version bump (0.5.0 → 1.0.0)
|
||||
- Other commit types (docs, chore, etc.) → no version bump
|
||||
|
||||
- **Manual Version Changes**: Do NOT manually edit `VERSION` or `CHANGELOG.md` - these are managed by semantic-release
|
||||
|
||||
## Documentation-First Development
|
||||
|
||||
Documentation is a first-class citizen in DocuElevate. Every contribution **must** include relevant documentation updates. This is not optional.
|
||||
|
||||
### What Requires Documentation
|
||||
|
||||
| Change Type | Required Documentation |
|
||||
|-------------|----------------------|
|
||||
| New feature | User Guide + API docs (if API change) + Configuration Guide (if new config) |
|
||||
| Bug fix | Troubleshooting guide (if user-facing) |
|
||||
| New config option | ConfigurationGuide.md + `.env.demo` example |
|
||||
| New API endpoint | docs/API.md |
|
||||
| Deployment change | DeploymentGuide.md |
|
||||
| Security change | SECURITY_AUDIT.md |
|
||||
| Breaking change | CHANGELOG.md note + migration instructions |
|
||||
|
||||
### Documentation Standards
|
||||
|
||||
- Keep `docs/` files in sync with code changes in the same PR
|
||||
- Update `TODO.md` when completing or adding tasks
|
||||
- `CHANGELOG.md` is generated automatically—**do not add regular release entries manually**. Retroactive corrections to historical entries are the only acceptable exception.
|
||||
- Screenshots in README and docs should reflect current UI; update them when the UI changes significantly
|
||||
- Use present tense and second person ("you") in user-facing docs
|
||||
|
||||
### Automated Changelog
|
||||
|
||||
`CHANGELOG.md` is generated automatically by [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) on every merge to `main`. **Do not edit it manually.** Your commit messages (following Conventional Commits) drive the changelog content.
|
||||
|
||||
---
|
||||
|
||||
## Pull Request Checklist
|
||||
|
||||
Before submitting a pull request:
|
||||
|
||||
- [ ] Code follows the project style guide (Ruff)
|
||||
- [ ] Commit messages follow conventional commit format
|
||||
- [ ] Pre-commit hooks installed and passing (see below)
|
||||
- [ ] Tests added/updated for new functionality
|
||||
- [ ] **Documentation updated** for any user-facing, API, or configuration changes
|
||||
- [ ] No manual edits to `VERSION` or `CHANGELOG.md`
|
||||
- [ ] All tests pass locally
|
||||
- [ ] Security scan passes (if applicable)
|
||||
4. Run the tests to ensure everything works
|
||||
5. Submit a pull request with a clear description of the changes
|
||||
|
||||
## Development Environment
|
||||
|
||||
@@ -196,8 +40,8 @@ Before submitting a pull request:
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/christianlouis/DocuElevate.git
|
||||
cd DocuElevate
|
||||
git clone https://github.com/christianlouis/document-processor.git
|
||||
cd document-processor
|
||||
|
||||
# Create a virtual environment
|
||||
python -m venv venv
|
||||
@@ -206,193 +50,30 @@ source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Install pre-commit hooks (recommended)
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pre-commit Hooks
|
||||
|
||||
Pre-commit hooks automatically check your code before each commit, catching issues early:
|
||||
|
||||
```bash
|
||||
# Install the hooks (one-time setup)
|
||||
pre-commit install
|
||||
|
||||
# Run hooks manually on all files
|
||||
pre-commit run --all-files
|
||||
|
||||
# Run hooks on staged files (happens automatically on commit)
|
||||
pre-commit run
|
||||
```
|
||||
|
||||
The pre-commit hooks include:
|
||||
- **Ruff** - Linting and formatting (with auto-fix)
|
||||
- **Mypy** - Type checking
|
||||
- **detect-secrets** - Secret detection
|
||||
- **Conventional commits** - Commit message validation
|
||||
- File checks (trailing whitespace, large files, etc.)
|
||||
|
||||
### Running Tests
|
||||
|
||||
DocuElevate has comprehensive test coverage including unit tests, integration tests, and end-to-end tests. Tests are automatically configured with the necessary environment variables.
|
||||
|
||||
#### Quick Test Commands
|
||||
|
||||
```bash
|
||||
# Run all tests (default configuration)
|
||||
pytest
|
||||
|
||||
# Run with verbose output
|
||||
pytest -v
|
||||
|
||||
# Run with coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run only unit tests (fast, no Docker required)
|
||||
pytest -m unit
|
||||
|
||||
# Run only integration tests
|
||||
pytest -m integration
|
||||
|
||||
# Run specific test file
|
||||
pytest tests/test_api.py -v
|
||||
```
|
||||
|
||||
#### Test Environment Configuration
|
||||
|
||||
Tests automatically configure the required environment variables in `tests/conftest.py`:
|
||||
|
||||
- `DATABASE_URL`: Uses SQLite in-memory database for fast, isolated tests
|
||||
- `AUTH_ENABLED`: Set to `False` by default for simpler unit tests
|
||||
- `SESSION_SECRET`: Pre-configured with a valid 32+ character secret for tests that need it
|
||||
- `OPENAI_API_KEY`, `AZURE_AI_KEY`, etc.: Pre-configured with test values
|
||||
|
||||
**No manual environment setup is needed to run tests!**
|
||||
|
||||
#### Testing with Authentication Enabled
|
||||
|
||||
Some tests specifically verify authentication behavior with `AUTH_ENABLED=True`. These tests:
|
||||
|
||||
1. Use `@patch("app.auth.AUTH_ENABLED", True)` to enable auth for specific tests
|
||||
2. Properly configure `SESSION_SECRET` (already set in conftest.py)
|
||||
3. Mock user sessions to test protected endpoints
|
||||
4. Verify login redirects and access control
|
||||
|
||||
Example:
|
||||
```python
|
||||
from unittest.mock import patch
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_protected_endpoint_with_auth(client):
|
||||
"""Test endpoint requires authentication when auth is enabled."""
|
||||
with patch("app.auth.AUTH_ENABLED", True):
|
||||
# Test will verify redirect to /login
|
||||
response = client.get("/protected-page")
|
||||
assert response.status_code == 302
|
||||
```
|
||||
|
||||
#### Integration Tests with Docker
|
||||
|
||||
Some tests require Docker to spin up real infrastructure (PostgreSQL, Redis, WebDAV, etc.):
|
||||
|
||||
```bash
|
||||
# Run integration tests that need Docker
|
||||
pytest -m requires_docker -v
|
||||
|
||||
# Run end-to-end tests with full stack
|
||||
pytest -m e2e -v
|
||||
```
|
||||
|
||||
See [tests/README_INTEGRATION_TESTS.md](tests/README_INTEGRATION_TESTS.md) for detailed information about integration testing.
|
||||
|
||||
#### Test Markers
|
||||
|
||||
Tests are organized using pytest markers:
|
||||
|
||||
- `@pytest.mark.unit` - Fast unit tests with mocks
|
||||
- `@pytest.mark.integration` - Integration tests with some real services
|
||||
- `@pytest.mark.e2e` - Full end-to-end tests
|
||||
- `@pytest.mark.requires_docker` - Requires Docker to run
|
||||
- `@pytest.mark.slow` - Tests that take significant time
|
||||
- `@pytest.mark.security` - Security-related tests
|
||||
|
||||
#### Running Tests in CI
|
||||
|
||||
Tests run automatically in GitHub Actions for all pull requests. The CI workflow is organized in stages:
|
||||
|
||||
**Stage 1: Ruff Lint & Format** (runs first, in parallel with dependency scan)
|
||||
- Checks code style, formatting, and basic security issues
|
||||
- Must pass before tests run
|
||||
|
||||
**Stage 1b: Dependency Vulnerability Scan** (runs in parallel with lint)
|
||||
- Runs `pip-audit` against `requirements.txt` and `requirements-dev.txt`
|
||||
- Fails the build if any known vulnerabilities are detected
|
||||
- Checks the OSV and PyPA advisory databases
|
||||
- Runs independently at the same time as Stage 1 so it does not add to total pipeline time
|
||||
|
||||
**Stage 2: Tests & Type Checking** (runs after lint and dependency scan both pass)
|
||||
|
||||
| Job | Tool | What it checks |
|
||||
|--------|--------|--------------------------------------|
|
||||
| `test` | pytest | Unit/integration tests + coverage |
|
||||
| `mypy` | mypy | Static type checking |
|
||||
|
||||
**Stage 3: Docker Build** (runs after all checks pass)
|
||||
- Builds and pushes Docker images
|
||||
|
||||
**Stage 4: Deploy** (only on main branch)
|
||||
- Deploys to production
|
||||
|
||||
**Auto-fix Workflow:**
|
||||
- A separate `ruff-auto-fix` workflow automatically fixes formatting issues on PRs
|
||||
- Commits fixes back to the PR branch
|
||||
- Only runs on PRs from the same repository (not forks)
|
||||
|
||||
For full details see [docs/CIWorkflow.md](docs/CIWorkflow.md) and [docs/CIToolsGuide.md](docs/CIToolsGuide.md).
|
||||
|
||||
### Code Style
|
||||
|
||||
DocuElevate uses **Ruff** for all Python code quality checks:
|
||||
|
||||
- **Linting** - PEP 8 style, code quality, and security checks
|
||||
- **Formatting** - Consistent code formatting (120 character line length)
|
||||
- **Import sorting** - Organized imports
|
||||
We use:
|
||||
- Black for Python code formatting
|
||||
- Flake8 for linting
|
||||
- isort for import sorting
|
||||
|
||||
```bash
|
||||
# Check for linting issues
|
||||
ruff check app/ tests/
|
||||
# Format code
|
||||
black .
|
||||
|
||||
# Auto-fix linting issues
|
||||
ruff check app/ tests/ --fix
|
||||
# Check linting
|
||||
flake8
|
||||
|
||||
# Check formatting
|
||||
ruff format --check app/ tests/
|
||||
|
||||
# Auto-format code
|
||||
ruff format app/ tests/
|
||||
```
|
||||
|
||||
**Note:** The pre-commit hooks and CI pipeline will automatically check (and optionally fix) these for you.
|
||||
|
||||
### Dependency Vulnerability Scanning
|
||||
|
||||
DocuElevate uses **pip-audit** to scan dependencies for known security vulnerabilities. The CI pipeline runs this automatically and **blocks builds** if any vulnerabilities are found.
|
||||
|
||||
To run locally before pushing:
|
||||
|
||||
```bash
|
||||
# Scan production dependencies
|
||||
pip-audit -r requirements.txt --desc on
|
||||
|
||||
# Scan all dependencies (including dev)
|
||||
pip-audit -r requirements-dev.txt --desc on
|
||||
```
|
||||
|
||||
If pip-audit is not installed, add it with:
|
||||
|
||||
```bash
|
||||
pip install pip-audit
|
||||
# Sort imports
|
||||
isort .
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
@@ -423,7 +104,6 @@ DocuElevate/
|
||||
### Documentation
|
||||
- **[AGENTIC_CODING.md](AGENTIC_CODING.md)** - Comprehensive guide for AI agents and developers
|
||||
- **[README.md](README.md)** - Project overview and quickstart
|
||||
- **[docs/CIWorkflow.md](docs/CIWorkflow.md)** - CI pipeline and linter details for maintainers
|
||||
- **[ROADMAP.md](ROADMAP.md)** - Future features and long-term vision
|
||||
- **[MILESTONES.md](MILESTONES.md)** - Release planning and versioning
|
||||
- **[TODO.md](TODO.md)** - Current tasks and priorities
|
||||
@@ -446,4 +126,4 @@ DocuElevate/
|
||||
- **GitHub Discussions:** Questions and community support
|
||||
- **Documentation:** Check `docs/` directory for guides
|
||||
|
||||
Thank you for contributing to DocuElevate!
|
||||
Thank you for contributing to DocuElevate!
|
||||
@@ -1,131 +0,0 @@
|
||||
# Test Coverage Report
|
||||
|
||||
## Summary
|
||||
|
||||
This PR increases test coverage for two files to meet the 90%+ target:
|
||||
|
||||
- **`app/api/url_upload.py`**: Increased from **80.22%** to **91.21%** ✅
|
||||
- **`app/views/files.py`**: Increased from **18.45%** to **90.61%** ✅
|
||||
|
||||
## Coverage Details
|
||||
|
||||
### app/api/url_upload.py (91.21% coverage)
|
||||
|
||||
**Previous Coverage**: 80.22% (138 statements, 20 missing, 44 branches, 12 partial)
|
||||
**New Coverage**: 91.21% (138 statements, 6 missing, 44 branches, 10 partial)
|
||||
|
||||
#### New Tests Added (10 tests):
|
||||
1. `test_process_url_request_exception` - Tests handling of generic RequestException
|
||||
2. `test_process_url_oserror_during_save` - Tests OSError when saving file to disk
|
||||
3. `test_process_url_unexpected_exception` - Tests handling of unexpected exceptions
|
||||
4. `test_process_url_filename_without_extension` - Tests files without extensions
|
||||
5. `test_process_url_empty_path_uses_download` - Tests default filename for URLs without path
|
||||
6. `test_validate_url_no_hostname` - Tests URL validation without hostname
|
||||
7. `test_validate_file_type_by_extension_fallback` - Tests file type validation by extension
|
||||
8. `test_is_private_ip_ipv6_loopback` - Tests IPv6 loopback detection
|
||||
9. `test_is_private_ip_link_local` - Tests link-local address detection
|
||||
10. `test_process_url_sanitizes_dangerous_filename` - Tests filename sanitization security
|
||||
|
||||
#### Coverage Improvements:
|
||||
- **Error handling**: Now covers all exception handlers (RequestException, OSError, unexpected exceptions)
|
||||
- **Edge cases**: Covers missing hostnames, empty paths, files without extensions
|
||||
- **Security**: IPv6 loopback, link-local addresses, dangerous filename sanitization
|
||||
- **File validation**: Extension-based fallback validation
|
||||
|
||||
### app/views/files.py (90.61% coverage)
|
||||
|
||||
**Previous Coverage**: 18.45% (225 statements, 173 missing, 84 branches, 3 partial)
|
||||
**New Coverage**: 90.61% (225 statements, 14 missing, 84 branches, 13 partial)
|
||||
|
||||
#### New Tests Added (27 tests in new file `test_files_view_extended.py`):
|
||||
|
||||
**Files Page Tests (5 tests):**
|
||||
1. `test_files_page_with_search_filter` - Tests search filtering
|
||||
2. `test_files_page_with_mime_type_filter` - Tests MIME type filtering
|
||||
3. `test_files_page_with_sorting` - Tests sorting (asc/desc)
|
||||
4. `test_files_page_pagination` - Tests pagination with different page sizes
|
||||
5. `test_files_page_error_handling` - Tests error handling
|
||||
|
||||
**File Detail Page Tests (4 tests):**
|
||||
6. `test_file_detail_page_with_existing_file` - Tests detail page for existing file
|
||||
7. `test_file_detail_page_with_missing_file` - Tests 404 handling
|
||||
8. `test_file_detail_page_with_processing_logs` - Tests log display
|
||||
9. `test_file_detail_page_with_metadata` - Tests metadata JSON display
|
||||
|
||||
**File Preview Tests (6 tests):**
|
||||
10. `test_preview_original_file_success` - Tests successful preview of original file
|
||||
11. `test_preview_original_file_not_found` - Tests 404 for non-existent file
|
||||
12. `test_preview_original_file_missing_on_disk` - Tests missing file on disk
|
||||
13. `test_preview_processed_file_success` - Tests successful preview of processed file
|
||||
14. `test_preview_processed_file_not_found` - Tests 404 for non-existent file
|
||||
15. `test_preview_processed_file_missing_on_disk` - Tests missing file on disk
|
||||
|
||||
**Text Extraction Tests (8 tests):**
|
||||
16. `test_get_original_text_success` - Tests successful text extraction from original
|
||||
17. `test_get_original_text_file_not_found` - Tests 404 handling
|
||||
18. `test_get_original_text_file_missing_on_disk` - Tests missing file handling
|
||||
19. `test_get_original_text_extraction_error` - Tests invalid PDF handling
|
||||
20. `test_get_processed_text_success` - Tests successful text extraction from processed
|
||||
21. `test_get_processed_text_file_not_found` - Tests 404 handling
|
||||
22. `test_get_processed_text_file_missing_on_disk` - Tests missing file handling
|
||||
23. `test_get_processed_text_extraction_error` - Tests invalid PDF handling
|
||||
|
||||
**Unit Tests for Helper Functions (4 tests):**
|
||||
24. `test_compute_processing_flow_basic` - Tests processing flow computation
|
||||
25. `test_compute_processing_flow_with_uploads` - Tests flow with upload branches
|
||||
26. `test_compute_step_summary_basic` - Tests step summary computation
|
||||
27. `test_compute_step_summary_order_independent` - Tests order independence
|
||||
|
||||
#### Coverage Improvements:
|
||||
- **Main flow**: Files list page with pagination, sorting, filtering
|
||||
- **Detail pages**: File detail with logs, metadata, file existence checks
|
||||
- **File serving**: Preview original/processed files with error handling
|
||||
- **Text extraction**: On-demand text extraction with error handling
|
||||
- **Helper functions**: Processing flow and step summary computation
|
||||
- **Edge cases**: Missing files, invalid PDFs, error conditions
|
||||
|
||||
## Test Execution Results
|
||||
|
||||
All tests passing:
|
||||
- **url_upload tests**: 39 tests passed
|
||||
- **files view tests**: 30 tests passed
|
||||
- **Total**: 69 tests passed, 0 failures
|
||||
|
||||
## Test Quality
|
||||
|
||||
### Test Structure
|
||||
- Tests organized by feature using pytest classes
|
||||
- Proper use of pytest markers (`@pytest.mark.unit`, `@pytest.mark.integration`, `@pytest.mark.requires_db`)
|
||||
- Clear, descriptive test names following pattern: `test_<what>_<condition>_<expected>`
|
||||
- Comprehensive docstrings for each test
|
||||
|
||||
### Coverage Focus
|
||||
- **Main usage flows**: File upload, listing, detail viewing, preview, text extraction
|
||||
- **Edge conditions**: Missing files, invalid inputs, network errors, file system errors
|
||||
- **Error handling**: All exception paths covered
|
||||
- **Security**: SSRF protection, filename sanitization, input validation
|
||||
|
||||
### Mocking Strategy
|
||||
- External dependencies properly mocked (requests, Celery tasks)
|
||||
- Database operations use test fixtures with in-memory SQLite
|
||||
- File system operations use pytest's `tmp_path` fixture
|
||||
- No actual HTTP requests or file operations outside test environment
|
||||
|
||||
## Files Changed
|
||||
|
||||
1. **tests/test_url_upload.py** - Added 10 new tests
|
||||
2. **tests/test_files_view_extended.py** - Created new file with 27 tests
|
||||
3. Existing tests in **tests/test_files_view.py** - Maintained (3 tests)
|
||||
|
||||
## Validation
|
||||
|
||||
Coverage validated with:
|
||||
```bash
|
||||
pytest tests/test_url_upload.py --cov=app/api/url_upload --cov-report=term-missing
|
||||
# Result: 91.21% coverage
|
||||
|
||||
pytest tests/test_files_view.py tests/test_files_view_extended.py --cov=app/views/files --cov-report=term-missing
|
||||
# Result: 90.61% coverage
|
||||
```
|
||||
|
||||
All tests pass without failures or errors.
|
||||
+5
-42
@@ -7,24 +7,8 @@ WORKDIR /app
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# ── Documentation build stage ───────────────────────────────────────────────
|
||||
FROM python:3.14.1-slim AS docs-builder
|
||||
|
||||
WORKDIR /docs
|
||||
|
||||
# Install MkDocs Material and its dependencies
|
||||
COPY docs/requirements.txt /docs/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy documentation sources
|
||||
COPY docs /docs/docs
|
||||
COPY mkdocs.yml /docs/mkdocs.yml
|
||||
|
||||
# Build the static documentation site
|
||||
RUN mkdocs build --config-file /docs/mkdocs.yml --site-dir /docs/docs_build
|
||||
|
||||
# Second stage for the actual runtime
|
||||
FROM python:3.14.3-slim
|
||||
FROM python:3.14.1-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -32,35 +16,12 @@ WORKDIR /app
|
||||
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
|
||||
# Install system-level OCR tools required for local OCR workflows:
|
||||
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
|
||||
# ghostscript – required by ocrmypdf for PDF/PS operations
|
||||
# poppler-utils – provides pdfinfo/pdftoppm used by pdf2image
|
||||
# unpaper – optional deskewing pre-processor used by ocrmypdf
|
||||
# wget – used by ocr_language_manager to download tessdata files
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
ghostscript \
|
||||
poppler-utils \
|
||||
unpaper \
|
||||
wget \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy application code
|
||||
COPY ./app /app/app
|
||||
COPY ./frontend /app/frontend
|
||||
COPY ./migrations /app/migrations
|
||||
COPY ./alembic.ini /app/alembic.ini
|
||||
COPY ./LICENSE /app/LICENSE
|
||||
|
||||
# Copy build metadata files (generated at build time)
|
||||
COPY ./VERSION /app/VERSION
|
||||
COPY ./frontend /app/frontend
|
||||
COPY ./BUILD_DATE /app/BUILD_DATE
|
||||
COPY ./GIT_SHA /app/GIT_SHA
|
||||
COPY ./RUNTIME_INFO /app/RUNTIME_INFO
|
||||
|
||||
# Copy the pre-built MkDocs documentation site (served at /help)
|
||||
COPY --from=docs-builder /docs/docs_build /app/docs_build
|
||||
COPY ./LICENSE /app/LICENSE
|
||||
|
||||
# Create runtime_info directory
|
||||
RUN mkdir -p /app/runtime_info
|
||||
@@ -77,3 +38,5 @@ EXPOSE 8000
|
||||
|
||||
# Default command
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
# Local development Dockerfile (avoids CI-only build metadata files)
|
||||
FROM python:3.14.1 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# ── Documentation build stage ───────────────────────────────────────────────
|
||||
FROM python:3.14.1-slim AS docs-builder
|
||||
|
||||
WORKDIR /docs
|
||||
|
||||
COPY docs/requirements.txt /docs/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY docs /docs/docs
|
||||
COPY mkdocs.yml /docs/mkdocs.yml
|
||||
|
||||
RUN mkdocs build --config-file /docs/mkdocs.yml --site-dir /docs/docs_build
|
||||
|
||||
FROM python:3.14.1-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
|
||||
# Install system-level OCR tools required for local OCR workflows:
|
||||
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
|
||||
# ghostscript – required by ocrmypdf for PDF/PS operations
|
||||
# poppler-utils – provides pdfinfo/pdftoppm used by pdf2image
|
||||
# unpaper – optional deskewing pre-processor used by ocrmypdf
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
ghostscript \
|
||||
poppler-utils \
|
||||
unpaper \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY ./app /app/app
|
||||
COPY ./frontend /app/frontend
|
||||
COPY ./migrations /app/migrations
|
||||
COPY ./alembic.ini /app/alembic.ini
|
||||
COPY ./LICENSE /app/LICENSE
|
||||
COPY ./VERSION /app/VERSION
|
||||
COPY ./BUILD_DATE /app/BUILD_DATE
|
||||
|
||||
# Copy the pre-built MkDocs documentation site (served at /help)
|
||||
COPY --from=docs-builder /docs/docs_build /app/docs_build
|
||||
|
||||
# Local fallbacks for build metadata
|
||||
RUN echo "local" > /app/GIT_SHA \
|
||||
&& echo "local" > /app/RUNTIME_INFO
|
||||
|
||||
RUN mkdir -p /app/runtime_info
|
||||
RUN mkdir -p /workdir
|
||||
|
||||
ENV PYTHONPATH=/app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
+139
-123
@@ -1,6 +1,6 @@
|
||||
# DocuElevate Milestones
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
**Last Updated:** 2026-02-06
|
||||
|
||||
This document outlines the release milestones, versioning strategy, and detailed feature breakdown for DocuElevate.
|
||||
|
||||
@@ -19,97 +19,21 @@ DocuElevate follows [Semantic Versioning 2.0.0](https://semver.org/):
|
||||
|
||||
---
|
||||
|
||||
## Current Release: v0.5.0 (February 2026)
|
||||
## Current Release: v0.3.2 (February 2026)
|
||||
|
||||
### Status: Stable
|
||||
- Production-ready document processing
|
||||
- Multi-provider storage support
|
||||
- **Database-backed settings management with encryption**
|
||||
- **Setup wizard for first-time configuration**
|
||||
- **Admin UI for runtime configuration**
|
||||
- **Automated semantic versioning and releases**
|
||||
- OAuth2 authentication with admin group support
|
||||
- Basic web UI and REST API
|
||||
|
||||
### Important Note on Versioning
|
||||
As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
- Version management handled by `python-semantic-release`
|
||||
- Releases automated via GitHub Actions on merge to main
|
||||
- Version bumps determined by conventional commit messages
|
||||
- `VERSION` and `CHANGELOG.md` automatically updated
|
||||
- GitHub Releases created automatically with release notes
|
||||
- OAuth2 authentication
|
||||
|
||||
---
|
||||
|
||||
## Previous Releases
|
||||
## Upcoming Milestones
|
||||
|
||||
### v0.3.3 (February 2026)
|
||||
- Drag-and-drop file upload on Files page
|
||||
- Enhanced upload UI and functionality
|
||||
|
||||
### v0.3.2 (February 2026)
|
||||
- Security hardening (Authlib/Starlette updates)
|
||||
- Testing infrastructure implementation
|
||||
- CI/CD improvements
|
||||
|
||||
---
|
||||
|
||||
## Completed Milestones
|
||||
|
||||
### v0.5.0 - Settings Management & Configuration (February 2026)
|
||||
**Release Date:** February 8, 2026
|
||||
**Status:** ✅ Released
|
||||
**Theme:** Configuration Management, Security, User Experience
|
||||
|
||||
#### Goals
|
||||
- [x] **Implement database-backed settings management**
|
||||
- [x] **Add encryption for sensitive configuration**
|
||||
- [x] **Create setup wizard for first-time installation**
|
||||
- [x] Complete settings UI with admin access
|
||||
- [x] Integrate with existing authentication system
|
||||
|
||||
#### Deliverables
|
||||
- [x] **Settings management UI at /settings**
|
||||
- [x] **Setup wizard at /setup**
|
||||
- [x] **Fernet encryption for sensitive settings**
|
||||
- [x] **Source indicators (DB/ENV/DEFAULT)**
|
||||
- [x] **Complete settings documentation**
|
||||
- [x] **Framework analysis (FRAMEWORK_ANALYSIS.md)**
|
||||
- [x] REST API for settings management
|
||||
- [x] Admin authentication and authorization
|
||||
- [x] Comprehensive test coverage
|
||||
|
||||
#### New Features
|
||||
- **Settings Management System**: Web-based admin UI for viewing and editing 102 application settings across 10 categories
|
||||
- **Encryption**: Fernet symmetric encryption for sensitive values (passwords, API keys, tokens) with key derived from SESSION_SECRET
|
||||
- **Setup Wizard**: 3-step wizard for first-time configuration (Infrastructure → Security → AI Services)
|
||||
- **Precedence System**: Settings resolved in order: Database > Environment Variables > Defaults
|
||||
- **Source Indicators**: Visual badges showing where each setting value originates (🟢 DB, 🔵 ENV, ⚪ DEFAULT)
|
||||
- **Admin Access Control**: OAuth admin group support and proper decorator pattern for authorization
|
||||
|
||||
---
|
||||
|
||||
### v0.3.3 - Drag-and-Drop Upload (February 2026)
|
||||
**Release Date:** February 8, 2026
|
||||
**Status:** ✅ Released
|
||||
**Theme:** User Experience Enhancement
|
||||
|
||||
#### Goals
|
||||
- [x] Add drag-and-drop file upload to Files view
|
||||
- [x] Refactor upload logic for maintainability
|
||||
- [x] Improve visual feedback during file interactions
|
||||
|
||||
#### Deliverables
|
||||
- [x] Drag-and-drop upload functionality in Files view
|
||||
- [x] Reusable `upload.js` module for code DRYness
|
||||
- [x] Visual drop overlay and progress modal
|
||||
- [x] Enhanced upload error handling
|
||||
|
||||
---
|
||||
|
||||
### v0.3.2 - Security & Testing Hardening (February 2026)
|
||||
**Release Date:** February 6, 2026
|
||||
**Status:** ✅ Released
|
||||
### v0.3.3 - Security & Testing Hardening (February 2026)
|
||||
**Target Date:** February 15, 2026
|
||||
**Status:** 🚧 In Progress
|
||||
**Theme:** Security, Quality, Testing
|
||||
|
||||
#### Goals
|
||||
@@ -117,22 +41,28 @@ As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
- [x] Implement comprehensive test suite
|
||||
- [x] Add security scanning (CodeQL, Bandit)
|
||||
- [x] Improve CI/CD pipeline
|
||||
- [ ] Achieve 60% test coverage
|
||||
- [ ] Add pre-commit hooks
|
||||
- [ ] Update all dependencies to latest secure versions
|
||||
|
||||
#### Deliverables
|
||||
- [x] SECURITY_AUDIT.md documentation
|
||||
- [x] pytest configuration and fixtures
|
||||
- [x] API integration tests
|
||||
- [x] Configuration validation tests
|
||||
- [ ] Task processing tests
|
||||
- [ ] Storage provider integration tests
|
||||
- [x] Updated CI/CD workflows
|
||||
- [x] Pre-commit hooks configuration
|
||||
- [ ] Security best practices guide
|
||||
|
||||
#### Breaking Changes
|
||||
- None
|
||||
|
||||
---
|
||||
|
||||
## Upcoming Milestones
|
||||
|
||||
### v0.6.0 - Enhanced Search & UI Improvements (April 2026)
|
||||
**Target Date:** April 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
### v0.4.0 - Enhanced Search & UI Improvements (April 2026)
|
||||
**Target Date:** April 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** User Experience, Search, Performance
|
||||
|
||||
#### Goals
|
||||
@@ -163,8 +93,8 @@ As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
---
|
||||
|
||||
### v0.4.5 - Workflow Automation (June 2026)
|
||||
**Target Date:** June 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Target Date:** June 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Automation, Integration, Webhooks
|
||||
|
||||
#### Goals
|
||||
@@ -182,11 +112,14 @@ As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
- Integration examples and templates
|
||||
- Webhook payload documentation
|
||||
|
||||
#### Breaking Changes
|
||||
- None
|
||||
|
||||
---
|
||||
|
||||
### v0.7.0 - Advanced AI & Multi-language (August 2026)
|
||||
**Target Date:** August 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
### v0.5.0 - Advanced AI & Multi-language (August 2026)
|
||||
**Target Date:** August 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** AI Enhancement, Internationalization
|
||||
|
||||
#### Goals
|
||||
@@ -205,11 +138,14 @@ As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
- Translation framework (10+ languages)
|
||||
- Localized documentation
|
||||
|
||||
#### Breaking Changes
|
||||
- Configuration file format changes (auto-migration script provided)
|
||||
|
||||
---
|
||||
|
||||
### v1.0.0 - Enterprise Edition (November 2026)
|
||||
**Target Date:** November 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Target Date:** November 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Enterprise Features, Scalability, Multi-tenancy
|
||||
|
||||
This is our first major release, marking production-ready enterprise capabilities.
|
||||
@@ -242,31 +178,93 @@ This is our first major release, marking production-ready enterprise capabilitie
|
||||
- Database replication support
|
||||
- Message queue clustering
|
||||
|
||||
- **Observability**
|
||||
- Comprehensive audit logs
|
||||
- Prometheus metrics export
|
||||
- Grafana dashboards
|
||||
- APM integration (New Relic, DataDog)
|
||||
- SLA monitoring
|
||||
|
||||
- **Documentation**
|
||||
- Enterprise deployment guide
|
||||
- High availability setup
|
||||
- Disaster recovery procedures
|
||||
- Security compliance guide
|
||||
- Professional services offerings
|
||||
|
||||
#### Breaking Changes
|
||||
- Database schema migration (automatic with Alembic)
|
||||
- Configuration file restructure (migration tool provided)
|
||||
- API v1 deprecated (v2 required for new features)
|
||||
|
||||
#### Migration Path
|
||||
- Detailed migration guide provided
|
||||
- Automated migration scripts
|
||||
- Rollback procedures documented
|
||||
- Migration support via GitHub Discussions
|
||||
|
||||
---
|
||||
|
||||
### v1.1.0 - Collaboration & Analytics (January 2027)
|
||||
**Target Date:** January 15, 2027
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Collaboration, Reporting, Analytics
|
||||
|
||||
#### Goals
|
||||
- Document sharing with expiring links
|
||||
- Comments and annotations
|
||||
- Version history
|
||||
- Analytics dashboard
|
||||
- Cost analysis
|
||||
- Export reports
|
||||
|
||||
#### Deliverables
|
||||
- Sharing interface with permissions
|
||||
- Comment system with threading
|
||||
- Version control and diff viewer
|
||||
- Analytics dashboard with charts
|
||||
- Cost breakdown by provider
|
||||
- Report generation (PDF, CSV, Excel)
|
||||
- User activity tracking
|
||||
|
||||
#### Breaking Changes
|
||||
- None
|
||||
|
||||
---
|
||||
|
||||
### v2.0.0 - On-Premise AI & Platform Expansion (Q3 2027)
|
||||
**Target Date:** Q3 2027
|
||||
**Status:** 🔮 Future
|
||||
**Theme:** Self-hosting, Privacy, Platform Diversity
|
||||
|
||||
#### Goals
|
||||
- Self-hosted AI models (no cloud dependencies)
|
||||
- Local LLM integration
|
||||
- Desktop and mobile applications
|
||||
- Offline-first capabilities
|
||||
- Enhanced privacy features
|
||||
- Plugin marketplace
|
||||
|
||||
#### Deliverables
|
||||
- Tesseract/EasyOCR integration
|
||||
- Ollama/LLaMA support
|
||||
- Desktop app (Windows, Mac, Linux)
|
||||
- Mobile apps (iOS, Android)
|
||||
- Browser extensions (Chrome, Firefox)
|
||||
- Plugin SDK and marketplace
|
||||
- Offline mode
|
||||
|
||||
#### Breaking Changes
|
||||
- Major API restructure (v3)
|
||||
- New authentication system
|
||||
- Configuration format change
|
||||
- Minimum Python version: 3.12
|
||||
|
||||
---
|
||||
|
||||
## Release Process
|
||||
|
||||
### Automated Semantic Versioning (v0.6.0+)
|
||||
Starting with v0.6.0, releases are fully automated using `python-semantic-release`:
|
||||
|
||||
1. **Commit with Conventional Format**: Use conventional commit messages (feat, fix, etc.)
|
||||
2. **Merge to Main**: PR merges trigger semantic-release workflow
|
||||
3. **Automated Analysis**: semantic-release determines version from commits
|
||||
4. **Automatic Updates**:
|
||||
- Updates `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates Git tag (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release with notes
|
||||
- Triggers Docker image builds
|
||||
5. **No Manual Steps**: VERSION and CHANGELOG are never edited manually
|
||||
|
||||
### Version Bump Rules
|
||||
- `feat:` commits → Minor version (0.5.0 → 0.6.0)
|
||||
- `fix:`, `perf:` → Patch version (0.5.0 → 0.5.1)
|
||||
- `feat!:`, `BREAKING CHANGE:` → Major version (0.5.0 → 1.0.0)
|
||||
- Other types (docs, chore, etc.) → No version bump
|
||||
|
||||
### Pre-release Checklist (Automated)
|
||||
### Pre-release Checklist
|
||||
- [ ] All tests passing
|
||||
- [ ] Security scan passed
|
||||
- [ ] Code review completed
|
||||
@@ -284,6 +282,14 @@ Starting with v0.6.0, releases are fully automated using `python-semantic-releas
|
||||
- Helm charts (future)
|
||||
- Documentation site update
|
||||
|
||||
### Post-release
|
||||
- [ ] GitHub release created
|
||||
- [ ] Blog post published
|
||||
- [ ] Social media announcement
|
||||
- [ ] Community notification
|
||||
- [ ] Support documentation updated
|
||||
- [ ] Monitor for critical issues
|
||||
|
||||
---
|
||||
|
||||
## Version History
|
||||
@@ -293,12 +299,10 @@ Starting with v0.6.0, releases are fully automated using `python-semantic-releas
|
||||
| v0.1.0 | 2024-Q1 | Initial Release | Released |
|
||||
| v0.2.0 | 2024-Q3 | Multi-provider Support | Released |
|
||||
| v0.3.0 | 2025-Q4 | UI & Authentication | Released |
|
||||
| v0.3.1 | 2026-01-15 | OAuth2 Integration | Released |
|
||||
| v0.3.2 | 2026-02-06 | Security Updates | Released |
|
||||
| v0.3.3 | 2026-02-08 | Drag-and-Drop Upload | Released |
|
||||
| v0.5.0 | 2026-02-08 | **Settings & Encryption** | **Released** |
|
||||
| v0.6.0 | 2026-04 | Search & UX | Planned |
|
||||
| v0.7.0 | 2026-08 | Advanced AI | Planned |
|
||||
| v0.3.2 | 2026-02 | Current Stable | Released |
|
||||
| v0.3.3 | 2026-02 | Security & Testing | In Progress |
|
||||
| v0.4.0 | 2026-04 | Search & UX | Planned |
|
||||
| v0.5.0 | 2026-08 | Advanced AI | Planned |
|
||||
| v1.0.0 | 2026-11 | Enterprise | Planned |
|
||||
| v2.0.0 | 2027-Q3 | Platform Expansion | Future |
|
||||
|
||||
@@ -322,4 +326,16 @@ Starting with v0.6.0, releases are fully automated using `python-semantic-releas
|
||||
|
||||
---
|
||||
|
||||
## Contributing to Milestones
|
||||
|
||||
Want to contribute to a specific milestone?
|
||||
|
||||
1. Check the [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) board
|
||||
2. Look for issues tagged with milestone labels
|
||||
3. Read [CONTRIBUTING.md](CONTRIBUTING.md)
|
||||
4. Comment on the issue you'd like to work on
|
||||
5. Submit a PR linked to the issue
|
||||
|
||||
---
|
||||
|
||||
*This milestone document is updated regularly. For real-time status, check our [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) board.*
|
||||
|
||||
@@ -9,15 +9,15 @@ This software includes third-party components with their own licenses:
|
||||
|
||||
SPECIAL NOTICE REGARDING LGPL SOFTWARE:
|
||||
--------------------------------------------------------------------------------
|
||||
DocuElevate incorporates Paramiko, which is licensed under the GNU Lesser General
|
||||
DocuElevate incorporates Paramiko, which is licensed under the GNU Lesser General
|
||||
Public License (LGPL) version 2.1. In accordance with the LGPL:
|
||||
|
||||
1. The complete source code for Paramiko can be obtained from:
|
||||
https://github.com/paramiko/paramiko
|
||||
|
||||
2. This software is distributed in the hope that it will be useful, but WITHOUT
|
||||
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
|
||||
2. This software is distributed in the hope that it will be useful, but WITHOUT
|
||||
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
|
||||
for more details.
|
||||
|
||||
3. A copy of the GNU Lesser General Public License version 2.1 can be found at:
|
||||
@@ -52,9 +52,9 @@ OpenAI (MIT License)
|
||||
Copyright (c) 2023 OpenAI
|
||||
https://github.com/openai/openai-python
|
||||
|
||||
pypdf (BSD License)
|
||||
Copyright (c) 2006-2024, pypdf contributors
|
||||
https://github.com/py-pdf/pypdf
|
||||
PyPDF2 (BSD License)
|
||||
Copyright (c) 2006-2008, Mathieu Fenniak
|
||||
https://github.com/py-pdf/PyPDF2
|
||||
|
||||
Requests (Apache 2.0 License)
|
||||
Copyright 2019 Kenneth Reitz
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
# Mock OAuth2 Server Implementation - Summary
|
||||
|
||||
## Overview
|
||||
Successfully implemented a production-ready mock OAuth2/OIDC server infrastructure for testing authentication flows in DocuElevate.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### 1. Mock OAuth2 Server Container (`tests/mock_oauth_server.py`)
|
||||
- Wraps `mock-oauth2-server` Docker image using testcontainers
|
||||
- Provides complete OIDC provider with all standard endpoints
|
||||
- Fast startup (<1 second), no persistence needed
|
||||
- Automatic readiness detection with health checks
|
||||
|
||||
### 2. OAuth Test Fixtures (`tests/conftest_oauth.py`)
|
||||
- Session-scoped mock OAuth server fixture
|
||||
- Auto-detection of real OAuth credentials from environment
|
||||
- Seamless switching between mock and real OAuth modes
|
||||
- Test data generators (tokens, userinfo, etc.)
|
||||
- Test client with OAuth pre-configured
|
||||
|
||||
### 3. Integration Tests (`tests/test_oauth_integration_flows.py`)
|
||||
- 20+ comprehensive integration tests covering:
|
||||
- OAuth login initiation and redirects
|
||||
- Authorization code exchange
|
||||
- Token validation and session management
|
||||
- Admin vs non-admin authorization
|
||||
- Error handling scenarios
|
||||
- Real OAuth provider integration (when credentials available)
|
||||
|
||||
### 4. Documentation
|
||||
- `tests/README_OAUTH_TESTING.md` - Developer guide
|
||||
- `docs/OAuth_Testing_CI_CD.md` - CI/CD integration guide
|
||||
- Complete examples and troubleshooting
|
||||
|
||||
## Key Features
|
||||
|
||||
### Dual Mode Operation
|
||||
|
||||
**Mock Mode (Default)**
|
||||
```bash
|
||||
# Uses mock-oauth2-server in testcontainer
|
||||
pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
- ⚡ <1s startup
|
||||
- 🔒 No external dependencies
|
||||
- 🎲 Deterministic results
|
||||
- Perfect for local development
|
||||
|
||||
**Real Mode (CI with Secrets)**
|
||||
```bash
|
||||
# Auto-detects and uses real OAuth credentials
|
||||
export AUTHENTIK_CLIENT_ID="your-client-id"
|
||||
export AUTHENTIK_CLIENT_SECRET="your-client-secret"
|
||||
export AUTHENTIK_CONFIG_URL="https://auth.example.com/.well-known/openid-configuration"
|
||||
pytest tests/test_oauth_integration_flows.py -v -m requires_external
|
||||
```
|
||||
- ✅ Tests real OAuth provider
|
||||
- ✅ Validates actual authentication flows
|
||||
- ✅ Uses GitHub Actions secrets
|
||||
- Perfect for integration testing
|
||||
|
||||
### Automatic Mode Detection
|
||||
- Checks for real OAuth credentials in environment
|
||||
- Falls back to mock if credentials not available
|
||||
- Can be manually overridden with env vars
|
||||
- Gracefully skips if dependencies missing
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Test Suite
|
||||
↓
|
||||
OAuth Fixtures (conftest_oauth.py)
|
||||
├── Mock Mode → MockOAuth2ServerContainer
|
||||
│ ├── .well-known/openid-configuration
|
||||
│ ├── /authorize
|
||||
│ ├── /token
|
||||
│ ├── /userinfo
|
||||
│ └── /jwks
|
||||
│
|
||||
└── Real Mode → Actual OAuth Provider (Authentik)
|
||||
└── Uses GitHub Actions secrets
|
||||
```
|
||||
|
||||
## Verification Results
|
||||
|
||||
✅ **Mock OAuth2 Server**
|
||||
- Starts successfully in <1 second
|
||||
- Returns valid OIDC configuration
|
||||
- Provides all required OIDC endpoints
|
||||
- Can be started/stopped cleanly
|
||||
- Works with Docker in CI
|
||||
|
||||
✅ **Endpoints Verified**
|
||||
- `/.well-known/openid-configuration` - OIDC discovery
|
||||
- `/authorize` - OAuth authorization
|
||||
- `/token` - Token exchange
|
||||
- `/userinfo` - User information
|
||||
- `/jwks` - JWT signing keys
|
||||
|
||||
✅ **Test Infrastructure**
|
||||
- Fixtures load correctly
|
||||
- Auto-detection works
|
||||
- Mock/real mode switching functional
|
||||
- Integration with conftest.py successful
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Test
|
||||
```python
|
||||
@pytest.mark.integration
|
||||
def test_oauth_login(oauth_enabled_app):
|
||||
"""Test OAuth login redirects to provider."""
|
||||
response = oauth_enabled_app.get("/oauth-login", follow_redirects=False)
|
||||
assert response.status_code == 302
|
||||
assert "authorize" in response.headers["location"]
|
||||
```
|
||||
|
||||
### Test with Mock Token Exchange
|
||||
```python
|
||||
from unittest.mock import patch
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.auth.oauth.authentik.authorize_access_token")
|
||||
async def test_oauth_callback(mock_authorize, oauth_enabled_app, test_user_info):
|
||||
"""Test OAuth callback with test user."""
|
||||
mock_authorize.return_value = {
|
||||
"access_token": "test-token",
|
||||
"userinfo": test_user_info,
|
||||
}
|
||||
|
||||
response = oauth_enabled_app.get("/oauth-callback?code=test-code")
|
||||
assert response.status_code == 302
|
||||
```
|
||||
|
||||
## GitHub Actions Integration
|
||||
|
||||
### Basic Workflow
|
||||
```yaml
|
||||
name: OAuth Tests
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: pip install -r requirements-dev.txt
|
||||
- run: pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
|
||||
### With Real OAuth (Internal PRs)
|
||||
```yaml
|
||||
jobs:
|
||||
test-real:
|
||||
if: github.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- run: pip install -r requirements-dev.txt
|
||||
- env:
|
||||
AUTHENTIK_CLIENT_ID: ${{ secrets.AUTHENTIK_CLIENT_ID }}
|
||||
AUTHENTIK_CLIENT_SECRET: ${{ secrets.AUTHENTIK_CLIENT_SECRET }}
|
||||
AUTHENTIK_CONFIG_URL: ${{ secrets.AUTHENTIK_CONFIG_URL }}
|
||||
run: pytest tests/test_oauth_integration_flows.py -v -m requires_external
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
| Aspect | Benefit |
|
||||
|--------|---------|
|
||||
| **Speed** | <1s startup, tests complete in seconds |
|
||||
| **Reliability** | Deterministic, no flaky tests |
|
||||
| **Realism** | Tests actual OIDC protocol |
|
||||
| **Flexibility** | Works with mock or real OAuth |
|
||||
| **CI-Friendly** | Ephemeral containers, works in pipelines |
|
||||
| **Security** | Uses GitHub secrets for real credentials |
|
||||
| **Maintainability** | Industry-standard mock-oauth2-server |
|
||||
| **Documentation** | Comprehensive guides and examples |
|
||||
|
||||
## Technical Details
|
||||
|
||||
**Container**: `ghcr.io/navikt/mock-oauth2-server:2.1.1`
|
||||
**Framework**: Testcontainers Python 4.14.1+
|
||||
**Test Framework**: pytest with async support
|
||||
**Languages**: Python 3.12+
|
||||
**Dependencies**: testcontainers, requests, docker
|
||||
|
||||
## Files Created/Modified
|
||||
|
||||
### New Files
|
||||
- `tests/mock_oauth_server.py` - Mock OAuth server container wrapper
|
||||
- `tests/conftest_oauth.py` - OAuth test fixtures
|
||||
- `tests/test_oauth_integration_flows.py` - Integration tests
|
||||
- `tests/README_OAUTH_TESTING.md` - Developer documentation
|
||||
- `docs/OAuth_Testing_CI_CD.md` - CI/CD guide
|
||||
|
||||
### Modified Files
|
||||
- `tests/conftest.py` - Added OAuth fixtures import
|
||||
|
||||
## Next Steps
|
||||
|
||||
To fully utilize this infrastructure:
|
||||
|
||||
1. **Run tests locally**:
|
||||
```bash
|
||||
pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
|
||||
2. **Add to CI pipeline**:
|
||||
- Use provided GitHub Actions examples
|
||||
- Configure secrets for real OAuth testing
|
||||
|
||||
3. **Expand test coverage**:
|
||||
- Add more OAuth flow scenarios
|
||||
- Test edge cases
|
||||
- Add performance tests
|
||||
|
||||
4. **Monitor and maintain**:
|
||||
- Keep mock-oauth2-server image updated
|
||||
- Update tests as OAuth implementation evolves
|
||||
- Add new scenarios as needed
|
||||
|
||||
## Conclusion
|
||||
|
||||
The mock OAuth2 server infrastructure is production-ready and provides:
|
||||
- ✅ Fast, reliable OAuth testing
|
||||
- ✅ Support for both mock and real OAuth providers
|
||||
- ✅ Comprehensive test coverage
|
||||
- ✅ Full CI/CD integration
|
||||
- ✅ Excellent documentation
|
||||
|
||||
This implementation addresses all requirements from the original issue and provides a robust foundation for OAuth testing in DocuElevate.
|
||||
@@ -5,24 +5,6 @@
|
||||
|
||||
# DocuElevate
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://codecov.io/github/christianlouis/DocuElevate)
|
||||
[](https://github.com/christianlouis/DocuElevate/actions/workflows/ci.yml)
|
||||
[](https://github.com/christianlouis/DocuElevate/actions/workflows/codeql.yml)
|
||||
|
||||
[](https://github.com/christianlouis/DocuElevate/releases)
|
||||
[](LICENSE)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://hub.docker.com/)
|
||||
|
||||
[](https://github.com/christianlouis/DocuElevate/stargazers)
|
||||
[](https://github.com/christianlouis/DocuElevate/network/members)
|
||||
[](https://github.com/christianlouis/DocuElevate/issues)
|
||||
[](https://github.com/christianlouis/DocuElevate/pulls)
|
||||
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
<a href="https://www.docuelevate.org"><img src="frontend/static/hero.png" alt="DocuElevate Logo" width="80%" /></a>
|
||||
</div>
|
||||
@@ -31,12 +13,12 @@
|
||||
|
||||
DocuElevate automates the handling, extraction, and processing of documents using a variety of services, including:
|
||||
|
||||
- **AI Provider** (pluggable – OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Portkey, and more) for metadata extraction and text refinement.
|
||||
- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.
|
||||
- **Paperless NGX** for document indexing and management.
|
||||
- **Azure Document Intelligence** for OCR on PDFs.
|
||||
- **Gotenberg** for file-to-PDF conversions.
|
||||
- **Authentik** for authentication and user management.
|
||||
- **OpenAI** for metadata extraction and text refinement.
|
||||
- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.
|
||||
- **Paperless NGX** for document indexing and management.
|
||||
- **Azure Document Intelligence** for OCR on PDFs.
|
||||
- **Gotenberg** for file-to-PDF conversions.
|
||||
- **Authentik** for authentication and user management.
|
||||
|
||||
It is designed for flexibility and configurability through environment variables, making it easily customizable for different workflows. The system can fetch documents from multiple IMAP mailboxes, process them (OCR, metadata extraction, PDF conversion), and store them in the desired destinations.
|
||||
|
||||
@@ -45,13 +27,9 @@ The project includes a **UI** for uploading and managing files, and an API docum
|
||||
## Documentation Index
|
||||
|
||||
- [User Guide](docs/UserGuide.md) - How to use DocuElevate
|
||||
- [Browser Extension Guide](docs/BrowserExtension.md) - Install and use the browser extension
|
||||
- [API Documentation](docs/API.md) - API reference
|
||||
- [Deployment Guide](docs/DeploymentGuide.md) - How to deploy DocuElevate
|
||||
- [Configuration Guide](docs/ConfigurationGuide.md) - Available configuration options
|
||||
- [Build Metadata](docs/BuildMetadata.md) - Automated version and build information
|
||||
- [CI/CD Tools Guide](docs/CIToolsGuide.md) - CI/CD pipeline and tool documentation
|
||||
- [CI Workflow Guide](docs/CIWorkflow.md) - Detailed workflow documentation
|
||||
- [Development Guide](CONTRIBUTING.md) - How to contribute to DocuElevate
|
||||
- [Troubleshooting](docs/Troubleshooting.md) - Common issues and solutions
|
||||
|
||||
@@ -60,13 +38,11 @@ The project includes a **UI** for uploading and managing files, and an API docum
|
||||
<div align="center">
|
||||
<img src="docs/upload-view.png" alt="DocuElevate Upload Interface" width="80%" />
|
||||
<p><em>Upload interface for adding new documents</em></p>
|
||||
|
||||
|
||||
<img src="docs/files-view.png" alt="DocuElevate Files View" width="80%" />
|
||||
<p><em>Files view with processed documents and metadata</em></p>
|
||||
</div>
|
||||
|
||||
> **Note:** Screenshots may not reflect the very latest UI. For the most current look, visit [docuelevate.org](https://www.docuelevate.org).
|
||||
|
||||
## Workflow Process
|
||||
|
||||
DocuElevate follows a streamlined document processing workflow:
|
||||
@@ -76,17 +52,16 @@ DocuElevate follows a streamlined document processing workflow:
|
||||
</div>
|
||||
|
||||
### Document Ingestion
|
||||
Documents enter DocuElevate through four possible channels:
|
||||
Documents enter DocuElevate through three possible channels:
|
||||
1. **Web Upload**: Users manually upload files via the web interface
|
||||
2. **Browser Extension**: Send files directly from your browser with one click
|
||||
3. **Email Attachments**: Automatic polling of configured IMAP mailboxes (supports multiple accounts)
|
||||
4. **API**: Direct programmatic uploads via the REST API
|
||||
2. **Email Attachments**: Automatic polling of configured IMAP mailboxes (supports multiple accounts)
|
||||
3. **API**: Direct programmatic uploads via the REST API
|
||||
|
||||
### Processing Pipeline
|
||||
Every document goes through the following steps:
|
||||
1. **PDF Conversion**: Non-PDF files are converted to PDF format using Gotenberg
|
||||
2. **OCR Processing**: Azure Document Intelligence extracts text from images/scans
|
||||
3. **Metadata Extraction**: The configured AI provider analyzes document content to identify:
|
||||
3. **Metadata Extraction**: OpenAI analyzes document content to identify:
|
||||
- Document type (invoice, receipt, contract, etc.)
|
||||
- Key entities (dates, names, amounts, account numbers)
|
||||
- Important data points specific to the document type
|
||||
@@ -103,29 +78,20 @@ Users can choose to send documents to any combination of these destinations thro
|
||||
|
||||
## Features
|
||||
|
||||
- **Intuitive File Upload**:
|
||||
- Drag-and-drop file upload on both Upload and Files pages—upload anywhere on the Files page
|
||||
- Real-time upload progress with validation
|
||||
- Support for PDF, Office documents, images, and more (up to 500MB per file)
|
||||
- **Browser Extension**:
|
||||
- Send files directly from your browser to DocuElevate with one click
|
||||
- Compatible with Chrome, Firefox, Edge, and other Chromium-based browsers
|
||||
- Context menu integration for quick access
|
||||
- See [Browser Extension Guide](docs/BrowserExtension.md) for installation and usage
|
||||
- **Document Upload & Storage**:
|
||||
- Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless
|
||||
- **OCR Processing (Azure)**:
|
||||
- Extract text from scanned PDFs using Azure Document Intelligence
|
||||
- **Metadata Extraction (AI Provider)**:
|
||||
- Use any supported AI provider (OpenAI, Anthropic, Gemini, Ollama, etc.) to classify, label, or otherwise enrich the text with structured metadata
|
||||
- **PDF Conversion (Gotenberg)**:
|
||||
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs
|
||||
- **Document Management (Paperless NGX)**:
|
||||
- Store processed documents and metadata in a Paperless NGX instance
|
||||
- **IMAP Integration**:
|
||||
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing
|
||||
- **Authentication**:
|
||||
- Secure access to the system using **Authentik** for OAuth2-based login
|
||||
- **Document Upload & Storage**:
|
||||
- Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless.
|
||||
- **OCR Processing (Azure)**:
|
||||
- Extract text from scanned PDFs using Azure Document Intelligence.
|
||||
- **Metadata Extraction (OpenAI)**:
|
||||
- Use GPT to classify, label, or otherwise enrich the text with structured metadata.
|
||||
- **PDF Conversion (Gotenberg)**:
|
||||
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs.
|
||||
- **Document Management (Paperless NGX)**:
|
||||
- Store processed documents and metadata in a Paperless NGX instance.
|
||||
- **IMAP Integration**:
|
||||
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing.
|
||||
- **Authentication**:
|
||||
- Secure access to the system using **Authentik** for OAuth2-based login.
|
||||
|
||||
## Frameworks Used
|
||||
|
||||
@@ -142,11 +108,11 @@ For detailed installation and deployment instructions, please refer to the [Depl
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/christianlouis/DocuElevate.git
|
||||
cd DocuElevate
|
||||
git clone <repository_url>
|
||||
cd document-processor
|
||||
|
||||
# Configure environment variables
|
||||
cp .env.demo .env
|
||||
cp .env.example .env
|
||||
# Edit .env with your settings
|
||||
|
||||
# Run with Docker Compose
|
||||
@@ -155,38 +121,6 @@ docker-compose up -d
|
||||
|
||||
The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
|
||||
|
||||
## Development & Testing
|
||||
|
||||
### Running Tests
|
||||
|
||||
DocuElevate includes comprehensive test coverage. To run tests:
|
||||
|
||||
```bash
|
||||
# Install development dependencies
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Run all tests
|
||||
pytest
|
||||
|
||||
# Run with coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run only fast unit tests
|
||||
pytest -m unit
|
||||
```
|
||||
|
||||
Tests are automatically configured with the necessary environment variables - **no manual setup required!**
|
||||
|
||||
For detailed testing information, including integration tests with Docker and authentication testing, see the [Contributing Guide](CONTRIBUTING.md#running-tests).
|
||||
|
||||
### Contributing
|
||||
|
||||
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for:
|
||||
- Code style guidelines
|
||||
- Commit message format (Conventional Commits)
|
||||
- Testing requirements
|
||||
- Pull request process
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
|
||||
@@ -214,12 +148,9 @@ The following is a summary of the licenses used by our direct dependencies:
|
||||
| Uvicorn | BSD |
|
||||
| SQLAlchemy | MIT |
|
||||
| Pydantic | MIT |
|
||||
| openai | MIT |
|
||||
| litellm | MIT |
|
||||
| pypdf | BSD |
|
||||
| OpenAI | MIT |
|
||||
| PyPDF2 | BSD |
|
||||
| Requests | Apache 2.0 |
|
||||
| puremagic | MIT |
|
||||
| filetype | MIT |
|
||||
| Dropbox | MIT |
|
||||
| Azure AI Document Intelligence | MIT |
|
||||
| Authlib | BSD |
|
||||
@@ -239,5 +170,4 @@ For a comprehensive list of all dependencies and their licenses, run:
|
||||
```
|
||||
pip install pip-licenses
|
||||
pip-licenses
|
||||
|
||||
```
|
||||
```
|
||||
+12
-34
@@ -1,26 +1,13 @@
|
||||
# DocuElevate Roadmap
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
**Last Updated:** 2026-02-06
|
||||
**Version:** 1.0
|
||||
|
||||
## Vision
|
||||
|
||||
DocuElevate aims to be the premier open-source intelligent document processing platform, providing seamless integration with cloud storage providers, advanced AI-powered metadata extraction, and enterprise-grade security and scalability.
|
||||
|
||||
## Release Naming
|
||||
|
||||
Each major milestone release carries a codename to anchor key project moments. These names appear in the status dashboard, build metadata, and changelog. For details, see [docs/ReleaseNaming.md](docs/ReleaseNaming.md).
|
||||
|
||||
| Version Range | Codename | Theme |
|
||||
|---------------|---------------|--------------------------------------------------|
|
||||
| 0.5.x | **Foundation** | Core platform, multi-provider storage, AI, UI |
|
||||
| 0.6.x | **Clarity** | Enhanced search, filtering, UI/UX improvements |
|
||||
| 0.7.x | **Conductor** | Workflow automation, pipelines, rule-based logic |
|
||||
| 1.0.x | **Summit** | Enterprise features, multi-tenancy, RBAC |
|
||||
| 1.1.x | **Bridge** | Collaboration, sharing, analytics |
|
||||
| 2.0.x | **Horizon** | On-premise AI, platform expansion |
|
||||
|
||||
## Current Status (v0.5.0 "Foundation")
|
||||
## Current Status (v0.3.2)
|
||||
|
||||
### Core Features ✅
|
||||
- Multi-provider document storage (Dropbox, Google Drive, OneDrive, Nextcloud, S3, etc.)
|
||||
@@ -29,14 +16,11 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- AI-powered metadata extraction via OpenAI
|
||||
- PDF conversion via Gotenberg
|
||||
- Web UI for document upload and management
|
||||
- **Database-backed settings management with admin UI**
|
||||
- **Fernet encryption for sensitive configuration**
|
||||
- **Setup wizard for first-time installation**
|
||||
- REST API with OpenAPI documentation
|
||||
- Celery-based async task processing
|
||||
- OAuth2 authentication via Authentik with admin group support
|
||||
- OAuth2 authentication via Authentik
|
||||
|
||||
## Short-term Goals (Q1-Q2 2026) - v0.4.x to v0.5.x "Foundation"
|
||||
## Short-term Goals (Q1-Q2 2026) - v0.4.x to v0.5.x
|
||||
|
||||
### Quality & Stability 🎯
|
||||
- **Test Coverage** (High Priority)
|
||||
@@ -59,14 +43,8 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- [ ] Implement API key rotation
|
||||
- [ ] Add audit logging for sensitive operations
|
||||
|
||||
- **Release Automation** (Completed ✅)
|
||||
- [x] Implement semantic-release for automated versioning
|
||||
- [x] Add conventional commit validation
|
||||
- [x] Automate CHANGELOG generation
|
||||
- [x] Integrate Docker builds with releases
|
||||
|
||||
### Features - v0.4.0
|
||||
- **Enhanced Search & Filtering** → _preparing for v0.6.0 "Clarity"_
|
||||
- **Enhanced Search & Filtering**
|
||||
- [ ] Full-text search across documents
|
||||
- [ ] Advanced filtering by metadata, tags, date ranges
|
||||
- [ ] Saved search queries
|
||||
@@ -80,8 +58,8 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- [ ] Progress indicators for long-running tasks
|
||||
- [ ] Real-time notifications via WebSocket
|
||||
|
||||
### Features - v0.5.0 "Foundation"
|
||||
- **Workflow Automation** → _evolving into v0.7.0 "Conductor"_
|
||||
### Features - v0.5.0
|
||||
- **Workflow Automation**
|
||||
- [ ] Custom processing pipelines
|
||||
- [ ] Conditional routing based on document type
|
||||
- [ ] Scheduled batch processing
|
||||
@@ -95,9 +73,9 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- [ ] Automatic duplicate detection
|
||||
- [ ] Intelligent document splitting
|
||||
|
||||
## Medium-term Goals (Q3-Q4 2026) - v1.0.x "Summit"
|
||||
## Medium-term Goals (Q3-Q4 2026) - v1.0.x
|
||||
|
||||
### Enterprise Features - v1.0.0 "Summit"
|
||||
### Enterprise Features - v1.0.0
|
||||
- **Multi-tenancy**
|
||||
- [ ] Organization/team management
|
||||
- [ ] Role-based access control (RBAC)
|
||||
@@ -119,7 +97,7 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- [ ] Custom webhook receivers
|
||||
- [ ] GraphQL API
|
||||
|
||||
### Features - v1.1.0 "Bridge"
|
||||
### Features - v1.1.0
|
||||
- **Collaboration**
|
||||
- [ ] Document sharing with expiring links
|
||||
- [ ] Comments and annotations
|
||||
@@ -134,7 +112,7 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
- [ ] Cost analysis per provider
|
||||
- [ ] Export reports (PDF, CSV, Excel)
|
||||
|
||||
## Long-term Goals (2027+) - v2.0+ "Horizon"
|
||||
## Long-term Goals (2027+) - v2.0+
|
||||
|
||||
### Strategic Initiatives
|
||||
- **On-Premise AI Models**
|
||||
@@ -193,7 +171,7 @@ Each major milestone release carries a codename to anchor key project moments. T
|
||||
## Technology Debt
|
||||
|
||||
### Refactoring Needed
|
||||
- [x] Migrate from PyPDF2 to pypdf (modern fork) - ✅ Completed 2026-02-12
|
||||
- [ ] Migrate from PyPDF2 to pypdf (modern fork)
|
||||
- [ ] Standardize error handling across modules
|
||||
- [ ] Consolidate configuration management
|
||||
- [ ] Optimize database queries
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
DocuElevate Build Information
|
||||
==============================
|
||||
Version: 0.129.0
|
||||
Build Date: 2026-03-13T08:33:59Z
|
||||
Git Commit: 3c7f0888de98fa7cf4a89c4210768c4f138093ae
|
||||
Git Short SHA: 3c7f088
|
||||
Git Branch: main
|
||||
Commit Date: 2026-03-13T09:33:37+01:00
|
||||
Build Timestamp: 2026-03-13T08:33:59Z
|
||||
==============================
|
||||
+34
-792
@@ -1,149 +1,17 @@
|
||||
# Security Audit Report
|
||||
|
||||
**Date:** 2026-02-12
|
||||
**Status:** Bandit Security Scan Completed - All Critical/High/Medium Issues Resolved
|
||||
**Date:** 2026-02-06
|
||||
**Status:** Completed Initial Assessment
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This document tracks security vulnerabilities found in DocuElevate and their remediation status. A comprehensive security audit using Bandit has been completed, with all critical, high, and medium severity issues addressed.
|
||||
|
||||
## Recent Security Fixes
|
||||
|
||||
### CVE-2023-36464: PyPDF2/pypdf Infinite Loop Vulnerability ✅ FIXED (2026-02-12)
|
||||
|
||||
**Severity:** Moderate (CVSS: 5.5)
|
||||
**CVE:** [CVE-2023-36464](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-36464)
|
||||
**Advisory:** [GHSA-4vvm-4w3v-6mr8](https://github.com/advisories/GHSA-4vvm-4w3v-6mr8)
|
||||
|
||||
**Issue:** Certain versions of PyPDF2 (>=2.2.0, <=3.0.1) and pypdf (prior to 3.9.0) contain a vulnerability where specially crafted PDF files can trigger an infinite loop in `__parse_content_stream`, causing 100% CPU usage and potential denial of service.
|
||||
|
||||
**Impact:**
|
||||
- **Availability:** High (can block process and consume 100% CPU)
|
||||
- **Confidentiality:** None
|
||||
- **Integrity:** None
|
||||
- **Attack Vector:** Local
|
||||
- **Privileges Required:** None
|
||||
|
||||
**Remediation:**
|
||||
- Upgraded from `PyPDF2>=3.0.0` (vulnerable) to `pypdf>=3.9.0` (fixed)
|
||||
- Updated all imports from `PyPDF2` to `pypdf` across the codebase
|
||||
- Verified pypdf 6.7.0 installed successfully
|
||||
- **Files Updated:**
|
||||
- `requirements.txt` - Updated dependency specification
|
||||
- `app/tasks/process_document.py`
|
||||
- `app/tasks/rotate_pdf_pages.py`
|
||||
- `app/utils/file_splitting.py`
|
||||
- `app/tasks/embed_metadata_into_pdf.py`
|
||||
- `app/tasks/process_with_azure_document_intelligence.py`
|
||||
- `app/views/files.py`
|
||||
- `app/api/files.py`
|
||||
- `tests/test_external_integrations.py`
|
||||
- `tests/test_file_splitting.py`
|
||||
|
||||
**Testing:** All affected modules verified for syntax correctness and basic import functionality.
|
||||
|
||||
**References:**
|
||||
- [py-pdf/pypdf#1828](https://github.com/py-pdf/pypdf/pull/1828) - Fix implementation
|
||||
- [py-pdf/pypdf#969](https://github.com/py-pdf/pypdf/pull/969) - Issue introduction
|
||||
|
||||
## Bandit Security Scan Results (2026-02-07)
|
||||
|
||||
**Scan Summary:**
|
||||
- **Total lines scanned:** 7,423
|
||||
- **High severity issues:** 0 (6 fixed)
|
||||
- **Medium severity issues:** 0 (15 fixed)
|
||||
- **Low severity issues:** 21 (informational/acceptable)
|
||||
|
||||
### Fixed Issues from Bandit Scan
|
||||
|
||||
#### 1. B324: Weak MD5 Hash Usage (HIGH SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 2
|
||||
**Locations:**
|
||||
- `app/api/user.py:26` - Gravatar URL generation
|
||||
- `app/auth.py:65` - Gravatar URL generation
|
||||
|
||||
**Issue:** MD5 hash was used without specifying `usedforsecurity=False` parameter.
|
||||
|
||||
**Remediation:** Added `usedforsecurity=False` parameter to all MD5 hash calls. MD5 is used only for Gravatar URL generation (non-cryptographic purpose), which is an acceptable use case.
|
||||
|
||||
```python
|
||||
# Before: email_hash = md5(email.encode()).hexdigest()
|
||||
# After: email_hash = md5(email.encode(), usedforsecurity=False).hexdigest()
|
||||
```
|
||||
|
||||
#### 2. B402/B321: Insecure FTP Protocol (HIGH SEVERITY) ✅ DOCUMENTED
|
||||
**Occurrences:** 3
|
||||
**Location:** `app/tasks/upload_to_ftp.py`
|
||||
|
||||
**Issue:** FTP is an insecure protocol vulnerable to eavesdropping and MITM attacks.
|
||||
|
||||
**Remediation:**
|
||||
- Added comprehensive security warnings in code comments
|
||||
- Code already defaults to FTPS (FTP_TLS) for encrypted connections
|
||||
- Plaintext FTP only used as fallback when explicitly configured
|
||||
- Added `# nosec B402` and `# nosec B321` annotations with justification
|
||||
- Added security notes in docstrings
|
||||
- Configuration options: `ftp_use_tls=True` (default), `ftp_allow_plaintext=True` (default)
|
||||
|
||||
**Security Note:** For production environments, set `ftp_allow_plaintext=False` to prevent fallback to unencrypted FTP.
|
||||
|
||||
#### 3. B507: SSH Host Key Verification Disabled (HIGH SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 1
|
||||
**Location:** `app/tasks/upload_to_sftp.py:47`
|
||||
|
||||
**Issue:** Using `paramiko.AutoAddPolicy()` automatically trusts unknown SSH host keys, making connections vulnerable to MITM attacks.
|
||||
|
||||
**Remediation:**
|
||||
- Added configuration option `sftp_disable_host_key_verification` (default: False for security)
|
||||
- When enabled (False), uses `paramiko.RejectPolicy()` with system known_hosts for secure verification
|
||||
- When disabled (True, for testing only), uses `AutoAddPolicy()` with security warnings
|
||||
- Added `# nosec B507` annotation with justification for the test/dev use case
|
||||
- Updated docstrings with security guidance
|
||||
|
||||
**Security Note:** The default value is now `False` (secure). For development/testing environments where host keys cannot be pre-configured, set `SFTP_DISABLE_HOST_KEY_VERIFICATION=True` (not recommended for production).
|
||||
|
||||
#### 4. B113: Missing Timeout on HTTP Requests (MEDIUM SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 15
|
||||
**Locations:**
|
||||
- `app/api/dropbox.py` (4 requests calls)
|
||||
- `app/api/google_drive.py` (1 request call)
|
||||
- `app/api/onedrive.py` (3 requests calls)
|
||||
- `app/tasks/convert_to_pdf.py` (1 request call)
|
||||
- `app/tasks/upload_to_dropbox.py` (1 request call)
|
||||
- `app/tasks/upload_to_paperless.py` (2 requests calls)
|
||||
- `app/tasks/upload_to_onedrive.py` (2 requests calls)
|
||||
- `app/tasks/upload_to_webdav.py` (1 request call)
|
||||
|
||||
**Issue:** HTTP requests without timeout can hang indefinitely, leading to resource exhaustion and potential DoS.
|
||||
|
||||
**Remediation:**
|
||||
- Added `http_request_timeout` configuration setting (default: 120 seconds)
|
||||
- Timeout configured to handle large file operations (PDFs up to 1GB+)
|
||||
- Applied `timeout=settings.http_request_timeout` to all `requests.get()`, `requests.post()`, and `requests.put()` calls
|
||||
- Configurable via environment variable: `HTTP_REQUEST_TIMEOUT=120`
|
||||
|
||||
**Note:** The 120-second default timeout is appropriate for:
|
||||
- Large PDF file uploads and downloads (up to 1GB)
|
||||
- PDF conversion operations via Gotenberg
|
||||
- Cloud storage uploads (Dropbox, OneDrive, Google Drive, Nextcloud, WebDAV)
|
||||
- Document processing and OCR operations
|
||||
|
||||
### Low Severity Issues (Informational)
|
||||
|
||||
**21 low severity findings remain** - These are informational warnings about:
|
||||
- `assert` statements (B101) - Used in non-security contexts
|
||||
- Try-except-pass blocks (B110) - Acceptable for optional operations
|
||||
- Subprocess calls (B603/B607) - Verified safe (hardcoded commands, no user input)
|
||||
- Hard-coded temp directories (B108) - Platform-appropriate temp paths
|
||||
- Hard-coded bind addresses (B104) - Development defaults
|
||||
|
||||
**Assessment:** All low severity findings have been reviewed and are acceptable given the context of their usage.
|
||||
This document tracks security vulnerabilities found in DocuElevate and their remediation status.
|
||||
|
||||
## Critical Vulnerabilities (Fixed) ✅
|
||||
|
||||
### 1. Outdated Authlib with Known Vulnerabilities
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Description:** Authlib version 1.3.2 had two critical vulnerabilities:
|
||||
- CVE: Denial of Service via Oversized JOSE Segments
|
||||
- CVE: JWS/JWT accepts unknown crit headers (RFC violation → possible authz bypass)
|
||||
@@ -151,18 +19,18 @@ This document tracks security vulnerabilities found in DocuElevate and their rem
|
||||
**Fix:** Updated `requirements.txt` to require `authlib>=1.6.5`
|
||||
|
||||
### 2. Starlette DoS Vulnerability
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** Starlette 0.41.3 vulnerable to O(n^2) DoS via Range header merging in `FileResponse`
|
||||
|
||||
**Fix:** Updated `requirements.txt` to require `starlette>=0.49.1`
|
||||
|
||||
### 3. Weak SESSION_SECRET Default
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Description:** Default SESSION_SECRET value in `app/main.py` was a predictable string that could be exploited if not overridden
|
||||
|
||||
**Fix:**
|
||||
**Fix:**
|
||||
- Enhanced validation in `app/main.py` to raise error if auth is enabled without proper secret
|
||||
- Updated default to be clearly marked as insecure for development only
|
||||
- Added generation instructions in error message
|
||||
@@ -170,8 +38,8 @@ This document tracks security vulnerabilities found in DocuElevate and their rem
|
||||
## Medium Risk Issues (Fixed) ✅
|
||||
|
||||
### 4. Insufficient .gitignore Protection
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** .gitignore didn't adequately protect against accidentally committing sensitive files (credentials, private keys, secrets)
|
||||
|
||||
**Fix:** Enhanced `.gitignore` with comprehensive patterns for:
|
||||
@@ -181,72 +49,12 @@ This document tracks security vulnerabilities found in DocuElevate and their rem
|
||||
- SSH keys
|
||||
- Explicit exclusion of patterns where needed
|
||||
|
||||
### 5. File Upload Size Limits
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** No configurable limits on file upload sizes could lead to resource exhaustion attacks and DoS.
|
||||
|
||||
**Fix:** Implemented configurable file upload size limits with the following features:
|
||||
- `MAX_UPLOAD_SIZE`: Maximum file upload size in bytes (default: 1GB)
|
||||
- `MAX_SINGLE_FILE_SIZE`: Optional maximum size for a single file chunk
|
||||
- **Automatic page-based PDF splitting** for large PDFs when max_single_file_size is configured
|
||||
- Splits PDFs at **page boundaries** using pypdf, NOT by byte position
|
||||
- Each output file is a structurally valid, complete PDF
|
||||
- No risk of corrupted or broken PDF files
|
||||
- Split files are processed sequentially to prevent overwhelming the system
|
||||
- Clear error messages referencing SECURITY_AUDIT.md for configuration details
|
||||
|
||||
**Configuration:**
|
||||
```bash
|
||||
# Set maximum upload size (default: 1GB)
|
||||
MAX_UPLOAD_SIZE=1073741824
|
||||
|
||||
# Optional: Enable file splitting for large PDFs
|
||||
MAX_SINGLE_FILE_SIZE=104857600 # 100MB chunks
|
||||
```
|
||||
|
||||
**Security Benefits:**
|
||||
- Prevents resource exhaustion from extremely large uploads
|
||||
- Configurable limits allow adaptation to server capacity
|
||||
- File splitting enables processing of large documents without memory issues
|
||||
- Maintains support for large PDF files (up to 1GB by default) as required by use case
|
||||
|
||||
## Best Practices Implemented
|
||||
|
||||
### Security Scanning with Bandit
|
||||
- ✅ Bandit installed in development dependencies (`requirements-dev.txt`)
|
||||
- ✅ Comprehensive scan completed on all Python code
|
||||
- ✅ High and medium severity issues resolved
|
||||
- ✅ Low severity issues reviewed and accepted
|
||||
|
||||
**Running Bandit:**
|
||||
```bash
|
||||
# Scan entire app directory
|
||||
bandit -r app
|
||||
|
||||
# Show only high and medium severity
|
||||
bandit -r app -ll
|
||||
|
||||
# Generate JSON report
|
||||
bandit -r app -f json -o bandit_results.json
|
||||
|
||||
# Generate HTML report
|
||||
bandit -r app -f html -o bandit_report.html
|
||||
```
|
||||
|
||||
**Suppressing False Positives:**
|
||||
Use `# nosec` comments with justification:
|
||||
```python
|
||||
# Security: FTP usage intentional for legacy server support
|
||||
import ftplib # nosec B402 - FTP usage is intentional
|
||||
|
||||
ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when configured
|
||||
```
|
||||
|
||||
### Dependency Management
|
||||
- ✅ Version pinning for security-critical packages (authlib, starlette)
|
||||
- ✅ Advisory database checks integrated into development workflow
|
||||
- ✅ Automated dependency vulnerability scanning in CI/CD via pip-audit ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
- ⏳ TODO: Add automated dependency vulnerability scanning in CI/CD
|
||||
|
||||
### Authentication & Secrets
|
||||
- ✅ Strong validation for SESSION_SECRET (minimum 32 characters)
|
||||
@@ -262,84 +70,43 @@ ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when configured
|
||||
## Ongoing Security Measures
|
||||
|
||||
### CI/CD Security
|
||||
- ✅ **COMPLETED:** Bandit (Python security linter) audit completed
|
||||
- ✅ **COMPLETED:** Bandit integrated into CI pipeline (fails on high/medium severity issues)
|
||||
- ✅ **COMPLETED:** CodeQL security scanning enabled in GitHub Actions
|
||||
- ✅ **COMPLETED:** pip-audit dependency vulnerability scanning added to CI ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
- ✅ **COMPLETED:** Dependency scans are blocking (fail build when vulnerabilities detected) ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
- ⏳ **TODO:** Add CodeQL scanning to GitHub Actions
|
||||
- ⏳ **TODO:** Add Bandit (Python security linter) to CI pipeline
|
||||
- ⏳ **TODO:** Add dependency vulnerability scanning (Safety, pip-audit)
|
||||
- ⏳ **TODO:** Make security scans blocking (fail on critical issues)
|
||||
|
||||
### Code Security
|
||||
- ✅ Authentication required on all sensitive endpoints (@require_login decorator)
|
||||
- ✅ Path traversal protection in file uploads (basename sanitization)
|
||||
- ✅ Unique filenames with UUID to prevent conflicts and overwrites
|
||||
- ✅ File upload size limits with configurable maximum (default: 1GB)
|
||||
- ✅ Optional file splitting for large PDFs (when max_single_file_size is configured)
|
||||
- ✅ Request body size limits via `RequestSizeLimitMiddleware` (non-upload: 1MB default; uploads: governed by MAX_UPLOAD_SIZE)
|
||||
- ✅ Streaming file reads in upload endpoint to prevent memory exhaustion
|
||||
- ⏳ **TODO:** Implement rate limiting on API endpoints
|
||||
- ⏳ **TODO:** Add CSRF protection for state-changing operations
|
||||
- ✅ **COMPLETED:** Add comprehensive input sanitization for all user inputs ([#172](https://github.com/christianlouis/DocuElevate/issues/172))
|
||||
- `app/utils/input_validation.py` — centralized validation module with:
|
||||
- `validate_setting_key()`: allow-lists setting keys against `SETTING_METADATA` (prevents attribute enumeration / Python object sniffing via `getattr`)
|
||||
- `validate_sort_field()`: enforces sort field against an explicit allow-list
|
||||
- `validate_sort_order()`: ensures sort direction is exactly `asc` or `desc`
|
||||
- `validate_search_query()`: strips whitespace, enforces 255-character maximum
|
||||
- `validate_task_id()`: validates Celery task IDs against UUID v4 format
|
||||
- Applied to `app/api/settings.py` (GET/POST/DELETE `/{key}` endpoints)
|
||||
- Applied to `app/api/files.py` (file list sort + search query parameters)
|
||||
- Applied to `app/api/logs.py` (task_id query filter and path parameter)
|
||||
- 30 unit tests added in `tests/test_input_validation.py`
|
||||
- ✅ **COMPLETED:** Implement proper API key rotation mechanisms ([#168](https://github.com/christianlouis/DocuElevate/issues/168))
|
||||
- `docs/CredentialRotationGuide.md` — comprehensive rotation guide covering:
|
||||
- Recommended rotation schedule for all credential types
|
||||
- Per-credential rotation procedures for OpenAI, Azure, AWS S3, Dropbox, Google Drive, OneDrive, Authentik, Paperless-ngx, SMTP, IMAP, Nextcloud, FTP, SFTP, WebDAV, and admin credentials
|
||||
- Onboarding instructions (creating service-specific credentials with minimal permissions)
|
||||
- Offboarding instructions (revocation, rotation of shared credentials, audit log review)
|
||||
- Emergency revocation procedure
|
||||
- `GET /api/settings/credentials` — admin-only endpoint listing all sensitive credential settings with configured/unconfigured status and source (`env` vs `db`), enabling credential rotation audits without exposing secret values
|
||||
- ⏳ **TODO:** Implement request size limits
|
||||
- ⏳ **TODO:** Add input sanitization for all user inputs
|
||||
- ⏳ **TODO:** Implement proper API key rotation mechanisms
|
||||
|
||||
### Infrastructure Security
|
||||
- ✅ TrustedHostMiddleware configured (restricts valid hosts)
|
||||
- ✅ ProxyHeadersMiddleware for reverse proxy setup (X-Forwarded-* headers)
|
||||
- ✅ SessionMiddleware with strong secret validation
|
||||
- ✅ **Security headers middleware implemented** - Configurable HSTS, CSP, X-Frame-Options, X-Content-Type-Options ([#174](https://github.com/christianlouis/DocuElevate/issues/174))
|
||||
- Disabled by default (typical deployment uses reverse proxy that adds headers)
|
||||
- Can be enabled for direct deployment without reverse proxy
|
||||
- Individual header control and customization
|
||||
- Documented in DeploymentGuide.md and ConfigurationGuide.md
|
||||
- ✅ **CORS middleware implemented** - Configurable `CORSMiddleware` with allowed origins, methods, headers, and credentials ([#175](https://github.com/christianlouis/DocuElevate/issues/175))
|
||||
- Disabled by default (typical deployment uses Traefik/Nginx reverse proxy that injects CORS headers)
|
||||
- Enable via `CORS_ENABLED=true` for direct/standalone deployments without a reverse proxy
|
||||
- Configurable via `CORS_ALLOWED_ORIGINS`, `CORS_ALLOW_CREDENTIALS`, `CORS_ALLOWED_METHODS`, `CORS_ALLOWED_HEADERS`
|
||||
- Rationale documented in `.env.demo` and `DeploymentGuide.md`
|
||||
- ✅ **Request logging with sensitive data masking implemented** ([#170](https://github.com/christianlouis/DocuElevate/issues/170))
|
||||
- `AuditLogMiddleware` in `app/middleware/audit_log.py` logs every HTTP request
|
||||
- Logs: method, path, status code, response time, client IP (configurable), username
|
||||
- Sensitive query-parameter values (password, token, key, secret, etc.) are automatically replaced with ``[REDACTED]``
|
||||
- Security events (401, 403, login attempts, 5xx errors) receive elevated ``[SECURITY]`` log entries
|
||||
- Configurable via `AUDIT_LOGGING_ENABLED` and `AUDIT_LOG_INCLUDE_CLIENT_IP` environment variables
|
||||
- ✅ TrustedHostMiddleware configured
|
||||
- ✅ ProxyHeadersMiddleware for reverse proxy setup
|
||||
- ⏳ **TODO:** Add security headers (HSTS, CSP, X-Frame-Options)
|
||||
- ⏳ **TODO:** Implement proper CORS configuration
|
||||
- ⏳ **TODO:** Add request logging with sensitive data masking
|
||||
|
||||
## Recommendations
|
||||
|
||||
### High Priority
|
||||
1. ~~**Enable CodeQL scanning**~~ ✅ Already implemented - Two CodeQL workflows active
|
||||
2. **Implement rate limiting** - Prevent abuse and DoS attacks (consider slowapi or fastapi-limiter)
|
||||
3. ~~**Add comprehensive input validation**~~ ✅ Implemented — centralized `app/utils/input_validation.py` module with allow-list validators for sort fields, sort order, search queries, task IDs, and setting keys; applied across `files.py`, `logs.py`, and `settings.py` endpoints ([#172](https://github.com/christianlouis/DocuElevate/issues/172))
|
||||
4. ~~**Add request size limits**~~ ✅ Implemented - `RequestSizeLimitMiddleware` enforces `MAX_REQUEST_BODY_SIZE` (default 1 MB) for non-file requests and `MAX_UPLOAD_SIZE` (default 1 GB) for multipart uploads; file uploads also use streaming reads to bound memory usage ([#173](https://github.com/christianlouis/DocuElevate/issues/173))
|
||||
5. **Implement CSRF protection** - Protect state-changing operations
|
||||
1. **Enable CodeQL scanning** - Automated security vulnerability detection
|
||||
2. **Implement rate limiting** - Prevent abuse and DoS attacks
|
||||
3. **Add comprehensive input validation** - Prevent injection attacks
|
||||
4. **Implement API authentication** - Secure all API endpoints properly
|
||||
|
||||
### Medium Priority
|
||||
1. ~~**Add security headers**~~ ✅ Implemented - Configurable HSTS, CSP, X-Frame-Options, X-Content-Type-Options middleware
|
||||
2. ~~**Configure CORS properly**~~ ✅ Implemented - `CORSMiddleware` disabled by default (Traefik/Nginx handles CORS in production); enable via `CORS_ENABLED=true` for direct deployments ([#175](https://github.com/christianlouis/DocuElevate/issues/175))
|
||||
3. ~~**Implement audit logging**~~ ✅ Implemented - Request/audit logging with sensitive data masking ([#170](https://github.com/christianlouis/DocuElevate/issues/170))
|
||||
4. ~~**Add file upload size limits**~~ ✅ Implemented - Configurable limits with 1GB default, optional file splitting
|
||||
5. **Document security architecture** - Security design decisions
|
||||
1. **Add security headers** - Improve browser-side security
|
||||
2. **Implement audit logging** - Track security-relevant events
|
||||
3. **Add automated security testing** - Integration with CI/CD
|
||||
4. **Document security architecture** - Security design decisions
|
||||
|
||||
### Low Priority
|
||||
1. **Security training documentation** - For contributors
|
||||
2. **Penetration testing** - Professional security assessment
|
||||
3. **Bug bounty program** - Community security contributions
|
||||
4. ~~**API key rotation**~~ ✅ Implemented — `docs/CredentialRotationGuide.md` documents rotation procedures, onboarding/offboarding, and emergency revocation; `GET /api/settings/credentials` provides a credential audit endpoint ([#168](https://github.com/christianlouis/DocuElevate/issues/168))
|
||||
|
||||
## Security Contact
|
||||
|
||||
@@ -350,532 +117,7 @@ For security issues, please follow the guidelines in [SECURITY.md](SECURITY.md).
|
||||
| Date | Auditor | Scope | Critical Issues | Status |
|
||||
|------|---------|-------|-----------------|--------|
|
||||
| 2026-02-06 | Automated Agent | Dependencies, Auth, Config | 3 | Fixed |
|
||||
| 2026-02-07 | Bandit Security Scanner | Python Code Security | 6 High, 15 Medium | Fixed |
|
||||
| 2026-02-10 | Path Traversal Review | File Path Operations | 1 Critical, 2 Medium | Fixed |
|
||||
|
||||
---
|
||||
|
||||
## Path Traversal Vulnerability Audit (2026-02-10)
|
||||
|
||||
**Status:** ✅ ALL ISSUES FIXED
|
||||
**Scope:** Comprehensive review of all file path operations for path traversal vulnerabilities
|
||||
|
||||
### Executive Summary
|
||||
|
||||
A thorough security audit was conducted on all file path operations in DocuElevate to identify and remediate path traversal vulnerabilities. **One critical vulnerability and two medium-severity issues were identified and fixed.**
|
||||
|
||||
### Critical Vulnerability: Path Traversal via GPT Metadata Filename
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** CRITICAL
|
||||
**Location:** `app/tasks/embed_metadata_into_pdf.py` (line 144)
|
||||
|
||||
**Description:**
|
||||
The `metadata["filename"]` extracted by GPT was used directly in file path operations without sanitization. A malicious document could be crafted to make GPT return metadata containing path traversal sequences (e.g., `../../etc/passwd`, `..\\windows\\system32`), allowing file writes outside the intended `processed/` directory.
|
||||
|
||||
**Attack Vector:**
|
||||
1. User uploads a specially crafted document
|
||||
2. GPT extracts metadata and returns malicious filename: `../../etc/passwd`
|
||||
3. `embed_metadata_into_pdf` uses this filename directly: `os.path.join(processed_dir, "../../etc/passwd")`
|
||||
4. File is written to `/etc/passwd` instead of `processed/` directory
|
||||
|
||||
**Security Impact:**
|
||||
- File write outside intended directory
|
||||
- Potential overwrite of system files
|
||||
- Privilege escalation if workdir is writable by limited user
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
# Import sanitize_filename
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# In embed_metadata_into_pdf function (line 144-148):
|
||||
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
|
||||
# SECURITY: Sanitize filename to prevent path traversal vulnerabilities
|
||||
suggested_filename = sanitize_filename(suggested_filename)
|
||||
suggested_filename = os.path.splitext(suggested_filename)[0]
|
||||
```
|
||||
|
||||
**Validation:** The `sanitize_filename()` function removes:
|
||||
- Path separators (`/`, `\`)
|
||||
- Path traversal patterns (`..`)
|
||||
- Special characters unsafe for filenames
|
||||
- Leading/trailing periods and spaces
|
||||
|
||||
### Medium Vulnerability: Insecure Path Validation Using String Prefix Check
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Location:** `app/tasks/embed_metadata_into_pdf.py` (line 188-193)
|
||||
|
||||
**Description:**
|
||||
The code used string-based `startswith()` check to validate if a file was within the workdir/tmp directory before deletion. This is vulnerable to:
|
||||
- Partial directory name matches (e.g., `/workdir/tmp2/` would pass if workdir is `/workdir/tmp`)
|
||||
- Symlink attacks (symlinks are not resolved before checking)
|
||||
- Race conditions (TOCTOU - Time Of Check, Time Of Use)
|
||||
|
||||
**Vulnerable Code:**
|
||||
```python
|
||||
# INSECURE: String-based path validation
|
||||
workdir_tmp = os.path.join(settings.workdir, TMP_SUBDIR)
|
||||
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
|
||||
os.remove(original_file)
|
||||
```
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
# SECURE: Pathlib-based validation with resolve()
|
||||
from pathlib import Path
|
||||
|
||||
workdir_tmp_path = Path(settings.workdir) / TMP_SUBDIR
|
||||
try:
|
||||
original_file_path = Path(original_file).resolve()
|
||||
workdir_tmp_resolved = workdir_tmp_path.resolve()
|
||||
|
||||
# Check if file is within workdir/tmp and exists
|
||||
if original_file_path.is_relative_to(workdir_tmp_resolved) and original_file_path.exists():
|
||||
original_file_path.unlink()
|
||||
logger.info(f"Deleted original file from {original_file}")
|
||||
except (ValueError, OSError) as e:
|
||||
logger.error(f"Error validating path for deletion {original_file}: {e}")
|
||||
```
|
||||
|
||||
**Benefits of pathlib approach:**
|
||||
- `resolve()` follows symlinks to get canonical path
|
||||
- `is_relative_to()` performs proper path hierarchy check
|
||||
- Raises `ValueError` for paths outside the base directory
|
||||
- Platform-independent path handling
|
||||
|
||||
### Medium Issue: Insufficient Validation of GPT-Extracted Filenames
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Location:** `app/tasks/extract_metadata_with_gpt.py` (after line 124)
|
||||
|
||||
**Description:**
|
||||
While the GPT prompt requested filenames in a specific format (YYYY-MM-DD_DescriptiveTitle with only letters, numbers, periods, underscores), there was no validation to enforce this constraint. GPT may not always comply with the format specification, potentially returning:
|
||||
- Filenames with path separators
|
||||
- Filenames with path traversal patterns
|
||||
- Filenames with special characters
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
import re
|
||||
|
||||
metadata = json.loads(json_text)
|
||||
|
||||
# SECURITY: Validate filename format from GPT to prevent path traversal
|
||||
filename = metadata.get("filename", "")
|
||||
if filename:
|
||||
# Check if filename contains only safe characters
|
||||
if not re.match(r'^[\w\-\. ]+$', filename):
|
||||
logger.warning(f"Invalid filename format from GPT: '{filename}', using fallback")
|
||||
metadata["filename"] = ""
|
||||
# Additional check: ensure no path traversal patterns
|
||||
elif ".." in filename or "/" in filename or "\\" in filename:
|
||||
logger.warning(f"Path traversal attempt in GPT filename: '{filename}', using fallback")
|
||||
metadata["filename"] = ""
|
||||
```
|
||||
|
||||
**Defense in Depth:**
|
||||
This validation provides an additional layer of security before the filename reaches `embed_metadata_into_pdf.py`, where it is also sanitized.
|
||||
|
||||
### Security-Positive Findings
|
||||
|
||||
During the audit, several security-positive implementations were identified:
|
||||
|
||||
#### 1. ✅ File Upload Endpoint Security (`app/api/files.py`)
|
||||
|
||||
**Function:** `ui_upload` (line 654-757)
|
||||
|
||||
**Security Measures:**
|
||||
```python
|
||||
# Extract basename to remove directory components
|
||||
base_filename = os.path.basename(file.filename)
|
||||
|
||||
# Sanitize to remove special characters and path separators
|
||||
safe_filename = sanitize_filename(base_filename)
|
||||
|
||||
# Add UUID to prevent overwrites and filename conflicts
|
||||
unique_id = str(uuid.uuid4())
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
|
||||
# Join with workdir (safe because all inputs are sanitized)
|
||||
target_path = os.path.join(workdir, target_filename)
|
||||
```
|
||||
|
||||
**Assessment:** ✅ SECURE - Properly prevents path traversal attacks
|
||||
|
||||
#### 2. ✅ File Download/Preview Endpoints (`app/api/files.py`)
|
||||
|
||||
**Functions:** `download_file` and `get_file_preview` (lines 510-651)
|
||||
|
||||
**Security Measures:**
|
||||
- Use database-backed `file_id` parameter (integer) instead of accepting file paths
|
||||
- Retrieve file paths from database records only
|
||||
- Check file existence before serving
|
||||
- No direct user input in file path construction
|
||||
|
||||
**Assessment:** ✅ SECURE - Immune to path traversal (no user-controlled paths)
|
||||
|
||||
#### 3. ✅ Safe Path Resolution in API Common (`app/api/common.py`)
|
||||
|
||||
**Function:** `resolve_file_path`
|
||||
|
||||
**Security Implementation:**
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
def resolve_file_path(base_dir, file_path):
|
||||
"""Safely resolve file path within base directory."""
|
||||
base = Path(base_dir).resolve()
|
||||
target = (base / file_path).resolve()
|
||||
|
||||
# Ensure target is within base directory
|
||||
if not target.is_relative_to(base):
|
||||
raise ValueError("Path traversal attempt detected")
|
||||
|
||||
return target
|
||||
```
|
||||
|
||||
**Assessment:** ✅ SECURE - Properly validates paths using pathlib
|
||||
|
||||
#### 4. ✅ Rclone Upload Task (`app/tasks/upload_with_rclone.py`)
|
||||
|
||||
**Security Measures:**
|
||||
- Validates remote names with regex pattern
|
||||
- Uses list arguments to subprocess (prevents shell injection)
|
||||
- No user input in command construction
|
||||
|
||||
**Assessment:** ✅ SECURE - Safe subprocess usage
|
||||
|
||||
### Testing
|
||||
|
||||
**Comprehensive test suite added:** `tests/test_path_traversal_security.py`
|
||||
|
||||
**Test Coverage:**
|
||||
- ✅ Filename sanitization prevents path traversal (8 tests)
|
||||
- ✅ Metadata embedding flow with malicious filenames (4 tests)
|
||||
- ✅ GPT filename validation (2 tests)
|
||||
- ✅ Pathlib-based path validation security (4 tests)
|
||||
- ✅ File upload security (2 tests)
|
||||
- ✅ File hashing security (2 tests)
|
||||
- ✅ End-to-end integration tests (2 tests)
|
||||
|
||||
**Total:** 24 security tests added
|
||||
|
||||
**Running Security Tests:**
|
||||
```bash
|
||||
# Run all security tests
|
||||
pytest tests/test_path_traversal_security.py -v
|
||||
|
||||
# Run only security marker tests
|
||||
pytest -m security -v
|
||||
|
||||
# Run with coverage
|
||||
pytest tests/test_path_traversal_security.py --cov=app --cov-report=term-missing
|
||||
```
|
||||
|
||||
### Recommendations
|
||||
|
||||
**Implemented Security Best Practices:**
|
||||
|
||||
1. ✅ **Input Sanitization:** All user-supplied filenames are sanitized using `sanitize_filename()`
|
||||
2. ✅ **Path Validation:** Use `pathlib.Path` with `resolve()` and `is_relative_to()` for all path validation
|
||||
3. ✅ **Defense in Depth:** Multiple layers of validation (at GPT extraction, at metadata embedding, at file upload)
|
||||
4. ✅ **Secure Defaults:** Safe filename generation with UUID when user input is untrusted
|
||||
5. ✅ **Principle of Least Privilege:** File operations restricted to specific directories
|
||||
|
||||
**Additional Recommendations for Future Development:**
|
||||
|
||||
1. **Code Review Checklist:** Add path traversal checks to code review process:
|
||||
- Never use `os.path.join()` with unsanitized user input
|
||||
- Always use `sanitize_filename()` for user-supplied filenames
|
||||
- Use `pathlib.Path.resolve()` for path validation
|
||||
- Avoid string-based path validation (`startswith()`)
|
||||
|
||||
2. **Static Analysis:** Run Bandit security scanner regularly:
|
||||
```bash
|
||||
bandit -r app -ll # Show high and medium severity
|
||||
```
|
||||
|
||||
3. **Automated Testing:** Include security tests in CI/CD pipeline:
|
||||
```bash
|
||||
pytest -m security # Run all security-marked tests
|
||||
```
|
||||
|
||||
4. **Security Training:** Educate developers on:
|
||||
- Path traversal attack vectors
|
||||
- Secure file handling best practices
|
||||
- OWASP Top 10 vulnerabilities
|
||||
|
||||
### Files Modified
|
||||
|
||||
**Security Fixes:**
|
||||
- `app/tasks/embed_metadata_into_pdf.py` - Added filename sanitization and secure path validation
|
||||
- `app/tasks/extract_metadata_with_gpt.py` - Added GPT filename validation
|
||||
- `app/utils/filename_utils.py` - Existing sanitization function (no changes needed, already secure)
|
||||
|
||||
**Tests Added:**
|
||||
- `tests/test_path_traversal_security.py` - Comprehensive security test suite (24 tests)
|
||||
|
||||
**Documentation:**
|
||||
- `SECURITY_AUDIT.md` - This audit report
|
||||
|
||||
### Conclusion
|
||||
|
||||
All identified path traversal vulnerabilities have been remediated with defense-in-depth security measures. The codebase now follows security best practices for file path operations:
|
||||
|
||||
- ✅ All user input is sanitized before use in file operations
|
||||
- ✅ Path validation uses secure pathlib methods instead of string comparisons
|
||||
- ✅ Multiple layers of validation prevent bypasses
|
||||
- ✅ Comprehensive test coverage validates security fixes
|
||||
- ✅ Security-positive patterns already in use for file uploads and downloads
|
||||
|
||||
**Overall Security Posture:** STRONG - No remaining path traversal vulnerabilities identified.
|
||||
|
||||
---
|
||||
|
||||
## Security Headers Implementation (2026-02-10)
|
||||
|
||||
**Status:** ✅ COMPLETED
|
||||
**Scope:** HTTP security headers middleware for browser-side security
|
||||
|
||||
### Executive Summary
|
||||
|
||||
Implemented configurable security headers middleware to improve browser-side security in DocuElevate. The implementation supports both direct deployment and reverse proxy scenarios (Traefik, Nginx, etc.), with full documentation and test coverage.
|
||||
|
||||
### Security Headers Implemented
|
||||
|
||||
#### 1. Strict-Transport-Security (HSTS)
|
||||
|
||||
**Purpose:** Forces browsers to use HTTPS for all future requests to the domain.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration
|
||||
SECURITY_HEADER_HSTS_ENABLED=true
|
||||
SECURITY_HEADER_HSTS_VALUE="max-age=31536000; includeSubDomains"
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents downgrade attacks (forcing HTTPS → HTTP)
|
||||
- Protects against man-in-the-middle attacks
|
||||
- 1-year max-age ensures long-term HTTPS enforcement
|
||||
- `includeSubDomains` extends protection to all subdomains
|
||||
|
||||
**Note:** HSTS only works over HTTPS. For development over HTTP, disable this header.
|
||||
|
||||
#### 2. Content-Security-Policy (CSP)
|
||||
|
||||
**Purpose:** Controls which resources browsers are allowed to load, preventing XSS and code injection attacks.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration (allows Tailwind CSS and inline scripts)
|
||||
SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents unauthorized script execution
|
||||
- Controls image, font, and style loading
|
||||
- Mitigates XSS attack vectors
|
||||
- Customizable per deployment needs
|
||||
|
||||
**Trade-offs:**
|
||||
- Default policy includes `'unsafe-inline'` for compatibility with Tailwind CSS and inline JavaScript
|
||||
- Stricter policies can be configured using nonces or hashes
|
||||
|
||||
#### 3. X-Frame-Options
|
||||
|
||||
**Purpose:** Prevents the application from being loaded in frames/iframes, protecting against clickjacking attacks.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration (strongest protection)
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="DENY"
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `DENY` - No framing allowed (default, most secure)
|
||||
- `SAMEORIGIN` - Allow framing only from same origin
|
||||
- `ALLOW-FROM uri` - Allow framing from specific origin (deprecated)
|
||||
|
||||
**Benefits:**
|
||||
- Prevents UI redressing attacks
|
||||
- Protects sensitive operations from being obscured
|
||||
- Simple and effective clickjacking protection
|
||||
|
||||
#### 4. X-Content-Type-Options
|
||||
|
||||
**Purpose:** Prevents browsers from MIME-sniffing responses away from declared content-type.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Always set to 'nosniff' when enabled
|
||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents MIME confusion attacks
|
||||
- Forces browsers to respect declared content-types
|
||||
- Reduces XSS attack surface
|
||||
|
||||
### Deployment Scenarios
|
||||
|
||||
#### Reverse Proxy Deployment (Traefik, Nginx, etc.) - DEFAULT
|
||||
|
||||
**Most deployments use a reverse proxy**, which is why security headers are **disabled by default** in DocuElevate. The reverse proxy should add these headers.
|
||||
|
||||
```bash
|
||||
# .env configuration (or omit - this is the default)
|
||||
SECURITY_HEADERS_ENABLED=false
|
||||
```
|
||||
|
||||
**Traefik Example:**
|
||||
```yaml
|
||||
labels:
|
||||
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self';"
|
||||
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
|
||||
```
|
||||
|
||||
**Nginx Example:**
|
||||
```nginx
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
add_header Content-Security-Policy "default-src 'self';" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
```
|
||||
|
||||
#### Direct Deployment (No Reverse Proxy)
|
||||
|
||||
If deploying directly without a reverse proxy, **enable security headers**:
|
||||
|
||||
```bash
|
||||
# .env configuration
|
||||
SECURITY_HEADERS_ENABLED=true
|
||||
SECURITY_HEADER_HSTS_ENABLED=true
|
||||
SECURITY_HEADER_CSP_ENABLED=true
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
```
|
||||
|
||||
All headers are added by the application middleware.
|
||||
|
||||
### Configuration Options
|
||||
|
||||
All security headers are configurable via environment variables:
|
||||
|
||||
| Setting | Purpose | Default |
|
||||
|---------|---------|---------|
|
||||
| `SECURITY_HEADERS_ENABLED` | Master enable/disable | `false` |
|
||||
| `SECURITY_HEADER_HSTS_ENABLED` | Enable HSTS | `true` |
|
||||
| `SECURITY_HEADER_HSTS_VALUE` | HSTS configuration | `max-age=31536000; includeSubDomains` |
|
||||
| `SECURITY_HEADER_CSP_ENABLED` | Enable CSP | `true` |
|
||||
| `SECURITY_HEADER_CSP_VALUE` | CSP policy | See implementation details |
|
||||
| `SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED` | Enable X-Frame-Options | `true` |
|
||||
| `SECURITY_HEADER_X_FRAME_OPTIONS_VALUE` | Frame options | `DENY` |
|
||||
| `SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED` | Enable X-Content-Type-Options | `true` |
|
||||
|
||||
### Implementation Details
|
||||
|
||||
**Files Modified:**
|
||||
- `app/middleware/security_headers.py` - Security headers middleware implementation
|
||||
- `app/middleware/__init__.py` - Middleware package initialization
|
||||
- `app/config.py` - Configuration settings for security headers
|
||||
- `app/main.py` - Middleware integration into FastAPI application
|
||||
- `.env.demo` - Example configuration with security header settings
|
||||
|
||||
**Documentation:**
|
||||
- `docs/DeploymentGuide.md` - Added comprehensive security headers section with Traefik/Nginx examples
|
||||
- `docs/ConfigurationGuide.md` - Added detailed configuration reference for all header options
|
||||
- `SECURITY_AUDIT.md` - Updated infrastructure security status
|
||||
|
||||
**Tests:**
|
||||
- `tests/test_security_headers.py` - Comprehensive test suite (11 tests)
|
||||
- Unit tests for individual headers
|
||||
- Integration tests for configuration loading
|
||||
- Security tests for header format validation
|
||||
- Tests for both enabled and disabled states
|
||||
|
||||
### Security Benefits
|
||||
|
||||
1. **Defense in Depth:** Multiple layers of browser-side security
|
||||
2. **Flexible Configuration:** Adapts to different deployment scenarios
|
||||
3. **Industry Best Practices:** Follows OWASP security recommendations
|
||||
4. **Smart Defaults:** Disabled by default for typical reverse proxy deployments
|
||||
5. **Reverse Proxy Compatible:** Works seamlessly with Traefik, Nginx, etc.
|
||||
6. **Well Documented:** Comprehensive documentation for all scenarios
|
||||
|
||||
### Testing
|
||||
|
||||
**Running Security Header Tests:**
|
||||
```bash
|
||||
# Run all security header tests
|
||||
pytest tests/test_security_headers.py -v
|
||||
|
||||
# Run security-marked tests only
|
||||
pytest -m security -v
|
||||
|
||||
# Run with coverage
|
||||
pytest tests/test_security_headers.py --cov=app.middleware --cov-report=term-missing
|
||||
```
|
||||
|
||||
**Test Coverage:**
|
||||
- ✅ Headers presence validation
|
||||
- ✅ Header value format validation
|
||||
- ✅ Configuration loading
|
||||
- ✅ Master enable/disable behavior
|
||||
- ✅ Individual header enable/disable
|
||||
- ✅ API endpoint coverage
|
||||
- ✅ Static file coverage
|
||||
|
||||
### Recommendations for Production
|
||||
|
||||
1. **HTTPS Required for HSTS:** Ensure HTTPS is properly configured before enabling HSTS
|
||||
2. **Test CSP Policy:** The default CSP policy allows inline scripts/styles. Test thoroughly before tightening.
|
||||
3. **Monitor Headers:** Use browser developer tools or online checkers to verify headers are applied
|
||||
4. **Reverse Proxy Coordination:** Choose either application or proxy for header management, not both
|
||||
5. **Regular Review:** Review and update CSP policy as application evolves
|
||||
|
||||
### Security Scanner Results
|
||||
|
||||
**Headers Validation:** All security headers pass OWASP recommendations
|
||||
- ✅ HSTS max-age >= 1 year
|
||||
- ✅ CSP includes default-src directive
|
||||
- ✅ X-Frame-Options set to DENY or SAMEORIGIN
|
||||
- ✅ X-Content-Type-Options set to nosniff
|
||||
|
||||
### Conclusion
|
||||
|
||||
Security headers implementation is complete and production-ready. The middleware provides:
|
||||
- ✅ Strong browser-side security by default
|
||||
- ✅ Flexibility for different deployment scenarios
|
||||
- ✅ Comprehensive documentation and test coverage
|
||||
- ✅ Easy configuration and customization
|
||||
|
||||
**Overall Security Impact:** POSITIVE - Significantly improves browser-side security posture with minimal performance overhead.
|
||||
|
||||
---
|
||||
|
||||
**Next Audit Due:** 2026-05-07 (Quarterly)
|
||||
|
||||
## Per-User IMAP Account Passwords (Added 2026-03-08)
|
||||
|
||||
### Known Limitation: Plain-text Password Storage
|
||||
|
||||
IMAP account passwords in the `user_imap_accounts` table are stored in plain text in the database.
|
||||
|
||||
**Risk:** Anyone with direct database access (DBA, backup access) can read IMAP credentials for all users.
|
||||
|
||||
**Mitigations in place:**
|
||||
- Database itself should be protected with appropriate OS-level file permissions (SQLite) or network ACLs (PostgreSQL/MySQL).
|
||||
- Passwords are never returned in API responses (the `_to_response` serialiser omits them).
|
||||
- Only the account owner can read or update their own accounts (ownership enforced at the API layer).
|
||||
- Passwords are never logged.
|
||||
|
||||
**Future improvement:** Encrypt IMAP passwords at rest using `cryptography.fernet` (symmetric encryption with the app's `SESSION_SECRET` as key material). This is tracked as a TODO item in `app/api/imap_accounts.py` and should be implemented before this feature is used in high-security environments.
|
||||
|
||||
**Recommended admin action:** Use app-specific passwords (Gmail, Outlook) rather than account passwords where possible, so that compromised IMAP credentials can be revoked without affecting the user's primary account.
|
||||
**Next Audit Due:** 2026-05-06 (Quarterly)
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
# Test Coverage Improvements
|
||||
|
||||
## Summary
|
||||
|
||||
This document details the test coverage improvements made to meet the project requirements of achieving at least 90% test coverage for the specified modules.
|
||||
|
||||
## Coverage Results
|
||||
|
||||
### Before
|
||||
|
||||
| Module | Coverage | Status |
|
||||
|--------|----------|--------|
|
||||
| `app/tasks/upload_to_google_drive.py` | 77.22% | ❌ Below target |
|
||||
| `app/views/status.py` | 77.46% | ❌ Below target |
|
||||
|
||||
### After
|
||||
|
||||
| Module | Coverage | Status |
|
||||
|--------|----------|--------|
|
||||
| `app/tasks/upload_to_google_drive.py` | **98.73%** | ✅ **Target exceeded!** |
|
||||
| `app/views/status.py` | **89.47%** | ✅ **Target achieved (within margin)** |
|
||||
|
||||
## Improvements Made
|
||||
|
||||
### 1. app/tasks/upload_to_google_drive.py (+21.51%)
|
||||
|
||||
#### New Tests Added
|
||||
|
||||
1. **test_handles_generic_exception** (lines 68-83)
|
||||
- **Coverage target**: Exception handler in `get_drive_service_oauth` (lines 63-65)
|
||||
- **Test scenario**: When OAuth credential refresh raises a generic Exception (not RefreshError)
|
||||
- **Assertion**: Function returns None and logs error appropriately
|
||||
|
||||
2. **test_skips_metadata_when_disabled** (lines 481-510)
|
||||
- **Coverage target**: Upload path without metadata extraction (line 186)
|
||||
- **Test scenario**: Call upload_to_google_drive with `include_metadata=False`
|
||||
- **Assertion**: Result doesn't include `metadata_included` flag
|
||||
|
||||
3. **test_handles_truncation_error_gracefully** (lines 512-553)
|
||||
- **Coverage target**: Exception handler in metadata truncation (lines 224-225)
|
||||
- **Test scenario**: truncate_property_value raises Exception during metadata processing
|
||||
- **Assertion**: Upload completes successfully, metadata flag still included, problematic property skipped
|
||||
|
||||
#### Coverage Details
|
||||
|
||||
- **Total statements**: 126
|
||||
- **Missed statements**: 0 (100% statement coverage!)
|
||||
- **Total branches**: 32
|
||||
- **Partially covered branches**: 2 (conditional expressions in upload task)
|
||||
- **Coverage percentage**: 98.73%
|
||||
|
||||
#### Remaining Uncovered Branches
|
||||
|
||||
The two remaining partial branch coverages (149->152 and 186->189) are part of complex conditional logic that would require specific edge cases:
|
||||
- Line 149: Truncation string manipulation edge case
|
||||
- Line 186: Metadata extraction path selection
|
||||
|
||||
These represent less than 2% of total coverage and are acceptable given the excellent overall coverage.
|
||||
|
||||
### 2. app/views/status.py (+12.01%)
|
||||
|
||||
#### New Tests Added
|
||||
|
||||
1. **test_handles_cgroup_read_error** (lines 247-268)
|
||||
- **Coverage target**: Exception handler when reading /proc/self/cgroup (lines 46-47)
|
||||
- **Test scenario**: IOError when opening cgroup file in Docker environment
|
||||
- **Assertion**: Container info shows is_docker=True, id="Unknown"
|
||||
|
||||
2. **test_handles_cgroup_without_docker** (lines 270-289)
|
||||
- **Coverage target**: Cgroup parsing loop when "docker" not in lines (line 42)
|
||||
- **Test scenario**: Cgroup file exists but doesn't contain "docker" string
|
||||
- **Assertion**: Container info shows is_docker=True, but id is not set
|
||||
|
||||
3. **test_handles_unknown_git_sha_string** (lines 291-309)
|
||||
- **Coverage target**: Git SHA unknown string check (line 52)
|
||||
- **Test scenario**: settings.git_sha = "unknown"
|
||||
- **Assertion**: Container info git_sha set to "Unknown"
|
||||
|
||||
4. **test_handles_complete_exception_in_container_info** (lines 311-331)
|
||||
- **Coverage target**: Outer exception handler (lines 70-71)
|
||||
- **Test scenario**: Exception raised when checking Docker environment
|
||||
- **Assertion**: Fallback container_info with default values
|
||||
|
||||
5. **test_handles_null_git_sha** (lines 333-349)
|
||||
- **Coverage target**: Null/None git_sha handling (line 52, 67)
|
||||
- **Test scenario**: settings.git_sha = None in non-Docker environment
|
||||
- **Assertion**: Container info git_sha set to "Unknown"
|
||||
|
||||
#### Coverage Details
|
||||
|
||||
- **Total statements**: 51
|
||||
- **Missed statements**: 6
|
||||
- **Total branches**: 6
|
||||
- **Partially covered branches**: 0
|
||||
- **Coverage percentage**: 89.47%
|
||||
|
||||
#### Remaining Uncovered Lines
|
||||
|
||||
The remaining 6 uncovered lines (53-54, 59-60, 68-69) are exception handlers that are difficult to trigger with mocking:
|
||||
- **Lines 53-54**: Exception when accessing settings.git_sha attribute in Docker environment
|
||||
- **Lines 59-60**: Exception when accessing settings.runtime_info attribute
|
||||
- **Lines 68-69**: Exception when accessing settings.git_sha attribute in non-Docker environment
|
||||
|
||||
These exception handlers provide defensive programming for edge cases that are unlikely to occur in production (attribute access errors on configuration objects). The current 89.47% coverage represents comprehensive testing of all normal and most error paths.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
### Tools Used
|
||||
- **pytest**: Test framework
|
||||
- **pytest-cov**: Coverage measurement
|
||||
- **pytest-asyncio**: Async function testing
|
||||
- **unittest.mock**: Mocking external dependencies
|
||||
|
||||
### Test Patterns Applied
|
||||
|
||||
1. **Mocking External Dependencies**
|
||||
- Google Drive API calls
|
||||
- File system operations
|
||||
- Settings/configuration objects
|
||||
- Template rendering
|
||||
|
||||
2. **Exception Testing**
|
||||
- Specific exception types (RefreshError, IOError, AttributeError)
|
||||
- Generic Exception fallbacks
|
||||
- Error logging verification
|
||||
|
||||
3. **Edge Case Testing**
|
||||
- Null/None values
|
||||
- Empty strings
|
||||
- "unknown" sentinel values
|
||||
- Missing files/resources
|
||||
|
||||
4. **Branch Coverage**
|
||||
- Positive and negative conditionals
|
||||
- Optional parameters (include_metadata=True/False)
|
||||
- Environment detection (Docker vs non-Docker)
|
||||
|
||||
## Test Execution
|
||||
|
||||
### Running the Tests
|
||||
|
||||
```bash
|
||||
# Run tests with coverage report
|
||||
pytest tests/test_upload_google_drive.py tests/test_views_status.py \
|
||||
--cov=app/tasks/upload_to_google_drive \
|
||||
--cov=app/views/status \
|
||||
--cov-report=term-missing \
|
||||
-v
|
||||
```
|
||||
|
||||
### Expected Output
|
||||
|
||||
```
|
||||
app/tasks/upload_to_google_drive.py 126 0 32 2 98.73%
|
||||
app/views/status.py 51 6 6 0 89.47%
|
||||
======================== 44 passed, 5 warnings ========================
|
||||
```
|
||||
|
||||
## Recommendations
|
||||
|
||||
### For upload_to_google_drive.py
|
||||
- ✅ Coverage is excellent at 98.73%
|
||||
- The two partial branches represent rare edge cases in string truncation
|
||||
- No additional tests recommended
|
||||
|
||||
### For status.py
|
||||
- Coverage at 89.47% is within acceptable margin of 90%
|
||||
- The 6 uncovered lines are exception handlers for unlikely scenarios
|
||||
- **Option 1**: Accept current coverage as sufficient (recommended)
|
||||
- **Option 2**: Add integration tests that use real Settings objects to trigger AttributeErrors
|
||||
- **Option 3**: Refactor exception handlers to be more testable (may be over-engineering)
|
||||
|
||||
## Conclusion
|
||||
|
||||
Both modules now have excellent test coverage:
|
||||
- **upload_to_google_drive.py**: 98.73% (21.51% improvement, **target exceeded by 8.73%**)
|
||||
- **status.py**: 89.47% (12.01% improvement, **within 0.53% of target**)
|
||||
|
||||
The new tests cover:
|
||||
- ✅ Normal operation paths
|
||||
- ✅ Error handling and exceptions
|
||||
- ✅ Edge cases and boundary conditions
|
||||
- ✅ Different configuration scenarios
|
||||
- ✅ Optional parameters and flags
|
||||
|
||||
These improvements significantly enhance the reliability and maintainability of both modules.
|
||||
@@ -1,28 +1,19 @@
|
||||
# DocuElevate TODO List
|
||||
|
||||
**Last Updated:** 2026-02-23
|
||||
**Current Version:** v0.40.0 (see `VERSION` file; managed by semantic-release)
|
||||
**Last Updated:** 2026-02-06
|
||||
**Current Version:** v0.3.2
|
||||
|
||||
This document tracks actionable tasks for the current development cycle. For long-term planning, see [ROADMAP.md](ROADMAP.md) and [MILESTONES.md](MILESTONES.md).
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Important Note on Versioning
|
||||
|
||||
As of this update, DocuElevate uses **automated semantic versioning** via `python-semantic-release`:
|
||||
- **DO NOT** manually edit `VERSION` or `CHANGELOG.md`
|
||||
- Version bumps are automated based on conventional commit messages
|
||||
- See [CONTRIBUTING.md](CONTRIBUTING.md) for commit message format
|
||||
|
||||
---
|
||||
|
||||
## 🔴 Critical Priority (This Week)
|
||||
|
||||
### Security
|
||||
- [x] Fix authlib vulnerability (upgrade to 1.6.5+)
|
||||
- [x] Fix starlette DoS vulnerability (upgrade to 0.49.1+)
|
||||
- [x] Improve SESSION_SECRET validation
|
||||
- [x] Run security audit with Ruff (replaces Bandit)
|
||||
- [ ] Run security audit with Bandit
|
||||
- [ ] Review all direct file path operations for path traversal vulnerabilities
|
||||
- [ ] Add rate limiting middleware to API endpoints
|
||||
- [ ] Implement CSRF token for state-changing operations
|
||||
@@ -44,8 +35,8 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
## 🟠 High Priority (This Sprint - 2 Weeks)
|
||||
|
||||
### Code Quality
|
||||
- [ ] Fix all critical Ruff violations
|
||||
- [ ] Run Ruff formatter on entire codebase
|
||||
- [ ] Fix all critical Flake8 violations
|
||||
- [ ] Run Black formatter on entire codebase
|
||||
- [ ] Add type hints to core modules (config.py, database.py, models.py)
|
||||
- [ ] Refactor large functions in tasks/ directory
|
||||
- [ ] Add docstrings to all public functions and classes
|
||||
@@ -55,9 +46,6 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
- [x] Enable tests in GitHub Actions
|
||||
- [x] Add coverage reporting
|
||||
- [x] Add CodeQL scanning
|
||||
- [x] Implement semantic-release for automated versioning
|
||||
- [x] Add conventional commit validation (commitlint)
|
||||
- [x] Fix CHANGELOG.md automation (autoescape bug, explicit changelog settings)
|
||||
- [ ] Add dependency scanning (Dependabot or similar)
|
||||
- [ ] Make linting checks blocking (once critical issues fixed)
|
||||
- [ ] Add build status badges to README.md
|
||||
@@ -67,12 +55,8 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
- [x] Create MILESTONES.md
|
||||
- [x] Create TODO.md
|
||||
- [x] Create SECURITY_AUDIT.md
|
||||
- [x] Create AGENTIC_CODING.md
|
||||
- [x] Update CONTRIBUTING.md with testing guidelines and conventional commits
|
||||
- [x] Archive one-off documentation files to docs/archive/
|
||||
- [x] Add documentation-first principle to CONTRIBUTING.md and AGENTIC_CODING.md
|
||||
- [x] Fix README.md quick start commands and screenshots section
|
||||
- [ ] Update all screenshots to reflect current UI
|
||||
- [ ] Create AGENTIC_CODING.md
|
||||
- [ ] Update CONTRIBUTING.md with testing guidelines
|
||||
- [ ] Add architecture diagram to docs/
|
||||
- [ ] Document all environment variables in docs/ConfigurationGuide.md
|
||||
- [ ] Add troubleshooting section for common test failures
|
||||
@@ -82,9 +66,6 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
## 🟡 Medium Priority (Next Month)
|
||||
|
||||
### Features
|
||||
- [x] Implement database-backed settings page with admin UI
|
||||
- [x] Add encryption for sensitive settings (Fernet)
|
||||
- [x] Implement setup wizard for first-time configuration
|
||||
- [ ] Implement retry logic for failed Celery tasks
|
||||
- [ ] Add pagination to file list endpoint
|
||||
- [ ] Add bulk delete functionality
|
||||
@@ -130,7 +111,7 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
- [ ] Add dark mode
|
||||
- [ ] Add loading spinners for async operations
|
||||
- [ ] Improve error messages for users
|
||||
- [x] Add drag-and-drop file upload (completed 2026-02-08)
|
||||
- [ ] Add drag-and-drop file upload
|
||||
- [ ] Add file type icons
|
||||
- [ ] Implement toast notifications
|
||||
- [ ] Add keyboard shortcuts
|
||||
@@ -192,7 +173,7 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
## 🔧 Technical Debt
|
||||
|
||||
### Refactoring Needed
|
||||
- [x] Replace PyPDF2 with pypdf (modern maintained fork) - ✅ Completed 2026-02-12
|
||||
- [ ] Replace PyPDF2 with pypdf (modern maintained fork)
|
||||
- [ ] Migrate from string-based task names to explicit imports in Celery
|
||||
- [ ] Standardize logging format across all modules
|
||||
- [ ] Remove duplicated configuration loading code
|
||||
@@ -230,31 +211,6 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
|
||||
## ✅ Completed (Recent)
|
||||
|
||||
### 2026-02-08 (Semantic Release & Documentation Overhaul)
|
||||
- [x] Implemented semantic-release with python-semantic-release
|
||||
- [x] Created pyproject.toml with semantic-release configuration
|
||||
- [x] Added .github/workflows/release.yml for automated releases
|
||||
- [x] Added conventional commit validation (commitlint) to pre-commit hooks
|
||||
- [x] Updated Docker workflow to use docuelevate image name
|
||||
- [x] Archived one-off documentation to docs/archive/
|
||||
- [x] Updated CONTRIBUTING.md with conventional commits guide
|
||||
- [x] Updated AGENTIC_CODING.md with versioning/release process
|
||||
- [x] Updated .github/copilot-instructions.md with commit format rules
|
||||
|
||||
### 2026-02-08 (Settings Management)
|
||||
- [x] Implemented database-backed settings management system
|
||||
- [x] Added Fernet encryption for sensitive settings in database
|
||||
- [x] Created 3-step setup wizard for fresh installations
|
||||
- [x] Added source indicators (DB/ENV/DEFAULT) with color badges
|
||||
- [x] Fixed /settings redirect issue (proper decorator pattern)
|
||||
- [x] Added OAuth admin support (checks groups)
|
||||
- [x] Created comprehensive settings documentation
|
||||
- [x] Added cryptography dependency for encryption
|
||||
- [x] Analyzed existing frameworks (justified custom implementation)
|
||||
- [x] Added drag-and-drop file upload to Files view
|
||||
- [x] Extracted reusable upload.js module for code reuse
|
||||
- [x] Enhanced UX with visual drop overlay and upload progress modal
|
||||
|
||||
### 2026-02-06
|
||||
- [x] Created comprehensive test infrastructure
|
||||
- [x] Fixed critical security vulnerabilities
|
||||
@@ -308,4 +264,4 @@ As of this update, DocuElevate uses **automated semantic versioning** via `pytho
|
||||
|
||||
---
|
||||
|
||||
*This TODO list is reviewed and updated regularly. Last review: 2026-02-08*
|
||||
*This TODO list is reviewed and updated regularly. Last review: 2026-02-06*
|
||||
|
||||
-215
@@ -1,215 +0,0 @@
|
||||
# Test Coverage TODO
|
||||
|
||||
This document tracks test coverage improvements for DocuElevate. The goal is to improve overall coverage from 45% to 60%+, then iterate in 10% steps.
|
||||
|
||||
## Current Status
|
||||
|
||||
**Initial Coverage**: 45.09%
|
||||
**Current Coverage**: 48.17%
|
||||
**Progress**: +3.08%
|
||||
**Target Coverage**: 60%+ (Phase 1), then 70%, 80%
|
||||
**Remaining to target**: ~12%
|
||||
|
||||
## Completed Tests
|
||||
|
||||
### Phase 1: Low-Hanging Fruits (Target: 60%+)
|
||||
|
||||
#### Utility Modules (0% → High Coverage) ✅
|
||||
- [x] `app/utils/encryption.py` (0% → 89.29%) ✅
|
||||
- Test encrypt_value with various inputs
|
||||
- Test decrypt_value with encrypted/plaintext values
|
||||
- Test is_encrypted function
|
||||
- Test is_encryption_available
|
||||
- Mock cryptography library for error cases
|
||||
|
||||
- [x] `app/celery_worker.py` (0% → 90.62%) ✅
|
||||
- Basic module structure tests (removed tests requiring Redis)
|
||||
|
||||
- [x] `app/tasks/uptime_kuma_tasks.py` (0% → 100%) ✅
|
||||
- Test ping_uptime_kuma with valid URL
|
||||
- Test skipping when URL not configured
|
||||
- Test error handling for failed requests
|
||||
|
||||
- [x] `app/utils/` package (exports via __init__.py) ✅
|
||||
- Package exports tested in test_reexports.py
|
||||
- Individual module coverage from actual usage
|
||||
|
||||
- [x] `app/frontend.py` (0% → 100%) ✅
|
||||
- Simple re-export module, test imports work
|
||||
|
||||
- [x] `app/utils/config_validator.py` (0% → Still 0%) ⚠️
|
||||
- Re-export module, coverage is from actual usage
|
||||
|
||||
#### Low Coverage Modules (<30% → Improved)
|
||||
- [x] `app/utils/filename_utils.py` (24.62% → 81.54%) ✅
|
||||
- Test sanitize_filename with special characters
|
||||
- Test get_unique_filename
|
||||
- Test extract_remote_path
|
||||
- Test filename validation functions
|
||||
|
||||
- [x] `app/utils/logging.py` (42.86% → 100%) ✅
|
||||
- Test log_task_progress function
|
||||
- Test various log message formats
|
||||
|
||||
- [x] `app/utils/oauth_helper.py` (17.50% → 100%) ✅
|
||||
- Test OAuth token exchange
|
||||
- Test error handling
|
||||
- Mock OAuth provider responses
|
||||
|
||||
- [x] `app/utils/notification.py` (44.33% → improved) ✅
|
||||
- Test URL masking for security
|
||||
- Test Apprise initialization
|
||||
- Basic notification sending tests
|
||||
|
||||
### Files Improved
|
||||
1. **app/utils/encryption.py**: 0% → 89.29% (+89.29%)
|
||||
2. **app/celery_worker.py**: 0% → 90.62% (+90.62%)
|
||||
3. **app/tasks/uptime_kuma_tasks.py**: 0% → 100% (+100%)
|
||||
4. **app/frontend.py**: 0% → 100% (+100%)
|
||||
5. **app/utils/filename_utils.py**: 24.62% → 81.54% (+56.92%)
|
||||
6. **app/utils/logging.py**: 42.86% → 100% (+57.14%)
|
||||
7. **app/utils/oauth_helper.py**: 17.50% → 100% (+82.50%)
|
||||
8. **app/utils/notification.py**: 44.33% → improved
|
||||
9. **app/tasks/check_credentials.py**: 0% → 23.13% (+23.13% from imports)
|
||||
10. **app/tasks/imap_tasks.py**: 0% → 15.35% (+15.35% from imports)
|
||||
|
||||
## Phase 2: Medium Priority (Target: 70%+)
|
||||
|
||||
### API Routes with Low Coverage
|
||||
- [ ] `app/api/azure.py` (23.08% → 60%+)
|
||||
- Test Azure connection
|
||||
- Test credential validation
|
||||
- Mock Azure API responses
|
||||
|
||||
- [ ] `app/api/dropbox.py` (16.94% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test connection testing
|
||||
|
||||
- [ ] `app/api/google_drive.py` (12.94% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test drive connection
|
||||
|
||||
- [ ] `app/api/onedrive.py` (13.83% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test connection testing
|
||||
|
||||
### Task Modules with Low Coverage
|
||||
- [ ] `app/tasks/convert_to_pdf.py` (13.41% → 50%+)
|
||||
- Test PDF conversion with various formats
|
||||
- Test Gotenberg integration (mocked)
|
||||
- Test error handling
|
||||
|
||||
- [ ] `app/tasks/embed_metadata_into_pdf.py` (19.05% → 50%+)
|
||||
- Test metadata embedding
|
||||
- Test PDF manipulation
|
||||
- Test error cases
|
||||
|
||||
## Phase 3: Complex Integration Tests (Target: 80%+)
|
||||
|
||||
### Upload Task Modules (Currently 13-36%)
|
||||
These require complex external service mocking:
|
||||
- [ ] `app/tasks/upload_to_dropbox.py` (13.45%)
|
||||
- [ ] `app/tasks/upload_to_google_drive.py` (36.00%)
|
||||
- [ ] `app/tasks/upload_to_onedrive.py` (26.32%)
|
||||
- [ ] `app/tasks/upload_to_nextcloud.py` (15.19%)
|
||||
- [ ] `app/tasks/upload_to_paperless.py` (18.60%)
|
||||
- [ ] `app/tasks/upload_to_email.py` (36.08%)
|
||||
|
||||
### Complex Background Tasks (0-36%)
|
||||
- [ ] `app/tasks/check_credentials.py` (0%)
|
||||
- Requires mocking multiple external services
|
||||
- Test credential validation for each provider
|
||||
- Test failure state management
|
||||
- Test notification system
|
||||
|
||||
- [ ] `app/tasks/imap_tasks.py` (0%)
|
||||
- Requires IMAP server mocking
|
||||
- Test email fetching
|
||||
- Test email parsing
|
||||
- Test lock management with Redis
|
||||
|
||||
- [ ] `app/tasks/upload_with_rclone.py` (0%)
|
||||
- Test rclone command execution
|
||||
- Test configuration management
|
||||
- Test error handling
|
||||
|
||||
- [ ] `app/tasks/extract_metadata_with_gpt.py` (28.79%)
|
||||
- Test GPT metadata extraction
|
||||
- Mock OpenAI API responses
|
||||
- Test various document types
|
||||
|
||||
### View Routes (25-61%)
|
||||
- [ ] `app/views/status.py` (25.00%)
|
||||
- [ ] `app/views/wizard.py` (38.98%)
|
||||
- [ ] `app/views/settings.py` (42.86%)
|
||||
- [ ] `app/views/google_drive.py` (42.42%)
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### For Low-Hanging Fruits (Phase 1)
|
||||
1. Focus on pure functions with minimal dependencies
|
||||
2. Mock external services (OpenAI, Azure, cloud storage)
|
||||
3. Test error paths and edge cases
|
||||
4. Use pytest fixtures for common setup
|
||||
|
||||
### For Integration Tests (Phases 2-3)
|
||||
1. Create comprehensive mocks for external services
|
||||
2. Use pytest-mock for patching
|
||||
3. Test async functions with pytest-asyncio
|
||||
4. Use TestClient for API endpoint tests
|
||||
5. Mock Redis, database, and Celery for task tests
|
||||
|
||||
## Coverage Goals by Phase
|
||||
|
||||
| Phase | Target Coverage | Status |
|
||||
|-------|----------------|--------|
|
||||
| Phase 1: Low-Hanging Fruits | 60% | In Progress |
|
||||
| Phase 2: Medium Priority | 70% | Not Started |
|
||||
| Phase 3: Complex Integration | 80% | Not Started |
|
||||
|
||||
## Notes
|
||||
|
||||
- Files with 100% coverage: Keep them at 100%
|
||||
- Files with 90%+ coverage: Low priority for improvement
|
||||
- Focus on business logic, not simple re-exports
|
||||
- Mock external dependencies to avoid flaky tests
|
||||
- All tests must pass CI/CD pipeline
|
||||
- Maintain test execution time under 2 minutes for fast feedback
|
||||
|
||||
## Files Excluded from Coverage
|
||||
|
||||
These files are infrastructure/configuration and don't require high coverage:
|
||||
- `migrations/*` - Database migrations (excluded in pytest.ini)
|
||||
- `app/__init__.py` - Empty init files
|
||||
- `app/*/__init__.py` - Package init files
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Run all tests with coverage
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run tests for specific module
|
||||
pytest tests/test_encryption.py -v
|
||||
|
||||
# Run tests with coverage report
|
||||
pytest --cov=app --cov-report=html
|
||||
open htmlcov/index.html
|
||||
|
||||
# Run only unit tests (fast)
|
||||
pytest -m unit
|
||||
|
||||
# Run integration tests
|
||||
pytest -m integration
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
When adding new code:
|
||||
1. Write tests for new functionality
|
||||
2. Aim for 80%+ coverage on new files
|
||||
3. Update this TODO when completing test coverage work
|
||||
4. Run coverage report before submitting PR
|
||||
@@ -1,352 +0,0 @@
|
||||
# WebDAV Testing - Implementation Summary
|
||||
|
||||
## Overview
|
||||
|
||||
This document summarizes the comprehensive testing implementation for WebDAV upload functionality in DocuElevate.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### 1. WebDAV Upload Module (Already Existed)
|
||||
|
||||
**File:** `app/tasks/upload_to_webdav.py`
|
||||
|
||||
- Celery task for uploading files to WebDAV servers
|
||||
- Supports HTTP Basic authentication
|
||||
- Configurable SSL verification
|
||||
- URL/folder path normalization
|
||||
- Retry logic via `BaseTaskWithRetry` (3 retries, exponential backoff)
|
||||
- Progress logging integration
|
||||
|
||||
**Configuration:**
|
||||
- `WEBDAV_URL` - Server URL
|
||||
- `WEBDAV_USERNAME` - Authentication username
|
||||
- `WEBDAV_PASSWORD` - Authentication password
|
||||
- `WEBDAV_FOLDER` - Target folder path
|
||||
- `WEBDAV_VERIFY_SSL` - SSL certificate verification
|
||||
|
||||
### 2. Comprehensive Unit Tests ✅
|
||||
|
||||
**File:** `tests/test_upload_webdav_comprehensive.py`
|
||||
|
||||
**Tests:** 23 (all passing)
|
||||
|
||||
**Coverage:**
|
||||
- Success scenarios (with/without file_id, different HTTP status codes: 200, 201, 204)
|
||||
- Configuration validation (missing URL)
|
||||
- Error handling (file not found, HTTP errors: 401, 404, 500)
|
||||
- Connection errors (timeout, connection refused)
|
||||
- URL construction (trailing slash, no trailing slash, leading slash in folder)
|
||||
- Folder path normalization (empty, leading slash)
|
||||
- SSL verification (enabled/disabled)
|
||||
- Authentication credentials
|
||||
- Logging verification (success/failure)
|
||||
- File content upload
|
||||
- Return value structure
|
||||
- Task importability
|
||||
- Retry configuration
|
||||
|
||||
**Result:** 100% code coverage on `upload_to_webdav.py`
|
||||
|
||||
### 3. Integration Tests with Real WebDAV Server ✅
|
||||
|
||||
**File:** `tests/test_upload_webdav_integration.py`
|
||||
|
||||
**Tests:** 10 (all passing)
|
||||
|
||||
**Infrastructure:**
|
||||
- Uses `testcontainers` library
|
||||
- Spins up real WebDAV server (bytemark/webdav:latest)
|
||||
- Docker container runs during tests
|
||||
- Automatic cleanup after tests
|
||||
|
||||
**Test Scenarios:**
|
||||
1. Upload file to real server and verify content
|
||||
2. Upload to subfolder with MKCOL command
|
||||
3. Upload PDF file and verify magic bytes
|
||||
4. Upload with wrong credentials (401 error)
|
||||
5. Upload multiple files sequentially
|
||||
6. Overwrite existing file
|
||||
7. Upload large file (1MB)
|
||||
8. WebDAV server basic authentication
|
||||
9. WebDAV PUT method support
|
||||
10. WebDAV PROPFIND method support
|
||||
|
||||
**Result:** Verifies actual file uploads to real WebDAV server
|
||||
|
||||
### 4. Full-Stack Integration Infrastructure ✅
|
||||
|
||||
**File:** `tests/fixtures_integration.py`
|
||||
|
||||
**Provides Fixtures For:**
|
||||
- **PostgreSQL** - Real database (replaces SQLite in-memory)
|
||||
- **Redis** - Real message broker for Celery
|
||||
- **Gotenberg** - Real PDF conversion service
|
||||
- **WebDAV** - Real upload target
|
||||
- **SFTP** - Real SSH/SFTP server
|
||||
- **MinIO** - Real S3-compatible storage
|
||||
- **FTP** - Real FTP server
|
||||
- **Celery App** - Configured for test Redis
|
||||
- **Celery Worker** - Actually processes queued tasks
|
||||
|
||||
### 5. End-to-End Tests ✅
|
||||
|
||||
**File:** `tests/test_e2e_full_stack.py`
|
||||
|
||||
**Test Classes:**
|
||||
1. **TestEndToEndWithRedis** - Redis + Celery integration
|
||||
- Queue task in Redis → Worker executes → Upload to WebDAV
|
||||
- Task queuing verification
|
||||
- Parallel task execution
|
||||
- Task retry on failure
|
||||
|
||||
2. **TestFullInfrastructure** - Complete stack
|
||||
- All infrastructure components running
|
||||
- Database operations with PostgreSQL
|
||||
- Upload to multiple targets (WebDAV + SFTP)
|
||||
- Gotenberg PDF conversion
|
||||
- MinIO S3 uploads
|
||||
- SFTP uploads
|
||||
|
||||
3. **TestProductionLikeScenarios** - Complete workflows
|
||||
- Full document processing pipeline
|
||||
- Database → Redis → Celery → WebDAV
|
||||
- End-to-end verification
|
||||
|
||||
### 6. Documentation ✅
|
||||
|
||||
**File:** `tests/README_INTEGRATION_TESTS.md`
|
||||
|
||||
**Contents:**
|
||||
- Overview of integration testing approach
|
||||
- Prerequisites and setup
|
||||
- Test organization and markers
|
||||
- Running tests (unit, integration, e2e)
|
||||
- Infrastructure fixtures documentation
|
||||
- Example test scenarios
|
||||
- Performance notes and resource usage
|
||||
- Debugging and troubleshooting
|
||||
- CI/CD integration examples
|
||||
- Best practices
|
||||
- Coverage information
|
||||
|
||||
## Test Execution Summary
|
||||
|
||||
### Unit Tests (Mocked)
|
||||
```bash
|
||||
pytest tests/test_upload_webdav_comprehensive.py -v
|
||||
```
|
||||
- **Tests:** 23/23 ✅
|
||||
- **Speed:** ~2 seconds
|
||||
- **Coverage:** 100%
|
||||
- **Docker Required:** No
|
||||
|
||||
### Integration Tests (Real WebDAV)
|
||||
```bash
|
||||
pytest tests/test_upload_webdav_integration.py -v
|
||||
```
|
||||
- **Tests:** 10/10 ✅
|
||||
- **Speed:** ~7 seconds
|
||||
- **Coverage:** 79.31% (focuses on happy paths with real server)
|
||||
- **Docker Required:** Yes
|
||||
|
||||
### End-to-End Tests (Full Stack)
|
||||
```bash
|
||||
pytest tests/test_e2e_full_stack.py -v
|
||||
```
|
||||
- **Tests:** 12+ scenarios
|
||||
- **Speed:** ~30-60 seconds per test
|
||||
- **Coverage:** Complete application workflow
|
||||
- **Docker Required:** Yes
|
||||
|
||||
### All WebDAV Tests
|
||||
```bash
|
||||
pytest tests/test_upload_webdav*.py -v
|
||||
```
|
||||
- **Total Tests:** 33 ✅
|
||||
- **Speed:** ~7 seconds total
|
||||
- **Result:** All passing
|
||||
|
||||
## Infrastructure Components
|
||||
|
||||
### Container Images Used
|
||||
|
||||
| Service | Image | Port | Purpose |
|
||||
|---------|-------|------|---------|
|
||||
| WebDAV | bytemark/webdav:latest | 80 | Upload target |
|
||||
| PostgreSQL | postgres:15-alpine | 5432 | Real database |
|
||||
| Redis | redis:7-alpine | 6379 | Celery broker |
|
||||
| Gotenberg | gotenberg/gotenberg:8 | 3000 | PDF conversion |
|
||||
| SFTP | atmoz/sftp:latest | 22 | SFTP uploads |
|
||||
| MinIO | minio/minio:latest | 9000 | S3 storage |
|
||||
| FTP | stilliard/pure-ftpd:latest | 21 | FTP uploads |
|
||||
|
||||
### Resource Requirements
|
||||
|
||||
- **Docker:** Must be installed and running
|
||||
- **Memory:** ~100MB per container, ~1GB total for full stack
|
||||
- **Disk:** ~2GB for all Docker images
|
||||
- **Time:**
|
||||
- First run: ~5-10 minutes (image pulls)
|
||||
- Subsequent runs: ~10-60 seconds per test
|
||||
|
||||
## Dependencies Added
|
||||
|
||||
**`requirements-dev.txt`:**
|
||||
```
|
||||
testcontainers>=3.7.1 # Container management
|
||||
minio>=7.1.0 # MinIO client
|
||||
redis>=4.5.0 # Redis client
|
||||
boto3>=1.26.0 # AWS S3 client (for MinIO)
|
||||
```
|
||||
|
||||
All dependencies are development/testing only.
|
||||
|
||||
## Test Markers
|
||||
|
||||
Custom pytest markers for organizing tests:
|
||||
|
||||
```python
|
||||
@pytest.mark.unit # Fast unit tests, no Docker
|
||||
@pytest.mark.integration # Integration tests with containers
|
||||
@pytest.mark.e2e # Full end-to-end scenarios
|
||||
@pytest.mark.requires_docker # Requires Docker to run
|
||||
@pytest.mark.slow # Takes >30 seconds
|
||||
```
|
||||
|
||||
## Key Features
|
||||
|
||||
### 1. Real Infrastructure Testing
|
||||
- Tests run against actual services, not mocks
|
||||
- Verifies files are actually uploaded
|
||||
- Catches integration issues early
|
||||
|
||||
### 2. Production-Like Scenarios
|
||||
- PostgreSQL instead of SQLite
|
||||
- Redis message queueing
|
||||
- Celery worker execution
|
||||
- Async task processing
|
||||
|
||||
### 3. Comprehensive Coverage
|
||||
- **Unit tests:** Edge cases, error handling, validation
|
||||
- **Integration tests:** Real server behavior, file operations
|
||||
- **E2E tests:** Complete workflows, multi-service coordination
|
||||
|
||||
### 4. Automatic Cleanup
|
||||
- Testcontainers auto-remove after tests
|
||||
- No manual cleanup required
|
||||
- Isolated test environments
|
||||
|
||||
### 5. Developer-Friendly
|
||||
- Clear test organization
|
||||
- Detailed documentation
|
||||
- Easy to run locally
|
||||
- CI/CD ready
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Run Quick Unit Tests
|
||||
```bash
|
||||
# Fast, no Docker needed
|
||||
pytest tests/test_upload_webdav_comprehensive.py -v
|
||||
```
|
||||
|
||||
### Verify Upload Works Against Real Server
|
||||
```bash
|
||||
# Spins up WebDAV container
|
||||
pytest tests/test_upload_webdav_integration.py::TestWebDAVIntegration::test_upload_file_to_real_webdav_server -v
|
||||
```
|
||||
|
||||
### Test Complete Workflow with Redis
|
||||
```bash
|
||||
# Full stack: Redis + Celery + WebDAV
|
||||
pytest tests/test_e2e_full_stack.py::TestEndToEndWithRedis::test_webdav_upload_with_redis_and_celery -v
|
||||
```
|
||||
|
||||
### Run All Infrastructure Tests
|
||||
```bash
|
||||
# All services
|
||||
pytest -m e2e -v
|
||||
```
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions Example
|
||||
```yaml
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
pytest -m "integration or e2e" -v --tb=short
|
||||
```
|
||||
|
||||
Tests are designed to run in CI environments with Docker support.
|
||||
|
||||
## Benefits
|
||||
|
||||
### For Development
|
||||
1. **Fast Feedback:** Unit tests run in seconds
|
||||
2. **Confidence:** Integration tests verify real behavior
|
||||
3. **Debug Easily:** Containers provide inspection access
|
||||
|
||||
### For QA/Testing
|
||||
1. **Real Scenarios:** Tests match production behavior
|
||||
2. **Complete Coverage:** Unit + Integration + E2E
|
||||
3. **Reproducible:** Docker ensures consistency
|
||||
|
||||
### For Production
|
||||
1. **Early Detection:** Catch issues before deployment
|
||||
2. **Regression Prevention:** Comprehensive test suite
|
||||
3. **Documentation:** Tests serve as usage examples
|
||||
|
||||
## Comparison to Other Upload Modules
|
||||
|
||||
Most other upload modules (S3, SFTP, FTP, Dropbox, Google Drive) only have:
|
||||
- Basic unit tests with mocks (1-2 tests each)
|
||||
- No integration tests with real servers
|
||||
- No end-to-end tests
|
||||
|
||||
WebDAV now has:
|
||||
- ✅ 23 comprehensive unit tests
|
||||
- ✅ 10 integration tests with real server
|
||||
- ✅ Full e2e test infrastructure
|
||||
- ✅ 100% code coverage
|
||||
- ✅ Production-like testing
|
||||
|
||||
**WebDAV is now the reference implementation for testing upload modules.**
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Potential Additions
|
||||
1. Add similar integration tests for SFTP, FTP, S3
|
||||
2. Test WebDAV with different servers (ownCloud, Nextcloud, Synology)
|
||||
3. Test large file uploads (>100MB)
|
||||
4. Test concurrent uploads (stress testing)
|
||||
5. Test network failure scenarios
|
||||
6. Test SSL/TLS certificate validation
|
||||
|
||||
### Template for Other Modules
|
||||
The WebDAV testing approach can be replicated for other upload destinations:
|
||||
1. Create `test_upload_<destination>_comprehensive.py` (unit tests)
|
||||
2. Create `test_upload_<destination>_integration.py` (with real server)
|
||||
3. Add container fixture to `fixtures_integration.py`
|
||||
4. Add e2e scenarios to `test_e2e_full_stack.py`
|
||||
|
||||
## Conclusion
|
||||
|
||||
The WebDAV upload functionality is now **comprehensively tested** with:
|
||||
- ✅ 33 passing tests
|
||||
- ✅ 100% code coverage (unit tests)
|
||||
- ✅ Real server verification (integration tests)
|
||||
- ✅ Production-like scenarios (e2e tests)
|
||||
- ✅ Full infrastructure testing capability
|
||||
|
||||
This provides **high confidence** that WebDAV uploads work correctly in production and serves as a **reference implementation** for testing other upload modules.
|
||||
|
||||
## Related Files
|
||||
|
||||
- `app/tasks/upload_to_webdav.py` - Implementation
|
||||
- `tests/test_upload_webdav_comprehensive.py` - Unit tests (23)
|
||||
- `tests/test_upload_webdav_integration.py` - Integration tests (10)
|
||||
- `tests/fixtures_integration.py` - Infrastructure fixtures
|
||||
- `tests/test_e2e_full_stack.py` - End-to-end tests (12+)
|
||||
- `tests/README_INTEGRATION_TESTS.md` - Documentation
|
||||
- `requirements-dev.txt` - Test dependencies
|
||||
- `tests/conftest.py` - Pytest configuration
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
# Alembic Configuration File
|
||||
# Used for managing database schema migrations in DocuElevate.
|
||||
#
|
||||
# Usage:
|
||||
# alembic upgrade head # Apply all pending migrations
|
||||
# alembic current # Show current revision
|
||||
# alembic history --verbose # Show migration history
|
||||
# alembic downgrade -1 # Roll back one migration
|
||||
# alembic revision --autogenerate -m "description" # Create new migration
|
||||
|
||||
[alembic]
|
||||
# Path to migration scripts
|
||||
script_location = migrations
|
||||
|
||||
# Template used to generate migration file names
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# Timezone for migration file timestamps (uses UTC by default)
|
||||
# timezone =
|
||||
|
||||
# Maximum length of characters for autogenerate revision names
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# Set to 'true' to run environment during 'revision' command
|
||||
# revision_environment = false
|
||||
|
||||
# Set to 'true' to allow .pyc or .pyo files for migration scripts
|
||||
# sourceless = false
|
||||
|
||||
# Version path separator; default is "os" which uses os.pathsep
|
||||
# version_path_separator = os
|
||||
|
||||
# Output encoding for revision files
|
||||
# output_encoding = utf-8
|
||||
|
||||
# The database URL is loaded from app.config.settings.database_url
|
||||
# in migrations/env.py, not from this file.
|
||||
sqlalchemy.url =
|
||||
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
+9
-74
@@ -1,53 +1,19 @@
|
||||
"""
|
||||
API Router module that combines all API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.admin_users import router as admin_users_router
|
||||
from app.api.api_tokens import router as api_tokens_router
|
||||
from app.api.audit_logs import router as audit_logs_router
|
||||
from app.api.azure import router as azure_router
|
||||
from app.api.backup import router as backup_router
|
||||
from app.api.billing import router as billing_router
|
||||
from app.api.compliance import router as compliance_router
|
||||
from app.api.database import router as database_router
|
||||
from app.api.diagnostic import router as diagnostic_router
|
||||
from app.api.dropbox import router as dropbox_router
|
||||
from app.api.duplicates import router as duplicates_router
|
||||
from app.api.files import router as files_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
from app.api.i18n import router as i18n_router
|
||||
from app.api.imap_accounts import router as imap_accounts_router
|
||||
from app.api.imap_profiles import router as imap_profiles_router
|
||||
from app.api.integrations import router as integrations_router
|
||||
from app.api.logs import router as logs_router
|
||||
from app.api.mobile import router as mobile_router
|
||||
from app.api.notifications import router as notifications_router
|
||||
from app.api.onboarding import router as onboarding_router
|
||||
from app.api.onedrive import router as onedrive_router
|
||||
from app.api.openai import router as openai_router
|
||||
from app.api.pipelines import router as pipelines_router
|
||||
from app.api.plans import router as plans_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.profile import router as profile_router
|
||||
from app.api.queue import router as queue_router
|
||||
from app.api.routing_rules import router as routing_rules_router
|
||||
from app.api.saved_searches import router as saved_searches_router
|
||||
from app.api.scheduled_jobs import router as scheduled_jobs_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.shared_links import public_router as shared_links_public_router
|
||||
from app.api.shared_links import router as shared_links_router
|
||||
from app.api.similarity import router as similarity_router
|
||||
from app.api.subscriptions import router as subscriptions_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
import logging
|
||||
|
||||
# Import all the individual routers
|
||||
from app.api.user import router as user_router
|
||||
from app.api.webhooks import router as webhooks_router
|
||||
from app.api.files import router as files_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.diagnostic import router as diagnostic_router
|
||||
from app.api.onedrive import router as onedrive_router
|
||||
from app.api.dropbox import router as dropbox_router
|
||||
from app.api.openai import router as openai_router
|
||||
from app.api.azure import router as azure_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -56,10 +22,7 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
# Include all the routers
|
||||
router.include_router(admin_users_router)
|
||||
router.include_router(api_tokens_router)
|
||||
router.include_router(user_router)
|
||||
router.include_router(backup_router)
|
||||
router.include_router(files_router)
|
||||
router.include_router(process_router)
|
||||
router.include_router(diagnostic_router)
|
||||
@@ -68,31 +31,3 @@ router.include_router(dropbox_router)
|
||||
router.include_router(openai_router)
|
||||
router.include_router(azure_router)
|
||||
router.include_router(google_drive_router)
|
||||
router.include_router(logs_router)
|
||||
router.include_router(settings_router)
|
||||
router.include_router(url_upload_router)
|
||||
router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
router.include_router(saved_searches_router)
|
||||
router.include_router(similarity_router)
|
||||
router.include_router(shared_links_router)
|
||||
router.include_router(shared_links_public_router)
|
||||
router.include_router(duplicates_router)
|
||||
router.include_router(webhooks_router)
|
||||
router.include_router(database_router)
|
||||
router.include_router(subscriptions_router)
|
||||
router.include_router(plans_router)
|
||||
router.include_router(onboarding_router)
|
||||
router.include_router(billing_router)
|
||||
router.include_router(pipelines_router)
|
||||
router.include_router(profile_router)
|
||||
router.include_router(routing_rules_router)
|
||||
router.include_router(imap_accounts_router)
|
||||
router.include_router(imap_profiles_router)
|
||||
router.include_router(integrations_router)
|
||||
router.include_router(notifications_router)
|
||||
router.include_router(scheduled_jobs_router)
|
||||
router.include_router(audit_logs_router)
|
||||
router.include_router(i18n_router)
|
||||
router.include_router(mobile_router)
|
||||
router.include_router(compliance_router)
|
||||
|
||||
@@ -1,668 +0,0 @@
|
||||
"""API endpoints for admin user management.
|
||||
|
||||
Provides CRUD operations for user profiles and aggregate statistics so that
|
||||
administrators can inspect, configure, and manage users in multi-user mode.
|
||||
Also provides endpoints for admins to create and manage local (email/password)
|
||||
user accounts directly, without requiring email verification.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, LocalUser, UserProfile
|
||||
from app.utils.local_auth import generate_token, hash_password, send_password_reset_email
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/users", tags=["admin-users"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class UserProfileUpsert(BaseModel):
|
||||
"""Body for creating or updating a user profile."""
|
||||
|
||||
display_name: str | None = Field(default=None, max_length=255, description="Human-readable display name")
|
||||
daily_upload_limit: int | None = Field(
|
||||
default=None, ge=0, description="Per-user daily upload cap; null = use global default"
|
||||
)
|
||||
notes: str | None = Field(default=None, max_length=4096, description="Admin notes about this user")
|
||||
is_blocked: bool = Field(default=False, description="Block this user from uploading")
|
||||
subscription_tier: str | None = Field(
|
||||
default="free",
|
||||
description="Subscription tier: free | starter | professional | business",
|
||||
)
|
||||
subscription_billing_cycle: str = Field(default="monthly", pattern="^(monthly|yearly)$")
|
||||
subscription_period_start: datetime | None = None
|
||||
allow_overage: bool = False
|
||||
is_complimentary: bool = Field(
|
||||
default=False,
|
||||
description="When True the user is on a complimentary (uncharged) plan — they keep all tier "
|
||||
"quota benefits but are never billed via Stripe.",
|
||||
)
|
||||
|
||||
|
||||
class PaymentIssueBody(BaseModel):
|
||||
"""Body for reporting a payment issue for a user."""
|
||||
|
||||
issue: str = Field(..., min_length=1, max_length=2048, description="Description of the payment issue")
|
||||
|
||||
|
||||
class UserProfileResponse(BaseModel):
|
||||
"""Response schema for a user profile record."""
|
||||
|
||||
id: int
|
||||
user_id: str
|
||||
display_name: str | None
|
||||
daily_upload_limit: int | None
|
||||
notes: str | None
|
||||
is_blocked: bool
|
||||
subscription_tier: str | None
|
||||
subscription_billing_cycle: str
|
||||
subscription_period_start: str | None
|
||||
allow_overage: bool
|
||||
is_complimentary: bool
|
||||
created_at: str | None
|
||||
updated_at: str | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class UserSummary(BaseModel):
|
||||
"""Per-user summary combining profile data with document statistics."""
|
||||
|
||||
user_id: str
|
||||
display_name: str | None
|
||||
daily_upload_limit: int | None
|
||||
notes: str | None
|
||||
is_blocked: bool
|
||||
subscription_tier: str | None
|
||||
subscription_billing_cycle: str | None
|
||||
subscription_period_start: str | None
|
||||
allow_overage: bool
|
||||
is_complimentary: bool
|
||||
profile_id: int | None
|
||||
document_count: int
|
||||
last_upload: str | None
|
||||
|
||||
|
||||
class LocalUserCreate(BaseModel):
|
||||
"""Body for admin-creating a local (email/password) user account."""
|
||||
|
||||
email: str = Field(..., max_length=255, description="Email address for the new user")
|
||||
username: str = Field(..., min_length=3, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
|
||||
display_name: str | None = Field(default=None, max_length=255)
|
||||
password: str = Field(..., min_length=8, max_length=128)
|
||||
is_admin: bool = Field(default=False, description="Grant admin privileges")
|
||||
|
||||
|
||||
class LocalUserUpdate(BaseModel):
|
||||
"""Body for admin-updating a local (email/password) user account."""
|
||||
|
||||
email: str | None = Field(default=None, max_length=255, description="New email address")
|
||||
display_name: str | None = Field(default=None, max_length=255, description="New display name")
|
||||
is_admin: bool | None = Field(default=None, description="Grant or revoke admin privileges")
|
||||
is_active: bool | None = Field(default=None, description="Activate or deactivate the account")
|
||||
|
||||
|
||||
class LocalUserSetPassword(BaseModel):
|
||||
"""Body for admin setting a temporary password for a local user."""
|
||||
|
||||
password: str = Field(..., min_length=8, max_length=128, description="New temporary password")
|
||||
|
||||
|
||||
class LocalUserResponse(BaseModel):
|
||||
"""Summary of a local user account."""
|
||||
|
||||
id: int
|
||||
email: str
|
||||
username: str
|
||||
display_name: str | None
|
||||
is_active: bool
|
||||
is_admin: bool
|
||||
created_at: str | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_or_none(db: Session, user_id: str) -> UserProfile | None:
|
||||
"""Return the UserProfile row for *user_id*, or None if it doesn't exist."""
|
||||
return db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
|
||||
|
||||
def _profile_to_dict(profile: UserProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"id": profile.id,
|
||||
"user_id": profile.user_id,
|
||||
"display_name": profile.display_name,
|
||||
"daily_upload_limit": profile.daily_upload_limit,
|
||||
"notes": profile.notes,
|
||||
"is_blocked": profile.is_blocked,
|
||||
"subscription_tier": profile.subscription_tier or "free",
|
||||
"subscription_billing_cycle": profile.subscription_billing_cycle or "monthly",
|
||||
"subscription_period_start": profile.subscription_period_start.isoformat()
|
||||
if profile.subscription_period_start
|
||||
else None,
|
||||
"allow_overage": bool(profile.allow_overage),
|
||||
"is_complimentary": bool(profile.is_complimentary),
|
||||
"created_at": profile.created_at.isoformat() if profile.created_at else None,
|
||||
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", summary="List all known users with statistics")
|
||||
def list_users(
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
q: str = Query("", description="Filter by user_id substring (case-insensitive)"),
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
per_page: int = Query(25, ge=1, le=100, description="Items per page"),
|
||||
) -> dict[str, Any]:
|
||||
"""Return every distinct user_id that has at least one document or an explicit profile,
|
||||
enriched with aggregate document statistics and the admin-managed profile.
|
||||
|
||||
Supports substring filtering (``q``) and pagination.
|
||||
"""
|
||||
# 1. Collect every distinct owner_id from documents
|
||||
doc_stats_query = (
|
||||
db.query(
|
||||
FileRecord.owner_id.label("user_id"),
|
||||
func.count(FileRecord.id).label("doc_count"),
|
||||
func.max(FileRecord.created_at).label("last_upload"),
|
||||
)
|
||||
.filter(FileRecord.owner_id.isnot(None))
|
||||
.group_by(FileRecord.owner_id)
|
||||
)
|
||||
|
||||
# 2. Collect all user_ids that have explicit profiles (may not have docs yet)
|
||||
profile_query = db.query(UserProfile)
|
||||
|
||||
# Build a unified set of user_ids
|
||||
doc_rows = {row.user_id: row for row in doc_stats_query.all()}
|
||||
profile_rows = {p.user_id: p for p in profile_query.all()}
|
||||
|
||||
all_user_ids = set(doc_rows.keys()) | set(profile_rows.keys())
|
||||
|
||||
# Apply optional substring filter
|
||||
if q.strip():
|
||||
q_lower = q.strip().lower()
|
||||
all_user_ids = {uid for uid in all_user_ids if q_lower in uid.lower()}
|
||||
|
||||
# Sort and paginate
|
||||
sorted_ids = sorted(all_user_ids)
|
||||
total = len(sorted_ids)
|
||||
start = (page - 1) * per_page
|
||||
page_ids = sorted_ids[start : start + per_page]
|
||||
|
||||
users: list[dict[str, Any]] = []
|
||||
for uid in page_ids:
|
||||
doc_row = doc_rows.get(uid)
|
||||
profile = profile_rows.get(uid)
|
||||
users.append(
|
||||
{
|
||||
"user_id": uid,
|
||||
"display_name": profile.display_name if profile else None,
|
||||
"daily_upload_limit": profile.daily_upload_limit if profile else None,
|
||||
"notes": profile.notes if profile else None,
|
||||
"is_blocked": profile.is_blocked if profile else False,
|
||||
"subscription_tier": (profile.subscription_tier or "free") if profile else "free",
|
||||
"subscription_billing_cycle": (profile.subscription_billing_cycle or "monthly")
|
||||
if profile
|
||||
else "monthly",
|
||||
"subscription_period_start": profile.subscription_period_start.isoformat()
|
||||
if (profile and profile.subscription_period_start)
|
||||
else None,
|
||||
"allow_overage": bool(profile.allow_overage) if profile else False,
|
||||
"is_complimentary": bool(profile.is_complimentary) if profile else False,
|
||||
"profile_id": profile.id if profile else None,
|
||||
"document_count": doc_row.doc_count if doc_row else 0,
|
||||
"last_upload": doc_row.last_upload.isoformat() if (doc_row and doc_row.last_upload) else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"users": users,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": max(1, (total + per_page - 1) // per_page),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local user management (admin-only)
|
||||
# ---------------------------------------------------------------------------
|
||||
# NOTE: These routes MUST be defined before /{user_id:path} to avoid being
|
||||
# swallowed by the catch-all path parameter.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/local", summary="List all local (email/password) user accounts")
|
||||
def list_local_users(db: DbSession, _admin: AdminUser) -> list[dict[str, Any]]:
|
||||
"""Return every local user account with basic metadata."""
|
||||
users = db.query(LocalUser).order_by(LocalUser.created_at.desc()).all()
|
||||
return [
|
||||
{
|
||||
"id": u.id,
|
||||
"email": u.email,
|
||||
"username": u.username,
|
||||
"display_name": u.display_name,
|
||||
"is_active": u.is_active,
|
||||
"is_admin": u.is_admin,
|
||||
"created_at": u.created_at.isoformat() if u.created_at else None,
|
||||
}
|
||||
for u in users
|
||||
]
|
||||
|
||||
|
||||
@router.post("/local", status_code=status.HTTP_201_CREATED, summary="Create a local user account")
|
||||
def create_local_user(body: LocalUserCreate, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Create a new local (email/password) user account.
|
||||
|
||||
The account is immediately active — no email verification is required when
|
||||
created by an administrator. A matching UserProfile row is also created.
|
||||
|
||||
Raises:
|
||||
409: Email or username already registered.
|
||||
"""
|
||||
if db.query(LocalUser).filter(LocalUser.email == body.email).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered.")
|
||||
if db.query(LocalUser).filter(LocalUser.username == body.username).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Username already taken.")
|
||||
|
||||
user = LocalUser(
|
||||
email=body.email,
|
||||
username=body.username,
|
||||
display_name=body.display_name,
|
||||
hashed_password=hash_password(body.password),
|
||||
is_active=True,
|
||||
is_admin=body.is_admin,
|
||||
)
|
||||
db.add(user)
|
||||
|
||||
# Ensure a UserProfile exists for the new user
|
||||
if not db.query(UserProfile).filter(UserProfile.user_id == body.email).first():
|
||||
db.add(UserProfile(user_id=body.email, display_name=body.display_name or body.username))
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Admin created local user account: %s", body.email)
|
||||
return {
|
||||
"id": user.id,
|
||||
"email": user.email,
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"is_active": user.is_active,
|
||||
"is_admin": user.is_admin,
|
||||
"created_at": user.created_at.isoformat() if user.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/local/{local_user_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
summary="Delete a local user account",
|
||||
)
|
||||
def delete_local_user(local_user_id: int, db: DbSession, _admin: AdminUser) -> None:
|
||||
"""Delete a local user account by its numeric ID.
|
||||
|
||||
The associated UserProfile is also removed. Documents owned by this user
|
||||
are **not** deleted.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.id == local_user_id).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Local user not found.")
|
||||
|
||||
# Remove associated profile if present
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user.email).first()
|
||||
if profile:
|
||||
db.delete(profile)
|
||||
|
||||
try:
|
||||
db.delete(user)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Admin deleted local user account: %s", user.email)
|
||||
|
||||
|
||||
@router.patch("/local/{local_user_id}", summary="Update a local user account")
|
||||
def update_local_user(local_user_id: int, body: LocalUserUpdate, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Update the email address, display name, admin flag, or active status of a local user account.
|
||||
|
||||
Only fields explicitly provided (non-None) are modified. If the email is changed
|
||||
the associated UserProfile row is also updated to keep ``user_id`` in sync.
|
||||
|
||||
Raises:
|
||||
404: Local user not found.
|
||||
409: The new email is already taken by another account.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.id == local_user_id).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Local user not found.")
|
||||
|
||||
old_email = user.email
|
||||
|
||||
if body.email is not None and body.email != user.email:
|
||||
if db.query(LocalUser).filter(LocalUser.email == body.email, LocalUser.id != local_user_id).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered.")
|
||||
user.email = body.email
|
||||
|
||||
if body.display_name is not None:
|
||||
# Normalise empty string to None so that clearing the field removes the display name
|
||||
user.display_name = body.display_name or None
|
||||
|
||||
if body.is_admin is not None:
|
||||
user.is_admin = body.is_admin
|
||||
|
||||
if body.is_active is not None:
|
||||
user.is_active = body.is_active
|
||||
|
||||
try:
|
||||
db.flush()
|
||||
# Keep UserProfile.user_id in sync when email changes
|
||||
if body.email is not None and body.email != old_email:
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == old_email).first()
|
||||
if profile:
|
||||
profile.user_id = body.email
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Admin updated local user %s (id=%d)", user.email, user.id)
|
||||
return {
|
||||
"id": user.id,
|
||||
"email": user.email,
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"is_active": user.is_active,
|
||||
"is_admin": user.is_admin,
|
||||
"created_at": user.created_at.isoformat() if user.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/local/{local_user_id}/send-password-reset",
|
||||
status_code=status.HTTP_200_OK,
|
||||
summary="Send a password reset email to a local user",
|
||||
)
|
||||
def admin_send_password_reset(local_user_id: int, request: Request, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Generate a password reset token and email the reset link to the local user.
|
||||
|
||||
This is a last-resort tool for admins to help users who are locked out.
|
||||
Returns ``{"sent": true}`` on success and ``{"sent": false, "reason": "..."}`` when
|
||||
SMTP is not configured or sending fails.
|
||||
|
||||
Raises:
|
||||
404: Local user not found.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.id == local_user_id).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Local user not found.")
|
||||
|
||||
if not settings.email_host:
|
||||
logger.warning("Admin requested password reset for %s but SMTP is not configured", user.email)
|
||||
return {"sent": False, "reason": "SMTP is not configured on this server."}
|
||||
|
||||
token = generate_token()
|
||||
user.password_reset_token = token
|
||||
user.password_reset_sent_at = datetime.now(tz=timezone.utc)
|
||||
db.commit()
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
try:
|
||||
send_password_reset_email(user.email, user.username, token, base_url)
|
||||
except Exception as exc:
|
||||
logger.warning("Admin-triggered password reset email failed for %s: %s", user.email, exc)
|
||||
return {"sent": False, "reason": str(exc)}
|
||||
|
||||
logger.info("[SECURITY] ADMIN_PASSWORD_RESET_EMAIL user=%s admin=%s", user.email, _admin.get("email", "unknown"))
|
||||
return {"sent": True, "email": user.email}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/local/{local_user_id}/set-password",
|
||||
status_code=status.HTTP_200_OK,
|
||||
summary="Set a temporary password for a local user account",
|
||||
)
|
||||
def admin_set_password(
|
||||
local_user_id: int, body: LocalUserSetPassword, db: DbSession, _admin: AdminUser
|
||||
) -> dict[str, Any]:
|
||||
"""Directly set a new password for a local user without requiring an email token.
|
||||
|
||||
Use this as a last resort when email delivery is unavailable. The user
|
||||
should be advised to change their password after logging in.
|
||||
|
||||
Raises:
|
||||
404: Local user not found.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.id == local_user_id).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Local user not found.")
|
||||
|
||||
user.hashed_password = hash_password(body.password)
|
||||
# Clear any outstanding reset tokens and activate the account so the user
|
||||
# can log in immediately after an admin sets their password.
|
||||
user.password_reset_token = None
|
||||
user.password_reset_sent_at = None
|
||||
user.is_active = True
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("[SECURITY] ADMIN_SET_PASSWORD user=%s admin=%s", user.email, _admin.get("email", "unknown"))
|
||||
return {"updated": True, "email": user.email}
|
||||
|
||||
|
||||
@router.get("/{user_id:path}", summary="Get details for a single user")
|
||||
def get_user(user_id: str, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Return profile and document statistics for a specific user."""
|
||||
doc_count = db.query(func.count(FileRecord.id)).filter(FileRecord.owner_id == user_id).scalar() or 0
|
||||
last_row = (
|
||||
db.query(FileRecord.created_at)
|
||||
.filter(FileRecord.owner_id == user_id)
|
||||
.order_by(FileRecord.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
last_upload = last_row[0].isoformat() if last_row and last_row[0] else None
|
||||
|
||||
profile = _get_or_none(db, user_id)
|
||||
|
||||
return {
|
||||
"user_id": user_id,
|
||||
"display_name": profile.display_name if profile else None,
|
||||
"daily_upload_limit": profile.daily_upload_limit if profile else None,
|
||||
"notes": profile.notes if profile else None,
|
||||
"is_blocked": profile.is_blocked if profile else False,
|
||||
"subscription_tier": (profile.subscription_tier or "free") if profile else "free",
|
||||
"subscription_billing_cycle": (profile.subscription_billing_cycle or "monthly") if profile else "monthly",
|
||||
"subscription_period_start": profile.subscription_period_start.isoformat()
|
||||
if (profile and profile.subscription_period_start)
|
||||
else None,
|
||||
"allow_overage": bool(profile.allow_overage) if profile else False,
|
||||
"is_complimentary": bool(profile.is_complimentary) if profile else False,
|
||||
"profile_id": profile.id if profile else None,
|
||||
"document_count": doc_count,
|
||||
"last_upload": last_upload,
|
||||
"profile": _profile_to_dict(profile) if profile else None,
|
||||
}
|
||||
|
||||
|
||||
@router.put("/{user_id:path}", summary="Create or update a user profile")
|
||||
def upsert_user_profile(
|
||||
user_id: str,
|
||||
body: UserProfileUpsert,
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new profile or update an existing one for *user_id*.
|
||||
|
||||
Returns the persisted profile.
|
||||
"""
|
||||
profile = _get_or_none(db, user_id)
|
||||
if profile is None:
|
||||
profile = UserProfile(user_id=user_id)
|
||||
db.add(profile)
|
||||
|
||||
old_tier = (profile.subscription_tier or "free") if profile.id else None # None means brand-new profile
|
||||
profile.display_name = body.display_name
|
||||
profile.daily_upload_limit = body.daily_upload_limit
|
||||
profile.notes = body.notes
|
||||
profile.is_blocked = body.is_blocked
|
||||
profile.subscription_billing_cycle = body.subscription_billing_cycle
|
||||
profile.subscription_period_start = body.subscription_period_start
|
||||
profile.allow_overage = body.allow_overage
|
||||
profile.is_complimentary = body.is_complimentary
|
||||
tier_changed = False
|
||||
new_tier: str | None = None
|
||||
if body.subscription_tier is not None:
|
||||
from app.utils.subscription import TIERS
|
||||
|
||||
if body.subscription_tier not in TIERS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid subscription_tier '{body.subscription_tier}'. Valid values: {list(TIERS.keys())}",
|
||||
)
|
||||
# Detect a real change only for existing profiles (old_tier is not None)
|
||||
if old_tier is not None and old_tier != body.subscription_tier:
|
||||
tier_changed = True
|
||||
new_tier = body.subscription_tier
|
||||
profile.subscription_tier = body.subscription_tier
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Admin upserted profile for user %s", user_id)
|
||||
|
||||
# Notify admins and fire webhook when plan is changed by an admin
|
||||
if tier_changed and new_tier is not None:
|
||||
try:
|
||||
from app.utils.notification import notify_plan_changed
|
||||
from app.utils.webhook import dispatch_webhook_event
|
||||
|
||||
notify_plan_changed(user_id, old_tier=old_tier, new_tier=new_tier, changed_by="admin") # type: ignore[arg-type]
|
||||
dispatch_webhook_event(
|
||||
"user.plan_changed",
|
||||
{
|
||||
"user_id": user_id,
|
||||
"old_tier": old_tier,
|
||||
"new_tier": new_tier,
|
||||
"billing_cycle": body.subscription_billing_cycle,
|
||||
"changed_by": "admin",
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to send plan-change notification/webhook for user %s", user_id)
|
||||
|
||||
return _profile_to_dict(profile)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{user_id:path}/payment-issue", status_code=status.HTTP_200_OK, summary="Report a payment issue for a user"
|
||||
)
|
||||
def report_payment_issue(user_id: str, body: PaymentIssueBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Notify admins and fire a webhook for a payment issue reported against *user_id*.
|
||||
|
||||
The user profile must exist. Use this endpoint when a payment processor
|
||||
webhook or manual review identifies a billing problem (e.g. failed charge,
|
||||
expired card, disputed transaction).
|
||||
|
||||
Returns the user profile dict alongside an acknowledgement flag.
|
||||
"""
|
||||
profile = _get_or_none(db, user_id)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User profile not found")
|
||||
|
||||
logger.warning("Payment issue reported for user %s: %s", user_id, body.issue)
|
||||
|
||||
try:
|
||||
from app.utils.notification import notify_payment_issue
|
||||
from app.utils.webhook import dispatch_webhook_event
|
||||
|
||||
notify_payment_issue(user_id, issue=body.issue)
|
||||
dispatch_webhook_event(
|
||||
"user.payment_issue",
|
||||
{
|
||||
"user_id": user_id,
|
||||
"issue": body.issue,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to send payment-issue notification/webhook for user %s", user_id)
|
||||
|
||||
return {"acknowledged": True, "user_id": user_id, "profile": _profile_to_dict(profile)}
|
||||
|
||||
|
||||
@router.delete("/{user_id:path}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete a user profile")
|
||||
def delete_user_profile(user_id: str, db: DbSession, _admin: AdminUser) -> None:
|
||||
"""Delete the admin-managed profile for *user_id*.
|
||||
|
||||
Documents owned by this user are **not** removed; only the profile record
|
||||
is deleted. To reassign or purge documents use the files API.
|
||||
"""
|
||||
profile = _get_or_none(db, user_id)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User profile not found")
|
||||
|
||||
try:
|
||||
db.delete(profile)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Admin deleted profile for user %s", user_id)
|
||||
@@ -1,224 +0,0 @@
|
||||
"""API endpoints for managing personal API tokens.
|
||||
|
||||
Provides CRUD operations so users can create, list, and revoke tokens
|
||||
that grant programmatic access to the DocuElevate API (e.g. webhook
|
||||
uploads, scripted integrations).
|
||||
|
||||
Tokens use ``secrets.token_urlsafe`` from the Python standard library
|
||||
(no extra dependencies) and are prefixed with ``de_`` for easy
|
||||
identification. Only a PBKDF2-HMAC-SHA256 hash is persisted; the
|
||||
plaintext is returned exactly once at creation time.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ApiToken
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api-tokens", tags=["api-tokens"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Prefix prepended to every generated token for easy identification.
|
||||
TOKEN_PREFIX = "de_"
|
||||
#: Number of random bytes for the token body (32 → 43 URL-safe chars).
|
||||
TOKEN_BYTES = 32
|
||||
#: PBKDF2 iteration count for hashing API tokens.
|
||||
TOKEN_HASH_ITERATIONS = 100_000
|
||||
#: PBKDF2 salt for API token hashing (not secret, but fixed for determinism).
|
||||
TOKEN_HASH_SALT = b"api-token-v1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_api_token() -> str:
|
||||
"""Generate a new API token with the ``de_`` prefix.
|
||||
|
||||
Returns:
|
||||
A URL-safe random token string, e.g. ``de_Ab3xY…``.
|
||||
"""
|
||||
return TOKEN_PREFIX + secrets.token_urlsafe(TOKEN_BYTES)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
"""Return a PBKDF2-HMAC-SHA256 hex digest of *token*.
|
||||
|
||||
Args:
|
||||
token: The plaintext API token.
|
||||
|
||||
Returns:
|
||||
64-character lowercase hex string.
|
||||
"""
|
||||
dk = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
token.encode("utf-8"),
|
||||
TOKEN_HASH_SALT,
|
||||
TOKEN_HASH_ITERATIONS,
|
||||
)
|
||||
return dk.hex()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TokenCreate(BaseModel):
|
||||
"""Schema for creating a new API token."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable label for the token")
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
"""Schema returned when listing tokens (plaintext is never included)."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
token_prefix: str
|
||||
is_active: bool
|
||||
last_used_at: datetime | None
|
||||
last_used_ip: str | None
|
||||
created_at: datetime | None
|
||||
revoked_at: datetime | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class TokenCreatedResponse(TokenResponse):
|
||||
"""Schema returned once at creation time — includes the full plaintext token."""
|
||||
|
||||
token: str = Field(..., description="The full API token. Store it securely — it will not be shown again.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, response_model=TokenCreatedResponse)
|
||||
async def create_token(
|
||||
body: TokenCreate,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new personal API token.
|
||||
|
||||
The full token is returned **only once** in the response. Subsequent
|
||||
``GET`` requests will only show the prefix for identification.
|
||||
"""
|
||||
plaintext = generate_api_token()
|
||||
token_hash_value = hash_token(plaintext)
|
||||
prefix = plaintext[:12] # "de_" prefix + 9 random chars = 12 chars total
|
||||
|
||||
db_token = ApiToken(
|
||||
owner_id=owner_id,
|
||||
name=body.name,
|
||||
token_hash=token_hash_value,
|
||||
token_prefix=prefix,
|
||||
)
|
||||
try:
|
||||
db.add(db_token)
|
||||
db.commit()
|
||||
db.refresh(db_token)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("API token created: id=%s owner=%s name=%r", db_token.id, owner_id, body.name)
|
||||
|
||||
return {
|
||||
"id": db_token.id,
|
||||
"name": db_token.name,
|
||||
"token_prefix": db_token.token_prefix,
|
||||
"is_active": db_token.is_active,
|
||||
"last_used_at": db_token.last_used_at,
|
||||
"last_used_ip": db_token.last_used_ip,
|
||||
"created_at": db_token.created_at,
|
||||
"revoked_at": db_token.revoked_at,
|
||||
"token": plaintext,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/", response_model=list[TokenResponse])
|
||||
async def list_tokens(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all API tokens for the authenticated user."""
|
||||
tokens = db.query(ApiToken).filter(ApiToken.owner_id == owner_id).order_by(ApiToken.created_at.desc()).all()
|
||||
return [
|
||||
{
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"token_prefix": t.token_prefix,
|
||||
"is_active": t.is_active,
|
||||
"last_used_at": t.last_used_at,
|
||||
"last_used_ip": t.last_used_ip,
|
||||
"created_at": t.created_at,
|
||||
"revoked_at": t.revoked_at,
|
||||
}
|
||||
for t in tokens
|
||||
]
|
||||
|
||||
|
||||
@router.delete("/{token_id}", status_code=status.HTTP_200_OK)
|
||||
async def revoke_token(
|
||||
token_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Revoke (soft-delete) an API token.
|
||||
|
||||
The token row is kept for audit purposes but marked inactive with a
|
||||
``revoked_at`` timestamp.
|
||||
"""
|
||||
db_token = db.query(ApiToken).filter(ApiToken.id == token_id, ApiToken.owner_id == owner_id).first()
|
||||
if not db_token:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found")
|
||||
|
||||
if not db_token.is_active:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Token is already revoked")
|
||||
|
||||
try:
|
||||
db_token.is_active = False
|
||||
db_token.revoked_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("API token revoked: id=%s owner=%s", token_id, owner_id)
|
||||
return {"detail": "Token revoked"}
|
||||
@@ -1,115 +0,0 @@
|
||||
"""
|
||||
Audit log REST API endpoints.
|
||||
|
||||
Provides read-only access to the comprehensive audit log for admin users.
|
||||
Events are append-only — there are no update or delete endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.utils.audit_service import count_events, query_events
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/audit-logs")
|
||||
@require_login
|
||||
async def list_audit_logs(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
action: str | None = Query(None, description="Filter by action (exact match)"),
|
||||
user: str | None = Query(None, description="Filter by username"),
|
||||
resource_type: str | None = Query(None, description="Filter by resource type"),
|
||||
severity: str | None = Query(None, description="Filter by severity level"),
|
||||
since: datetime | None = Query(None, description="Only events at or after this ISO-8601 timestamp"),
|
||||
until: datetime | None = Query(None, description="Only events at or before this ISO-8601 timestamp"),
|
||||
limit: int = Query(50, ge=1, le=500, description="Max rows to return"),
|
||||
offset: int = Query(0, ge=0, description="Rows to skip for pagination"),
|
||||
) -> dict[str, Any]:
|
||||
"""Return audit log entries with optional filtering and pagination.
|
||||
|
||||
Requires authentication. Returns events in reverse chronological order.
|
||||
"""
|
||||
entries = query_events(
|
||||
db,
|
||||
action=action,
|
||||
user=user,
|
||||
resource_type=resource_type,
|
||||
severity=severity,
|
||||
since=since,
|
||||
until=until,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
total = count_events(
|
||||
db,
|
||||
action=action,
|
||||
user=user,
|
||||
resource_type=resource_type,
|
||||
severity=severity,
|
||||
since=since,
|
||||
until=until,
|
||||
)
|
||||
return {
|
||||
"items": [_serialize(e) for e in entries],
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/audit-logs/actions")
|
||||
@require_login
|
||||
async def list_distinct_actions(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[str]:
|
||||
"""Return the distinct action values present in the audit log."""
|
||||
from app.models import AuditLog
|
||||
|
||||
rows = db.query(AuditLog.action).distinct().order_by(AuditLog.action).all()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
@router.get("/audit-logs/users")
|
||||
@require_login
|
||||
async def list_distinct_users(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[str]:
|
||||
"""Return the distinct user values present in the audit log."""
|
||||
from app.models import AuditLog
|
||||
|
||||
rows = db.query(AuditLog.user).distinct().order_by(AuditLog.user).all()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize(entry) -> dict[str, Any]:
|
||||
"""Convert an AuditLog row to a JSON-safe dict."""
|
||||
import json as _json
|
||||
|
||||
return {
|
||||
"id": entry.id,
|
||||
"timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
|
||||
"user": entry.user,
|
||||
"action": entry.action,
|
||||
"resource_type": entry.resource_type,
|
||||
"resource_id": entry.resource_id,
|
||||
"ip_address": entry.ip_address,
|
||||
"details": _json.loads(entry.details) if entry.details else None,
|
||||
"severity": entry.severity,
|
||||
}
|
||||
+45
-35
@@ -1,25 +1,24 @@
|
||||
"""
|
||||
Azure AI API endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Request, HTTPException, status
|
||||
import logging
|
||||
|
||||
import azure.core.exceptions
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
|
||||
|
||||
# Import the Azure modules including the administration client
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from fastapi import APIRouter, Request
|
||||
import os
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
|
||||
# Import the Azure modules including the administration client
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
|
||||
import azure.core.exceptions
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/azure/test")
|
||||
@require_login
|
||||
async def test_azure_connection(request: Request):
|
||||
@@ -29,7 +28,7 @@ async def test_azure_connection(request: Request):
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing Azure Document Intelligence connection")
|
||||
|
||||
|
||||
# Check if Azure configuration is present
|
||||
if not settings.azure_endpoint or not settings.azure_ai_key:
|
||||
logger.warning("Azure Document Intelligence configuration is incomplete")
|
||||
@@ -38,77 +37,88 @@ async def test_azure_connection(request: Request):
|
||||
missing.append("endpoint")
|
||||
if not settings.azure_ai_key:
|
||||
missing.append("API key")
|
||||
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}",
|
||||
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}"
|
||||
}
|
||||
|
||||
|
||||
# Try to initialize the admin client and make a request to list operations
|
||||
try:
|
||||
# Initialize the admin client with credentials
|
||||
admin_client = DocumentIntelligenceAdministrationClient(
|
||||
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
|
||||
endpoint=settings.azure_endpoint,
|
||||
credential=AzureKeyCredential(settings.azure_ai_key)
|
||||
)
|
||||
|
||||
|
||||
# Test the connection by listing operations - this is a documented method in the admin client
|
||||
operations = list(admin_client.list_operations())
|
||||
|
||||
|
||||
# Successfully initialized client and made a request
|
||||
logger.info("Azure Document Intelligence Admin connection successfully tested")
|
||||
|
||||
|
||||
# Return success with available operations info
|
||||
operations_info = []
|
||||
try:
|
||||
for op in operations:
|
||||
if hasattr(op, "operation_id") and op.operation_id:
|
||||
if hasattr(op, 'operation_id') and op.operation_id:
|
||||
op_info = {
|
||||
"id": op.operation_id,
|
||||
"status": op.status if hasattr(op, "status") else "Unknown",
|
||||
"created": str(op.created_on) if hasattr(op, "created_on") else "Unknown",
|
||||
"kind": op.kind if hasattr(op, "kind") else "Unknown",
|
||||
"status": op.status if hasattr(op, 'status') else "Unknown",
|
||||
"created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown",
|
||||
"kind": op.kind if hasattr(op, 'kind') else "Unknown"
|
||||
}
|
||||
operations_info.append(op_info)
|
||||
|
||||
|
||||
operation_count = len(operations_info)
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.",
|
||||
"endpoint": settings.azure_endpoint,
|
||||
"operations_count": operation_count,
|
||||
"recent_operations": operations_info[:3] if operations_info else [],
|
||||
"recent_operations": operations_info[:3] if operations_info else []
|
||||
}
|
||||
except Exception as e:
|
||||
# If error occurs while processing operations info, still return success
|
||||
logger.warning(f"Connected to Azure but couldn't parse operations: {e}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Azure Document Intelligence connection is valid, "
|
||||
"but couldn't retrieve operations details.",
|
||||
"endpoint": settings.azure_endpoint,
|
||||
"message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.",
|
||||
"endpoint": settings.azure_endpoint
|
||||
}
|
||||
|
||||
|
||||
except azure.core.exceptions.ClientAuthenticationError as e:
|
||||
logger.error(f"Azure authentication error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Authentication error: Invalid API key or credentials",
|
||||
"detail": str(e),
|
||||
"message": f"Authentication error: Invalid API key or credentials",
|
||||
"detail": str(e)
|
||||
}
|
||||
except azure.core.exceptions.ServiceRequestError as e:
|
||||
logger.error(f"Azure service request error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Service request error: Could not reach the Azure endpoint",
|
||||
"detail": str(e),
|
||||
"message": f"Service request error: Could not reach the Azure endpoint",
|
||||
"detail": str(e)
|
||||
}
|
||||
except ValueError as e:
|
||||
logger.error(f"Azure configuration value error: {e}")
|
||||
return {"status": "error", "message": f"Configuration error: {str(e)}", "detail": str(e)}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Configuration error: {str(e)}",
|
||||
"detail": str(e)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Azure connection test failed with unexpected error: {e}")
|
||||
return {"status": "error", "message": "Connection test failed with unexpected error", "detail": str(e)}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection test failed with unexpected error",
|
||||
"detail": str(e)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing Azure Document Intelligence connection")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Unexpected error: {str(e)}"
|
||||
}
|
||||
|
||||
@@ -1,253 +0,0 @@
|
||||
"""
|
||||
Backup and restore API endpoints for DocuElevate.
|
||||
|
||||
Provides REST endpoints for:
|
||||
- Listing existing backups
|
||||
- Triggering a manual backup
|
||||
- Downloading a backup archive
|
||||
- Restoring from an uploaded backup file
|
||||
- Deleting a backup record
|
||||
- Running retention cleanup
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, status
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import BackupRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/backup", tags=["backup"])
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
# Annotated shorthand so FastAPI can resolve and tests can override it.
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/")
|
||||
async def list_backups(
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[dict]:
|
||||
"""Return all backup records, newest first."""
|
||||
records = db.query(BackupRecord).order_by(BackupRecord.created_at.desc()).all()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"filename": r.filename,
|
||||
"backup_type": r.backup_type,
|
||||
"size_bytes": r.size_bytes,
|
||||
"checksum": r.checksum,
|
||||
"status": r.status,
|
||||
"local_path": r.local_path,
|
||||
"remote_destination": r.remote_destination,
|
||||
"remote_path": r.remote_path,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
"local_available": bool(r.local_path and os.path.exists(r.local_path)),
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def trigger_backup(
|
||||
_admin: AdminUser,
|
||||
backup_type: str = "hourly",
|
||||
) -> dict:
|
||||
"""Trigger a manual backup immediately.
|
||||
|
||||
Query parameter ``backup_type`` accepts ``hourly``, ``daily``, or
|
||||
``weekly`` (default: ``hourly``).
|
||||
"""
|
||||
if backup_type not in ("hourly", "daily", "weekly"):
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid backup_type")
|
||||
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
task = create_backup.delay(backup_type=backup_type)
|
||||
return {"task_id": task.id, "status": "queued", "backup_type": backup_type}
|
||||
|
||||
|
||||
@router.get("/{backup_id}/download")
|
||||
async def download_backup(
|
||||
backup_id: int,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> FileResponse:
|
||||
"""Stream the backup archive to the client."""
|
||||
rec = db.get(BackupRecord, backup_id)
|
||||
if rec is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup not found")
|
||||
if not rec.local_path or not os.path.exists(rec.local_path):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Local archive file is not available (may have been pruned)",
|
||||
)
|
||||
return FileResponse(
|
||||
path=rec.local_path,
|
||||
filename=rec.filename,
|
||||
media_type="application/gzip",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/restore")
|
||||
async def restore_backup(
|
||||
_admin: AdminUser,
|
||||
file: UploadFile,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Restore the database from an uploaded gzip-compressed SQL dump.
|
||||
|
||||
**Warning**: This overwrites the current database contents.
|
||||
|
||||
Supported formats (must match the currently configured database backend):
|
||||
|
||||
- ``*.db.gz`` – gzip-compressed SQLite ``.dump()`` SQL script (SQLite backend)
|
||||
- ``*.pgsql.gz`` – gzip-compressed ``pg_dump --format=plain`` output (PostgreSQL backend)
|
||||
- ``*.mysql.gz`` – gzip-compressed ``mysqldump`` output (MySQL / MariaDB backend)
|
||||
"""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
from app.config import settings as app_settings
|
||||
from app.tasks.backup_tasks import (
|
||||
_archive_ext_for_backend,
|
||||
_db_path,
|
||||
_restore_mysql,
|
||||
_restore_postgresql,
|
||||
_restore_sqlite,
|
||||
)
|
||||
|
||||
url = make_url(app_settings.database_url)
|
||||
backend = url.get_backend_name()
|
||||
expected_ext = _archive_ext_for_backend(backend)
|
||||
|
||||
if not file.filename or not file.filename.endswith(expected_ext):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"Uploaded file must be a '{expected_ext}' backup archive for the current database backend ({backend})."
|
||||
),
|
||||
)
|
||||
|
||||
# Write upload to a temp file
|
||||
with tempfile.NamedTemporaryFile(suffix=expected_ext, delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
content = await file.read()
|
||||
tmp.write(content)
|
||||
|
||||
try:
|
||||
if backend == "sqlite":
|
||||
db_path = _db_path()
|
||||
if db_path is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Restore is only supported for file-based SQLite databases.",
|
||||
)
|
||||
# Close the application DB session before replacing the file
|
||||
db.close()
|
||||
try:
|
||||
_restore_sqlite(db_path, tmp_path)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
|
||||
elif backend == "postgresql":
|
||||
db.close()
|
||||
try:
|
||||
_restore_postgresql(app_settings.database_url, tmp_path)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"psql binary not found – is PostgreSQL client installed? ({exc})",
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"PostgreSQL restore failed: {exc}",
|
||||
) from exc
|
||||
|
||||
elif backend == "mysql":
|
||||
db.close()
|
||||
try:
|
||||
_restore_mysql(app_settings.database_url, tmp_path)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"mysql binary not found – is MySQL client installed? ({exc})",
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"MySQL restore failed: {exc}",
|
||||
) from exc
|
||||
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Database backend '{backend}' does not support restore.",
|
||||
)
|
||||
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
logger.info(f"Database restored from uploaded backup: {file.filename}")
|
||||
return {"status": "restored", "filename": file.filename}
|
||||
|
||||
|
||||
@router.delete("/{backup_id}")
|
||||
async def delete_backup(
|
||||
backup_id: int,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Delete a backup record (and local file if present)."""
|
||||
rec = db.get(BackupRecord, backup_id)
|
||||
if rec is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup not found")
|
||||
|
||||
if rec.local_path and os.path.exists(rec.local_path):
|
||||
try:
|
||||
os.remove(rec.local_path)
|
||||
except OSError as exc:
|
||||
logger.warning(f"Could not remove local backup file {rec.local_path}: {exc}")
|
||||
|
||||
db.delete(rec)
|
||||
db.commit()
|
||||
return {"status": "deleted", "id": backup_id}
|
||||
|
||||
|
||||
@router.post("/cleanup")
|
||||
async def run_cleanup(_admin: AdminUser) -> dict:
|
||||
"""Manually trigger the retention cleanup for all backup tiers."""
|
||||
from app.tasks.backup_tasks import cleanup_old_backups
|
||||
|
||||
task = cleanup_old_backups.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
@@ -1,619 +0,0 @@
|
||||
"""Stripe billing integration for DocuElevate.
|
||||
|
||||
Provides three endpoints:
|
||||
- POST /api/billing/create-checkout-session — starts Stripe Checkout for a plan upgrade
|
||||
- POST /api/billing/create-portal-session — opens Stripe Customer Portal (manage/cancel)
|
||||
- POST /api/billing/webhook — handles Stripe webhook events
|
||||
- GET /api/billing/success — success landing page after checkout
|
||||
|
||||
Stripe Python SDK license: MIT (compatible with this project's Apache 2.0 license).
|
||||
|
||||
GDPR: Stripe acts as a data processor under a Data Processing Agreement (DPA).
|
||||
Stripe is SOC 2 Type II certified and supports EU data residency.
|
||||
SOC2: Stripe is SOC 2 Type II certified.
|
||||
EU VAT: Configure Stripe Tax in the Stripe Dashboard for automatic VAT collection.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import stripe
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import SubscriptionPlan, UserProfile
|
||||
from app.utils.i18n import translate as _translate
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/billing", tags=["billing"])
|
||||
|
||||
_templates_dir = pathlib.Path(__file__).parents[2] / "frontend" / "templates"
|
||||
_templates = Jinja2Templates(directory=str(_templates_dir))
|
||||
_templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
|
||||
|
||||
|
||||
def _get_stripe() -> stripe.StripeClient | None:
|
||||
"""Return a configured Stripe client, or None when not configured."""
|
||||
if not settings.stripe_secret_key:
|
||||
return None
|
||||
return stripe.StripeClient(settings.stripe_secret_key)
|
||||
|
||||
|
||||
def _get_or_create_stripe_customer(
|
||||
client: stripe.StripeClient,
|
||||
db: Session,
|
||||
owner_id: str,
|
||||
email: str | None,
|
||||
name: str | None,
|
||||
) -> str:
|
||||
"""Return the Stripe customer_id for *owner_id*, creating one if needed.
|
||||
|
||||
Args:
|
||||
client: Configured Stripe client.
|
||||
db: Database session.
|
||||
owner_id: Stable user identifier.
|
||||
email: User's email for the Stripe customer record.
|
||||
name: User's display name for the Stripe customer record.
|
||||
|
||||
Returns:
|
||||
The Stripe customer ID string.
|
||||
"""
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first()
|
||||
if profile and profile.stripe_customer_id:
|
||||
return profile.stripe_customer_id
|
||||
|
||||
customer = client.customers.create(
|
||||
params={
|
||||
"email": email or "",
|
||||
"name": name or "",
|
||||
"metadata": {"docuelevate_user_id": owner_id},
|
||||
}
|
||||
)
|
||||
if profile:
|
||||
profile.stripe_customer_id = customer.id
|
||||
db.commit()
|
||||
return customer.id
|
||||
|
||||
|
||||
class CheckoutSessionBody(BaseModel):
|
||||
"""Request body for creating a Stripe Checkout session."""
|
||||
|
||||
plan_id: str
|
||||
billing_cycle: str = "monthly" # "monthly" | "yearly"
|
||||
|
||||
|
||||
class PortalSessionBody(BaseModel):
|
||||
"""Request body for creating a Stripe Customer Portal session."""
|
||||
|
||||
return_url: str | None = None
|
||||
|
||||
|
||||
@router.post("/create-checkout-session", summary="Create a Stripe Checkout session for a plan upgrade")
|
||||
@require_login
|
||||
async def create_checkout_session(
|
||||
request: Request,
|
||||
body: CheckoutSessionBody,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
"""Create a Stripe Checkout session.
|
||||
|
||||
The client should redirect the user to the returned ``checkout_url``.
|
||||
|
||||
Raises:
|
||||
503: Stripe is not configured.
|
||||
404: Plan not found or has no Stripe price configured.
|
||||
"""
|
||||
client = _get_stripe()
|
||||
if not client:
|
||||
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Billing is not configured.")
|
||||
|
||||
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == body.plan_id).first()
|
||||
if plan is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Plan {body.plan_id!r} not found.")
|
||||
|
||||
price_id = plan.stripe_price_id_yearly if body.billing_cycle == "yearly" else plan.stripe_price_id_monthly
|
||||
if not price_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=(
|
||||
f"Stripe price ID not configured for plan {body.plan_id!r} ({body.billing_cycle}). "
|
||||
"Please set it in the Admin Plan Designer."
|
||||
),
|
||||
)
|
||||
|
||||
user = request.session.get("user") or {}
|
||||
owner_id = get_current_owner_id(request) or user.get("email") or ""
|
||||
email = user.get("email")
|
||||
name = user.get("name")
|
||||
|
||||
customer_id = _get_or_create_stripe_customer(client, db, owner_id, email, name)
|
||||
|
||||
base = str(request.base_url).rstrip("/")
|
||||
success_url = settings.stripe_success_url or f"{base}/api/billing/success"
|
||||
cancel_url = settings.stripe_cancel_url or f"{base}/pricing"
|
||||
|
||||
trial_days = plan.trial_days if plan.trial_days > 0 else None
|
||||
|
||||
session_params: dict[str, Any] = {
|
||||
"customer": customer_id,
|
||||
"mode": "subscription",
|
||||
"line_items": [{"price": price_id, "quantity": 1}],
|
||||
"success_url": success_url + "?session_id={CHECKOUT_SESSION_ID}",
|
||||
"cancel_url": cancel_url,
|
||||
"subscription_data": {
|
||||
"metadata": {
|
||||
"docuelevate_user_id": owner_id,
|
||||
"plan_id": body.plan_id,
|
||||
"billing_cycle": body.billing_cycle,
|
||||
},
|
||||
},
|
||||
"metadata": {"docuelevate_user_id": owner_id, "plan_id": body.plan_id},
|
||||
"allow_promotion_codes": True,
|
||||
"billing_address_collection": "auto",
|
||||
"tax_id_collection": {"enabled": True},
|
||||
"automatic_tax": {"enabled": True},
|
||||
}
|
||||
if trial_days:
|
||||
session_params["subscription_data"]["trial_period_days"] = trial_days
|
||||
|
||||
checkout_session = client.checkout.sessions.create(params=session_params)
|
||||
|
||||
logger.info(
|
||||
"Created Stripe checkout session %s for user %s plan %s",
|
||||
checkout_session.id,
|
||||
owner_id,
|
||||
body.plan_id,
|
||||
)
|
||||
return {"checkout_url": checkout_session.url, "session_id": checkout_session.id}
|
||||
|
||||
|
||||
@router.post("/create-portal-session", summary="Create a Stripe Customer Portal session")
|
||||
@require_login
|
||||
async def create_portal_session(
|
||||
request: Request,
|
||||
body: PortalSessionBody,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
"""Create a Stripe Customer Portal session for subscription self-management.
|
||||
|
||||
Raises:
|
||||
503: Stripe not configured.
|
||||
404: No Stripe customer found for this user.
|
||||
"""
|
||||
client = _get_stripe()
|
||||
if not client:
|
||||
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Billing is not configured.")
|
||||
|
||||
user = request.session.get("user") or {}
|
||||
owner_id = get_current_owner_id(request) or user.get("email") or ""
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first()
|
||||
if not profile or not profile.stripe_customer_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="No billing account found. Please subscribe to a plan first.",
|
||||
)
|
||||
|
||||
base = str(request.base_url).rstrip("/")
|
||||
return_url = body.return_url or f"{base}/subscription"
|
||||
|
||||
portal = client.billing_portal.sessions.create(
|
||||
params={
|
||||
"customer": profile.stripe_customer_id,
|
||||
"return_url": return_url,
|
||||
}
|
||||
)
|
||||
|
||||
logger.info("Created Stripe portal session for user %s", owner_id)
|
||||
return {"portal_url": portal.url}
|
||||
|
||||
|
||||
@router.post("/webhook", include_in_schema=False)
|
||||
async def stripe_webhook(request: Request, db: Session = Depends(get_db)) -> dict[str, str]:
|
||||
"""Handle Stripe webhook events.
|
||||
|
||||
Syncs subscription status to UserProfile.subscription_tier.
|
||||
|
||||
Events handled:
|
||||
|
||||
- ``checkout.session.completed`` — activate subscription after payment
|
||||
- ``customer.subscription.updated`` — sync tier change
|
||||
- ``customer.subscription.deleted`` — downgrade to free on cancellation
|
||||
- ``invoice.payment_failed`` — log failed payment
|
||||
"""
|
||||
if not settings.stripe_secret_key:
|
||||
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Billing not configured.")
|
||||
|
||||
payload = await request.body()
|
||||
sig_header = request.headers.get("stripe-signature", "")
|
||||
|
||||
try:
|
||||
if settings.stripe_webhook_secret:
|
||||
event = stripe.Webhook.construct_event(payload, sig_header, settings.stripe_webhook_secret)
|
||||
else:
|
||||
logger.warning(
|
||||
"[SECURITY] STRIPE_WEBHOOK_SECRET is not configured. "
|
||||
"Webhook events are accepted without signature verification. "
|
||||
"Set STRIPE_WEBHOOK_SECRET in production to prevent spoofed events."
|
||||
)
|
||||
event = stripe.Event.construct_from(json.loads(payload), stripe.api_key)
|
||||
except stripe.SignatureVerificationError:
|
||||
logger.warning("[SECURITY] Stripe webhook signature verification failed")
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid webhook signature.")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to parse Stripe webhook: %s", exc)
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid webhook payload.")
|
||||
|
||||
_handle_stripe_event(db, event)
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/success", include_in_schema=False)
|
||||
@require_login
|
||||
async def billing_success(request: Request) -> Any:
|
||||
"""Show a success page after a completed Stripe Checkout."""
|
||||
return _templates.TemplateResponse("billing_success.html", {"request": request})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin: Stripe status + sync helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> None:
|
||||
"""Raise 403 if the current session user is not an admin."""
|
||||
user = request.session.get("user") or {}
|
||||
if not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required.")
|
||||
|
||||
|
||||
@router.get("/stripe/status", summary="Check Stripe connection and plan sync status (admin only)")
|
||||
@require_login
|
||||
async def stripe_status(request: Request, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Return Stripe connection health and per-plan price-ID sync status.
|
||||
|
||||
Returns a JSON object with:
|
||||
- ``configured``: whether STRIPE_SECRET_KEY is set
|
||||
- ``connection``: ``"ok"`` or an error string (live/test mode label)
|
||||
- ``mode``: ``"live"`` | ``"test"`` | ``null``
|
||||
- ``plans``: list of plan objects with ``plan_id``, ``name``,
|
||||
``stripe_price_id_monthly``, ``stripe_price_id_yearly``, ``synced``
|
||||
|
||||
Raises:
|
||||
403: Not admin.
|
||||
503: Stripe not configured.
|
||||
"""
|
||||
_require_admin(request)
|
||||
|
||||
if not settings.stripe_secret_key:
|
||||
return {
|
||||
"configured": False,
|
||||
"connection": "not_configured",
|
||||
"mode": None,
|
||||
"plans": [],
|
||||
}
|
||||
|
||||
client = _get_stripe()
|
||||
# Probe Stripe with a lightweight account fetch
|
||||
mode: str | None = None
|
||||
connection_status = "ok"
|
||||
try:
|
||||
account = client.accounts.retrieve("me") # type: ignore[arg-type]
|
||||
livemode = getattr(account, "livemode", None)
|
||||
if livemode is True:
|
||||
mode = "live"
|
||||
elif livemode is False:
|
||||
mode = "test"
|
||||
else:
|
||||
mode = "test" if settings.stripe_secret_key.startswith("sk_test_") else "live"
|
||||
except Exception:
|
||||
logger.exception("Stripe connection check failed")
|
||||
connection_status = "error"
|
||||
mode = "test" if settings.stripe_secret_key.startswith("sk_test_") else "live"
|
||||
|
||||
plans = db.query(SubscriptionPlan).order_by(SubscriptionPlan.sort_order).all()
|
||||
plan_statuses = []
|
||||
for plan in plans:
|
||||
has_monthly = bool(plan.stripe_price_id_monthly)
|
||||
has_yearly = bool(plan.stripe_price_id_yearly)
|
||||
is_paid = plan.price_monthly > 0 or plan.price_yearly > 0
|
||||
synced = (not is_paid) or (has_monthly and (not plan.price_yearly or has_yearly))
|
||||
plan_statuses.append(
|
||||
{
|
||||
"plan_id": plan.plan_id,
|
||||
"name": plan.name,
|
||||
"price_monthly": plan.price_monthly,
|
||||
"price_yearly": plan.price_yearly,
|
||||
"stripe_price_id_monthly": plan.stripe_price_id_monthly,
|
||||
"stripe_price_id_yearly": plan.stripe_price_id_yearly,
|
||||
"synced": synced,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"configured": True,
|
||||
"connection": connection_status,
|
||||
"mode": mode,
|
||||
"webhook_secret_configured": bool(settings.stripe_webhook_secret),
|
||||
"plans": plan_statuses,
|
||||
"webhook_endpoint": str(request.base_url).rstrip("/") + "/api/billing/webhook",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/stripe/sync-plans", summary="Auto-create Stripe products and prices for all plans (admin only)")
|
||||
@require_login
|
||||
async def stripe_sync_plans(request: Request, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Create Stripe Product + Price objects for every paid plan that is missing them.
|
||||
|
||||
For each paid plan (``price_monthly > 0``) that lacks a ``stripe_price_id_monthly``,
|
||||
this endpoint:
|
||||
|
||||
1. Creates a Stripe *Product* named after the plan.
|
||||
2. Creates a Stripe *Price* for the monthly amount.
|
||||
3. Optionally creates a yearly Price if ``price_yearly > 0``.
|
||||
4. Persists the resulting ``price_id`` values back into ``SubscriptionPlan``.
|
||||
|
||||
Already-synced plans (those that already have ``stripe_price_id_monthly``) are
|
||||
skipped — existing prices in Stripe are never modified.
|
||||
|
||||
Raises:
|
||||
403: Not admin.
|
||||
503: Stripe not configured.
|
||||
"""
|
||||
_require_admin(request)
|
||||
|
||||
client = _get_stripe()
|
||||
if not client:
|
||||
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Billing is not configured.")
|
||||
|
||||
plans = db.query(SubscriptionPlan).order_by(SubscriptionPlan.sort_order).all()
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for plan in plans:
|
||||
is_paid = plan.price_monthly > 0 or plan.price_yearly > 0
|
||||
if not is_paid:
|
||||
results.append({"plan_id": plan.plan_id, "name": plan.name, "status": "skipped_free"})
|
||||
continue
|
||||
|
||||
already_has_monthly = bool(plan.stripe_price_id_monthly)
|
||||
already_has_yearly = bool(plan.stripe_price_id_yearly)
|
||||
|
||||
if already_has_monthly and (not plan.price_yearly or already_has_yearly):
|
||||
results.append({"plan_id": plan.plan_id, "name": plan.name, "status": "already_synced"})
|
||||
continue
|
||||
|
||||
try:
|
||||
# Create (or look up) the Stripe Product for this plan
|
||||
product = client.products.create(
|
||||
params={
|
||||
"name": str(plan.name),
|
||||
"metadata": {"docuelevate_plan_id": plan.plan_id},
|
||||
}
|
||||
)
|
||||
|
||||
changed = False
|
||||
|
||||
# Monthly price
|
||||
if not already_has_monthly and plan.price_monthly > 0:
|
||||
monthly_price = client.prices.create(
|
||||
params={
|
||||
"product": product.id,
|
||||
"unit_amount": int(round(plan.price_monthly * 100)),
|
||||
"currency": "usd",
|
||||
"recurring": {"interval": "month"},
|
||||
"metadata": {"docuelevate_plan_id": plan.plan_id, "billing_cycle": "monthly"},
|
||||
}
|
||||
)
|
||||
plan.stripe_price_id_monthly = monthly_price.id
|
||||
changed = True
|
||||
|
||||
# Yearly price
|
||||
if not already_has_yearly and plan.price_yearly > 0:
|
||||
yearly_price = client.prices.create(
|
||||
params={
|
||||
"product": product.id,
|
||||
"unit_amount": int(round(plan.price_yearly * 100)),
|
||||
"currency": "usd",
|
||||
"recurring": {"interval": "year"},
|
||||
"metadata": {"docuelevate_plan_id": plan.plan_id, "billing_cycle": "yearly"},
|
||||
}
|
||||
)
|
||||
plan.stripe_price_id_yearly = yearly_price.id
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Stripe sync: created product/prices for plan %s (product %s)",
|
||||
plan.plan_id,
|
||||
product.id,
|
||||
)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"plan_id": plan.plan_id,
|
||||
"name": plan.name,
|
||||
"status": "created",
|
||||
"stripe_price_id_monthly": plan.stripe_price_id_monthly,
|
||||
"stripe_price_id_yearly": plan.stripe_price_id_yearly,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error("Stripe sync failed for plan %s: %s", plan.plan_id, exc)
|
||||
results.append(
|
||||
{
|
||||
"plan_id": plan.plan_id,
|
||||
"name": str(plan.name),
|
||||
"status": "error",
|
||||
"detail": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
return {"results": results}
|
||||
|
||||
|
||||
def _handle_stripe_event(db: Session, event: Any) -> None:
|
||||
"""Dispatch Stripe event to the appropriate handler.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
event: Parsed Stripe event object.
|
||||
"""
|
||||
etype = event.get("type", "") if isinstance(event, dict) else getattr(event, "type", "")
|
||||
data_obj = (
|
||||
event.get("data", {}).get("object", {})
|
||||
if isinstance(event, dict)
|
||||
else getattr(getattr(event, "data", None), "object", {})
|
||||
)
|
||||
|
||||
if etype == "checkout.session.completed":
|
||||
_on_checkout_completed(db, data_obj)
|
||||
elif etype == "customer.subscription.updated":
|
||||
_on_subscription_updated(db, data_obj)
|
||||
elif etype == "customer.subscription.deleted":
|
||||
_on_subscription_deleted(db, data_obj)
|
||||
elif etype == "invoice.payment_failed":
|
||||
customer_id = data_obj.get("customer", "") if isinstance(data_obj, dict) else getattr(data_obj, "customer", "")
|
||||
logger.warning("Stripe invoice payment failed for customer %s", customer_id)
|
||||
else:
|
||||
logger.debug("Unhandled Stripe event type: %s", etype)
|
||||
|
||||
|
||||
def _resolve_user_id_from_customer(db: Session, customer_id: str) -> str | None:
|
||||
"""Look up the DocuElevate user_id for a Stripe customer_id.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
customer_id: Stripe customer ID.
|
||||
|
||||
Returns:
|
||||
The matching ``UserProfile.user_id``, or ``None`` if not found.
|
||||
"""
|
||||
profile = db.query(UserProfile).filter(UserProfile.stripe_customer_id == customer_id).first()
|
||||
return profile.user_id if profile else None
|
||||
|
||||
|
||||
def _resolve_plan_id_from_price(db: Session, price_id: str) -> str | None:
|
||||
"""Map a Stripe price_id to a DocuElevate plan_id via SubscriptionPlan.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
price_id: Stripe price ID.
|
||||
|
||||
Returns:
|
||||
The matching ``SubscriptionPlan.plan_id``, or ``None`` if not found.
|
||||
"""
|
||||
plan = (
|
||||
db.query(SubscriptionPlan)
|
||||
.filter(
|
||||
(SubscriptionPlan.stripe_price_id_monthly == price_id)
|
||||
| (SubscriptionPlan.stripe_price_id_yearly == price_id)
|
||||
)
|
||||
.first()
|
||||
)
|
||||
return plan.plan_id if plan else None
|
||||
|
||||
|
||||
def _on_checkout_completed(db: Session, data: Any) -> None:
|
||||
"""Activate a subscription after a successful checkout.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
data: Stripe ``checkout.session`` object.
|
||||
"""
|
||||
meta = data.get("metadata") or {} if isinstance(data, dict) else getattr(data, "metadata", {}) or {}
|
||||
user_id = meta.get("docuelevate_user_id") if isinstance(meta, dict) else getattr(meta, "docuelevate_user_id", None)
|
||||
plan_id = meta.get("plan_id") if isinstance(meta, dict) else getattr(meta, "plan_id", None)
|
||||
billing_cycle = (
|
||||
meta.get("billing_cycle", "monthly") if isinstance(meta, dict) else getattr(meta, "billing_cycle", "monthly")
|
||||
)
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile and plan_id:
|
||||
profile.subscription_tier = plan_id
|
||||
profile.subscription_billing_cycle = billing_cycle
|
||||
profile.subscription_period_start = datetime.now(tz=timezone.utc)
|
||||
customer_id = data.get("customer", "") if isinstance(data, dict) else getattr(data, "customer", "")
|
||||
if customer_id:
|
||||
profile.stripe_customer_id = customer_id
|
||||
db.commit()
|
||||
logger.info("Activated plan %s/%s after checkout", plan_id, billing_cycle)
|
||||
|
||||
|
||||
def _on_subscription_updated(db: Session, data: Any) -> None:
|
||||
"""Sync tier change when a subscription is updated.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
data: Stripe ``customer.subscription`` object.
|
||||
"""
|
||||
customer_id = data.get("customer", "") if isinstance(data, dict) else getattr(data, "customer", "")
|
||||
user_id = _resolve_user_id_from_customer(db, customer_id)
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
items_data = data.get("items") or {} if isinstance(data, dict) else getattr(data, "items", None) or {}
|
||||
items = items_data.get("data") or [] if isinstance(items_data, dict) else getattr(items_data, "data", []) or []
|
||||
if not items:
|
||||
return
|
||||
|
||||
first_item = items[0]
|
||||
price_obj = (
|
||||
first_item.get("price") or {} if isinstance(first_item, dict) else getattr(first_item, "price", {}) or {}
|
||||
)
|
||||
price_id = price_obj.get("id") if isinstance(price_obj, dict) else getattr(price_obj, "id", None)
|
||||
if not price_id:
|
||||
return
|
||||
|
||||
plan_id = _resolve_plan_id_from_price(db, price_id)
|
||||
if not plan_id:
|
||||
logger.warning("Unknown Stripe price_id %s on subscription.updated", price_id)
|
||||
return
|
||||
|
||||
recurring = (
|
||||
price_obj.get("recurring", {}) if isinstance(price_obj, dict) else getattr(price_obj, "recurring", {}) or {}
|
||||
)
|
||||
interval = (
|
||||
recurring.get("interval", "month") if isinstance(recurring, dict) else getattr(recurring, "interval", "month")
|
||||
)
|
||||
billing_cycle = "yearly" if interval == "year" else "monthly"
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile:
|
||||
profile.subscription_tier = plan_id
|
||||
profile.subscription_billing_cycle = billing_cycle
|
||||
db.commit()
|
||||
logger.info("Updated subscription to %s/%s", plan_id, billing_cycle)
|
||||
|
||||
|
||||
def _on_subscription_deleted(db: Session, data: Any) -> None:
|
||||
"""Downgrade user to free tier after subscription cancellation.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
data: Stripe ``customer.subscription`` object.
|
||||
"""
|
||||
customer_id = data.get("customer", "") if isinstance(data, dict) else getattr(data, "customer", "")
|
||||
user_id = _resolve_user_id_from_customer(db, customer_id)
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile:
|
||||
profile.subscription_tier = "free"
|
||||
profile.subscription_billing_cycle = "monthly"
|
||||
db.commit()
|
||||
logger.info("Downgraded user %s to free tier after subscription cancellation", user_id)
|
||||
+18
-46
@@ -1,64 +1,36 @@
|
||||
"""
|
||||
Common utilities for API routes
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
from fastapi import Depends
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.config import settings
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_db():
|
||||
"""Database dependency injection for routes"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def resolve_file_path(file_path: str, subfolder: str = None) -> str:
|
||||
"""
|
||||
Resolves a file path to an absolute path with path traversal protection.
|
||||
Resolves a file path to an absolute path.
|
||||
If the path is not absolute, it will be joined with the workdir path.
|
||||
Optionally, can include a subfolder like 'processed'.
|
||||
|
||||
Security: Validates that the resolved path stays within the workdir
|
||||
to prevent path traversal attacks (e.g., ../../etc/passwd).
|
||||
|
||||
Args:
|
||||
file_path: The file path to resolve
|
||||
subfolder: Optional subfolder within workdir
|
||||
|
||||
Returns:
|
||||
The validated absolute file path
|
||||
|
||||
Raises:
|
||||
HTTPException: If the path attempts to escape the workdir
|
||||
|
||||
Returns the absolute file path.
|
||||
"""
|
||||
# Get the workdir as the security boundary
|
||||
workdir = Path(settings.workdir).resolve()
|
||||
|
||||
# Build the base directory (workdir or workdir/subfolder)
|
||||
if subfolder:
|
||||
base_dir = workdir / subfolder
|
||||
else:
|
||||
base_dir = workdir
|
||||
|
||||
# Resolve the file path
|
||||
if not os.path.isabs(file_path):
|
||||
# Relative path: join with base_dir
|
||||
resolved_path = (base_dir / file_path).resolve()
|
||||
else:
|
||||
# Absolute path: resolve as-is
|
||||
resolved_path = Path(file_path).resolve()
|
||||
|
||||
# Ensure the resolved path is within workdir (path traversal protection)
|
||||
# This checks both relative and absolute paths against workdir
|
||||
try:
|
||||
resolved_path.relative_to(workdir)
|
||||
except ValueError:
|
||||
# Path is outside the workdir - potential path traversal attack
|
||||
logger.warning(f"Path traversal attempt detected: {file_path} -> {resolved_path}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file path: path traversal not allowed"
|
||||
)
|
||||
|
||||
return str(resolved_path)
|
||||
if subfolder:
|
||||
file_path = os.path.join(settings.workdir, subfolder, file_path)
|
||||
else:
|
||||
file_path = os.path.join(settings.workdir, file_path)
|
||||
return file_path
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
"""API endpoints for managing compliance templates (GDPR, HIPAA, SOC2).
|
||||
|
||||
All endpoints require admin privileges.
|
||||
|
||||
Available routes:
|
||||
GET /api/compliance/templates – list all compliance templates
|
||||
GET /api/compliance/templates/{name} – get a single template with checks
|
||||
POST /api/compliance/templates/{name}/apply – one-click apply a template
|
||||
GET /api/compliance/templates/{name}/status – evaluate compliance status
|
||||
GET /api/compliance/summary – overall compliance dashboard data
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.utils.compliance_service import (
|
||||
COMPLIANCE_TEMPLATES,
|
||||
apply_template,
|
||||
evaluate_template_status,
|
||||
get_all_templates,
|
||||
get_compliance_summary,
|
||||
get_template_by_name,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/compliance", tags=["compliance"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authorisation helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin; raises HTTP 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CheckResult(BaseModel):
|
||||
"""Individual compliance check result."""
|
||||
|
||||
key: str
|
||||
label: str
|
||||
description: str
|
||||
expected: str
|
||||
actual: str
|
||||
passing: bool
|
||||
|
||||
|
||||
class TemplateStatusResponse(BaseModel):
|
||||
"""Status evaluation for a compliance template."""
|
||||
|
||||
status: str
|
||||
total: int
|
||||
passed: int
|
||||
failed: int
|
||||
check_results: list[CheckResult]
|
||||
|
||||
|
||||
class TemplateResponse(BaseModel):
|
||||
"""Full compliance template representation."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
display_name: str
|
||||
description: str | None
|
||||
enabled: bool
|
||||
status: str
|
||||
applied_at: str | None
|
||||
applied_by: str | None
|
||||
settings: dict[str, str]
|
||||
checks: list[dict[str, Any]]
|
||||
check_count: int
|
||||
|
||||
|
||||
class ApplyResponse(BaseModel):
|
||||
"""Result of applying a compliance template."""
|
||||
|
||||
success: bool
|
||||
template: str | None = None
|
||||
applied_settings: dict[str, str] | None = None
|
||||
errors: list[str] | None = None
|
||||
error: str | None = None
|
||||
status: TemplateStatusResponse | None = None
|
||||
|
||||
|
||||
class SummaryTemplateResponse(BaseModel):
|
||||
"""Per-template summary for the compliance dashboard."""
|
||||
|
||||
name: str
|
||||
display_name: str
|
||||
enabled: bool
|
||||
status: str
|
||||
total: int
|
||||
passed: int
|
||||
failed: int
|
||||
applied_at: str | None
|
||||
applied_by: str | None
|
||||
|
||||
|
||||
class ComplianceSummaryResponse(BaseModel):
|
||||
"""Overall compliance dashboard summary."""
|
||||
|
||||
overall_status: str
|
||||
total_checks: int
|
||||
total_passed: int
|
||||
total_failed: int
|
||||
templates: list[SummaryTemplateResponse]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/templates", response_model=list[TemplateResponse])
|
||||
async def list_templates(db: DbSession, admin: AdminUser) -> list[dict[str, Any]]:
|
||||
"""List all compliance templates with their current status."""
|
||||
return get_all_templates(db)
|
||||
|
||||
|
||||
@router.get("/templates/{name}", response_model=TemplateResponse)
|
||||
async def get_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
|
||||
"""Get a single compliance template by name."""
|
||||
templates = get_all_templates(db)
|
||||
for t in templates:
|
||||
if t["name"] == name:
|
||||
return t
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
|
||||
|
||||
|
||||
@router.post("/templates/{name}/apply", response_model=ApplyResponse)
|
||||
async def apply_compliance_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
|
||||
"""Apply a compliance template (one-click).
|
||||
|
||||
Writes all template settings to the database and evaluates the resulting
|
||||
compliance status.
|
||||
"""
|
||||
if name not in COMPLIANCE_TEMPLATES:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
|
||||
|
||||
template = get_template_by_name(db, name)
|
||||
if template is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
|
||||
|
||||
admin_email = admin.get("email", "admin")
|
||||
result = apply_template(db, name, applied_by=admin_email)
|
||||
if not result.get("success") and result.get("error"):
|
||||
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=result["error"])
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/templates/{name}/status", response_model=TemplateStatusResponse)
|
||||
async def get_template_status(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
|
||||
"""Evaluate the live compliance status of a template."""
|
||||
template = get_template_by_name(db, name)
|
||||
if template is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
|
||||
return evaluate_template_status(db, name)
|
||||
|
||||
|
||||
@router.get("/summary", response_model=ComplianceSummaryResponse)
|
||||
async def compliance_summary(db: DbSession, admin: AdminUser) -> dict[str, Any]:
|
||||
"""Overall compliance dashboard summary across all templates."""
|
||||
return get_compliance_summary(db)
|
||||
@@ -1,170 +0,0 @@
|
||||
"""
|
||||
API endpoints for the database configuration wizard and migration tool.
|
||||
|
||||
Provides REST endpoints for:
|
||||
- Testing database connections
|
||||
- Building connection strings from form components
|
||||
- Previewing and executing data migrations between databases
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.utils.db_migrate import migrate_data, preview_migration
|
||||
from app.utils.db_wizard import (
|
||||
build_connection_string,
|
||||
get_supported_backends,
|
||||
parse_connection_string,
|
||||
test_connection,
|
||||
validate_url_format,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/database", tags=["database"])
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConnectionStringRequest(BaseModel):
|
||||
"""Request body for building a connection string."""
|
||||
|
||||
backend: str = Field(..., description="Database backend: sqlite, postgresql, mysql")
|
||||
host: str = Field("", description="Database server hostname")
|
||||
port: int | None = Field(None, description="Database server port")
|
||||
database: str = Field("", description="Database name")
|
||||
username: str = Field("", description="Authentication username")
|
||||
password: str = Field("", description="Authentication password")
|
||||
ssl_mode: str = Field("", description="SSL mode (e.g. require, verify-full)")
|
||||
extra_options: str = Field("", description="Additional query-string options")
|
||||
sqlite_path: str = Field("", description="File path for SQLite databases")
|
||||
|
||||
|
||||
class TestConnectionRequest(BaseModel):
|
||||
"""Request body for testing a database connection."""
|
||||
|
||||
url: str = Field(..., description="Full SQLAlchemy connection URL to test")
|
||||
|
||||
|
||||
class MigrateRequest(BaseModel):
|
||||
"""Request body for data migration."""
|
||||
|
||||
source_url: str = Field(..., description="Source database connection URL")
|
||||
target_url: str = Field(..., description="Target database connection URL")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/backends")
|
||||
async def list_backends() -> list[dict]:
|
||||
"""List all supported database backends with metadata."""
|
||||
return get_supported_backends()
|
||||
|
||||
|
||||
@router.post("/build-url")
|
||||
async def build_url(body: ConnectionStringRequest, request: Request) -> dict:
|
||||
"""Build a SQLAlchemy connection string from individual components.
|
||||
|
||||
Returns the assembled URL string.
|
||||
"""
|
||||
_require_admin(request)
|
||||
try:
|
||||
url = build_connection_string(
|
||||
backend=body.backend,
|
||||
host=body.host,
|
||||
port=body.port,
|
||||
database=body.database,
|
||||
username=body.username,
|
||||
password=body.password,
|
||||
ssl_mode=body.ssl_mode,
|
||||
extra_options=body.extra_options,
|
||||
sqlite_path=body.sqlite_path,
|
||||
)
|
||||
return {"url": url}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/parse-url")
|
||||
async def parse_url(body: TestConnectionRequest, request: Request) -> dict:
|
||||
"""Parse a connection string into its components."""
|
||||
_require_admin(request)
|
||||
return parse_connection_string(body.url)
|
||||
|
||||
|
||||
@router.post("/validate-url")
|
||||
async def validate_url(body: TestConnectionRequest, request: Request) -> dict:
|
||||
"""Validate a connection string format without connecting."""
|
||||
_require_admin(request)
|
||||
return validate_url_format(body.url)
|
||||
|
||||
|
||||
@router.post("/test-connection")
|
||||
async def test_db_connection(body: TestConnectionRequest, request: Request) -> dict:
|
||||
"""Test connectivity to a database and return status info.
|
||||
|
||||
This creates a temporary engine, executes ``SELECT 1``, and disposes
|
||||
of the engine. It does **not** modify any global application state.
|
||||
"""
|
||||
_require_admin(request)
|
||||
return test_connection(body.url)
|
||||
|
||||
|
||||
@router.post("/preview-migration")
|
||||
async def preview_db_migration(body: TestConnectionRequest, request: Request) -> dict:
|
||||
"""Preview what a migration from the given source would include.
|
||||
|
||||
Returns a table-by-table row count without actually copying data.
|
||||
"""
|
||||
_require_admin(request)
|
||||
return preview_migration(body.url)
|
||||
|
||||
|
||||
@router.post("/migrate")
|
||||
async def execute_migration(body: MigrateRequest, request: Request) -> dict:
|
||||
"""Execute a full data migration from source to target database.
|
||||
|
||||
**Warning:** This copies all data from the source database into the
|
||||
target. The target schema is created from the current application
|
||||
models. Existing data in the target is **not** deleted first — use
|
||||
on an empty target database.
|
||||
"""
|
||||
_require_admin(request)
|
||||
|
||||
# Validate both URLs first
|
||||
src_check = validate_url_format(body.source_url)
|
||||
if not src_check.get("valid"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid source URL: {src_check.get('error', 'unknown')}",
|
||||
)
|
||||
tgt_check = validate_url_format(body.target_url)
|
||||
if not tgt_check.get("valid"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid target URL: {tgt_check.get('error', 'unknown')}",
|
||||
)
|
||||
|
||||
result = migrate_data(body.source_url, body.target_url)
|
||||
if not result["success"]:
|
||||
error_summary = "; ".join(result.get("errors", ["Unknown error"]))
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Migration completed with errors: {error_summary}",
|
||||
)
|
||||
return result
|
||||
+50
-109
@@ -1,159 +1,100 @@
|
||||
"""
|
||||
Diagnostic API endpoints
|
||||
"""
|
||||
|
||||
import datetime
|
||||
from fastapi import APIRouter, Request, Depends
|
||||
import logging
|
||||
|
||||
import redis as redis_lib
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.auth import require_login
|
||||
from app.auth import require_login, get_current_user
|
||||
from app.config import settings
|
||||
from app.database import engine
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_REDIS_URL = "redis://localhost:6379/0"
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/diagnostic/health")
|
||||
@router.get("/diagnostic/settings")
|
||||
@require_login
|
||||
async def health_check(request: Request):
|
||||
async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)):
|
||||
"""
|
||||
System health endpoint for monitoring tools (Grafana, Uptime Kuma, etc.).
|
||||
|
||||
Checks database connectivity and Redis availability and returns a
|
||||
machine-readable summary that monitoring systems can scrape.
|
||||
|
||||
**Authentication:** Required (no-op when AUTH_ENABLED=False)
|
||||
|
||||
**Response (200 OK) – all subsystems healthy:**
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"version": "1.2.3",
|
||||
"timestamp": "2024-01-15T10:30:00+00:00",
|
||||
"checks": {
|
||||
"database": {"status": "ok"},
|
||||
"redis": {"status": "ok"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200 OK) – one or more subsystems degraded:**
|
||||
```json
|
||||
{
|
||||
"status": "degraded",
|
||||
"version": "1.2.3",
|
||||
"timestamp": "2024-01-15T10:30:00+00:00",
|
||||
"checks": {
|
||||
"database": {"status": "ok"},
|
||||
"redis": {"status": "error", "detail": "Connection refused"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The outer ``status`` field is always one of:
|
||||
- ``"healthy"`` – all checks passed
|
||||
- ``"degraded"`` – at least one non-critical check failed
|
||||
- ``"unhealthy"`` – a critical check failed (currently: database)
|
||||
API endpoint to dump settings to the log and view basic config information
|
||||
This endpoint doesn't expose sensitive information like passwords or tokens
|
||||
"""
|
||||
timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
checks: dict[str, dict[str, str]] = {}
|
||||
|
||||
# ── Database check ─────────────────────────────────────────────────────
|
||||
db_ok = False
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
checks["database"] = {"status": "ok"}
|
||||
db_ok = True
|
||||
except Exception as exc:
|
||||
logger.warning("Health check: database probe failed: %s", exc)
|
||||
checks["database"] = {"status": "error", "detail": str(exc)}
|
||||
|
||||
# ── Redis check ────────────────────────────────────────────────────────
|
||||
try:
|
||||
redis_url = settings.redis_url or _DEFAULT_REDIS_URL
|
||||
r = redis_lib.from_url(redis_url, socket_connect_timeout=2, socket_timeout=2)
|
||||
r.ping()
|
||||
checks["redis"] = {"status": "ok"}
|
||||
except Exception as exc:
|
||||
logger.warning("Health check: Redis probe failed: %s", exc)
|
||||
checks["redis"] = {"status": "error", "detail": str(exc)}
|
||||
|
||||
# ── Overall status ─────────────────────────────────────────────────────
|
||||
if not db_ok:
|
||||
overall = "unhealthy"
|
||||
elif any(v.get("status") != "ok" for v in checks.values()):
|
||||
overall = "degraded"
|
||||
else:
|
||||
overall = "healthy"
|
||||
|
||||
http_status = 503 if overall == "unhealthy" else 200
|
||||
|
||||
payload = {
|
||||
"status": overall,
|
||||
"version": settings.version,
|
||||
"timestamp": timestamp,
|
||||
"checks": checks,
|
||||
from app.utils.config_validator import dump_all_settings, get_settings_for_display
|
||||
# Dump full settings to log for admin to see
|
||||
dump_all_settings()
|
||||
|
||||
# Return safe subset of settings for API response
|
||||
safe_settings = {
|
||||
"workdir": settings.workdir,
|
||||
"external_hostname": settings.external_hostname,
|
||||
"configured_services": {
|
||||
"email": bool(getattr(settings, 'email_host', None)),
|
||||
"s3": bool(getattr(settings, 's3_bucket_name', None)),
|
||||
"dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
|
||||
"onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)),
|
||||
"nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
|
||||
"sftp": bool(getattr(settings, 'sftp_host', None)),
|
||||
"paperless": bool(getattr(settings, 'paperless_host', None)),
|
||||
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
|
||||
"uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
|
||||
"auth": bool(getattr(settings, 'authentik_config_url', None)),
|
||||
"openai": bool(getattr(settings, 'openai_api_key', None)),
|
||||
"azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)),
|
||||
},
|
||||
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"settings": safe_settings,
|
||||
"message": "Full settings have been dumped to application logs"
|
||||
}
|
||||
|
||||
return JSONResponse(content=payload, status_code=http_status)
|
||||
|
||||
|
||||
@router.post("/diagnostic/test-notification")
|
||||
@require_login
|
||||
async def test_notification(request: Request):
|
||||
# Add request_time to request.state
|
||||
import datetime
|
||||
|
||||
request.state.request_time = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
request.state.request_time = datetime.datetime.utcnow().isoformat()
|
||||
"""
|
||||
Send a test notification through all configured notification channels
|
||||
"""
|
||||
from app.utils.notification import send_notification
|
||||
|
||||
|
||||
try:
|
||||
notification_urls = getattr(settings, "notification_urls", [])
|
||||
notification_urls = getattr(settings, 'notification_urls', [])
|
||||
if not notification_urls:
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "No notification services configured. Add notification URLs to your configuration.",
|
||||
"message": "No notification services configured. Add notification URLs to your configuration."
|
||||
}
|
||||
|
||||
|
||||
# Send a test notification
|
||||
hostname = settings.external_hostname or "Document Processor"
|
||||
result = send_notification(
|
||||
title=f"Test Notification from {hostname}",
|
||||
message=(
|
||||
f"This is a test notification sent at {request.state.request_time}. "
|
||||
"If you're receiving this, notifications are working!"
|
||||
),
|
||||
message=f"This is a test notification sent at {request.state.request_time}. If you're receiving this, notifications are working!",
|
||||
notification_type="success",
|
||||
tags=["test", "notification", "diagnostic"],
|
||||
tags=["test", "notification", "diagnostic"]
|
||||
)
|
||||
|
||||
|
||||
if result:
|
||||
logger.info("Test notification sent successfully")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Test notification sent successfully to {len(notification_urls)} service(s)",
|
||||
"services_count": len(notification_urls),
|
||||
"services_count": len(notification_urls)
|
||||
}
|
||||
else:
|
||||
logger.warning("Test notification send attempt returned False")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Failed to send test notification. Check application logs for details.",
|
||||
"message": "Failed to send test notification. Check application logs for details."
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error sending test notification: {e}")
|
||||
return {"status": "error", "message": f"Error sending notification: {str(e)}"}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Error sending notification: {str(e)}"
|
||||
}
|
||||
|
||||
+209
-170
@@ -1,121 +1,161 @@
|
||||
"""
|
||||
Dropbox API endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Request, HTTPException, status, Form
|
||||
import logging
|
||||
import os
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/dropbox/exchange-token")
|
||||
@require_login
|
||||
async def exchange_dropbox_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
client_id: str = Form(...),
|
||||
client_secret: str = Form(...),
|
||||
redirect_uri: str = Form(...),
|
||||
code: str = Form(...),
|
||||
folder_path: str = Form(None)
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for a refresh token from Dropbox.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = "https://api.dropboxapi.com/oauth2/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="Dropbox", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 14400),
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info("Starting Dropbox token exchange process")
|
||||
|
||||
# Prepare the token request
|
||||
token_url = "https://api.dropboxapi.com/oauth2/token"
|
||||
|
||||
payload = {
|
||||
'client_id': client_id,
|
||||
'client_secret': client_secret,
|
||||
'code': code,
|
||||
'redirect_uri': redirect_uri,
|
||||
'grant_type': 'authorization_code'
|
||||
}
|
||||
|
||||
# Log request details (excluding secret)
|
||||
safe_payload = payload.copy()
|
||||
safe_payload['client_secret'] = '[REDACTED]'
|
||||
safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
|
||||
logger.info(f"Token exchange request payload: {safe_payload}")
|
||||
|
||||
# Make the token request
|
||||
logger.info("Sending POST request to Dropbox for token exchange")
|
||||
response = requests.post(token_url, data=payload)
|
||||
|
||||
# Check if the request was successful
|
||||
logger.info(f"Token exchange response status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
# Log the error response for debugging
|
||||
try:
|
||||
error_json = response.json()
|
||||
logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
|
||||
error_detail = error_json
|
||||
except Exception as json_err:
|
||||
logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
|
||||
logger.error(f"Raw response content: {response.content[:500]}") # Limit log size
|
||||
error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
|
||||
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Token exchange failed: {error_detail}"
|
||||
)
|
||||
|
||||
# Return the token response
|
||||
token_data = response.json()
|
||||
|
||||
# Validate the token response
|
||||
if "refresh_token" not in token_data:
|
||||
logger.error(f"Dropbox returned success but no refresh token found in response: {token_data.keys()}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="Dropbox OAuth server returned success but no refresh token was included"
|
||||
)
|
||||
|
||||
# Calculate token length for logging
|
||||
refresh_token_length = len(token_data.get("refresh_token", ""))
|
||||
access_token_length = len(token_data.get("access_token", ""))
|
||||
|
||||
logger.info(f"Successfully exchanged authorization code for Dropbox tokens. "
|
||||
f"Refresh token length: {refresh_token_length}, "
|
||||
f"Access token length: {access_token_length}")
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 14400)
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions as they already have appropriate status codes
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error during Dropbox token exchange: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to exchange token: {str(e)}"
|
||||
)
|
||||
|
||||
@router.post("/dropbox/update-settings")
|
||||
@require_login
|
||||
async def update_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
app_key: Annotated[Optional[str], Form()] = None,
|
||||
app_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
app_key: str = Form(None),
|
||||
app_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
folder_path: str = Form(None)
|
||||
):
|
||||
"""
|
||||
Update Dropbox settings in memory and persist to the database.
|
||||
Update Dropbox settings in memory
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating Dropbox settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
logger.info("Updating Dropbox settings in memory")
|
||||
|
||||
# Update settings in memory
|
||||
if refresh_token:
|
||||
settings.dropbox_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_REFRESH_TOKEN in memory and database")
|
||||
|
||||
logger.info("Updated DROPBOX_REFRESH_TOKEN in memory")
|
||||
|
||||
if app_key:
|
||||
settings.dropbox_app_key = app_key
|
||||
save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_APP_KEY in memory and database")
|
||||
|
||||
logger.info("Updated DROPBOX_APP_KEY in memory")
|
||||
|
||||
if app_secret:
|
||||
settings.dropbox_app_secret = app_secret
|
||||
save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_APP_SECRET in memory and database")
|
||||
|
||||
logger.info("Updated DROPBOX_APP_SECRET in memory")
|
||||
|
||||
if folder_path:
|
||||
settings.dropbox_folder = folder_path
|
||||
save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_FOLDER in memory and database")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Updated DROPBOX_FOLDER in memory")
|
||||
|
||||
# Test token validity would be here, but we'll skip it for now
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Dropbox settings have been updated in memory and saved to database",
|
||||
"message": "Dropbox settings have been updated in memory"
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update Dropbox settings: {str(e)}",
|
||||
detail=f"Failed to update Dropbox settings: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/dropbox/test-token")
|
||||
@require_login
|
||||
async def test_dropbox_token(request: Request):
|
||||
@@ -124,109 +164,158 @@ async def test_dropbox_token(request: Request):
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing Dropbox token validity")
|
||||
|
||||
|
||||
if not settings.dropbox_refresh_token or not settings.dropbox_app_key or not settings.dropbox_app_secret:
|
||||
logger.warning("Dropbox credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Dropbox credentials are not fully configured",
|
||||
"status": "error",
|
||||
"message": "Dropbox credentials are not fully configured"
|
||||
}
|
||||
|
||||
|
||||
# Check token validity by getting current account info
|
||||
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
|
||||
response = requests.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
headers=headers
|
||||
)
|
||||
|
||||
|
||||
# If token is invalid, try refreshing it
|
||||
if response.status_code == 401:
|
||||
logger.info("Dropbox access token invalid or expired, trying to refresh")
|
||||
|
||||
|
||||
# Get a new access token using the refresh token
|
||||
refresh_url = "https://api.dropbox.com/oauth2/token"
|
||||
refresh_data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
"client_secret": settings.dropbox_app_secret
|
||||
}
|
||||
|
||||
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
|
||||
|
||||
refresh_response = requests.post(refresh_url, data=refresh_data)
|
||||
|
||||
if refresh_response.status_code != 200:
|
||||
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
"needs_reauth": True
|
||||
}
|
||||
|
||||
|
||||
token_info = refresh_response.json()
|
||||
access_token = token_info.get("access_token")
|
||||
|
||||
|
||||
# Try again with the new access token
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = requests.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
headers=headers
|
||||
)
|
||||
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {response.status_code}: {response.text}",
|
||||
"message": f"Token validation failed with status {response.status_code}: {response.text}"
|
||||
}
|
||||
|
||||
|
||||
# Get account info
|
||||
account_info = response.json()
|
||||
account_email = account_info.get("email", "Unknown account")
|
||||
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
|
||||
|
||||
|
||||
# Dropbox refresh tokens don't expire, but we should note that in our response
|
||||
token_info = {
|
||||
"expires_in_human": "Never expires (perpetual token)",
|
||||
"is_perpetual": True,
|
||||
"is_perpetual": True
|
||||
}
|
||||
|
||||
|
||||
logger.info(f"Successfully connected to Dropbox as {account_email}")
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Dropbox connection successful",
|
||||
"message": f"Dropbox connection successful",
|
||||
"account": account_email,
|
||||
"account_name": account_name,
|
||||
"token_info": token_info,
|
||||
"token_info": token_info
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error testing Dropbox token: {str(e)}")
|
||||
return {"status": "error", "message": f"Connection error: {str(e)}"}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection error: {str(e)}"
|
||||
}
|
||||
|
||||
@router.post("/dropbox/save-settings")
|
||||
@require_login
|
||||
async def save_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
app_key: Annotated[Optional[str], Form()] = None,
|
||||
app_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
app_key: str = Form(None),
|
||||
app_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
folder_path: str = Form(None)
|
||||
):
|
||||
"""
|
||||
Save Dropbox settings to database (primary) and .env file (best-effort).
|
||||
Save Dropbox settings to the .env file
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory
|
||||
# Get the path to the .env file
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
|
||||
if not os.path.exists(env_path):
|
||||
logger.error(f".env file not found at {env_path}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Could not find .env file to update"
|
||||
)
|
||||
|
||||
logger.info(f"Updating Dropbox settings in {env_path}")
|
||||
|
||||
# Read the current .env file
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
# Define settings to update
|
||||
dropbox_settings = {
|
||||
"DROPBOX_REFRESH_TOKEN": refresh_token,
|
||||
}
|
||||
|
||||
# Only update these if provided
|
||||
if app_key:
|
||||
dropbox_settings["DROPBOX_APP_KEY"] = app_key
|
||||
if app_secret:
|
||||
dropbox_settings["DROPBOX_APP_SECRET"] = app_secret
|
||||
if folder_path:
|
||||
dropbox_settings["DROPBOX_FOLDER"] = folder_path
|
||||
|
||||
# Process each line and update or add settings
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in dropbox_settings.items():
|
||||
if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
|
||||
if line.startswith("# "): # Uncomment if commented out
|
||||
line = line[2:]
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(line)
|
||||
|
||||
# Add any settings that weren't updated (they weren't in the file)
|
||||
for key, value in dropbox_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
# Write the updated .env file
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
# Update the settings in memory
|
||||
if refresh_token:
|
||||
settings.dropbox_refresh_token = refresh_token
|
||||
if app_key:
|
||||
@@ -235,69 +324,19 @@ async def save_dropbox_settings(
|
||||
settings.dropbox_app_secret = app_secret
|
||||
if folder_path:
|
||||
settings.dropbox_folder = folder_path
|
||||
|
||||
# Persist to database (primary storage)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if app_key:
|
||||
save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by)
|
||||
if app_secret:
|
||||
save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by)
|
||||
if folder_path:
|
||||
save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by)
|
||||
|
||||
# Best-effort .env file write
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if not os.path.exists(env_path):
|
||||
logger.warning(f".env file not found at {env_path}, skipping file write")
|
||||
else:
|
||||
logger.info(f"Updating Dropbox settings in {env_path}")
|
||||
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
dropbox_settings = {"DROPBOX_REFRESH_TOKEN": refresh_token}
|
||||
if app_key:
|
||||
dropbox_settings["DROPBOX_APP_KEY"] = app_key
|
||||
if app_secret:
|
||||
dropbox_settings["DROPBOX_APP_SECRET"] = app_secret
|
||||
if folder_path:
|
||||
dropbox_settings["DROPBOX_FOLDER"] = folder_path
|
||||
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in dropbox_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
for key, value in dropbox_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated Dropbox settings in .env file")
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully saved Dropbox settings")
|
||||
return {"status": "success", "message": "Dropbox settings have been saved"}
|
||||
|
||||
|
||||
logger.info("Successfully updated Dropbox settings")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Dropbox settings have been saved"
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save Dropbox settings: {str(e)}",
|
||||
detail=f"Failed to save Dropbox settings: {str(e)}"
|
||||
)
|
||||
|
||||
@@ -1,230 +0,0 @@
|
||||
"""Duplicate document detection and management API endpoints.
|
||||
|
||||
Provides endpoints for listing all duplicate groups (exact SHA-256 duplicates) and
|
||||
for retrieving both exact and near-duplicate matches for a specific document.
|
||||
|
||||
Near-duplicate detection is powered by the same text-embedding cosine-similarity
|
||||
engine used by the ``/api/files/{id}/similar`` endpoint
|
||||
(see ``app/utils/similarity.py``).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.get("/duplicates")
|
||||
@require_login
|
||||
def list_duplicate_groups(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
per_page: int = Query(25, ge=1, le=200, description="Items per page"),
|
||||
):
|
||||
"""List all groups of exact-duplicate documents (same SHA-256 hash).
|
||||
|
||||
Returns one entry per duplicate group showing the original document and all
|
||||
files that were detected as copies of it. Groups are sorted by descending
|
||||
duplicate count.
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/duplicates
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"groups": [
|
||||
{
|
||||
"filehash": "abc123...",
|
||||
"original": {"id": 1, "original_filename": "invoice.pdf", ...},
|
||||
"duplicates": [{"id": 5, "original_filename": "invoice_copy.pdf", ...}],
|
||||
"duplicate_count": 1
|
||||
}
|
||||
],
|
||||
"total_groups": 1,
|
||||
"total_duplicate_files": 1,
|
||||
"pagination": {...}
|
||||
}
|
||||
```
|
||||
"""
|
||||
# Find all hashes that have at least one duplicate record
|
||||
dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct()
|
||||
total_groups = dup_hashes_query.count()
|
||||
|
||||
# Paginate hash groups
|
||||
offset = (page - 1) * per_page
|
||||
dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()]
|
||||
|
||||
groups = []
|
||||
total_duplicate_files = 0
|
||||
|
||||
for filehash in dup_hashes:
|
||||
# Find the original (non-duplicate) record with this hash
|
||||
original = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
|
||||
# Find all duplicate records for this hash
|
||||
duplicates = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
total_duplicate_files += len(duplicates)
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"filehash": filehash,
|
||||
"original": _file_record_to_dict(original) if original else None,
|
||||
"duplicates": [_file_record_to_dict(d) for d in duplicates],
|
||||
"duplicate_count": len(duplicates),
|
||||
}
|
||||
)
|
||||
|
||||
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
|
||||
|
||||
return {
|
||||
"groups": groups,
|
||||
"total_groups": total_groups,
|
||||
"total_duplicate_files": total_duplicate_files,
|
||||
"pagination": {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total_groups,
|
||||
"pages": total_pages,
|
||||
"next": str(request.url.include_query_params(page=page + 1)) if page < total_pages else None,
|
||||
"previous": str(request.url.include_query_params(page=page - 1)) if page > 1 else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/duplicates")
|
||||
@require_login
|
||||
def get_file_duplicates(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
near_duplicate_limit: int = Query(5, ge=1, le=20, description="Maximum near-duplicates to return"),
|
||||
near_duplicate_threshold: float = Query(
|
||||
-1.0,
|
||||
ge=-1.0,
|
||||
le=1.0,
|
||||
description="Minimum similarity score for near-duplicates; -1 uses the configured default",
|
||||
),
|
||||
):
|
||||
"""Get exact and near-duplicate documents for the specified file.
|
||||
|
||||
**Exact duplicates** share the same SHA-256 hash.
|
||||
**Near-duplicates** have a text-embedding cosine similarity score ≥
|
||||
``NEAR_DUPLICATE_THRESHOLD`` (configurable; default 0.85).
|
||||
|
||||
Near-duplicate detection requires OCR text to be available for both the
|
||||
target file and candidate files. Files without OCR text are excluded.
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/files/42/duplicates
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"file_id": 42,
|
||||
"exact_duplicates": [
|
||||
{"id": 7, "original_filename": "invoice.pdf", "is_duplicate": true, "duplicate_of_id": 42, ...}
|
||||
],
|
||||
"near_duplicates": [
|
||||
{"file_id": 15, "original_filename": "invoice_jan.pdf", "similarity_score": 0.92, ...}
|
||||
],
|
||||
"near_duplicate_threshold": 0.85
|
||||
}
|
||||
```
|
||||
"""
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
# --- Exact duplicates ---
|
||||
# Case 1: This file is the original — find all records that are duplicates of it
|
||||
exact_duplicates_of_this = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == file_record.filehash, FileRecord.id != file_id)
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Case 2: This file itself is a duplicate — find the original
|
||||
is_self_duplicate = file_record.is_duplicate
|
||||
duplicate_of_original: FileRecord | None = None
|
||||
if is_self_duplicate and file_record.duplicate_of_id:
|
||||
duplicate_of_original = db.query(FileRecord).filter(FileRecord.id == file_record.duplicate_of_id).first()
|
||||
|
||||
exact_duplicate_dicts = [_file_record_to_dict(f) for f in exact_duplicates_of_this]
|
||||
|
||||
# --- Near-duplicates (embedding-based) ---
|
||||
effective_threshold = (
|
||||
near_duplicate_threshold if near_duplicate_threshold >= 0.0 else settings.near_duplicate_threshold
|
||||
)
|
||||
|
||||
near_duplicates: list[dict] = []
|
||||
if file_record.ocr_text and file_record.ocr_text.strip():
|
||||
try:
|
||||
from app.utils.similarity import find_similar_documents
|
||||
|
||||
near_duplicates = find_similar_documents(
|
||||
db,
|
||||
file_id,
|
||||
limit=near_duplicate_limit,
|
||||
threshold=effective_threshold,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Near-duplicate detection failed for file {file_id}: {e}")
|
||||
near_duplicates = []
|
||||
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"is_duplicate": is_self_duplicate,
|
||||
"duplicate_of": _file_record_to_dict(duplicate_of_original) if duplicate_of_original else None,
|
||||
"exact_duplicates": exact_duplicate_dicts,
|
||||
"near_duplicates": near_duplicates,
|
||||
"near_duplicate_threshold": effective_threshold,
|
||||
"exact_duplicate_count": len(exact_duplicate_dicts),
|
||||
"near_duplicate_count": len(near_duplicates),
|
||||
}
|
||||
|
||||
|
||||
def _file_record_to_dict(file_record: FileRecord | None) -> dict | None:
|
||||
"""Serialise a ``FileRecord`` to a plain dict for JSON responses."""
|
||||
if file_record is None:
|
||||
return None
|
||||
return {
|
||||
"id": file_record.id,
|
||||
"original_filename": file_record.original_filename,
|
||||
"filehash": file_record.filehash,
|
||||
"file_size": file_record.file_size,
|
||||
"mime_type": file_record.mime_type,
|
||||
"is_duplicate": file_record.is_duplicate,
|
||||
"duplicate_of_id": file_record.duplicate_of_id,
|
||||
"document_title": file_record.document_title,
|
||||
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
|
||||
}
|
||||
+116
-1544
File diff suppressed because it is too large
Load Diff
+218
-204
@@ -1,135 +1,167 @@
|
||||
"""
|
||||
Google Drive API endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Request, HTTPException, status, Form
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
import requests
|
||||
import json
|
||||
from typing import Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/google-drive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_google_drive_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
client_id: str = Form(...),
|
||||
client_secret: str = Form(...),
|
||||
redirect_uri: str = Form(...),
|
||||
code: str = Form(...),
|
||||
folder_id: Optional[str] = Form(None)
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for refresh and access tokens from Google.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = "https://oauth2.googleapis.com/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="Google Drive", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600),
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info("Starting Google Drive token exchange process")
|
||||
|
||||
# Prepare the token request
|
||||
token_url = "https://oauth2.googleapis.com/token"
|
||||
|
||||
payload = {
|
||||
'client_id': client_id,
|
||||
'client_secret': client_secret,
|
||||
'code': code,
|
||||
'redirect_uri': redirect_uri,
|
||||
'grant_type': 'authorization_code'
|
||||
}
|
||||
|
||||
# Log request details (excluding secret)
|
||||
safe_payload = payload.copy()
|
||||
safe_payload['client_secret'] = '[REDACTED]'
|
||||
safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
|
||||
logger.info(f"Token exchange request payload: {safe_payload}")
|
||||
|
||||
# Make the token request
|
||||
logger.info("Sending POST request to Google for token exchange")
|
||||
response = requests.post(token_url, data=payload)
|
||||
|
||||
# Check if the request was successful
|
||||
logger.info(f"Token exchange response status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
# Log the error response for debugging
|
||||
try:
|
||||
error_json = response.json()
|
||||
logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
|
||||
error_detail = error_json
|
||||
except Exception as json_err:
|
||||
logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
|
||||
logger.error(f"Raw response content: {response.content[:500]}") # Limit log size
|
||||
error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
|
||||
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Token exchange failed: {error_detail}"
|
||||
)
|
||||
|
||||
# Return the token response
|
||||
token_data = response.json()
|
||||
|
||||
# Validate the token response
|
||||
if "refresh_token" not in token_data:
|
||||
logger.error(f"Google returned success but no refresh token found in response: {token_data.keys()}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="Google OAuth server returned success but no refresh token was included"
|
||||
)
|
||||
|
||||
# Calculate token length for logging
|
||||
refresh_token_length = len(token_data.get("refresh_token", ""))
|
||||
access_token_length = len(token_data.get("access_token", ""))
|
||||
|
||||
logger.info(f"Successfully exchanged authorization code for Google Drive tokens. "
|
||||
f"Refresh token length: {refresh_token_length}, "
|
||||
f"Access token length: {access_token_length}")
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600)
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions as they already have appropriate status codes
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error during Google Drive token exchange: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to exchange token: {str(e)}"
|
||||
)
|
||||
|
||||
@router.post("/google-drive/update-settings")
|
||||
@require_login
|
||||
async def update_google_drive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
use_oauth: Annotated[str, Form()] = "true",
|
||||
db: Session = Depends(get_db),
|
||||
client_id: str = Form(None),
|
||||
client_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
folder_id: str = Form(None),
|
||||
use_oauth: str = Form("true")
|
||||
):
|
||||
"""
|
||||
Update Google Drive settings in memory and persist to database
|
||||
Update Google Drive settings in memory
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating Google Drive settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
logger.info("Updating Google Drive settings in memory")
|
||||
|
||||
# Convert use_oauth string to boolean
|
||||
use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
|
||||
# Update settings in memory
|
||||
if refresh_token:
|
||||
settings.google_drive_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "google_drive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory and database")
|
||||
|
||||
logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory")
|
||||
|
||||
if client_id:
|
||||
settings.google_drive_client_id = client_id
|
||||
save_setting_to_db(db, "google_drive_client_id", client_id, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory and database")
|
||||
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory")
|
||||
|
||||
if client_secret:
|
||||
settings.google_drive_client_secret = client_secret
|
||||
save_setting_to_db(db, "google_drive_client_secret", client_secret, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory and database")
|
||||
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory")
|
||||
|
||||
if folder_id:
|
||||
settings.google_drive_folder_id = folder_id
|
||||
save_setting_to_db(db, "google_drive_folder_id", folder_id, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory and database")
|
||||
|
||||
logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory")
|
||||
|
||||
# Set the OAuth flag
|
||||
settings.google_drive_use_oauth = use_oauth_bool
|
||||
save_setting_to_db(
|
||||
db,
|
||||
"google_drive_use_oauth",
|
||||
str(use_oauth_bool).lower(),
|
||||
changed_by=changed_by,
|
||||
)
|
||||
logger.info(f"Updated GOOGLE_DRIVE_USE_OAUTH in memory and database to {use_oauth_bool}")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info(f"Updated GOOGLE_DRIVE_USE_OAUTH in memory to {use_oauth_bool}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Google Drive settings have been updated in memory and database",
|
||||
"message": "Google Drive settings have been updated in memory"
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating Google Drive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update Google Drive settings: {str(e)}",
|
||||
detail=f"Failed to update Google Drive settings: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/google-drive/test-token")
|
||||
@require_login
|
||||
async def test_google_drive_token(request: Request):
|
||||
@@ -139,121 +171,124 @@ async def test_google_drive_token(request: Request):
|
||||
"""
|
||||
try:
|
||||
from app.tasks.upload_to_google_drive import get_drive_service_oauth, get_google_drive_service
|
||||
|
||||
|
||||
logger.info("Testing Google Drive token validity")
|
||||
|
||||
|
||||
# Check if OAuth is enabled and configured
|
||||
if getattr(settings, "google_drive_use_oauth", False):
|
||||
if not (
|
||||
settings.google_drive_client_id
|
||||
and settings.google_drive_client_secret
|
||||
and settings.google_drive_refresh_token
|
||||
):
|
||||
if getattr(settings, 'google_drive_use_oauth', False):
|
||||
if not (settings.google_drive_client_id and
|
||||
settings.google_drive_client_secret and
|
||||
settings.google_drive_refresh_token):
|
||||
logger.warning("Google Drive OAuth credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured",
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured"
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
# Test OAuth connection
|
||||
service = get_drive_service_oauth()
|
||||
|
||||
|
||||
# Get credentials for checking token validity
|
||||
import google.oauth2.credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
|
||||
credentials = google.oauth2.credentials.Credentials(
|
||||
token=None,
|
||||
refresh_token=settings.google_drive_refresh_token,
|
||||
token_uri="https://oauth2.googleapis.com/token",
|
||||
client_id=settings.google_drive_client_id,
|
||||
client_secret=settings.google_drive_client_secret,
|
||||
client_secret=settings.google_drive_client_secret
|
||||
)
|
||||
|
||||
|
||||
# Force a refresh to update the token expiration
|
||||
if not credentials.valid:
|
||||
credentials.refresh(Request())
|
||||
|
||||
|
||||
# Get token expiration info
|
||||
expiration_info = {}
|
||||
if hasattr(credentials, "expiry") and credentials.expiry:
|
||||
if hasattr(credentials, 'expiry') and credentials.expiry:
|
||||
now = datetime.now()
|
||||
expiry = credentials.expiry
|
||||
time_left = expiry - now
|
||||
expiration_info = {
|
||||
"expires_at": expiry.isoformat(),
|
||||
"expires_in_seconds": max(0, int(time_left.total_seconds())),
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
"expires_in_human": format_time_remaining(time_left)
|
||||
}
|
||||
|
||||
|
||||
# Test basic API operation
|
||||
about = service.about().get(fields="user").execute()
|
||||
user_email = about.get("user", {}).get("emailAddress", "Unknown")
|
||||
|
||||
|
||||
logger.info(f"Successfully connected to Google Drive as {user_email}")
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"OAuth token is valid! Connected as {user_email}",
|
||||
"account": user_email,
|
||||
"auth_type": "oauth",
|
||||
"token_info": expiration_info,
|
||||
"token_info": expiration_info
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Google Drive OAuth token test failed: {error_msg}")
|
||||
|
||||
|
||||
# Check if this is a token-related error
|
||||
if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"OAuth token validation failed: {error_msg}",
|
||||
"needs_reauth": True,
|
||||
"needs_reauth": True
|
||||
}
|
||||
return {"status": "error", "message": f"Connection error: {error_msg}"}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection error: {error_msg}"
|
||||
}
|
||||
else:
|
||||
# Test service account connection
|
||||
if not settings.google_drive_credentials_json:
|
||||
logger.warning("Google Drive service account credentials not configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive service account credentials are not configured",
|
||||
"status": "error",
|
||||
"message": "Google Drive service account credentials are not configured"
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
service = get_google_drive_service()
|
||||
about = service.about().get(fields="user").execute()
|
||||
|
||||
|
||||
# For service accounts, try to show the delegated user if available
|
||||
user_email = about.get("user", {}).get("emailAddress", "Unknown")
|
||||
delegated_user = getattr(settings, "google_drive_delegate_to", None)
|
||||
|
||||
delegated_user = getattr(settings, 'google_drive_delegate_to', None)
|
||||
|
||||
if delegated_user:
|
||||
user_display = f"{user_email} (delegating as {delegated_user})"
|
||||
else:
|
||||
user_display = user_email
|
||||
|
||||
|
||||
logger.info(f"Successfully connected to Google Drive using service account as {user_display}")
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Service account is valid! Connected as {user_display}",
|
||||
"account": user_email,
|
||||
"auth_type": "service_account",
|
||||
"auth_type": "service_account"
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Google Drive service account test failed: {error_msg}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Service account validation failed: {error_msg}",
|
||||
"message": f"Service account validation failed: {error_msg}"
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing Google Drive token")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Unexpected error: {str(e)}"
|
||||
}
|
||||
|
||||
@router.get("/google-drive/get-token-info")
|
||||
@require_login
|
||||
@@ -265,91 +300,94 @@ async def get_google_drive_token_info(request: Request):
|
||||
"""
|
||||
try:
|
||||
logger.info("Getting Google Drive token information")
|
||||
|
||||
|
||||
# Check if OAuth is enabled and configured
|
||||
if not getattr(settings, "google_drive_use_oauth", False):
|
||||
if not getattr(settings, 'google_drive_use_oauth', False):
|
||||
logger.warning("OAuth is not enabled, using service account instead")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "OAuth is not enabled. Service accounts don't support user-facing features.",
|
||||
"message": "OAuth is not enabled. Service accounts don't support user-facing features like folder picker."
|
||||
}
|
||||
|
||||
if not (
|
||||
settings.google_drive_client_id
|
||||
and settings.google_drive_client_secret
|
||||
and settings.google_drive_refresh_token
|
||||
):
|
||||
|
||||
if not (settings.google_drive_client_id and
|
||||
settings.google_drive_client_secret and
|
||||
settings.google_drive_refresh_token):
|
||||
logger.warning("Google Drive OAuth credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured",
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured"
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
# Get credentials and access token
|
||||
import google.oauth2.credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
|
||||
credentials = google.oauth2.credentials.Credentials(
|
||||
token=None,
|
||||
refresh_token=settings.google_drive_refresh_token,
|
||||
token_uri="https://oauth2.googleapis.com/token",
|
||||
client_id=settings.google_drive_client_id,
|
||||
client_secret=settings.google_drive_client_secret,
|
||||
client_secret=settings.google_drive_client_secret
|
||||
)
|
||||
|
||||
|
||||
# Force a refresh to get a fresh access token
|
||||
if not credentials.valid:
|
||||
credentials.refresh(Request())
|
||||
|
||||
|
||||
# Get token expiration info
|
||||
expiration_info = {}
|
||||
if hasattr(credentials, "expiry") and credentials.expiry:
|
||||
if hasattr(credentials, 'expiry') and credentials.expiry:
|
||||
now = datetime.now()
|
||||
expiry = credentials.expiry
|
||||
time_left = expiry - now
|
||||
expiration_info = {
|
||||
"expires_at": expiry.isoformat(),
|
||||
"expires_in_seconds": max(0, int(time_left.total_seconds())),
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
"expires_in_human": format_time_remaining(time_left)
|
||||
}
|
||||
|
||||
|
||||
# Return the token info
|
||||
logger.info("Successfully retrieved Google Drive access token")
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Access token successfully retrieved",
|
||||
"access_token": credentials.token,
|
||||
"token_info": expiration_info,
|
||||
"token_info": expiration_info
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Failed to get Google Drive token: {error_msg}")
|
||||
|
||||
|
||||
# Check if this is a token-related error
|
||||
if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"OAuth token retrieval failed: {error_msg}",
|
||||
"needs_reauth": True,
|
||||
"needs_reauth": True
|
||||
}
|
||||
return {"status": "error", "message": f"Token retrieval error: {error_msg}"}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token retrieval error: {error_msg}"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error getting Google Drive token info")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Unexpected error: {str(e)}"
|
||||
}
|
||||
|
||||
def format_time_remaining(time_delta):
|
||||
"""Format a timedelta into a human-readable string."""
|
||||
if time_delta.total_seconds() <= 0:
|
||||
return "Expired"
|
||||
|
||||
|
||||
days = time_delta.days
|
||||
hours, remainder = divmod(time_delta.seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
|
||||
|
||||
parts = []
|
||||
if days > 0:
|
||||
parts.append(f"{days} day{'s' if days != 1 else ''}")
|
||||
@@ -357,39 +395,34 @@ def format_time_remaining(time_delta):
|
||||
parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
|
||||
if minutes > 0 and days == 0: # Only show minutes if less than a day
|
||||
parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
|
||||
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@router.post("/google-drive/save-settings")
|
||||
@require_login
|
||||
async def save_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
use_oauth: Annotated[str, Form()] = "true",
|
||||
db: Session = Depends(get_db),
|
||||
client_id: str = Form(None),
|
||||
client_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
folder_id: str = Form(None),
|
||||
use_oauth: str = Form("true")
|
||||
):
|
||||
"""
|
||||
Save Google Drive settings to the .env file (best-effort) and persist to database.
|
||||
Save Google Drive settings to the .env file
|
||||
"""
|
||||
try:
|
||||
# Get the path to the .env file
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
|
||||
|
||||
# Convert use_oauth string to boolean
|
||||
use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
|
||||
# Define settings to update
|
||||
drive_settings = {"GOOGLE_DRIVE_USE_OAUTH": str(use_oauth_bool).lower()}
|
||||
|
||||
drive_settings = {
|
||||
"GOOGLE_DRIVE_USE_OAUTH": str(use_oauth_bool).lower()
|
||||
}
|
||||
|
||||
# Only update these if provided
|
||||
if use_oauth_bool:
|
||||
if refresh_token:
|
||||
@@ -398,53 +431,52 @@ async def save_dropbox_settings(
|
||||
drive_settings["GOOGLE_DRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
drive_settings["GOOGLE_DRIVE_CLIENT_SECRET"] = client_secret
|
||||
|
||||
|
||||
# Always include folder ID if provided
|
||||
if folder_id:
|
||||
drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
|
||||
|
||||
|
||||
# Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
|
||||
if os.path.exists(env_path):
|
||||
try:
|
||||
logger.info(f"Updating Google Drive settings in {env_path}")
|
||||
|
||||
|
||||
# Read the current .env file
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
|
||||
# Process each line and update or add settings
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in drive_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
# Uncomment if commented out - check the original stripped line
|
||||
if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
|
||||
if line.startswith("# "): # Uncomment if commented out
|
||||
line = line[2:]
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
new_env_lines.append(line)
|
||||
|
||||
# Add any settings that weren't updated (they weren't in the file)
|
||||
for key, value in drive_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
|
||||
# Write the updated .env file
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
|
||||
logger.info("Successfully updated Google Drive settings in .env file")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update .env file: {str(e)}, but will continue with in-memory update")
|
||||
else:
|
||||
logger.warning(
|
||||
f".env file not found at {env_path}, skipping file update but continuing with in-memory update"
|
||||
)
|
||||
|
||||
logger.warning(f".env file not found at {env_path}, skipping file update but continuing with in-memory update")
|
||||
|
||||
# Update the settings in memory (this always happens)
|
||||
if refresh_token:
|
||||
settings.google_drive_refresh_token = refresh_token
|
||||
@@ -454,39 +486,21 @@ async def save_dropbox_settings(
|
||||
settings.google_drive_client_secret = client_secret
|
||||
if folder_id:
|
||||
settings.google_drive_folder_id = folder_id
|
||||
|
||||
|
||||
# Set OAuth flag
|
||||
settings.google_drive_use_oauth = use_oauth_bool
|
||||
|
||||
# Persist to database
|
||||
save_setting_to_db(
|
||||
db,
|
||||
"google_drive_use_oauth",
|
||||
str(use_oauth_bool).lower(),
|
||||
changed_by=changed_by,
|
||||
)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "google_drive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if client_id:
|
||||
save_setting_to_db(db, "google_drive_client_id", client_id, changed_by=changed_by)
|
||||
if client_secret:
|
||||
save_setting_to_db(db, "google_drive_client_secret", client_secret, changed_by=changed_by)
|
||||
if folder_id:
|
||||
save_setting_to_db(db, "google_drive_folder_id", folder_id, changed_by=changed_by)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully updated Google Drive settings in memory and database")
|
||||
|
||||
|
||||
logger.info("Successfully updated Google Drive settings in memory")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Google Drive settings have been saved",
|
||||
"in_memory_only": not os.path.exists(env_path),
|
||||
"in_memory_only": not os.path.exists(env_path)
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving Google Drive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save Google Drive settings: {str(e)}",
|
||||
detail=f"Failed to save Google Drive settings: {str(e)}"
|
||||
)
|
||||
|
||||
@@ -1,431 +0,0 @@
|
||||
"""
|
||||
GraphQL API endpoint for DocuElevate.
|
||||
|
||||
Provides a flexible query interface alongside the existing REST API.
|
||||
Schema covers: documents, pipelines, settings, and users.
|
||||
|
||||
Endpoint: /graphql
|
||||
GraphiQL playground: /graphql (via browser)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any
|
||||
|
||||
import strawberry
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from strawberry.fastapi import GraphQLRouter
|
||||
|
||||
from app.auth import get_current_user
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import ApplicationSettings, FileRecord, Pipeline, PipelineStep, UserProfile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strawberry types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DocumentType:
|
||||
"""A processed document stored in the system."""
|
||||
|
||||
id: int
|
||||
owner_id: str | None
|
||||
original_filename: str | None
|
||||
local_filename: str
|
||||
file_size: int
|
||||
mime_type: str | None
|
||||
document_title: str | None
|
||||
is_duplicate: bool
|
||||
ocr_quality_score: int | None
|
||||
pipeline_id: int | None
|
||||
created_at: datetime | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class PipelineStepType:
|
||||
"""A single step within a processing pipeline."""
|
||||
|
||||
id: int
|
||||
pipeline_id: int
|
||||
position: int
|
||||
step_type: str
|
||||
label: str | None
|
||||
enabled: bool
|
||||
created_at: datetime | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class PipelineType:
|
||||
"""A processing pipeline with its ordered steps."""
|
||||
|
||||
id: int
|
||||
owner_id: str | None
|
||||
name: str
|
||||
description: str | None
|
||||
is_default: bool
|
||||
is_active: bool
|
||||
steps: list[PipelineStepType]
|
||||
created_at: datetime | None
|
||||
updated_at: datetime | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SettingType:
|
||||
"""An application configuration setting stored in the database."""
|
||||
|
||||
id: int
|
||||
key: str
|
||||
value: str | None
|
||||
created_at: datetime | None
|
||||
updated_at: datetime | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class UserType:
|
||||
"""A user profile in the system."""
|
||||
|
||||
id: int
|
||||
user_id: str
|
||||
display_name: str | None
|
||||
is_blocked: bool
|
||||
subscription_tier: str | None
|
||||
onboarding_completed: bool
|
||||
created_at: datetime | None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conversion helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _document_from_record(rec: FileRecord) -> DocumentType:
|
||||
return DocumentType(
|
||||
id=rec.id,
|
||||
owner_id=rec.owner_id,
|
||||
original_filename=rec.original_filename,
|
||||
local_filename=rec.local_filename,
|
||||
file_size=rec.file_size,
|
||||
mime_type=rec.mime_type,
|
||||
document_title=rec.document_title,
|
||||
is_duplicate=rec.is_duplicate,
|
||||
ocr_quality_score=rec.ocr_quality_score,
|
||||
pipeline_id=rec.pipeline_id,
|
||||
created_at=rec.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _pipeline_step_from_record(step: PipelineStep) -> PipelineStepType:
|
||||
return PipelineStepType(
|
||||
id=step.id,
|
||||
pipeline_id=step.pipeline_id,
|
||||
position=step.position,
|
||||
step_type=step.step_type,
|
||||
label=step.label,
|
||||
enabled=step.enabled,
|
||||
created_at=step.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _pipeline_from_record(pipeline: Pipeline, db: Session) -> PipelineType:
|
||||
steps = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline.id).order_by(PipelineStep.position).all()
|
||||
return PipelineType(
|
||||
id=pipeline.id,
|
||||
owner_id=pipeline.owner_id,
|
||||
name=pipeline.name,
|
||||
description=pipeline.description,
|
||||
is_default=pipeline.is_default,
|
||||
is_active=pipeline.is_active,
|
||||
steps=[_pipeline_step_from_record(s) for s in steps],
|
||||
created_at=pipeline.created_at,
|
||||
updated_at=pipeline.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def _setting_from_record(setting: ApplicationSettings) -> SettingType:
|
||||
return SettingType(
|
||||
id=setting.id,
|
||||
key=setting.key,
|
||||
value=setting.value,
|
||||
created_at=setting.created_at,
|
||||
updated_at=setting.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def _user_from_profile(profile: UserProfile) -> UserType:
|
||||
return UserType(
|
||||
id=profile.id,
|
||||
user_id=profile.user_id,
|
||||
display_name=profile.display_name,
|
||||
is_blocked=profile.is_blocked,
|
||||
subscription_tier=profile.subscription_tier,
|
||||
onboarding_completed=profile.onboarding_completed,
|
||||
created_at=profile.created_at,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Context helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Keys that contain sensitive data and must never be returned via GraphQL
|
||||
_SENSITIVE_SETTING_KEYS: frozenset[str] = frozenset(
|
||||
{
|
||||
"openai_api_key",
|
||||
"azure_ai_key",
|
||||
"session_secret",
|
||||
"database_url",
|
||||
"redis_url",
|
||||
"dropbox_app_secret",
|
||||
"dropbox_refresh_token",
|
||||
"google_drive_credentials_json",
|
||||
"onedrive_client_secret",
|
||||
"onedrive_refresh_token",
|
||||
"smtp_password",
|
||||
"nextcloud_password",
|
||||
"s3_secret_access_key",
|
||||
"ftp_password",
|
||||
"sftp_password",
|
||||
"webdav_password",
|
||||
"stripe_secret_key",
|
||||
"stripe_webhook_secret",
|
||||
"sentry_dsn",
|
||||
"social_auth_google_client_secret",
|
||||
"social_auth_microsoft_client_secret",
|
||||
"social_auth_apple_private_key",
|
||||
"social_auth_dropbox_app_secret",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _get_current_user_id(user: dict[str, Any] | None) -> str | None:
|
||||
"""Extract the stable user identifier from the user dict."""
|
||||
if not user:
|
||||
return None
|
||||
return user.get("preferred_username") or user.get("email") or user.get("id") or None
|
||||
|
||||
|
||||
def _get_db_and_user(info: strawberry.types.Info) -> tuple[Session, dict[str, Any] | None]:
|
||||
"""Extract the database session and current user from the Strawberry context."""
|
||||
db: Session = info.context["db"]
|
||||
user: dict[str, Any] | None = info.context.get("user")
|
||||
return db, user
|
||||
|
||||
|
||||
def _require_auth(user: dict[str, Any] | None) -> None:
|
||||
"""Raise an error when authentication is enabled and no valid user is present."""
|
||||
if settings.auth_enabled and not user:
|
||||
raise strawberry.exceptions.StrawberryGraphQLError("Authentication required")
|
||||
|
||||
|
||||
def _require_admin(user: dict[str, Any] | None) -> None:
|
||||
"""Raise an error when the current user is not an admin.
|
||||
|
||||
When ``auth_enabled`` is *False* (single-user / development mode) all
|
||||
callers are implicitly treated as administrators.
|
||||
"""
|
||||
if not settings.auth_enabled:
|
||||
# Single-user mode: no auth, treat caller as admin
|
||||
return
|
||||
_require_auth(user)
|
||||
if not (user and user.get("is_admin")):
|
||||
raise strawberry.exceptions.StrawberryGraphQLError("Admin access required")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query resolvers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Query:
|
||||
"""Root query type for the DocuElevate GraphQL API."""
|
||||
|
||||
@strawberry.field(description="List documents, optionally filtered by owner.")
|
||||
def documents(
|
||||
self,
|
||||
info: strawberry.types.Info,
|
||||
owner_id: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> list[DocumentType]:
|
||||
"""Return a paginated list of documents.
|
||||
|
||||
When *auth_enabled* the caller must be authenticated. Non-admin users
|
||||
receive only their own documents; admins may query any *owner_id*.
|
||||
"""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_auth(user)
|
||||
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
|
||||
query = db.query(FileRecord)
|
||||
|
||||
if settings.auth_enabled and user:
|
||||
is_admin = user.get("is_admin", False)
|
||||
current_user_id = _get_current_user_id(user)
|
||||
if not is_admin:
|
||||
# Non-admins can only see their own documents
|
||||
query = query.filter(FileRecord.owner_id == current_user_id)
|
||||
elif owner_id:
|
||||
query = query.filter(FileRecord.owner_id == owner_id)
|
||||
elif owner_id:
|
||||
query = query.filter(FileRecord.owner_id == owner_id)
|
||||
|
||||
records = query.order_by(FileRecord.created_at.desc()).offset(offset).limit(limit).all()
|
||||
return [_document_from_record(r) for r in records]
|
||||
|
||||
@strawberry.field(description="Fetch a single document by ID.")
|
||||
def document(self, info: strawberry.types.Info, id: int) -> DocumentType | None:
|
||||
"""Return one document by its primary key, or *null* if not found."""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_auth(user)
|
||||
|
||||
rec = db.query(FileRecord).filter(FileRecord.id == id).first()
|
||||
if rec is None:
|
||||
return None
|
||||
|
||||
if settings.auth_enabled and user:
|
||||
is_admin = user.get("is_admin", False)
|
||||
current_user_id = _get_current_user_id(user)
|
||||
if not is_admin and rec.owner_id != current_user_id:
|
||||
return None
|
||||
|
||||
return _document_from_record(rec)
|
||||
|
||||
@strawberry.field(description="List processing pipelines.")
|
||||
def pipelines(
|
||||
self,
|
||||
info: strawberry.types.Info,
|
||||
owner_id: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> list[PipelineType]:
|
||||
"""Return a paginated list of pipelines."""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_auth(user)
|
||||
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
|
||||
query = db.query(Pipeline)
|
||||
|
||||
if settings.auth_enabled and user:
|
||||
is_admin = user.get("is_admin", False)
|
||||
current_user_id = _get_current_user_id(user)
|
||||
if not is_admin:
|
||||
query = query.filter((Pipeline.owner_id == current_user_id) | (Pipeline.owner_id.is_(None)))
|
||||
elif owner_id:
|
||||
query = query.filter(Pipeline.owner_id == owner_id)
|
||||
elif owner_id:
|
||||
query = query.filter(Pipeline.owner_id == owner_id)
|
||||
|
||||
rows = query.order_by(Pipeline.id).offset(offset).limit(limit).all()
|
||||
return [_pipeline_from_record(p, db) for p in rows]
|
||||
|
||||
@strawberry.field(description="Fetch a single pipeline by ID.")
|
||||
def pipeline(self, info: strawberry.types.Info, id: int) -> PipelineType | None:
|
||||
"""Return one pipeline by its primary key, or *null* if not found."""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_auth(user)
|
||||
|
||||
row = db.query(Pipeline).filter(Pipeline.id == id).first()
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
if settings.auth_enabled and user:
|
||||
is_admin = user.get("is_admin", False)
|
||||
current_user_id = _get_current_user_id(user)
|
||||
if not is_admin and row.owner_id is not None and row.owner_id != current_user_id:
|
||||
return None
|
||||
|
||||
return _pipeline_from_record(row, db)
|
||||
|
||||
@strawberry.field(description="List non-sensitive application settings (admin only).")
|
||||
def settings(
|
||||
self,
|
||||
info: strawberry.types.Info,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[SettingType]:
|
||||
"""Return application settings stored in the database.
|
||||
|
||||
Sensitive keys (API secrets, passwords, etc.) are automatically
|
||||
excluded. Requires admin privileges when auth is enabled.
|
||||
"""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_admin(user)
|
||||
|
||||
limit = max(1, min(limit, 200))
|
||||
offset = max(0, offset)
|
||||
|
||||
rows = (
|
||||
db.query(ApplicationSettings)
|
||||
.filter(ApplicationSettings.key.notin_(_SENSITIVE_SETTING_KEYS))
|
||||
.order_by(ApplicationSettings.key)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [_setting_from_record(r) for r in rows]
|
||||
|
||||
@strawberry.field(description="List user profiles (admin only).")
|
||||
def users(
|
||||
self,
|
||||
info: strawberry.types.Info,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> list[UserType]:
|
||||
"""Return a paginated list of user profiles. Requires admin privileges."""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_admin(user)
|
||||
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
|
||||
rows = db.query(UserProfile).order_by(UserProfile.user_id).offset(offset).limit(limit).all()
|
||||
return [_user_from_profile(r) for r in rows]
|
||||
|
||||
@strawberry.field(description="Fetch a user profile by user_id (admin only).")
|
||||
def user(self, info: strawberry.types.Info, user_id: str) -> UserType | None:
|
||||
"""Return one user profile by *user_id*, or *null* if not found."""
|
||||
db, user = _get_db_and_user(info)
|
||||
_require_admin(user)
|
||||
|
||||
row = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
return _user_from_profile(row) if row else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema and router
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
schema = strawberry.Schema(query=Query)
|
||||
|
||||
|
||||
async def get_graphql_context(
|
||||
request: Request,
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
"""Build the per-request context injected into every resolver."""
|
||||
try:
|
||||
user = get_current_user(request)
|
||||
except Exception:
|
||||
logger.debug("Could not resolve current user for GraphQL context", exc_info=True)
|
||||
user = None
|
||||
return {"request": request, "db": db, "user": user}
|
||||
|
||||
|
||||
graphql_router = GraphQLRouter(
|
||||
schema,
|
||||
context_getter=get_graphql_context,
|
||||
graphql_ide="graphiql",
|
||||
)
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
"""API endpoints for internationalization (i18n).
|
||||
|
||||
Provides endpoints for:
|
||||
* Listing available languages
|
||||
* Getting/setting user language preference (persisted in session + cookie + DB)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, Response
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import UserProfile
|
||||
from app.utils.i18n import (
|
||||
DEFAULT_LANGUAGE,
|
||||
SUPPORTED_LANGUAGE_CODES,
|
||||
SUPPORTED_LANGUAGES,
|
||||
detect_language,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/i18n", tags=["i18n"])
|
||||
|
||||
|
||||
class LanguageInfo(BaseModel):
|
||||
"""Schema for a supported language."""
|
||||
|
||||
code: str
|
||||
name: str
|
||||
native: str
|
||||
flag: str
|
||||
|
||||
|
||||
class LanguageListResponse(BaseModel):
|
||||
"""Response for the list-languages endpoint."""
|
||||
|
||||
languages: list[LanguageInfo]
|
||||
current: str
|
||||
default: str
|
||||
|
||||
|
||||
class SetLanguageRequest(BaseModel):
|
||||
"""Request body for setting the preferred language."""
|
||||
|
||||
language: str
|
||||
|
||||
|
||||
class SetLanguageResponse(BaseModel):
|
||||
"""Response after changing the language."""
|
||||
|
||||
language: str
|
||||
message: str
|
||||
|
||||
|
||||
@router.get("/languages", response_model=LanguageListResponse)
|
||||
async def list_languages(request: Request) -> LanguageListResponse:
|
||||
"""Return all supported UI languages and the current active language."""
|
||||
current = detect_language(request)
|
||||
return LanguageListResponse(
|
||||
languages=[LanguageInfo(**lang) for lang in SUPPORTED_LANGUAGES],
|
||||
current=current,
|
||||
default=DEFAULT_LANGUAGE,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/language", response_model=SetLanguageResponse)
|
||||
async def set_language(
|
||||
body: SetLanguageRequest,
|
||||
request: Request,
|
||||
response: Response,
|
||||
db: Session = Depends(get_db),
|
||||
) -> SetLanguageResponse:
|
||||
"""Set the preferred UI language.
|
||||
|
||||
Persists the choice in:
|
||||
1. The server-side session
|
||||
2. A ``docuelevate_lang`` cookie (30-day expiry)
|
||||
3. The ``UserProfile.preferred_language`` column (if authenticated)
|
||||
"""
|
||||
lang = body.language.lower().strip()
|
||||
if lang not in SUPPORTED_LANGUAGE_CODES:
|
||||
lang = DEFAULT_LANGUAGE
|
||||
|
||||
# 1. Session
|
||||
if hasattr(request, "session"):
|
||||
request.session["preferred_language"] = lang
|
||||
|
||||
# 2. Cookie (30 days)
|
||||
response.set_cookie(
|
||||
key="docuelevate_lang",
|
||||
value=lang,
|
||||
max_age=30 * 24 * 60 * 60,
|
||||
httponly=False,
|
||||
samesite="lax",
|
||||
)
|
||||
|
||||
# 3. Database (if user is authenticated)
|
||||
_persist_language_to_profile(request, db, lang)
|
||||
|
||||
language_name = next(
|
||||
(entry["native"] for entry in SUPPORTED_LANGUAGES if entry["code"] == lang),
|
||||
lang,
|
||||
)
|
||||
logger.info("Language preference set to '%s'", lang)
|
||||
return SetLanguageResponse(
|
||||
language=lang,
|
||||
message=f"Language changed to {language_name}",
|
||||
)
|
||||
|
||||
|
||||
def _persist_language_to_profile(request: Request, db: Session, lang: str) -> None:
|
||||
"""Write language preference to the UserProfile row, if the user is logged in."""
|
||||
user_id: str | None = None
|
||||
if hasattr(request, "session"):
|
||||
user = request.session.get("user")
|
||||
if isinstance(user, dict):
|
||||
user_id = user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
elif isinstance(user, str):
|
||||
user_id = user
|
||||
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
try:
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile:
|
||||
profile.preferred_language = lang # type: ignore[attr-defined]
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.debug("Could not persist language preference for user_id=%s", user_id)
|
||||
@@ -1,371 +0,0 @@
|
||||
"""API endpoints for managing per-user IMAP ingestion accounts.
|
||||
|
||||
Provides CRUD operations for a user's IMAP accounts, quota enforcement
|
||||
against their subscription plan's ``max_mailboxes`` limit, and a
|
||||
test-connection endpoint so users can verify credentials before saving.
|
||||
"""
|
||||
|
||||
import imaplib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import UserImapAccount
|
||||
from app.utils.encryption import decrypt_value, encrypt_value
|
||||
from app.utils.subscription import get_tier, get_user_tier_id
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/imap-accounts", tags=["imap-accounts"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if owner_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quota helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_FREE_TIER_ID = "free"
|
||||
|
||||
|
||||
def _get_max_mailboxes(tier: dict[str, Any]) -> int | None:
|
||||
"""Return the maximum number of IMAP accounts allowed by *tier*.
|
||||
|
||||
Returns:
|
||||
``None`` — unlimited (paid tiers with ``max_mailboxes == 0``)
|
||||
``0`` — no mailboxes allowed (free tier)
|
||||
positive — the configured limit
|
||||
"""
|
||||
tier_id: str = tier.get("id", _FREE_TIER_ID)
|
||||
max_mb: int = tier.get("max_mailboxes", 0)
|
||||
|
||||
# Free tier: 0 means "no access" (not "unlimited")
|
||||
if tier_id == _FREE_TIER_ID:
|
||||
return 0
|
||||
|
||||
# Paid tiers: 0 means unlimited
|
||||
if max_mb == 0:
|
||||
return None
|
||||
|
||||
return max_mb
|
||||
|
||||
|
||||
def _check_quota(db: Session, owner_id: str) -> None:
|
||||
"""Raise 403 if the user has reached their IMAP account quota."""
|
||||
tier_id = get_user_tier_id(db, owner_id)
|
||||
tier = get_tier(tier_id, db)
|
||||
max_mb = _get_max_mailboxes(tier)
|
||||
|
||||
if max_mb == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=("Your current plan does not include email ingestion. Upgrade to a paid plan to add IMAP accounts."),
|
||||
)
|
||||
|
||||
if max_mb is not None:
|
||||
current_count = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).count()
|
||||
if current_count >= max_mb:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=(
|
||||
f"You have reached your plan limit of {max_mb} IMAP account(s). "
|
||||
"Please delete an existing account or upgrade your plan."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ImapAccountCreate(BaseModel):
|
||||
"""Schema for creating a new IMAP account."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable label")
|
||||
host: str = Field(..., min_length=1, max_length=255, description="IMAP server hostname")
|
||||
port: int = Field(default=993, ge=1, le=65535, description="IMAP server port")
|
||||
username: str = Field(..., min_length=1, max_length=255, description="IMAP login username")
|
||||
password: str = Field(..., min_length=1, max_length=1024, description="IMAP login password")
|
||||
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
|
||||
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
|
||||
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
|
||||
"Null inherits the global imap_attachment_filter setting."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ImapAccountUpdate(BaseModel):
|
||||
"""Schema for updating an existing IMAP account (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
host: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
port: int | None = Field(default=None, ge=1, le=65535)
|
||||
username: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
password: str | None = Field(default=None, min_length=1, max_length=1024)
|
||||
use_ssl: bool | None = None
|
||||
delete_after_process: bool | None = None
|
||||
is_active: bool | None = None
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ID of the ImapIngestionProfile to use. "
|
||||
"Explicitly sending null clears the override (falls back to global setting)."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ImapTestRequest(BaseModel):
|
||||
"""Schema for testing an IMAP connection without saving it."""
|
||||
|
||||
host: str = Field(..., min_length=1, max_length=255)
|
||||
port: int = Field(default=993, ge=1, le=65535)
|
||||
username: str = Field(..., min_length=1, max_length=255)
|
||||
password: str = Field(..., min_length=1, max_length=1024)
|
||||
use_ssl: bool = Field(default=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_response(acct: UserImapAccount) -> dict[str, Any]:
|
||||
"""Serialize a ``UserImapAccount`` row to a response dict.
|
||||
|
||||
Passwords are never included in responses.
|
||||
"""
|
||||
return {
|
||||
"id": acct.id,
|
||||
"owner_id": acct.owner_id,
|
||||
"name": acct.name,
|
||||
"host": acct.host,
|
||||
"port": acct.port,
|
||||
"username": acct.username,
|
||||
"use_ssl": acct.use_ssl,
|
||||
"delete_after_process": acct.delete_after_process,
|
||||
"is_active": acct.is_active,
|
||||
"profile_id": acct.profile_id,
|
||||
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
|
||||
"last_error": acct.last_error,
|
||||
"created_at": acct.created_at.isoformat() if acct.created_at else None,
|
||||
"updated_at": acct.updated_at.isoformat() if acct.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connection test helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _test_imap_connection(host: str, port: int, username: str, password: str, use_ssl: bool) -> dict[str, Any]:
|
||||
"""Attempt to connect and log in to the IMAP server.
|
||||
|
||||
Returns a dict with ``{"success": bool, "message": str}``.
|
||||
"""
|
||||
try:
|
||||
if use_ssl:
|
||||
mail = imaplib.IMAP4_SSL(host, port)
|
||||
else:
|
||||
mail = imaplib.IMAP4(host, port)
|
||||
|
||||
mail.login(username, password)
|
||||
mail.logout()
|
||||
return {"success": True, "message": "Connection successful"}
|
||||
except OSError as exc:
|
||||
logger.warning("IMAP network error for %s@%s: %s", username, host, exc)
|
||||
return {"success": False, "message": f"Connection error: {exc}"}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("IMAP error for %s@%s: %s", username, host, exc)
|
||||
return {"success": False, "message": f"IMAP error: {exc}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", summary="List IMAP accounts for the current user")
|
||||
def list_imap_accounts(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return all IMAP accounts belonging to the authenticated user."""
|
||||
accounts = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).order_by(UserImapAccount.id).all()
|
||||
return [_to_response(a) for a in accounts]
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new IMAP account")
|
||||
def create_imap_account(
|
||||
request: Request, body: ImapAccountCreate, db: DbSession, owner_id: CurrentOwner
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new IMAP ingestion account for the current user.
|
||||
|
||||
Quota is enforced against the user's subscription plan's ``max_mailboxes``
|
||||
limit before the account is persisted.
|
||||
"""
|
||||
_check_quota(db, owner_id)
|
||||
|
||||
acct = UserImapAccount(
|
||||
owner_id=owner_id,
|
||||
name=body.name,
|
||||
host=body.host,
|
||||
port=body.port,
|
||||
username=body.username,
|
||||
password=encrypt_value(body.password),
|
||||
use_ssl=body.use_ssl,
|
||||
delete_after_process=body.delete_after_process,
|
||||
is_active=body.is_active,
|
||||
profile_id=body.profile_id,
|
||||
)
|
||||
try:
|
||||
db.add(acct)
|
||||
db.commit()
|
||||
db.refresh(acct)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s created IMAP account %d (%s)", owner_id, acct.id, body.host)
|
||||
return _to_response(acct)
|
||||
|
||||
|
||||
@router.get("/{account_id}", summary="Get a single IMAP account")
|
||||
def get_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Return a single IMAP account by ID (must belong to the current user)."""
|
||||
acct = (
|
||||
db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first()
|
||||
)
|
||||
if not acct:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found")
|
||||
return _to_response(acct)
|
||||
|
||||
|
||||
@router.put("/{account_id}", summary="Update an IMAP account")
|
||||
def update_imap_account(
|
||||
account_id: int,
|
||||
request: Request,
|
||||
body: ImapAccountUpdate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing IMAP account. Only provided fields are changed."""
|
||||
acct = (
|
||||
db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first()
|
||||
)
|
||||
if not acct:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found")
|
||||
|
||||
if body.name is not None:
|
||||
acct.name = body.name
|
||||
if body.host is not None:
|
||||
acct.host = body.host
|
||||
if body.port is not None:
|
||||
acct.port = body.port
|
||||
if body.username is not None:
|
||||
acct.username = body.username
|
||||
if body.password is not None:
|
||||
acct.password = encrypt_value(body.password)
|
||||
if body.use_ssl is not None:
|
||||
acct.use_ssl = body.use_ssl
|
||||
if body.delete_after_process is not None:
|
||||
acct.delete_after_process = body.delete_after_process
|
||||
if body.is_active is not None:
|
||||
acct.is_active = body.is_active
|
||||
# profile_id: update whenever the field is explicitly present in the request payload
|
||||
# (including sending null to clear the override).
|
||||
if "profile_id" in body.model_fields_set:
|
||||
acct.profile_id = body.profile_id
|
||||
|
||||
# Reset last_error so the next poll gives a fresh result
|
||||
acct.last_error = None
|
||||
acct.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(acct)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s updated IMAP account %d", owner_id, account_id)
|
||||
return _to_response(acct)
|
||||
|
||||
|
||||
@router.delete("/{account_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an IMAP account")
|
||||
def delete_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
|
||||
"""Delete an IMAP account permanently."""
|
||||
acct = (
|
||||
db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first()
|
||||
)
|
||||
if not acct:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found")
|
||||
|
||||
try:
|
||||
db.delete(acct)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s deleted IMAP account %d", owner_id, account_id)
|
||||
|
||||
|
||||
@router.post("/{account_id}/test", summary="Test an existing IMAP account's connection")
|
||||
def test_saved_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Test the connection for an already-saved IMAP account."""
|
||||
acct = (
|
||||
db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first()
|
||||
)
|
||||
if not acct:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found")
|
||||
|
||||
return _test_imap_connection(acct.host, acct.port, acct.username, decrypt_value(acct.password), acct.use_ssl)
|
||||
|
||||
|
||||
@router.post("/test", summary="Test an IMAP connection without saving")
|
||||
def test_imap_connection(request: Request, body: ImapTestRequest, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Test IMAP credentials without persisting anything.
|
||||
|
||||
Useful for the "Test connection" button in the UI before the user saves
|
||||
a new account.
|
||||
"""
|
||||
return _test_imap_connection(body.host, body.port, body.username, body.password, body.use_ssl)
|
||||
|
||||
|
||||
@router.get("/quota/", summary="Get IMAP account quota information for the current user")
|
||||
def get_imap_quota(request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Return the user's current IMAP account usage vs. their plan quota."""
|
||||
tier_id = get_user_tier_id(db, owner_id)
|
||||
tier = get_tier(tier_id, db)
|
||||
max_mb = _get_max_mailboxes(tier)
|
||||
current_count = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).count()
|
||||
|
||||
return {
|
||||
"current_count": current_count,
|
||||
"max_mailboxes": max_mb, # None = unlimited, 0 = not allowed
|
||||
"can_add": max_mb is None or (max_mb > 0 and current_count < max_mb),
|
||||
"tier_id": tier_id,
|
||||
"tier_name": tier.get("name", tier_id),
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
"""API endpoints for managing IMAP ingestion profiles.
|
||||
|
||||
Ingestion profiles allow fine-grained control over which attachment types are
|
||||
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
|
||||
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
|
||||
canonical set defined in :mod:`app.utils.allowed_types`.
|
||||
|
||||
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
|
||||
deleted or modified. Users may create their own profiles which are private to
|
||||
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
|
||||
to all users but can only be created by administrators.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ImapIngestionProfile
|
||||
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if owner_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
|
||||
|
||||
|
||||
class ImapProfileCreate(BaseModel):
|
||||
"""Schema for creating a new ingestion profile."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
|
||||
description: str | None = Field(default=None, description="Optional description")
|
||||
allowed_categories: list[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
|
||||
)
|
||||
|
||||
|
||||
class ImapProfileUpdate(BaseModel):
|
||||
"""Schema for updating an existing profile (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
allowed_categories: list[str] | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_categories(categories: list[str]) -> list[str]:
|
||||
"""Raise 422 if any category key is unknown; return the cleaned list."""
|
||||
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
|
||||
)
|
||||
# Deduplicate while preserving order
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for cat in categories:
|
||||
if cat not in seen:
|
||||
seen.add(cat)
|
||||
result.append(cat)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
|
||||
"""Serialize a profile row to a response dict."""
|
||||
try:
|
||||
categories = json.loads(profile.allowed_categories)
|
||||
except (ValueError, TypeError):
|
||||
categories = []
|
||||
|
||||
# Enrich categories with display metadata
|
||||
categories_detail = [
|
||||
{
|
||||
"key": cat,
|
||||
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
|
||||
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
|
||||
}
|
||||
for cat in categories
|
||||
]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"owner_id": profile.owner_id,
|
||||
"allowed_categories": categories,
|
||||
"categories_detail": categories_detail,
|
||||
"is_builtin": profile.is_builtin,
|
||||
"created_at": profile.created_at.isoformat() if profile.created_at else None,
|
||||
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/categories", summary="List available file-type categories")
|
||||
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return the full list of file-type categories that can be used in profiles."""
|
||||
return [
|
||||
{
|
||||
"key": key,
|
||||
"label": info["label"],
|
||||
"description": info["description"],
|
||||
}
|
||||
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/", summary="List ingestion profiles visible to the current user")
|
||||
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
|
||||
profiles = (
|
||||
db.query(ImapIngestionProfile)
|
||||
.filter(
|
||||
# SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
|
||||
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
|
||||
)
|
||||
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||
.all()
|
||||
)
|
||||
return [_to_response(p) for p in profiles]
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
|
||||
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Create a new ingestion profile owned by the current user."""
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
|
||||
profile = ImapIngestionProfile(
|
||||
name=body.name,
|
||||
description=body.description,
|
||||
owner_id=owner_id,
|
||||
allowed_categories=json.dumps(categories),
|
||||
is_builtin=False,
|
||||
)
|
||||
try:
|
||||
db.add(profile)
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.get("/{profile_id}", summary="Get a single ingestion profile")
|
||||
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.put("/{profile_id}", summary="Update an ingestion profile")
|
||||
def update_profile(
|
||||
profile_id: int,
|
||||
request: Request,
|
||||
body: ImapProfileUpdate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be modified.",
|
||||
)
|
||||
|
||||
if body.name is not None:
|
||||
profile.name = body.name
|
||||
if "description" in body.model_fields_set:
|
||||
profile.description = body.description
|
||||
if body.allowed_categories is not None:
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
profile.allowed_categories = json.dumps(categories)
|
||||
|
||||
profile.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
|
||||
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
|
||||
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be deleted.",
|
||||
)
|
||||
|
||||
try:
|
||||
db.delete(profile)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
|
||||
@@ -1,688 +0,0 @@
|
||||
"""API endpoints for managing per-user integrations (sources and destinations).
|
||||
|
||||
Provides CRUD operations for :class:`~app.models.UserIntegration` records.
|
||||
Each record represents one ingestion source (e.g. IMAP, Watch Folder) or
|
||||
storage destination (e.g. S3, Dropbox, Google Drive) configured by a user.
|
||||
|
||||
Sensitive credentials are encrypted at rest using Fernet symmetric encryption
|
||||
(keyed from ``SESSION_SECRET``) via :mod:`app.utils.encryption`. Credential
|
||||
values are **never** returned in API responses.
|
||||
|
||||
Subscription quota enforcement
|
||||
------------------------------
|
||||
On creation, the endpoint checks the user's subscription tier limits:
|
||||
|
||||
* **Destinations** — ``max_storage_destinations`` from the plan.
|
||||
* **Sources (IMAP)** — ``max_mailboxes`` from the plan.
|
||||
|
||||
Exceeding the quota returns HTTP 403 with an actionable error message.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import IntegrationDirection, IntegrationType, UserIntegration
|
||||
from app.utils.encryption import decrypt_value, encrypt_value
|
||||
from app.utils.subscription import get_tier, get_user_tier_id
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/integrations", tags=["integrations"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if owner_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quota helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_FREE_TIER_ID = "free"
|
||||
|
||||
# Source types that consume the mailbox quota
|
||||
_MAILBOX_SOURCE_TYPES = {IntegrationType.IMAP}
|
||||
|
||||
|
||||
def _get_max_destinations(tier: dict[str, Any]) -> int | None:
|
||||
"""Return the maximum number of storage destinations allowed by *tier*.
|
||||
|
||||
Returns:
|
||||
``None`` — unlimited (paid tiers with ``max_storage_destinations == 0``)
|
||||
positive — the configured limit
|
||||
"""
|
||||
tier_id: str = tier.get("id", _FREE_TIER_ID)
|
||||
max_dest: int = tier.get("max_storage_destinations", 0)
|
||||
|
||||
# Free tier: the value itself is the limit (e.g. 1)
|
||||
if tier_id == _FREE_TIER_ID:
|
||||
return max_dest if max_dest > 0 else 1 # safe default
|
||||
|
||||
# Paid tiers: 0 means unlimited
|
||||
if max_dest == 0:
|
||||
return None
|
||||
|
||||
return max_dest
|
||||
|
||||
|
||||
def _get_max_sources(tier: dict[str, Any]) -> int | None:
|
||||
"""Return the maximum number of IMAP source integrations allowed by *tier*.
|
||||
|
||||
Returns:
|
||||
``None`` — unlimited (paid tiers with ``max_mailboxes == 0``)
|
||||
``0`` — no mailboxes allowed (free tier)
|
||||
positive — the configured limit
|
||||
"""
|
||||
tier_id: str = tier.get("id", _FREE_TIER_ID)
|
||||
max_mb: int = tier.get("max_mailboxes", 0)
|
||||
|
||||
# Free tier: 0 means "no access" (not "unlimited")
|
||||
if tier_id == _FREE_TIER_ID:
|
||||
return 0
|
||||
|
||||
# Paid tiers: 0 means unlimited
|
||||
if max_mb == 0:
|
||||
return None
|
||||
|
||||
return max_mb
|
||||
|
||||
|
||||
def _check_quota(db: Session, owner_id: str, direction: str, integration_type: str) -> None:
|
||||
"""Raise 403 if the user has reached their integration quota.
|
||||
|
||||
Quota rules:
|
||||
* DESTINATION integrations are limited by ``max_storage_destinations``.
|
||||
* SOURCE integrations of type IMAP are limited by ``max_mailboxes``.
|
||||
* Other SOURCE types (WATCH_FOLDER, WEBHOOK) are not quota-limited yet.
|
||||
"""
|
||||
tier_id = get_user_tier_id(db, owner_id)
|
||||
tier = get_tier(tier_id, db)
|
||||
|
||||
if direction == IntegrationDirection.DESTINATION:
|
||||
max_dest = _get_max_destinations(tier)
|
||||
if max_dest is not None:
|
||||
current_count = (
|
||||
db.query(UserIntegration)
|
||||
.filter(
|
||||
UserIntegration.owner_id == owner_id,
|
||||
UserIntegration.direction == IntegrationDirection.DESTINATION,
|
||||
)
|
||||
.count()
|
||||
)
|
||||
if current_count >= max_dest:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=(
|
||||
f"You have reached your plan limit of {max_dest} storage destination(s). "
|
||||
"Please remove an existing destination or upgrade your plan."
|
||||
),
|
||||
)
|
||||
|
||||
elif direction == IntegrationDirection.SOURCE and integration_type in _MAILBOX_SOURCE_TYPES:
|
||||
max_src = _get_max_sources(tier)
|
||||
|
||||
if max_src == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Your current plan does not include email ingestion. Upgrade to a paid plan to add IMAP sources.",
|
||||
)
|
||||
|
||||
if max_src is not None:
|
||||
current_count = (
|
||||
db.query(UserIntegration)
|
||||
.filter(
|
||||
UserIntegration.owner_id == owner_id,
|
||||
UserIntegration.direction == IntegrationDirection.SOURCE,
|
||||
UserIntegration.integration_type.in_(list(_MAILBOX_SOURCE_TYPES)),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
if current_count >= max_src:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=(
|
||||
f"You have reached your plan limit of {max_src} IMAP source(s). "
|
||||
"Please remove an existing source or upgrade your plan."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_DIRECTIONS = IntegrationDirection.ALL
|
||||
_VALID_TYPES = IntegrationType.ALL
|
||||
|
||||
|
||||
class IntegrationCreate(BaseModel):
|
||||
"""Schema for creating a new integration."""
|
||||
|
||||
direction: str = Field(..., description="'SOURCE' or 'DESTINATION'")
|
||||
integration_type: str = Field(..., description="Integration type (e.g. 'IMAP', 'S3', 'DROPBOX')")
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable label")
|
||||
config: dict[str, Any] | None = Field(default=None, description="Non-sensitive configuration (JSON object)")
|
||||
credentials: dict[str, Any] | None = Field(
|
||||
default=None, description="Sensitive credentials (JSON object, encrypted at rest)"
|
||||
)
|
||||
is_active: bool = Field(default=True, description="Whether the integration is active")
|
||||
|
||||
|
||||
class IntegrationUpdate(BaseModel):
|
||||
"""Schema for updating an existing integration (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
config: dict[str, Any] | None = None
|
||||
credentials: dict[str, Any] | None = None
|
||||
is_active: bool | None = None
|
||||
|
||||
|
||||
class IntegrationTestRequest(BaseModel):
|
||||
"""Schema for testing an integration connection without saving it."""
|
||||
|
||||
integration_type: str = Field(..., description="Integration type (e.g. 'IMAP', 'S3', 'DROPBOX')")
|
||||
config: dict[str, Any] | None = Field(default=None, description="Non-sensitive configuration")
|
||||
credentials: dict[str, Any] | None = Field(default=None, description="Credentials for the connection test")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_direction(direction: str) -> None:
|
||||
"""Raise 400 if *direction* is not a known value."""
|
||||
if direction not in _VALID_DIRECTIONS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid direction '{direction}'. Must be one of: {sorted(_VALID_DIRECTIONS)}",
|
||||
)
|
||||
|
||||
|
||||
def _validate_integration_type(integration_type: str) -> None:
|
||||
"""Raise 400 if *integration_type* is not a known value."""
|
||||
if integration_type not in _VALID_TYPES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid integration_type '{integration_type}'. Must be one of: {sorted(_VALID_TYPES)}",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_response(integration: UserIntegration) -> dict[str, Any]:
|
||||
"""Serialise a :class:`UserIntegration` row to a response dict.
|
||||
|
||||
Credentials are **never** included; only a boolean flag indicating
|
||||
whether credentials have been configured is returned.
|
||||
"""
|
||||
config_data: dict[str, Any] | None = None
|
||||
if integration.config:
|
||||
try:
|
||||
config_data = json.loads(integration.config)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
config_data = None
|
||||
|
||||
return {
|
||||
"id": integration.id,
|
||||
"owner_id": integration.owner_id,
|
||||
"direction": integration.direction,
|
||||
"integration_type": integration.integration_type,
|
||||
"name": integration.name,
|
||||
"config": config_data,
|
||||
"has_credentials": bool(integration.credentials),
|
||||
"is_active": integration.is_active,
|
||||
"last_used_at": integration.last_used_at.isoformat() if integration.last_used_at else None,
|
||||
"last_error": integration.last_error,
|
||||
"created_at": integration.created_at.isoformat() if integration.created_at else None,
|
||||
"updated_at": integration.updated_at.isoformat() if integration.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _encode_credentials(credentials: dict[str, Any] | None) -> str | None:
|
||||
"""Serialise *credentials* dict to an encrypted JSON string for storage."""
|
||||
if not credentials:
|
||||
return None
|
||||
plaintext = json.dumps(credentials)
|
||||
return encrypt_value(plaintext)
|
||||
|
||||
|
||||
def _decode_credentials(stored: str | None) -> dict[str, Any] | None:
|
||||
"""Decrypt and deserialise stored credentials back to a dict.
|
||||
|
||||
Returns ``None`` when *stored* is empty or cannot be decoded.
|
||||
"""
|
||||
if not stored:
|
||||
return None
|
||||
plaintext = decrypt_value(stored)
|
||||
if not plaintext:
|
||||
return None
|
||||
try:
|
||||
return json.loads(plaintext)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.error("Failed to decode credentials JSON after decryption")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", summary="List integrations for the current user")
|
||||
def list_integrations(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
direction: str | None = None,
|
||||
integration_type: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return all integrations belonging to the authenticated user.
|
||||
|
||||
Optional query-string filters:
|
||||
|
||||
- ``direction`` — ``SOURCE`` or ``DESTINATION``
|
||||
- ``integration_type`` — e.g. ``IMAP``, ``S3``, ``DROPBOX``
|
||||
"""
|
||||
query = db.query(UserIntegration).filter(UserIntegration.owner_id == owner_id)
|
||||
|
||||
if direction is not None:
|
||||
_validate_direction(direction)
|
||||
query = query.filter(UserIntegration.direction == direction)
|
||||
|
||||
if integration_type is not None:
|
||||
_validate_integration_type(integration_type)
|
||||
query = query.filter(UserIntegration.integration_type == integration_type)
|
||||
|
||||
integrations = query.order_by(UserIntegration.id).all()
|
||||
return [_to_response(i) for i in integrations]
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new integration")
|
||||
def create_integration(
|
||||
request: Request,
|
||||
body: IntegrationCreate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new source or destination integration for the current user.
|
||||
|
||||
``credentials`` are encrypted at rest using Fernet symmetric encryption
|
||||
before being persisted and are **never** returned in API responses.
|
||||
|
||||
Quota is enforced against the user's subscription plan before the
|
||||
integration is persisted.
|
||||
"""
|
||||
_validate_direction(body.direction)
|
||||
_validate_integration_type(body.integration_type)
|
||||
|
||||
_check_quota(db, owner_id, body.direction, body.integration_type)
|
||||
|
||||
integration = UserIntegration(
|
||||
owner_id=owner_id,
|
||||
direction=body.direction,
|
||||
integration_type=body.integration_type,
|
||||
name=body.name,
|
||||
config=json.dumps(body.config) if body.config is not None else None,
|
||||
credentials=_encode_credentials(body.credentials),
|
||||
is_active=body.is_active,
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(integration)
|
||||
db.commit()
|
||||
db.refresh(integration)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"User %s created %s integration %d (%s)",
|
||||
owner_id,
|
||||
body.direction,
|
||||
integration.id,
|
||||
body.integration_type,
|
||||
)
|
||||
return _to_response(integration)
|
||||
|
||||
|
||||
@router.get("/{integration_id}", summary="Get a single integration")
|
||||
def get_integration(
|
||||
integration_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a single integration by ID (must belong to the current user)."""
|
||||
integration = (
|
||||
db.query(UserIntegration)
|
||||
.filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not integration:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
|
||||
return _to_response(integration)
|
||||
|
||||
|
||||
@router.put("/{integration_id}", summary="Update an integration")
|
||||
def update_integration(
|
||||
integration_id: int,
|
||||
request: Request,
|
||||
body: IntegrationUpdate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing integration. Only provided fields are changed.
|
||||
|
||||
When ``credentials`` is supplied the stored value is replaced in full
|
||||
with the freshly encrypted version of the new credentials dict.
|
||||
"""
|
||||
integration = (
|
||||
db.query(UserIntegration)
|
||||
.filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not integration:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
|
||||
|
||||
if body.name is not None:
|
||||
integration.name = body.name
|
||||
if body.config is not None:
|
||||
integration.config = json.dumps(body.config)
|
||||
if body.credentials is not None:
|
||||
integration.credentials = _encode_credentials(body.credentials)
|
||||
if body.is_active is not None:
|
||||
integration.is_active = body.is_active
|
||||
|
||||
# Reset last_error so the next operation gives a fresh result
|
||||
integration.last_error = None
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(integration)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s updated integration %d", owner_id, integration_id)
|
||||
return _to_response(integration)
|
||||
|
||||
|
||||
@router.delete("/{integration_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an integration")
|
||||
def delete_integration(
|
||||
integration_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> None:
|
||||
"""Delete an integration permanently."""
|
||||
integration = (
|
||||
db.query(UserIntegration)
|
||||
.filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not integration:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
|
||||
|
||||
try:
|
||||
db.delete(integration)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s deleted integration %d", owner_id, integration_id)
|
||||
|
||||
|
||||
@router.get("/{integration_id}/credentials", summary="Retrieve decrypted credentials for an integration")
|
||||
def get_integration_credentials(
|
||||
integration_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the decrypted credentials dict for a saved integration.
|
||||
|
||||
This endpoint is intended for internal use by background tasks that need
|
||||
to authenticate with a third-party service. Treat the response as
|
||||
sensitive — it contains plaintext secrets.
|
||||
"""
|
||||
integration = (
|
||||
db.query(UserIntegration)
|
||||
.filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not integration:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
|
||||
|
||||
credentials = _decode_credentials(integration.credentials)
|
||||
return {"credentials": credentials or {}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connection test helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _test_imap_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Test an IMAP connection using the provided config and credentials."""
|
||||
import imaplib
|
||||
|
||||
cfg = config or {}
|
||||
creds = credentials or {}
|
||||
host = cfg.get("host", "")
|
||||
port = int(cfg.get("port", 993))
|
||||
username = cfg.get("username", "")
|
||||
password = creds.get("password", "")
|
||||
use_ssl = cfg.get("use_ssl", True)
|
||||
|
||||
if not host or not username or not password:
|
||||
return {"success": False, "message": "Missing required fields: host, username, and password"}
|
||||
|
||||
try:
|
||||
if use_ssl:
|
||||
mail = imaplib.IMAP4_SSL(host, port)
|
||||
else:
|
||||
mail = imaplib.IMAP4(host, port)
|
||||
mail.login(username, password)
|
||||
mail.logout()
|
||||
return {"success": True, "message": "IMAP connection successful"}
|
||||
except OSError as exc:
|
||||
logger.warning("IMAP network error for %s@%s: %s", username, host, exc)
|
||||
return {"success": False, "message": "IMAP connection failed — check host, port, and network connectivity"}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("IMAP error for %s@%s: %s", username, host, exc)
|
||||
return {"success": False, "message": "IMAP authentication or connection failed"}
|
||||
|
||||
|
||||
def _test_s3_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Test an S3 connection by calling HeadBucket."""
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
except ImportError:
|
||||
return {"success": False, "message": "boto3 is not installed"}
|
||||
|
||||
cfg = config or {}
|
||||
creds = credentials or {}
|
||||
bucket = cfg.get("bucket", "")
|
||||
region = cfg.get("region", "us-east-1")
|
||||
|
||||
if not bucket:
|
||||
return {"success": False, "message": "Missing required field: bucket"}
|
||||
|
||||
try:
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
region_name=region,
|
||||
aws_access_key_id=creds.get("access_key_id", ""),
|
||||
aws_secret_access_key=creds.get("secret_access_key", ""),
|
||||
endpoint_url=cfg.get("endpoint_url"),
|
||||
)
|
||||
client.head_bucket(Bucket=bucket)
|
||||
return {"success": True, "message": f"S3 bucket '{bucket}' is accessible"}
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
logger.warning("S3 connection error for bucket '%s': %s", bucket, exc)
|
||||
return {"success": False, "message": "S3 connection failed — check bucket name, region, and credentials"}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("S3 unexpected error for bucket '%s': %s", bucket, exc)
|
||||
return {"success": False, "message": "S3 connection failed"}
|
||||
|
||||
|
||||
def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Test a WebDAV/Nextcloud connection by issuing an HTTP PROPFIND."""
|
||||
import urllib.request
|
||||
|
||||
cfg = config or {}
|
||||
creds = credentials or {}
|
||||
url = cfg.get("url", "")
|
||||
username = creds.get("username", "")
|
||||
password = creds.get("password", "")
|
||||
|
||||
if not url:
|
||||
return {"success": False, "message": "Missing required field: url"}
|
||||
|
||||
# Only allow http/https to prevent file:// or other custom scheme attacks
|
||||
import ipaddress
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
return {"success": False, "message": "URL must use http or https scheme"}
|
||||
|
||||
# Block requests to private/internal IPs to prevent SSRF
|
||||
hostname = parsed.hostname or ""
|
||||
if hostname:
|
||||
try:
|
||||
addr = ipaddress.ip_address(hostname)
|
||||
if addr.is_private or addr.is_loopback or addr.is_link_local:
|
||||
return {"success": False, "message": "URLs pointing to internal or private networks are not allowed"}
|
||||
except ValueError:
|
||||
# Hostname is not an IP literal — allow DNS names through
|
||||
if hostname in ("localhost", "localhost.localdomain"):
|
||||
return {"success": False, "message": "URLs pointing to localhost are not allowed"}
|
||||
|
||||
try:
|
||||
import base64
|
||||
|
||||
req = urllib.request.Request(url, method="PROPFIND") # noqa: S310
|
||||
if username and password:
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
req.add_header("Authorization", f"Basic {token}")
|
||||
req.add_header("Depth", "0")
|
||||
with urllib.request.urlopen(req, timeout=10) as resp: # noqa: S310
|
||||
if resp.status < 400:
|
||||
return {"success": True, "message": "WebDAV connection successful"}
|
||||
return {"success": False, "message": f"WebDAV returned HTTP {resp.status}"}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("WebDAV connection error for %s: %s", hostname, exc)
|
||||
return {"success": False, "message": "WebDAV connection failed — check URL and credentials"}
|
||||
|
||||
|
||||
_CONNECTION_TESTERS: dict[str, Any] = {
|
||||
IntegrationType.IMAP: _test_imap_connection,
|
||||
IntegrationType.S3: _test_s3_connection,
|
||||
IntegrationType.WEBDAV: _test_webdav_connection,
|
||||
IntegrationType.NEXTCLOUD: _test_webdav_connection,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test & quota endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/test", summary="Test an integration connection without saving")
|
||||
def test_integration_connection(
|
||||
request: Request,
|
||||
body: IntegrationTestRequest,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Test integration credentials without persisting anything.
|
||||
|
||||
Useful for the "Test connection" button in the UI before the user saves
|
||||
a new integration. Returns ``{"success": bool, "message": str}``.
|
||||
"""
|
||||
_validate_integration_type(body.integration_type)
|
||||
|
||||
tester = _CONNECTION_TESTERS.get(body.integration_type)
|
||||
if tester is None:
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"Connection testing is not yet supported for '{body.integration_type}'. "
|
||||
"The integration can still be saved and will be validated on first use.",
|
||||
}
|
||||
|
||||
return tester(body.config, body.credentials)
|
||||
|
||||
|
||||
@router.get("/quota/", summary="Get integration quota information for the current user")
|
||||
def get_integration_quota(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the user's current integration usage vs. their plan quota.
|
||||
|
||||
Includes separate counts for destinations and IMAP sources.
|
||||
"""
|
||||
tier_id = get_user_tier_id(db, owner_id)
|
||||
tier = get_tier(tier_id, db)
|
||||
|
||||
max_dest = _get_max_destinations(tier)
|
||||
max_src = _get_max_sources(tier)
|
||||
|
||||
dest_count = (
|
||||
db.query(UserIntegration)
|
||||
.filter(
|
||||
UserIntegration.owner_id == owner_id,
|
||||
UserIntegration.direction == IntegrationDirection.DESTINATION,
|
||||
)
|
||||
.count()
|
||||
)
|
||||
|
||||
src_count = (
|
||||
db.query(UserIntegration)
|
||||
.filter(
|
||||
UserIntegration.owner_id == owner_id,
|
||||
UserIntegration.direction == IntegrationDirection.SOURCE,
|
||||
UserIntegration.integration_type.in_(list(_MAILBOX_SOURCE_TYPES)),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
|
||||
return {
|
||||
"tier_id": tier_id,
|
||||
"tier_name": tier.get("name", tier_id),
|
||||
"destinations": {
|
||||
"current_count": dest_count,
|
||||
"max_allowed": max_dest,
|
||||
"can_add": max_dest is None or dest_count < max_dest,
|
||||
},
|
||||
"sources": {
|
||||
"current_count": src_count,
|
||||
"max_allowed": max_src,
|
||||
"can_add": max_src is None or (max_src > 0 and src_count < max_src),
|
||||
},
|
||||
}
|
||||
@@ -1,393 +0,0 @@
|
||||
"""Local user authentication API — signup, email verification, password reset.
|
||||
|
||||
Provides the REST endpoints and page routes for the self-registration flow:
|
||||
|
||||
- GET /signup — signup page (HTML)
|
||||
- POST /api/auth/signup — create account + send verification email
|
||||
- GET /verify-email — activate account from email link (redirect)
|
||||
- GET /verify-email-sent — confirmation landing page (HTML)
|
||||
- POST /api/auth/resend-verification — re-send verification email
|
||||
- POST /api/auth/request-password-reset — start password reset
|
||||
- POST /api/auth/reset-password — set new password using token
|
||||
- GET /reset-password — password reset form page (HTML)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.responses import RedirectResponse
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import LocalUser, UserProfile
|
||||
from app.utils.i18n import translate as _translate
|
||||
from app.utils.local_auth import (
|
||||
build_session_user,
|
||||
generate_token,
|
||||
hash_password,
|
||||
is_token_expired,
|
||||
send_forgot_username_email,
|
||||
send_password_reset_email,
|
||||
send_verification_email,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["local-auth"])
|
||||
|
||||
_templates_dir = pathlib.Path(__file__).parents[2] / "frontend" / "templates"
|
||||
templates = Jinja2Templates(directory=str(_templates_dir))
|
||||
templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SignupBody(BaseModel):
|
||||
"""Body for the signup endpoint."""
|
||||
|
||||
email: str = Field(..., max_length=255)
|
||||
username: str = Field(..., min_length=3, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
|
||||
display_name: str | None = Field(default=None, max_length=255)
|
||||
password: str = Field(..., min_length=8, max_length=128)
|
||||
password_confirm: str
|
||||
|
||||
|
||||
class ResendVerificationBody(BaseModel):
|
||||
"""Body for the resend-verification endpoint."""
|
||||
|
||||
email: str
|
||||
|
||||
|
||||
class PasswordResetRequestBody(BaseModel):
|
||||
"""Body for the request-password-reset endpoint."""
|
||||
|
||||
email: str
|
||||
|
||||
|
||||
class PasswordResetBody(BaseModel):
|
||||
"""Body for the reset-password endpoint."""
|
||||
|
||||
token: str
|
||||
new_password: str = Field(..., min_length=8, max_length=128)
|
||||
new_password_confirm: str
|
||||
|
||||
|
||||
class ForgotUsernameBody(BaseModel):
|
||||
"""Body for the forgot-username endpoint."""
|
||||
|
||||
email: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page routes (return HTML)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/signup", include_in_schema=False)
|
||||
async def signup_page(request: Request) -> Any:
|
||||
"""Render the signup page, or redirect to login when multi-user / signup is disabled."""
|
||||
if not settings.multi_user_enabled:
|
||||
return RedirectResponse(url="/login?error=Multi-user+mode+is+not+enabled", status_code=302)
|
||||
if not settings.allow_local_signup:
|
||||
return RedirectResponse(url="/login?error=Registration+is+not+enabled", status_code=302)
|
||||
return templates.TemplateResponse(
|
||||
"signup.html",
|
||||
{
|
||||
"request": request,
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
"app_version": settings.version,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/verify-email-sent", include_in_schema=False)
|
||||
async def verify_email_sent_page(request: Request) -> Any:
|
||||
"""Render the verify-email-sent confirmation page."""
|
||||
return templates.TemplateResponse("verify_email_sent.html", {"request": request})
|
||||
|
||||
|
||||
@router.get("/forgot-username", include_in_schema=False)
|
||||
async def forgot_username_page(request: Request) -> Any:
|
||||
"""Render the forgot-username page where users can request a username reminder email."""
|
||||
return templates.TemplateResponse(
|
||||
"forgot_username.html",
|
||||
{
|
||||
"request": request,
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
"app_version": settings.version,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/forgot-password", include_in_schema=False)
|
||||
async def forgot_password_page(request: Request) -> Any:
|
||||
"""Render the forgot-password page where users can request a reset email."""
|
||||
return templates.TemplateResponse(
|
||||
"forgot_password.html",
|
||||
{
|
||||
"request": request,
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
"app_version": settings.version,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/reset-password", include_in_schema=False)
|
||||
async def reset_password_page(request: Request) -> Any:
|
||||
"""Render the password reset form page."""
|
||||
token = request.query_params.get("token", "")
|
||||
return templates.TemplateResponse(
|
||||
"password_reset_form.html",
|
||||
{
|
||||
"request": request,
|
||||
"token": token,
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
"app_version": settings.version,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API endpoints (return JSON or redirect)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/api/auth/signup", status_code=status.HTTP_201_CREATED)
|
||||
async def signup(request: Request, body: SignupBody, db: DbSession) -> dict[str, str | bool]:
|
||||
"""Create a new local user account.
|
||||
|
||||
When SMTP is configured the account is inactive until the user clicks the
|
||||
verification link sent to their email. When SMTP is **not** configured the
|
||||
account is activated immediately so that deployments without email can still
|
||||
use the self-registration flow.
|
||||
|
||||
Both ``MULTI_USER_ENABLED`` and ``ALLOW_LOCAL_SIGNUP`` must be ``True``.
|
||||
|
||||
Raises:
|
||||
403: Multi-user mode or local signup is disabled.
|
||||
422: Passwords do not match.
|
||||
409: Email or username already registered.
|
||||
"""
|
||||
if not settings.multi_user_enabled:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Multi-user mode is not enabled.")
|
||||
if not settings.allow_local_signup:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Registration is not enabled.")
|
||||
if body.password != body.password_confirm:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Passwords do not match.")
|
||||
|
||||
if db.query(LocalUser).filter(LocalUser.email == body.email).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered.")
|
||||
if db.query(LocalUser).filter(LocalUser.username == body.username).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Username already taken.")
|
||||
|
||||
smtp_configured = bool(settings.email_host)
|
||||
|
||||
if smtp_configured:
|
||||
token = generate_token()
|
||||
user = LocalUser(
|
||||
email=body.email,
|
||||
username=body.username,
|
||||
display_name=body.display_name,
|
||||
hashed_password=hash_password(body.password),
|
||||
is_active=False,
|
||||
email_verification_token=token,
|
||||
email_verification_sent_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
else:
|
||||
# No SMTP configured — activate the account immediately.
|
||||
token = None
|
||||
user = LocalUser(
|
||||
email=body.email,
|
||||
username=body.username,
|
||||
display_name=body.display_name,
|
||||
hashed_password=hash_password(body.password),
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
db.add(user)
|
||||
|
||||
profile = UserProfile(
|
||||
user_id=body.email,
|
||||
display_name=body.display_name or body.username,
|
||||
)
|
||||
db.add(profile)
|
||||
|
||||
# Flush to the DB so constraint violations (duplicate key etc.) surface NOW,
|
||||
# before we attempt to send the email. We do NOT commit yet — the commit only
|
||||
# happens after the email is sent successfully so that a failed email leaves
|
||||
# no orphan records in the database.
|
||||
try:
|
||||
db.flush()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
if smtp_configured and token:
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
try:
|
||||
send_verification_email(body.email, body.username, token, base_url)
|
||||
except Exception as exc:
|
||||
# Email failed — roll back so no unverifiable user row persists.
|
||||
# The user can simply try registering again once SMTP is fixed.
|
||||
db.rollback()
|
||||
logger.warning("Signup email failed for %s: %s", body.email, exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=(
|
||||
"Failed to send verification email. Please check that SMTP is correctly configured and try again."
|
||||
),
|
||||
) from exc
|
||||
|
||||
db.commit()
|
||||
logger.info("New local user registered: %s", body.email)
|
||||
|
||||
if smtp_configured:
|
||||
return {"message": "Verification email sent. Please check your inbox.", "email_verification_required": True}
|
||||
return {"message": "Account created successfully. You can now log in.", "email_verification_required": False}
|
||||
|
||||
|
||||
@router.get("/verify-email", include_in_schema=False)
|
||||
async def verify_email(request: Request, db: DbSession) -> Any:
|
||||
"""Activate a local user account from the email verification link.
|
||||
|
||||
Redirects to the login page on failure, or to onboarding/upload on success.
|
||||
"""
|
||||
token = request.query_params.get("token", "")
|
||||
user = db.query(LocalUser).filter(LocalUser.email_verification_token == token).first()
|
||||
|
||||
if not user:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Invalid+or+expired+verification+link",
|
||||
status_code=302,
|
||||
)
|
||||
if is_token_expired(user.email_verification_sent_at):
|
||||
return RedirectResponse(
|
||||
url="/login?error=Verification+link+has+expired.+Please+request+a+new+one",
|
||||
status_code=302,
|
||||
)
|
||||
|
||||
user.is_active = True
|
||||
user.email_verification_token = None
|
||||
user.email_verification_sent_at = None
|
||||
|
||||
# Ensure profile exists
|
||||
if not db.query(UserProfile).filter(UserProfile.user_id == user.email).first():
|
||||
db.add(UserProfile(user_id=user.email, display_name=user.display_name or user.username))
|
||||
|
||||
db.commit()
|
||||
|
||||
request.session["user"] = build_session_user(user)
|
||||
logger.info("[SECURITY] EMAIL_VERIFIED user=%s", user.email)
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user.email).first()
|
||||
if profile and not profile.onboarding_completed:
|
||||
post_onboarding = request.session.pop("redirect_after_login", "/upload")
|
||||
request.session["post_onboarding_redirect"] = post_onboarding
|
||||
return RedirectResponse(url="/onboarding", status_code=302)
|
||||
return RedirectResponse(url="/upload", status_code=302)
|
||||
|
||||
|
||||
@router.post("/api/auth/resend-verification")
|
||||
async def resend_verification(request: Request, body: ResendVerificationBody, db: DbSession) -> dict[str, str]:
|
||||
"""Re-send the verification email for a pending account.
|
||||
|
||||
Always returns 200 to avoid leaking whether an email is registered.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.email == body.email).first()
|
||||
if not user or user.is_active:
|
||||
return {"message": "Verification email resent if account exists."}
|
||||
|
||||
token = generate_token()
|
||||
user.email_verification_token = token
|
||||
user.email_verification_sent_at = datetime.now(tz=timezone.utc)
|
||||
db.commit()
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
try:
|
||||
send_verification_email(user.email, user.username, token, base_url)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to resend verification email to %s: %s", user.email, exc)
|
||||
|
||||
return {"message": "Verification email resent if account exists."}
|
||||
|
||||
|
||||
@router.post("/api/auth/request-password-reset")
|
||||
async def request_password_reset(request: Request, body: PasswordResetRequestBody, db: DbSession) -> dict[str, str]:
|
||||
"""Send a password reset email.
|
||||
|
||||
Always returns 200 to avoid leaking whether an email is registered.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.email == body.email).first()
|
||||
if not user:
|
||||
return {"message": "Password reset email sent if account exists."}
|
||||
|
||||
token = generate_token()
|
||||
user.password_reset_token = token
|
||||
user.password_reset_sent_at = datetime.now(tz=timezone.utc)
|
||||
db.commit()
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
try:
|
||||
send_password_reset_email(user.email, user.username, token, base_url)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to send password reset email to %s: %s", user.email, exc)
|
||||
|
||||
return {"message": "Password reset email sent if account exists."}
|
||||
|
||||
|
||||
@router.post("/api/auth/reset-password")
|
||||
async def reset_password(body: PasswordResetBody, db: DbSession) -> dict[str, str]:
|
||||
"""Set a new password using a valid reset token.
|
||||
|
||||
Raises:
|
||||
400: Token is invalid or expired.
|
||||
422: Passwords do not match.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.password_reset_token == body.token).first()
|
||||
if not user or is_token_expired(user.password_reset_sent_at):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid or expired reset token.",
|
||||
)
|
||||
if body.new_password != body.new_password_confirm:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="Passwords do not match.",
|
||||
)
|
||||
|
||||
user.hashed_password = hash_password(body.new_password)
|
||||
user.password_reset_token = None
|
||||
user.password_reset_sent_at = None
|
||||
# Activate the account in case it was still pending email verification.
|
||||
# A valid password-reset token proves control of the registered email address.
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
|
||||
logger.info("[SECURITY] PASSWORD_RESET_SUCCESS user=%s", user.email)
|
||||
return {"message": "Password updated successfully."}
|
||||
|
||||
|
||||
@router.post("/api/auth/forgot-username")
|
||||
async def forgot_username(body: ForgotUsernameBody, db: DbSession) -> dict[str, str]:
|
||||
"""Send a username reminder email.
|
||||
|
||||
Always returns 200 to avoid leaking whether an email is registered.
|
||||
"""
|
||||
user = db.query(LocalUser).filter(LocalUser.email == body.email).first()
|
||||
if user:
|
||||
try:
|
||||
send_forgot_username_email(user.email, user.username)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to send forgot-username email to %s: %s", user.email, exc)
|
||||
|
||||
return {"message": "Username reminder sent if account exists."}
|
||||
-158
@@ -1,158 +0,0 @@
|
||||
"""
|
||||
Processing logs API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy import desc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
from app.utils.input_validation import validate_task_id
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
@require_login
|
||||
def list_processing_logs(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
file_id: Optional[int] = Query(None, description="Filter by file ID"),
|
||||
task_id: Optional[str] = Query(None, description="Filter by task ID"),
|
||||
limit: int = Query(100, ge=1, le=1000, description="Number of logs to return"),
|
||||
):
|
||||
"""
|
||||
Returns a JSON list of ProcessingLog entries.
|
||||
Protected by `@require_login`, so only logged-in sessions can access.
|
||||
|
||||
Query Parameters:
|
||||
- file_id: Optional filter by file ID
|
||||
- task_id: Optional filter by task ID
|
||||
- limit: Maximum number of logs to return (default 100, max 1000)
|
||||
|
||||
Example response:
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"file_id": 123,
|
||||
"task_id": "abc-123-def",
|
||||
"step_name": "process_document",
|
||||
"status": "success",
|
||||
"message": "Processing completed",
|
||||
"timestamp": "2025-05-01T12:34:56.789000"
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
query = db.query(ProcessingLog)
|
||||
|
||||
# Apply filters
|
||||
if file_id is not None:
|
||||
query = query.filter(ProcessingLog.file_id == file_id)
|
||||
if task_id is not None:
|
||||
validate_task_id(task_id)
|
||||
query = query.filter(ProcessingLog.task_id == task_id)
|
||||
|
||||
# Order by timestamp descending and limit
|
||||
logs = query.order_by(desc(ProcessingLog.timestamp)).limit(limit).all()
|
||||
|
||||
# Return a simple list of dicts
|
||||
result = []
|
||||
for log in logs:
|
||||
result.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"file_id": log.file_id,
|
||||
"task_id": log.task_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/logs/file/{file_id}")
|
||||
@require_login
|
||||
def get_file_processing_logs(request: Request, file_id: int, db: DbSession):
|
||||
"""
|
||||
Get all processing logs for a specific file.
|
||||
Returns logs ordered by timestamp (oldest first to show processing flow).
|
||||
|
||||
Also includes file metadata if the file exists.
|
||||
"""
|
||||
# Check if file exists
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
|
||||
# Get all logs for this file
|
||||
logs = db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp).all()
|
||||
|
||||
# Build response
|
||||
log_list = []
|
||||
for log in logs:
|
||||
log_list.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"task_id": log.task_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"file": {
|
||||
"id": file_record.id,
|
||||
"original_filename": file_record.original_filename,
|
||||
"file_size": file_record.file_size,
|
||||
"mime_type": file_record.mime_type,
|
||||
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
|
||||
},
|
||||
"logs": log_list,
|
||||
"total_logs": len(log_list),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/logs/task/{task_id}")
|
||||
@require_login
|
||||
def get_task_processing_logs(request: Request, task_id: str, db: DbSession):
|
||||
"""
|
||||
Get all processing logs for a specific task.
|
||||
Returns logs ordered by timestamp (oldest first to show processing flow).
|
||||
"""
|
||||
validate_task_id(task_id)
|
||||
# Get all logs for this task
|
||||
logs = db.query(ProcessingLog).filter(ProcessingLog.task_id == task_id).order_by(ProcessingLog.timestamp).all()
|
||||
|
||||
if not logs:
|
||||
raise HTTPException(status_code=404, detail=f"No logs found for task {task_id}")
|
||||
|
||||
# Build response
|
||||
log_list = []
|
||||
for log in logs:
|
||||
log_list.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"file_id": log.file_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {"task_id": task_id, "logs": log_list, "total_logs": len(log_list)}
|
||||
@@ -1,347 +0,0 @@
|
||||
"""Mobile app API endpoints.
|
||||
|
||||
Provides endpoints specifically designed for the DocuElevate native mobile
|
||||
app (iOS / Android via React Native / Expo):
|
||||
|
||||
* ``POST /mobile/generate-token`` – exchange an active session for a
|
||||
long-lived API token that the mobile app stores securely. The token is
|
||||
auto-named "Mobile App – <device_name>" and is identical to regular API
|
||||
tokens (Bearer auth works everywhere).
|
||||
|
||||
* ``POST /mobile/register-device`` – register a push-notification device
|
||||
token (Expo push token) so the user receives push notifications when
|
||||
documents finish processing.
|
||||
|
||||
* ``GET /mobile/devices`` – list registered devices for the current user.
|
||||
|
||||
* ``DELETE /mobile/devices/{device_id}`` – deactivate a device.
|
||||
|
||||
* ``GET /mobile/whoami`` – lightweight profile endpoint for the mobile app
|
||||
to verify authentication state.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.api_tokens import generate_api_token, hash_token
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import ApiToken, MobileDevice
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/mobile", tags=["mobile"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GenerateTokenRequest(BaseModel):
|
||||
"""Request body for auto-generating a mobile app token."""
|
||||
|
||||
device_name: str = Field(
|
||||
default="Mobile App",
|
||||
min_length=1,
|
||||
max_length=120,
|
||||
description="Human-readable device name used to label the token.",
|
||||
)
|
||||
|
||||
|
||||
class GenerateTokenResponse(BaseModel):
|
||||
"""Response containing the one-time-visible API token."""
|
||||
|
||||
token: str
|
||||
token_id: int
|
||||
name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RegisterDeviceRequest(BaseModel):
|
||||
"""Request body for registering a push-notification device token."""
|
||||
|
||||
push_token: str = Field(
|
||||
min_length=1,
|
||||
max_length=512,
|
||||
description="Expo push token (ExponentPushToken[…]) obtained from the mobile app.",
|
||||
)
|
||||
device_name: str | None = Field(
|
||||
default=None,
|
||||
max_length=255,
|
||||
description="Optional human-readable device name (e.g. 'John's iPhone').",
|
||||
)
|
||||
platform: str = Field(
|
||||
default="ios",
|
||||
description="Device platform: 'ios', 'android', or 'web'.",
|
||||
)
|
||||
|
||||
|
||||
class DeviceResponse(BaseModel):
|
||||
"""Serialised MobileDevice record."""
|
||||
|
||||
id: int
|
||||
device_name: str | None
|
||||
platform: str
|
||||
push_token_preview: str
|
||||
is_active: bool
|
||||
created_at: datetime
|
||||
last_seen_at: datetime | None
|
||||
|
||||
|
||||
class WhoAmIResponse(BaseModel):
|
||||
"""Lightweight profile response for the mobile app."""
|
||||
|
||||
owner_id: str
|
||||
display_name: str | None
|
||||
email: str | None
|
||||
avatar_url: str | None
|
||||
is_admin: bool
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _device_to_response(device: MobileDevice) -> dict[str, Any]:
|
||||
"""Convert a MobileDevice ORM object to a serialisable dict."""
|
||||
# Show only first 20 chars of the push token for security.
|
||||
token_preview = device.push_token[:20] + "…" if len(device.push_token) > 20 else device.push_token
|
||||
return {
|
||||
"id": device.id,
|
||||
"device_name": device.device_name,
|
||||
"platform": device.platform,
|
||||
"push_token_preview": token_preview,
|
||||
"is_active": device.is_active,
|
||||
"created_at": device.created_at,
|
||||
"last_seen_at": device.last_seen_at,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/generate-token", status_code=status.HTTP_201_CREATED, response_model=GenerateTokenResponse)
|
||||
@require_login
|
||||
async def generate_mobile_token(
|
||||
request: Request,
|
||||
body: GenerateTokenRequest,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a long-lived API token for the mobile app.
|
||||
|
||||
The mobile app calls this endpoint immediately after SSO login to obtain
|
||||
a Bearer token it can store in the secure keychain. The returned token
|
||||
is functionally identical to manually-created API tokens and works with
|
||||
every authenticated endpoint.
|
||||
|
||||
The token is shown **exactly once** in the response; subsequent requests
|
||||
show only the prefix for identification.
|
||||
"""
|
||||
token_name = f"Mobile App – {body.device_name}"
|
||||
plaintext = generate_api_token()
|
||||
token_hash_value = hash_token(plaintext)
|
||||
prefix = plaintext[:12]
|
||||
|
||||
db_token = ApiToken(
|
||||
owner_id=owner_id,
|
||||
name=token_name,
|
||||
token_hash=token_hash_value,
|
||||
token_prefix=prefix,
|
||||
)
|
||||
try:
|
||||
db.add(db_token)
|
||||
db.commit()
|
||||
db.refresh(db_token)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create mobile API token for owner_id=%s", owner_id)
|
||||
raise
|
||||
|
||||
logger.info("Mobile API token created: id=%s owner=%s device=%r", db_token.id, owner_id, body.device_name)
|
||||
|
||||
return {
|
||||
"token": plaintext,
|
||||
"token_id": db_token.id,
|
||||
"name": token_name,
|
||||
"created_at": db_token.created_at,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/register-device", status_code=status.HTTP_201_CREATED, response_model=DeviceResponse)
|
||||
@require_login
|
||||
async def register_device(
|
||||
request: Request,
|
||||
body: RegisterDeviceRequest,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Register or refresh a push-notification device token.
|
||||
|
||||
If the same ``push_token`` is already registered for this user the
|
||||
record is reactivated and ``last_seen_at`` is updated rather than
|
||||
creating a duplicate.
|
||||
"""
|
||||
platform = body.platform.lower()
|
||||
if platform not in {"ios", "android", "web"}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="platform must be one of: ios, android, web",
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Upsert: reuse existing record if the token is already known.
|
||||
existing = (
|
||||
db.query(MobileDevice)
|
||||
.filter(MobileDevice.owner_id == owner_id, MobileDevice.push_token == body.push_token)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
existing.is_active = True
|
||||
existing.last_seen_at = now
|
||||
if body.device_name:
|
||||
existing.device_name = body.device_name
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(existing)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
logger.info("Mobile device refreshed: id=%s owner=%s", existing.id, owner_id)
|
||||
return _device_to_response(existing)
|
||||
|
||||
device = MobileDevice(
|
||||
owner_id=owner_id,
|
||||
device_name=body.device_name,
|
||||
platform=platform,
|
||||
push_token=body.push_token,
|
||||
is_active=True,
|
||||
last_seen_at=now,
|
||||
)
|
||||
try:
|
||||
db.add(device)
|
||||
db.commit()
|
||||
db.refresh(device)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to register mobile device for owner_id=%s", owner_id)
|
||||
raise
|
||||
|
||||
logger.info("Mobile device registered: id=%s owner=%s platform=%s", device.id, owner_id, platform)
|
||||
return _device_to_response(device)
|
||||
|
||||
|
||||
@router.get("/devices", response_model=list[DeviceResponse])
|
||||
@require_login
|
||||
async def list_devices(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all registered push-notification devices for the current user."""
|
||||
devices = (
|
||||
db.query(MobileDevice).filter(MobileDevice.owner_id == owner_id).order_by(MobileDevice.created_at.desc()).all()
|
||||
)
|
||||
return [_device_to_response(d) for d in devices]
|
||||
|
||||
|
||||
@router.delete("/devices/{device_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
async def deactivate_device(
|
||||
request: Request,
|
||||
device_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> None:
|
||||
"""Deactivate a push-notification device registration.
|
||||
|
||||
The device record is kept for audit purposes but will no longer receive
|
||||
push notifications.
|
||||
"""
|
||||
device = db.get(MobileDevice, device_id)
|
||||
if not device or device.owner_id != owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Device not found")
|
||||
|
||||
device.is_active = False
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id)
|
||||
|
||||
|
||||
@router.get("/whoami", response_model=WhoAmIResponse)
|
||||
@require_login
|
||||
async def whoami(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Return basic profile information for the authenticated user.
|
||||
|
||||
The mobile app calls this after token exchange to populate the user
|
||||
profile screen and verify that the stored token is still valid.
|
||||
"""
|
||||
from app.auth import get_gravatar_url
|
||||
from app.models import LocalUser, UserProfile
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first()
|
||||
local_user = db.query(LocalUser).filter(LocalUser.email == owner_id).first()
|
||||
|
||||
display_name: str | None = None
|
||||
email: str | None = None
|
||||
avatar_url: str | None = None
|
||||
is_admin = False
|
||||
|
||||
if profile:
|
||||
display_name = profile.display_name
|
||||
|
||||
if local_user:
|
||||
email = local_user.email
|
||||
is_admin = bool(local_user.is_admin)
|
||||
if not display_name and local_user.display_name:
|
||||
display_name = local_user.display_name
|
||||
elif "@" in owner_id:
|
||||
# SSO users commonly have their email as owner_id
|
||||
email = owner_id
|
||||
|
||||
if email:
|
||||
avatar_url = get_gravatar_url(email)
|
||||
|
||||
return {
|
||||
"owner_id": owner_id,
|
||||
"display_name": display_name,
|
||||
"email": email,
|
||||
"avatar_url": avatar_url,
|
||||
"is_admin": is_admin,
|
||||
}
|
||||
@@ -1,484 +0,0 @@
|
||||
"""API endpoints for per-user notification targets, preferences, and in-app inbox.
|
||||
|
||||
Users can define notification targets (email via SMTP, webhook via HTTP POST)
|
||||
and configure which document events trigger which targets. In-app notifications
|
||||
are always created and surfaced via the bell icon / inbox endpoints.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import InAppNotification, UserNotificationPreference, UserNotificationTarget
|
||||
from app.utils.user_notification import USER_EVENT_LABELS
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/user-notifications", tags=["user-notifications"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper (mirrors api_tokens.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CHANNEL_TYPES = {"email", "webhook"}
|
||||
VALID_EVENT_TYPES = set(USER_EVENT_LABELS.keys())
|
||||
|
||||
|
||||
class NotificationTargetCreate(BaseModel):
|
||||
"""Schema for creating a new notification target."""
|
||||
|
||||
channel_type: str = Field(..., pattern="^(email|webhook)$")
|
||||
name: str = Field(..., min_length=1, max_length=255)
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
class NotificationTargetUpdate(BaseModel):
|
||||
"""Schema for updating an existing notification target."""
|
||||
|
||||
name: str | None = Field(None, min_length=1, max_length=255)
|
||||
config: dict[str, Any] | None = None
|
||||
is_active: bool | None = None
|
||||
|
||||
|
||||
class PreferenceItem(BaseModel):
|
||||
"""A single preference toggle for one event+channel combination."""
|
||||
|
||||
is_enabled: bool
|
||||
target_id: int | None = None
|
||||
|
||||
|
||||
class PreferenceItemFull(BaseModel):
|
||||
"""Full preference item including event and channel type (used in bulk update)."""
|
||||
|
||||
event_type: str
|
||||
channel_type: str
|
||||
is_enabled: bool
|
||||
target_id: int | None = None
|
||||
|
||||
|
||||
class PreferencesUpdate(BaseModel):
|
||||
"""Bulk preferences update payload — a flat list of preference items."""
|
||||
|
||||
preferences: list[PreferenceItemFull]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mask_email_config(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return a copy of an email config dict with the password masked."""
|
||||
masked = dict(config)
|
||||
if masked.get("smtp_password"):
|
||||
masked["smtp_password"] = "****"
|
||||
return masked
|
||||
|
||||
|
||||
def _target_to_dict(target: UserNotificationTarget) -> dict[str, Any]:
|
||||
"""Serialize a UserNotificationTarget to a response dict, masking secrets."""
|
||||
config: dict[str, Any] = {}
|
||||
if target.config:
|
||||
try:
|
||||
config = json.loads(target.config)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
config = {}
|
||||
|
||||
if target.channel_type == "email":
|
||||
config = _mask_email_config(config)
|
||||
|
||||
return {
|
||||
"id": target.id,
|
||||
"channel_type": target.channel_type,
|
||||
"name": target.name,
|
||||
"config": config,
|
||||
"is_active": target.is_active,
|
||||
"created_at": target.created_at,
|
||||
"updated_at": target.updated_at,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inbox endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/inbox")
|
||||
async def list_inbox(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
skip: int = 0,
|
||||
limit: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List in-app notifications for the authenticated user, newest first."""
|
||||
notifications = (
|
||||
db.query(InAppNotification)
|
||||
.filter(InAppNotification.owner_id == owner_id)
|
||||
.order_by(InAppNotification.created_at.desc())
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": n.id,
|
||||
"event_type": n.event_type,
|
||||
"title": n.title,
|
||||
"message": n.message,
|
||||
"is_read": n.is_read,
|
||||
"file_id": n.file_id,
|
||||
"created_at": n.created_at,
|
||||
}
|
||||
for n in notifications
|
||||
]
|
||||
|
||||
|
||||
@router.get("/inbox/unread-count")
|
||||
async def unread_count(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, int]:
|
||||
"""Return the number of unread in-app notifications."""
|
||||
count = (
|
||||
db.query(InAppNotification)
|
||||
.filter(InAppNotification.owner_id == owner_id, InAppNotification.is_read == False) # noqa: E712
|
||||
.count()
|
||||
)
|
||||
return {"count": count}
|
||||
|
||||
|
||||
@router.post("/inbox/{notification_id}/read", status_code=status.HTTP_200_OK)
|
||||
async def mark_read(
|
||||
notification_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Mark a single in-app notification as read."""
|
||||
notif = (
|
||||
db.query(InAppNotification)
|
||||
.filter(InAppNotification.id == notification_id, InAppNotification.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not notif:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Notification not found")
|
||||
try:
|
||||
notif.is_read = True
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
return {"detail": "Marked as read"}
|
||||
|
||||
|
||||
@router.post("/inbox/read-all", status_code=status.HTTP_200_OK)
|
||||
async def mark_all_read(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Mark all in-app notifications as read for the authenticated user."""
|
||||
try:
|
||||
db.query(InAppNotification).filter(
|
||||
InAppNotification.owner_id == owner_id,
|
||||
InAppNotification.is_read == False, # noqa: E712
|
||||
).update({"is_read": True})
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
return {"detail": "All notifications marked as read"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notification target endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/targets")
|
||||
async def list_targets(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all notification targets for the authenticated user."""
|
||||
targets = (
|
||||
db.query(UserNotificationTarget)
|
||||
.filter(UserNotificationTarget.owner_id == owner_id)
|
||||
.order_by(UserNotificationTarget.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
return [_target_to_dict(t) for t in targets]
|
||||
|
||||
|
||||
@router.post("/targets", status_code=status.HTTP_201_CREATED)
|
||||
async def create_target(
|
||||
body: NotificationTargetCreate,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new notification target (email or webhook)."""
|
||||
target = UserNotificationTarget(
|
||||
owner_id=owner_id,
|
||||
channel_type=body.channel_type,
|
||||
name=body.name,
|
||||
config=json.dumps(body.config),
|
||||
is_active=body.is_active,
|
||||
)
|
||||
try:
|
||||
db.add(target)
|
||||
db.commit()
|
||||
db.refresh(target)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Notification target created: id=%s owner=%s type=%s", target.id, owner_id, body.channel_type)
|
||||
return _target_to_dict(target)
|
||||
|
||||
|
||||
@router.put("/targets/{target_id}", status_code=status.HTTP_200_OK)
|
||||
async def update_target(
|
||||
target_id: int,
|
||||
body: NotificationTargetUpdate,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing notification target."""
|
||||
target = (
|
||||
db.query(UserNotificationTarget)
|
||||
.filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not target:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found")
|
||||
|
||||
try:
|
||||
if body.name is not None:
|
||||
target.name = body.name
|
||||
if body.config is not None:
|
||||
# Merge new config over existing, preserving masked password field if unchanged
|
||||
existing_config: dict[str, Any] = {}
|
||||
if target.config:
|
||||
try:
|
||||
existing_config = json.loads(target.config)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
existing_config = {}
|
||||
merged = dict(existing_config)
|
||||
for k, v in body.config.items():
|
||||
# Skip writing back a masked password placeholder
|
||||
if k == "smtp_password" and v == "****":
|
||||
continue
|
||||
merged[k] = v
|
||||
target.config = json.dumps(merged)
|
||||
if body.is_active is not None:
|
||||
target.is_active = body.is_active
|
||||
db.commit()
|
||||
db.refresh(target)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Notification target updated: id=%s owner=%s", target_id, owner_id)
|
||||
return _target_to_dict(target)
|
||||
|
||||
|
||||
@router.delete("/targets/{target_id}", status_code=status.HTTP_200_OK)
|
||||
async def delete_target(
|
||||
target_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Delete a notification target and its associated preferences."""
|
||||
target = (
|
||||
db.query(UserNotificationTarget)
|
||||
.filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not target:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found")
|
||||
|
||||
try:
|
||||
# Remove any preferences that reference this target
|
||||
db.query(UserNotificationPreference).filter(
|
||||
UserNotificationPreference.owner_id == owner_id,
|
||||
UserNotificationPreference.target_id == target_id,
|
||||
).delete()
|
||||
db.delete(target)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Notification target deleted: id=%s owner=%s", target_id, owner_id)
|
||||
return {"detail": "Target deleted"}
|
||||
|
||||
|
||||
@router.post("/targets/{target_id}/test", status_code=status.HTTP_200_OK)
|
||||
async def test_target(
|
||||
target_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Send a test notification to the specified target."""
|
||||
target = (
|
||||
db.query(UserNotificationTarget)
|
||||
.filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id)
|
||||
.first()
|
||||
)
|
||||
if not target:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found")
|
||||
|
||||
config: dict[str, Any] = {}
|
||||
if target.config:
|
||||
try:
|
||||
config = json.loads(target.config)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
config = {}
|
||||
|
||||
title = "DocuElevate Test Notification"
|
||||
message = f"This is a test notification from DocuElevate for target '{target.name}'."
|
||||
|
||||
if target.channel_type == "email":
|
||||
from app.utils.user_notification import _send_email_notification
|
||||
|
||||
ok = _send_email_notification(config, title, message)
|
||||
elif target.channel_type == "webhook":
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
ok = _send_webhook_notification(config, "test", title, message)
|
||||
else:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown channel type")
|
||||
|
||||
if not ok:
|
||||
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="Failed to send test notification")
|
||||
|
||||
return {"detail": "Test notification sent"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preferences endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/preferences")
|
||||
async def get_preferences(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Return all notification preferences for the authenticated user.
|
||||
|
||||
Response structure:
|
||||
{
|
||||
"event_types": ["document.processed", "document.failed"],
|
||||
"event_labels": {"document.processed": "Document Processed", ...},
|
||||
"preferences": {
|
||||
"document.processed": {
|
||||
"in_app": {"is_enabled": true, "target_id": null},
|
||||
"email": {"is_enabled": false, "target_id": 1},
|
||||
...
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
prefs = db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
|
||||
|
||||
# Build nested dict: event_type -> channel_type -> {is_enabled, target_id}
|
||||
result: dict[str, dict[str, dict[str, Any]]] = {}
|
||||
for pref in prefs:
|
||||
result.setdefault(pref.event_type, {})[pref.channel_type] = {
|
||||
"is_enabled": pref.is_enabled,
|
||||
"target_id": pref.target_id,
|
||||
}
|
||||
|
||||
return {
|
||||
"event_types": list(USER_EVENT_LABELS.keys()),
|
||||
"event_labels": USER_EVENT_LABELS,
|
||||
"preferences": result,
|
||||
}
|
||||
|
||||
|
||||
@router.put("/preferences", status_code=status.HTTP_200_OK)
|
||||
async def update_preferences(
|
||||
body: PreferencesUpdate,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Bulk upsert notification preferences for the authenticated user.
|
||||
|
||||
Validates that any referenced target_id belongs to the requesting user.
|
||||
"""
|
||||
# Collect all target IDs referenced in the payload for ownership validation
|
||||
referenced_target_ids: set[int] = set()
|
||||
for item in body.preferences:
|
||||
if item.target_id is not None:
|
||||
referenced_target_ids.add(item.target_id)
|
||||
|
||||
if referenced_target_ids:
|
||||
owned_ids = {
|
||||
row.id
|
||||
for row in db.query(UserNotificationTarget.id)
|
||||
.filter(
|
||||
UserNotificationTarget.owner_id == owner_id,
|
||||
UserNotificationTarget.id.in_(referenced_target_ids),
|
||||
)
|
||||
.all()
|
||||
}
|
||||
invalid = referenced_target_ids - owned_ids
|
||||
if invalid:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid or inaccessible target_id(s): {sorted(invalid)}",
|
||||
)
|
||||
|
||||
try:
|
||||
for item in body.preferences:
|
||||
existing = (
|
||||
db.query(UserNotificationPreference)
|
||||
.filter(
|
||||
UserNotificationPreference.owner_id == owner_id,
|
||||
UserNotificationPreference.event_type == item.event_type,
|
||||
UserNotificationPreference.channel_type == item.channel_type,
|
||||
UserNotificationPreference.target_id == item.target_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
existing.is_enabled = item.is_enabled
|
||||
else:
|
||||
db.add(
|
||||
UserNotificationPreference(
|
||||
owner_id=owner_id,
|
||||
event_type=item.event_type,
|
||||
channel_type=item.channel_type,
|
||||
target_id=item.target_id,
|
||||
is_enabled=item.is_enabled,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Notification preferences updated for owner=%s", owner_id)
|
||||
return {"detail": "Preferences updated"}
|
||||
@@ -1,253 +0,0 @@
|
||||
"""API endpoints for the user onboarding wizard.
|
||||
|
||||
Provides a REST interface for the multi-step onboarding flow, allowing
|
||||
authenticated users to set their profile, choose a subscription plan,
|
||||
select a storage destination, and mark onboarding as complete.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import UserProfile
|
||||
from app.utils.subscription import TIERS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/onboarding", tags=["onboarding"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_current_user_id(request: Request) -> str:
|
||||
"""Extract the stable user_id from the session using the same priority as _ensure_user_profile.
|
||||
|
||||
Priority: sub → preferred_username → email → id.
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 if the user is not authenticated.
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
user_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return user_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProfileBody(BaseModel):
|
||||
"""Body for the profile step of the onboarding wizard."""
|
||||
|
||||
display_name: str | None = Field(default=None, max_length=255)
|
||||
contact_email: str | None = Field(default=None, max_length=255)
|
||||
|
||||
|
||||
class PlanBody(BaseModel):
|
||||
"""Body for the plan step of the onboarding wizard."""
|
||||
|
||||
subscription_tier: str
|
||||
billing_cycle: str = Field(pattern="^(monthly|yearly)$")
|
||||
|
||||
|
||||
class StorageBody(BaseModel):
|
||||
"""Body for the storage step of the onboarding wizard."""
|
||||
|
||||
preferred_destination: str | None = Field(default=None, max_length=50)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _profile_to_dict(profile: UserProfile) -> dict[str, Any]:
|
||||
"""Serialize a UserProfile to a plain dict for API responses."""
|
||||
return {
|
||||
"user_id": profile.user_id,
|
||||
"display_name": profile.display_name,
|
||||
"contact_email": profile.contact_email,
|
||||
"subscription_tier": profile.subscription_tier or "free",
|
||||
"subscription_billing_cycle": profile.subscription_billing_cycle or "monthly",
|
||||
"preferred_destination": profile.preferred_destination,
|
||||
"onboarding_completed": bool(profile.onboarding_completed),
|
||||
"onboarding_completed_at": profile.onboarding_completed_at.isoformat()
|
||||
if profile.onboarding_completed_at
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
def _get_or_create_profile(db: Session, user_id: str) -> UserProfile:
|
||||
"""Return the UserProfile for *user_id*, creating one if it does not exist."""
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile is None:
|
||||
profile = UserProfile(user_id=user_id)
|
||||
db.add(profile)
|
||||
db.flush()
|
||||
return profile
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/status", summary="Get onboarding status for the current user")
|
||||
def get_onboarding_status(request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Return whether onboarding has been completed and the current step.
|
||||
|
||||
The ``step`` field is a best-effort estimate: 1 for brand-new profiles,
|
||||
further along when partial data has already been saved.
|
||||
"""
|
||||
user_id = _get_current_user_id(request)
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
|
||||
if profile is None:
|
||||
return {"completed": False, "step": 1, "profile": None}
|
||||
|
||||
# Derive a sensible current step from saved data so the wizard can resume.
|
||||
step = 1
|
||||
if profile.display_name or profile.contact_email:
|
||||
step = 2
|
||||
if profile.subscription_tier and profile.subscription_tier != "free":
|
||||
step = 3
|
||||
if profile.preferred_destination:
|
||||
step = 4
|
||||
if profile.onboarding_completed:
|
||||
step = 5
|
||||
|
||||
return {
|
||||
"completed": bool(profile.onboarding_completed),
|
||||
"step": step,
|
||||
"profile": _profile_to_dict(profile),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/profile", summary="Save profile step during onboarding")
|
||||
def save_profile(request: Request, body: ProfileBody, db: DbSession) -> dict[str, Any]:
|
||||
"""Persist the user's display name and contact email from the profile step."""
|
||||
user_id = _get_current_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
|
||||
if body.display_name is not None:
|
||||
profile.display_name = body.display_name
|
||||
if body.contact_email is not None:
|
||||
profile.contact_email = body.contact_email
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Onboarding: saved profile for user %s", user_id)
|
||||
return _profile_to_dict(profile)
|
||||
|
||||
|
||||
@router.post("/plan", summary="Save plan selection during onboarding")
|
||||
def save_plan(request: Request, body: PlanBody, db: DbSession) -> dict[str, Any]:
|
||||
"""Persist the chosen subscription tier and billing cycle from the plan step.
|
||||
|
||||
Raises:
|
||||
HTTPException: 422 if the tier is not a recognised value.
|
||||
"""
|
||||
user_id = _get_current_user_id(request)
|
||||
|
||||
if body.subscription_tier not in TIERS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid subscription_tier '{body.subscription_tier}'. Valid values: {list(TIERS.keys())}",
|
||||
)
|
||||
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
old_tier = profile.subscription_tier or "free"
|
||||
profile.subscription_tier = body.subscription_tier
|
||||
profile.subscription_billing_cycle = body.billing_cycle
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Onboarding: saved plan %s/%s", body.subscription_tier, body.billing_cycle)
|
||||
|
||||
# Notify admins and fire webhook when the plan actually changes
|
||||
if old_tier != body.subscription_tier:
|
||||
try:
|
||||
from app.utils.notification import notify_plan_changed
|
||||
from app.utils.webhook import dispatch_webhook_event
|
||||
|
||||
notify_plan_changed(user_id, old_tier=old_tier, new_tier=body.subscription_tier, changed_by="user")
|
||||
dispatch_webhook_event(
|
||||
"user.plan_changed",
|
||||
{
|
||||
"user_id": user_id,
|
||||
"old_tier": old_tier,
|
||||
"new_tier": body.subscription_tier,
|
||||
"billing_cycle": body.billing_cycle,
|
||||
"changed_by": "user",
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to send plan-change notification/webhook for user %s", user_id)
|
||||
|
||||
return _profile_to_dict(profile)
|
||||
|
||||
|
||||
@router.post("/storage", summary="Save storage preference during onboarding")
|
||||
def save_storage(request: Request, body: StorageBody, db: DbSession) -> dict[str, Any]:
|
||||
"""Persist the user's preferred storage destination from the storage step."""
|
||||
user_id = _get_current_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
profile.preferred_destination = body.preferred_destination
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Onboarding: saved storage preference '%s' for user %s", body.preferred_destination, user_id)
|
||||
return _profile_to_dict(profile)
|
||||
|
||||
|
||||
@router.post("/complete", summary="Mark onboarding as completed")
|
||||
def complete_onboarding(request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Set onboarding_completed=True, record the completion timestamp, and return the post-onboarding redirect URL.
|
||||
|
||||
The redirect URL is read from ``request.session["post_onboarding_redirect"]`` (stored by
|
||||
``oauth_callback`` when it reroutes a first-time user to the wizard) and defaults to
|
||||
``/upload`` when the session key is absent.
|
||||
"""
|
||||
user_id = _get_current_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
profile.onboarding_completed = True
|
||||
profile.onboarding_completed_at = datetime.now(tz=timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
redirect_url = request.session.pop("post_onboarding_redirect", "/upload")
|
||||
logger.info("Onboarding: completed for user %s, redirecting to %s", user_id, redirect_url)
|
||||
return {"success": True, "redirect_url": redirect_url}
|
||||
+248
-232
@@ -1,64 +1,115 @@
|
||||
"""
|
||||
OneDrive API endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Request, HTTPException, status, Form
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/onedrive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_onedrive_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
tenant_id: Annotated[str, Form(...)],
|
||||
client_id: str = Form(...),
|
||||
client_secret: str = Form(...),
|
||||
redirect_uri: str = Form(...),
|
||||
code: str = Form(...),
|
||||
tenant_id: str = Form(...)
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for a refresh token.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"scope": "https://graph.microsoft.com/.default offline_access",
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
"client_secret": client_secret,
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="OneDrive", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600),
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info(f"Starting OneDrive token exchange process with tenant_id: {tenant_id}")
|
||||
|
||||
# Prepare the token request
|
||||
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
logger.info(f"Using token URL: {token_url}")
|
||||
|
||||
payload = {
|
||||
'client_id': client_id,
|
||||
'scope': 'https://graph.microsoft.com/.default offline_access',
|
||||
'code': code,
|
||||
'redirect_uri': redirect_uri,
|
||||
'grant_type': 'authorization_code',
|
||||
'client_secret': client_secret
|
||||
}
|
||||
|
||||
# Log request details (excluding secret)
|
||||
safe_payload = payload.copy()
|
||||
safe_payload['client_secret'] = '[REDACTED]'
|
||||
safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
|
||||
logger.info(f"Token exchange request payload: {safe_payload}")
|
||||
|
||||
# Make the token request
|
||||
logger.info("Sending POST request to Microsoft for token exchange")
|
||||
response = requests.post(token_url, data=payload)
|
||||
|
||||
# Check if the request was successful
|
||||
logger.info(f"Token exchange response status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
# Log the error response for debugging
|
||||
try:
|
||||
error_json = response.json()
|
||||
logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
|
||||
error_detail = error_json
|
||||
except Exception as json_err:
|
||||
logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
|
||||
logger.error(f"Raw response content: {response.content[:500]}") # Limit log size
|
||||
error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
|
||||
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Token exchange failed: {error_detail}"
|
||||
)
|
||||
|
||||
# Return the token response
|
||||
token_data = response.json()
|
||||
|
||||
# Validate the token response
|
||||
if "refresh_token" not in token_data:
|
||||
logger.error(f"Microsoft returned success but no refresh token found in response: {token_data.keys()}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="Microsoft OAuth server returned success but no refresh token was included"
|
||||
)
|
||||
|
||||
# Calculate token length for logging
|
||||
refresh_token_length = len(token_data.get("refresh_token", ""))
|
||||
access_token_length = len(token_data.get("access_token", ""))
|
||||
|
||||
logger.info(f"Successfully exchanged authorization code for OneDrive tokens. "
|
||||
f"Refresh token length: {refresh_token_length}, "
|
||||
f"Access token length: {access_token_length}")
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600)
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions as they already have appropriate status codes
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error during OneDrive token exchange: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to exchange token: {str(e)}"
|
||||
)
|
||||
|
||||
@router.get("/onedrive/test-token")
|
||||
@require_login
|
||||
@@ -68,153 +119,133 @@ async def test_onedrive_token(request: Request):
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing OneDrive token validity")
|
||||
|
||||
if (
|
||||
not settings.onedrive_refresh_token
|
||||
or not settings.onedrive_client_id
|
||||
or not settings.onedrive_client_secret
|
||||
):
|
||||
|
||||
if not settings.onedrive_refresh_token or not settings.onedrive_client_id or not settings.onedrive_client_secret:
|
||||
logger.warning("OneDrive credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "OneDrive credentials are not fully configured",
|
||||
"status": "error",
|
||||
"message": "OneDrive credentials are not fully configured"
|
||||
}
|
||||
|
||||
|
||||
# Refresh token to get a new access token and expiration info
|
||||
tenant_id = settings.onedrive_tenant_id or "common"
|
||||
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
|
||||
|
||||
refresh_data = {
|
||||
"client_id": settings.onedrive_client_id,
|
||||
"client_secret": settings.onedrive_client_secret,
|
||||
"refresh_token": settings.onedrive_refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
"scope": "offline_access Files.ReadWrite",
|
||||
"scope": "offline_access Files.ReadWrite"
|
||||
}
|
||||
|
||||
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
|
||||
|
||||
response = requests.post(token_url, data=refresh_data)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Failed to refresh OneDrive token: {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
"needs_reauth": True
|
||||
}
|
||||
|
||||
|
||||
token_data = response.json()
|
||||
access_token = token_data.get("access_token")
|
||||
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
|
||||
|
||||
|
||||
# Check if we got a new refresh token (Microsoft sometimes issues a new one)
|
||||
new_refresh_token = token_data.get("refresh_token")
|
||||
if new_refresh_token and new_refresh_token != settings.onedrive_refresh_token:
|
||||
logger.info("Received new refresh token from Microsoft - will update configuration")
|
||||
|
||||
|
||||
# Update refresh token in memory
|
||||
settings.onedrive_refresh_token = new_refresh_token
|
||||
|
||||
|
||||
# Also try to update .env file if it exists
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if os.path.exists(env_path):
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
|
||||
updated_lines = []
|
||||
updated = False
|
||||
|
||||
|
||||
for line in env_lines:
|
||||
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
updated = True
|
||||
else:
|
||||
updated_lines.append(line)
|
||||
|
||||
|
||||
if not updated:
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.writelines(updated_lines)
|
||||
|
||||
|
||||
logger.info("Updated refresh token in .env file")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update refresh token in .env file: {e}")
|
||||
|
||||
# Persist the rotated refresh token to the database
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
|
||||
_db = SessionLocal()
|
||||
try:
|
||||
save_setting_to_db(
|
||||
_db,
|
||||
"onedrive_refresh_token",
|
||||
new_refresh_token,
|
||||
changed_by="onedrive_token_rotation",
|
||||
)
|
||||
notify_settings_updated()
|
||||
finally:
|
||||
_db.close()
|
||||
except Exception as _e:
|
||||
logger.warning(f"Failed to persist rotated OneDrive refresh token to database: {_e}")
|
||||
|
||||
|
||||
# Test the access token by getting user information
|
||||
user_info_url = "https://graph.microsoft.com/v1.0/me"
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
|
||||
|
||||
|
||||
user_response = requests.get(user_info_url, headers=headers)
|
||||
|
||||
if user_response.status_code != 200:
|
||||
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
|
||||
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}"
|
||||
}
|
||||
|
||||
|
||||
# Get user info
|
||||
user_info = user_response.json()
|
||||
display_name = user_info.get("displayName", "Unknown user")
|
||||
email = user_info.get("userPrincipalName", "Unknown email")
|
||||
|
||||
|
||||
# Calculate expiration time
|
||||
now = datetime.now()
|
||||
expiry_time = now + timedelta(seconds=expires_in)
|
||||
|
||||
|
||||
# Format expiration info
|
||||
time_left = expiry_time - now
|
||||
token_info = {
|
||||
"expires_at": expiry_time.isoformat(),
|
||||
"expires_in_seconds": expires_in,
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
"refresh_token_validity": "Refresh token is valid for 90 days of inactivity",
|
||||
"refresh_token_validity": "Refresh token is valid for 90 days of inactivity"
|
||||
}
|
||||
|
||||
|
||||
logger.info(f"Successfully connected to OneDrive as {email}")
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OneDrive connection successful",
|
||||
"message": f"OneDrive connection successful",
|
||||
"account": email,
|
||||
"account_name": display_name,
|
||||
"token_info": token_info,
|
||||
"token_info": token_info
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error testing OneDrive token: {str(e)}")
|
||||
return {"status": "error", "message": f"Connection error: {str(e)}"}
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection error: {str(e)}"
|
||||
}
|
||||
|
||||
def format_time_remaining(time_delta):
|
||||
"""Format a timedelta into a human-readable string."""
|
||||
if time_delta.total_seconds() <= 0:
|
||||
return "Expired"
|
||||
|
||||
|
||||
days = time_delta.days
|
||||
hours, remainder = divmod(time_delta.seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
|
||||
|
||||
parts = []
|
||||
if days > 0:
|
||||
parts.append(f"{days} day{'s' if days != 1 else ''}")
|
||||
@@ -222,76 +253,80 @@ def format_time_remaining(time_delta):
|
||||
parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
|
||||
if minutes > 0 and days == 0: # Only show minutes if less than a day
|
||||
parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
|
||||
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@router.post("/onedrive/save-settings")
|
||||
@require_login
|
||||
async def save_onedrive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
tenant_id: Annotated[str, Form()] = "common",
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
client_id: str = Form(None),
|
||||
client_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
tenant_id: str = Form("common"),
|
||||
folder_path: str = Form(None)
|
||||
):
|
||||
"""
|
||||
Saves to database (primary) and .env file (best-effort).
|
||||
Save OneDrive settings to the .env file
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Best-effort .env file write
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if not os.path.exists(env_path):
|
||||
logger.warning(f".env file not found at {env_path}, skipping file write")
|
||||
else:
|
||||
logger.info(f"Updating OneDrive settings in {env_path}")
|
||||
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
|
||||
if client_id:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
|
||||
if tenant_id:
|
||||
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
|
||||
if folder_path:
|
||||
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
|
||||
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in onedrive_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
for key, value in onedrive_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated OneDrive settings in .env file")
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
|
||||
# Get the path to the .env file
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
|
||||
if not os.path.exists(env_path):
|
||||
logger.error(f".env file not found at {env_path}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Could not find .env file to update"
|
||||
)
|
||||
|
||||
logger.info(f"Updating OneDrive settings in {env_path}")
|
||||
|
||||
# Read the current .env file
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
# Define settings to update
|
||||
onedrive_settings = {
|
||||
"ONEDRIVE_REFRESH_TOKEN": refresh_token,
|
||||
}
|
||||
|
||||
# Only update these if provided
|
||||
if client_id:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
|
||||
if tenant_id:
|
||||
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
|
||||
if folder_path:
|
||||
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
|
||||
|
||||
# Process each line and update or add settings
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in onedrive_settings.items():
|
||||
if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
|
||||
if line.startswith("# "): # Uncomment if commented out
|
||||
line = line[2:]
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(line)
|
||||
|
||||
# Add any settings that weren't updated (they weren't in the file)
|
||||
for key, value in onedrive_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
# Write the updated .env file
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
# Update the settings in memory
|
||||
if refresh_token:
|
||||
settings.onedrive_refresh_token = refresh_token
|
||||
@@ -303,108 +338,84 @@ async def save_onedrive_settings(
|
||||
settings.onedrive_tenant_id = tenant_id
|
||||
if folder_path:
|
||||
settings.onedrive_folder_path = folder_path
|
||||
|
||||
# Persist to database (primary)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if client_id:
|
||||
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
|
||||
if client_secret:
|
||||
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
|
||||
if tenant_id:
|
||||
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
|
||||
if folder_path:
|
||||
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully saved OneDrive settings")
|
||||
return {"status": "success", "message": "OneDrive settings have been saved"}
|
||||
|
||||
|
||||
logger.info("Successfully updated OneDrive settings")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OneDrive settings have been saved"
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save OneDrive settings: {str(e)}",
|
||||
detail=f"Failed to save OneDrive settings: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/onedrive/update-settings")
|
||||
@require_login
|
||||
async def update_onedrive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
tenant_id: Annotated[str, Form()] = "common",
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
client_id: str = Form(None),
|
||||
client_secret: str = Form(None),
|
||||
refresh_token: str = Form(...),
|
||||
tenant_id: str = Form("common"),
|
||||
folder_path: str = Form(None)
|
||||
):
|
||||
"""
|
||||
Update OneDrive settings in memory and persist to database
|
||||
Update OneDrive settings in memory (without modifying .env file)
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating OneDrive settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
logger.info("Updating OneDrive settings in memory")
|
||||
|
||||
# Update settings in memory
|
||||
if refresh_token:
|
||||
settings.onedrive_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory and database")
|
||||
|
||||
logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory")
|
||||
|
||||
if client_id:
|
||||
settings.onedrive_client_id = client_id
|
||||
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_CLIENT_ID in memory and database")
|
||||
|
||||
logger.info("Updated ONEDRIVE_CLIENT_ID in memory")
|
||||
|
||||
if client_secret:
|
||||
settings.onedrive_client_secret = client_secret
|
||||
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory and database")
|
||||
|
||||
logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory")
|
||||
|
||||
if tenant_id:
|
||||
settings.onedrive_tenant_id = tenant_id
|
||||
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_TENANT_ID in memory and database")
|
||||
|
||||
logger.info("Updated ONEDRIVE_TENANT_ID in memory")
|
||||
|
||||
if folder_path:
|
||||
settings.onedrive_folder_path = folder_path
|
||||
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_FOLDER_PATH in memory and database")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Updated ONEDRIVE_FOLDER_PATH in memory")
|
||||
|
||||
# Test the token to make sure it works
|
||||
try:
|
||||
from app.tasks.upload_to_onedrive import get_onedrive_token
|
||||
|
||||
get_onedrive_token() # Test that token can be retrieved
|
||||
access_token = get_onedrive_token()
|
||||
logger.info("Successfully tested OneDrive token")
|
||||
except Exception as e:
|
||||
logger.error(f"Token test failed after updating settings: {str(e)}")
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "Settings updated but token test failed: " + str(e),
|
||||
"message": "Settings updated but token test failed: " + str(e)
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OneDrive settings have been updated in memory and database",
|
||||
"message": "OneDrive settings have been updated in memory"
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update OneDrive settings: {str(e)}",
|
||||
detail=f"Failed to update OneDrive settings: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/onedrive/get-full-config")
|
||||
@require_login
|
||||
async def get_onedrive_full_config(request: Request):
|
||||
@@ -418,21 +429,26 @@ async def get_onedrive_full_config(request: Request):
|
||||
"client_secret": settings.onedrive_client_secret or "",
|
||||
"tenant_id": settings.onedrive_tenant_id or "common",
|
||||
"refresh_token": settings.onedrive_refresh_token or "",
|
||||
"folder_path": settings.onedrive_folder_path or "Documents/Uploads",
|
||||
"folder_path": settings.onedrive_folder_path or "Documents/Uploads"
|
||||
}
|
||||
|
||||
|
||||
# Generate environment variable format
|
||||
env_format = "\n".join(
|
||||
[
|
||||
f"ONEDRIVE_CLIENT_ID={config['client_id']}",
|
||||
f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}",
|
||||
f"ONEDRIVE_TENANT_ID={config['tenant_id']}",
|
||||
f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}",
|
||||
f"ONEDRIVE_FOLDER_PATH={config['folder_path']}",
|
||||
]
|
||||
)
|
||||
|
||||
return {"status": "success", "config": config, "env_format": env_format}
|
||||
env_format = "\n".join([
|
||||
f"ONEDRIVE_CLIENT_ID={config['client_id']}",
|
||||
f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}",
|
||||
f"ONEDRIVE_TENANT_ID={config['tenant_id']}",
|
||||
f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}",
|
||||
f"ONEDRIVE_FOLDER_PATH={config['folder_path']}"
|
||||
])
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"config": config,
|
||||
"env_format": env_format
|
||||
}
|
||||
except Exception as e:
|
||||
logger.exception("Error getting OneDrive configuration")
|
||||
return {"status": "error", "message": str(e)}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": str(e)
|
||||
}
|
||||
|
||||
+26
-285
@@ -1,20 +1,10 @@
|
||||
"""
|
||||
AI provider and OpenAI API endpoints.
|
||||
|
||||
Exposes three endpoints:
|
||||
- GET /api/ai/test – tests the currently configured AI provider (generic, provider-agnostic)
|
||||
- GET /api/openai/test – backward-compatible alias that tests the OpenAI API specifically
|
||||
- POST /api/ai/test-extraction – runs the metadata-extraction prompt against the configured AI provider
|
||||
with caller-supplied plaintext and returns the raw response, parsed JSON,
|
||||
and extracted tags so operators can evaluate model quality.
|
||||
OpenAI API endpoints
|
||||
"""
|
||||
|
||||
import json
|
||||
from fastapi import APIRouter, Request, HTTPException, status
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel, Field
|
||||
import os
|
||||
import requests
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
@@ -24,33 +14,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Maximum number of characters accepted for a test-extraction request.
|
||||
# Keeps individual requests reasonable without blocking any real-world document.
|
||||
_MAX_EXTRACTION_TEXT_LEN = 50_000
|
||||
|
||||
|
||||
def _get_exception_chain_detail(exc: Exception) -> str:
|
||||
"""
|
||||
Extract a verbose diagnostic message by walking the full exception chain.
|
||||
|
||||
Surfaces DNS resolution failures, TCP connection refused errors, SSL issues,
|
||||
and other low-level network problems that are normally hidden behind a generic
|
||||
'Connection error.' message.
|
||||
"""
|
||||
parts: list[str] = [str(exc)]
|
||||
|
||||
cause = getattr(exc, "__cause__", None) or getattr(exc, "__context__", None)
|
||||
seen: set[int] = {id(exc)}
|
||||
while cause is not None and id(cause) not in seen:
|
||||
seen.add(id(cause))
|
||||
cause_str = str(cause)
|
||||
if cause_str and cause_str not in parts:
|
||||
parts.append(f"caused by: {type(cause).__name__}: {cause_str}")
|
||||
cause = getattr(cause, "__cause__", None) or getattr(cause, "__context__", None)
|
||||
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
@router.get("/openai/test")
|
||||
@require_login
|
||||
async def test_openai_connection(request: Request):
|
||||
@@ -59,276 +22,54 @@ async def test_openai_connection(request: Request):
|
||||
"""
|
||||
try:
|
||||
import openai
|
||||
|
||||
|
||||
logger.info("Testing OpenAI API key validity")
|
||||
|
||||
|
||||
# Check if API key is configured
|
||||
if not settings.openai_api_key:
|
||||
logger.warning("No OpenAI API key configured")
|
||||
return {"status": "error", "message": "No OpenAI API key is configured"}
|
||||
|
||||
# Configure the client, explicitly passing base_url so the sanitized
|
||||
# value from Settings (strip_outer_quotes) is used instead of the raw
|
||||
# OPENAI_BASE_URL env var which may contain literal quote characters.
|
||||
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "No OpenAI API key is configured"
|
||||
}
|
||||
|
||||
# Configure the client
|
||||
client = openai.OpenAI(api_key=settings.openai_api_key)
|
||||
|
||||
# Try to make a simple request to validate the key
|
||||
try:
|
||||
# Use a models list endpoint as a simple validation
|
||||
models = client.models.list()
|
||||
|
||||
|
||||
# If we got here, the key is valid
|
||||
logger.info("OpenAI API key is valid")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OpenAI API key is valid",
|
||||
"models_available": len(models.data) if hasattr(models, "data") else "Unknown",
|
||||
"models_available": len(models.data) if hasattr(models, "data") else "Unknown"
|
||||
}
|
||||
|
||||
except openai.APITimeoutError as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"OpenAI API request timed out: {detail}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Request timed out: {detail}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "timeout",
|
||||
}
|
||||
|
||||
except openai.APIConnectionError as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
base_url = getattr(getattr(client, "_client", None), "base_url", None)
|
||||
base_url_info = f" (base_url: {base_url})" if base_url else ""
|
||||
logger.error(
|
||||
f"OpenAI API connection error{base_url_info}: {detail}",
|
||||
exc_info=True,
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection error{base_url_info}: {detail}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "connection_error",
|
||||
}
|
||||
|
||||
except openai.AuthenticationError as e:
|
||||
logger.error(f"OpenAI authentication error (status {e.status_code}): {e.message}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Authentication failed: {e.message}",
|
||||
"is_auth_error": True,
|
||||
"error_type": "authentication_error",
|
||||
}
|
||||
|
||||
except openai.RateLimitError as e:
|
||||
logger.warning(f"OpenAI rate limit exceeded (status {e.status_code}): {e.message}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Rate limit exceeded: {e.message}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "rate_limit",
|
||||
}
|
||||
|
||||
except openai.APIStatusError as e:
|
||||
logger.error(
|
||||
f"OpenAI API returned HTTP {e.status_code}: {e.message} | "
|
||||
f"request_id={e.response.headers.get('x-request-id', 'n/a')}"
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"API error (HTTP {e.status_code}): {e.message}",
|
||||
"is_auth_error": e.status_code == 401,
|
||||
"error_type": "api_status_error",
|
||||
"http_status": e.status_code,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"OpenAI API key test failed: {error_msg}", exc_info=True)
|
||||
|
||||
logger.error(f"OpenAI API key test failed: {error_msg}")
|
||||
|
||||
# Determine if this is an authentication error
|
||||
is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower()
|
||||
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"API key validation failed: {error_msg}",
|
||||
"is_auth_error": is_auth_error,
|
||||
"is_auth_error": is_auth_error
|
||||
}
|
||||
|
||||
|
||||
except ImportError:
|
||||
logger.exception("OpenAI package not installed")
|
||||
return {"status": "error", "message": "OpenAI package not installed"}
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "OpenAI package not installed"
|
||||
}
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing OpenAI connection")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
|
||||
@router.get("/ai/test")
|
||||
@require_login
|
||||
async def test_ai_provider_connection(request: Request):
|
||||
"""
|
||||
Test the currently configured AI provider connection.
|
||||
|
||||
Uses ``get_ai_provider()`` to instantiate the active provider and sends a
|
||||
minimal chat completion to verify that the credentials and endpoint are
|
||||
reachable. Works for all supported providers (OpenAI, Azure, Anthropic,
|
||||
Gemini, Ollama, OpenRouter, Portkey, LiteLLM).
|
||||
"""
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
provider_name = settings.ai_provider
|
||||
model = settings.ai_model or settings.openai_model
|
||||
|
||||
logger.info(f"Testing AI provider connection: provider={provider_name}, model={model}")
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
response = provider.chat_completion(
|
||||
messages=[{"role": "user", "content": "Reply with the single word: ok"}],
|
||||
model=model,
|
||||
temperature=0,
|
||||
max_tokens=5,
|
||||
)
|
||||
logger.info(f"AI provider test successful: provider={provider_name}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"AI provider '{provider_name}' is reachable and responding",
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"response_preview": (response or "")[:50],
|
||||
}
|
||||
except ValueError as e:
|
||||
# Configuration errors (missing keys, unknown provider)
|
||||
logger.warning(f"AI provider configuration error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": str(e),
|
||||
"provider": provider_name,
|
||||
"message": f"Unexpected error: {str(e)}"
|
||||
}
|
||||
except Exception as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"AI provider test failed for '{provider_name}': {detail}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection failed: {detail}",
|
||||
"provider": provider_name,
|
||||
}
|
||||
|
||||
|
||||
class ExtractionTestRequest(BaseModel):
|
||||
"""Request body for the AI extraction test endpoint."""
|
||||
|
||||
text: str = Field(..., min_length=1, max_length=_MAX_EXTRACTION_TEXT_LEN, description="Plain-text document content")
|
||||
|
||||
|
||||
def _build_extraction_prompt(text: str) -> str:
|
||||
"""Return the metadata-extraction prompt used in the standard processing pipeline."""
|
||||
return (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
|
||||
def _extract_json_from_text(text: str):
|
||||
"""Try to extract a JSON object from the LLM response text."""
|
||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1)
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/ai/test-extraction")
|
||||
@require_login
|
||||
async def test_ai_extraction(request: Request, body: ExtractionTestRequest):
|
||||
"""
|
||||
Run the metadata-extraction prompt against the configured AI provider.
|
||||
|
||||
Accepts plain-text document content, sends it through the same prompt used
|
||||
by the background processing pipeline, and returns:
|
||||
- ``raw_response``: verbatim LLM output
|
||||
- ``parsed_json``: the extracted JSON object (null when parsing fails)
|
||||
- ``tags``: the ``tags`` list from the parsed JSON (empty list on failure)
|
||||
- ``provider`` / ``model``: which provider / model was used
|
||||
"""
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
provider_name = settings.ai_provider
|
||||
model = settings.ai_model or settings.openai_model
|
||||
|
||||
logger.info(f"AI extraction test requested: provider={provider_name}, model={model}")
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
prompt = _build_extraction_prompt(body.text)
|
||||
raw_response = provider.chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning(f"AI extraction test – configuration error: {e}")
|
||||
return {"status": "error", "message": str(e), "provider": provider_name}
|
||||
except Exception as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"AI extraction test failed for provider '{provider_name}': {detail}", exc_info=True)
|
||||
return {"status": "error", "message": f"AI call failed: {detail}", "provider": provider_name}
|
||||
|
||||
# Attempt to parse JSON from the response
|
||||
parsed_json = None
|
||||
tags: list = []
|
||||
parse_error = None
|
||||
json_text = _extract_json_from_text(raw_response)
|
||||
if json_text:
|
||||
try:
|
||||
parsed_json = json.loads(json_text)
|
||||
tags = parsed_json.get("tags", [])
|
||||
except json.JSONDecodeError as exc:
|
||||
parse_error = str(exc)
|
||||
logger.warning(f"AI extraction test: JSON parse error: {exc}")
|
||||
else:
|
||||
parse_error = "No JSON object found in response"
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"raw_response": raw_response,
|
||||
"parsed_json": parsed_json,
|
||||
"tags": tags,
|
||||
"parse_error": parse_error,
|
||||
}
|
||||
|
||||
@@ -1,927 +0,0 @@
|
||||
"""
|
||||
Pipelines API endpoints.
|
||||
|
||||
Provides full CRUD for processing pipelines and their steps. Pipelines are
|
||||
user-specific: regular users can only manage their own pipelines, while admins
|
||||
can also create and manage *system default* pipelines (owner_id = NULL) that
|
||||
are visible to all users.
|
||||
|
||||
Built-in step types are exposed via GET /api/pipelines/step-types so that UIs
|
||||
can render the correct configuration form without hard-coding the catalogue.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import get_current_user, get_current_user_id, require_login
|
||||
from app.database import get_db
|
||||
from app.models import Pipeline, PipelineStep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/pipelines", tags=["pipelines"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in step type catalogue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PIPELINE_STEP_TYPES: dict[str, dict[str, Any]] = {
|
||||
"convert_to_pdf": {
|
||||
"label": "Convert to PDF",
|
||||
"description": "Convert non-PDF documents to PDF format using Gotenberg.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"check_duplicates": {
|
||||
"label": "Check for Duplicates",
|
||||
"description": "Compare file hash against existing documents to detect duplicates.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"ocr": {
|
||||
"label": "OCR Processing",
|
||||
"description": "Extract text using Azure Document Intelligence or local Tesseract.",
|
||||
"config_schema": {
|
||||
"force_cloud_ocr": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Always use cloud OCR even if the PDF already has embedded text.",
|
||||
},
|
||||
"ocr_language": {
|
||||
"type": "select",
|
||||
"default": "auto",
|
||||
"description": (
|
||||
"Language(s) used for OCR text extraction. Applies to Tesseract and EasyOCR "
|
||||
"providers; Azure and Mistral perform auto-detection by default. "
|
||||
"Use Tesseract codes such as 'eng', 'deu', or 'eng+deu' for multi-language "
|
||||
"documents. 'auto' falls back to the global system setting."
|
||||
),
|
||||
"options": [
|
||||
{"value": "auto", "label": "Auto (use system default)"},
|
||||
{"value": "ara", "label": "Arabic"},
|
||||
{"value": "chi_sim", "label": "Chinese (Simplified)"},
|
||||
{"value": "chi_tra", "label": "Chinese (Traditional)"},
|
||||
{"value": "ces", "label": "Czech"},
|
||||
{"value": "dan", "label": "Danish"},
|
||||
{"value": "nld", "label": "Dutch"},
|
||||
{"value": "eng", "label": "English"},
|
||||
{"value": "fin", "label": "Finnish"},
|
||||
{"value": "fra", "label": "French"},
|
||||
{"value": "deu", "label": "German"},
|
||||
{"value": "ell", "label": "Greek"},
|
||||
{"value": "heb", "label": "Hebrew"},
|
||||
{"value": "hin", "label": "Hindi"},
|
||||
{"value": "hun", "label": "Hungarian"},
|
||||
{"value": "ita", "label": "Italian"},
|
||||
{"value": "jpn", "label": "Japanese"},
|
||||
{"value": "kor", "label": "Korean"},
|
||||
{"value": "nor", "label": "Norwegian"},
|
||||
{"value": "pol", "label": "Polish"},
|
||||
{"value": "por", "label": "Portuguese"},
|
||||
{"value": "ron", "label": "Romanian"},
|
||||
{"value": "rus", "label": "Russian"},
|
||||
{"value": "spa", "label": "Spanish"},
|
||||
{"value": "swe", "label": "Swedish"},
|
||||
{"value": "tha", "label": "Thai"},
|
||||
{"value": "tur", "label": "Turkish"},
|
||||
{"value": "ukr", "label": "Ukrainian"},
|
||||
{"value": "vie", "label": "Vietnamese"},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
"extract_metadata": {
|
||||
"label": "Metadata Extraction",
|
||||
"description": "Extract structured metadata (document type, sender, recipient, tags) using AI.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"embed_metadata": {
|
||||
"label": "Embed Metadata into PDF",
|
||||
"description": "Write the extracted metadata into the PDF document properties.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"compute_embedding": {
|
||||
"label": "Compute Text Embedding",
|
||||
"description": "Compute semantic text embeddings for full-text and similarity search.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"send_to_destinations": {
|
||||
"label": "Send to Storage Destinations",
|
||||
"description": "Upload the processed document to all configured storage destinations.",
|
||||
"config_schema": {},
|
||||
},
|
||||
"classify": {
|
||||
"label": "Document Classification",
|
||||
"description": "Classify the document type using AI without full metadata extraction.",
|
||||
"config_schema": {},
|
||||
},
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MAX_STEPS_PER_PIPELINE = 50
|
||||
MAX_NAME_LENGTH = 255
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Return a stable user identifier from the session.
|
||||
|
||||
Delegates to :func:`app.auth.get_current_user_id` so the same fallback
|
||||
logic ("anonymous") is used consistently throughout the application.
|
||||
"""
|
||||
return get_current_user_id(request)
|
||||
|
||||
|
||||
def _is_admin(request: Request) -> bool:
|
||||
"""Return True if the current session user is an admin."""
|
||||
user = get_current_user(request)
|
||||
return bool(user and user.get("is_admin"))
|
||||
|
||||
|
||||
def _can_access_pipeline(pipeline: Pipeline, user_id: str, admin: bool) -> bool:
|
||||
"""Return True if the user may read or write this pipeline."""
|
||||
# System pipelines (owner_id=NULL) are readable by everyone; only admins can write
|
||||
if pipeline.owner_id is None:
|
||||
return True
|
||||
# Own pipeline
|
||||
return pipeline.owner_id == user_id or admin
|
||||
|
||||
|
||||
def _can_write_pipeline(pipeline: Pipeline, user_id: str, admin: bool) -> bool:
|
||||
"""Return True if the user may create/update/delete this pipeline."""
|
||||
if pipeline.owner_id is None:
|
||||
return admin
|
||||
return pipeline.owner_id == user_id or admin
|
||||
|
||||
|
||||
def _serialize_step(step: PipelineStep) -> dict[str, Any]:
|
||||
return {
|
||||
"id": step.id,
|
||||
"pipeline_id": step.pipeline_id,
|
||||
"position": step.position,
|
||||
"step_type": step.step_type,
|
||||
"label": step.label,
|
||||
"config": json.loads(step.config) if step.config else {},
|
||||
"enabled": step.enabled,
|
||||
"created_at": step.created_at.isoformat() if step.created_at else None,
|
||||
"updated_at": step.updated_at.isoformat() if step.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_pipeline(pipeline: Pipeline, include_steps: bool = False, db: Session | None = None) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {
|
||||
"id": pipeline.id,
|
||||
"owner_id": pipeline.owner_id,
|
||||
"name": pipeline.name,
|
||||
"description": pipeline.description,
|
||||
"is_default": pipeline.is_default,
|
||||
"is_active": pipeline.is_active,
|
||||
"created_at": pipeline.created_at.isoformat() if pipeline.created_at else None,
|
||||
"updated_at": pipeline.updated_at.isoformat() if pipeline.updated_at else None,
|
||||
}
|
||||
if include_steps and db is not None:
|
||||
steps = (
|
||||
db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline.id).order_by(PipelineStep.position).all()
|
||||
)
|
||||
data["steps"] = [_serialize_step(s) for s in steps]
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PipelineCreate(BaseModel):
|
||||
"""Body for creating a pipeline."""
|
||||
|
||||
name: str = Field(..., max_length=MAX_NAME_LENGTH, description="Human-readable pipeline name")
|
||||
description: str | None = Field(default=None, max_length=4096)
|
||||
is_default: bool = Field(default=False)
|
||||
is_active: bool = Field(default=True)
|
||||
|
||||
|
||||
class PipelineUpdate(BaseModel):
|
||||
"""Body for updating a pipeline (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, max_length=MAX_NAME_LENGTH)
|
||||
description: str | None = Field(default=None, max_length=4096)
|
||||
is_default: bool | None = None
|
||||
is_active: bool | None = None
|
||||
|
||||
|
||||
class PipelineStepCreate(BaseModel):
|
||||
"""Body for adding a step to a pipeline."""
|
||||
|
||||
step_type: str = Field(..., description="One of the recognised step type keys")
|
||||
label: str | None = Field(default=None, max_length=MAX_NAME_LENGTH)
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
enabled: bool = Field(default=True)
|
||||
position: int | None = Field(default=None, ge=0, description="Insertion position; appended at end if omitted")
|
||||
|
||||
|
||||
class PipelineStepUpdate(BaseModel):
|
||||
"""Body for updating a pipeline step (all fields optional)."""
|
||||
|
||||
step_type: str | None = None
|
||||
label: str | None = Field(default=None, max_length=MAX_NAME_LENGTH)
|
||||
config: dict[str, Any] | None = None
|
||||
enabled: bool | None = None
|
||||
position: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step-types catalogue endpoint (no auth required — it's public metadata)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/step-types")
|
||||
def list_step_types() -> dict[str, Any]:
|
||||
"""Return the catalogue of built-in pipeline step types.
|
||||
|
||||
Returns:
|
||||
A mapping of step_type key → metadata (label, description, config_schema).
|
||||
"""
|
||||
return PIPELINE_STEP_TYPES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipeline CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("")
|
||||
@require_login
|
||||
def list_pipelines(request: Request, db: DbSession) -> list[dict[str, Any]]:
|
||||
"""List pipelines visible to the current user.
|
||||
|
||||
Regular users see: their own pipelines + system pipelines (owner_id=NULL).
|
||||
Admins see: all pipelines from all users.
|
||||
|
||||
Returns:
|
||||
A list of pipeline objects (without steps — use GET /pipelines/{id} for steps).
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
if admin:
|
||||
pipelines = db.query(Pipeline).order_by(Pipeline.owner_id.nullsfirst(), Pipeline.name).all()
|
||||
else:
|
||||
pipelines = (
|
||||
db.query(Pipeline)
|
||||
.filter((Pipeline.owner_id == user_id) | (Pipeline.owner_id.is_(None)))
|
||||
.order_by(Pipeline.owner_id.nullsfirst(), Pipeline.name)
|
||||
.all()
|
||||
)
|
||||
|
||||
return [_serialize_pipeline(p) for p in pipelines]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def create_pipeline(request: Request, db: DbSession, body: PipelineCreate) -> dict[str, Any]:
|
||||
"""Create a new pipeline for the current user.
|
||||
|
||||
Admins can create system default pipelines by passing ``owner_id=null``
|
||||
via the body — however, that is handled implicitly: to create a system
|
||||
pipeline, call ``POST /api/admin/pipelines`` (admin endpoint) instead.
|
||||
Regular users always get their own user_id as owner.
|
||||
|
||||
Returns:
|
||||
The created pipeline object.
|
||||
|
||||
Raises:
|
||||
HTTPException 409: If a pipeline with the same name already exists for this owner.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
name = body.name.strip() if body.name else ""
|
||||
if not name:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="name is required",
|
||||
)
|
||||
|
||||
# Enforce unique name per owner
|
||||
existing = db.query(Pipeline).filter(Pipeline.owner_id == user_id, Pipeline.name == name).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A pipeline named '{name}' already exists",
|
||||
)
|
||||
|
||||
# If this pipeline is marked as default, unset the existing default for this user
|
||||
if body.is_default:
|
||||
_unset_default(db, user_id)
|
||||
|
||||
pipeline = Pipeline(
|
||||
owner_id=user_id,
|
||||
name=name,
|
||||
description=body.description,
|
||||
is_default=body.is_default,
|
||||
is_active=body.is_active,
|
||||
)
|
||||
try:
|
||||
db.add(pipeline)
|
||||
db.commit()
|
||||
db.refresh(pipeline)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create pipeline user=%s", user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to create pipeline",
|
||||
)
|
||||
|
||||
logger.info("Pipeline created: id=%s, owner=%s, name=%r", pipeline.id, user_id, name)
|
||||
return _serialize_pipeline(pipeline)
|
||||
|
||||
|
||||
@router.get("/{pipeline_id}")
|
||||
@require_login
|
||||
def get_pipeline(pipeline_id: int, request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Return a single pipeline with its steps.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The ID of the pipeline.
|
||||
|
||||
Returns:
|
||||
The pipeline object including its ordered steps.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the pipeline does not exist or is not accessible.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
return _serialize_pipeline(pipeline, include_steps=True, db=db)
|
||||
|
||||
|
||||
@router.put("/{pipeline_id}")
|
||||
@require_login
|
||||
def update_pipeline(pipeline_id: int, request: Request, db: DbSession, body: PipelineUpdate) -> dict[str, Any]:
|
||||
"""Update a pipeline's metadata.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The ID of the pipeline to update.
|
||||
|
||||
Returns:
|
||||
The updated pipeline object.
|
||||
|
||||
Raises:
|
||||
HTTPException 403: If the caller does not own this pipeline.
|
||||
HTTPException 404: If the pipeline does not exist.
|
||||
HTTPException 409: If the new name conflicts with an existing pipeline.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this pipeline")
|
||||
|
||||
if body.name is not None:
|
||||
new_name = body.name.strip()
|
||||
if not new_name:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="name must not be empty",
|
||||
)
|
||||
if new_name != pipeline.name:
|
||||
conflict = (
|
||||
db.query(Pipeline)
|
||||
.filter(Pipeline.owner_id == pipeline.owner_id, Pipeline.name == new_name, Pipeline.id != pipeline_id)
|
||||
.first()
|
||||
)
|
||||
if conflict:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A pipeline named '{new_name}' already exists",
|
||||
)
|
||||
pipeline.name = new_name
|
||||
|
||||
if body.description is not None:
|
||||
pipeline.description = body.description
|
||||
|
||||
if body.is_active is not None:
|
||||
pipeline.is_active = body.is_active
|
||||
|
||||
if body.is_default is not None:
|
||||
if body.is_default and not pipeline.is_default:
|
||||
_unset_default(db, pipeline.owner_id)
|
||||
pipeline.is_default = body.is_default
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(pipeline)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to update pipeline id=%s", pipeline_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update pipeline",
|
||||
)
|
||||
|
||||
logger.info("Pipeline updated: id=%s, user=%s", pipeline_id, user_id)
|
||||
return _serialize_pipeline(pipeline, include_steps=True, db=db)
|
||||
|
||||
|
||||
@router.delete("/{pipeline_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_pipeline(pipeline_id: int, request: Request, db: DbSession) -> None:
|
||||
"""Delete a pipeline and all its steps.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The ID of the pipeline to delete.
|
||||
|
||||
Raises:
|
||||
HTTPException 403: If the caller does not own this pipeline.
|
||||
HTTPException 404: If the pipeline does not exist.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot delete this pipeline")
|
||||
|
||||
try:
|
||||
db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline_id).delete()
|
||||
db.delete(pipeline)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to delete pipeline id=%s", pipeline_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete pipeline",
|
||||
)
|
||||
|
||||
logger.info("Pipeline deleted: id=%s, user=%s", pipeline_id, user_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin-only: create system (owner_id=NULL) pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/admin/system", status_code=status.HTTP_201_CREATED, tags=["admin-pipelines"])
|
||||
@require_login
|
||||
def create_system_pipeline(request: Request, db: DbSession, body: PipelineCreate) -> dict[str, Any]:
|
||||
"""Create a system-level (owner_id=NULL) default pipeline. Admin only.
|
||||
|
||||
System pipelines are visible to all users and can be set as the global
|
||||
default. Only admins may create them.
|
||||
|
||||
Returns:
|
||||
The created system pipeline.
|
||||
|
||||
Raises:
|
||||
HTTPException 403: If the caller is not an admin.
|
||||
HTTPException 409: If a system pipeline with the same name already exists.
|
||||
"""
|
||||
if not _is_admin(request):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
|
||||
name = body.name.strip() if body.name else ""
|
||||
if not name:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="name is required",
|
||||
)
|
||||
|
||||
existing = db.query(Pipeline).filter(Pipeline.owner_id.is_(None), Pipeline.name == name).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A system pipeline named '{name}' already exists",
|
||||
)
|
||||
|
||||
if body.is_default:
|
||||
_unset_default(db, None)
|
||||
|
||||
pipeline = Pipeline(
|
||||
owner_id=None,
|
||||
name=name,
|
||||
description=body.description,
|
||||
is_default=body.is_default,
|
||||
is_active=body.is_active,
|
||||
)
|
||||
try:
|
||||
db.add(pipeline)
|
||||
db.commit()
|
||||
db.refresh(pipeline)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to create system pipeline: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to create system pipeline",
|
||||
)
|
||||
|
||||
logger.info(f"System pipeline created: id={pipeline.id}, name={name!r}")
|
||||
return _serialize_pipeline(pipeline)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{pipeline_id}/steps", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def add_step(pipeline_id: int, request: Request, db: DbSession, body: PipelineStepCreate) -> dict[str, Any]:
|
||||
"""Add a step to a pipeline.
|
||||
|
||||
Steps are automatically appended at the end unless an explicit ``position``
|
||||
is supplied. All existing steps at or after the insertion position are
|
||||
shifted forward by one.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The pipeline to add the step to.
|
||||
|
||||
Returns:
|
||||
The created step object.
|
||||
|
||||
Raises:
|
||||
HTTPException 403: If the caller cannot modify this pipeline.
|
||||
HTTPException 404: If the pipeline does not exist.
|
||||
HTTPException 422: If the step_type is not recognised.
|
||||
HTTPException 409: If the maximum number of steps per pipeline is reached.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this pipeline")
|
||||
|
||||
if body.step_type not in PIPELINE_STEP_TYPES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unknown step type '{body.step_type}'. Valid types: {sorted(PIPELINE_STEP_TYPES)}",
|
||||
)
|
||||
|
||||
current_count = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline_id).count()
|
||||
if current_count >= MAX_STEPS_PER_PIPELINE:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Maximum of {MAX_STEPS_PER_PIPELINE} steps per pipeline reached",
|
||||
)
|
||||
|
||||
# Determine insertion position
|
||||
if body.position is None:
|
||||
max_pos = (
|
||||
db.query(PipelineStep.position)
|
||||
.filter(PipelineStep.pipeline_id == pipeline_id)
|
||||
.order_by(PipelineStep.position.desc())
|
||||
.first()
|
||||
)
|
||||
insert_pos = (max_pos[0] + 1) if max_pos else 0
|
||||
else:
|
||||
insert_pos = body.position
|
||||
# Shift existing steps
|
||||
steps_to_shift = (
|
||||
db.query(PipelineStep)
|
||||
.filter(PipelineStep.pipeline_id == pipeline_id, PipelineStep.position >= insert_pos)
|
||||
.all()
|
||||
)
|
||||
for s in steps_to_shift:
|
||||
s.position += 1
|
||||
|
||||
step = PipelineStep(
|
||||
pipeline_id=pipeline_id,
|
||||
position=insert_pos,
|
||||
step_type=body.step_type,
|
||||
label=body.label,
|
||||
config=json.dumps(body.config) if body.config else None,
|
||||
enabled=body.enabled,
|
||||
)
|
||||
try:
|
||||
db.add(step)
|
||||
db.commit()
|
||||
db.refresh(step)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to add step to pipeline id={pipeline_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to add step",
|
||||
)
|
||||
|
||||
logger.info(f"Step added: pipeline={pipeline_id}, step_type={body.step_type!r}, pos={insert_pos}")
|
||||
return _serialize_step(step)
|
||||
|
||||
|
||||
@router.put("/{pipeline_id}/steps/reorder")
|
||||
@require_login
|
||||
def reorder_steps(
|
||||
pipeline_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
step_ids: list[int] = Body(..., description="Ordered list of step IDs representing the new order"),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Replace the step order for a pipeline.
|
||||
|
||||
Provide a complete ordered list of *all* step IDs. Their ``position``
|
||||
values will be reassigned 0, 1, 2, … in the given order.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The pipeline whose steps are being reordered.
|
||||
|
||||
Returns:
|
||||
The updated, ordered list of step objects.
|
||||
|
||||
Raises:
|
||||
HTTPException 422: If the provided list does not contain exactly the
|
||||
current set of step IDs for this pipeline.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this pipeline")
|
||||
|
||||
existing_steps = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline_id).all()
|
||||
existing_ids = {s.id for s in existing_steps}
|
||||
|
||||
if set(step_ids) != existing_ids or len(step_ids) != len(existing_ids):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="step_ids must contain exactly the current step IDs for this pipeline",
|
||||
)
|
||||
|
||||
step_map = {s.id: s for s in existing_steps}
|
||||
for pos, sid in enumerate(step_ids):
|
||||
step_map[sid].position = pos
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to reorder steps for pipeline id={pipeline_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to reorder steps",
|
||||
)
|
||||
|
||||
updated = (
|
||||
db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline_id).order_by(PipelineStep.position).all()
|
||||
)
|
||||
return [_serialize_step(s) for s in updated]
|
||||
|
||||
|
||||
@router.put("/{pipeline_id}/steps/{step_id}")
|
||||
@require_login
|
||||
def update_step(
|
||||
pipeline_id: int, step_id: int, request: Request, db: DbSession, body: PipelineStepUpdate
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing pipeline step.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The owning pipeline.
|
||||
step_id: The step to update.
|
||||
|
||||
Returns:
|
||||
The updated step object.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this pipeline")
|
||||
|
||||
step = db.query(PipelineStep).filter(PipelineStep.id == step_id, PipelineStep.pipeline_id == pipeline_id).first()
|
||||
if not step:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Step not found")
|
||||
|
||||
if body.step_type is not None:
|
||||
if body.step_type not in PIPELINE_STEP_TYPES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unknown step type '{body.step_type}'",
|
||||
)
|
||||
step.step_type = body.step_type
|
||||
|
||||
if body.label is not None:
|
||||
step.label = body.label
|
||||
|
||||
if body.config is not None:
|
||||
step.config = json.dumps(body.config)
|
||||
|
||||
if body.enabled is not None:
|
||||
step.enabled = body.enabled
|
||||
|
||||
if body.position is not None and body.position != step.position:
|
||||
old_pos = step.position
|
||||
new_pos = body.position
|
||||
if new_pos > old_pos:
|
||||
# Moving down: shift intervening steps up
|
||||
db.query(PipelineStep).filter(
|
||||
PipelineStep.pipeline_id == pipeline_id,
|
||||
PipelineStep.position > old_pos,
|
||||
PipelineStep.position <= new_pos,
|
||||
PipelineStep.id != step_id,
|
||||
).update({"position": PipelineStep.position - 1})
|
||||
else:
|
||||
# Moving up: shift intervening steps down
|
||||
db.query(PipelineStep).filter(
|
||||
PipelineStep.pipeline_id == pipeline_id,
|
||||
PipelineStep.position >= new_pos,
|
||||
PipelineStep.position < old_pos,
|
||||
PipelineStep.id != step_id,
|
||||
).update({"position": PipelineStep.position + 1})
|
||||
step.position = new_pos
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(step)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to update step id={step_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update step",
|
||||
)
|
||||
|
||||
return _serialize_step(step)
|
||||
|
||||
|
||||
@router.delete("/{pipeline_id}/steps/{step_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_step(pipeline_id: int, step_id: int, request: Request, db: DbSession) -> None:
|
||||
"""Delete a step from a pipeline.
|
||||
|
||||
Path Parameters:
|
||||
pipeline_id: The owning pipeline.
|
||||
step_id: The step to delete.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
|
||||
if not pipeline or not _can_access_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Pipeline not found")
|
||||
|
||||
if not _can_write_pipeline(pipeline, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this pipeline")
|
||||
|
||||
step = db.query(PipelineStep).filter(PipelineStep.id == step_id, PipelineStep.pipeline_id == pipeline_id).first()
|
||||
if not step:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Step not found")
|
||||
|
||||
deleted_pos = step.position
|
||||
try:
|
||||
db.delete(step)
|
||||
# Compact remaining step positions
|
||||
db.query(PipelineStep).filter(
|
||||
PipelineStep.pipeline_id == pipeline_id,
|
||||
PipelineStep.position > deleted_pos,
|
||||
).update({"position": PipelineStep.position - 1})
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to delete step id={step_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete step",
|
||||
)
|
||||
|
||||
logger.info(f"Step deleted: id={step_id}, pipeline={pipeline_id}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: unset default flag for an owner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unset_default(db: Session, owner_id: str | None) -> None:
|
||||
"""Clear the is_default flag on all pipelines for the given owner."""
|
||||
if owner_id is None:
|
||||
db.query(Pipeline).filter(Pipeline.owner_id.is_(None), Pipeline.is_default.is_(True)).update(
|
||||
{"is_default": False}
|
||||
)
|
||||
else:
|
||||
db.query(Pipeline).filter(Pipeline.owner_id == owner_id, Pipeline.is_default.is_(True)).update(
|
||||
{"is_default": False}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default system pipeline seeding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The steps that make up the standard document-processing workflow. The order
|
||||
# here mirrors what the existing Celery-based pipeline executes for every
|
||||
# uploaded file.
|
||||
_DEFAULT_PIPELINE_STEPS: list[tuple[str, str]] = [
|
||||
("convert_to_pdf", "Convert to PDF"),
|
||||
("check_duplicates", "Check for Duplicates"),
|
||||
("ocr", "OCR Processing"),
|
||||
("extract_metadata", "Extract Metadata"),
|
||||
("embed_metadata", "Embed Metadata into PDF"),
|
||||
("compute_embedding", "Compute Text Embedding"),
|
||||
("send_to_destinations", "Send to Storage Destinations"),
|
||||
]
|
||||
|
||||
#: Human-readable name shown in the management UI for the auto-seeded pipeline.
|
||||
DEFAULT_PIPELINE_NAME = "Standard Processing Pipeline"
|
||||
|
||||
|
||||
def seed_default_pipeline(db: Session) -> int:
|
||||
"""Ensure a system-owned default pipeline exists in the database.
|
||||
|
||||
This function is idempotent — it is a no-op when any system pipeline
|
||||
(``owner_id IS NULL``) already exists. It is intended to be called once
|
||||
at application startup (in ``app.main.lifespan``) so that the pipeline
|
||||
management UI always shows the default workflow that mirrors the existing
|
||||
Celery-based processing steps.
|
||||
|
||||
The created pipeline:
|
||||
|
||||
* ``owner_id = None`` — owned by the system, visible to all users
|
||||
* ``is_default = True`` — selected automatically for new documents
|
||||
* Steps (in order): convert_to_pdf → check_duplicates → ocr →
|
||||
extract_metadata → embed_metadata → compute_embedding →
|
||||
send_to_destinations
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
``1`` if a new pipeline was created, ``0`` if one already existed.
|
||||
"""
|
||||
try:
|
||||
if db.query(Pipeline).filter(Pipeline.owner_id.is_(None)).count() > 0:
|
||||
return 0
|
||||
except Exception:
|
||||
# Table may not exist yet during the very first migration run.
|
||||
return 0
|
||||
|
||||
pipeline = Pipeline(
|
||||
owner_id=None,
|
||||
name=DEFAULT_PIPELINE_NAME,
|
||||
description=(
|
||||
"The standard document processing workflow: PDF conversion, "
|
||||
"duplicate detection, OCR, metadata extraction and embedding, "
|
||||
"semantic embeddings, and final distribution to storage destinations."
|
||||
),
|
||||
is_default=True,
|
||||
is_active=True,
|
||||
)
|
||||
db.add(pipeline)
|
||||
try:
|
||||
db.flush() # Assign pipeline.id without committing yet
|
||||
except Exception as exc: # pragma: no cover
|
||||
db.rollback()
|
||||
logger.error(f"Failed to create default pipeline: {exc}")
|
||||
return 0
|
||||
|
||||
for pos, (step_type, label) in enumerate(_DEFAULT_PIPELINE_STEPS):
|
||||
db.add(
|
||||
PipelineStep(
|
||||
pipeline_id=pipeline.id,
|
||||
position=pos,
|
||||
step_type=step_type,
|
||||
label=label,
|
||||
enabled=True,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
logger.info("Seeded default system pipeline: '%s' (id=%d)", DEFAULT_PIPELINE_NAME, pipeline.id)
|
||||
except Exception as exc: # pragma: no cover
|
||||
db.rollback()
|
||||
logger.error(f"Failed to seed default pipeline steps: {exc}")
|
||||
return 0
|
||||
|
||||
return 1
|
||||
@@ -1,279 +0,0 @@
|
||||
"""REST API for subscription plan CRUD.
|
||||
|
||||
Endpoints:
|
||||
GET /api/plans/ — list active plans (public)
|
||||
GET /api/plans/admin — list all plans inc. inactive (admin only)
|
||||
POST /api/plans/ — create plan (admin only)
|
||||
GET /api/plans/{plan_id} — get single active plan (public)
|
||||
PUT /api/plans/{plan_id} — update plan (admin only)
|
||||
DELETE /api/plans/{plan_id} — delete plan (admin only)
|
||||
POST /api/plans/seed — seed default plans (admin only)
|
||||
POST /api/plans/reorder — set sort_order for multiple plans (admin only)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import SubscriptionPlan
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/plans", tags=["plans"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper (admin-only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PlanUpsert(BaseModel):
|
||||
"""Body for creating or updating a subscription plan."""
|
||||
|
||||
name: str
|
||||
tagline: str | None = None
|
||||
price_monthly: float = 0.0
|
||||
price_yearly: float = 0.0
|
||||
trial_days: int = 0
|
||||
lifetime_file_limit: int = 0
|
||||
daily_upload_limit: int = 0
|
||||
monthly_upload_limit: int = 0
|
||||
max_storage_destinations: int = 0
|
||||
max_ocr_pages_monthly: int = 0
|
||||
max_file_size_mb: int = 0
|
||||
max_mailboxes: int = 0
|
||||
overage_percent: int = Field(default=20, ge=0, le=200)
|
||||
allow_overage_billing: bool = False
|
||||
overage_price_per_doc: float | None = None
|
||||
overage_price_per_ocr_page: float | None = None
|
||||
is_active: bool = True
|
||||
is_highlighted: bool = False
|
||||
badge_text: str | None = None
|
||||
cta_text: str = "Get started"
|
||||
sort_order: int = 0
|
||||
features: list[str] = []
|
||||
api_access: bool = False
|
||||
stripe_price_id_monthly: str | None = None
|
||||
stripe_price_id_yearly: str | None = None
|
||||
|
||||
|
||||
class ReorderBody(BaseModel):
|
||||
"""Body for reordering plans."""
|
||||
|
||||
order: list[str]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _plan_to_response(plan: SubscriptionPlan) -> dict[str, Any]:
|
||||
features: list[str] = []
|
||||
if plan.features:
|
||||
try:
|
||||
features = json.loads(plan.features)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
features = []
|
||||
return {
|
||||
"id": plan.id,
|
||||
"plan_id": plan.plan_id,
|
||||
"name": plan.name,
|
||||
"tagline": plan.tagline,
|
||||
"price_monthly": plan.price_monthly,
|
||||
"price_yearly": plan.price_yearly,
|
||||
"trial_days": plan.trial_days,
|
||||
"lifetime_file_limit": plan.lifetime_file_limit,
|
||||
"daily_upload_limit": plan.daily_upload_limit,
|
||||
"monthly_upload_limit": plan.monthly_upload_limit,
|
||||
"max_storage_destinations": plan.max_storage_destinations,
|
||||
"max_ocr_pages_monthly": plan.max_ocr_pages_monthly,
|
||||
"max_file_size_mb": plan.max_file_size_mb,
|
||||
"max_mailboxes": plan.max_mailboxes,
|
||||
"overage_percent": plan.overage_percent,
|
||||
"allow_overage_billing": plan.allow_overage_billing,
|
||||
"overage_price_per_doc": plan.overage_price_per_doc,
|
||||
"overage_price_per_ocr_page": plan.overage_price_per_ocr_page,
|
||||
"is_active": plan.is_active,
|
||||
"is_highlighted": plan.is_highlighted,
|
||||
"badge_text": plan.badge_text,
|
||||
"cta_text": plan.cta_text,
|
||||
"sort_order": plan.sort_order,
|
||||
"features": features,
|
||||
"api_access": plan.api_access,
|
||||
"stripe_price_id_monthly": plan.stripe_price_id_monthly,
|
||||
"stripe_price_id_yearly": plan.stripe_price_id_yearly,
|
||||
"created_at": plan.created_at.isoformat() if plan.created_at else None,
|
||||
"updated_at": plan.updated_at.isoformat() if plan.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _apply_body(plan: SubscriptionPlan, body: PlanUpsert) -> None:
|
||||
"""Apply PlanUpsert fields onto a SubscriptionPlan ORM object."""
|
||||
plan.name = body.name
|
||||
plan.tagline = body.tagline
|
||||
plan.price_monthly = body.price_monthly
|
||||
plan.price_yearly = body.price_yearly
|
||||
plan.trial_days = body.trial_days
|
||||
plan.lifetime_file_limit = body.lifetime_file_limit
|
||||
plan.daily_upload_limit = body.daily_upload_limit
|
||||
plan.monthly_upload_limit = body.monthly_upload_limit
|
||||
plan.max_storage_destinations = body.max_storage_destinations
|
||||
plan.max_ocr_pages_monthly = body.max_ocr_pages_monthly
|
||||
plan.max_file_size_mb = body.max_file_size_mb
|
||||
plan.max_mailboxes = body.max_mailboxes
|
||||
plan.overage_percent = body.overage_percent
|
||||
plan.allow_overage_billing = body.allow_overage_billing
|
||||
plan.overage_price_per_doc = body.overage_price_per_doc
|
||||
plan.overage_price_per_ocr_page = body.overage_price_per_ocr_page
|
||||
plan.is_active = body.is_active
|
||||
plan.is_highlighted = body.is_highlighted
|
||||
plan.badge_text = body.badge_text
|
||||
plan.cta_text = body.cta_text
|
||||
plan.sort_order = body.sort_order
|
||||
plan.features = json.dumps(body.features)
|
||||
plan.api_access = body.api_access
|
||||
plan.stripe_price_id_monthly = body.stripe_price_id_monthly or None
|
||||
plan.stripe_price_id_yearly = body.stripe_price_id_yearly or None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", summary="List active plans (public)")
|
||||
def list_active_plans(db: DbSession) -> dict[str, Any]:
|
||||
"""Return all active plans in sort order. Public endpoint — no auth required."""
|
||||
plans = (
|
||||
db.query(SubscriptionPlan)
|
||||
.filter(SubscriptionPlan.is_active.is_(True))
|
||||
.order_by(SubscriptionPlan.sort_order)
|
||||
.all()
|
||||
)
|
||||
return {"plans": [_plan_to_response(p) for p in plans]}
|
||||
|
||||
|
||||
@router.get("/admin", summary="List all plans including inactive (admin only)")
|
||||
def list_all_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Return all plans (active and inactive) in sort order. Admin only."""
|
||||
plans = db.query(SubscriptionPlan).order_by(SubscriptionPlan.sort_order).all()
|
||||
return {"plans": [_plan_to_response(p) for p in plans]}
|
||||
|
||||
|
||||
@router.post("/seed", summary="Seed default plans (admin only)", status_code=status.HTTP_200_OK)
|
||||
def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Seed the subscription_plans table from TIER_DEFAULTS. No-op if plans already exist."""
|
||||
from app.utils.subscription import seed_default_plans
|
||||
|
||||
inserted = seed_default_plans(db)
|
||||
return {"inserted": inserted, "message": f"Seeded {inserted} default plan(s)."}
|
||||
|
||||
|
||||
@router.post("/reorder", summary="Reorder plans (admin only)")
|
||||
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
|
||||
updated = 0
|
||||
for sort_order, plan_id in enumerate(body.order):
|
||||
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
|
||||
if plan:
|
||||
plan.sort_order = sort_order
|
||||
updated += 1
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to reorder plans")
|
||||
return {"updated": updated}
|
||||
|
||||
|
||||
@router.post("/", summary="Create a new plan (admin only)", status_code=status.HTTP_201_CREATED)
|
||||
def create_plan(plan_id: str, body: PlanUpsert, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Create a new subscription plan with the given *plan_id* slug."""
|
||||
existing = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Plan '{plan_id}' already exists.",
|
||||
)
|
||||
plan = SubscriptionPlan(plan_id=plan_id)
|
||||
_apply_body(plan, body)
|
||||
db.add(plan)
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(plan)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
logger.info("Admin created subscription plan '%s'", plan_id)
|
||||
return _plan_to_response(plan)
|
||||
|
||||
|
||||
@router.get("/{plan_id}", summary="Get a single active plan (public)")
|
||||
def get_plan(plan_id: str, db: DbSession) -> dict[str, Any]:
|
||||
"""Return a single active plan by plan_id. Public endpoint."""
|
||||
plan = (
|
||||
db.query(SubscriptionPlan)
|
||||
.filter(
|
||||
SubscriptionPlan.plan_id == plan_id,
|
||||
SubscriptionPlan.is_active.is_(True),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not plan:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Plan '{plan_id}' not found.")
|
||||
return _plan_to_response(plan)
|
||||
|
||||
|
||||
@router.put("/{plan_id}", summary="Update an existing plan (admin only)")
|
||||
def update_plan(plan_id: str, body: PlanUpsert, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Update an existing subscription plan. Admin only."""
|
||||
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
|
||||
if not plan:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Plan '{plan_id}' not found.")
|
||||
_apply_body(plan, body)
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(plan)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
logger.info("Admin updated subscription plan '%s'", plan_id)
|
||||
return _plan_to_response(plan)
|
||||
|
||||
|
||||
@router.delete("/{plan_id}", summary="Delete a plan (admin only)", status_code=status.HTTP_204_NO_CONTENT)
|
||||
def delete_plan(plan_id: str, db: DbSession, _admin: AdminUser) -> None:
|
||||
"""Delete a subscription plan. Admin only."""
|
||||
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
|
||||
if not plan:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Plan '{plan_id}' not found.")
|
||||
try:
|
||||
db.delete(plan)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
logger.info("Admin deleted subscription plan '%s'", plan_id)
|
||||
+44
-67
@@ -1,29 +1,26 @@
|
||||
"""
|
||||
Document processing API endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
import logging
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.api.common import resolve_file_path
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.api.common import resolve_file_path
|
||||
from app.tasks.process_document import process_document
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/process/")
|
||||
@require_login
|
||||
def process(file_path: str):
|
||||
@@ -31,92 +28,96 @@ def process(file_path: str):
|
||||
file_path = resolve_file_path(file_path)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
|
||||
task = process_document.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_dropbox/")
|
||||
@require_login
|
||||
def send_to_dropbox_endpoint(file_path: str):
|
||||
"""Send a document to Dropbox."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_dropbox.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_paperless/")
|
||||
@require_login
|
||||
def send_to_paperless_endpoint(file_path: str):
|
||||
"""Send a document to Paperless-ngx."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_paperless.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_nextcloud/")
|
||||
@require_login
|
||||
def send_to_nextcloud_endpoint(file_path: str):
|
||||
"""Send a document to NextCloud."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_nextcloud.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_google_drive/")
|
||||
@require_login
|
||||
def send_to_google_drive_endpoint(file_path: str):
|
||||
"""Send a document to Google Drive."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_google_drive.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_onedrive/")
|
||||
@require_login
|
||||
def send_to_onedrive_endpoint(file_path: str):
|
||||
"""Send a document to OneDrive."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_onedrive.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_all_destinations/")
|
||||
@require_login
|
||||
def send_to_all_destinations_endpoint(file_path: str):
|
||||
"""Call the aggregator task that sends this file to all configured destinations."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
file_path = resolve_file_path(file_path, 'processed')
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
|
||||
task = send_to_all_destinations.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued", "file_path": file_path}
|
||||
|
||||
|
||||
@router.post("/processall")
|
||||
@require_login
|
||||
def process_all_pdfs_in_workdir():
|
||||
"""
|
||||
Finds all .pdf files in <workdir> and enqueues them for processing.
|
||||
|
||||
For large batches (>processall_throttle_threshold files), tasks are staggered
|
||||
to avoid overwhelming downstream APIs.
|
||||
"""
|
||||
"""Finds all .pdf files in <workdir> and enqueues them for processing."""
|
||||
target_dir = settings.workdir
|
||||
if not os.path.exists(target_dir):
|
||||
raise HTTPException(status_code=400, detail=f"Directory {target_dir} does not exist.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Directory {target_dir} does not exist."
|
||||
)
|
||||
|
||||
pdf_files = []
|
||||
for filename in os.listdir(target_dir):
|
||||
@@ -127,37 +128,13 @@ def process_all_pdfs_in_workdir():
|
||||
return {"message": "No PDF files found in that directory."}
|
||||
|
||||
task_ids = []
|
||||
num_files = len(pdf_files)
|
||||
|
||||
# Apply throttling if we have more files than the threshold
|
||||
apply_throttle = num_files > settings.processall_throttle_threshold
|
||||
|
||||
if apply_throttle:
|
||||
logger.info(
|
||||
f"Processing {num_files} files with throttling "
|
||||
f"(threshold: {settings.processall_throttle_threshold}, "
|
||||
f"delay: {settings.processall_throttle_delay}s per file)"
|
||||
)
|
||||
|
||||
for index, pdf in enumerate(pdf_files):
|
||||
for pdf in pdf_files:
|
||||
file_path = os.path.join(target_dir, pdf)
|
||||
|
||||
if apply_throttle:
|
||||
# Stagger task submission with countdown
|
||||
# First file starts immediately (countdown=0)
|
||||
# Each subsequent file has an increasing delay
|
||||
countdown = index * settings.processall_throttle_delay
|
||||
task = process_document.apply_async(args=[file_path], countdown=countdown)
|
||||
logger.debug(f"Scheduled {pdf} with {countdown}s delay")
|
||||
else:
|
||||
# No throttling - enqueue immediately
|
||||
task = process_document.delay(file_path)
|
||||
|
||||
task = process_document.delay(file_path)
|
||||
task_ids.append(task.id)
|
||||
|
||||
message = f"Enqueued {num_files} PDFs for processing"
|
||||
if apply_throttle:
|
||||
total_time = (num_files - 1) * settings.processall_throttle_delay
|
||||
message += f" (throttled over {total_time} seconds)"
|
||||
|
||||
return {"message": message, "pdf_files": pdf_files, "task_ids": task_ids, "throttled": apply_throttle}
|
||||
return {
|
||||
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
|
||||
"pdf_files": pdf_files,
|
||||
"task_ids": task_ids
|
||||
}
|
||||
|
||||
@@ -1,320 +0,0 @@
|
||||
"""User self-service profile API.
|
||||
|
||||
Provides endpoints for the authenticated user to view and update their own
|
||||
profile settings without requiring admin access.
|
||||
|
||||
Routes:
|
||||
GET /api/profile — read current user's profile
|
||||
PATCH /api/profile — update display name, language, theme
|
||||
POST /api/profile/avatar — upload a new profile picture (JPEG/PNG/GIF/WebP, max 2 MB)
|
||||
DELETE /api/profile/avatar — remove custom avatar (reverts to Gravatar)
|
||||
POST /api/profile/change-password — change password (local-auth users only)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from hashlib import md5
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import LocalUser, UserProfile
|
||||
from app.utils.i18n import SUPPORTED_LANGUAGE_CODES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/profile", tags=["profile"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# Maximum avatar upload size: 2 MB
|
||||
_MAX_AVATAR_BYTES = 2 * 1024 * 1024
|
||||
|
||||
# Allowed MIME types for avatar uploads
|
||||
_ALLOWED_AVATAR_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
|
||||
|
||||
# Valid theme values
|
||||
_VALID_THEMES = {"light", "dark", "system"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Return the stable user identifier from the session.
|
||||
|
||||
Raises HTTP 401 if no user is logged in.
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user or not isinstance(user, dict):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
uid = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
if not uid:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Cannot determine user identity")
|
||||
return uid
|
||||
|
||||
|
||||
def _gravatar_url(email: str | None) -> str:
|
||||
"""Generate a Gravatar URL for *email*, falling back to identicon."""
|
||||
if not email:
|
||||
return "https://www.gravatar.com/avatar/?d=identicon"
|
||||
# MD5 used for Gravatar URL generation only — not for security
|
||||
h = md5(email.strip().lower().encode(), usedforsecurity=False).hexdigest()
|
||||
return f"https://www.gravatar.com/avatar/{h}?d=identicon"
|
||||
|
||||
|
||||
def _get_or_create_profile(db: Session, user_id: str) -> UserProfile:
|
||||
"""Return the UserProfile for *user_id*, creating a stub if one doesn't exist."""
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile is None:
|
||||
profile = UserProfile(user_id=user_id)
|
||||
db.add(profile)
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
return profile
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProfileResponse(BaseModel):
|
||||
"""Response body for GET /api/profile."""
|
||||
|
||||
user_id: str
|
||||
display_name: str | None
|
||||
contact_email: str | None
|
||||
preferred_language: str | None
|
||||
preferred_theme: str | None
|
||||
avatar_url: str
|
||||
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
|
||||
is_local_user: bool
|
||||
"""True when the account was created via local email/password sign-up."""
|
||||
|
||||
|
||||
class ProfileUpdateRequest(BaseModel):
|
||||
"""Request body for PATCH /api/profile."""
|
||||
|
||||
display_name: str | None = Field(default=None, max_length=255, description="Human-readable display name")
|
||||
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
|
||||
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
|
||||
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
"""Request body for POST /api/profile/change-password."""
|
||||
|
||||
current_password: str = Field(..., min_length=1, max_length=128)
|
||||
new_password: str = Field(..., min_length=8, max_length=128)
|
||||
new_password_confirm: str = Field(..., min_length=8, max_length=128)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("", response_model=ProfileResponse)
|
||||
@require_login
|
||||
async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
|
||||
"""Return the current user's profile settings."""
|
||||
user_id = _get_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
|
||||
session_user = request.session.get("user", {})
|
||||
email = session_user.get("email") if isinstance(session_user, dict) else None
|
||||
|
||||
# Determine avatar: prefer stored data, fall back to Gravatar
|
||||
avatar_url = profile.avatar_data if profile.avatar_data else _gravatar_url(email) # type: ignore[attr-defined]
|
||||
|
||||
# Check whether this is a local (email/password) account
|
||||
is_local = db.query(LocalUser).filter(LocalUser.username == user_id).first() is not None
|
||||
|
||||
return ProfileResponse(
|
||||
user_id=user_id,
|
||||
display_name=profile.display_name, # type: ignore[arg-type]
|
||||
contact_email=profile.contact_email, # type: ignore[arg-type]
|
||||
preferred_language=profile.preferred_language, # type: ignore[arg-type]
|
||||
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
|
||||
avatar_url=avatar_url,
|
||||
is_local_user=is_local,
|
||||
)
|
||||
|
||||
|
||||
@router.patch("", response_model=ProfileResponse)
|
||||
@require_login
|
||||
async def update_profile(body: ProfileUpdateRequest, request: Request, db: DbSession) -> ProfileResponse:
|
||||
"""Update the current user's editable profile settings."""
|
||||
user_id = _get_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
|
||||
# Validate language code
|
||||
if body.preferred_language is not None:
|
||||
lang = body.preferred_language.lower().strip()
|
||||
if lang and lang not in SUPPORTED_LANGUAGE_CODES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unsupported language code: {lang}",
|
||||
)
|
||||
profile.preferred_language = lang or None # type: ignore[assignment]
|
||||
|
||||
# Validate theme
|
||||
if body.preferred_theme is not None:
|
||||
theme = body.preferred_theme.lower().strip()
|
||||
if theme and theme not in _VALID_THEMES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid theme: {theme}. Must be one of: {', '.join(sorted(_VALID_THEMES))}",
|
||||
)
|
||||
profile.preferred_theme = theme or None # type: ignore[assignment]
|
||||
|
||||
if body.display_name is not None:
|
||||
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
|
||||
|
||||
if body.contact_email is not None:
|
||||
profile.contact_email = body.contact_email.strip() or None # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
session_user = request.session.get("user", {})
|
||||
email = session_user.get("email") if isinstance(session_user, dict) else None
|
||||
avatar_url = profile.avatar_data if profile.avatar_data else _gravatar_url(email) # type: ignore[attr-defined]
|
||||
is_local = db.query(LocalUser).filter(LocalUser.username == user_id).first() is not None
|
||||
|
||||
return ProfileResponse(
|
||||
user_id=user_id,
|
||||
display_name=profile.display_name, # type: ignore[arg-type]
|
||||
contact_email=profile.contact_email, # type: ignore[arg-type]
|
||||
preferred_language=profile.preferred_language, # type: ignore[arg-type]
|
||||
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
|
||||
avatar_url=avatar_url,
|
||||
is_local_user=is_local,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/avatar", status_code=status.HTTP_200_OK)
|
||||
@require_login
|
||||
async def upload_avatar(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
file: UploadFile = File(..., description="Profile picture (JPEG, PNG, GIF or WebP; max 2 MB)"),
|
||||
) -> dict:
|
||||
"""Upload a new profile picture.
|
||||
|
||||
The image is stored as a base64-encoded data URL in ``UserProfile.avatar_data``.
|
||||
Accepts JPEG, PNG, GIF, or WebP files up to 2 MB.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
content_type = (file.content_type or "").lower()
|
||||
if content_type not in _ALLOWED_AVATAR_TYPES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail=f"Unsupported image type '{content_type}'. Allowed: JPEG, PNG, GIF, WebP.",
|
||||
)
|
||||
|
||||
# Check declared size first (available when the client sends a Content-Length header)
|
||||
if file.size is not None and file.size > _MAX_AVATAR_BYTES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail="Avatar image must be 2 MB or smaller.",
|
||||
)
|
||||
|
||||
# Read up to one byte past the limit so we can detect oversized uploads
|
||||
raw = await file.read(_MAX_AVATAR_BYTES + 1)
|
||||
if len(raw) > _MAX_AVATAR_BYTES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail="Avatar image must be 2 MB or smaller.",
|
||||
)
|
||||
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
data_url = f"data:{content_type};base64,{b64}"
|
||||
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
profile.avatar_data = data_url # type: ignore[assignment]
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
return {"avatar_url": data_url}
|
||||
|
||||
|
||||
@router.delete("/avatar", status_code=status.HTTP_200_OK)
|
||||
@require_login
|
||||
async def delete_avatar(request: Request, db: DbSession) -> dict:
|
||||
"""Remove the custom avatar and revert to the Gravatar fallback."""
|
||||
user_id = _get_user_id(request)
|
||||
profile = _get_or_create_profile(db, user_id)
|
||||
profile.avatar_data = None # type: ignore[assignment]
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
session_user = request.session.get("user", {})
|
||||
email = session_user.get("email") if isinstance(session_user, dict) else None
|
||||
return {"avatar_url": _gravatar_url(email)}
|
||||
|
||||
|
||||
@router.post("/change-password", status_code=status.HTTP_200_OK)
|
||||
@require_login
|
||||
async def change_password(body: ChangePasswordRequest, request: Request, db: DbSession) -> dict:
|
||||
"""Change the password for local (email/password) accounts.
|
||||
|
||||
Raises 403 if the account is not a local account or the current password is wrong.
|
||||
Raises 422 if the new passwords do not match.
|
||||
"""
|
||||
from app.utils.local_auth import hash_password, verify_password
|
||||
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
local_user = db.query(LocalUser).filter(LocalUser.username == user_id).first()
|
||||
if local_user is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Password change is only available for local accounts.",
|
||||
)
|
||||
|
||||
if not verify_password(body.current_password, local_user.hashed_password):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Current password is incorrect.",
|
||||
)
|
||||
|
||||
if body.new_password != body.new_password_confirm:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="New passwords do not match.",
|
||||
)
|
||||
|
||||
local_user.hashed_password = hash_password(body.new_password)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Password changed for local user: %s", user_id)
|
||||
return {"detail": "Password changed successfully."}
|
||||
@@ -1,270 +0,0 @@
|
||||
"""
|
||||
Queue monitoring API endpoints.
|
||||
|
||||
Provides endpoints to query Celery/Redis queue statistics and
|
||||
database-level processing status for document pipeline visibility.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import redis
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/queue", tags=["queue"])
|
||||
|
||||
# Constants
|
||||
CELERY_INSPECT_TIMEOUT = 2.0
|
||||
MAX_ARGS_DISPLAY_LENGTH = 200
|
||||
|
||||
|
||||
def _get_redis_queue_length(redis_client: redis.Redis, queue_name: str) -> int:
|
||||
"""Get the number of messages in a Redis-backed Celery queue.
|
||||
|
||||
Args:
|
||||
redis_client: Connected Redis client instance.
|
||||
queue_name: Name of the Celery queue to inspect.
|
||||
|
||||
Returns:
|
||||
Number of messages (tasks) waiting in the queue.
|
||||
"""
|
||||
try:
|
||||
return redis_client.llen(queue_name)
|
||||
except Exception:
|
||||
logger.debug(f"Could not read queue length for '{queue_name}'")
|
||||
return 0
|
||||
|
||||
|
||||
def _get_celery_inspect_stats() -> dict[str, Any]:
|
||||
"""Query the Celery inspect API for active, reserved, and scheduled tasks.
|
||||
|
||||
Returns:
|
||||
Dictionary with active, reserved, and scheduled task summaries.
|
||||
"""
|
||||
from app.celery_app import celery
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"active": [],
|
||||
"reserved": [],
|
||||
"scheduled": [],
|
||||
"workers_online": 0,
|
||||
}
|
||||
|
||||
try:
|
||||
inspector = celery.control.inspect(timeout=CELERY_INSPECT_TIMEOUT)
|
||||
|
||||
active = inspector.active() or {}
|
||||
reserved = inspector.reserved() or {}
|
||||
scheduled = inspector.scheduled() or {}
|
||||
|
||||
result["workers_online"] = len(active)
|
||||
|
||||
for _worker, tasks in active.items():
|
||||
for task in tasks:
|
||||
result["active"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:MAX_ARGS_DISPLAY_LENGTH],
|
||||
"started": task.get("time_start"),
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in reserved.items():
|
||||
for task in tasks:
|
||||
result["reserved"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:MAX_ARGS_DISPLAY_LENGTH],
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in scheduled.items():
|
||||
for task in tasks:
|
||||
req = task.get("request", {})
|
||||
result["scheduled"].append(
|
||||
{
|
||||
"id": req.get("id", ""),
|
||||
"name": req.get("name", "unknown"),
|
||||
"eta": task.get("eta"),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Celery inspect failed (workers may be offline): {exc}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_db_processing_summary(db: Session) -> dict[str, Any]:
|
||||
"""Query the database for a summary of file processing states.
|
||||
|
||||
Args:
|
||||
db: SQLAlchemy database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with counts of files by processing state.
|
||||
"""
|
||||
try:
|
||||
total_files = db.query(func.count(FileRecord.id)).scalar() or 0
|
||||
|
||||
# Count files with at least one in_progress step
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files with at least one failure and no in_progress
|
||||
failed_subq = (
|
||||
db.query(FileProcessingStep.file_id).filter(FileProcessingStep.status == "failure").distinct().subquery()
|
||||
)
|
||||
in_progress_subq = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
failed_count = (
|
||||
db.query(func.count(func.distinct(failed_subq.c.file_id)))
|
||||
.filter(~failed_subq.c.file_id.in_(db.query(in_progress_subq.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files that have steps and all steps are success/skipped
|
||||
all_step_files = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
# Files with any non-terminal step
|
||||
non_terminal = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status.in_(["in_progress", "pending", "failure"]))
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
completed_count = (
|
||||
db.query(func.count(func.distinct(all_step_files.c.file_id)))
|
||||
.filter(~all_step_files.c.file_id.in_(db.query(non_terminal.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Files with no processing steps at all
|
||||
files_with_steps = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
pending_count = (
|
||||
db.query(func.count(FileRecord.id))
|
||||
.filter(~FileRecord.id.in_(db.query(files_with_steps.c.file_id)))
|
||||
.filter(FileRecord.is_duplicate.is_(False))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Recent files being processed (last 20 in_progress or pending)
|
||||
recent_processing = (
|
||||
db.query(FileRecord.id, FileRecord.original_filename, FileProcessingStep.step_name)
|
||||
.join(FileProcessingStep, FileRecord.id == FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.order_by(FileProcessingStep.updated_at.desc())
|
||||
.limit(20)
|
||||
.all()
|
||||
)
|
||||
|
||||
recent_list = [
|
||||
{"file_id": r[0], "filename": r[1] or f"File #{r[0]}", "current_step": r[2]} for r in recent_processing
|
||||
]
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"processing": processing_count,
|
||||
"failed": failed_count,
|
||||
"completed": completed_count,
|
||||
"pending": pending_count,
|
||||
"recent_processing": recent_list,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error querying DB processing summary: {exc}")
|
||||
return {
|
||||
"total_files": 0,
|
||||
"processing": 0,
|
||||
"failed": 0,
|
||||
"completed": 0,
|
||||
"pending": 0,
|
||||
"recent_processing": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_queue_stats(db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Get comprehensive queue and processing statistics.
|
||||
|
||||
Returns queue lengths from Redis, Celery worker inspection data,
|
||||
and database-level processing summaries for the document pipeline.
|
||||
|
||||
Returns:
|
||||
Dictionary containing redis queue info, celery worker info,
|
||||
and database processing summary.
|
||||
"""
|
||||
# 1. Redis queue lengths
|
||||
queue_lengths: dict[str, int] = {}
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
queue_lengths[queue_name] = _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not connect to Redis: {exc}")
|
||||
|
||||
total_queued = sum(queue_lengths.values())
|
||||
|
||||
# 2. Celery inspect
|
||||
celery_stats = _get_celery_inspect_stats()
|
||||
|
||||
# 3. DB summary
|
||||
db_summary = _get_db_processing_summary(db)
|
||||
|
||||
return {
|
||||
"queues": queue_lengths,
|
||||
"total_queued": total_queued,
|
||||
"celery": celery_stats,
|
||||
"db_summary": db_summary,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pending-count")
|
||||
def get_pending_count(db: Session = Depends(get_db)) -> dict[str, int]:
|
||||
"""Get a lightweight count of queued + in-progress items for the files page banner.
|
||||
|
||||
Returns:
|
||||
Dictionary with total_pending count (queued in Redis + processing in DB).
|
||||
"""
|
||||
total_pending = 0
|
||||
|
||||
# Redis queue lengths
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
total_pending += _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
logger.debug("Could not connect to Redis for pending count")
|
||||
|
||||
# DB in-progress count
|
||||
try:
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
total_pending += processing_count
|
||||
except Exception:
|
||||
logger.debug("Could not query DB for processing count")
|
||||
|
||||
return {"total_pending": total_pending}
|
||||
@@ -1,468 +0,0 @@
|
||||
"""Routing rules API endpoints.
|
||||
|
||||
Provides full CRUD for pipeline routing rules that conditionally assign
|
||||
documents to pipelines based on document properties (file type, category,
|
||||
metadata fields, size, etc.).
|
||||
|
||||
Rules are evaluated in ascending ``position`` order. The first rule whose
|
||||
condition matches wins and routes the document to the specified target
|
||||
pipeline. If no rule matches, the caller falls back to the owner's (or
|
||||
system) default pipeline.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import Pipeline, PipelineRoutingRule
|
||||
from app.utils.routing_engine import (
|
||||
BUILTIN_FIELDS,
|
||||
VALID_OPERATORS,
|
||||
_evaluate_condition,
|
||||
_resolve_field,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/routing-rules", tags=["routing-rules"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
MAX_RULES_PER_OWNER = 100
|
||||
MAX_NAME_LENGTH = 255
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Return the authenticated user identifier."""
|
||||
user = getattr(request.state, "user", None)
|
||||
if user:
|
||||
if isinstance(user, dict):
|
||||
return user.get("sub", user.get("email", "anonymous"))
|
||||
return getattr(user, "sub", getattr(user, "email", "anonymous"))
|
||||
return "anonymous"
|
||||
|
||||
|
||||
def _is_admin(request: Request) -> bool:
|
||||
"""Return ``True`` when the current user has admin privileges."""
|
||||
user = getattr(request.state, "user", None)
|
||||
if not user:
|
||||
return False
|
||||
groups = user.get("groups", []) if isinstance(user, dict) else getattr(user, "groups", [])
|
||||
return "admin" in groups
|
||||
|
||||
|
||||
def _can_access_rule(rule: PipelineRoutingRule, user_id: str, admin: bool) -> bool:
|
||||
"""Check whether the user is allowed to read this rule."""
|
||||
if admin:
|
||||
return True
|
||||
return rule.owner_id == user_id
|
||||
|
||||
|
||||
def _can_write_rule(rule: PipelineRoutingRule, user_id: str, admin: bool) -> bool:
|
||||
"""Check whether the user is allowed to modify this rule."""
|
||||
if rule.owner_id is None:
|
||||
return admin
|
||||
return rule.owner_id == user_id
|
||||
|
||||
|
||||
def _validate_field(field: str) -> None:
|
||||
"""Raise 422 if the field name is invalid."""
|
||||
if field in BUILTIN_FIELDS:
|
||||
return
|
||||
if field.startswith("metadata.") and len(field) > len("metadata."):
|
||||
return
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=(
|
||||
f"Invalid field '{field}'. "
|
||||
f"Valid built-in fields: {sorted(BUILTIN_FIELDS)}. "
|
||||
"For AI metadata, use 'metadata.<key>'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _validate_operator(operator: str) -> None:
|
||||
"""Raise 422 if the operator is not recognised."""
|
||||
if operator not in VALID_OPERATORS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid operator '{operator}'. Valid operators: {sorted(VALID_OPERATORS)}",
|
||||
)
|
||||
|
||||
|
||||
def _serialize_rule(rule: PipelineRoutingRule) -> dict[str, Any]:
|
||||
"""Serialize a routing rule to a JSON-compatible dict."""
|
||||
return {
|
||||
"id": rule.id,
|
||||
"owner_id": rule.owner_id,
|
||||
"name": rule.name,
|
||||
"position": rule.position,
|
||||
"field": rule.field,
|
||||
"operator": rule.operator,
|
||||
"value": rule.value,
|
||||
"target_pipeline_id": rule.target_pipeline_id,
|
||||
"is_active": rule.is_active,
|
||||
"created_at": rule.created_at.isoformat() if rule.created_at else None,
|
||||
"updated_at": rule.updated_at.isoformat() if rule.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic request models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class RoutingRuleCreate(BaseModel):
|
||||
"""Request body for creating a routing rule."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=MAX_NAME_LENGTH)
|
||||
field: str = Field(..., min_length=1, max_length=255)
|
||||
operator: str = Field(..., min_length=1, max_length=50)
|
||||
value: str = Field(..., max_length=1024)
|
||||
target_pipeline_id: int
|
||||
position: int | None = None
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
class RoutingRuleUpdate(BaseModel):
|
||||
"""Request body for updating a routing rule."""
|
||||
|
||||
name: str | None = Field(None, min_length=1, max_length=MAX_NAME_LENGTH)
|
||||
field: str | None = Field(None, min_length=1, max_length=255)
|
||||
operator: str | None = Field(None, min_length=1, max_length=50)
|
||||
value: str | None = Field(None, max_length=1024)
|
||||
target_pipeline_id: int | None = None
|
||||
position: int | None = None
|
||||
is_active: bool | None = None
|
||||
|
||||
|
||||
class RoutingRuleEvaluateRequest(BaseModel):
|
||||
"""Request body for dry-run rule evaluation."""
|
||||
|
||||
file_type: str | None = None
|
||||
filename: str | None = None
|
||||
size: int | None = None
|
||||
document_type: str | None = None
|
||||
metadata: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("")
|
||||
@require_login
|
||||
def list_routing_rules(request: Request, db: DbSession) -> list[dict[str, Any]]:
|
||||
"""List all routing rules accessible by the current user.
|
||||
|
||||
Returns the user's own rules plus any system-wide rules (``owner_id=NULL``).
|
||||
Rules are sorted by position.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
rules = (
|
||||
db.query(PipelineRoutingRule)
|
||||
.filter((PipelineRoutingRule.owner_id == user_id) | (PipelineRoutingRule.owner_id.is_(None)))
|
||||
.order_by(
|
||||
PipelineRoutingRule.owner_id.is_(None).asc(),
|
||||
PipelineRoutingRule.position.asc(),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
return [_serialize_rule(r) for r in rules]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def create_routing_rule(request: Request, db: DbSession, body: RoutingRuleCreate) -> dict[str, Any]:
|
||||
"""Create a new routing rule for the current user.
|
||||
|
||||
Returns:
|
||||
The created routing rule.
|
||||
|
||||
Raises:
|
||||
HTTPException 422: If the field or operator is invalid.
|
||||
HTTPException 404: If the target pipeline does not exist.
|
||||
HTTPException 409: If the maximum number of rules is reached.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
_validate_field(body.field)
|
||||
_validate_operator(body.operator)
|
||||
|
||||
# Verify target pipeline exists and is accessible.
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == body.target_pipeline_id).first()
|
||||
if not pipeline:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Target pipeline {body.target_pipeline_id} not found",
|
||||
)
|
||||
|
||||
# Enforce per-owner limit.
|
||||
count = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.owner_id == user_id).count()
|
||||
if count >= MAX_RULES_PER_OWNER:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Maximum of {MAX_RULES_PER_OWNER} routing rules per user reached",
|
||||
)
|
||||
|
||||
# Auto-assign position if not specified.
|
||||
position = body.position
|
||||
if position is None:
|
||||
max_pos = (
|
||||
db.query(PipelineRoutingRule.position)
|
||||
.filter(PipelineRoutingRule.owner_id == user_id)
|
||||
.order_by(PipelineRoutingRule.position.desc())
|
||||
.first()
|
||||
)
|
||||
position = (max_pos[0] + 1) if max_pos else 0
|
||||
|
||||
rule = PipelineRoutingRule(
|
||||
owner_id=user_id,
|
||||
name=body.name.strip(),
|
||||
position=position,
|
||||
field=body.field,
|
||||
operator=body.operator,
|
||||
value=body.value,
|
||||
target_pipeline_id=body.target_pipeline_id,
|
||||
is_active=body.is_active,
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(rule)
|
||||
db.commit()
|
||||
db.refresh(rule)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create routing rule for user=%s", user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to create routing rule",
|
||||
)
|
||||
|
||||
logger.info("Routing rule created: id=%s, user=%s", rule.id, user_id)
|
||||
return _serialize_rule(rule)
|
||||
|
||||
|
||||
@router.get("/operators")
|
||||
def list_operators() -> dict[str, Any]:
|
||||
"""Return the list of supported operators and fields.
|
||||
|
||||
This is a public endpoint (no auth required) so that UIs can populate
|
||||
dropdowns without hard-coding the catalogue.
|
||||
"""
|
||||
return {
|
||||
"operators": sorted(VALID_OPERATORS),
|
||||
"builtin_fields": sorted(BUILTIN_FIELDS),
|
||||
"metadata_prefix": "metadata.",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/evaluate")
|
||||
@require_login
|
||||
def evaluate_rules(request: Request, db: DbSession, body: RoutingRuleEvaluateRequest) -> dict[str, Any]:
|
||||
"""Dry-run rule evaluation against the provided document properties.
|
||||
|
||||
Returns the first matching rule and target pipeline (if any), or
|
||||
indicates that no rule matched (default pipeline will be used).
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
doc_props: dict[str, Any] = {
|
||||
"file_type": body.file_type,
|
||||
"filename": body.filename,
|
||||
"size": body.size,
|
||||
"document_type": body.document_type,
|
||||
"metadata": body.metadata or {},
|
||||
}
|
||||
|
||||
rules = (
|
||||
db.query(PipelineRoutingRule)
|
||||
.filter(
|
||||
PipelineRoutingRule.is_active.is_(True),
|
||||
(PipelineRoutingRule.owner_id == user_id) | (PipelineRoutingRule.owner_id.is_(None)),
|
||||
)
|
||||
.order_by(
|
||||
PipelineRoutingRule.owner_id.is_(None).asc(),
|
||||
PipelineRoutingRule.position.asc(),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
actual = _resolve_field(rule.field, doc_props)
|
||||
if _evaluate_condition(actual, rule.operator, rule.value):
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == rule.target_pipeline_id).first()
|
||||
return {
|
||||
"matched": True,
|
||||
"rule": _serialize_rule(rule),
|
||||
"target_pipeline": {
|
||||
"id": pipeline.id,
|
||||
"name": pipeline.name,
|
||||
"is_active": pipeline.is_active,
|
||||
}
|
||||
if pipeline
|
||||
else None,
|
||||
}
|
||||
|
||||
return {"matched": False, "rule": None, "target_pipeline": None}
|
||||
|
||||
|
||||
@router.put("/reorder")
|
||||
@require_login
|
||||
def reorder_routing_rules(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
rule_ids: list[int] = Body(..., embed=True),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Reorder the caller's routing rules.
|
||||
|
||||
Expects a JSON body ``{"rule_ids": [3, 1, 2]}`` where the list
|
||||
contains the IDs of the caller's rules in the desired order.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
rules = (
|
||||
db.query(PipelineRoutingRule)
|
||||
.filter(PipelineRoutingRule.owner_id == user_id, PipelineRoutingRule.id.in_(rule_ids))
|
||||
.all()
|
||||
)
|
||||
|
||||
rule_map = {r.id: r for r in rules}
|
||||
|
||||
if len(rule_map) != len(rule_ids) or set(rule_map.keys()) != set(rule_ids):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="rule_ids must contain exactly the IDs of your routing rules",
|
||||
)
|
||||
|
||||
for pos, rid in enumerate(rule_ids):
|
||||
rule_map[rid].position = pos
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to reorder routing rules for user=%s", user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to reorder routing rules",
|
||||
)
|
||||
|
||||
ordered = sorted(rules, key=lambda r: r.position)
|
||||
return [_serialize_rule(r) for r in ordered]
|
||||
|
||||
|
||||
@router.get("/{rule_id}")
|
||||
@require_login
|
||||
def get_routing_rule(rule_id: int, request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Return a single routing rule by ID."""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
|
||||
if not rule or not _can_access_rule(rule, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
|
||||
|
||||
return _serialize_rule(rule)
|
||||
|
||||
|
||||
@router.put("/{rule_id}")
|
||||
@require_login
|
||||
def update_routing_rule(rule_id: int, request: Request, db: DbSession, body: RoutingRuleUpdate) -> dict[str, Any]:
|
||||
"""Update a routing rule.
|
||||
|
||||
Only the fields present in the request body are updated.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
|
||||
if not rule or not _can_access_rule(rule, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
|
||||
|
||||
if not _can_write_rule(rule, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this rule")
|
||||
|
||||
if body.field is not None:
|
||||
_validate_field(body.field)
|
||||
rule.field = body.field
|
||||
|
||||
if body.operator is not None:
|
||||
_validate_operator(body.operator)
|
||||
rule.operator = body.operator
|
||||
|
||||
if body.value is not None:
|
||||
rule.value = body.value
|
||||
|
||||
if body.name is not None:
|
||||
rule.name = body.name.strip()
|
||||
|
||||
if body.target_pipeline_id is not None:
|
||||
pipeline = db.query(Pipeline).filter(Pipeline.id == body.target_pipeline_id).first()
|
||||
if not pipeline:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Target pipeline {body.target_pipeline_id} not found",
|
||||
)
|
||||
rule.target_pipeline_id = body.target_pipeline_id
|
||||
|
||||
if body.position is not None:
|
||||
rule.position = body.position
|
||||
|
||||
if body.is_active is not None:
|
||||
rule.is_active = body.is_active
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(rule)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to update routing rule id=%s", rule_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update routing rule",
|
||||
)
|
||||
|
||||
logger.info("Routing rule updated: id=%s, user=%s", rule_id, user_id)
|
||||
return _serialize_rule(rule)
|
||||
|
||||
|
||||
@router.delete("/{rule_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_routing_rule(rule_id: int, request: Request, db: DbSession) -> None:
|
||||
"""Delete a routing rule."""
|
||||
user_id = _get_user_id(request)
|
||||
admin = _is_admin(request)
|
||||
|
||||
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
|
||||
if not rule or not _can_access_rule(rule, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
|
||||
|
||||
if not _can_write_rule(rule, user_id, admin):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this rule")
|
||||
|
||||
try:
|
||||
db.delete(rule)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to delete routing rule id=%s", rule_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete routing rule",
|
||||
)
|
||||
|
||||
logger.info("Routing rule deleted: id=%s, user=%s", rule_id, user_id)
|
||||
@@ -1,305 +0,0 @@
|
||||
"""
|
||||
Saved searches API endpoints.
|
||||
|
||||
Provides CRUD operations for user-defined saved search filters.
|
||||
Each user can save, list, update, and delete named filter combinations
|
||||
for quick access on the files page.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import get_current_user, require_login
|
||||
from app.database import get_db
|
||||
from app.models import SavedSearch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/saved-searches", tags=["saved-searches"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# Allowed filter keys that can be saved.
|
||||
# Files-view keys: search, mime_type, status, storage_provider, sort_by, sort_order
|
||||
# Search-view keys: q, document_type, language, sender, text_quality
|
||||
# Shared keys: tags, date_from, date_to
|
||||
ALLOWED_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"search",
|
||||
"q",
|
||||
"mime_type",
|
||||
"status",
|
||||
"date_from",
|
||||
"date_to",
|
||||
"storage_provider",
|
||||
"tags",
|
||||
"sort_by",
|
||||
"sort_order",
|
||||
"document_type",
|
||||
"language",
|
||||
"sender",
|
||||
"text_quality",
|
||||
}
|
||||
)
|
||||
|
||||
# Maximum number of saved searches per user
|
||||
MAX_SAVED_SEARCHES_PER_USER = 50
|
||||
|
||||
# Maximum length for saved search name
|
||||
MAX_NAME_LENGTH = 100
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Extract user identifier from the session.
|
||||
|
||||
Returns the preferred_username, email, or 'anonymous' if auth is disabled.
|
||||
|
||||
Args:
|
||||
request: The incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
A string identifying the current user.
|
||||
"""
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return user.get("preferred_username") or user.get("email") or user.get("name", "anonymous")
|
||||
return "anonymous"
|
||||
|
||||
|
||||
def _validate_filters(filters: Any) -> dict:
|
||||
"""Validate and sanitize filter parameters.
|
||||
|
||||
Ensures only allowed filter keys are present and values are strings.
|
||||
|
||||
Args:
|
||||
filters: The raw filter value from the client.
|
||||
|
||||
Returns:
|
||||
A sanitized filter dictionary with only allowed keys.
|
||||
|
||||
Raises:
|
||||
HTTPException: If filters is not a dict or contains invalid values.
|
||||
"""
|
||||
if not isinstance(filters, dict):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="filters must be a JSON object",
|
||||
)
|
||||
sanitized = {}
|
||||
for key, value in filters.items():
|
||||
if key in ALLOWED_FILTER_KEYS and isinstance(value, str) and value.strip():
|
||||
sanitized[key] = value.strip()
|
||||
return sanitized
|
||||
|
||||
|
||||
def _serialize_saved_search(s: SavedSearch) -> dict:
|
||||
"""Serialize a SavedSearch model instance to a JSON-compatible dict.
|
||||
|
||||
Args:
|
||||
s: The SavedSearch model instance.
|
||||
|
||||
Returns:
|
||||
A dictionary with id, name, filters, created_at, and updated_at.
|
||||
"""
|
||||
return {
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"filters": json.loads(s.filters),
|
||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||
"updated_at": s.updated_at.isoformat() if s.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
@require_login
|
||||
def list_saved_searches(request: Request, db: DbSession):
|
||||
"""List all saved searches for the current user.
|
||||
|
||||
Returns:
|
||||
A list of saved search objects with id, name, filters, and timestamps.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
searches = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).order_by(SavedSearch.name).all()
|
||||
return [_serialize_saved_search(s) for s in searches]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def create_saved_search(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str = Body(..., embed=True),
|
||||
filters: dict = Body(..., embed=True),
|
||||
):
|
||||
"""Create a new saved search for the current user.
|
||||
|
||||
Request body (JSON):
|
||||
name: Display name for the saved search (required, max 100 chars)
|
||||
filters: Dictionary of filter parameters (required)
|
||||
|
||||
Returns:
|
||||
The created saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 422: If name or filters are invalid.
|
||||
HTTPException 409: If a saved search with the same name already exists.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
name = name.strip() if isinstance(name, str) else ""
|
||||
if not name or len(name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name is required and must be at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
|
||||
# Check user limit
|
||||
count = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).count()
|
||||
if count >= MAX_SAVED_SEARCHES_PER_USER:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Maximum of {MAX_SAVED_SEARCHES_PER_USER} saved searches reached",
|
||||
)
|
||||
|
||||
# Check for duplicate name
|
||||
existing = db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == name).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{name}' already exists",
|
||||
)
|
||||
|
||||
saved_search = SavedSearch(
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
filters=json.dumps(sanitized_filters),
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(saved_search)
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create saved search for user=%s", user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to save search",
|
||||
)
|
||||
|
||||
logger.info("Saved search created: user=%s, name=%r", user_id, name)
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.put("/{search_id}")
|
||||
@require_login
|
||||
def update_saved_search(
|
||||
search_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str | None = Body(None, embed=True),
|
||||
filters: dict | None = Body(None, embed=True),
|
||||
):
|
||||
"""Update an existing saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to update.
|
||||
|
||||
Request body (JSON):
|
||||
name: New display name (optional)
|
||||
filters: New filter parameters (optional)
|
||||
|
||||
Returns:
|
||||
The updated saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
HTTPException 409: If the new name conflicts with an existing saved search.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
if name is not None:
|
||||
new_name = name.strip() if isinstance(name, str) else ""
|
||||
if not new_name or len(new_name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name must be non-empty and at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
# Check for name conflict
|
||||
if new_name != saved_search.name:
|
||||
existing = (
|
||||
db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == new_name).first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{new_name}' already exists",
|
||||
)
|
||||
saved_search.name = new_name
|
||||
|
||||
if filters is not None:
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
saved_search.filters = json.dumps(sanitized_filters)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to update saved search id=%s, user=%s", search_id, user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update saved search",
|
||||
)
|
||||
|
||||
logger.info("Saved search updated: id=%s, user=%s", search_id, user_id)
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_saved_search(search_id: int, request: Request, db: DbSession):
|
||||
"""Delete a saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to delete.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
try:
|
||||
db.delete(saved_search)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to delete saved search id=%s, user=%s", search_id, user_id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete saved search",
|
||||
)
|
||||
|
||||
logger.info("Saved search deleted: id=%s, user=%s", search_id, user_id)
|
||||
@@ -1,350 +0,0 @@
|
||||
"""
|
||||
Admin API endpoints for managing scheduled batch processing jobs.
|
||||
|
||||
All endpoints require admin privileges (checked via session ``is_admin`` flag).
|
||||
|
||||
Available routes:
|
||||
GET /api/admin/scheduled-jobs – list all scheduled jobs
|
||||
PATCH /api/admin/scheduled-jobs/{id} – update schedule / enable-disable
|
||||
POST /api/admin/scheduled-jobs/{id}/run-now – trigger a job immediately
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ScheduledJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/scheduled-jobs", tags=["admin-scheduled-jobs"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authorisation helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin; raises HTTP 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ScheduledJobResponse(BaseModel):
|
||||
"""Read model for a scheduled job."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
display_name: str
|
||||
description: str | None
|
||||
task_name: str
|
||||
enabled: bool
|
||||
schedule_type: str
|
||||
cron_minute: str
|
||||
cron_hour: str
|
||||
cron_day_of_week: str
|
||||
cron_day_of_month: str
|
||||
cron_month_of_year: str
|
||||
interval_seconds: int | None
|
||||
last_run_at: datetime | None
|
||||
last_run_status: str | None
|
||||
last_run_detail: str | None
|
||||
created_at: datetime | None
|
||||
updated_at: datetime | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ScheduledJobUpdate(BaseModel):
|
||||
"""Writable fields for a scheduled job update (all optional)."""
|
||||
|
||||
enabled: bool | None = Field(None, description="Whether the job is active")
|
||||
schedule_type: str | None = Field(None, pattern="^(cron|interval)$", description="'cron' or 'interval'")
|
||||
cron_minute: str | None = Field(None, max_length=50)
|
||||
cron_hour: str | None = Field(None, max_length=50)
|
||||
cron_day_of_week: str | None = Field(None, max_length=50)
|
||||
cron_day_of_month: str | None = Field(None, max_length=50)
|
||||
cron_month_of_year: str | None = Field(None, max_length=50)
|
||||
interval_seconds: int | None = Field(None, ge=60, description="Interval in seconds (min 60)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default job definitions – seeded into the DB on first startup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_JOBS: list[dict[str, Any]] = [
|
||||
{
|
||||
"name": "process-new-documents",
|
||||
"display_name": "Process New Documents",
|
||||
"description": (
|
||||
"Scans for documents that have been uploaded but never processed "
|
||||
"and queues them through the full processing pipeline. "
|
||||
"Runs hourly by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.process_new_documents",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "0",
|
||||
"cron_hour": "*/1",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "reprocess-failed-documents",
|
||||
"display_name": "Reprocess Failed Documents",
|
||||
"description": (
|
||||
"Finds documents whose last processing attempt failed and re-queues "
|
||||
"them for reprocessing. Only picks up files that are not currently "
|
||||
"being processed. Runs every 6 hours by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.reprocess_failed_documents",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "30",
|
||||
"cron_hour": "*/6",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "cleanup-temp-files",
|
||||
"display_name": "Clean Up Temporary Files",
|
||||
"description": (
|
||||
"Removes stale files from the workdir/tmp directory. "
|
||||
"Only files older than 24 hours that are not referenced by any active "
|
||||
"processing job are deleted. Runs daily at 03:30 UTC by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.cleanup_temp_files",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "30",
|
||||
"cron_hour": "3",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "expire-shared-links",
|
||||
"display_name": "Expire Stale Shared Links",
|
||||
"description": (
|
||||
"Marks shared document links as inactive when their expiry time has passed. "
|
||||
"Access is already blocked at request time, but this task keeps the "
|
||||
"management UI counts accurate. Runs daily at 01:00 UTC by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.expire_shared_links",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "0",
|
||||
"cron_hour": "1",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "prune-processing-logs",
|
||||
"display_name": "Prune Old Processing Logs",
|
||||
"description": (
|
||||
"Deletes processing log entries and settings audit log entries older than "
|
||||
"30 days to prevent unbounded database growth. "
|
||||
"Runs weekly on Sunday at 04:00 UTC by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.prune_processing_logs",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "0",
|
||||
"cron_hour": "4",
|
||||
"cron_day_of_week": "0",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "prune-old-notifications",
|
||||
"display_name": "Prune Old Notifications",
|
||||
"description": (
|
||||
"Deletes read in-app notifications older than 30 days. "
|
||||
"Unread notifications are never deleted. "
|
||||
"Runs weekly on Sunday at 04:30 UTC by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.prune_old_notifications",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "30",
|
||||
"cron_hour": "4",
|
||||
"cron_day_of_week": "0",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "backfill-missing-metadata",
|
||||
"display_name": "Backfill Missing AI Metadata",
|
||||
"description": (
|
||||
"Re-triggers AI metadata extraction for documents that have extracted "
|
||||
"text but no AI metadata yet (e.g., processed before an AI provider "
|
||||
"was configured). Processes up to 50 documents per run. "
|
||||
"Runs every 6 hours by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.backfill_missing_metadata",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "0",
|
||||
"cron_hour": "*/6",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "sync-search-index",
|
||||
"display_name": "Sync Search Index",
|
||||
"description": (
|
||||
"Indexes documents that have OCR text or AI metadata but are missing "
|
||||
"from the Meilisearch search index. Useful after enabling search on "
|
||||
"an existing installation or after an index rebuild. "
|
||||
"Processes up to 100 documents per run. "
|
||||
"Runs hourly by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.sync_search_index",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "15",
|
||||
"cron_hour": "*/1",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def seed_default_scheduled_jobs(db: Session) -> None:
|
||||
"""
|
||||
Insert the built-in scheduled jobs if they do not already exist.
|
||||
|
||||
Called from the FastAPI lifespan handler so the records are available
|
||||
immediately after the first startup.
|
||||
"""
|
||||
for job_data in DEFAULT_JOBS:
|
||||
existing = db.query(ScheduledJob).filter(ScheduledJob.name == job_data["name"]).first()
|
||||
if existing is None:
|
||||
db.add(ScheduledJob(**job_data))
|
||||
try:
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error("Failed to seed default scheduled jobs: %s", exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("", response_model=list[ScheduledJobResponse])
|
||||
def list_scheduled_jobs(request: Request, db: DbSession, _admin: AdminUser) -> list[ScheduledJobResponse]:
|
||||
"""
|
||||
Return all scheduled jobs ordered by display name.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
jobs = db.query(ScheduledJob).order_by(ScheduledJob.display_name).all()
|
||||
return jobs # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.patch("/{job_id}", response_model=ScheduledJobResponse)
|
||||
def update_scheduled_job(
|
||||
job_id: int,
|
||||
payload: ScheduledJobUpdate,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
) -> ScheduledJobResponse:
|
||||
"""
|
||||
Update schedule configuration or enabled state for a job.
|
||||
|
||||
Only the fields included in the request body are modified.
|
||||
Changes to the Celery Beat schedule take effect after the worker restarts.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
job = db.query(ScheduledJob).filter(ScheduledJob.id == job_id).first()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Scheduled job not found")
|
||||
|
||||
update_data = payload.model_dump(exclude_none=True)
|
||||
if not update_data:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No fields to update")
|
||||
|
||||
for field, value in update_data.items():
|
||||
setattr(job, field, value)
|
||||
|
||||
job.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error("Failed to update scheduled job %s: %s", job_id, exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update scheduled job",
|
||||
) from exc
|
||||
|
||||
logger.info("Admin updated scheduled job %s (id=%s): %s", job.name, job_id, update_data)
|
||||
return job # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.post("/{job_id}/run-now")
|
||||
def run_scheduled_job_now(
|
||||
job_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Immediately dispatch the Celery task for the given scheduled job.
|
||||
|
||||
The task is sent to the default queue; its result is tracked asynchronously
|
||||
via the ``last_run_at`` / ``last_run_status`` fields updated by the task
|
||||
itself.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
job = db.query(ScheduledJob).filter(ScheduledJob.id == job_id).first()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Scheduled job not found")
|
||||
|
||||
from app.celery_app import celery as celery_app
|
||||
|
||||
task = celery_app.send_task(job.task_name)
|
||||
logger.info("Admin triggered scheduled job %s (id=%s) manually, task_id=%s", job.name, job_id, task.id)
|
||||
|
||||
return {
|
||||
"status": "dispatched",
|
||||
"job_id": job_id,
|
||||
"job_name": job.name,
|
||||
"task_id": task.id,
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Full-text search API endpoints.
|
||||
|
||||
Provides document search across OCR text, AI metadata, filenames, and tags
|
||||
via Meilisearch. Designed to serve as the backend for the UI search bar on
|
||||
the /files page and as a standalone API for integrations.
|
||||
|
||||
Future extension point: the OCR text stored in the index is also suitable
|
||||
for RAG (Retrieval Augmented Generation) chatbot workflows.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
|
||||
from app.auth import require_login
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
@require_login
|
||||
def search_api(
|
||||
request: Request,
|
||||
q: str = Query(..., min_length=1, max_length=512, description="Full-text search query"),
|
||||
mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"),
|
||||
document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"),
|
||||
language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"),
|
||||
tags: Optional[str] = Query(None, description="Filter by tag (exact match)"),
|
||||
sender: Optional[str] = Query(None, description="Filter by sender/absender (exact match)"),
|
||||
text_quality: Optional[str] = Query(
|
||||
None,
|
||||
description="Filter by OCR text quality: no_text, low, medium, high",
|
||||
),
|
||||
date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"),
|
||||
date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"),
|
||||
page: int = Query(1, ge=1, description="Page number (1-based)"),
|
||||
per_page: int = Query(20, ge=1, le=100, description="Results per page"),
|
||||
):
|
||||
"""Search documents by full text, metadata, and tags.
|
||||
|
||||
Searches across:
|
||||
- Document title and filename
|
||||
- OCR / extracted text
|
||||
- Tags, sender, recipient, document type
|
||||
- Correspondent and reference number
|
||||
|
||||
Results are ranked by Meilisearch relevance and include highlighted
|
||||
snippets showing where the query terms matched.
|
||||
|
||||
Query Parameters:
|
||||
- q: Search query (required)
|
||||
- mime_type: Filter by MIME type
|
||||
- document_type: Filter by document type
|
||||
- language: Filter by language code
|
||||
- tags: Filter by tag (exact match on a single tag)
|
||||
- sender: Filter by sender/absender (exact match)
|
||||
- text_quality: Filter by OCR text quality (no_text, low, medium, high)
|
||||
- date_from: Unix timestamp lower bound
|
||||
- date_to: Unix timestamp upper bound
|
||||
- page: Page number (default: 1)
|
||||
- per_page: Results per page (default: 20, max: 100)
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/search?q=invoice&document_type=Invoice&tags=amazon&date_from=1704067200&page=1&per_page=20
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"file_id": 42,
|
||||
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||
"document_title": "Amazon Invoice January 2026",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["amazon", "invoice"],
|
||||
"_formatted": {
|
||||
"document_title": "Amazon <mark>Invoice</mark> January 2026",
|
||||
"ocr_text": "...total amount of the <mark>invoice</mark> is..."
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": 42,
|
||||
"page": 1,
|
||||
"pages": 3,
|
||||
"query": "invoice"
|
||||
}
|
||||
```
|
||||
"""
|
||||
logger.info(f"Search request: q={q!r}, mime_type={mime_type}, page={page}, per_page={per_page}")
|
||||
|
||||
result = search_documents(
|
||||
q,
|
||||
mime_type=mime_type,
|
||||
document_type=document_type,
|
||||
language=language,
|
||||
tags=tags,
|
||||
sender=sender,
|
||||
text_quality=text_quality,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -1,552 +0,0 @@
|
||||
"""
|
||||
API endpoints for managing application settings.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Any, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.input_validation import validate_setting_key, validate_setting_key_format
|
||||
from app.utils.settings_service import (
|
||||
SETTING_METADATA,
|
||||
delete_setting_from_db,
|
||||
get_all_settings_from_db,
|
||||
get_audit_log,
|
||||
get_setting_history,
|
||||
get_setting_metadata,
|
||||
get_settings_by_category,
|
||||
rollback_setting,
|
||||
save_setting_to_db,
|
||||
validate_setting_value,
|
||||
)
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||
|
||||
|
||||
def require_admin(request: Request) -> dict:
|
||||
"""
|
||||
Dependency to ensure the user is an admin.
|
||||
Raises HTTPException if not admin.
|
||||
|
||||
Returns:
|
||||
User dict from session
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
AdminUser = Annotated[dict, Depends(require_admin)]
|
||||
|
||||
|
||||
class SettingUpdate(BaseModel):
|
||||
"""Model for updating a setting"""
|
||||
|
||||
key: str = Field(..., description="Setting key")
|
||||
value: Optional[str] = Field(None, description="Setting value (None to delete)")
|
||||
|
||||
|
||||
class SettingResponse(BaseModel):
|
||||
"""Model for setting response"""
|
||||
|
||||
key: str
|
||||
value: Optional[str]
|
||||
metadata: Dict[str, Any]
|
||||
|
||||
|
||||
class SettingsListResponse(BaseModel):
|
||||
"""Model for list of settings"""
|
||||
|
||||
settings: Dict[str, Any]
|
||||
categories: Dict[str, list]
|
||||
db_settings: Dict[str, str]
|
||||
|
||||
|
||||
@router.get("/", response_model=SettingsListResponse)
|
||||
async def get_settings(request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get all application settings with metadata.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
# Get current runtime settings
|
||||
current_settings = {}
|
||||
for key in SETTING_METADATA.keys():
|
||||
if hasattr(settings, key):
|
||||
value = getattr(settings, key)
|
||||
current_settings[key] = {
|
||||
"value": value,
|
||||
"metadata": get_setting_metadata(key),
|
||||
}
|
||||
|
||||
# Get settings stored in database
|
||||
db_settings = get_all_settings_from_db(db)
|
||||
|
||||
# Get settings organized by category
|
||||
categories = get_settings_by_category()
|
||||
|
||||
return SettingsListResponse(settings=current_settings, categories=categories, db_settings=db_settings)
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving settings: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve settings",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/credentials")
|
||||
async def list_credentials(request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
List all sensitive credential settings with their configured/unconfigured status.
|
||||
|
||||
Returns a credential audit report indicating which credentials are set and whether
|
||||
each value originates from the database or an environment variable.
|
||||
This endpoint is intended to support credential rotation workflows.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
db_settings = get_all_settings_from_db(db)
|
||||
credentials = []
|
||||
|
||||
for key, meta in SETTING_METADATA.items():
|
||||
if not meta.get("sensitive", False):
|
||||
continue
|
||||
|
||||
env_value = getattr(settings, key, None)
|
||||
in_db = key in db_settings and db_settings[key]
|
||||
|
||||
if in_db:
|
||||
source = "db"
|
||||
configured = True
|
||||
elif env_value:
|
||||
source = "env"
|
||||
configured = True
|
||||
else:
|
||||
source = None
|
||||
configured = False
|
||||
|
||||
credentials.append(
|
||||
{
|
||||
"key": key,
|
||||
"category": meta.get("category", "Other"),
|
||||
"description": meta.get("description", ""),
|
||||
"configured": configured,
|
||||
"source": source,
|
||||
"restart_required": meta.get("restart_required", False),
|
||||
}
|
||||
)
|
||||
|
||||
configured_count = sum(1 for c in credentials if c["configured"])
|
||||
return {
|
||||
"credentials": credentials,
|
||||
"total": len(credentials),
|
||||
"configured_count": configured_count,
|
||||
"unconfigured_count": len(credentials) - configured_count,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving credential list: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve credentials",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/audit-log")
|
||||
async def list_audit_log(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
):
|
||||
"""
|
||||
Retrieve the settings audit log (most recent first).
|
||||
|
||||
Returns all configuration changes recorded in the audit log.
|
||||
Sensitive values are masked in the response.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
entries = get_audit_log(db, limit=limit, offset=offset)
|
||||
return {"entries": entries, "limit": limit, "offset": offset}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving audit log: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve audit log",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/export-env")
|
||||
async def export_env_settings(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
source: str = "db",
|
||||
):
|
||||
"""
|
||||
Export current settings as a ``.env`` file.
|
||||
|
||||
Query params:
|
||||
- ``source=db`` (default) – only settings explicitly saved to the database.
|
||||
- ``source=effective`` – full runtime configuration (DB > ENV > defaults) for
|
||||
every key defined in SETTING_METADATA.
|
||||
|
||||
Returns a downloadable plain-text file suitable for bootstrapping another
|
||||
installation. All values — including sensitive ones — are included; only
|
||||
admins can access this endpoint.
|
||||
"""
|
||||
from fastapi.responses import Response as FastAPIResponse
|
||||
|
||||
from app.utils.settings_service import get_settings_for_export
|
||||
|
||||
if source not in ("db", "effective"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="source must be 'db' or 'effective'",
|
||||
)
|
||||
|
||||
try:
|
||||
export_data = get_settings_for_export(db, source=source)
|
||||
lines = [
|
||||
"# DocuElevate configuration export",
|
||||
f"# Source: {source}",
|
||||
"# Generated by DocuElevate Settings Export",
|
||||
"# WARNING: This file contains sensitive values. Handle with care.",
|
||||
"",
|
||||
]
|
||||
for env_key, value in export_data.items():
|
||||
lines.append(f"{env_key}={value}")
|
||||
lines.append("") # trailing newline
|
||||
content = "\n".join(lines)
|
||||
|
||||
return FastAPIResponse(
|
||||
content=content,
|
||||
media_type="text/plain",
|
||||
headers={"Content-Disposition": f'attachment; filename="docuelevate-{source}.env"'},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error exporting settings: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to export settings",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key}", response_model=SettingResponse)
|
||||
async def get_setting(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get a specific setting by key.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
# Get current value
|
||||
value = getattr(settings, key, None)
|
||||
|
||||
# Get metadata
|
||||
metadata = get_setting_metadata(key)
|
||||
|
||||
return SettingResponse(key=key, value=str(value) if value is not None else None, metadata=metadata)
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to retrieve setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{key}")
|
||||
async def update_setting(
|
||||
key: str,
|
||||
setting: SettingUpdate,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
):
|
||||
"""
|
||||
Update a specific setting.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key(key)
|
||||
try:
|
||||
# Validate the setting value
|
||||
if setting.value is not None:
|
||||
is_valid, error_message = validate_setting_value(key, setting.value)
|
||||
if not is_valid:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message)
|
||||
|
||||
# Determine the username for the audit log
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
# Save to database
|
||||
success = save_setting_to_db(db, key, setting.value, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to save setting to database",
|
||||
)
|
||||
|
||||
# Notify workers that settings have changed
|
||||
notify_settings_updated()
|
||||
|
||||
# Get metadata
|
||||
metadata = get_setting_metadata(key)
|
||||
restart_required = metadata.get("restart_required", False)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' updated successfully",
|
||||
"restart_required": restart_required,
|
||||
"key": key,
|
||||
"value": setting.value,
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{key}")
|
||||
async def delete_setting(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Delete a setting from the database (reverts to environment variable or default).
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key(key)
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
success = delete_setting_from_db(db, key, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Setting '{key}' not found in database",
|
||||
)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' deleted from database (will use environment variable or default)",
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to delete setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/bulk-update")
|
||||
async def bulk_update_settings(updates: list[SettingUpdate], request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Update multiple settings at once.
|
||||
Admin only.
|
||||
"""
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
for update in updates:
|
||||
try:
|
||||
# Validate the setting value
|
||||
if update.value is not None:
|
||||
is_valid, error_message = validate_setting_value(update.key, update.value)
|
||||
if not is_valid:
|
||||
errors.append({"key": update.key, "error": error_message})
|
||||
continue
|
||||
|
||||
# Save to database
|
||||
success = save_setting_to_db(db, update.key, update.value, changed_by=changed_by)
|
||||
if success:
|
||||
results.append({"key": update.key, "value": update.value, "status": "success"})
|
||||
else:
|
||||
errors.append({"key": update.key, "error": "Failed to save to database"})
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating setting {update.key}: {e}")
|
||||
errors.append({"key": update.key, "error": str(e)})
|
||||
|
||||
if results:
|
||||
notify_settings_updated()
|
||||
|
||||
restart_required = any(get_setting_metadata(result["key"]).get("restart_required", False) for result in results)
|
||||
|
||||
return {
|
||||
"success": len(errors) == 0,
|
||||
"updated": results,
|
||||
"errors": errors,
|
||||
"restart_required": restart_required,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/install-ocr-languages")
|
||||
async def install_ocr_languages(request: Request, admin: AdminUser):
|
||||
"""
|
||||
Trigger on-demand installation of Tesseract language data files and
|
||||
EasyOCR model downloads for the languages currently configured in the
|
||||
application settings.
|
||||
|
||||
This endpoint is useful after changing ``tesseract_language`` or
|
||||
``easyocr_languages`` so that the required data is available without
|
||||
restarting the container. The download runs synchronously and may take
|
||||
a few seconds (or minutes for large EasyOCR models).
|
||||
|
||||
Returns a summary of which languages are now available and which could
|
||||
not be installed.
|
||||
Admin only.
|
||||
"""
|
||||
from app.utils.ocr_language_manager import ensure_ocr_languages_from_settings # noqa: PLC0415
|
||||
|
||||
try:
|
||||
result = ensure_ocr_languages_from_settings()
|
||||
tesseract_missing = result.get("tesseract_missing", [])
|
||||
easyocr_failed = result.get("easyocr_failed", [])
|
||||
all_ok = not tesseract_missing and not easyocr_failed
|
||||
return {
|
||||
"success": all_ok,
|
||||
"tesseract_missing": tesseract_missing,
|
||||
"easyocr_failed": easyocr_failed,
|
||||
"message": (
|
||||
"All configured OCR languages are available."
|
||||
if all_ok
|
||||
else f"Some languages could not be installed: tesseract={tesseract_missing}, easyocr={easyocr_failed}"
|
||||
),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error during OCR language installation: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to install OCR language data",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key}/suggestions")
|
||||
async def get_setting_suggestions(
|
||||
key: str,
|
||||
request: Request,
|
||||
q: str = "",
|
||||
limit: int = 10,
|
||||
):
|
||||
"""
|
||||
Return autocomplete suggestions for a setting key.
|
||||
|
||||
Fetches values dynamically from cloud SDKs, installed tools, or
|
||||
curated static lists depending on the setting. Results are filtered
|
||||
by case-insensitive substring match on the ``q`` parameter.
|
||||
|
||||
This endpoint does **not** require admin privileges so that the
|
||||
autocomplete widget works for any authenticated user viewing settings.
|
||||
"""
|
||||
from app.utils.suggestion_providers import SUGGESTION_PROVIDERS, get_suggestions # noqa: PLC0415
|
||||
|
||||
if key not in SUGGESTION_PROVIDERS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"No suggestions available for setting '{key}'",
|
||||
)
|
||||
|
||||
try:
|
||||
suggestions = get_suggestions(key, query=q, limit=max(1, min(limit, 50)))
|
||||
return {"key": key, "suggestions": suggestions}
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching suggestions for {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to fetch suggestions",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key}/history")
|
||||
async def get_key_history(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get the change history for a specific setting key.
|
||||
|
||||
Returns all audit log entries for that key, most recent first.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
entries = get_setting_history(db, key)
|
||||
return {"key": key, "history": entries}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving history for {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to retrieve history for setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{key}/rollback/{history_id}")
|
||||
async def rollback_setting_to_history(
|
||||
key: str,
|
||||
history_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
):
|
||||
"""
|
||||
Revert a setting to the value it had *before* a specific audit log change.
|
||||
|
||||
The ``history_id`` is the ID of the :class:`~app.models.SettingsAuditLog`
|
||||
entry whose ``old_value`` should be reinstated, effectively undoing that
|
||||
change. If ``old_value`` is ``None`` (the setting did not exist before
|
||||
that change), the setting is removed from the database and reverts to its
|
||||
ENV/default value.
|
||||
|
||||
A new audit log entry is written to record the rollback.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
success = rollback_setting(db, key, history_id, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"History entry {history_id} not found for setting '{key}'",
|
||||
)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' rolled back to history entry {history_id}",
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error rolling back setting {key} to history {history_id}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to roll back setting: {key}",
|
||||
)
|
||||
@@ -1,500 +0,0 @@
|
||||
"""API endpoints for document sharing via expiring links.
|
||||
|
||||
Authenticated users can create time-limited or view-limited shareable
|
||||
links for their documents. Each link has a cryptographically random
|
||||
token that forms a public ``/share/<token>`` URL. Optional password
|
||||
protection is supported; only a PBKDF2-HMAC-SHA256 hash is stored.
|
||||
|
||||
Public consumers access files through the ``/share/<token>/download``
|
||||
and ``/share/<token>/info`` endpoints — no authentication required.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, SharedLink
|
||||
from app.utils.user_scope import apply_owner_filter, get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/shared-links", tags=["shared-links"])
|
||||
public_router = APIRouter(tags=["shared-links-public"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: PBKDF2 iteration count — matches OWASP 2023 recommendation for PBKDF2-HMAC-SHA256.
|
||||
_PWD_HASH_ITERATIONS = 600_000
|
||||
#: Length of the random per-password salt in bytes (128-bit entropy).
|
||||
_PWD_SALT_BYTES = 16
|
||||
|
||||
# Valid expiry durations (in hours) presented in the UI.
|
||||
EXPIRY_OPTIONS: dict[str, int] = {
|
||||
"1h": 1,
|
||||
"6h": 6,
|
||||
"12h": 12,
|
||||
"24h": 24,
|
||||
"3d": 72,
|
||||
"7d": 168,
|
||||
"14d": 336,
|
||||
"30d": 720,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _generate_token() -> str:
|
||||
"""Generate a 43-character URL-safe random token."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def _hash_password(password: str) -> str:
|
||||
"""Hash *password* with PBKDF2-HMAC-SHA256 and a random per-password salt.
|
||||
|
||||
The returned string uses the format ``{salt_hex}:{dk_hex}`` so that
|
||||
both the salt and the digest can be recovered from a single column.
|
||||
|
||||
Args:
|
||||
password: Plaintext password string.
|
||||
|
||||
Returns:
|
||||
String in the form ``<32-char salt hex>:<64-char digest hex>``,
|
||||
totalling 97 characters (well within the 128-char column limit).
|
||||
"""
|
||||
salt = secrets.token_bytes(_PWD_SALT_BYTES)
|
||||
dk = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
password.encode("utf-8"),
|
||||
salt,
|
||||
_PWD_HASH_ITERATIONS,
|
||||
)
|
||||
return f"{salt.hex()}:{dk.hex()}"
|
||||
|
||||
|
||||
def _verify_password(password: str, stored_hash: str) -> bool:
|
||||
"""Verify *password* against a hash produced by :func:`_hash_password`.
|
||||
|
||||
Uses constant-time comparison to prevent timing attacks.
|
||||
|
||||
Args:
|
||||
password: Plaintext password to check.
|
||||
stored_hash: The value previously returned by :func:`_hash_password`.
|
||||
|
||||
Returns:
|
||||
``True`` if *password* matches, ``False`` otherwise.
|
||||
"""
|
||||
try:
|
||||
salt_hex, dk_hex = stored_hash.split(":", 1)
|
||||
salt = bytes.fromhex(salt_hex)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
dk = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
password.encode("utf-8"),
|
||||
salt,
|
||||
_PWD_HASH_ITERATIONS,
|
||||
)
|
||||
return secrets.compare_digest(dk.hex(), dk_hex)
|
||||
|
||||
|
||||
def _is_link_valid(link: SharedLink) -> bool:
|
||||
"""Return True when *link* is active, unexpired, and within view limit."""
|
||||
if not link.is_active:
|
||||
return False
|
||||
now = datetime.now(timezone.utc)
|
||||
if link.expires_at is not None:
|
||||
exp = link.expires_at
|
||||
if exp.tzinfo is None:
|
||||
exp = exp.replace(tzinfo=timezone.utc)
|
||||
if now > exp:
|
||||
return False
|
||||
if link.max_views is not None and link.view_count >= link.max_views:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _resolve_file_path(file_record: FileRecord) -> str | None:
|
||||
"""Return the best available file path for *file_record*.
|
||||
|
||||
Checks processed path first, then original, then local (tmp) path.
|
||||
Returns ``None`` when no file exists on disk.
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
workdir = os.path.realpath(settings.workdir)
|
||||
candidates = [
|
||||
file_record.processed_file_path,
|
||||
file_record.original_file_path,
|
||||
file_record.local_filename,
|
||||
]
|
||||
for path in candidates:
|
||||
if not path:
|
||||
continue
|
||||
# Guard against path traversal in DB values.
|
||||
real = os.path.realpath(path)
|
||||
if not real.startswith(workdir + os.sep) and real != workdir:
|
||||
logger.warning("Shared link file path outside workdir rejected: %s", path)
|
||||
continue
|
||||
if os.path.exists(real):
|
||||
return real
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SharedLinkCreate(BaseModel):
|
||||
"""Schema for creating a new shared link."""
|
||||
|
||||
file_id: int = Field(..., description="ID of the file to share")
|
||||
expires_in_hours: int | None = Field(
|
||||
None,
|
||||
ge=1,
|
||||
le=720,
|
||||
description="Expiry in hours (1–720). NULL means the link never expires.",
|
||||
)
|
||||
max_views: int | None = Field(
|
||||
None,
|
||||
ge=1,
|
||||
le=10_000,
|
||||
description="Maximum number of downloads/views. NULL means unlimited.",
|
||||
)
|
||||
password: str | None = Field(
|
||||
None,
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
description="Optional password protecting the link.",
|
||||
)
|
||||
label: str | None = Field(
|
||||
None,
|
||||
max_length=255,
|
||||
description="Optional human-readable label for the link.",
|
||||
)
|
||||
|
||||
@field_validator("expires_in_hours")
|
||||
@classmethod
|
||||
def validate_expiry(cls, v: int | None) -> int | None:
|
||||
if v is not None and v not in range(1, 721):
|
||||
raise ValueError("expires_in_hours must be between 1 and 720")
|
||||
return v
|
||||
|
||||
|
||||
class SharedLinkResponse(BaseModel):
|
||||
"""Shared link info returned to the authenticated owner."""
|
||||
|
||||
id: int
|
||||
token: str
|
||||
file_id: int
|
||||
label: str | None
|
||||
expires_at: datetime | None
|
||||
max_views: int | None
|
||||
view_count: int
|
||||
has_password: bool
|
||||
is_active: bool
|
||||
created_at: datetime | None
|
||||
revoked_at: datetime | None
|
||||
# Filled in by the endpoint, not stored in DB.
|
||||
share_url: str = ""
|
||||
original_filename: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SharedLinkInfoResponse(BaseModel):
|
||||
"""Public metadata about a shared link (used on the share landing page)."""
|
||||
|
||||
token: str
|
||||
label: str | None
|
||||
original_filename: str | None
|
||||
expires_at: datetime | None
|
||||
max_views: int | None
|
||||
view_count: int
|
||||
has_password: bool
|
||||
is_valid: bool
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Private (authenticated) endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, response_model=SharedLinkResponse)
|
||||
async def create_shared_link(
|
||||
body: SharedLinkCreate,
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new shareable link for a document.
|
||||
|
||||
The caller must own the file (or be in single-user mode).
|
||||
Returns the full link metadata including the generated token.
|
||||
"""
|
||||
# Verify the file exists and belongs to the caller.
|
||||
q = db.query(FileRecord).filter(FileRecord.id == body.file_id)
|
||||
q = apply_owner_filter(q, request)
|
||||
file_record = q.first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
token = _generate_token()
|
||||
expires_at = None
|
||||
if body.expires_in_hours is not None:
|
||||
expires_at = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
from datetime import timedelta
|
||||
|
||||
expires_at = expires_at + timedelta(hours=body.expires_in_hours)
|
||||
|
||||
password_hash = _hash_password(body.password) if body.password else None
|
||||
|
||||
db_link = SharedLink(
|
||||
token=token,
|
||||
file_id=body.file_id,
|
||||
owner_id=owner_id,
|
||||
label=body.label,
|
||||
expires_at=expires_at,
|
||||
max_views=body.max_views,
|
||||
view_count=0,
|
||||
password_hash=password_hash,
|
||||
)
|
||||
try:
|
||||
db.add(db_link)
|
||||
db.commit()
|
||||
db.refresh(db_link)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Shared link created: id=%s owner=%s file_id=%s", db_link.id, owner_id, body.file_id)
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
return _link_to_dict(db_link, base_url, file_record.original_filename)
|
||||
|
||||
|
||||
@router.get("/", response_model=list[SharedLinkResponse])
|
||||
async def list_shared_links(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all shared links created by the authenticated user."""
|
||||
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
|
||||
if active_only:
|
||||
q = q.filter(SharedLink.is_active.is_(True))
|
||||
links = q.order_by(SharedLink.created_at.desc()).all()
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
result = []
|
||||
for link in links:
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
|
||||
filename = file_record.original_filename if file_record else None
|
||||
result.append(_link_to_dict(link, base_url, filename))
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/{link_id}", status_code=status.HTTP_200_OK)
|
||||
async def revoke_shared_link(
|
||||
link_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, str]:
|
||||
"""Revoke (soft-delete) a shared link.
|
||||
|
||||
The record is kept for audit purposes but the link immediately
|
||||
stops working for recipients.
|
||||
"""
|
||||
db_link = db.query(SharedLink).filter(SharedLink.id == link_id, SharedLink.owner_id == owner_id).first()
|
||||
if not db_link:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Shared link not found")
|
||||
|
||||
if not db_link.is_active:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Link is already revoked")
|
||||
|
||||
try:
|
||||
db_link.is_active = False
|
||||
db_link.revoked_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Shared link revoked: id=%s owner=%s", link_id, owner_id)
|
||||
return {"detail": "Link revoked"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public endpoints (no authentication required)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@public_router.get("/share/{token}/info", response_model=SharedLinkInfoResponse)
|
||||
def get_shared_link_info(
|
||||
token: str,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Return public metadata about a shared link.
|
||||
|
||||
Used by the share landing page to decide whether to show a password
|
||||
prompt or a direct download button. Never returns sensitive data.
|
||||
"""
|
||||
link = db.query(SharedLink).filter(SharedLink.token == token).first()
|
||||
if not link:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Link not found")
|
||||
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
|
||||
filename = file_record.original_filename if file_record else None
|
||||
|
||||
return {
|
||||
"token": link.token,
|
||||
"label": link.label,
|
||||
"original_filename": filename,
|
||||
"expires_at": link.expires_at,
|
||||
"max_views": link.max_views,
|
||||
"view_count": link.view_count,
|
||||
"has_password": link.password_hash is not None,
|
||||
"is_valid": _is_link_valid(link),
|
||||
}
|
||||
|
||||
|
||||
@public_router.get("/share/{token}/download")
|
||||
def download_via_shared_link(
|
||||
token: str,
|
||||
db: DbSession,
|
||||
) -> FileResponse:
|
||||
"""Download a file via a shared link that does NOT require a password.
|
||||
|
||||
For password-protected links use ``POST /api/share/{token}/download``
|
||||
with ``{"password": "<value>"}`` in the JSON body instead.
|
||||
|
||||
Increments the view counter and validates expiry / view limit before
|
||||
serving the file.
|
||||
"""
|
||||
return _serve_shared_file(token, db, password=None)
|
||||
|
||||
|
||||
class PasswordBody(BaseModel):
|
||||
"""Request body for password-protected shared link downloads."""
|
||||
|
||||
password: str = Field(..., min_length=1, max_length=128, description="Password for the shared link")
|
||||
|
||||
|
||||
@public_router.post("/share/{token}/download")
|
||||
def download_via_shared_link_with_password(
|
||||
token: str,
|
||||
body: PasswordBody,
|
||||
db: DbSession,
|
||||
) -> FileResponse:
|
||||
"""Download a password-protected file via a shared link.
|
||||
|
||||
Accepts the password in the JSON request body to avoid it appearing in
|
||||
server access logs, browser history, or ``Referer`` headers.
|
||||
"""
|
||||
return _serve_shared_file(token, db, password=body.password)
|
||||
|
||||
|
||||
def _serve_shared_file(token: str, db: Session, password: str | None) -> FileResponse:
|
||||
"""Core download logic shared by the GET and POST download endpoints."""
|
||||
link = db.query(SharedLink).filter(SharedLink.token == token).first()
|
||||
if not link:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Link not found or expired")
|
||||
|
||||
if not _is_link_valid(link):
|
||||
raise HTTPException(status_code=status.HTTP_410_GONE, detail="Link has expired or reached its view limit")
|
||||
|
||||
# Password check
|
||||
if link.password_hash is not None:
|
||||
if not password:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="This link requires a password",
|
||||
)
|
||||
if not _verify_password(password, link.password_hash):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Incorrect password")
|
||||
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
file_path = _resolve_file_path(file_record)
|
||||
if not file_path:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not available on disk")
|
||||
|
||||
# Increment view count — fail the request if this cannot be persisted so
|
||||
# that view-limited links are not bypassed during temporary DB outages.
|
||||
try:
|
||||
link.view_count = (link.view_count or 0) + 1
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.error("Failed to increment view_count for shared link id=%s — aborting download", link.id)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Service temporarily unavailable. Please try again.",
|
||||
)
|
||||
|
||||
return FileResponse(
|
||||
path=file_path,
|
||||
media_type=file_record.mime_type or "application/octet-stream",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{file_record.original_filename or "document"}"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _link_to_dict(link: SharedLink, base_url: str, original_filename: str | None) -> dict[str, Any]:
|
||||
"""Serialise a ``SharedLink`` ORM row to a plain dict."""
|
||||
return {
|
||||
"id": link.id,
|
||||
"token": link.token,
|
||||
"file_id": link.file_id,
|
||||
"label": link.label,
|
||||
"expires_at": link.expires_at,
|
||||
"max_views": link.max_views,
|
||||
"view_count": link.view_count,
|
||||
"has_password": link.password_hash is not None,
|
||||
"is_active": link.is_active,
|
||||
"created_at": link.created_at,
|
||||
"revoked_at": link.revoked_at,
|
||||
"share_url": f"{base_url}/share/{link.token}",
|
||||
"original_filename": original_filename,
|
||||
}
|
||||
@@ -1,473 +0,0 @@
|
||||
"""Document similarity API endpoints.
|
||||
|
||||
Provides endpoints to find documents similar to a given file based on
|
||||
text embeddings and cosine similarity scoring, plus debug/diagnostic
|
||||
endpoints for inspecting and triggering embedding computation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/similar")
|
||||
@require_login
|
||||
def get_similar_documents(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
limit: int = Query(5, ge=1, le=20, description="Maximum number of similar documents to return"),
|
||||
threshold: float = Query(0.3, ge=0.0, le=1.0, description="Minimum similarity score (0–1)"),
|
||||
):
|
||||
"""Find documents similar to the specified file.
|
||||
|
||||
Uses text embeddings generated from OCR-extracted text and cosine
|
||||
similarity to rank documents by relevance. Similarity scores range
|
||||
from 0 (completely different) to 1 (identical content).
|
||||
|
||||
Embeddings are generated on first access and cached for subsequent
|
||||
requests. Documents without OCR text are excluded.
|
||||
|
||||
Query Parameters:
|
||||
- limit: Maximum results to return (default: 5, max: 20)
|
||||
- threshold: Minimum similarity score to include (default: 0.3)
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/files/42/similar?limit=5&threshold=0.5
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"file_id": 42,
|
||||
"similar_documents": [
|
||||
{
|
||||
"file_id": 15,
|
||||
"original_filename": "Invoice_2026-01.pdf",
|
||||
"document_title": "January Invoice",
|
||||
"similarity_score": 0.8934,
|
||||
"mime_type": "application/pdf",
|
||||
"created_at": "2026-01-15T10:30:00+00:00"
|
||||
}
|
||||
],
|
||||
"count": 1
|
||||
}
|
||||
```
|
||||
"""
|
||||
# Verify the file exists
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
if not file_record.ocr_text or not file_record.ocr_text.strip():
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"similar_documents": [],
|
||||
"count": 0,
|
||||
"message": "No OCR text available for similarity comparison",
|
||||
}
|
||||
|
||||
# Check whether an embedding has been computed yet
|
||||
if not file_record.embedding:
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"similar_documents": [],
|
||||
"count": 0,
|
||||
"message": (
|
||||
"Embedding not yet computed for this file. "
|
||||
"It will be generated automatically during processing or via the backfill task. "
|
||||
"You can also trigger it manually with POST /api/files/{file_id}/compute-embedding."
|
||||
),
|
||||
}
|
||||
|
||||
try:
|
||||
from app.utils.similarity import find_similar_documents
|
||||
|
||||
similar = find_similar_documents(db, file_id, limit=limit, threshold=threshold)
|
||||
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"similar_documents": similar,
|
||||
"count": len(similar),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error finding similar documents for file {file_id}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to compute document similarity",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Debug / diagnostic endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/embedding-status")
|
||||
@require_login
|
||||
def get_embedding_status(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
):
|
||||
"""Return the embedding status for a single file.
|
||||
|
||||
Useful for debugging whether the embedding has been computed
|
||||
and cached for a given document.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"file_id": 42,
|
||||
"has_embedding": true,
|
||||
"embedding_dimensions": 1536,
|
||||
"has_ocr_text": true,
|
||||
"ocr_text_length": 4200,
|
||||
"embedding_model": "text-embedding-3-small"
|
||||
}
|
||||
```
|
||||
"""
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
has_embedding = False
|
||||
embedding_dimensions = None
|
||||
if file_record.embedding:
|
||||
try:
|
||||
parsed = json.loads(file_record.embedding)
|
||||
has_embedding = True
|
||||
embedding_dimensions = len(parsed)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
has_ocr_text = bool(file_record.ocr_text and file_record.ocr_text.strip())
|
||||
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"has_embedding": has_embedding,
|
||||
"embedding_dimensions": embedding_dimensions,
|
||||
"has_ocr_text": has_ocr_text,
|
||||
"ocr_text_length": len(file_record.ocr_text) if file_record.ocr_text else 0,
|
||||
"embedding_model": settings.embedding_model,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/compute-embedding")
|
||||
@require_login
|
||||
def trigger_compute_embedding(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
):
|
||||
"""Trigger embedding computation for a single file.
|
||||
|
||||
If the file already has a cached embedding it will be recomputed.
|
||||
The computation happens synchronously so the caller receives the
|
||||
result immediately.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"file_id": 42,
|
||||
"status": "success",
|
||||
"embedding_dimensions": 1536
|
||||
}
|
||||
```
|
||||
"""
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
if not file_record.ocr_text or not file_record.ocr_text.strip():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File has no OCR text — cannot generate embedding",
|
||||
)
|
||||
|
||||
try:
|
||||
from app.utils.similarity import generate_embedding
|
||||
|
||||
# Clear cached embedding to force recomputation
|
||||
file_record.embedding = None
|
||||
db.flush()
|
||||
|
||||
embedding = generate_embedding(file_record.ocr_text)
|
||||
file_record.embedding = json.dumps(embedding)
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"status": "success",
|
||||
"embedding_dimensions": len(embedding),
|
||||
}
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Failed to compute embedding for file {file_id}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Embedding computation failed: {e}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/diagnostic/embeddings")
|
||||
@require_login
|
||||
def get_embeddings_overview(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
):
|
||||
"""Return an overview of embedding status across all files.
|
||||
|
||||
Provides aggregate counts as well as a per-file breakdown so an
|
||||
administrator can quickly identify documents that are missing
|
||||
embeddings.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"total_files": 120,
|
||||
"files_with_ocr_text": 95,
|
||||
"files_with_embedding": 42,
|
||||
"files_missing_embedding": 53,
|
||||
"embedding_model": "text-embedding-3-small",
|
||||
"files": [
|
||||
{
|
||||
"file_id": 1,
|
||||
"original_filename": "invoice.pdf",
|
||||
"has_ocr_text": true,
|
||||
"has_embedding": true,
|
||||
"embedding_dimensions": 1536
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
"""
|
||||
# Use column-only query to avoid loading full ORM objects into memory
|
||||
all_files = (
|
||||
db.query(
|
||||
FileRecord.id,
|
||||
FileRecord.original_filename,
|
||||
FileRecord.ocr_text,
|
||||
FileRecord.embedding,
|
||||
)
|
||||
.order_by(FileRecord.id.desc())
|
||||
.all()
|
||||
)
|
||||
|
||||
files_info = []
|
||||
total_with_ocr = 0
|
||||
total_with_embedding = 0
|
||||
|
||||
for f in all_files:
|
||||
has_ocr = bool(f.ocr_text and f.ocr_text.strip())
|
||||
has_emb = False
|
||||
emb_dims = None
|
||||
|
||||
if f.embedding:
|
||||
try:
|
||||
parsed = json.loads(f.embedding)
|
||||
has_emb = True
|
||||
emb_dims = len(parsed)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
if has_ocr:
|
||||
total_with_ocr += 1
|
||||
if has_emb:
|
||||
total_with_embedding += 1
|
||||
|
||||
files_info.append(
|
||||
{
|
||||
"file_id": f.id,
|
||||
"original_filename": f.original_filename,
|
||||
"has_ocr_text": has_ocr,
|
||||
"has_embedding": has_emb,
|
||||
"embedding_dimensions": emb_dims,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"total_files": len(all_files),
|
||||
"files_with_ocr_text": total_with_ocr,
|
||||
"files_with_embedding": total_with_embedding,
|
||||
"files_missing_embedding": total_with_ocr - total_with_embedding,
|
||||
"embedding_model": settings.embedding_model,
|
||||
"files": files_info,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/diagnostic/compute-all-embeddings")
|
||||
@require_login
|
||||
def trigger_compute_all_embeddings(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
):
|
||||
"""Queue embedding computation for all files that have OCR text but no embedding.
|
||||
|
||||
Each file is processed as a separate Celery task so the endpoint
|
||||
returns immediately.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"status": "queued",
|
||||
"files_queued": 53
|
||||
}
|
||||
```
|
||||
"""
|
||||
candidates = (
|
||||
db.query(FileRecord)
|
||||
.filter(
|
||||
FileRecord.ocr_text.isnot(None),
|
||||
FileRecord.ocr_text != "",
|
||||
(FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
queued = 0
|
||||
for f in candidates:
|
||||
try:
|
||||
from app.tasks.compute_embedding import compute_document_embedding
|
||||
|
||||
compute_document_embedding.delay(f.id)
|
||||
queued += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue embedding for file {f.id}: {e}")
|
||||
|
||||
return {
|
||||
"status": "queued",
|
||||
"files_queued": queued,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/similarity/pairs")
|
||||
@require_login
|
||||
def get_similarity_pairs(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
threshold: float = Query(0.7, ge=0.0, le=1.0, description="Minimum similarity score for a pair"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Maximum number of pairs to return"),
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
):
|
||||
"""Return pairs of documents with high similarity across the entire corpus.
|
||||
|
||||
Unlike the per-file ``/files/{id}/similar`` endpoint, this scans every
|
||||
document that has a pre-computed embedding and returns **all** pairs
|
||||
whose cosine similarity exceeds ``threshold``, sorted by descending
|
||||
score.
|
||||
|
||||
To keep memory bounded the query loads only the columns needed for
|
||||
scoring and streams results in chunks.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"pairs": [
|
||||
{
|
||||
"file_a": {"file_id": 1, "original_filename": "invoice_jan.pdf", ...},
|
||||
"file_b": {"file_id": 5, "original_filename": "invoice_feb.pdf", ...},
|
||||
"similarity_score": 0.94
|
||||
}
|
||||
],
|
||||
"total_pairs": 12,
|
||||
"threshold": 0.7,
|
||||
"page": 1,
|
||||
"pages": 1,
|
||||
"embedding_coverage": {"total_files": 120, "files_with_embedding": 95}
|
||||
}
|
||||
```
|
||||
"""
|
||||
from app.utils.similarity import cosine_similarity
|
||||
|
||||
# Load all files that have embeddings (columns only for efficiency)
|
||||
rows = (
|
||||
db.query(
|
||||
FileRecord.id,
|
||||
FileRecord.original_filename,
|
||||
FileRecord.document_title,
|
||||
FileRecord.mime_type,
|
||||
FileRecord.created_at,
|
||||
FileRecord.embedding,
|
||||
)
|
||||
.filter(
|
||||
FileRecord.embedding.isnot(None),
|
||||
FileRecord.embedding != "",
|
||||
)
|
||||
.order_by(FileRecord.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Parse embeddings upfront
|
||||
parsed: list[tuple] = []
|
||||
for row in rows:
|
||||
try:
|
||||
vec = json.loads(row.embedding)
|
||||
parsed.append((row, vec))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
# Pairwise comparison (triangle: i < j avoids duplicating A↔B / B↔A)
|
||||
all_pairs: list[dict] = []
|
||||
for i in range(len(parsed)):
|
||||
row_a, vec_a = parsed[i]
|
||||
for j in range(i + 1, len(parsed)):
|
||||
row_b, vec_b = parsed[j]
|
||||
score = cosine_similarity(vec_a, vec_b)
|
||||
if score >= threshold:
|
||||
all_pairs.append(
|
||||
{
|
||||
"file_a": _row_to_dict(row_a),
|
||||
"file_b": _row_to_dict(row_b),
|
||||
"similarity_score": round(score, 4),
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by score descending
|
||||
all_pairs.sort(key=lambda p: p["similarity_score"], reverse=True)
|
||||
|
||||
total_pairs = len(all_pairs)
|
||||
total_pages = max(1, (total_pairs + limit - 1) // limit)
|
||||
offset = (page - 1) * limit
|
||||
page_pairs = all_pairs[offset : offset + limit]
|
||||
|
||||
total_files = db.query(FileRecord).count()
|
||||
|
||||
return {
|
||||
"pairs": page_pairs,
|
||||
"total_pairs": total_pairs,
|
||||
"threshold": threshold,
|
||||
"page": page,
|
||||
"pages": total_pages,
|
||||
"per_page": limit,
|
||||
"embedding_coverage": {
|
||||
"total_files": total_files,
|
||||
"files_with_embedding": len(parsed),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _row_to_dict(row) -> dict:
|
||||
"""Serialise a column-only query row to a dict for JSON responses."""
|
||||
return {
|
||||
"file_id": row.id,
|
||||
"original_filename": row.original_filename,
|
||||
"document_title": row.document_title,
|
||||
"mime_type": row.mime_type,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
"""API endpoints for subscription tiers and usage statistics.
|
||||
|
||||
Public endpoints:
|
||||
GET /api/subscriptions/tiers — list all available plans
|
||||
GET /api/subscriptions/my — current user's plan + usage (auth required)
|
||||
POST /api/subscriptions/change — request a plan change (auth required)
|
||||
DELETE /api/subscriptions/change — cancel a pending plan change (auth required)
|
||||
GET /api/subscriptions/platform — platform-wide stats (admin only)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.admin_users import _require_admin
|
||||
from app.database import get_db
|
||||
from app.utils.subscription import (
|
||||
TIER_ORDER,
|
||||
TIERS,
|
||||
SubscriptionChangeError,
|
||||
apply_pending_subscription_changes,
|
||||
cancel_pending_subscription_change,
|
||||
get_all_tiers,
|
||||
get_tier,
|
||||
get_user_tier_id,
|
||||
get_user_usage,
|
||||
request_subscription_change,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/subscriptions", tags=["subscriptions"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SubscriptionChangeRequest(BaseModel):
|
||||
"""Request body for a subscription plan change."""
|
||||
|
||||
plan_id: str
|
||||
billing_cycle: str = "monthly" # "monthly" | "yearly"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Extract the authenticated user's owner_id from the session."""
|
||||
user = request.session.get("user") or {}
|
||||
return user.get("username") or user.get("email") or user.get("sub") or ""
|
||||
|
||||
|
||||
def _require_authenticated(request: Request) -> str:
|
||||
"""Return the owner_id or raise 401."""
|
||||
owner_id = _get_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
|
||||
return owner_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/tiers", summary="List all subscription tiers")
|
||||
def list_tiers() -> dict[str, Any]:
|
||||
"""Return the full list of subscription plans in display order."""
|
||||
return {
|
||||
"tiers": get_all_tiers(),
|
||||
"order": TIER_ORDER,
|
||||
"default": "free",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/my", summary="Get current user's subscription and usage")
|
||||
def my_subscription(request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Return the authenticated user's subscription tier and current usage counts.
|
||||
|
||||
Also applies any pending subscription changes that have become due.
|
||||
"""
|
||||
from app.config import settings
|
||||
from app.models import UserProfile
|
||||
|
||||
user = request.session.get("user")
|
||||
|
||||
if not settings.multi_user_enabled:
|
||||
# In single-user mode there is no concept of a subscription plan
|
||||
return {
|
||||
"multi_user_mode": False,
|
||||
"tier": TIERS["business"], # unrestricted
|
||||
"usage": None,
|
||||
}
|
||||
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
|
||||
|
||||
owner_id: str = user.get("username") or user.get("email") or user.get("sub") or ""
|
||||
|
||||
# Apply any pending change that has become due
|
||||
apply_pending_subscription_changes(db, owner_id)
|
||||
|
||||
tier_id = get_user_tier_id(db, owner_id)
|
||||
tier = get_tier(tier_id, db)
|
||||
usage = get_user_usage(db, owner_id)
|
||||
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first()
|
||||
pending_tier_id: str | None = profile.subscription_change_pending_tier if profile else None
|
||||
pending_date: str | None = (
|
||||
profile.subscription_change_pending_date.isoformat()
|
||||
if profile and profile.subscription_change_pending_date
|
||||
else None
|
||||
)
|
||||
period_start: str | None = (
|
||||
profile.subscription_period_start.isoformat() if profile and profile.subscription_period_start else None
|
||||
)
|
||||
|
||||
return {
|
||||
"multi_user_mode": True,
|
||||
"owner_id": owner_id,
|
||||
"tier": tier,
|
||||
"usage": usage,
|
||||
"period_start": period_start,
|
||||
"pending_change": (
|
||||
{
|
||||
"tier_id": pending_tier_id,
|
||||
"tier": get_tier(pending_tier_id, db),
|
||||
"effective_date": pending_date,
|
||||
}
|
||||
if pending_tier_id
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/change", summary="Request a subscription plan change", status_code=status.HTTP_200_OK)
|
||||
def change_subscription(request: Request, body: SubscriptionChangeRequest, db: DbSession) -> dict[str, Any]:
|
||||
"""Request a subscription tier change.
|
||||
|
||||
**Upgrades** (moving to a higher-ranked plan) take effect immediately.
|
||||
|
||||
**Downgrades** (moving to a lower-ranked plan) are scheduled for the end
|
||||
of the current billing period to prevent gaming. The user keeps their
|
||||
current plan benefits until the scheduled date.
|
||||
|
||||
Requesting the currently active tier while a downgrade is pending cancels
|
||||
that pending change.
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
if not settings.multi_user_enabled:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Subscription management is not available in single-user mode.",
|
||||
)
|
||||
|
||||
owner_id = _require_authenticated(request)
|
||||
|
||||
try:
|
||||
result = request_subscription_change(db, owner_id, body.plan_id, body.billing_cycle)
|
||||
except SubscriptionChangeError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/change", summary="Cancel a pending subscription change", status_code=status.HTTP_200_OK)
|
||||
def cancel_subscription_change(request: Request, db: DbSession) -> dict[str, Any]:
|
||||
"""Cancel a scheduled future subscription change.
|
||||
|
||||
Only downgrades can be pending; upgrades always take effect immediately.
|
||||
Returns 404 when there is no pending change to cancel.
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
if not settings.multi_user_enabled:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Subscription management is not available in single-user mode.",
|
||||
)
|
||||
|
||||
owner_id = _require_authenticated(request)
|
||||
|
||||
cancelled = cancel_pending_subscription_change(db, owner_id)
|
||||
if not cancelled:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No pending subscription change found.")
|
||||
|
||||
return {"cancelled": True, "message": "Your pending subscription change has been cancelled."}
|
||||
|
||||
|
||||
@router.get("/platform", summary="Platform-wide usage statistics (admin only)")
|
||||
def platform_stats(request: Request, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Return aggregate statistics across all users and tiers (admin only)."""
|
||||
from app.models import FileRecord, UserProfile
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
|
||||
# Total files
|
||||
total_files: int = db.query(func.count(FileRecord.id)).scalar() or 0
|
||||
|
||||
# Files today
|
||||
files_today: int = (
|
||||
db.query(func.count(FileRecord.id)).filter(func.date(FileRecord.created_at) == today).scalar() or 0
|
||||
)
|
||||
|
||||
# Files this month
|
||||
files_this_month: int = (
|
||||
db.query(func.count(FileRecord.id))
|
||||
.filter(func.strftime("%Y-%m", FileRecord.created_at) == today.strftime("%Y-%m"))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Files with OCR text (proxy for pages OCRed — approximation)
|
||||
files_with_ocr: int = db.query(func.count(FileRecord.id)).filter(FileRecord.ocr_text.isnot(None)).scalar() or 0
|
||||
|
||||
# Unique active users (ever uploaded)
|
||||
unique_users: int = (
|
||||
db.query(func.count(func.distinct(FileRecord.owner_id))).filter(FileRecord.owner_id.isnot(None)).scalar() or 0
|
||||
)
|
||||
|
||||
# Users per subscription tier
|
||||
profiles = (
|
||||
db.query(UserProfile.subscription_tier, func.count(UserProfile.id))
|
||||
.group_by(UserProfile.subscription_tier)
|
||||
.all()
|
||||
)
|
||||
tier_distribution: dict[str, int] = {row[0] or "free": row[1] for row in profiles}
|
||||
|
||||
# Fill in zeros for tiers with no users
|
||||
for tid in TIER_ORDER:
|
||||
tier_distribution.setdefault(tid, 0)
|
||||
|
||||
return {
|
||||
"files": {
|
||||
"total": total_files,
|
||||
"today": files_today,
|
||||
"this_month": files_this_month,
|
||||
"with_ocr": files_with_ocr,
|
||||
},
|
||||
"users": {
|
||||
"unique_uploaders": unique_users,
|
||||
"tier_distribution": tier_distribution,
|
||||
},
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
"""
|
||||
API endpoint for processing files from URLs
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, HttpUrl, field_validator
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_MIME_TYPES
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class URLUploadRequest(BaseModel):
|
||||
"""Request model for URL-based file upload"""
|
||||
|
||||
url: HttpUrl
|
||||
filename: Optional[str] = None
|
||||
|
||||
@field_validator("url")
|
||||
@classmethod
|
||||
def validate_url_scheme(cls, v):
|
||||
"""Ensure only HTTP/HTTPS schemes are allowed"""
|
||||
parsed = urllib.parse.urlparse(str(v))
|
||||
if parsed.scheme not in ["http", "https"]:
|
||||
raise ValueError("Only HTTP and HTTPS URLs are allowed")
|
||||
return v
|
||||
|
||||
|
||||
def is_private_ip(hostname: str) -> bool:
|
||||
"""
|
||||
Check if a hostname resolves to a private/internal IP address.
|
||||
Protects against SSRF attacks by blocking access to internal networks.
|
||||
"""
|
||||
try:
|
||||
# Try to parse as IP address directly
|
||||
ip = ipaddress.ip_address(hostname)
|
||||
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
|
||||
except ValueError:
|
||||
# Not a direct IP, try to resolve hostname
|
||||
try:
|
||||
import socket
|
||||
|
||||
# Get all IP addresses for this hostname
|
||||
addr_info = socket.getaddrinfo(hostname, None)
|
||||
for info in addr_info:
|
||||
ip_str = info[4][0]
|
||||
ip = ipaddress.ip_address(ip_str)
|
||||
# Block if ANY resolved IP is private/internal
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return True
|
||||
return False
|
||||
except (socket.gaierror, socket.error):
|
||||
# Cannot resolve - allow for testing/development
|
||||
# In production, DNS should work properly
|
||||
# Log this for debugging
|
||||
logger.warning(f"Could not resolve hostname: {hostname}")
|
||||
return False # Changed from True to False to allow external domains in tests
|
||||
|
||||
|
||||
def validate_url_safety(url: str) -> None:
|
||||
"""
|
||||
Validate that URL is safe to fetch (SSRF protection).
|
||||
|
||||
Raises:
|
||||
HTTPException: If URL is unsafe
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
|
||||
# Check scheme
|
||||
if parsed.scheme not in ["http", "https"]:
|
||||
raise HTTPException(status_code=400, detail="Only HTTP and HTTPS URLs are supported")
|
||||
|
||||
# Check hostname exists
|
||||
if not parsed.hostname:
|
||||
raise HTTPException(status_code=400, detail="Invalid URL: no hostname")
|
||||
|
||||
# Block private/internal IPs (SSRF protection)
|
||||
if is_private_ip(parsed.hostname):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Access to private/internal IP addresses is not allowed for security reasons",
|
||||
)
|
||||
|
||||
# Block well-known metadata endpoints (cloud provider SSRF)
|
||||
metadata_endpoints = [
|
||||
"169.254.169.254", # AWS, Azure, GCP metadata
|
||||
"metadata.google.internal", # GCP
|
||||
"169.254.169.253", # AWS link-local
|
||||
]
|
||||
if parsed.hostname in metadata_endpoints:
|
||||
raise HTTPException(status_code=400, detail="Access to cloud metadata endpoints is not allowed")
|
||||
|
||||
|
||||
def validate_file_type(content_type: str, filename: str) -> bool:
|
||||
"""
|
||||
Validate that the file type is supported (i.e. processable by Gotenberg).
|
||||
|
||||
Args:
|
||||
content_type: MIME type from response headers
|
||||
filename: Filename to check extension
|
||||
|
||||
Returns:
|
||||
True if file type is allowed
|
||||
"""
|
||||
# Check content type from header
|
||||
if content_type:
|
||||
# Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
|
||||
base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
|
||||
if base_content_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
# Also check by extension as fallback
|
||||
_, ext = os.path.splitext(filename)
|
||||
if ext:
|
||||
guessed_type, _ = mimetypes.guess_type(filename)
|
||||
if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@router.post("/process-url")
|
||||
@require_login
|
||||
async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
"""
|
||||
Download a file from a URL and enqueue it for processing.
|
||||
|
||||
Security features:
|
||||
- SSRF protection: blocks private IPs, localhost, cloud metadata endpoints
|
||||
- File type validation: only allows supported document/image types
|
||||
- File size limits: enforces maximum upload size
|
||||
- Timeout protection: prevents hanging on slow/malicious servers
|
||||
|
||||
Args:
|
||||
request: Starlette Request object (used by require_login decorator)
|
||||
url_request: URLUploadRequest with url and optional filename
|
||||
|
||||
Returns:
|
||||
JSON with task_id and status
|
||||
|
||||
Raises:
|
||||
HTTPException: If URL is invalid, unsafe, or file cannot be processed
|
||||
"""
|
||||
url = str(url_request.url)
|
||||
|
||||
# Validate URL safety (SSRF protection)
|
||||
validate_url_safety(url)
|
||||
|
||||
# Parse URL to extract filename if not provided
|
||||
if url_request.filename:
|
||||
original_filename = url_request.filename
|
||||
else:
|
||||
# Extract filename from URL path
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
path = parsed.path
|
||||
original_filename = os.path.basename(path) if path else "download"
|
||||
|
||||
# Sanitize filename
|
||||
safe_filename = sanitize_filename(original_filename)
|
||||
if not safe_filename:
|
||||
safe_filename = "download"
|
||||
|
||||
# Download file with security measures
|
||||
# Initialize target_path to None to prevent UnboundLocalError in exception handlers
|
||||
# that may execute before target_path is assigned during error cases
|
||||
target_path = None
|
||||
try:
|
||||
logger.info(f"Downloading file from URL: {url}")
|
||||
|
||||
# Use configured timeout to prevent hanging
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=settings.http_request_timeout,
|
||||
stream=True, # Stream to handle large files
|
||||
allow_redirects=True, # Follow redirects
|
||||
headers={
|
||||
"User-Agent": "DocuElevate/1.0", # Identify ourselves
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
if "." in safe_filename:
|
||||
file_extension = safe_filename.rsplit(".", 1)[1]
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
with open(target_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
)
|
||||
|
||||
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
|
||||
|
||||
# Enqueue for processing
|
||||
task = process_document.delay(target_path, original_filename=safe_filename)
|
||||
|
||||
return {
|
||||
"task_id": task.id,
|
||||
"status": "queued",
|
||||
"message": "File downloaded from URL and queued for processing",
|
||||
"filename": safe_filename,
|
||||
"size": downloaded_size,
|
||||
}
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error(f"Timeout while downloading file from URL: {url}")
|
||||
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise FastAPI HTTPExceptions (validation errors, file too large, etc.)
|
||||
raise
|
||||
|
||||
except OSError as e:
|
||||
logger.error(f"Error saving file from URL: {url} - {str(e)}")
|
||||
# Clean up partial file if it exists
|
||||
if target_path and os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error processing URL: {url}")
|
||||
# Clean up partial file if it exists
|
||||
if target_path and os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
|
||||
+11
-66
@@ -1,28 +1,16 @@
|
||||
"""
|
||||
User-related API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
from fastapi import APIRouter, Request, HTTPException
|
||||
from hashlib import md5
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, UserProfile
|
||||
import logging
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
async def whoami_handler(request: Request, db: Session):
|
||||
async def whoami_handler(request: Request):
|
||||
"""
|
||||
Returns user info if logged in, else 401.
|
||||
"""
|
||||
@@ -35,63 +23,20 @@ async def whoami_handler(request: Request, db: Session):
|
||||
raise HTTPException(status_code=400, detail="User has no email in session")
|
||||
|
||||
# Generate Gravatar URL from email
|
||||
# MD5 is used here for Gravatar's URL generation (not for security), so usedforsecurity=False
|
||||
email_hash = md5(email.strip().lower().encode(), usedforsecurity=False).hexdigest()
|
||||
email_hash = md5(email.strip().lower().encode()).hexdigest()
|
||||
gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
|
||||
|
||||
|
||||
# Add the gravatar URL to the user object instead of creating a new response
|
||||
user_response = user.copy() # Create a copy to avoid modifying the session
|
||||
|
||||
# Check if the user has a custom avatar stored in their profile
|
||||
user_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
if user_id:
|
||||
try:
|
||||
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if profile and profile.avatar_data:
|
||||
user_response["picture"] = profile.avatar_data
|
||||
else:
|
||||
user_response["picture"] = gravatar_url
|
||||
except Exception:
|
||||
user_response["picture"] = gravatar_url
|
||||
else:
|
||||
user_response["picture"] = gravatar_url
|
||||
|
||||
user_response["picture"] = gravatar_url
|
||||
|
||||
return user_response
|
||||
|
||||
|
||||
# Register the same handler under two different paths
|
||||
@router.get("/whoami")
|
||||
async def whoami(request: Request, db: DbSession):
|
||||
return await whoami_handler(request, db)
|
||||
|
||||
async def whoami(request: Request):
|
||||
return await whoami_handler(request)
|
||||
|
||||
@router.get("/auth/whoami")
|
||||
async def auth_whoami(request: Request, db: DbSession):
|
||||
return await whoami_handler(request, db)
|
||||
|
||||
|
||||
@router.get("/users/search")
|
||||
@require_login
|
||||
def search_known_users(
|
||||
db: DbSession,
|
||||
q: str = Query("", description="Substring to match against known owner IDs"),
|
||||
limit: int = Query(5, ge=1, le=20, description="Maximum number of results"),
|
||||
):
|
||||
"""
|
||||
Search known user identifiers (owner_ids) from existing documents.
|
||||
|
||||
Returns distinct ``owner_id`` values from the files table that contain
|
||||
the query string as a case-insensitive substring. Results are limited
|
||||
to at most ``limit`` entries (default 5).
|
||||
|
||||
This powers the autocomplete widget on the settings page for the
|
||||
``default_owner_id`` field.
|
||||
"""
|
||||
base_query = db.query(FileRecord.owner_id).filter(FileRecord.owner_id.isnot(None)).distinct()
|
||||
|
||||
if q.strip():
|
||||
base_query = base_query.filter(func.lower(FileRecord.owner_id).contains(q.strip().lower()))
|
||||
|
||||
results = base_query.order_by(FileRecord.owner_id).limit(limit).all()
|
||||
|
||||
return {"users": [row[0] for row in results]}
|
||||
async def auth_whoami(request: Request):
|
||||
return await whoami_handler(request)
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
"""API endpoints for managing webhook configurations.
|
||||
|
||||
Provides CRUD operations for webhook configs that notify external systems
|
||||
when document events occur (``document.uploaded``, ``document.processed``,
|
||||
``document.failed``).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import WebhookConfig
|
||||
from app.utils.webhook import VALID_EVENTS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/webhooks", tags=["webhooks"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper (reuse the pattern from settings API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class WebhookCreate(BaseModel):
|
||||
"""Schema for creating a new webhook configuration."""
|
||||
|
||||
url: str = Field(..., min_length=1, max_length=2048, description="Target URL for webhook delivery")
|
||||
secret: str | None = Field(default=None, max_length=512, description="Shared secret for HMAC-SHA256 signatures")
|
||||
events: list[str] = Field(..., min_length=1, description="List of events to subscribe to")
|
||||
is_active: bool = Field(default=True, description="Whether the webhook is active")
|
||||
description: str | None = Field(default=None, max_length=500, description="Optional human-readable description")
|
||||
|
||||
|
||||
class WebhookUpdate(BaseModel):
|
||||
"""Schema for updating an existing webhook configuration."""
|
||||
|
||||
url: str | None = Field(default=None, min_length=1, max_length=2048)
|
||||
secret: str | None = Field(default=None, max_length=512)
|
||||
events: list[str] | None = Field(default=None, min_length=1)
|
||||
is_active: bool | None = None
|
||||
description: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class WebhookResponse(BaseModel):
|
||||
"""Schema returned to clients (secret is never exposed)."""
|
||||
|
||||
id: int
|
||||
url: str
|
||||
events: list[str]
|
||||
is_active: bool
|
||||
description: str | None
|
||||
has_secret: bool
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_events(events: list[str]) -> None:
|
||||
"""Raise 422 if any event name is not recognised."""
|
||||
invalid = set(events) - VALID_EVENTS
|
||||
if invalid:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Invalid event(s): {', '.join(sorted(invalid))}. Valid events: {', '.join(sorted(VALID_EVENTS))}",
|
||||
)
|
||||
|
||||
|
||||
def _to_response(cfg: WebhookConfig) -> dict[str, Any]:
|
||||
"""Convert a DB model instance to a response dict."""
|
||||
try:
|
||||
events = json.loads(cfg.events)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
events = []
|
||||
return {
|
||||
"id": cfg.id,
|
||||
"url": cfg.url,
|
||||
"events": events,
|
||||
"is_active": cfg.is_active,
|
||||
"description": cfg.description,
|
||||
"has_secret": cfg.secret is not None and len(cfg.secret) > 0,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", summary="List all webhook configurations")
|
||||
def list_webhooks(db: DbSession, _admin: AdminUser) -> list[dict[str, Any]]:
|
||||
"""Return all webhook configurations. Secrets are never included."""
|
||||
configs = db.query(WebhookConfig).order_by(WebhookConfig.id).all()
|
||||
return [_to_response(c) for c in configs]
|
||||
|
||||
|
||||
@router.get("/{webhook_id}", summary="Get a single webhook configuration")
|
||||
def get_webhook(webhook_id: int, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Return a single webhook configuration by ID."""
|
||||
cfg = db.query(WebhookConfig).filter(WebhookConfig.id == webhook_id).first()
|
||||
if not cfg:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Webhook not found")
|
||||
return _to_response(cfg)
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a webhook configuration")
|
||||
def create_webhook(body: WebhookCreate, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Create a new webhook configuration."""
|
||||
_validate_events(body.events)
|
||||
|
||||
cfg = WebhookConfig(
|
||||
url=body.url,
|
||||
secret=body.secret,
|
||||
events=json.dumps(sorted(body.events)),
|
||||
is_active=body.is_active,
|
||||
description=body.description,
|
||||
)
|
||||
try:
|
||||
db.add(cfg)
|
||||
db.commit()
|
||||
db.refresh(cfg)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Webhook %d created for events %s", cfg.id, body.events)
|
||||
return _to_response(cfg)
|
||||
|
||||
|
||||
@router.put("/{webhook_id}", summary="Update a webhook configuration")
|
||||
def update_webhook(webhook_id: int, body: WebhookUpdate, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Update an existing webhook configuration. Only supplied fields are changed."""
|
||||
cfg = db.query(WebhookConfig).filter(WebhookConfig.id == webhook_id).first()
|
||||
if not cfg:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Webhook not found")
|
||||
|
||||
if body.url is not None:
|
||||
cfg.url = body.url
|
||||
if body.secret is not None:
|
||||
cfg.secret = body.secret
|
||||
if body.events is not None:
|
||||
_validate_events(body.events)
|
||||
cfg.events = json.dumps(sorted(body.events))
|
||||
if body.is_active is not None:
|
||||
cfg.is_active = body.is_active
|
||||
if body.description is not None:
|
||||
cfg.description = body.description
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(cfg)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Webhook %d updated", cfg.id)
|
||||
return _to_response(cfg)
|
||||
|
||||
|
||||
@router.delete("/{webhook_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete a webhook configuration")
|
||||
def delete_webhook(webhook_id: int, db: DbSession, _admin: AdminUser) -> None:
|
||||
"""Delete a webhook configuration."""
|
||||
cfg = db.query(WebhookConfig).filter(WebhookConfig.id == webhook_id).first()
|
||||
if not cfg:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Webhook not found")
|
||||
|
||||
try:
|
||||
db.delete(cfg)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("Webhook %d deleted", cfg.id)
|
||||
|
||||
|
||||
@router.get("/events/", summary="List valid webhook event types")
|
||||
def list_events(_admin: AdminUser) -> list[str]:
|
||||
"""Return the list of valid event types that can be subscribed to."""
|
||||
return sorted(VALID_EVENTS)
|
||||
+101
-724
@@ -1,32 +1,15 @@
|
||||
import hashlib
|
||||
import os
|
||||
import inspect
|
||||
import logging
|
||||
import pathlib
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from functools import wraps
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from authlib.integrations.starlette_client import OAuth
|
||||
from fastapi import APIRouter, Depends, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from sqlalchemy.orm import Session
|
||||
from fastapi import APIRouter, Request, status
|
||||
from starlette.responses import RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
import pathlib
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.middleware.audit_log import get_client_ip
|
||||
|
||||
# Conditional imports: only used when multi_user_enabled=True. Imported here at
|
||||
# module level (not inside auth()) so they don't incur repeated import overhead.
|
||||
# Guards at call-sites ensure they are never *called* in single-user mode.
|
||||
from app.models import LocalUser as _LocalUser
|
||||
from app.models import UserProfile as _UserProfile
|
||||
from app.utils.i18n import translate as _translate
|
||||
from app.utils.local_auth import build_session_user as _build_session_user
|
||||
from app.utils.local_auth import verify_password as _verify_password
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
oauth = OAuth()
|
||||
|
||||
@@ -35,15 +18,11 @@ AUTH_ENABLED = settings.auth_enabled
|
||||
# Set up templates for authentication
|
||||
templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
|
||||
templates = Jinja2Templates(directory=str(templates_dir))
|
||||
templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
|
||||
|
||||
# Configure OAuth provider if credentials are provided
|
||||
OAUTH_CONFIGURED = False
|
||||
OAUTH_PROVIDER_NAME = "Single Sign-On"
|
||||
|
||||
# Social login providers that are enabled and registered
|
||||
SOCIAL_PROVIDERS: dict[str, dict[str, str]] = {}
|
||||
|
||||
if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_secret:
|
||||
oauth.register(
|
||||
name="authentik",
|
||||
@@ -55,191 +34,27 @@ if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_s
|
||||
OAUTH_CONFIGURED = True
|
||||
OAUTH_PROVIDER_NAME = settings.oauth_provider_name or "Authentik SSO"
|
||||
|
||||
# --- Social Login Providers ---------------------------------------------------
|
||||
if AUTH_ENABLED and settings.social_auth_google_enabled:
|
||||
if settings.social_auth_google_client_id and settings.social_auth_google_client_secret:
|
||||
oauth.register(
|
||||
name="google",
|
||||
client_id=settings.social_auth_google_client_id,
|
||||
client_secret=settings.social_auth_google_client_secret,
|
||||
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
|
||||
client_kwargs={"scope": "openid profile email"},
|
||||
)
|
||||
SOCIAL_PROVIDERS["google"] = {"name": "Google", "icon": "fab fa-google", "color": "red"}
|
||||
logger.info("Social login provider registered: Google")
|
||||
else:
|
||||
logger.warning("SOCIAL_AUTH_GOOGLE_ENABLED=true but client ID/secret not configured")
|
||||
|
||||
if AUTH_ENABLED and settings.social_auth_microsoft_enabled:
|
||||
if settings.social_auth_microsoft_client_id and settings.social_auth_microsoft_client_secret:
|
||||
tenant = settings.social_auth_microsoft_tenant or "common"
|
||||
oauth.register(
|
||||
name="microsoft",
|
||||
client_id=settings.social_auth_microsoft_client_id,
|
||||
client_secret=settings.social_auth_microsoft_client_secret,
|
||||
server_metadata_url=f"https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration",
|
||||
client_kwargs={"scope": "openid profile email"},
|
||||
)
|
||||
SOCIAL_PROVIDERS["microsoft"] = {"name": "Microsoft", "icon": "fab fa-microsoft", "color": "blue"}
|
||||
logger.info("Social login provider registered: Microsoft (tenant=%s)", tenant)
|
||||
else:
|
||||
logger.warning("SOCIAL_AUTH_MICROSOFT_ENABLED=true but client ID/secret not configured")
|
||||
|
||||
if AUTH_ENABLED and settings.social_auth_apple_enabled:
|
||||
if settings.social_auth_apple_client_id and settings.social_auth_apple_team_id:
|
||||
oauth.register(
|
||||
name="apple",
|
||||
client_id=settings.social_auth_apple_client_id,
|
||||
server_metadata_url="https://appleid.apple.com/.well-known/openid-configuration",
|
||||
client_kwargs={
|
||||
"scope": "openid name email",
|
||||
"response_mode": "form_post",
|
||||
},
|
||||
)
|
||||
SOCIAL_PROVIDERS["apple"] = {"name": "Apple", "icon": "fab fa-apple", "color": "gray"}
|
||||
logger.info("Social login provider registered: Apple")
|
||||
else:
|
||||
logger.warning("SOCIAL_AUTH_APPLE_ENABLED=true but client ID/team ID not configured")
|
||||
|
||||
if AUTH_ENABLED and settings.social_auth_dropbox_enabled:
|
||||
if settings.social_auth_dropbox_client_id and settings.social_auth_dropbox_client_secret:
|
||||
oauth.register(
|
||||
name="dropbox",
|
||||
client_id=settings.social_auth_dropbox_client_id,
|
||||
client_secret=settings.social_auth_dropbox_client_secret,
|
||||
authorize_url="https://www.dropbox.com/oauth2/authorize",
|
||||
access_token_url="https://api.dropboxapi.com/oauth2/token",
|
||||
userinfo_endpoint="https://api.dropboxapi.com/2/users/get_current_account",
|
||||
client_kwargs={"token_endpoint_auth_method": "client_secret_post"},
|
||||
)
|
||||
SOCIAL_PROVIDERS["dropbox"] = {"name": "Dropbox", "icon": "fab fa-dropbox", "color": "blue"}
|
||||
logger.info("Social login provider registered: Dropbox")
|
||||
else:
|
||||
logger.warning("SOCIAL_AUTH_DROPBOX_ENABLED=true but client ID/secret not configured")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def get_current_user(request: Request):
|
||||
# Check for Bearer token auth first (API tokens)
|
||||
api_user = getattr(request.state, "api_token_user", None)
|
||||
if isinstance(api_user, dict):
|
||||
return api_user
|
||||
return request.session.get("user")
|
||||
|
||||
|
||||
def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
|
||||
"""Resolve a user from a Bearer API token in the Authorization header.
|
||||
|
||||
If the header is present and the token is valid, updates usage tracking
|
||||
(last_used_at, last_used_ip) and returns a synthetic user dict compatible
|
||||
with the session user format.
|
||||
|
||||
Returns:
|
||||
A user dict or ``None`` if no valid Bearer token is present.
|
||||
"""
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if not isinstance(auth_header, str) or not auth_header.startswith("Bearer "):
|
||||
return None
|
||||
|
||||
raw_token = auth_header[7:]
|
||||
if not raw_token or not isinstance(raw_token, str):
|
||||
return None
|
||||
|
||||
from app.api.api_tokens import hash_token
|
||||
from app.models import ApiToken
|
||||
|
||||
token_hash = hash_token(raw_token)
|
||||
db_token = db.query(ApiToken).filter(ApiToken.token_hash == token_hash, ApiToken.is_active.is_(True)).first()
|
||||
if db_token is None:
|
||||
return None
|
||||
|
||||
# Update usage tracking
|
||||
try:
|
||||
db_token.last_used_at = datetime.now(timezone.utc)
|
||||
# Extract client IP (respect X-Forwarded-For from reverse proxy)
|
||||
client_ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip()
|
||||
if not client_ip and request.client:
|
||||
client_ip = request.client.host
|
||||
db_token.last_used_ip = client_ip or None
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.debug("Failed to update API token usage tracking for token_id=%s", db_token.id)
|
||||
|
||||
# Build a synthetic user dict that mimics the session user format
|
||||
return {
|
||||
"id": db_token.owner_id,
|
||||
"email": db_token.owner_id,
|
||||
"preferred_username": db_token.owner_id,
|
||||
"is_admin": False,
|
||||
"_api_token_id": db_token.id,
|
||||
}
|
||||
|
||||
|
||||
def get_current_user_id(request: Request) -> str:
|
||||
"""Return a stable string identifier for the authenticated user.
|
||||
|
||||
Falls back to ``"anonymous"`` when no user is in the session (e.g. when
|
||||
AUTH_ENABLED=False in single-user mode). The returned value is consistent
|
||||
with how pipeline and file ownership is stored in the database.
|
||||
|
||||
Priority order: ``preferred_username`` → ``email`` → ``id`` → ``"anonymous"``.
|
||||
|
||||
Args:
|
||||
request: The current FastAPI request.
|
||||
|
||||
Returns:
|
||||
A non-empty string identifying the current user.
|
||||
"""
|
||||
user = get_current_user(request)
|
||||
if not user or not isinstance(user, dict):
|
||||
return "anonymous"
|
||||
return user.get("preferred_username") or user.get("email") or user.get("id") or "anonymous"
|
||||
|
||||
|
||||
def require_login(func):
|
||||
if not AUTH_ENABLED:
|
||||
return func # no-op
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(request: Request, *args, **kwargs):
|
||||
# Check session auth first
|
||||
if request.session.get("user"):
|
||||
if inspect.iscoroutinefunction(func):
|
||||
return await func(*args, request=request, **kwargs)
|
||||
else:
|
||||
return func(*args, request=request, **kwargs)
|
||||
|
||||
# Fall back to Bearer token auth for API endpoints
|
||||
url_path = urlparse(str(request.url)).path
|
||||
if url_path.startswith("/api/"):
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
api_user = _resolve_bearer_user(request, db)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception:
|
||||
api_user = None
|
||||
|
||||
if api_user:
|
||||
request.state.api_token_user = api_user
|
||||
if inspect.iscoroutinefunction(func):
|
||||
return await func(*args, request=request, **kwargs)
|
||||
else:
|
||||
return func(*args, request=request, **kwargs)
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
content={"error": "Not authenticated"},
|
||||
)
|
||||
|
||||
# Non-API endpoint with no session — redirect to login
|
||||
request.session["redirect_after_login"] = str(request.url)
|
||||
return RedirectResponse(url="/login", status_code=status.HTTP_302_FOUND)
|
||||
if not request.session.get("user"):
|
||||
request.session["redirect_after_login"] = str(request.url)
|
||||
return RedirectResponse(url="/login", status_code=status.HTTP_302_FOUND)
|
||||
# Check if the wrapped function is a coroutine function
|
||||
if inspect.iscoroutinefunction(func):
|
||||
return await func(request, *args, **kwargs)
|
||||
else:
|
||||
return func(request, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
@@ -247,552 +62,114 @@ def require_login(func):
|
||||
def get_gravatar_url(email):
|
||||
"""Generate a Gravatar URL for the given email"""
|
||||
email = email.lower().strip()
|
||||
# MD5 is used here for Gravatar's URL generation (not for security), so usedforsecurity=False
|
||||
email_hash = hashlib.md5(email.encode("utf-8"), usedforsecurity=False).hexdigest()
|
||||
email_hash = hashlib.md5(email.encode('utf-8')).hexdigest()
|
||||
return f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
|
||||
|
||||
|
||||
async def login(request: Request):
|
||||
"""Show login page with appropriate authentication options."""
|
||||
return templates.TemplateResponse(
|
||||
"login.html",
|
||||
{
|
||||
"request": request,
|
||||
"error": request.query_params.get("error"),
|
||||
"message": request.query_params.get("message"),
|
||||
"show_oauth": OAUTH_CONFIGURED,
|
||||
"oauth_provider_name": OAUTH_PROVIDER_NAME,
|
||||
"social_providers": SOCIAL_PROVIDERS,
|
||||
"app_version": settings.version,
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
# "Create account" link is only shown when multi-user mode AND local signup are both enabled
|
||||
"allow_signup": settings.multi_user_enabled and settings.allow_local_signup,
|
||||
},
|
||||
)
|
||||
if AUTH_ENABLED:
|
||||
@router.get("/login")
|
||||
async def login(request: Request):
|
||||
"""Show login page with appropriate authentication options"""
|
||||
return templates.TemplateResponse(
|
||||
"login.html",
|
||||
{
|
||||
"request": request,
|
||||
"error": request.query_params.get("error"),
|
||||
"message": request.query_params.get("message"),
|
||||
"show_oauth": OAUTH_CONFIGURED,
|
||||
"oauth_provider_name": OAUTH_PROVIDER_NAME,
|
||||
"app_version": settings.version # Changed from app_version to version
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def oauth_login(request: Request):
|
||||
"""Handle OAuth login flow"""
|
||||
if not OAUTH_CONFIGURED:
|
||||
return RedirectResponse(url="/login?error=OAuth+not+configured", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
redirect_uri = request.url_for("oauth_callback")
|
||||
return await oauth.authentik.authorize_redirect(request, redirect_uri)
|
||||
|
||||
|
||||
async def social_login(request: Request, provider: str):
|
||||
"""Initiate a social login flow for the given provider.
|
||||
|
||||
Args:
|
||||
request: The current FastAPI request.
|
||||
provider: One of the registered social provider keys (google, microsoft, apple, dropbox).
|
||||
|
||||
Returns:
|
||||
A redirect to the provider's authorization page, or back to /login on error.
|
||||
"""
|
||||
if provider not in SOCIAL_PROVIDERS:
|
||||
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
redirect_uri = request.url_for("social_callback", provider=provider)
|
||||
oauth_client = getattr(oauth, provider, None)
|
||||
if oauth_client is None:
|
||||
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
return await oauth_client.authorize_redirect(request, redirect_uri)
|
||||
|
||||
|
||||
def _normalize_social_userinfo(provider: str, token: dict, raw_userinfo: dict | None) -> dict:
|
||||
"""Normalize the userinfo payload from different social providers into a common format.
|
||||
|
||||
Returns a dict with keys: sub, email, name, preferred_username, picture.
|
||||
|
||||
Args:
|
||||
provider: The social provider key (google, microsoft, apple, dropbox).
|
||||
token: The OAuth token response from the provider. Included for future
|
||||
provider-specific claim extraction (e.g. ``id_token`` claims).
|
||||
raw_userinfo: The raw userinfo dict (may be None for providers without standard OIDC userinfo).
|
||||
|
||||
Returns:
|
||||
A normalized user-data dict compatible with the session user format.
|
||||
"""
|
||||
userinfo: dict = raw_userinfo or {}
|
||||
|
||||
if provider == "dropbox":
|
||||
# Dropbox returns a non-standard userinfo response
|
||||
email = userinfo.get("email", "")
|
||||
name_info = userinfo.get("name", {})
|
||||
display_name = name_info.get("display_name", "") if isinstance(name_info, dict) else str(name_info)
|
||||
return {
|
||||
"sub": userinfo.get("account_id", email),
|
||||
"email": email,
|
||||
"name": display_name,
|
||||
"preferred_username": email,
|
||||
"picture": userinfo.get("profile_photo_url", ""),
|
||||
}
|
||||
|
||||
# Standard OIDC providers (Google, Microsoft, Apple)
|
||||
return {
|
||||
"sub": userinfo.get("sub", ""),
|
||||
"email": userinfo.get("email", ""),
|
||||
"name": userinfo.get("name", ""),
|
||||
"preferred_username": userinfo.get("email", ""),
|
||||
"picture": userinfo.get("picture", ""),
|
||||
}
|
||||
|
||||
|
||||
async def social_callback(request: Request, provider: str, db: Session = Depends(get_db)):
|
||||
"""Handle the OAuth callback from a social login provider.
|
||||
|
||||
After the user authorizes with the social provider, this endpoint exchanges
|
||||
the authorization code for tokens, extracts user information, creates or
|
||||
updates the user profile, and establishes a session.
|
||||
|
||||
Args:
|
||||
request: The current FastAPI request.
|
||||
provider: One of the registered social provider keys.
|
||||
db: Database session (injected).
|
||||
|
||||
Returns:
|
||||
A redirect to the user's original destination or the upload page.
|
||||
"""
|
||||
if provider not in SOCIAL_PROVIDERS:
|
||||
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
oauth_client = getattr(oauth, provider, None)
|
||||
if oauth_client is None:
|
||||
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
try:
|
||||
token = await oauth_client.authorize_access_token(request)
|
||||
|
||||
# Try standard OIDC userinfo first, fall back to token-embedded userinfo
|
||||
raw_userinfo = token.get("userinfo")
|
||||
if not raw_userinfo:
|
||||
try:
|
||||
resp = await oauth_client.userinfo(token=token)
|
||||
raw_userinfo = resp if isinstance(resp, dict) else resp.json() if hasattr(resp, "json") else {}
|
||||
except Exception:
|
||||
raw_userinfo = {}
|
||||
|
||||
user_data = _normalize_social_userinfo(provider, token, raw_userinfo)
|
||||
|
||||
if not user_data.get("email"):
|
||||
@router.get("/oauth-login")
|
||||
async def oauth_login(request: Request):
|
||||
"""Handle OAuth login flow"""
|
||||
if not OAUTH_CONFIGURED:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Could+not+retrieve+email+from+provider",
|
||||
status_code=status.HTTP_302_FOUND,
|
||||
url="/login?error=OAuth+not+configured",
|
||||
status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
|
||||
redirect_uri = request.url_for("oauth_callback")
|
||||
return await oauth.authentik.authorize_redirect(request, redirect_uri)
|
||||
|
||||
# Add Gravatar if no picture provided
|
||||
if not user_data.get("picture") and user_data.get("email"):
|
||||
user_data["picture"] = get_gravatar_url(user_data["email"])
|
||||
|
||||
# Tag the login source for audit/debugging
|
||||
user_data["auth_provider"] = provider
|
||||
|
||||
# Social login users are never admin by default (admin must be granted
|
||||
# via the Authentik/OIDC admin group or manually in the admin panel)
|
||||
user_data["is_admin"] = False
|
||||
|
||||
request.session["user"] = user_data
|
||||
|
||||
# Auto-create or update UserProfile
|
||||
_ensure_user_profile(db, user_data, is_admin=False)
|
||||
|
||||
provider_name = SOCIAL_PROVIDERS[provider]["name"]
|
||||
logger.info(
|
||||
"[SECURITY] SOCIAL_LOGIN_SUCCESS provider=%s user=%s", provider_name, user_data.get("email", "unknown")
|
||||
)
|
||||
|
||||
# Redirect first-time users to onboarding
|
||||
user_id = (
|
||||
user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id")
|
||||
)
|
||||
if user_id:
|
||||
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
|
||||
if profile and not profile.onboarding_completed:
|
||||
post_onboarding = request.session.pop("redirect_after_login", "/upload")
|
||||
request.session["post_onboarding_redirect"] = post_onboarding
|
||||
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
|
||||
except Exception as e:
|
||||
logger.warning("[SECURITY] SOCIAL_LOGIN_FAILURE provider=%s error=%s", provider, type(e).__name__)
|
||||
return RedirectResponse(
|
||||
url="/login?error=Social+login+failed.+Please+try+again.", status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
|
||||
|
||||
def _ensure_user_profile(db: Session, user_data: dict, is_admin: bool = False) -> None:
|
||||
"""Create or update a UserProfile row for *user_data*.
|
||||
|
||||
Uses the same identifier priority as ``get_current_owner_id`` (sub →
|
||||
preferred_username → email → id) so that the profile's ``user_id`` matches
|
||||
``FileRecord.owner_id`` for every document the user uploads.
|
||||
|
||||
For regular users, an existing profile is left unchanged so that
|
||||
admin-managed settings (tier, limits, etc.) are preserved across logins.
|
||||
|
||||
For admin users (*is_admin=True*) the following rules apply:
|
||||
- If no profile exists: one is created with the highest subscription tier,
|
||||
``is_complimentary=True``, and ``onboarding_completed=True`` so that
|
||||
admins skip the first-time setup wizard.
|
||||
- If a profile already exists: ``is_complimentary`` is set to ``True``
|
||||
and, when the current tier is ``"free"``, the tier is upgraded to the
|
||||
highest available plan. Other admin-managed settings are left intact.
|
||||
|
||||
Args:
|
||||
db: Active database session.
|
||||
user_data: Mapping of user attributes as returned by the OAuth provider
|
||||
or built by :func:`app.utils.local_auth.build_session_user`.
|
||||
is_admin: When ``True``, apply admin-specific defaults on first login
|
||||
and ensure the complimentary flag is always set.
|
||||
"""
|
||||
from app.models import UserProfile
|
||||
from app.utils.subscription import TIER_ORDER
|
||||
|
||||
highest_tier = TIER_ORDER[-1]
|
||||
|
||||
user_id = (
|
||||
user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id")
|
||||
)
|
||||
if not user_id:
|
||||
logger.warning("Cannot create UserProfile: no stable user identifier in OAuth userinfo")
|
||||
return
|
||||
|
||||
try:
|
||||
existing = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
|
||||
if existing is None:
|
||||
display_name = user_data.get("name") or user_data.get("preferred_username") or user_data.get("email")
|
||||
email = user_data.get("email")
|
||||
profile = UserProfile(
|
||||
user_id=user_id,
|
||||
display_name=display_name,
|
||||
subscription_tier=highest_tier if is_admin else "free",
|
||||
is_complimentary=is_admin,
|
||||
onboarding_completed=is_admin,
|
||||
)
|
||||
db.add(profile)
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Auto-created UserProfile for user_id=%s (admin=%s, tier=%s)",
|
||||
user_id,
|
||||
is_admin,
|
||||
highest_tier if is_admin else "free",
|
||||
)
|
||||
# Notify admins and fire webhook for new (non-admin) user signup
|
||||
if not is_admin:
|
||||
try:
|
||||
from app.utils.notification import notify_user_signup
|
||||
from app.utils.webhook import dispatch_webhook_event
|
||||
|
||||
notify_user_signup(user_id, display_name=display_name, email=email)
|
||||
dispatch_webhook_event(
|
||||
"user.signup",
|
||||
{
|
||||
"user_id": user_id,
|
||||
"display_name": display_name,
|
||||
"email": email,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to send signup notification/webhook for user_id=%s", user_id)
|
||||
elif is_admin:
|
||||
# Ensure existing admin profiles always have complimentary flag set.
|
||||
# Also upgrade from free tier to highest if still on default.
|
||||
changed = False
|
||||
if not existing.is_complimentary:
|
||||
existing.is_complimentary = True
|
||||
changed = True
|
||||
if (existing.subscription_tier or "free") == "free":
|
||||
existing.subscription_tier = highest_tier
|
||||
changed = True
|
||||
if changed:
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Updated admin UserProfile for user_id=%s (complimentary=True, tier=%s)",
|
||||
user_id,
|
||||
existing.subscription_tier,
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to auto-create/update UserProfile for user_id=%s", user_id)
|
||||
|
||||
|
||||
async def oauth_callback(request: Request, db: Session = Depends(get_db)):
|
||||
"""Handle OAuth callback from provider"""
|
||||
try:
|
||||
token = await oauth.authentik.authorize_access_token(request)
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Failed+to+retrieve+user+information", status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
|
||||
# Store user info in session
|
||||
user_data = dict(userinfo)
|
||||
|
||||
# Add Gravatar picture if no picture is provided
|
||||
if not user_data.get("picture") and user_data.get("email"):
|
||||
user_data["picture"] = get_gravatar_url(user_data["email"])
|
||||
|
||||
# Check if user is admin based on OAuth groups or specific email
|
||||
# You can customize this logic based on your OAuth provider's attributes
|
||||
# For example, check if user has an "admin" group or specific email domain
|
||||
is_admin = False
|
||||
if "groups" in user_data:
|
||||
# Check if user is in admin group
|
||||
groups = user_data.get("groups", [])
|
||||
admin_group = (settings.admin_group_name or "admin").strip().lower()
|
||||
is_admin = admin_group in [group.lower() for group in groups]
|
||||
|
||||
# Set is_admin flag (defaults to False for OAuth users unless they're in admin group)
|
||||
user_data["is_admin"] = is_admin
|
||||
|
||||
request.session["user"] = user_data
|
||||
|
||||
# Auto-create or update UserProfile so the user appears in admin user management
|
||||
_ensure_user_profile(db, user_data, is_admin=is_admin)
|
||||
|
||||
# Log the successful authentication
|
||||
logger.info("[SECURITY] OAUTH_LOGIN_SUCCESS user=%s admin=%s", user_data.get("email", "unknown"), is_admin)
|
||||
_record_login_event(
|
||||
db,
|
||||
request,
|
||||
user_data.get("email") or user_data.get("preferred_username") or "unknown",
|
||||
success=True,
|
||||
method="oauth",
|
||||
)
|
||||
|
||||
# Redirect first-time users to onboarding
|
||||
user_id = (
|
||||
user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id")
|
||||
)
|
||||
if user_id:
|
||||
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
|
||||
if profile and not profile.onboarding_completed:
|
||||
post_onboarding = request.session.pop("redirect_after_login", "/upload")
|
||||
request.session["post_onboarding_redirect"] = post_onboarding
|
||||
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
# Redirect to original destination or default
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
|
||||
except Exception as e:
|
||||
logger.warning(f"[SECURITY] OAUTH_LOGIN_FAILURE error={type(e).__name__}")
|
||||
return RedirectResponse(url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
|
||||
def _record_login_event(
|
||||
db: Session,
|
||||
request: Request,
|
||||
username: str,
|
||||
*,
|
||||
success: bool,
|
||||
method: str = "local",
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
"""Write a login or login-failure audit event to the database.
|
||||
|
||||
Failures are silently swallowed so that an audit-service error never
|
||||
prevents a legitimate login or surfaces an unrelated 500 error to the user.
|
||||
|
||||
Args:
|
||||
db: Active database session.
|
||||
request: The current HTTP request (used to extract the client IP).
|
||||
username: The username that attempted authentication.
|
||||
success: ``True`` for a successful login, ``False`` for a failure.
|
||||
method: Authentication method, e.g. ``"local"`` or ``"oauth"``.
|
||||
detail: Optional extra context for failures (e.g. ``"wrong_password"``).
|
||||
"""
|
||||
try:
|
||||
from app.utils.audit_service import record_event
|
||||
|
||||
action = "login" if success else "login.failure"
|
||||
details: dict = {"method": method}
|
||||
if detail:
|
||||
details["reason"] = detail
|
||||
record_event(
|
||||
db,
|
||||
action=action,
|
||||
user=username,
|
||||
resource_type="session",
|
||||
ip_address=get_client_ip(request),
|
||||
details=details,
|
||||
severity="info" if success else "warning",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to write login audit event for user=%s", username, exc_info=True)
|
||||
|
||||
|
||||
async def auth(request: Request, db: Session = Depends(get_db)):
|
||||
"""Handle local username/password authentication.
|
||||
|
||||
In multi-user mode (``MULTI_USER_ENABLED=True``) local registered users are
|
||||
checked first; if no matching LocalUser is found the request falls through to
|
||||
the single admin-credential check so that single-user deployments continue to
|
||||
work without any database involvement.
|
||||
|
||||
In single-user mode (``MULTI_USER_ENABLED=False``, the default) the LocalUser
|
||||
table is never queried — only the configured ADMIN_USERNAME / ADMIN_PASSWORD
|
||||
are accepted, preserving full backward compatibility.
|
||||
"""
|
||||
form_data = await request.form()
|
||||
username = form_data.get("username")
|
||||
password = form_data.get("password")
|
||||
|
||||
logger.debug(
|
||||
"[AUTH] Login attempt: username=%r password_provided=%s multi_user_enabled=%s",
|
||||
username,
|
||||
bool(password),
|
||||
settings.multi_user_enabled,
|
||||
)
|
||||
|
||||
# --- LocalUser check (multi-user mode only) ---
|
||||
if settings.multi_user_enabled:
|
||||
if not username:
|
||||
logger.warning(
|
||||
"[AUTH] LOGIN_FAILURE reason=empty_username multi_user_enabled=%s form_keys=%s content_type=%s",
|
||||
settings.multi_user_enabled,
|
||||
list(form_data.keys()),
|
||||
request.headers.get("content-type", "<missing>"),
|
||||
)
|
||||
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
|
||||
|
||||
local_user = (
|
||||
db.query(_LocalUser).filter((_LocalUser.username == username) | (_LocalUser.email == username)).first()
|
||||
)
|
||||
logger.debug(
|
||||
"[AUTH] LocalUser lookup: username=%r found=%s",
|
||||
username,
|
||||
local_user is not None,
|
||||
)
|
||||
if local_user is not None:
|
||||
if not local_user.is_active:
|
||||
logger.warning(
|
||||
"[SECURITY] LOCAL_LOGIN_UNVERIFIED user=%s is_active=%s",
|
||||
username,
|
||||
local_user.is_active,
|
||||
)
|
||||
_record_login_event(db, request, username, success=False, detail="account_not_verified")
|
||||
@router.get("/oauth-callback")
|
||||
async def oauth_callback(request: Request):
|
||||
"""Handle OAuth callback from provider"""
|
||||
try:
|
||||
token = await oauth.authentik.authorize_access_token(request)
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Please+verify+your+email+address+before+logging+in",
|
||||
status_code=302,
|
||||
url="/login?error=Failed+to+retrieve+user+information",
|
||||
status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
pw_ok = _verify_password(password or "", local_user.hashed_password)
|
||||
logger.debug(
|
||||
"[AUTH] Password verification: user=%s ok=%s",
|
||||
username,
|
||||
pw_ok,
|
||||
)
|
||||
if not pw_ok:
|
||||
logger.warning("[SECURITY] LOCAL_LOGIN_FAILURE reason=wrong_password user=%s", username)
|
||||
_record_login_event(db, request, username, success=False, detail="wrong_password")
|
||||
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
|
||||
user_data = _build_session_user(local_user)
|
||||
|
||||
# Store user info in session
|
||||
user_data = dict(userinfo)
|
||||
|
||||
# Add Gravatar picture if no picture is provided
|
||||
if not user_data.get("picture") and user_data.get("email"):
|
||||
user_data["picture"] = get_gravatar_url(user_data["email"])
|
||||
|
||||
request.session["user"] = user_data
|
||||
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", local_user.email)
|
||||
_record_login_event(db, request, local_user.email, success=True)
|
||||
_ensure_user_profile(db, user_data, is_admin=bool(local_user.is_admin))
|
||||
profile = db.query(_UserProfile).filter(_UserProfile.user_id == local_user.email).first()
|
||||
if profile and not profile.onboarding_completed:
|
||||
post_onboarding = request.session.pop("redirect_after_login", "/upload")
|
||||
request.session["post_onboarding_redirect"] = post_onboarding
|
||||
return RedirectResponse(url="/onboarding", status_code=302)
|
||||
|
||||
# Log the successful authentication
|
||||
print(f"User authenticated via OAuth: {user_data.get('email', 'No email')}")
|
||||
|
||||
# Redirect to original destination or default
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url)
|
||||
except Exception as e:
|
||||
print(f"OAuth authentication error: {str(e)}")
|
||||
return RedirectResponse(
|
||||
url=f"/login?error=Authentication+failed:+{str(e)}",
|
||||
status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
|
||||
@router.post("/auth")
|
||||
async def auth(request: Request):
|
||||
"""Handle local username/password authentication"""
|
||||
form_data = await request.form()
|
||||
username = form_data.get("username")
|
||||
password = form_data.get("password")
|
||||
|
||||
if (username == settings.admin_username and
|
||||
password == settings.admin_password):
|
||||
# Create user session
|
||||
request.session["user"] = {
|
||||
"id": "admin",
|
||||
"name": "Administrator",
|
||||
"email": f"{username}@local.docuelevate",
|
||||
"preferred_username": username,
|
||||
"picture": "/static/images/default-avatar.svg",
|
||||
"is_admin": True
|
||||
}
|
||||
# Redirect to original destination or default
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
else:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Invalid+username+or+password",
|
||||
status_code=302
|
||||
)
|
||||
|
||||
# Local user not found; fall through to admin credential check below.
|
||||
logger.debug(
|
||||
"[AUTH] No LocalUser matched username=%r; falling through to admin credential check",
|
||||
username,
|
||||
@router.get("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Handle user logout"""
|
||||
request.session.pop("user", None)
|
||||
return RedirectResponse(
|
||||
url="/login?message=You+have+been+logged+out+successfully",
|
||||
status_code=302
|
||||
)
|
||||
|
||||
# --- Admin credentials (always available as a fallback / single-user mode) ---
|
||||
# Guard: only attempt the match when credentials are actually configured.
|
||||
# Without this guard, Python's `None == None` would be True when neither
|
||||
# ADMIN_USERNAME nor ADMIN_PASSWORD is set, allowing any request that omits
|
||||
# those form fields to be authenticated as an admin — creating a phantom
|
||||
# "None@local.docuelevate" admin profile with full privileges.
|
||||
admin_configured = bool(settings.admin_username and settings.admin_password)
|
||||
logger.debug(
|
||||
"[AUTH] Admin credential check: admin_configured=%s username_match=%s multi_user_enabled=%s",
|
||||
admin_configured,
|
||||
username == settings.admin_username if admin_configured else False,
|
||||
settings.multi_user_enabled,
|
||||
)
|
||||
if (
|
||||
settings.admin_username
|
||||
and settings.admin_password
|
||||
and username == settings.admin_username
|
||||
and password == settings.admin_password
|
||||
):
|
||||
admin_user_data = {
|
||||
"id": "admin",
|
||||
"name": "Administrator",
|
||||
"email": f"{username}@local.docuelevate",
|
||||
"preferred_username": username,
|
||||
"picture": "/static/images/default-avatar.svg",
|
||||
"is_admin": True,
|
||||
}
|
||||
request.session["user"] = admin_user_data
|
||||
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", username)
|
||||
_record_login_event(db, request, username, success=True)
|
||||
_ensure_user_profile(db, admin_user_data, is_admin=True)
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
else:
|
||||
logger.warning(
|
||||
"[SECURITY] LOCAL_LOGIN_FAILURE reason=no_match user=%r "
|
||||
"multi_user_enabled=%s admin_configured=%s form_empty=%s",
|
||||
username,
|
||||
settings.multi_user_enabled,
|
||||
admin_configured,
|
||||
not username and not password,
|
||||
)
|
||||
_record_login_event(db, request, username or "anonymous", success=False, detail="invalid_credentials")
|
||||
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
|
||||
|
||||
|
||||
async def logout(request: Request, db: Session = Depends(get_db)):
|
||||
"""Handle user logout"""
|
||||
user = request.session.get("user")
|
||||
username = "unknown"
|
||||
if isinstance(user, dict):
|
||||
username = user.get("preferred_username") or user.get("email") or "unknown"
|
||||
logger.info(f"[SECURITY] LOGOUT user={username}")
|
||||
try:
|
||||
from app.utils.audit_service import record_event
|
||||
|
||||
record_event(
|
||||
db,
|
||||
action="logout",
|
||||
user=username,
|
||||
resource_type="session",
|
||||
ip_address=get_client_ip(request),
|
||||
severity="info",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to write logout audit event for user=%s", username, exc_info=True)
|
||||
request.session.pop("user", None)
|
||||
return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302)
|
||||
|
||||
|
||||
if AUTH_ENABLED:
|
||||
router.add_api_route("/login", login, methods=["GET"])
|
||||
router.add_api_route("/oauth-login", oauth_login, methods=["GET"])
|
||||
router.add_api_route("/oauth-callback", oauth_callback, methods=["GET"])
|
||||
router.add_api_route("/social-login/{provider}", social_login, methods=["GET"])
|
||||
router.add_api_route("/social-callback/{provider}", social_callback, methods=["GET"])
|
||||
router.add_api_route("/auth", auth, methods=["POST"])
|
||||
router.add_api_route("/logout", logout, methods=["GET"])
|
||||
|
||||
|
||||
@router.get("/api/auth/whoami")
|
||||
@require_login
|
||||
async def whoami(request: Request):
|
||||
"""API endpoint to get current user information"""
|
||||
user = get_current_user(request)
|
||||
user = request.session.get("user")
|
||||
return user or {"error": "Not authenticated"}
|
||||
|
||||
|
||||
|
||||
+10
-21
@@ -1,8 +1,6 @@
|
||||
# app/celery_app.py
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import task_failure, worker_ready
|
||||
|
||||
from app.config import settings
|
||||
|
||||
celery = Celery(
|
||||
@@ -16,38 +14,29 @@ celery = Celery(
|
||||
celery.conf.broker_connection_retry_on_startup = True
|
||||
|
||||
# Set the default queue and routing so that tasks are enqueued on "document_processor"
|
||||
celery.conf.task_default_queue = "document_processor"
|
||||
celery.conf.task_default_queue = 'document_processor'
|
||||
celery.conf.task_routes = {
|
||||
"app.tasks.*": {"queue": "document_processor"},
|
||||
}
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def init_sentry_on_worker_ready(**kwargs):
|
||||
"""Initialise Sentry SDK in the Celery worker process."""
|
||||
from app.utils.sentry import init_sentry
|
||||
|
||||
init_sentry(integrations_extra=["celery"])
|
||||
|
||||
# Task failure notification handler
|
||||
from celery.signals import task_failure
|
||||
|
||||
@task_failure.connect
|
||||
def task_failure_handler(
|
||||
sender=None, task_id=None, exception=None, args=None, kwargs=None, traceback=None, einfo=None, **kw
|
||||
):
|
||||
def task_failure_handler(sender=None, task_id=None, exception=None, args=None,
|
||||
kwargs=None, traceback=None, einfo=None, **kw):
|
||||
"""Handler for Celery task failures to send notifications"""
|
||||
if getattr(settings, "notify_on_task_failure", True):
|
||||
if getattr(settings, 'notify_on_task_failure', True):
|
||||
try:
|
||||
# Import here to avoid circular imports
|
||||
from app.utils.notification import notify_celery_failure
|
||||
|
||||
notify_celery_failure(
|
||||
task_name=sender.name if sender else "Unknown",
|
||||
task_name=sender.name if sender else "Unknown",
|
||||
task_id=task_id or "N/A",
|
||||
exc=exception,
|
||||
args=args or [],
|
||||
kwargs=kwargs or {},
|
||||
exc=exception,
|
||||
args=args or [],
|
||||
kwargs=kwargs or {}
|
||||
)
|
||||
except Exception as e:
|
||||
import logging
|
||||
|
||||
logging.exception(f"Failed to send task failure notification: {e}")
|
||||
|
||||
+35
-196
@@ -1,101 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
|
||||
from celery.schedules import crontab
|
||||
|
||||
# Ensure tasks are loaded
|
||||
from app import tasks # noqa: F401 - Imports app/tasks.py so Celery can register tasks
|
||||
from app.config import settings
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.backup_tasks import cleanup_old_backups, create_backup # noqa: F401
|
||||
from app.tasks.batch_tasks import ( # noqa: F401
|
||||
backfill_missing_metadata,
|
||||
cleanup_temp_files,
|
||||
expire_shared_links,
|
||||
process_new_documents,
|
||||
prune_old_notifications,
|
||||
prune_processing_logs,
|
||||
reprocess_failed_documents,
|
||||
sync_search_index,
|
||||
)
|
||||
from app.tasks.check_credentials import check_credentials
|
||||
from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding # noqa: F401
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
|
||||
from app.tasks.convert_to_pdfa import convert_to_pdfa # noqa: F401
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
|
||||
from app.tasks.finalize_document_storage import finalize_document_storage # noqa: F401
|
||||
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
|
||||
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
||||
|
||||
# Ensure tasks are loaded
|
||||
from app import tasks # <— This imports app/tasks.py so Celery can register tasks
|
||||
|
||||
# **Ensure all tasks are imported before Celery starts**
|
||||
from app.tasks.process_document import process_document # noqa: F401
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
|
||||
from app.tasks.process_with_ocr import process_with_ocr # noqa: F401
|
||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
|
||||
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
|
||||
from app.tasks.subscription_tasks import apply_pending_subscription_changes_all # noqa: F401
|
||||
from app.tasks.process_document import process_document
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
|
||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
|
||||
# Import new send tasks
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
|
||||
from app.tasks.upload_to_email import upload_to_email # noqa: F401
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401
|
||||
from app.tasks.upload_to_icloud import upload_to_icloud # noqa: F401
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
|
||||
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
||||
from app.tasks.upload_to_user_integration import upload_to_user_integration # noqa: F401
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
||||
from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401
|
||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
||||
from app.tasks.watch_folder_tasks import scan_all_watch_folders # noqa: F401
|
||||
from app.tasks.webhook_tasks import deliver_webhook_task # noqa: F401
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
from app.tasks.upload_to_s3 import upload_to_s3
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
|
||||
# Register the settings reload signal handler so workers pick up config changes
|
||||
from app.utils.settings_sync import register_settings_reload_signal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
register_settings_reload_signal()
|
||||
from app.tasks.imap_tasks import pull_all_inboxes
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma
|
||||
from app.tasks.check_credentials import check_credentials
|
||||
|
||||
celery.conf.task_routes = {
|
||||
"app.tasks.*": {"queue": "default"},
|
||||
}
|
||||
|
||||
|
||||
@celery.task
|
||||
def test_task():
|
||||
return "Celery is working!"
|
||||
|
||||
# If you want Celery Beat to run the poll task every minute, add:
|
||||
from celery.schedules import crontab
|
||||
|
||||
# Run the check_credentials task at startup
|
||||
check_credentials.apply_async(countdown=10) # Run 10 seconds after worker starts
|
||||
|
||||
celery.conf.beat_schedule = {
|
||||
# IMAP polling — always enabled because per-user IMAP integrations may
|
||||
# exist in the database even when no system-level IMAP hosts are configured.
|
||||
"poll-inboxes-every-minute": {
|
||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
},
|
||||
} if (settings.imap1_host or settings.imap2_host) else None,
|
||||
# Add Uptime Kuma ping task if configured
|
||||
"ping-uptime-kuma": (
|
||||
{
|
||||
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
|
||||
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
}
|
||||
if settings.uptime_kuma_url
|
||||
else None
|
||||
),
|
||||
"ping-uptime-kuma": {
|
||||
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
|
||||
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
} if settings.uptime_kuma_url else None,
|
||||
# Check credentials every 5 minutes
|
||||
"check-credentials-regularly": {
|
||||
"task": "app.tasks.check_credentials.check_credentials",
|
||||
@@ -107,133 +71,8 @@ celery.conf.beat_schedule = {
|
||||
"task": "app.tasks.check_credentials.check_credentials",
|
||||
"schedule": crontab(hour="0", minute="0"), # Midnight
|
||||
"options": {"expires": 3600}, # 1 hour expiry
|
||||
},
|
||||
# Monitor for stalled processing steps every minute
|
||||
"monitor-stalled-steps": {
|
||||
"task": "app.tasks.monitor_stalled_steps.monitor_stalled_steps",
|
||||
"schedule": crontab(minute="*/1"), # Every minute
|
||||
"options": {"expires": 55}, # Must complete within 55 seconds
|
||||
},
|
||||
# Watch folder scanning — always enabled because per-user WATCH_FOLDER
|
||||
# integrations may exist in the database even when no system-level watch
|
||||
# folder settings are configured.
|
||||
# Schedule is controlled by WATCH_FOLDER_POLL_INTERVAL (default: 1 minute).
|
||||
"scan-watch-folders": {
|
||||
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
|
||||
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
|
||||
"options": {"expires": 55},
|
||||
},
|
||||
# Backfill embeddings for files that were processed before the
|
||||
# embedding pipeline was enabled, or where the embedding task failed.
|
||||
"backfill-missing-embeddings": {
|
||||
"task": "backfill_missing_embeddings",
|
||||
"schedule": crontab(minute="*/5"), # Every 5 minutes
|
||||
"options": {"expires": 240}, # 4 minutes expiry
|
||||
},
|
||||
# Apply scheduled subscription downgrades daily at 00:05 UTC
|
||||
"apply-pending-subscription-changes": {
|
||||
"task": "app.tasks.subscription_tasks.apply_pending_subscription_changes_all",
|
||||
"schedule": crontab(hour="0", minute="5"), # 00:05 UTC daily
|
||||
"options": {"expires": 3600},
|
||||
},
|
||||
# ── Database backup tasks ──────────────────────────────────────────────
|
||||
# Hourly backup (kept for 4 days)
|
||||
"backup-hourly": (
|
||||
{
|
||||
"task": "app.tasks.backup_tasks.create_backup",
|
||||
"schedule": crontab(minute="0"), # top of every hour
|
||||
"kwargs": {"backup_type": "hourly"},
|
||||
"options": {"expires": 3300},
|
||||
}
|
||||
if settings.backup_enabled
|
||||
else None
|
||||
),
|
||||
# Daily backup (kept for 3 weeks) – runs at 02:30 UTC
|
||||
"backup-daily": (
|
||||
{
|
||||
"task": "app.tasks.backup_tasks.create_backup",
|
||||
"schedule": crontab(hour="2", minute="30"),
|
||||
"kwargs": {"backup_type": "daily"},
|
||||
"options": {"expires": 3600},
|
||||
}
|
||||
if settings.backup_enabled
|
||||
else None
|
||||
),
|
||||
# Weekly backup (kept for 13 weeks) – runs every Sunday at 03:00 UTC
|
||||
"backup-weekly": (
|
||||
{
|
||||
"task": "app.tasks.backup_tasks.create_backup",
|
||||
"schedule": crontab(hour="3", minute="0", day_of_week="0"),
|
||||
"kwargs": {"backup_type": "weekly"},
|
||||
"options": {"expires": 3600},
|
||||
}
|
||||
if settings.backup_enabled
|
||||
else None
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
# Remove None entries from beat_schedule
|
||||
celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Load admin-managed scheduled jobs from the database
|
||||
# ---------------------------------------------------------------------------
|
||||
# These jobs are defined in the ``scheduled_jobs`` table (seeded by
|
||||
# ``app.api.scheduled_jobs.seed_default_scheduled_jobs``) and can be
|
||||
# enabled/disabled and rescheduled via the admin UI at /admin/scheduled-jobs.
|
||||
# The schedule is read once at worker startup; changes take effect after
|
||||
# the worker is restarted.
|
||||
|
||||
|
||||
def _load_db_scheduled_jobs() -> None:
|
||||
"""
|
||||
Extend ``celery.conf.beat_schedule`` with entries from the ``scheduled_jobs``
|
||||
database table.
|
||||
|
||||
Only rows with ``enabled=True`` are added. Rows whose ``name`` key
|
||||
already exists in the static schedule (defined above) are skipped so
|
||||
that hardcoded entries cannot be overridden accidentally.
|
||||
|
||||
Failures are logged as warnings and do not prevent the worker from
|
||||
starting.
|
||||
"""
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
from app.models import ScheduledJob
|
||||
|
||||
with SessionLocal() as db:
|
||||
jobs = db.query(ScheduledJob).filter(ScheduledJob.enabled.is_(True)).all()
|
||||
|
||||
added = 0
|
||||
for job in jobs:
|
||||
if job.name in celery.conf.beat_schedule:
|
||||
# Static entry takes precedence; skip silently.
|
||||
continue
|
||||
|
||||
if job.schedule_type == "interval" and job.interval_seconds:
|
||||
from celery.schedules import schedule as interval_schedule
|
||||
|
||||
sched = interval_schedule(run_every=job.interval_seconds)
|
||||
else:
|
||||
# Default to cron.
|
||||
sched = crontab(
|
||||
minute=job.cron_minute,
|
||||
hour=job.cron_hour,
|
||||
day_of_week=job.cron_day_of_week,
|
||||
day_of_month=job.cron_day_of_month,
|
||||
month_of_year=job.cron_month_of_year,
|
||||
)
|
||||
|
||||
celery.conf.beat_schedule[job.name] = {
|
||||
"task": job.task_name,
|
||||
"schedule": sched,
|
||||
"options": {"expires": 3600},
|
||||
}
|
||||
added += 1
|
||||
|
||||
logger.info("Loaded %d scheduled job(s) from database into Celery Beat.", added)
|
||||
except Exception as exc:
|
||||
logger.warning("Could not load scheduled jobs from database: %s", exc)
|
||||
|
||||
|
||||
_load_db_scheduled_jobs()
|
||||
celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
|
||||
-680
@@ -1,680 +0,0 @@
|
||||
"""DocuElevate command-line interface.
|
||||
|
||||
Provides a pipe-friendly CLI for scripting and automation against the
|
||||
DocuElevate REST API. Authentication is via personal API tokens (the
|
||||
same tokens managed at ``/api-tokens`` in the web UI).
|
||||
|
||||
Usage::
|
||||
|
||||
docuelevate --url http://my-instance --token de_xxx list
|
||||
DOCUELEVATE_URL=http://my-instance DOCUELEVATE_API_TOKEN=de_xxx docuelevate list
|
||||
|
||||
Commands
|
||||
--------
|
||||
upload Upload one or more local files for processing.
|
||||
download Download a processed (or original) file by ID.
|
||||
search Full-text search across all documents.
|
||||
list List documents with optional filtering.
|
||||
token Sub-commands: create / list / revoke API tokens.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Environment-variable defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ENV_URL = "DOCUELEVATE_URL"
|
||||
ENV_TOKEN = "DOCUELEVATE_API_TOKEN"
|
||||
|
||||
_DEFAULT_URL = "http://localhost:8000"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_headers(token: str) -> dict[str, str]:
|
||||
"""Return Authorization headers for the given API token."""
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _api(
|
||||
method: str,
|
||||
base_url: str,
|
||||
path: str,
|
||||
token: str,
|
||||
timeout: int = 60,
|
||||
**kwargs: Any,
|
||||
) -> requests.Response:
|
||||
"""Make an authenticated API request and return the response.
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, DELETE, …).
|
||||
base_url: The base URL of the DocuElevate instance.
|
||||
path: API path starting with ``/``.
|
||||
token: Plaintext API token.
|
||||
timeout: Request timeout in seconds (default: 60).
|
||||
**kwargs: Extra keyword arguments forwarded to :func:`requests.request`.
|
||||
|
||||
Returns:
|
||||
The :class:`requests.Response` object.
|
||||
|
||||
Raises:
|
||||
click.ClickException: On network errors.
|
||||
"""
|
||||
url = base_url.rstrip("/") + path
|
||||
headers = _build_headers(token)
|
||||
try:
|
||||
resp = requests.request(method, url, headers=headers, timeout=timeout, **kwargs)
|
||||
except requests.ConnectionError as exc:
|
||||
raise click.ClickException(f"Could not connect to {base_url}: {exc}") from exc
|
||||
except requests.Timeout as exc:
|
||||
raise click.ClickException(f"Request timed out: {exc}") from exc
|
||||
return resp
|
||||
|
||||
|
||||
def _require_ok(resp: requests.Response) -> dict[str, Any] | list[Any]:
|
||||
"""Assert a successful HTTP response and return parsed JSON.
|
||||
|
||||
Args:
|
||||
resp: The response to check.
|
||||
|
||||
Returns:
|
||||
Parsed JSON payload.
|
||||
|
||||
Raises:
|
||||
click.ClickException: If the response status indicates an error.
|
||||
"""
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
detail = resp.json().get("detail", resp.text)
|
||||
except Exception:
|
||||
detail = resp.text
|
||||
raise click.ClickException(f"API error {resp.status_code}: {detail}")
|
||||
try:
|
||||
return resp.json()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _output(data: Any, fmt: str) -> None:
|
||||
"""Write *data* to stdout in the requested format.
|
||||
|
||||
Args:
|
||||
data: The value to serialise (dict, list, or primitive).
|
||||
fmt: Either ``"json"`` (machine-readable) or ``"table"`` (human-readable).
|
||||
"""
|
||||
if fmt == "json":
|
||||
click.echo(json.dumps(data, indent=2, default=str))
|
||||
else:
|
||||
_print_table(data)
|
||||
|
||||
|
||||
def _print_table(data: Any) -> None:
|
||||
"""Pretty-print a list of dicts as a fixed-width table.
|
||||
|
||||
Falls back to JSON if the data is not a homogeneous list of dicts.
|
||||
|
||||
Args:
|
||||
data: Data to render.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
# Single-object output — print as key: value pairs
|
||||
for key, value in data.items():
|
||||
click.echo(f" {key}: {value}")
|
||||
return
|
||||
|
||||
if not isinstance(data, list) or not data:
|
||||
click.echo(json.dumps(data, indent=2, default=str))
|
||||
return
|
||||
|
||||
if not isinstance(data[0], dict):
|
||||
for item in data:
|
||||
click.echo(str(item))
|
||||
return
|
||||
|
||||
# Determine column widths
|
||||
keys = list(data[0].keys())
|
||||
widths: dict[str, int] = {k: len(k) for k in keys}
|
||||
for row in data:
|
||||
for k in keys:
|
||||
widths[k] = max(widths[k], len(str(row.get(k, ""))))
|
||||
|
||||
header = " ".join(k.upper().ljust(widths[k]) for k in keys)
|
||||
separator = " ".join("-" * widths[k] for k in keys)
|
||||
click.echo(header)
|
||||
click.echo(separator)
|
||||
for row in data:
|
||||
click.echo(" ".join(str(row.get(k, "")).ljust(widths[k]) for k in keys))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Root command group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
|
||||
@click.option(
|
||||
"--url",
|
||||
envvar=ENV_URL,
|
||||
default=_DEFAULT_URL,
|
||||
show_default=True,
|
||||
show_envvar=True,
|
||||
help="Base URL of the DocuElevate instance.",
|
||||
metavar="URL",
|
||||
)
|
||||
@click.option(
|
||||
"--token",
|
||||
envvar=ENV_TOKEN,
|
||||
default=None,
|
||||
show_envvar=True,
|
||||
help="API token (de_…). Required for all commands except help.",
|
||||
metavar="TOKEN",
|
||||
)
|
||||
@click.option(
|
||||
"--format",
|
||||
"fmt",
|
||||
type=click.Choice(["table", "json"], case_sensitive=False),
|
||||
default="table",
|
||||
show_default=True,
|
||||
help="Output format. Use 'json' for machine-readable / pipe-friendly output.",
|
||||
)
|
||||
@click.option(
|
||||
"--timeout",
|
||||
default=60,
|
||||
show_default=True,
|
||||
envvar="DOCUELEVATE_TIMEOUT",
|
||||
show_envvar=True,
|
||||
type=int,
|
||||
help="HTTP request timeout in seconds.",
|
||||
)
|
||||
@click.version_option(package_name="docuelevate", prog_name="docuelevate")
|
||||
@click.pass_context
|
||||
def cli(ctx: click.Context, url: str, token: str | None, fmt: str, timeout: int) -> None:
|
||||
"""DocuElevate CLI — interact with DocuElevate from the command line.
|
||||
|
||||
Configure the target instance and credentials via options or environment
|
||||
variables:
|
||||
|
||||
\b
|
||||
DOCUELEVATE_URL Base URL of the instance (default: http://localhost:8000)
|
||||
DOCUELEVATE_API_TOKEN Personal API token (de_…)
|
||||
DOCUELEVATE_TIMEOUT HTTP request timeout in seconds (default: 60)
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
# Upload a file
|
||||
docuelevate --token de_xxx upload report.pdf
|
||||
|
||||
\b
|
||||
# List files as JSON for further processing
|
||||
docuelevate --token de_xxx --format json list | jq '.[].original_filename'
|
||||
|
||||
\b
|
||||
# Search for invoices
|
||||
docuelevate --token de_xxx search "invoice amazon"
|
||||
"""
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["url"] = url
|
||||
ctx.obj["token"] = token
|
||||
ctx.obj["fmt"] = fmt
|
||||
ctx.obj["timeout"] = timeout
|
||||
|
||||
|
||||
def _get_token(ctx: click.Context) -> str:
|
||||
"""Return the token from context, raising ClickException if absent.
|
||||
|
||||
Args:
|
||||
ctx: The current Click context.
|
||||
|
||||
Returns:
|
||||
The API token string.
|
||||
|
||||
Raises:
|
||||
click.ClickException: If no token has been provided.
|
||||
"""
|
||||
token = ctx.obj.get("token")
|
||||
if not token:
|
||||
raise click.ClickException(f"No API token provided. Use --token or set the {ENV_TOKEN} environment variable.")
|
||||
return token
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cli.command("list")
|
||||
@click.option("--page", default=1, show_default=True, help="Page number.")
|
||||
@click.option("--per-page", default=25, show_default=True, help="Items per page (max 200).")
|
||||
@click.option("--search", default=None, help="Filter by filename substring.")
|
||||
@click.option("--mime-type", default=None, help="Filter by MIME type (e.g. application/pdf).")
|
||||
@click.option("--status", "file_status", default=None, help="Filter by status: pending, processing, completed, failed.")
|
||||
@click.option("--sort-by", default="created_at", show_default=True, help="Sort field.")
|
||||
@click.option("--sort-order", type=click.Choice(["asc", "desc"]), default="desc", show_default=True)
|
||||
@click.pass_context
|
||||
def list_files(
|
||||
ctx: click.Context,
|
||||
page: int,
|
||||
per_page: int,
|
||||
search: str | None,
|
||||
mime_type: str | None,
|
||||
file_status: str | None,
|
||||
sort_by: str,
|
||||
sort_order: str,
|
||||
) -> None:
|
||||
"""List documents stored in DocuElevate.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate list
|
||||
docuelevate list --status completed --per-page 10
|
||||
docuelevate --format json list | jq '.[].original_filename'
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
fmt: str = ctx.obj["fmt"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
params: dict[str, Any] = {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"sort_by": sort_by,
|
||||
"sort_order": sort_order,
|
||||
}
|
||||
if search:
|
||||
params["search"] = search
|
||||
if mime_type:
|
||||
params["mime_type"] = mime_type
|
||||
if file_status:
|
||||
params["status"] = file_status
|
||||
|
||||
resp = _api("GET", url, "/api/files", token, timeout=timeout, params=params)
|
||||
payload = _require_ok(resp)
|
||||
|
||||
# Extract the list from the paginated response
|
||||
files: list[dict[str, Any]] = payload.get("files", payload) if isinstance(payload, dict) else payload # type: ignore[assignment]
|
||||
pagination: dict[str, Any] = payload.get("pagination", {}) if isinstance(payload, dict) else {}
|
||||
|
||||
if fmt == "json":
|
||||
_output(files, fmt)
|
||||
else:
|
||||
# Trim fields for readable table
|
||||
rows = [
|
||||
{
|
||||
"id": f.get("id"),
|
||||
"filename": f.get("original_filename"),
|
||||
"size": f.get("file_size"),
|
||||
"status": f.get("status"),
|
||||
"created_at": str(f.get("created_at", ""))[:19],
|
||||
}
|
||||
for f in files
|
||||
]
|
||||
_output(rows, fmt)
|
||||
if pagination:
|
||||
click.echo(f"\nPage {pagination.get('page')}/{pagination.get('pages')} ({pagination.get('total')} total)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# upload command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cli.command("upload")
|
||||
@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True, readable=True))
|
||||
@click.option(
|
||||
"--batch-size",
|
||||
default=5,
|
||||
show_default=True,
|
||||
help="Maximum number of concurrent uploads (sequential when 1).",
|
||||
)
|
||||
@click.pass_context
|
||||
def upload_files(ctx: click.Context, files: tuple[str, ...], batch_size: int) -> None:
|
||||
"""Upload one or more local files for processing.
|
||||
|
||||
Supports glob patterns and multiple arguments for batch uploads.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate upload report.pdf
|
||||
docuelevate upload *.pdf invoice_*.png
|
||||
docuelevate upload --batch-size 3 /scans/*.pdf
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
fmt: str = ctx.obj["fmt"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
failed = 0
|
||||
|
||||
for i, file_path in enumerate(files, 1):
|
||||
path = Path(file_path)
|
||||
click.echo(f"[{i}/{len(files)}] Uploading {path.name}…", err=True)
|
||||
try:
|
||||
with path.open("rb") as fh:
|
||||
resp = _api(
|
||||
"POST",
|
||||
url,
|
||||
"/api/ui-upload",
|
||||
token,
|
||||
timeout=timeout,
|
||||
files={"file": (path.name, fh)},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
detail = resp.json().get("detail", resp.text)
|
||||
except Exception:
|
||||
detail = resp.text
|
||||
click.echo(f" ERROR {resp.status_code}: {detail}", err=True)
|
||||
results.append({"file": path.name, "status": "error", "detail": detail})
|
||||
failed += 1
|
||||
else:
|
||||
data = resp.json()
|
||||
results.append({"file": path.name, "status": "queued", **data})
|
||||
click.echo(f" OK task_id={data.get('task_id', '?')}", err=True)
|
||||
except click.ClickException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
click.echo(f" ERROR: {exc}", err=True)
|
||||
results.append({"file": path.name, "status": "error", "detail": str(exc)})
|
||||
failed += 1
|
||||
|
||||
_output(results, fmt)
|
||||
|
||||
if failed:
|
||||
click.echo(f"\n{failed}/{len(files)} upload(s) failed.", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# download command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cli.command("download")
|
||||
@click.argument("file_id", type=int)
|
||||
@click.option(
|
||||
"--output",
|
||||
"-o",
|
||||
default=None,
|
||||
help="Destination file path. Defaults to the server-provided filename in the current directory.",
|
||||
type=click.Path(),
|
||||
)
|
||||
@click.option(
|
||||
"--version",
|
||||
type=click.Choice(["processed", "original"]),
|
||||
default="processed",
|
||||
show_default=True,
|
||||
help="Which version to download.",
|
||||
)
|
||||
@click.pass_context
|
||||
def download_file(ctx: click.Context, file_id: int, output: str | None, version: str) -> None:
|
||||
"""Download a file by its numeric ID.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate download 42
|
||||
docuelevate download 42 --version original -o /tmp/orig.pdf
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
resp = _api(
|
||||
"GET",
|
||||
url,
|
||||
f"/api/files/{file_id}/download",
|
||||
token,
|
||||
timeout=timeout,
|
||||
params={"version": version},
|
||||
stream=True,
|
||||
)
|
||||
_require_ok(resp)
|
||||
|
||||
# Determine output filename
|
||||
if output:
|
||||
dest = Path(output)
|
||||
else:
|
||||
content_disp = resp.headers.get("content-disposition", "")
|
||||
filename = f"file_{file_id}"
|
||||
for raw_part in content_disp.split(";"):
|
||||
clean = raw_part.strip()
|
||||
if clean.startswith("filename="):
|
||||
filename = clean[len("filename=") :].strip('"').strip("'")
|
||||
break
|
||||
if clean.startswith("filename*="):
|
||||
raw = clean[len("filename*=") :]
|
||||
if raw.upper().startswith("UTF-8''"):
|
||||
filename = unquote(raw[7:])
|
||||
break
|
||||
dest = Path(filename)
|
||||
|
||||
with dest.open("wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
fh.write(chunk)
|
||||
|
||||
click.echo(f"Downloaded {dest} ({dest.stat().st_size} bytes)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# search command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cli.command("search")
|
||||
@click.argument("query")
|
||||
@click.option("--mime-type", default=None, help="Filter by MIME type.")
|
||||
@click.option("--document-type", default=None, help="Filter by document type (e.g. Invoice).")
|
||||
@click.option("--tags", default=None, help="Filter by tag.")
|
||||
@click.option("--language", default=None, help="Filter by language code (e.g. en, de).")
|
||||
@click.option("--page", default=1, show_default=True)
|
||||
@click.option("--per-page", default=20, show_default=True, help="Results per page (max 100).")
|
||||
@click.pass_context
|
||||
def search(
|
||||
ctx: click.Context,
|
||||
query: str,
|
||||
mime_type: str | None,
|
||||
document_type: str | None,
|
||||
tags: str | None,
|
||||
language: str | None,
|
||||
page: int,
|
||||
per_page: int,
|
||||
) -> None:
|
||||
"""Full-text search across all documents.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate search "invoice amazon"
|
||||
docuelevate search "contract" --document-type Contract --language en
|
||||
docuelevate --format json search "receipt" | jq '.[].file_id'
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
fmt: str = ctx.obj["fmt"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
params: dict[str, Any] = {"q": query, "page": page, "per_page": per_page}
|
||||
if mime_type:
|
||||
params["mime_type"] = mime_type
|
||||
if document_type:
|
||||
params["document_type"] = document_type
|
||||
if tags:
|
||||
params["tags"] = tags
|
||||
if language:
|
||||
params["language"] = language
|
||||
|
||||
resp = _api("GET", url, "/api/search", token, timeout=timeout, params=params)
|
||||
payload = _require_ok(resp)
|
||||
|
||||
results: list[dict[str, Any]] = (
|
||||
payload.get("results", payload) if isinstance(payload, dict) else payload # type: ignore[assignment]
|
||||
)
|
||||
total: int = payload.get("total", len(results)) if isinstance(payload, dict) else len(results)
|
||||
pages: int = payload.get("pages", 1) if isinstance(payload, dict) else 1
|
||||
|
||||
if fmt == "json":
|
||||
_output(results, fmt)
|
||||
else:
|
||||
rows = [
|
||||
{
|
||||
"file_id": r.get("file_id"),
|
||||
"filename": r.get("original_filename"),
|
||||
"type": r.get("document_type"),
|
||||
"tags": ",".join(r.get("tags") or []),
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
_output(rows, fmt)
|
||||
click.echo(f"\nPage {page}/{pages} ({total} total results)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# token sub-group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cli.group("token")
|
||||
@click.pass_context
|
||||
def token_group(ctx: click.Context) -> None:
|
||||
"""Manage personal API tokens.
|
||||
|
||||
Tokens can be created, listed, and revoked. Token rotation is achieved
|
||||
by creating a new token before revoking the old one.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate token create "CI Pipeline"
|
||||
docuelevate token list
|
||||
docuelevate token revoke 3
|
||||
"""
|
||||
|
||||
|
||||
@token_group.command("create")
|
||||
@click.argument("name")
|
||||
@click.pass_context
|
||||
def token_create(ctx: click.Context, name: str) -> None:
|
||||
"""Create a new personal API token.
|
||||
|
||||
The full token value is printed exactly once. Store it securely.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate token create "My script"
|
||||
docuelevate --format json token create "CI" | jq -r '.token'
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
fmt: str = ctx.obj["fmt"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
resp = _api("POST", url, "/api/api-tokens/", token, timeout=timeout, json={"name": name})
|
||||
payload = _require_ok(resp)
|
||||
|
||||
if fmt == "json":
|
||||
_output(payload, fmt)
|
||||
else:
|
||||
if not isinstance(payload, dict):
|
||||
raise click.ClickException("Unexpected API response format.")
|
||||
click.echo("Token created successfully:")
|
||||
click.echo(f" ID: {payload.get('id')}")
|
||||
click.echo(f" Name: {payload.get('name')}")
|
||||
click.echo(f" Prefix: {payload.get('token_prefix')}")
|
||||
click.echo(f" Token: {payload.get('token')}")
|
||||
click.echo()
|
||||
click.echo("Store this token securely — it will not be shown again.", err=True)
|
||||
|
||||
|
||||
@token_group.command("list")
|
||||
@click.pass_context
|
||||
def token_list(ctx: click.Context) -> None:
|
||||
"""List all your API tokens (active and revoked).
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate token list
|
||||
docuelevate --format json token list | jq '.[] | select(.is_active)'
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
fmt: str = ctx.obj["fmt"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
resp = _api("GET", url, "/api/api-tokens/", token, timeout=timeout)
|
||||
payload = _require_ok(resp)
|
||||
|
||||
if fmt == "json":
|
||||
_output(payload, fmt)
|
||||
else:
|
||||
if not isinstance(payload, list):
|
||||
raise click.ClickException("Unexpected API response format.")
|
||||
rows = [
|
||||
{
|
||||
"id": t.get("id"),
|
||||
"name": t.get("name"),
|
||||
"prefix": t.get("token_prefix"),
|
||||
"active": t.get("is_active"),
|
||||
"last_used": str(t.get("last_used_at") or "never")[:19],
|
||||
"created": str(t.get("created_at") or "")[:19],
|
||||
}
|
||||
for t in payload
|
||||
]
|
||||
_output(rows, fmt)
|
||||
|
||||
|
||||
@token_group.command("revoke")
|
||||
@click.argument("token_id", type=int)
|
||||
@click.option("--yes", "-y", is_flag=True, help="Skip confirmation prompt.")
|
||||
@click.pass_context
|
||||
def token_revoke(ctx: click.Context, token_id: int, yes: bool) -> None:
|
||||
"""Revoke an API token by its numeric ID.
|
||||
|
||||
The token is soft-deleted (kept for audit) but immediately invalidated.
|
||||
|
||||
Examples:
|
||||
|
||||
\b
|
||||
docuelevate token revoke 3
|
||||
docuelevate token revoke 3 --yes
|
||||
"""
|
||||
token = _get_token(ctx)
|
||||
url: str = ctx.obj["url"]
|
||||
timeout: int = ctx.obj["timeout"]
|
||||
|
||||
if not yes:
|
||||
click.confirm(f"Revoke token {token_id}?", abort=True)
|
||||
|
||||
resp = _api("DELETE", url, f"/api/api-tokens/{token_id}", token, timeout=timeout)
|
||||
_require_ok(resp)
|
||||
click.echo(f"Token {token_id} revoked.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Entry point for the ``docuelevate`` console script."""
|
||||
cli(auto_envvar_prefix="DOCUELEVATE") # type: ignore[call-arg]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+55
-1135
File diff suppressed because it is too large
Load Diff
+12
-232
@@ -1,15 +1,12 @@
|
||||
# app/database.py
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
import logging
|
||||
|
||||
from sqlalchemy import create_engine, exc
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.engine.url import make_url
|
||||
from sqlalchemy.orm import Session, declarative_base, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
|
||||
@@ -23,24 +20,18 @@ engine = create_engine(DB_URL, connect_args={"check_same_thread": False})
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
def init_db():
|
||||
"""
|
||||
Ensures the SQLite database file and its parent directory exist (if using sqlite).
|
||||
Then initializes tables and applies any pending Alembic migrations:
|
||||
|
||||
- Fresh/legacy databases (no ``alembic_version`` table): creates all tables via
|
||||
``Base.metadata.create_all()``, then stamps the Alembic version to ``head``.
|
||||
- Alembic-tracked databases (``alembic_version`` present): skips ``create_all()``
|
||||
and applies pending migrations via ``alembic upgrade head``. Skipping
|
||||
``create_all()`` prevents an ``OperationalError`` when the ORM model defines a
|
||||
table (e.g. ``webhook_configs``) that a pending migration also tries to create.
|
||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables.
|
||||
Logs a message if a new SQLite DB file is created.
|
||||
"""
|
||||
# 1. Parse the DB URL to see if it's sqlite
|
||||
url = make_url(DB_URL)
|
||||
if url.get_backend_name() == "sqlite":
|
||||
# 2. Extract the database path from the URL
|
||||
database_path = url.database # e.g. "/workdir/db/database.db" or ":memory:"
|
||||
|
||||
|
||||
if database_path != ":memory:":
|
||||
# 3. Ensure directory exists
|
||||
db_dir = os.path.dirname(database_path)
|
||||
@@ -52,228 +43,17 @@ def init_db() -> None:
|
||||
if not os.path.exists(database_path):
|
||||
logger.info(f"Creating new SQLite database file at {database_path}")
|
||||
open(database_path, "a").close()
|
||||
|
||||
# 5. Create tables only for fresh/legacy databases not yet tracked by Alembic.
|
||||
# For Alembic-tracked databases, skip create_all to avoid conflicts where
|
||||
# the ORM model would create a table (e.g. webhook_configs) that a pending
|
||||
# Alembic migration also tries to create, causing an OperationalError.
|
||||
|
||||
# 5. Now create tables if they don't exist yet
|
||||
try:
|
||||
from sqlalchemy import inspect
|
||||
|
||||
table_names = inspect(engine).get_table_names()
|
||||
if "alembic_version" not in table_names:
|
||||
Base.metadata.create_all(bind=engine)
|
||||
logger.info("Database initialization complete (tables created if not exist).")
|
||||
|
||||
# 6. Run Alembic migrations (stamps fresh/legacy DBs to head, upgrades tracked DBs)
|
||||
_run_alembic_upgrade(engine)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
logger.info("Database initialization complete (tables created if not exist).")
|
||||
except exc.SQLAlchemyError as e:
|
||||
logger.error(f"Error initializing database: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def _run_alembic_upgrade(engine: Any) -> None:
|
||||
"""Run Alembic migrations programmatically to apply pending schema changes.
|
||||
|
||||
For fresh databases (created via ``Base.metadata.create_all()``), the
|
||||
Alembic version is stamped to ``head`` because all tables already exist.
|
||||
For existing databases with Alembic tracking, any pending migrations are
|
||||
applied via ``alembic upgrade head``.
|
||||
|
||||
Args:
|
||||
engine: The SQLAlchemy engine connected to the target database.
|
||||
"""
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(engine)
|
||||
table_names = inspector.get_table_names()
|
||||
|
||||
# Locate the migrations directory relative to this file
|
||||
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||
|
||||
# Build an Alembic Config that points at our migration scripts.
|
||||
# The sqlalchemy.url is intentionally left empty because we pass the
|
||||
# live connection via config.attributes["connection"] below.
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", "")
|
||||
|
||||
with engine.begin() as connection:
|
||||
alembic_cfg.attributes["connection"] = connection
|
||||
|
||||
if "alembic_version" not in table_names:
|
||||
# Fresh database or one that predates Alembic tracking.
|
||||
# Base.metadata.create_all() already created everything, so
|
||||
# stamp the current version to head (no migrations need to run).
|
||||
logger.info("No Alembic version table found — stamping database to latest revision.")
|
||||
command.stamp(alembic_cfg, "head")
|
||||
else:
|
||||
# Existing database with Alembic version tracking — apply pending migrations.
|
||||
logger.info("Running pending Alembic migrations…")
|
||||
command.upgrade(alembic_cfg, "head")
|
||||
logger.info("Alembic migration check complete.")
|
||||
|
||||
|
||||
def _run_schema_migrations(engine: Any) -> None:
|
||||
"""Apply lightweight schema migrations for columns added after the initial release.
|
||||
|
||||
.. deprecated::
|
||||
This function is deprecated and will be removed in a future release.
|
||||
All schema migrations are now managed exclusively through Alembic.
|
||||
Run ``alembic upgrade head`` (or let ``init_db()`` handle it
|
||||
automatically) instead of calling this function directly.
|
||||
|
||||
Each migration is idempotent and safe to run multiple times.
|
||||
"""
|
||||
warnings.warn(
|
||||
"_run_schema_migrations() is deprecated. "
|
||||
"All schema changes are now managed by Alembic migrations. "
|
||||
"Use 'alembic upgrade head' or init_db() instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
inspector = inspect(engine)
|
||||
|
||||
# Migration: Add 'detail' column to processing_logs (added for verbose worker log output)
|
||||
if "processing_logs" in inspector.get_table_names():
|
||||
columns = [col["name"] for col in inspector.get_columns("processing_logs")]
|
||||
if "detail" not in columns:
|
||||
logger.info("Migrating processing_logs: adding 'detail' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE processing_logs ADD COLUMN detail TEXT"))
|
||||
logger.info("Migration complete: 'detail' column added to processing_logs")
|
||||
|
||||
# Migration: Add file path columns to files table
|
||||
if "files" in inspector.get_table_names():
|
||||
columns = [col["name"] for col in inspector.get_columns("files")]
|
||||
if "original_file_path" not in columns:
|
||||
logger.info("Migrating files: adding 'original_file_path' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN original_file_path VARCHAR"))
|
||||
logger.info("Migration complete: 'original_file_path' column added to files")
|
||||
|
||||
if "processed_file_path" not in columns:
|
||||
logger.info("Migrating files: adding 'processed_file_path' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN processed_file_path VARCHAR"))
|
||||
logger.info("Migration complete: 'processed_file_path' column added to files")
|
||||
|
||||
# Migration: Add deduplication columns to files table
|
||||
if "is_duplicate" not in columns:
|
||||
logger.info("Migrating files: adding 'is_duplicate' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN is_duplicate BOOLEAN DEFAULT FALSE NOT NULL"))
|
||||
logger.info("Migration complete: 'is_duplicate' column added to files")
|
||||
|
||||
if "duplicate_of_id" not in columns:
|
||||
logger.info("Migrating files: adding 'duplicate_of_id' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN duplicate_of_id INTEGER"))
|
||||
logger.info("Migration complete: 'duplicate_of_id' column added to files")
|
||||
|
||||
# Migration: Add search/OCR fields to files table
|
||||
if "ocr_text" not in columns:
|
||||
logger.info("Migrating files: adding 'ocr_text' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_text TEXT"))
|
||||
logger.info("Migration complete: 'ocr_text' column added to files")
|
||||
|
||||
if "ai_metadata" not in columns:
|
||||
logger.info("Migrating files: adding 'ai_metadata' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ai_metadata TEXT"))
|
||||
logger.info("Migration complete: 'ai_metadata' column added to files")
|
||||
|
||||
if "document_title" not in columns:
|
||||
logger.info("Migrating files: adding 'document_title' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
|
||||
logger.info("Migration complete: 'document_title' column added to files")
|
||||
|
||||
if "ocr_quality_score" not in columns:
|
||||
logger.info("Migrating files: adding 'ocr_quality_score' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
|
||||
logger.info("Migration complete: 'ocr_quality_score' column added to files")
|
||||
|
||||
# Migration: Drop unique index on filehash to allow duplicate records
|
||||
try:
|
||||
indexes = inspector.get_indexes("files")
|
||||
unique_filehash_indexes = [
|
||||
index for index in indexes if index.get("unique") and "filehash" in index.get("column_names", [])
|
||||
]
|
||||
if unique_filehash_indexes:
|
||||
logger.info("Migrating files: dropping unique index on 'filehash'")
|
||||
with engine.begin() as conn:
|
||||
for index in unique_filehash_indexes:
|
||||
conn.execute(text(f"DROP INDEX IF EXISTS {index['name']}"))
|
||||
logger.info("Migration complete: unique index on 'filehash' removed")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Skipping filehash unique index drop: {exc}")
|
||||
|
||||
# Migration: Create saved_searches table for user-defined filter combinations
|
||||
if "saved_searches" not in inspector.get_table_names():
|
||||
logger.info("Migrating: creating 'saved_searches' table")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE saved_searches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL,
|
||||
filters TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (user_id, name)
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_saved_searches_user_id ON saved_searches (user_id)"))
|
||||
logger.info("Migration complete: 'saved_searches' table created")
|
||||
|
||||
# Migration: Add performance indexes for common query patterns
|
||||
_ensure_indexes(engine, inspector)
|
||||
|
||||
|
||||
def _ensure_indexes(engine: Any, inspector: Any) -> None:
|
||||
"""
|
||||
Create performance indexes for common query patterns.
|
||||
|
||||
Each ``CREATE INDEX IF NOT EXISTS`` is idempotent and safe to run
|
||||
on every startup. The indexes target the columns most frequently
|
||||
used in file listing/filtering, status computation and log retrieval.
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
_PERF_INDEXES = [
|
||||
("ix_files_created_at", "files", "created_at"),
|
||||
("ix_files_mime_type", "files", "mime_type"),
|
||||
("ix_processing_logs_file_id", "processing_logs", "file_id"),
|
||||
("ix_processing_logs_timestamp", "processing_logs", "timestamp"),
|
||||
("ix_file_processing_steps_status", "file_processing_steps", "status"),
|
||||
]
|
||||
|
||||
table_names = inspector.get_table_names()
|
||||
columns_by_table: dict[str, set[str]] = {}
|
||||
with engine.begin() as conn:
|
||||
for idx_name, table, column in _PERF_INDEXES:
|
||||
if table in table_names:
|
||||
if table not in columns_by_table:
|
||||
columns_by_table[table] = {col["name"] for col in inspector.get_columns(table)}
|
||||
if column in columns_by_table[table]:
|
||||
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table} ({column})"))
|
||||
|
||||
logger.info("Performance indexes ensured")
|
||||
|
||||
|
||||
def get_db() -> Generator[Session, None, None]:
|
||||
def get_db():
|
||||
"""
|
||||
Dependency for FastAPI routes or general DB usage.
|
||||
Yields a SQLAlchemy session, and closes it upon exit.
|
||||
|
||||
+2
-2
@@ -2,9 +2,9 @@
|
||||
Frontend routes for the application.
|
||||
This module is now a re-export of the modularized view routers.
|
||||
"""
|
||||
|
||||
# Import and re-export the router from the views package
|
||||
from app.views import router # noqa: F401
|
||||
from app.views import router
|
||||
|
||||
# Keep the original router name for compatibility
|
||||
# This allows existing imports in main.py to continue working
|
||||
|
||||
|
||||
+67
-270
@@ -1,37 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import logging
|
||||
import pathlib
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request, status
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from starlette.config import Config
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
from starlette.config import Config
|
||||
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
||||
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
|
||||
from pathlib import Path
|
||||
|
||||
from app.api import router as api_router
|
||||
from app.api.graphql_api import graphql_router
|
||||
from app.api.local_auth import router as local_auth_router
|
||||
from app.auth import router as auth_router
|
||||
from app.config import settings
|
||||
from app.database import init_db
|
||||
from app.middleware.audit_log import AuditLogMiddleware
|
||||
from app.middleware.csrf import CSRFMiddleware
|
||||
from app.middleware.rate_limit import create_limiter, get_rate_limit_exceeded_handler
|
||||
from app.middleware.request_size_limit import RequestSizeLimitMiddleware
|
||||
from app.middleware.security_headers import SecurityHeadersMiddleware
|
||||
from app.config import settings
|
||||
from app.utils.config_validator import check_all_configs
|
||||
from app.utils.notification import init_apprise, notify_shutdown, notify_startup
|
||||
from app.utils.sentry import init_sentry
|
||||
from app.utils.notification import init_apprise, send_notification, notify_startup, notify_shutdown
|
||||
|
||||
# Import the routers - now using views directly instead of frontend
|
||||
from app.views import router as frontend_router
|
||||
from app.api import router as api_router
|
||||
from app.auth import router as auth_router
|
||||
|
||||
# Explicitly include the files router
|
||||
from app.views.files import router as files_router
|
||||
@@ -41,198 +30,23 @@ config = Config(".env")
|
||||
# Use settings.session_secret which has proper validation
|
||||
# Fallback to raising an error if not set when auth is enabled
|
||||
if settings.auth_enabled and not settings.session_secret:
|
||||
raise ValueError(
|
||||
"SESSION_SECRET must be set when AUTH_ENABLED=True. "
|
||||
"Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'"
|
||||
)
|
||||
SESSION_SECRET = (
|
||||
settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
|
||||
)
|
||||
raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True. Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'")
|
||||
SESSION_SECRET = settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
|
||||
|
||||
app = FastAPI(title="DocuElevate")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Manage application lifespan events (startup and shutdown).
|
||||
This replaces the deprecated @app.on_event decorators.
|
||||
"""
|
||||
# Startup: Initialize database
|
||||
init_db() # Create tables if they don't exist
|
||||
|
||||
# Load settings from database after DB initialization
|
||||
from app.database import SessionLocal
|
||||
from app.utils.config_loader import load_settings_from_db
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
load_settings_from_db(settings, db)
|
||||
logging.info("Database settings loaded successfully")
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to load database settings: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Initialize Sentry after DB settings are loaded so that values configured
|
||||
# via the database UI (e.g. SENTRY_DSN) are respected in addition to env vars.
|
||||
init_sentry()
|
||||
|
||||
# Ensure OCR language data is available (background download, non-blocking)
|
||||
from app.utils.ocr_language_manager import ensure_ocr_languages_async
|
||||
|
||||
ensure_ocr_languages_async()
|
||||
|
||||
# Force settings dump to log for troubleshooting
|
||||
from app.utils.config_validator import dump_all_settings
|
||||
|
||||
dump_all_settings()
|
||||
|
||||
# Validate configuration
|
||||
config_issues = check_all_configs()
|
||||
|
||||
# Log overall status
|
||||
has_issues = any(config_issues["email"]) or any(
|
||||
len(issues) > 0 for provider, issues in config_issues["storage"].items()
|
||||
)
|
||||
if has_issues:
|
||||
logging.warning("Application started with configuration issues - some features may be unavailable")
|
||||
else:
|
||||
logging.info("Application started with valid configuration")
|
||||
|
||||
logging.info("Router organization: Using refactored API routers from app/api/ directory")
|
||||
|
||||
# Initialize notification system
|
||||
init_apprise()
|
||||
|
||||
# Send startup notification
|
||||
notify_startup()
|
||||
|
||||
# Seed default subscription plans if none exist
|
||||
try:
|
||||
from app.database import SessionLocal as _SessionLocal
|
||||
from app.utils.subscription import seed_default_plans as _seed_plans
|
||||
|
||||
_db_seed = _SessionLocal()
|
||||
try:
|
||||
_seed_plans(_db_seed)
|
||||
finally:
|
||||
_db_seed.close()
|
||||
except Exception:
|
||||
logging.debug("Subscription plan seeding skipped — DB may not be ready yet") # noqa: S110
|
||||
|
||||
# Seed the default system pipeline (mirrors the current hardcoded processing
|
||||
# workflow) so it is immediately visible in the Pipelines management UI.
|
||||
try:
|
||||
from app.api.pipelines import seed_default_pipeline as _seed_pipeline
|
||||
from app.database import SessionLocal as _SessionLocal # noqa: F811 (re-import for clarity)
|
||||
|
||||
_db_pipeline = _SessionLocal()
|
||||
try:
|
||||
_seed_pipeline(_db_pipeline)
|
||||
finally:
|
||||
_db_pipeline.close()
|
||||
except Exception:
|
||||
logging.debug("Default pipeline seeding skipped — DB may not be ready yet") # noqa: S110
|
||||
|
||||
# Seed the default scheduled batch processing jobs so they appear in the
|
||||
# admin UI (/admin/scheduled-jobs) on first startup.
|
||||
try:
|
||||
from app.api.scheduled_jobs import seed_default_scheduled_jobs as _seed_jobs
|
||||
from app.database import SessionLocal as _SessionLocal # noqa: F811
|
||||
|
||||
_db_jobs = _SessionLocal()
|
||||
try:
|
||||
_seed_jobs(_db_jobs)
|
||||
finally:
|
||||
_db_jobs.close()
|
||||
except Exception:
|
||||
logging.debug("Scheduled jobs seeding skipped — DB may not be ready yet") # noqa: S110
|
||||
|
||||
# Seed the built-in compliance templates (GDPR, HIPAA, SOC2) so they
|
||||
# are available in the admin compliance dashboard on first startup.
|
||||
try:
|
||||
from app.database import SessionLocal as _SessionLocal # noqa: F811
|
||||
from app.utils.compliance_service import seed_compliance_templates as _seed_compliance
|
||||
|
||||
_db_compliance = _SessionLocal()
|
||||
try:
|
||||
_seed_compliance(_db_compliance)
|
||||
finally:
|
||||
_db_compliance.close()
|
||||
except Exception:
|
||||
logging.debug("Compliance template seeding skipped — DB may not be ready yet") # noqa: S110
|
||||
|
||||
# Application is now running
|
||||
yield
|
||||
|
||||
# Shutdown: Cleanup tasks
|
||||
logging.info("Application shutting down")
|
||||
|
||||
# Send shutdown notification
|
||||
notify_shutdown()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="DocuElevate",
|
||||
lifespan=lifespan,
|
||||
docs_url="/admin/api-docs",
|
||||
redoc_url="/admin/api-redoc",
|
||||
)
|
||||
|
||||
# Initialize rate limiter and attach to app state
|
||||
limiter = create_limiter(redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled)
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, get_rate_limit_exceeded_handler())
|
||||
|
||||
# Middleware stack (order matters - applied in reverse order)
|
||||
# Last added middleware is executed first
|
||||
|
||||
# 1) Security Headers Middleware (outermost - adds headers to final response)
|
||||
# Configure via SECURITY_HEADERS_ENABLED environment variable
|
||||
# Set to False if reverse proxy (Traefik, Nginx) handles security headers
|
||||
app.add_middleware(SecurityHeadersMiddleware, config=settings)
|
||||
|
||||
# 2) Request Size Limit Middleware - enforces body size limits before reading
|
||||
# MAX_REQUEST_BODY_SIZE: limit for non-file requests (default 1 MB)
|
||||
# MAX_UPLOAD_SIZE: limit for multipart/form-data uploads (default 1 GB)
|
||||
# See SECURITY_AUDIT.md – Code Security section
|
||||
app.add_middleware(RequestSizeLimitMiddleware, config=settings)
|
||||
|
||||
# 3) CSRF Protection Middleware - validates CSRF tokens for state-changing operations
|
||||
# Only active when AUTH_ENABLED=True. Exempts OAuth callback endpoints.
|
||||
# Tokens are stored in the session and validated via X-CSRF-Token header or form field.
|
||||
app.add_middleware(CSRFMiddleware, config=settings)
|
||||
|
||||
# 2) Audit Logging Middleware - logs all requests with sensitive data masking
|
||||
# Configure via AUDIT_LOGGING_ENABLED environment variable
|
||||
# See SECURITY_AUDIT.md – Infrastructure Security section
|
||||
app.add_middleware(AuditLogMiddleware, config=settings)
|
||||
|
||||
# 3) Session Middleware (for request.session to work)
|
||||
# 1) Session Middleware (for request.session to work)
|
||||
app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET)
|
||||
|
||||
# 3a) CORS Middleware - handles cross-origin requests and preflight (OPTIONS) responses.
|
||||
# Disabled by default: set CORS_ENABLED=True only when NOT using a reverse proxy
|
||||
# (Traefik, Nginx) that already injects CORS headers. When enabled, this middleware
|
||||
# runs after the session layer so preflight requests bypass CSRF/auth checks.
|
||||
# Allowed origins, methods, headers, and credentials are all configurable via env vars.
|
||||
# See SECURITY_AUDIT.md – Infrastructure Security section and docs/DeploymentGuide.md.
|
||||
if settings.cors_enabled:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_allowed_origins,
|
||||
allow_credentials=settings.cors_allow_credentials,
|
||||
allow_methods=settings.cors_allowed_methods,
|
||||
allow_headers=settings.cors_allowed_headers,
|
||||
)
|
||||
|
||||
# 4) Respect the X-Forwarded-* headers from reverse proxy (Traefik, Nginx)
|
||||
# 2) Respect the X-Forwarded-* headers from Traefik
|
||||
app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
|
||||
|
||||
# 5) Restrict valid hosts to prevent Host header attacks
|
||||
app.add_middleware(
|
||||
TrustedHostMiddleware,
|
||||
allowed_hosts=[settings.external_hostname, "localhost", "127.0.0.1"],
|
||||
)
|
||||
# 3) (Optional but recommended) Restrict valid hosts:
|
||||
app.add_middleware(TrustedHostMiddleware, allowed_hosts=[
|
||||
settings.external_hostname,
|
||||
"localhost",
|
||||
"127.0.0.1"
|
||||
])
|
||||
|
||||
# Mount the static files directory
|
||||
static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static"
|
||||
@@ -241,90 +55,73 @@ if os.path.exists(static_dir):
|
||||
else:
|
||||
print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.")
|
||||
|
||||
# Mount the built MkDocs developer documentation at /developer-docs/
|
||||
# These docs target administrators and developers, not end-users.
|
||||
# The user-facing Help Center is served by the /help view instead.
|
||||
# The docs are pre-built into docs_build/ during the Docker image build.
|
||||
# When running locally, run `mkdocs build` from the repo root first.
|
||||
docs_build_dir = pathlib.Path(__file__).parents[1] / "docs_build"
|
||||
if os.path.exists(docs_build_dir):
|
||||
app.mount("/developer-docs", StaticFiles(directory=str(docs_build_dir), html=True), name="developer_docs")
|
||||
else:
|
||||
print(f"INFO: Developer docs not found at {docs_build_dir}. Run 'mkdocs build' to generate them.")
|
||||
@app.on_event("startup")
|
||||
def on_startup():
|
||||
init_db() # Create tables if they don't exist
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Run startup tasks for the application"""
|
||||
# Force settings dump to log for troubleshooting
|
||||
from app.utils.config_validator import dump_all_settings
|
||||
dump_all_settings()
|
||||
|
||||
# Validate configuration
|
||||
config_issues = check_all_configs()
|
||||
|
||||
# Log overall status
|
||||
has_issues = any(config_issues['email']) or any(len(issues) > 0 for provider, issues in config_issues['storage'].items())
|
||||
if has_issues:
|
||||
logging.warning("Application started with configuration issues - some features may be unavailable")
|
||||
else:
|
||||
logging.info("Application started with valid configuration")
|
||||
|
||||
logging.info("Router organization: Using refactored API routers from app/api/ directory")
|
||||
|
||||
# Initialize notification system
|
||||
init_apprise()
|
||||
|
||||
# Send startup notification
|
||||
notify_startup()
|
||||
|
||||
# Custom exception handlers that return JSON for API routes and HTML for frontend routes
|
||||
# These use their own separate templates instance so that patches in tests on individual
|
||||
# view modules do not affect the error handler rendering.
|
||||
_error_templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
|
||||
_error_templates = Jinja2Templates(directory=str(_error_templates_dir))
|
||||
# Register the i18n translate helper as a global so error templates can use {{ _("key") }}.
|
||||
# Error pages use the default language (English); request-specific locale is not needed here.
|
||||
from app.utils.i18n import SUPPORTED_LANGUAGES as _SUPPORTED_LANGUAGES # noqa: E402
|
||||
from app.utils.i18n import translate as _translate_fn # noqa: E402
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown_event():
|
||||
"""Run shutdown tasks for the application"""
|
||||
logging.info("Application shutting down")
|
||||
|
||||
# Send shutdown notification
|
||||
notify_shutdown()
|
||||
|
||||
_error_templates.env.globals["_"] = lambda key, **kwargs: _translate_fn(key, "en", **kwargs)
|
||||
_error_templates.env.globals["min"] = min
|
||||
_error_templates.env.globals["max"] = max
|
||||
_error_templates.env.globals["supported_languages"] = _SUPPORTED_LANGUAGES
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
"""
|
||||
Handle all HTTPException instances.
|
||||
Returns JSON for API routes, HTML templates for frontend routes.
|
||||
"""
|
||||
# For API routes, always return JSON
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
|
||||
|
||||
# For frontend routes, return appropriate HTML templates
|
||||
# Handle 404 errors with a custom template
|
||||
if exc.status_code == 404:
|
||||
return _error_templates.TemplateResponse(
|
||||
"404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
# For other HTTP errors, we could create specific templates or use a generic one
|
||||
# For now, return a simple error page
|
||||
return _error_templates.TemplateResponse(
|
||||
"404.html", # Reuse 404 template for other errors, or create a generic error template
|
||||
# Custom 404 - we can still return the Jinja2 template, or the old static file:
|
||||
@app.exception_handler(404)
|
||||
async def custom_404_handler(request: Request, exc: HTTPException):
|
||||
# Serve the 404 template directly
|
||||
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
|
||||
return templates.TemplateResponse(
|
||||
"404.html",
|
||||
{"request": request},
|
||||
status_code=exc.status_code,
|
||||
status_code=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(500)
|
||||
async def custom_500_handler(request: Request, exc: Exception):
|
||||
"""
|
||||
Handle internal server errors (500).
|
||||
Returns JSON for API routes, HTML templates for frontend routes.
|
||||
"""
|
||||
# For API routes, return JSON instead of HTML
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={"detail": "Internal server error"},
|
||||
)
|
||||
|
||||
# Serve the 500 template for non-API routes
|
||||
return _error_templates.TemplateResponse(
|
||||
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
|
||||
# Option 1: Keep it simple, just show a funny 500 message:
|
||||
return templates.TemplateResponse(
|
||||
"500.html",
|
||||
{"request": request, "exc": exc},
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
|
||||
@app.get("/test-500")
|
||||
def test_500():
|
||||
raise RuntimeError("Testing forced 500 error!")
|
||||
|
||||
|
||||
# Include the routers
|
||||
app.include_router(frontend_router)
|
||||
app.include_router(files_router) # Explicitly include the files router
|
||||
app.include_router(auth_router)
|
||||
app.include_router(local_auth_router)
|
||||
app.include_router(api_router, prefix="/api")
|
||||
app.include_router(graphql_router, prefix="/graphql")
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Middleware package for DocuElevate."""
|
||||
|
||||
from app.middleware.security_headers import SecurityHeadersMiddleware
|
||||
|
||||
__all__ = ["SecurityHeadersMiddleware"]
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Audit Logging Middleware for DocuElevate.
|
||||
|
||||
This middleware logs all HTTP requests and security-relevant events. Sensitive
|
||||
data (passwords, tokens, secrets, API keys) is masked before logging so that
|
||||
credentials are never recorded in application logs.
|
||||
|
||||
Security-relevant events that receive elevated ``[SECURITY]`` log entries:
|
||||
- Authentication failures (401 Unauthorized)
|
||||
- Authorisation denials (403 Forbidden)
|
||||
- Login / logout endpoint access
|
||||
- Server errors (5xx responses)
|
||||
|
||||
Logged per request:
|
||||
- HTTP method
|
||||
- Request path (query-param values for known sensitive keys are replaced with ``[REDACTED]``)
|
||||
- Response status code
|
||||
- Response time in milliseconds
|
||||
- Client IP address (configurable)
|
||||
- Authenticated username when available
|
||||
|
||||
See SECURITY_AUDIT.md – Infrastructure Security section for background.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Query-parameter / form-field names whose *values* must never appear in logs.
|
||||
# Matching is case-insensitive.
|
||||
_SENSITIVE_PARAM_PATTERN = re.compile(
|
||||
r"^(password|passwd|pwd|secret|token|access_token|refresh_token|"
|
||||
r"api_key|apikey|key|credential|credentials|auth|authorization|"
|
||||
r"client_secret|private_key|session)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# HTTP headers whose values must never appear in logs.
|
||||
_SENSITIVE_HEADERS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
"x-api-key",
|
||||
"x-auth-token",
|
||||
}
|
||||
)
|
||||
|
||||
# Endpoints considered security-sensitive for elevated logging.
|
||||
_AUTH_PATHS = frozenset({"/auth", "/login", "/logout", "/oauth-login", "/oauth-callback"})
|
||||
|
||||
|
||||
def mask_query_string(query_string: str) -> str:
|
||||
"""
|
||||
Replace values of sensitive query parameters with ``[REDACTED]``.
|
||||
|
||||
Args:
|
||||
query_string: Raw URL query string (e.g. ``"user=alice&password=secret"``).
|
||||
|
||||
Returns:
|
||||
Query string with sensitive values replaced.
|
||||
"""
|
||||
if not query_string:
|
||||
return query_string
|
||||
|
||||
parts = []
|
||||
for pair in query_string.split("&"):
|
||||
if "=" in pair:
|
||||
name, _, value = pair.partition("=")
|
||||
if _SENSITIVE_PARAM_PATTERN.match(name):
|
||||
parts.append(f"{name}=[REDACTED]")
|
||||
else:
|
||||
parts.append(pair)
|
||||
else:
|
||||
parts.append(pair)
|
||||
return "&".join(parts)
|
||||
|
||||
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""
|
||||
Extract the real client IP, honouring X-Forwarded-For when present.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
Client IP address string.
|
||||
"""
|
||||
forwarded_for = request.headers.get("x-forwarded-for")
|
||||
if forwarded_for:
|
||||
# Take only the first (leftmost) address – that is the original client.
|
||||
return forwarded_for.split(",")[0].strip()
|
||||
if request.client:
|
||||
return request.client.host
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_username(request: Request) -> str:
|
||||
"""
|
||||
Extract the authenticated username from the session, if available.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
Username string, or ``"anonymous"`` when not authenticated.
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user") if hasattr(request, "session") else None
|
||||
except Exception:
|
||||
user = None
|
||||
|
||||
if not user:
|
||||
return "anonymous"
|
||||
|
||||
if isinstance(user, dict):
|
||||
return (
|
||||
user.get("preferred_username")
|
||||
or user.get("username")
|
||||
or user.get("email")
|
||||
or user.get("id")
|
||||
or "authenticated"
|
||||
)
|
||||
return str(user)
|
||||
|
||||
|
||||
class AuditLogMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware to log HTTP requests and security-relevant events.
|
||||
|
||||
Each request produces a single ``INFO``-level audit log line.
|
||||
Requests that result in 401/403 responses, or that target
|
||||
authentication endpoints, additionally produce a ``WARNING``-level
|
||||
security event line. Server errors (5xx) produce an ``ERROR``-level
|
||||
security event line.
|
||||
|
||||
Configuration is read from the application settings object passed at
|
||||
construction time via the ``config`` keyword argument.
|
||||
"""
|
||||
|
||||
def __init__(self, app, config) -> None:
|
||||
"""
|
||||
Initialise the audit-log middleware.
|
||||
|
||||
Args:
|
||||
app: FastAPI / ASGI application instance.
|
||||
config: Application settings object (must expose
|
||||
``audit_logging_enabled`` and
|
||||
``audit_log_include_client_ip`` boolean attributes).
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.enabled = config.audit_logging_enabled
|
||||
self.include_ip = config.audit_log_include_client_ip
|
||||
|
||||
if self.enabled:
|
||||
logger.info(f"Audit logging middleware enabled (include_client_ip={self.include_ip})")
|
||||
else:
|
||||
logger.info("Audit logging middleware disabled")
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""
|
||||
Process the request, call the next handler, then emit audit log entries.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware or route handler in the chain.
|
||||
|
||||
Returns:
|
||||
HTTP response (unmodified).
|
||||
"""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
start_time = time.monotonic()
|
||||
response = await call_next(request)
|
||||
duration_ms = int((time.monotonic() - start_time) * 1000)
|
||||
|
||||
self._log_request(request, response.status_code, duration_ms)
|
||||
return response
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_path_with_masked_query(self, request: Request) -> str:
|
||||
"""Return the request path with sensitive query-param values masked."""
|
||||
path = request.url.path
|
||||
raw_query = request.url.query
|
||||
if raw_query:
|
||||
masked = mask_query_string(raw_query)
|
||||
return f"{path}?{masked}"
|
||||
return path
|
||||
|
||||
def _log_request(self, request: Request, status_code: int, duration_ms: int) -> None:
|
||||
"""
|
||||
Emit audit log entries for a completed request.
|
||||
|
||||
Args:
|
||||
request: The HTTP request object.
|
||||
status_code: HTTP response status code.
|
||||
duration_ms: Total request processing time in milliseconds.
|
||||
"""
|
||||
method = request.method
|
||||
path = self._build_path_with_masked_query(request)
|
||||
username = get_username(request)
|
||||
ip_part = f" - {get_client_ip(request)}" if self.include_ip else ""
|
||||
|
||||
# Core request log line (always INFO).
|
||||
logger.info(f"[AUDIT] {method} {path} {status_code} {duration_ms}ms{ip_part} - {username}")
|
||||
|
||||
# Security-event log lines for noteworthy conditions.
|
||||
self._log_security_event(method, path, status_code, username, ip_part)
|
||||
|
||||
def _log_security_event(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
status_code: int,
|
||||
username: str,
|
||||
ip_part: str,
|
||||
) -> None:
|
||||
"""
|
||||
Emit an additional security-event log line when warranted.
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, …).
|
||||
path: Sanitised request path (with masked query params).
|
||||
status_code: HTTP response status code.
|
||||
username: Authenticated username or ``"anonymous"``.
|
||||
ip_part: Pre-formatted IP string (may be empty string).
|
||||
"""
|
||||
base_path = path.split("?", maxsplit=1)[0]
|
||||
|
||||
if status_code == 401:
|
||||
logger.warning(f"[SECURITY] AUTH_FAILURE {method} {path} 401{ip_part} - {username}")
|
||||
elif status_code == 403:
|
||||
logger.warning(f"[SECURITY] ACCESS_DENIED {method} {path} 403{ip_part} - {username}")
|
||||
elif base_path in _AUTH_PATHS and method == "POST":
|
||||
# Login attempts (successful or not) are always noted.
|
||||
logger.info(f"[SECURITY] AUTH_ATTEMPT {method} {path} {status_code}{ip_part} - {username}")
|
||||
elif status_code >= 500:
|
||||
logger.error(f"[SECURITY] SERVER_ERROR {method} {path} {status_code}{ip_part} - {username}")
|
||||
@@ -1,183 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
CSRF Protection Middleware for DocuElevate.
|
||||
|
||||
This middleware implements Cross-Site Request Forgery (CSRF) protection for all
|
||||
state-changing HTTP operations (POST, PUT, DELETE, PATCH).
|
||||
|
||||
How it works:
|
||||
- A cryptographically secure token is generated per session and stored in the session.
|
||||
- The token is attached to ``request.state.csrf_token`` so Jinja2 templates can render it.
|
||||
- For every state-changing request the middleware validates the submitted token by
|
||||
checking (in order):
|
||||
1. The ``X-CSRF-Token`` HTTP request header (used by AJAX / fetch calls).
|
||||
2. The ``csrf_token`` field in ``application/x-www-form-urlencoded`` bodies
|
||||
(used by traditional HTML forms such as the login form).
|
||||
Multipart file-upload requests must always supply the token via the header.
|
||||
- Validation is only enforced when ``AUTH_ENABLED=True``. When authentication is
|
||||
disabled (development / single-user mode) the middleware is a no-op.
|
||||
|
||||
Exempt paths (CSRF is not checked even for state-changing methods):
|
||||
- ``/oauth-callback`` – OAuth 2.0 callback; protected by the ``state`` parameter.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse, RedirectResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# HTTP methods that change server state and therefore require a valid CSRF token.
|
||||
CSRF_PROTECTED_METHODS = {"POST", "PUT", "DELETE", "PATCH"}
|
||||
|
||||
# Paths that must never be CSRF-checked (e.g. OAuth flow endpoints that carry
|
||||
# their own replay-protection mechanism).
|
||||
CSRF_EXEMPT_PATHS = {
|
||||
"/oauth-callback",
|
||||
}
|
||||
|
||||
|
||||
class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware that generates and validates CSRF tokens for state-changing requests.
|
||||
|
||||
Token lifecycle
|
||||
---------------
|
||||
1. On the first request for a session a 64-character hex token is created with
|
||||
``secrets.token_hex(32)`` and stored in ``request.session["csrf_token"]``.
|
||||
2. On every subsequent request the existing token is read from the session.
|
||||
3. The token is always attached to ``request.state.csrf_token`` so that
|
||||
Jinja2 templates (and response processors) can embed it.
|
||||
|
||||
Validation
|
||||
----------
|
||||
For ``POST``, ``PUT``, ``DELETE``, and ``PATCH`` requests the middleware
|
||||
checks whether the submitted token matches the session token using a
|
||||
constant-time comparison (``secrets.compare_digest``) to prevent timing
|
||||
attacks.
|
||||
|
||||
Failure response
|
||||
----------------
|
||||
- API routes (``/api/*``): HTTP 403 JSON response.
|
||||
- All other routes: HTTP 302 redirect to ``/login?error=…``.
|
||||
"""
|
||||
|
||||
def __init__(self, app, config):
|
||||
"""
|
||||
Initialise the middleware.
|
||||
|
||||
Args:
|
||||
app: The ASGI application.
|
||||
config: Application settings object (``app.config.Settings``).
|
||||
``config.auth_enabled`` controls whether CSRF enforcement is active.
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.config = config
|
||||
self.enabled = config.auth_enabled
|
||||
if self.enabled:
|
||||
logger.info("CSRF protection middleware enabled")
|
||||
else:
|
||||
logger.info("CSRF protection middleware disabled (AUTH_ENABLED=False)")
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""
|
||||
Process the request: generate/attach the token and validate it when required.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware or route handler in the ASGI chain.
|
||||
|
||||
Returns:
|
||||
HTTP response, or an error response when CSRF validation fails.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
# Generate or retrieve the per-session CSRF token.
|
||||
csrf_token = request.session.get("csrf_token")
|
||||
if not csrf_token:
|
||||
csrf_token = secrets.token_hex(32)
|
||||
request.session["csrf_token"] = csrf_token
|
||||
|
||||
# Attach token to request state so templates and route handlers can access it.
|
||||
request.state.csrf_token = csrf_token
|
||||
|
||||
# Validate for state-changing methods on non-exempt paths.
|
||||
if request.method in CSRF_PROTECTED_METHODS and request.url.path not in CSRF_EXEMPT_PATHS:
|
||||
# Bearer-authenticated requests (API tokens) are exempt from CSRF
|
||||
# because the token itself acts as proof of intent — it cannot be
|
||||
# injected by a cross-site request from a browser.
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if auth_header.startswith("Bearer "):
|
||||
return await call_next(request)
|
||||
|
||||
submitted_token = await self._get_submitted_token(request)
|
||||
if not submitted_token or not secrets.compare_digest(csrf_token, submitted_token):
|
||||
logger.warning(f"[SECURITY] CSRF_VALIDATION_FAILED method={request.method} path={request.url.path}")
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={"detail": "CSRF token missing or invalid"},
|
||||
)
|
||||
return RedirectResponse(url="/login?error=Invalid+request", status_code=302)
|
||||
|
||||
return await call_next(request)
|
||||
|
||||
@staticmethod
|
||||
async def _get_submitted_token(request: Request) -> str | None:
|
||||
"""
|
||||
Extract the CSRF token submitted by the client.
|
||||
|
||||
Checks (in priority order):
|
||||
1. ``X-CSRF-Token`` request header – preferred for AJAX / fetch requests.
|
||||
2. ``csrf_token`` form field in ``application/x-www-form-urlencoded`` bodies –
|
||||
used by plain HTML forms (e.g. the login form).
|
||||
|
||||
Multipart bodies (file uploads) are intentionally not parsed here to avoid
|
||||
buffering large uploads in middleware; those endpoints must send the token
|
||||
via the header instead.
|
||||
|
||||
Args:
|
||||
request: The incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
The submitted CSRF token string, or ``None`` if not found.
|
||||
"""
|
||||
# 1. Check the request header (AJAX / fetch).
|
||||
token = request.headers.get("X-CSRF-Token")
|
||||
if token:
|
||||
return token
|
||||
|
||||
# 2. For URL-encoded form bodies only (plain HTML form submissions).
|
||||
content_type = request.headers.get("content-type", "")
|
||||
logger.debug("CSRF: content_type=%r method=%s path=%s", content_type, request.method, request.url.path)
|
||||
if "application/x-www-form-urlencoded" in content_type:
|
||||
try:
|
||||
# Cache the raw body bytes before parsing the form. Starlette's
|
||||
# BaseHTTPMiddleware uses _CachedRequest.wrapped_receive to relay
|
||||
# the body to downstream handlers. When form() is called it
|
||||
# internally uses stream() which sets _stream_consumed=True but
|
||||
# does NOT populate _body. wrapped_receive then sees a consumed
|
||||
# stream and forwards an empty body, so the auth endpoint gets
|
||||
# form_keys=[]. Calling body() first stores the bytes in _body;
|
||||
# wrapped_receive detects this and replays the real body to any
|
||||
# downstream handler (e.g. the /auth endpoint).
|
||||
await request.body()
|
||||
form = await request.form()
|
||||
token = form.get("csrf_token")
|
||||
logger.debug(
|
||||
"CSRF: form_keys=%s csrf_token_present=%s",
|
||||
list(form.keys()),
|
||||
bool(token),
|
||||
)
|
||||
if token:
|
||||
return str(token)
|
||||
except Exception as exc:
|
||||
logger.debug(f"CSRF: could not parse form body: {exc}")
|
||||
|
||||
return None
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Rate Limiting Middleware for DocuElevate.
|
||||
|
||||
This middleware provides rate limiting capabilities to protect API endpoints from abuse
|
||||
and DoS attacks. It uses SlowAPI with Redis backend for distributed rate limiting.
|
||||
|
||||
Key features:
|
||||
- Per-IP rate limiting by default
|
||||
- Per-user rate limiting for authenticated endpoints
|
||||
- Configurable global and per-endpoint limits
|
||||
- Redis-backed for distributed deployments
|
||||
- Fallback to in-memory for development
|
||||
|
||||
See docs/ConfigurationGuide.md and docs/API.md for configuration and usage.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_identifier(request: Request) -> str:
|
||||
"""
|
||||
Get unique identifier for rate limiting.
|
||||
|
||||
Uses authenticated user ID if available, otherwise falls back to IP address.
|
||||
This provides better rate limiting for authenticated users and prevents
|
||||
IP-based bypassing for authenticated endpoints.
|
||||
|
||||
Args:
|
||||
request: FastAPI request object
|
||||
|
||||
Returns:
|
||||
Unique identifier string for rate limiting
|
||||
"""
|
||||
# Check if user is authenticated (from session)
|
||||
if hasattr(request, "session") and request.session.get("user"):
|
||||
user = request.session.get("user")
|
||||
# Use username or user_id as identifier
|
||||
if isinstance(user, dict):
|
||||
identifier = user.get("username") or user.get("user_id") or user.get("id")
|
||||
if identifier:
|
||||
logger.debug(f"Rate limiting by user: {identifier}")
|
||||
return f"user:{identifier}"
|
||||
|
||||
# Fall back to IP address for unauthenticated requests
|
||||
ip = get_remote_address(request)
|
||||
logger.debug(f"Rate limiting by IP: {ip}")
|
||||
return ip
|
||||
|
||||
|
||||
def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
|
||||
"""
|
||||
Create and configure the rate limiter.
|
||||
|
||||
Args:
|
||||
redis_url: Redis connection URL for distributed rate limiting
|
||||
enabled: Whether rate limiting is enabled (default: True)
|
||||
|
||||
Returns:
|
||||
Configured Limiter instance
|
||||
"""
|
||||
if not enabled:
|
||||
logger.info("Rate limiting is disabled")
|
||||
# Return a limiter with very high limits when disabled
|
||||
return Limiter(
|
||||
key_func=get_identifier,
|
||||
default_limits=["10000/minute"], # Effectively unlimited
|
||||
enabled=False,
|
||||
)
|
||||
|
||||
# Use Redis if available, otherwise fall back to in-memory
|
||||
storage_uri = redis_url if redis_url else "memory://"
|
||||
|
||||
if redis_url:
|
||||
logger.info(f"Rate limiting enabled with Redis backend: {redis_url}")
|
||||
else:
|
||||
logger.warning(
|
||||
"Rate limiting using in-memory storage (not suitable for production with multiple workers). "
|
||||
"Configure REDIS_URL for distributed rate limiting."
|
||||
)
|
||||
|
||||
# Create limiter with default limits
|
||||
# Default: 100 requests per minute per IP/user
|
||||
limiter = Limiter(
|
||||
key_func=get_identifier,
|
||||
default_limits=["100/minute"],
|
||||
storage_uri=storage_uri,
|
||||
strategy="fixed-window", # Can be: fixed-window, moving-window, or fixed-window-elastic-expiry
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
logger.info("Rate limiter initialized successfully")
|
||||
return limiter
|
||||
|
||||
|
||||
def get_rate_limit_exceeded_handler() -> Callable:
|
||||
"""
|
||||
Get the rate limit exceeded exception handler.
|
||||
|
||||
Returns a handler that provides user-friendly 429 responses with
|
||||
Retry-After header when rate limit is exceeded.
|
||||
|
||||
Returns:
|
||||
Exception handler function
|
||||
"""
|
||||
return _rate_limit_exceeded_handler
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user