gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: merge main branch and renumber migration 027→037

Browse Source 
Resolve 3 merge conflicts and renumber the automation_hooks migration
to follow main's migration chain (036_add_document_translation_fields).

Conflicts resolved:
- app/api/__init__.py: add automation_router alongside main's new routers
- app/utils/settings_service.py: add automation_hooks_enabled alongside compliance_enabled
- tests/conftest.py: add AutomationHook alongside AuditLog/ComplianceTemplate imports

Migration renumbered:
- 027_add_automation_hooks → 037_add_automation_hooks
- down_revision: 026_add_scheduled_jobs → 036_add_document_translation_fields

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-16 22:40:15 +00:00
parent 6a83d51d88
commit 204000aabc
318 changed files with 317516 additions and 2858 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+91
View File
@@ -0,0 +1,91 @@
# =============================================================================
# Docker build context exclusions
# Reducing the build context speeds up builds and prevents unnecessary cache
# invalidation when unrelated files change.
# =============================================================================
# ── Version control ──────────────────────────────────────────────────────────
.git
# ── GitHub / CI tooling ──────────────────────────────────────────────────────
.github
# ── IDE / local dev ──────────────────────────────────────────────────────────
.vscode
.jules
# ── Pre-commit / linting config (not needed at runtime) ──────────────────────
.pre-commit-config.yaml
pyproject.toml
codecov.yml
crowdin.yml
# ── Test suite ───────────────────────────────────────────────────────────────
tests/
requirements-dev.txt
coverage.json
COVERAGE_REPORT.md
.coverage
htmlcov/
.pytest_cache/
junit.xml
coverage.xml
# ── Mobile app / browser extension / legacy placeholder ─────────────────────
# backend/ is an empty placeholder directory not part of the Python application
mobile/
browser-extension/
backend/
# ── Helm charts ──────────────────────────────────────────────────────────────
helm/
# ── Scripts (run before Docker build, output files are COPYd separately) ─────
scripts/
# ── Benchmark and one-off utility scripts ────────────────────────────────────
benchmark_*.py
fix_test*.py
run_fast_tests.sh
# ── Root-level Markdown files (docs/ is kept for docs-builder stage) ─────────
# Note: *.md only matches files at the root level, not inside subdirectories
*.md
# ── Python bytecode / compiled artifacts ─────────────────────────────────────
__pycache__/
*.pyc
*.pyo
*.pyd
*.so
*.egg
*.egg-info/
# ── Virtual environments ──────────────────────────────────────────────────────
.venv/
venv/
env/
# ── Environment / secret files ───────────────────────────────────────────────
.env
.env.local
.env.*.local
# ── Runtime state files ───────────────────────────────────────────────────────
*.log
celerybeat-schedule
celerybeat.pid
# ── Build artifacts ───────────────────────────────────────────────────────────
build/
dist/
.cache/
.mypy_cache/
.ruff_cache/
site/
docs_build/
# ── Editor temp files ─────────────────────────────────────────────────────────
*.swp
*.swo
*~
.env.demo
+104
View File
@@ -5,6 +5,25 @@ REDIS_URL=redis://redis:6379/0
EXTERNAL_HOSTNAME=docuelevate.example.com
GOTENBERG_URL=http://gotenberg:3000
ALLOW_FILE_DELETE=true # Allow deletion of file records
COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2)
# **Logging**
# LOG_LEVEL controls the Python root-logger level.
# Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO).
# When DEBUG=true and LOG_LEVEL is not set, the level is automatically lowered to DEBUG.
# LOG_LEVEL=INFO
# DEBUG=false
# Log output format: "text" (human-readable, default) or "json" (structured JSON lines).
# Use "json" when shipping logs to Grafana Loki, Splunk, ELK, Datadog, or any SIEM.
# LOG_FORMAT=text
# Forward application logs to a syslog receiver (in addition to stdout).
# Useful for traditional (non-container) deployments and centralised SIEM ingestion.
# LOG_SYSLOG_ENABLED=false
# LOG_SYSLOG_HOST=localhost
# LOG_SYSLOG_PORT=514
# LOG_SYSLOG_PROTOCOL=udp # udp | tcp
# **UI / Appearance**
# Default colour scheme: system (follow OS), light, or dark
@@ -96,6 +115,22 @@ MAX_UPLOAD_SIZE=1073741824
# Allowed request headers (use * to allow all)
# CORS_ALLOWED_HEADERS=*
# **Audit Logging & SIEM Integration** (see docs/ConfigurationGuide.md#audit-logging)
# Enable HTTP request audit logging middleware
AUDIT_LOGGING_ENABLED=true
# Include client IP in audit log entries (disable for GDPR-sensitive deployments)
AUDIT_LOG_INCLUDE_CLIENT_IP=true
# Forward audit events to an external SIEM system (Syslog, Splunk, Logstash, Grafana, etc.)
# AUDIT_SIEM_ENABLED=false
# AUDIT_SIEM_TRANSPORT=syslog # syslog | http
# AUDIT_SIEM_SYSLOG_HOST=localhost
# AUDIT_SIEM_SYSLOG_PORT=514
# AUDIT_SIEM_SYSLOG_PROTOCOL=udp # udp | tcp
# AUDIT_SIEM_HTTP_URL= # e.g. https://splunk:8088/services/collector/event
# AUDIT_SIEM_HTTP_TOKEN= # Bearer / HEC token
# AUDIT_SIEM_HTTP_CUSTOM_HEADERS= # Comma-separated Key:Value pairs
# **Rate Limiting** (see SECURITY_AUDIT.md and docs/API.md)
# Protects against DoS attacks and API abuse by limiting request rates per IP/user
# Enabled by default - highly recommended for production
@@ -158,6 +193,33 @@ AUTHENTIK_CLIENT_SECRET=<yourAuthentikClientSecret>
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docuelevate/.well-known/openid-configuration>
OAUTH_PROVIDER_NAME="Authentik SSO"
# **Social Login Providers**
# Enable one or more social login providers to let users sign in with existing accounts.
# Each provider requires separate OAuth credentials. See docs/SocialLoginSetup.md for details.
# Google Sign-In (https://console.cloud.google.com/apis/credentials)
# SOCIAL_AUTH_GOOGLE_ENABLED=false
# SOCIAL_AUTH_GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com
# SOCIAL_AUTH_GOOGLE_CLIENT_SECRET=your-google-client-secret
# Microsoft Sign-In / Azure AD (https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps)
# SOCIAL_AUTH_MICROSOFT_ENABLED=false
# SOCIAL_AUTH_MICROSOFT_CLIENT_ID=your-microsoft-application-id
# SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=your-microsoft-client-secret
# SOCIAL_AUTH_MICROSOFT_TENANT=common # common | organizations | consumers | <tenant-id>
# Apple Sign-In (https://developer.apple.com/account/resources)
# SOCIAL_AUTH_APPLE_ENABLED=false
# SOCIAL_AUTH_APPLE_CLIENT_ID=com.example.docuelevate
# SOCIAL_AUTH_APPLE_TEAM_ID=ABCDE12345
# SOCIAL_AUTH_APPLE_KEY_ID=FGHIJ67890
# SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
# Dropbox Sign-In (https://www.dropbox.com/developers/apps)
# SOCIAL_AUTH_DROPBOX_ENABLED=false
# SOCIAL_AUTH_DROPBOX_CLIENT_ID=your-dropbox-app-key
# SOCIAL_AUTH_DROPBOX_CLIENT_SECRET=your-dropbox-app-secret
# **AI/ML Services**
# Select your AI provider: openai | azure | anthropic | gemini | ollama | openrouter | portkey | litellm
AI_PROVIDER=openai
@@ -196,6 +258,13 @@ OPENAI_MODEL=gpt-4o-mini
# AZURE_OPENAI_API_VERSION=2024-02-01
# AI_MODEL=gpt-4o # deployment name in Azure
# **Document Translation**
# After processing, documents whose detected language differs from the default
# target language are automatically translated. Only the original and this
# default-language version are persisted; other translations are on-the-fly.
# Users can override this in their profile settings.
# DEFAULT_DOCUMENT_LANGUAGE=en
# Azure Document Intelligence (OCR – separate from AI provider above)
# **Email Settings (shared SMTP – password reset, verification, and system notifications)**
EMAIL_HOST=smtp.example.com
@@ -210,6 +279,7 @@ EMAIL_SENDER=DocuElevate System <docuelevate@example.com>
# These settings are intentionally separate from the shared EMAIL_* settings above.
# Configuring EMAIL_HOST for password reset / notifications does NOT automatically
# enable the email destination – you must set DEST_EMAIL_HOST to activate it.
# DEST_EMAIL_ENABLED=true # Set to false to disable email delivery without removing credentials
DEST_EMAIL_HOST=smtp.example.com
DEST_EMAIL_PORT=587
DEST_EMAIL_USERNAME=docuelevate@example.com
@@ -294,8 +364,15 @@ IMAP2_DELETE_AFTER_PROCESS=false
# Use for pre-production instances that share a mailbox with production.
IMAP_READONLY_MODE=false
# Controls which attachment types are ingested from IMAP emails.
# 'documents_only' (default) – PDFs and office files only; images are skipped.
# 'all' – all supported file types including images.
# Per-user IMAP accounts can override this global default.
IMAP_ATTACHMENT_FILTER=documents_only
# **Storage/Document Services**
# Amazon S3
# S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials
AWS_REGION=us-east-1
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
@@ -305,12 +382,14 @@ S3_STORAGE_CLASS=STANDARD
S3_ACL=private
# NextCloud
# NEXTCLOUD_ENABLED=true # Set to false to disable NextCloud uploads without removing credentials
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
# Paperless-ngx
# PAPERLESS_ENABLED=true # Set to false to disable Paperless uploads without removing credentials
PAPERLESS_HOST=https://paperless.example.com
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
@@ -327,12 +406,14 @@ PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
# PAPERLESS_CUSTOM_FIELDS_MAPPING=
# Dropbox
# DROPBOX_ENABLED=true # Set to false to disable Dropbox uploads without removing credentials
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
DROPBOX_FOLDER="/Documents/Uploads"
# Google Drive
# GOOGLE_DRIVE_ENABLED=true # Set to false to disable Google Drive uploads without removing credentials
# Service Account Method:
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
@@ -345,6 +426,7 @@ GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret # Required for OAuth method
GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token # Required for OAuth method
# OneDrive
# ONEDRIVE_ENABLED=true # Set to false to disable OneDrive uploads without removing credentials
ONEDRIVE_CLIENT_ID=your-client-id
ONEDRIVE_CLIENT_SECRET=your-client-secret
ONEDRIVE_TENANT_ID=common
@@ -352,6 +434,7 @@ ONEDRIVE_REFRESH_TOKEN=your-refresh-token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
# WebDAV
# WEBDAV_ENABLED=true # Set to false to disable WebDAV uploads without removing credentials
WEBDAV_URL=https://webdav.example.com/path
WEBDAV_USERNAME=webdav_user
WEBDAV_PASSWORD=your_secure_webdav_password
@@ -359,6 +442,7 @@ WEBDAV_FOLDER=/Documents/Uploads
WEBDAV_VERIFY_SSL=True
# FTP
# FTP_ENABLED=true # Set to false to disable FTP uploads without removing credentials
# Security Note: FTP_USE_TLS=True is strongly recommended for secure connections
# Set FTP_ALLOW_PLAINTEXT=False in production to prevent unencrypted FTP
FTP_HOST=ftp.example.com
@@ -370,6 +454,7 @@ FTP_USE_TLS=True
FTP_ALLOW_PLAINTEXT=True
# SFTP
# SFTP_ENABLED=true # Set to false to disable SFTP uploads without removing credentials
# Security Note: Host key verification is enabled by default (False)
# Only set to True in development/testing environments if needed
# When false, configure SSH known_hosts for proper host key verification
@@ -382,6 +467,16 @@ SFTP_PASSWORD=your_secure_sftp_password
SFTP_FOLDER=/Documents/Uploads
SFTP_DISABLE_HOST_KEY_VERIFICATION=False # Default is False (secure); set to True only for testing
# iCloud Drive
# ICLOUD_ENABLED=true # Set to false to disable iCloud uploads without removing credentials
# Requires an Apple ID with iCloud Drive enabled.
# For accounts with two-factor authentication (most accounts), generate an
# app-specific password at https://appleid.apple.com/account/manage
ICLOUD_USERNAME=your_apple_id@example.com
ICLOUD_PASSWORD=your-app-specific-password
ICLOUD_FOLDER=Documents/Uploads
# ICLOUD_COOKIE_DIRECTORY=/path/to/cookie/dir # Optional: defaults to ~/.pyicloud
# **HTTP Request Settings**
# Timeout for HTTP requests - set higher to handle large PDF files (up to 1GB)
HTTP_REQUEST_TIMEOUT=120 # Timeout in seconds (default: 120 for large file operations)
@@ -510,3 +605,12 @@ EMBEDDING_MAX_TOKENS=8000
# Attach PII (IP addresses, user agents) to Sentry events.
# Disable (default) to stay GDPR/CCPA compliant.
# SENTRY_SEND_DEFAULT_PII=false
# **Mobile App – Push Notifications**
# Push notifications are delivered via Expo's push notification service
# (https://expo.dev/notifications) which routes to APNs (iOS) and FCM (Android).
# No additional credentials are required on the server side.
# The mobile app registers its Expo push token via POST /api/mobile/register-device.
#
# To use native FCM/APNs directly (without Expo relay), replace the
# send_expo_push_notification function in app/utils/push_notification.py.
.github/copilot-instructions.md
+7
View File
@@ -163,6 +163,13 @@ pytest --tb=short -q
- Keep JavaScript minimal - prefer server-side rendering
- Follow existing template structure and patterns
### Internationalization (i18n) & Localization (l10n)
- **Always** use the `_("key")` helper in Jinja2 templates and `translate("key", locale)` in Python for every user-visible string — never hardcode UI text.
- **Only add new keys to `frontend/translations/en.json`** — that is the one and only file you must touch when introducing new UI strings.
- Do **not** manually edit any non-English translation file (`de.json`, `fr.json`, etc.). An external automation script syncs all other language files from `en.json` automatically.
- Key naming convention: `<section>.<descriptor>` in snake_case, e.g. `language.search_placeholder`, `nav.help`, `common.cancel`.
- The `test_all_languages_have_same_keys` check has been intentionally removed — key completeness across locales is enforced by the external sync script, not by the test suite.
### Testing
- Write tests in `tests/` directory, mirroring `app/` structure
- Use pytest markers: `@pytest.mark.unit`, `@pytest.mark.integration`, etc.
.github/workflows/ci.yml
+1
View File
@@ -17,6 +17,7 @@ concurrency:
env:
IMAGE_NAME: christianlouis/docuelevate
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
# ══════════════════════════════════════════════════════════════════════════
.github/workflows/codeql.yml
+3
View File
@@ -8,6 +8,9 @@ on:
schedule:
- cron: '37 1 * * 1'
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
analyze:
name: Analyze (${{ matrix.language }})
.github/workflows/release.yml
+3
View File
@@ -12,6 +12,9 @@ permissions:
pull-requests: write
packages: write
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
release:
name: Semantic Release
.github/workflows/ruff-auto-fix.yml
+3
View File
@@ -16,6 +16,9 @@ permissions:
contents: write
pull-requests: write
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
ruff-auto-fix:
name: Auto-fix Ruff Issues
.jules/sentinel.md
+4
View File
@@ -0,0 +1,4 @@
## 2024-05-24 - SSRF in WebDAV connection test
**Vulnerability:** The `_test_webdav_connection` function had a custom SSRF check that failed to resolve DNS names, allowing attackers to bypass the check by providing a domain that resolves to an internal IP (e.g., `127.0.0.1`).
**Learning:** DNS resolution is required for robust SSRF protection when validating URLs provided by users.
**Prevention:** Use a centralized `is_private_ip` function (now in `app/utils/network.py`) that resolves the hostname to its IPs and checks if any are private.
BUILD_DATE
+1 -1
View File
@@ -1 +1 @@
2026-03-09T23:00:57Z
2026-03-16T21:40:03Z
CHANGELOG.md
+1408
View File
File diff suppressed because it is too large Load Diff
Dockerfile
+37 -19
View File
@@ -1,14 +1,34 @@
# Use multi-stage build for a smaller final image
FROM python:3.14.1 AS builder
# syntax=docker/dockerfile:1
WORKDIR /app
# ── Stage 1: Python dependency builder ──────────────────────────────────────
# Use the same slim variant as the runtime to keep Python versions in sync.
# build-essential + libffi-dev cover the few packages (e.g. cryptography) that
# need a C compiler; they are discarded after this stage.
FROM python:3.14.3-slim AS builder
# Copy requirements first for better layer caching
COPY requirements.txt /app/
RUN pip install --no-cache-dir -r requirements.txt
WORKDIR /build
# ── Documentation build stage ───────────────────────────────────────────────
FROM python:3.14.1-slim AS docs-builder
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
libffi-dev \
&& rm -rf /var/lib/apt/lists/*
# Create an isolated virtual environment so only installed packages are copied
# to the runtime image (no pip, setuptools, or other builder artefacts).
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1
COPY requirements.txt /build/
RUN pip install --no-cache-dir -r requirements.txt \
# Remove bytecode and cache to keep the venv lean
&& find /opt/venv -type f -name "*.pyc" -delete \
&& find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
# ── Stage 2: Documentation builder ──────────────────────────────────────────
FROM python:3.14.3-slim AS docs-builder
WORKDIR /docs
@@ -23,14 +43,13 @@ COPY mkdocs.yml /docs/mkdocs.yml
# Build the static documentation site
RUN mkdocs build --config-file /docs/mkdocs.yml --site-dir /docs/docs_build
# Second stage for the actual runtime
# ── Stage 3: Runtime image ───────────────────────────────────────────────────
FROM python:3.14.3-slim
WORKDIR /app
# Copy installed packages from builder stage
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
COPY --from=builder /usr/local/bin /usr/local/bin
# Copy only the pre-built virtual environment from the builder
COPY --from=builder /opt/venv /opt/venv
# Install system-level OCR tools required for local OCR workflows:
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
@@ -62,15 +81,14 @@ COPY ./RUNTIME_INFO /app/RUNTIME_INFO
# Copy the pre-built MkDocs documentation site (served at /help)
COPY --from=docs-builder /docs/docs_build /app/docs_build
# Create runtime_info directory
RUN mkdir -p /app/runtime_info
# Create necessary directories
RUN mkdir -p /workdir
# Create necessary runtime directories in a single layer
RUN mkdir -p /app/runtime_info /workdir
# Set environment variables
ENV PYTHONPATH=/app
ENV PYTHONUNBUFFERED=1
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONPATH=/app \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1
# Expose the port the app runs on
EXPOSE 8000
Dockerfile.local
+35 -13
View File
@@ -1,13 +1,31 @@
# syntax=docker/dockerfile:1
# Local development Dockerfile (avoids CI-only build metadata files)
FROM python:3.14.1 AS builder
WORKDIR /app
# ── Stage 1: Python dependency builder ──────────────────────────────────────
FROM python:3.14.3-slim AS builder
COPY requirements.txt /app/
RUN pip install --no-cache-dir -r requirements.txt
WORKDIR /build
# ── Documentation build stage ───────────────────────────────────────────────
FROM python:3.14.1-slim AS docs-builder
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
libffi-dev \
&& rm -rf /var/lib/apt/lists/*
# Create an isolated virtual environment
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1
COPY requirements.txt /build/
RUN pip install --no-cache-dir -r requirements.txt \
&& find /opt/venv -type f -name "*.pyc" -delete \
&& find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
# ── Stage 2: Documentation builder ──────────────────────────────────────────
FROM python:3.14.3-slim AS docs-builder
WORKDIR /docs
@@ -19,23 +37,25 @@ COPY mkdocs.yml /docs/mkdocs.yml
RUN mkdocs build --config-file /docs/mkdocs.yml --site-dir /docs/docs_build
FROM python:3.14.1-slim
# ── Stage 3: Runtime image ───────────────────────────────────────────────────
FROM python:3.14.3-slim
WORKDIR /app
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=builder /opt/venv /opt/venv
# Install system-level OCR tools required for local OCR workflows:
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
# ghostscript – required by ocrmypdf for PDF/PS operations
# poppler-utils – provides pdfinfo/pdftoppm used by pdf2image
# unpaper – optional deskewing pre-processor used by ocrmypdf
# wget – used by ocr_language_manager to download tessdata files
RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr \
ghostscript \
poppler-utils \
unpaper \
wget \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
COPY ./app /app/app
@@ -53,11 +73,13 @@ COPY --from=docs-builder /docs/docs_build /app/docs_build
RUN echo "local" > /app/GIT_SHA \
&& echo "local" > /app/RUNTIME_INFO
RUN mkdir -p /app/runtime_info
RUN mkdir -p /workdir
# Create necessary runtime directories in a single layer
RUN mkdir -p /app/runtime_info /workdir
ENV PYTHONPATH=/app
ENV PYTHONUNBUFFERED=1
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONPATH=/app \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1
EXPOSE 8000
GIT_SHA
+1 -1
View File
@@ -1 +1 @@
5fd3f06
793b6d3
README.md
+203 -97
View File
@@ -24,121 +24,154 @@
</div>
<div align="center">
<a href="https://www.docuelevate.org"><img src="frontend/static/hero.png" alt="DocuElevate Logo" width="80%" /></a>
<a href="https://www.docuelevate.org"><img src="frontend/static/hero.png" alt="DocuElevate Hero" width="80%" /></a>
</div>
## Overview
DocuElevate automates the handling, extraction, and processing of documents using a variety of services, including:
DocuElevate is an intelligent document processing system that automates the ingestion, OCR, AI-powered metadata extraction, and distribution of documents. It supports a wide range of AI providers, OCR engines, and cloud storage destinations out of the box.
- **AI Provider** (pluggable – OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Portkey, and more) for metadata extraction and text refinement.
- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.
- **Paperless NGX** for document indexing and management.
- **Azure Document Intelligence** for OCR on PDFs.
- **Gotenberg** for file-to-PDF conversions.
- **Authentik** for authentication and user management.
**Key capabilities:**
It is designed for flexibility and configurability through environment variables, making it easily customizable for different workflows. The system can fetch documents from multiple IMAP mailboxes, process them (OCR, metadata extraction, PDF conversion), and store them in the desired destinations.
- **AI-Powered Metadata Extraction** — pluggable AI providers including OpenAI, Anthropic Claude, Google Gemini, Ollama (local), OpenRouter, Portkey, and Azure OpenAI via LiteLLM
- **Multi-Engine OCR** — Azure Document Intelligence, Tesseract, EasyOCR, Mistral OCR, Google Cloud Document AI, and AWS Textract with configurable merge strategies
- **12 Storage Destinations** — Dropbox, Google Drive, OneDrive, Amazon S3, Nextcloud, WebDAV, FTP, SFTP, iCloud Drive, Email (SMTP), Paperless-ngx, and Rclone
- **Multi-Channel Ingestion** — web upload, browser extension, mobile app, CLI, REST API, IMAP email, and watched folders (local, cloud, FTP/SFTP)
- **Processing Pipelines** — customizable multi-step workflows with conditional routing rules
- **Full-Text Search** — powered by Meilisearch for instant document discovery
- **Multi-User with SSO** — local accounts, OAuth2/OIDC (Authentik), and social login (Google, Microsoft, Apple, Dropbox)
The project includes a **UI** for uploading and managing files, and an API documentation page is available at `/docs` (powered by **FastAPI**).
## Documentation Index
- [User Guide](docs/UserGuide.md) - How to use DocuElevate
- [Browser Extension Guide](docs/BrowserExtension.md) - Install and use the browser extension
- [API Documentation](docs/API.md) - API reference
- [Deployment Guide](docs/DeploymentGuide.md) - How to deploy DocuElevate
- [Configuration Guide](docs/ConfigurationGuide.md) - Available configuration options
- [Build Metadata](docs/BuildMetadata.md) - Automated version and build information
- [CI/CD Tools Guide](docs/CIToolsGuide.md) - CI/CD pipeline and tool documentation
- [CI Workflow Guide](docs/CIWorkflow.md) - Detailed workflow documentation
- [Development Guide](CONTRIBUTING.md) - How to contribute to DocuElevate
- [Troubleshooting](docs/Troubleshooting.md) - Common issues and solutions
The project ships with a web UI, a REST + GraphQL API, a CLI tool, a native mobile app (iOS & Android), a browser extension, and Helm charts for Kubernetes deployment.
## Screenshots
<div align="center">
<img src="docs/upload-view.png" alt="DocuElevate Upload Interface" width="80%" />
<p><em>Upload interface for adding new documents</em></p>
<p><em>Upload interface — drag-and-drop file upload with real-time progress</em></p>
<img src="docs/files-view.png" alt="DocuElevate Files View" width="80%" />
<p><em>Files view with processed documents and metadata</em></p>
<p><em>Files view — processed documents with AI-extracted metadata</em></p>
<img src="docs/status-view.png" alt="DocuElevate Status View" width="80%" />
<p><em>Status view — system health and service monitoring</em></p>
</div>
> **Note:** Screenshots may not reflect the very latest UI. For the most current look, visit [docuelevate.org](https://www.docuelevate.org).
## Workflow Process
DocuElevate follows a streamlined document processing workflow:
## Workflow
<div align="center">
<img src="docs/workflow-diagram.png" alt="DocuElevate Workflow" width="90%" />
</div>
### Document Ingestion
Documents enter DocuElevate through four possible channels:
1. **Web Upload**: Users manually upload files via the web interface
2. **Browser Extension**: Send files directly from your browser with one click
3. **Email Attachments**: Automatic polling of configured IMAP mailboxes (supports multiple accounts)
4. **API**: Direct programmatic uploads via the REST API
### Ingestion
Documents enter DocuElevate through multiple channels:
| Channel | Description |
|---------|-------------|
| **Web Upload** | Drag-and-drop interface with real-time progress (up to 1 GB per file) |
| **Browser Extension** | Clip web pages or send files from Chrome, Firefox, or Edge |
| **Mobile App** | Capture documents with the device camera or upload from the photo library |
| **CLI** | Batch uploads and scripted workflows via the `docuelevate` command-line tool |
| **REST API** | Programmatic uploads with full API-token authentication |
| **Email (IMAP)** | Automatic polling of multiple mailboxes with attachment filtering |
| **Watched Folders** | Monitor local paths, FTP, SFTP, S3, Dropbox, Google Drive, OneDrive, Nextcloud, or WebDAV for new files |
### Processing Pipeline
Every document goes through the following steps:
1. **PDF Conversion**: Non-PDF files are converted to PDF format using Gotenberg
2. **OCR Processing**: Azure Document Intelligence extracts text from images/scans
3. **Metadata Extraction**: The configured AI provider analyzes document content to identify:
- Document type (invoice, receipt, contract, etc.)
- Key entities (dates, names, amounts, account numbers)
- Important data points specific to the document type
4. **Enrichment**: Metadata is attached to the document in a structured format
Each document passes through a configurable set of steps:
1. **PDF Conversion** — Non-PDF files are converted using Gotenberg, with optional PDF/A archival conversion
2. **OCR** — Text extraction via one or more OCR engines (Azure, Tesseract, EasyOCR, Mistral, Google Document AI, AWS Textract) with configurable merge strategies
3. **AI Metadata Extraction** — The configured AI provider classifies the document and extracts structured metadata (type, dates, amounts, entities)
4. **Enrichment** — Metadata is embedded into the PDF and stored alongside the document
5. **Embedding Generation** — Vector embeddings for similarity search and duplicate detection
Steps can be customized using **Pipelines** and **Routing Rules** for conditional processing.
### Distribution
Processed documents with their metadata can be automatically sent to:
- **Dropbox**: For cloud storage and sharing
- **Nextcloud**: For self-hosted file storage
- **Google Drive**: For Google Workspace integration
- **Paperless-NGX**: For advanced document management with search capabilities
Users can choose to send documents to any combination of these destinations through configuration settings or manual selection.
Processed documents are distributed to any combination of configured destinations:
| Destination | Type |
|------------|------|
| **Dropbox** | Cloud storage |
| **Google Drive** | Cloud storage |
| **OneDrive** | Cloud storage |
| **Amazon S3** | Object storage |
| **Nextcloud** | Self-hosted cloud |
| **WebDAV** | Protocol-based |
| **FTP / SFTP** | File transfer |
| **iCloud Drive** | Apple cloud |
| **Email (SMTP)** | Send as attachment |
| **Paperless-ngx** | Document management system |
| **Rclone** | 70+ cloud providers via Rclone |
## Features
- **Intuitive File Upload**:
- Drag-and-drop file upload on both Upload and Files pages—upload anywhere on the Files page
- Real-time upload progress with validation
- Support for PDF, Office documents, images, and more (up to 500MB per file)
- **Browser Extension**:
- Send files directly from your browser to DocuElevate with one click
- Compatible with Chrome, Firefox, Edge, and other Chromium-based browsers
- Context menu integration for quick access
- See [Browser Extension Guide](docs/BrowserExtension.md) for installation and usage
- **Document Upload & Storage**:
- Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless
- **OCR Processing (Azure)**:
- Extract text from scanned PDFs using Azure Document Intelligence
- **Metadata Extraction (AI Provider)**:
- Use any supported AI provider (OpenAI, Anthropic, Gemini, Ollama, etc.) to classify, label, or otherwise enrich the text with structured metadata
- **PDF Conversion (Gotenberg)**:
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs
- **Document Management (Paperless NGX)**:
- Store processed documents and metadata in a Paperless NGX instance
- **IMAP Integration**:
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing
- **Authentication**:
- Secure access to the system using **Authentik** for OAuth2-based login
### Document Processing
- **Multi-engine OCR** with quality checks and configurable merge strategies (AI merge, longest, primary)
- **AI metadata extraction** using any supported provider (OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Portkey, Azure OpenAI)
- **PDF conversion** via Gotenberg with optional PDF/A archival format
- **Duplicate detection** — exact (SHA-256) and near-duplicate (content similarity with vector embeddings)
- **Customizable pipelines** — define multi-step processing workflows with conditional routing rules
## Frameworks Used
### Document Management
- **Full-text search** powered by Meilisearch with saved searches
- **File detail view** with metadata, text preview, processing history, and similarity analysis
- **Shared links** for public document access with expiration controls
- **Bulk operations** — reprocess, delete, or reassign documents in batch
- **FastAPI**: High-performance web framework for APIs.
- **Celery**: Task queue for asynchronous processing.
- **Redis**: Message broker and result backend.
- **SQLAlchemy**: ORM for database interactions.
- **Tailwind CSS**: Utility-first CSS framework.
- **Docker**: Containerization for easy deployment.
### Multi-Channel Ingestion
- **Web UI** — drag-and-drop upload with real-time progress
- **Browser extension** — clip web pages or send files from Chrome, Firefox, Edge ([guide](docs/BrowserExtension.md))
- **Mobile app** — iOS and Android with camera capture, push notifications, and SSO ([guide](docs/MobileApp.md))
- **CLI tool** — batch uploads, downloads, search, and API-token management ([guide](docs/CLIGuide.md))
- **REST API & GraphQL** — full programmatic access with Swagger documentation at `/docs`
- **IMAP email** — poll multiple mailboxes with attachment filtering and auto-processing
- **Watched folders** — local filesystem, FTP, SFTP, and cloud storage providers
### Administration
- **Multi-user mode** with per-user document isolation and ownership
- **Subscription & billing** — Stripe integration with configurable plans and quotas
- **Scheduled jobs** — IMAP polling, watched folder scans, automated backups, uptime monitoring
- **Audit logging** with SIEM integration support
- **Compliance templates** — GDPR, HIPAA, SOC 2
- **Admin dashboard** — user management, queue monitoring, credential management, backup/restore
### Authentication & Security
- **Local accounts** with self-service registration and password reset
- **OAuth2/OIDC** via Authentik or any OIDC provider
- **Social login** — Google, Microsoft, Apple, Dropbox
- **API tokens** for CLI, mobile, and automation access
- **Security headers** — HSTS, CSP, X-Frame-Options, X-Content-Type-Options
- **Rate limiting** with configurable per-endpoint controls
### Notifications
- **100+ notification backends** via Apprise — Discord, Telegram, Slack, Microsoft Teams, Email, webhooks, and more
- **Configurable events** — task failures, credential issues, file processed, user signup, payment issues
- **In-app notification inbox** with per-user preferences
- **Webhooks** — push events to external systems with HMAC signature verification and retry
## Tech Stack
| Component | Technology |
|-----------|-----------|
| **Backend** | FastAPI, Celery, Redis, SQLAlchemy, Alembic |
| **Frontend** | Jinja2, Tailwind CSS |
| **Search** | Meilisearch |
| **Mobile** | React Native (Expo) — iOS & Android |
| **AI** | LiteLLM (OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Portkey) |
| **OCR** | Azure Document Intelligence, Tesseract, EasyOCR, Mistral, Google Doc AI, AWS Textract |
| **PDF** | Gotenberg, pypdf |
| **Auth** | Authlib (OAuth2/OIDC), MSAL, social providers |
| **Infrastructure** | Docker, Docker Compose, Helm/Kubernetes |
| **Docs** | MkDocs Material |
## Quick Start
For detailed installation and deployment instructions, please refer to the [Deployment Guide](docs/DeploymentGuide.md).
For detailed installation and deployment instructions, see the [Deployment Guide](docs/DeploymentGuide.md).
```bash
# Clone the repository
@@ -147,20 +180,96 @@ cd DocuElevate
# Configure environment variables
cp .env.demo .env
# Edit .env with your settings
# Edit .env with your settings (see Configuration Guide for all options)
# Run with Docker Compose
docker-compose up -d
docker compose up -d
```
The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
The web UI is available at **`http://localhost:8000`** and the interactive API documentation at **`http://localhost:8000/docs`**.
### Kubernetes / Helm
```bash
helm repo add docuelevate https://christianlouis.github.io/DocuElevate
helm install docuelevate docuelevate/docuelevate -f values.yaml
```
See the [Kubernetes Deployment Guide](docs/KubernetesDeployment.md) for full details.
## Documentation
### Getting Started
| Guide | Description |
|-------|-------------|
| [Setup Wizard](docs/SetupWizard.md) | Interactive first-run setup |
| [User Guide](docs/UserGuide.md) | How to use DocuElevate |
| [Browser Extension](docs/BrowserExtension.md) | Install and use the browser extension |
| [Mobile App](docs/MobileApp.md) | iOS and Android mobile app |
| [CLI Guide](docs/CLIGuide.md) | Command-line tool for automation |
### How-To Guides
| Guide | Description |
|-------|-------------|
| [How-To Overview](docs/HowToGuides.md) | Index of all how-to guides |
| [Email Ingestion](docs/howto/EmailIngestion.md) | Set up IMAP email polling |
| [Watched Folder](docs/howto/WatchedFolderSetup.md) | Monitor local or remote folders |
| [Mobile Scanning](docs/howto/MobileScanning.md) | Scan documents with your phone |
### Reference
| Guide | Description |
|-------|-------------|
| [API Documentation](docs/API.md) | REST & GraphQL API reference |
| [Configuration Guide](docs/ConfigurationGuide.md) | All environment variables |
| [Configuration Master](docs/ConfigurationMaster.md) | Configuration overview |
| [Settings Management](docs/SettingsManagement.md) | Runtime settings UI |
### Deployment & Operations
| Guide | Description |
|-------|-------------|
| [Deployment Guide](docs/DeploymentGuide.md) | Docker Compose deployment |
| [Kubernetes / Helm](docs/KubernetesDeployment.md) | Kubernetes deployment with Helm charts |
| [Production Readiness](docs/ProductionReadiness.md) | Checklist for production environments |
| [Database Configuration](docs/DatabaseConfiguration.md) | Database setup and migration |
| [Backup & Restore](docs/ConfigurationGuide.md#backup--restore) | Automated backup configuration |
### Storage Integration Setup
| Guide | Description |
|-------|-------------|
| [Dropbox](docs/DropboxSetup.md) | Dropbox OAuth setup |
| [Google Drive](docs/GoogleDriveSetup.md) | Google Drive service account / OAuth |
| [OneDrive](docs/OneDriveSetup.md) | Microsoft OneDrive setup |
| [Amazon S3](docs/AmazonS3Setup.md) | S3 bucket configuration |
| [Authentication](docs/AuthenticationSetup.md) | OAuth2, OIDC, and social login |
| [Notifications](docs/NotificationsSetup.md) | Notification backend setup |
### Security & Compliance
| Guide | Description |
|-------|-------------|
| [Credential Rotation](docs/CredentialRotationGuide.md) | Rotate secrets safely |
| [Licensing Compliance](docs/LicensingCompliance.md) | Dependency licenses |
| [Privacy & GDPR](docs/PrivacyCompliance.md) | Privacy compliance |
### Development
| Guide | Description |
|-------|-------------|
| [Contributing](CONTRIBUTING.md) | Code style, commits, and PR process |
| [Troubleshooting](docs/Troubleshooting.md) | Common issues and solutions |
| [Configuration Troubleshooting](docs/ConfigurationTroubleshooting.md) | Configuration-specific issues |
| [Build Metadata](docs/BuildMetadata.md) | Version and build information |
| [Internationalization](docs/InternationalizationGuide.md) | Translation and localization |
## Development & Testing
### Running Tests
DocuElevate includes comprehensive test coverage. To run tests:
```bash
# Install development dependencies
pip install -r requirements-dev.txt
@@ -175,21 +284,21 @@ pytest --cov=app --cov-report=term-missing
pytest -m unit
```
Tests are automatically configured with the necessary environment variables - **no manual setup required!**
Tests are automatically configured with the necessary environment variables — **no manual setup required!**
For detailed testing information, including integration tests with Docker and authentication testing, see the [Contributing Guide](CONTRIBUTING.md#running-tests).
For detailed testing information, see the [Contributing Guide](CONTRIBUTING.md#running-tests).
### Contributing
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for:
- Code style guidelines
- Code style guidelines (Ruff for formatting and linting)
- Commit message format (Conventional Commits)
- Testing requirements
- Pull request process
## License
This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
This project is licensed under the Apache License 2.0 — see the [LICENSE](LICENSE) file for details.
## Third-Party Software
@@ -214,13 +323,10 @@ The following is a summary of the licenses used by our direct dependencies:
| Uvicorn | BSD |
| SQLAlchemy | MIT |
| Pydantic | MIT |
| openai | MIT |
| litellm | MIT |
| pypdf | BSD |
| Requests | Apache 2.0 |
| puremagic | MIT |
| filetype | MIT |
| Dropbox | MIT |
| Dropbox SDK | MIT |
| Azure AI Document Intelligence | MIT |
| Authlib | BSD |
| Starlette | BSD |
@@ -229,15 +335,15 @@ The following is a summary of the licenses used by our direct dependencies:
| Microsoft Graph Core | MIT |
| MSAL | MIT |
| Boto3 | Apache 2.0 |
| Paramiko | LGPL-2.1|
| Paramiko | LGPL-2.1 |
| Apprise | MIT |
| Redis | BSD |
| Gotenberg | MIT |
| Redis (py) | BSD |
| Gotenberg Client | MIT |
| Meilisearch | MIT |
For a comprehensive list of all dependencies and their licenses, run:
```
```bash
pip install pip-licenses
pip-licenses
```
RUNTIME_INFO
+6 -6
View File
@@ -1,10 +1,10 @@
DocuElevate Build Information
==============================
Version: 0.114.0
Build Date: 2026-03-09T23:00:57Z
Git Commit: 5fd3f0661b8d86ba6b3f92481675d820aec0d53c
Git Short SHA: 5fd3f06
Version: 0.152.0
Build Date: 2026-03-16T21:40:03Z
Git Commit: 793b6d3e2562518cb404579f656e82a6dc60da02
Git Short SHA: 793b6d3
Git Branch: main
Commit Date: 2026-03-10T00:00:39+01:00
Build Timestamp: 2026-03-09T23:00:57Z
Commit Date: 2026-03-16T22:39:40+01:00
Build Timestamp: 2026-03-16T21:40:03Z
==============================
VERSION
+1 -1
View File
@@ -1 +1 @@
0.114.0
0.152.0
app/api/__init__.py
+16
View File
@@ -8,19 +8,24 @@ from fastapi import APIRouter
from app.api.admin_users import router as admin_users_router
from app.api.api_tokens import router as api_tokens_router
from app.api.audit_logs import router as audit_logs_router
from app.api.automation import router as automation_router
from app.api.azure import router as azure_router
from app.api.backup import router as backup_router
from app.api.billing import router as billing_router
from app.api.compliance import router as compliance_router
from app.api.database import router as database_router
from app.api.diagnostic import router as diagnostic_router
from app.api.dropbox import router as dropbox_router
from app.api.duplicates import router as duplicates_router
from app.api.files import router as files_router
from app.api.google_drive import router as google_drive_router
from app.api.i18n import router as i18n_router
from app.api.imap_accounts import router as imap_accounts_router
from app.api.imap_profiles import router as imap_profiles_router
from app.api.integrations import router as integrations_router
from app.api.logs import router as logs_router
from app.api.mobile import router as mobile_router
from app.api.notifications import router as notifications_router
from app.api.onboarding import router as onboarding_router
from app.api.onedrive import router as onedrive_router
@@ -28,7 +33,9 @@ from app.api.openai import router as openai_router
from app.api.pipelines import router as pipelines_router
from app.api.plans import router as plans_router
from app.api.process import router as process_router
from app.api.profile import router as profile_router
from app.api.queue import router as queue_router
from app.api.routing_rules import router as routing_rules_router
from app.api.saved_searches import router as saved_searches_router
from app.api.scheduled_jobs import router as scheduled_jobs_router
from app.api.search import router as search_router
@@ -37,6 +44,7 @@ from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router
from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router
# Import all the individual routers
@@ -79,8 +87,16 @@ router.include_router(plans_router)
router.include_router(onboarding_router)
router.include_router(billing_router)
router.include_router(pipelines_router)
router.include_router(profile_router)
router.include_router(routing_rules_router)
router.include_router(imap_accounts_router)
router.include_router(imap_profiles_router)
router.include_router(integrations_router)
router.include_router(notifications_router)
router.include_router(scheduled_jobs_router)
router.include_router(audit_logs_router)
router.include_router(i18n_router)
router.include_router(mobile_router)
router.include_router(compliance_router)
router.include_router(translation_router)
router.include_router(automation_router)
app/api/audit_logs.py
+117
View File
@@ -0,0 +1,117 @@
"""
Audit log REST API endpoints.
Provides read-only access to the comprehensive audit log for admin users.
Events are append-only — there are no update or delete endpoints.
"""
import logging
from datetime import datetime
from typing import Annotated, Any
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.utils.audit_service import count_events, query_events
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
@router.get("/audit-logs")
@require_login
async def list_audit_logs(
request: Request,
db: DbSession,
action: Annotated[str | None, Query(description="Filter by action (exact match)")] = None,
user: Annotated[str | None, Query(description="Filter by username")] = None,
resource_type: Annotated[str | None, Query(description="Filter by resource type")] = None,
severity: Annotated[str | None, Query(description="Filter by severity level")] = None,
since: Annotated[datetime | None, Query(description="Only events at or after this ISO-8601 timestamp")] = None,
until: Annotated[datetime | None, Query(description="Only events at or before this ISO-8601 timestamp")] = None,
limit: Annotated[int, Query(ge=1, le=500, description="Max rows to return")] = 50,
offset: Annotated[int, Query(ge=0, description="Rows to skip for pagination")] = 0,
) -> dict[str, Any]:
"""Return audit log entries with optional filtering and pagination.
Requires authentication. Returns events in reverse chronological order.
"""
entries = query_events(
db,
action=action,
user=user,
resource_type=resource_type,
severity=severity,
since=since,
until=until,
limit=limit,
offset=offset,
)
total = count_events(
db,
action=action,
user=user,
resource_type=resource_type,
severity=severity,
since=since,
until=until,
)
return {
"items": [_serialize(e) for e in entries],
"total": total,
"limit": limit,
"offset": offset,
}
@router.get("/audit-logs/actions")
@require_login
async def list_distinct_actions(
request: Request,
db: DbSession,
) -> list[str]:
"""Return the distinct action values present in the audit log."""
from app.models import AuditLog
rows = db.query(AuditLog.action).distinct().order_by(AuditLog.action).all()
return [r[0] for r in rows]
@router.get("/audit-logs/users")
@require_login
async def list_distinct_users(
request: Request,
db: DbSession,
) -> list[str]:
"""Return the distinct user values present in the audit log."""
from app.models import AuditLog
rows = db.query(AuditLog.user).distinct().order_by(AuditLog.user).all()
return [r[0] for r in rows]
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _serialize(entry) -> dict[str, Any]:
"""Convert an AuditLog row to a JSON-safe dict."""
import json as _json
return {
"id": entry.id,
"timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
"user": entry.user,
"action": entry.action,
"resource_type": entry.resource_type,
"resource_id": entry.resource_id,
"ip_address": entry.ip_address,
"details": _json.loads(entry.details) if entry.details else None,
"severity": entry.severity,
}
app/api/billing.py
+4 -3
View File
@@ -30,6 +30,7 @@ from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import SubscriptionPlan, UserProfile
from app.utils.i18n import translate as _translate
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
@@ -37,6 +38,7 @@ router = APIRouter(prefix="/billing", tags=["billing"])
_templates_dir = pathlib.Path(__file__).parents[2] / "frontend" / "templates"
_templates = Jinja2Templates(directory=str(_templates_dir))
_templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
def _get_stripe() -> stripe.StripeClient | None:
@@ -166,9 +168,8 @@ async def create_checkout_session(
checkout_session = client.checkout.sessions.create(params=session_params)
logger.info(
"Created Stripe checkout session %s for user %s plan %s",
"Created Stripe checkout session %s for plan %s",
checkout_session.id,
owner_id,
body.plan_id,
)
return {"checkout_url": checkout_session.url, "session_id": checkout_session.id}
@@ -211,7 +212,7 @@ async def create_portal_session(
}
)
logger.info("Created Stripe portal session for user %s", owner_id)
logger.info("Created Stripe portal session for user")
return {"portal_url": portal.url}
app/api/compliance.py
+183
View File
@@ -0,0 +1,183 @@
"""API endpoints for managing compliance templates (GDPR, HIPAA, SOC2).
All endpoints require admin privileges.
Available routes:
GET /api/compliance/templates – list all compliance templates
GET /api/compliance/templates/{name} – get a single template with checks
POST /api/compliance/templates/{name}/apply – one-click apply a template
GET /api/compliance/templates/{name}/status – evaluate compliance status
GET /api/compliance/summary – overall compliance dashboard data
"""
import logging
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.database import get_db
from app.utils.compliance_service import (
COMPLIANCE_TEMPLATES,
apply_template,
evaluate_template_status,
get_all_templates,
get_compliance_summary,
get_template_by_name,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/compliance", tags=["compliance"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Authorisation helper
# ---------------------------------------------------------------------------
def _require_admin(request: Request) -> dict:
"""Ensure the caller is an admin; raises HTTP 403 otherwise."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
# ---------------------------------------------------------------------------
# Pydantic response models
# ---------------------------------------------------------------------------
class CheckResult(BaseModel):
"""Individual compliance check result."""
key: str
label: str
description: str
expected: str
actual: str
passing: bool
class TemplateStatusResponse(BaseModel):
"""Status evaluation for a compliance template."""
status: str
total: int
passed: int
failed: int
check_results: list[CheckResult]
class TemplateResponse(BaseModel):
"""Full compliance template representation."""
id: int
name: str
display_name: str
description: str | None
enabled: bool
status: str
applied_at: str | None
applied_by: str | None
settings: dict[str, str]
checks: list[dict[str, Any]]
check_count: int
class ApplyResponse(BaseModel):
"""Result of applying a compliance template."""
success: bool
template: str | None = None
applied_settings: dict[str, str] | None = None
errors: list[str] | None = None
error: str | None = None
status: TemplateStatusResponse | None = None
class SummaryTemplateResponse(BaseModel):
"""Per-template summary for the compliance dashboard."""
name: str
display_name: str
enabled: bool
status: str
total: int
passed: int
failed: int
applied_at: str | None
applied_by: str | None
class ComplianceSummaryResponse(BaseModel):
"""Overall compliance dashboard summary."""
overall_status: str
total_checks: int
total_passed: int
total_failed: int
templates: list[SummaryTemplateResponse]
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/templates", response_model=list[TemplateResponse])
async def list_templates(db: DbSession, admin: AdminUser) -> list[dict[str, Any]]:
"""List all compliance templates with their current status."""
return get_all_templates(db)
@router.get("/templates/{name}", response_model=TemplateResponse)
async def get_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
"""Get a single compliance template by name."""
templates = get_all_templates(db)
for t in templates:
if t["name"] == name:
return t
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
@router.post("/templates/{name}/apply", response_model=ApplyResponse)
async def apply_compliance_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
"""Apply a compliance template (one-click).
Writes all template settings to the database and evaluates the resulting
compliance status.
"""
if name not in COMPLIANCE_TEMPLATES:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
template = get_template_by_name(db, name)
if template is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
admin_email = admin.get("email", "admin")
result = apply_template(db, name, applied_by=admin_email)
if not result.get("success") and result.get("error"):
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=result["error"])
return result
@router.get("/templates/{name}/status", response_model=TemplateStatusResponse)
async def get_template_status(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]:
"""Evaluate the live compliance status of a template."""
template = get_template_by_name(db, name)
if template is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found")
return evaluate_template_status(db, name)
@router.get("/summary", response_model=ComplianceSummaryResponse)
async def compliance_summary(db: DbSession, admin: AdminUser) -> dict[str, Any]:
"""Overall compliance dashboard summary across all templates."""
return get_compliance_summary(db)
app/api/dropbox.py
+49 -46
View File
@@ -6,7 +6,7 @@ import logging
import os
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -132,57 +132,60 @@ async def test_dropbox_token(request: Request):
"message": "Dropbox credentials are not fully configured",
}
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = requests.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = requests.post(
async with httpx.AsyncClient() as client:
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get account info
account_info = response.json()
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = await client.post(
refresh_url, data=refresh_data, timeout=settings.http_request_timeout
)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# Get account info
account_info = response.json()
account_email = account_info.get("email", "Unknown account")
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
app/api/duplicates.py
+28 -23
View File
@@ -73,33 +73,38 @@ def list_duplicate_groups(
groups = []
total_duplicate_files = 0
for filehash in dup_hashes:
# Find the original (non-duplicate) record with this hash
original = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
if dup_hashes:
# Fetch all matching files (both original and duplicates) in a single batch query
all_records = (
db.query(FileRecord).filter(FileRecord.filehash.in_(dup_hashes)).order_by(FileRecord.id.asc()).all()
)
# Find all duplicate records for this hash
duplicates = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
.order_by(FileRecord.id.asc())
.all()
)
# Group records by hash
originals_by_hash = {}
duplicates_by_hash = {h: [] for h in dup_hashes}
total_duplicate_files += len(duplicates)
for record in all_records:
h = record.filehash
if not record.is_duplicate:
# Store only the first original record per hash, matching the old .first() behaviour
if h not in originals_by_hash:
originals_by_hash[h] = record
else:
duplicates_by_hash[h].append(record)
total_duplicate_files += 1
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
for filehash in dup_hashes:
original = originals_by_hash.get(filehash)
duplicates = duplicates_by_hash.get(filehash, [])
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
app/api/files.py
+68 -30
View File
@@ -11,6 +11,7 @@ import zipfile
from datetime import datetime, timezone
from typing import Annotated, List, Optional
import aiofiles
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status
from fastapi.responses import StreamingResponse
from sqlalchemy import asc, desc
@@ -1217,6 +1218,66 @@ def download_file(
raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")
async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: int) -> int:
"""Save an uploaded file in chunks and enforce the maximum size limit."""
try:
written_size = 0
with open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
if not chunk:
break
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
return written_size
except HTTPException:
raise
except Exception as e:
if os.path.exists(target_path):
os.remove(target_path)
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
if not settings.enable_deduplication:
return None
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
return {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
return None
@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
@@ -1277,7 +1338,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# enforcing the size limit during the read so memory usage stays bounded.
try:
written_size = 0
with open(target_path, "wb") as f:
async with aiofiles.open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
@@ -1286,14 +1347,14 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
await f.write(chunk)
except HTTPException:
raise
except Exception as e:
@@ -1384,29 +1445,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# Check for exact duplicates (same SHA-256 hash) before returning.
# This gives the caller an immediate warning without waiting for the pipeline.
# Only performed when deduplication is enabled in settings.
exact_duplicate_warning = None
if settings.enable_deduplication:
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
exact_duplicate_warning = {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
response: dict = {
"task_id": task.id,
@@ -1458,7 +1497,7 @@ def claim_file(request: Request, file_id: int, db: DbSession):
logger.exception(f"Error claiming file {file_id}: {e}")
raise HTTPException(status_code=500, detail="Failed to claim document")
logger.info(f"File {file_id} claimed by user '{owner_id}'")
logger.info("File %d claimed by user", file_id)
return {"status": "success", "message": "Document claimed successfully", "file_id": file_id, "owner_id": owner_id}
@@ -1498,7 +1537,7 @@ def bulk_claim_files(request: Request, file_ids: list[int], db: DbSession):
logger.exception(f"Error during bulk claim: {e}")
raise HTTPException(status_code=500, detail="Failed to claim documents")
logger.info(f"Bulk claim by '{owner_id}': claimed={claimed}, skipped={[s['file_id'] for s in skipped]}")
logger.info("Bulk claim: claimed=%s, skipped=%s", claimed, [s["file_id"] for s in skipped])
return {
"status": "success",
"claimed_count": len(claimed),
@@ -1551,8 +1590,7 @@ def assign_owner(request: Request, db: DbSession, owner_id: str = Query(...), fi
logger.exception(f"Error assigning owner: {e}")
raise HTTPException(status_code=500, detail="Failed to assign owner")
admin_name = get_current_owner_id(request) or "admin"
logger.info(f"Admin '{admin_name}' assigned owner_id='{owner_id}' to {updated} file(s)")
logger.info("Admin assigned owner to %d file(s)", updated)
return {
"status": "success",
"message": f"Assigned owner to {updated} document(s)",
app/api/google_drive.py
+1 -1
View File
@@ -363,7 +363,7 @@ def format_time_remaining(time_delta):
@router.post("/google-drive/save-settings")
@require_login
async def save_dropbox_settings(
async def save_google_drive_settings(
request: Request,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
app/api/graphql_api.py
+431
View File
@@ -0,0 +1,431 @@
"""
GraphQL API endpoint for DocuElevate.
Provides a flexible query interface alongside the existing REST API.
Schema covers: documents, pipelines, settings, and users.
Endpoint: /graphql
GraphiQL playground: /graphql (via browser)
"""
from __future__ import annotations
import logging
from datetime import datetime
from typing import Annotated, Any
import strawberry
from fastapi import Depends, Request
from sqlalchemy.orm import Session
from strawberry.fastapi import GraphQLRouter
from app.auth import get_current_user
from app.config import settings
from app.database import get_db
from app.models import ApplicationSettings, FileRecord, Pipeline, PipelineStep, UserProfile
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Strawberry types
# ---------------------------------------------------------------------------
@strawberry.type
class DocumentType:
"""A processed document stored in the system."""
id: int
owner_id: str | None
original_filename: str | None
local_filename: str
file_size: int
mime_type: str | None
document_title: str | None
is_duplicate: bool
ocr_quality_score: int | None
pipeline_id: int | None
created_at: datetime | None
@strawberry.type
class PipelineStepType:
"""A single step within a processing pipeline."""
id: int
pipeline_id: int
position: int
step_type: str
label: str | None
enabled: bool
created_at: datetime | None
@strawberry.type
class PipelineType:
"""A processing pipeline with its ordered steps."""
id: int
owner_id: str | None
name: str
description: str | None
is_default: bool
is_active: bool
steps: list[PipelineStepType]
created_at: datetime | None
updated_at: datetime | None
@strawberry.type
class SettingType:
"""An application configuration setting stored in the database."""
id: int
key: str
value: str | None
created_at: datetime | None
updated_at: datetime | None
@strawberry.type
class UserType:
"""A user profile in the system."""
id: int
user_id: str
display_name: str | None
is_blocked: bool
subscription_tier: str | None
onboarding_completed: bool
created_at: datetime | None
# ---------------------------------------------------------------------------
# Conversion helpers
# ---------------------------------------------------------------------------
def _document_from_record(rec: FileRecord) -> DocumentType:
return DocumentType(
id=rec.id,
owner_id=rec.owner_id,
original_filename=rec.original_filename,
local_filename=rec.local_filename,
file_size=rec.file_size,
mime_type=rec.mime_type,
document_title=rec.document_title,
is_duplicate=rec.is_duplicate,
ocr_quality_score=rec.ocr_quality_score,
pipeline_id=rec.pipeline_id,
created_at=rec.created_at,
)
def _pipeline_step_from_record(step: PipelineStep) -> PipelineStepType:
return PipelineStepType(
id=step.id,
pipeline_id=step.pipeline_id,
position=step.position,
step_type=step.step_type,
label=step.label,
enabled=step.enabled,
created_at=step.created_at,
)
def _pipeline_from_record(pipeline: Pipeline, db: Session) -> PipelineType:
steps = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline.id).order_by(PipelineStep.position).all()
return PipelineType(
id=pipeline.id,
owner_id=pipeline.owner_id,
name=pipeline.name,
description=pipeline.description,
is_default=pipeline.is_default,
is_active=pipeline.is_active,
steps=[_pipeline_step_from_record(s) for s in steps],
created_at=pipeline.created_at,
updated_at=pipeline.updated_at,
)
def _setting_from_record(setting: ApplicationSettings) -> SettingType:
return SettingType(
id=setting.id,
key=setting.key,
value=setting.value,
created_at=setting.created_at,
updated_at=setting.updated_at,
)
def _user_from_profile(profile: UserProfile) -> UserType:
return UserType(
id=profile.id,
user_id=profile.user_id,
display_name=profile.display_name,
is_blocked=profile.is_blocked,
subscription_tier=profile.subscription_tier,
onboarding_completed=profile.onboarding_completed,
created_at=profile.created_at,
)
# ---------------------------------------------------------------------------
# Context helpers
# ---------------------------------------------------------------------------
# Keys that contain sensitive data and must never be returned via GraphQL
_SENSITIVE_SETTING_KEYS: frozenset[str] = frozenset(
{
"openai_api_key",
"azure_ai_key",
"session_secret",
"database_url",
"redis_url",
"dropbox_app_secret",
"dropbox_refresh_token",
"google_drive_credentials_json",
"onedrive_client_secret",
"onedrive_refresh_token",
"smtp_password",
"nextcloud_password",
"s3_secret_access_key",
"ftp_password",
"sftp_password",
"webdav_password",
"stripe_secret_key",
"stripe_webhook_secret",
"sentry_dsn",
"social_auth_google_client_secret",
"social_auth_microsoft_client_secret",
"social_auth_apple_private_key",
"social_auth_dropbox_app_secret",
}
)
def _get_current_user_id(user: dict[str, Any] | None) -> str | None:
"""Extract the stable user identifier from the user dict."""
if not user:
return None
return user.get("preferred_username") or user.get("email") or user.get("id") or None
def _get_db_and_user(info: strawberry.types.Info) -> tuple[Session, dict[str, Any] | None]:
"""Extract the database session and current user from the Strawberry context."""
db: Session = info.context["db"]
user: dict[str, Any] | None = info.context.get("user")
return db, user
def _require_auth(user: dict[str, Any] | None) -> None:
"""Raise an error when authentication is enabled and no valid user is present."""
if settings.auth_enabled and not user:
raise strawberry.exceptions.StrawberryGraphQLError("Authentication required")
def _require_admin(user: dict[str, Any] | None) -> None:
"""Raise an error when the current user is not an admin.
When ``auth_enabled`` is *False* (single-user / development mode) all
callers are implicitly treated as administrators.
"""
if not settings.auth_enabled:
# Single-user mode: no auth, treat caller as admin
return
_require_auth(user)
if not (user and user.get("is_admin")):
raise strawberry.exceptions.StrawberryGraphQLError("Admin access required")
# ---------------------------------------------------------------------------
# Query resolvers
# ---------------------------------------------------------------------------
@strawberry.type
class Query:
"""Root query type for the DocuElevate GraphQL API."""
@strawberry.field(description="List documents, optionally filtered by owner.")
def documents(
self,
info: strawberry.types.Info,
owner_id: str | None = None,
limit: int = 20,
offset: int = 0,
) -> list[DocumentType]:
"""Return a paginated list of documents.
When *auth_enabled* the caller must be authenticated. Non-admin users
receive only their own documents; admins may query any *owner_id*.
"""
db, user = _get_db_and_user(info)
_require_auth(user)
limit = max(1, min(limit, 100))
offset = max(0, offset)
query = db.query(FileRecord)
if settings.auth_enabled and user:
is_admin = user.get("is_admin", False)
current_user_id = _get_current_user_id(user)
if not is_admin:
# Non-admins can only see their own documents
query = query.filter(FileRecord.owner_id == current_user_id)
elif owner_id:
query = query.filter(FileRecord.owner_id == owner_id)
elif owner_id:
query = query.filter(FileRecord.owner_id == owner_id)
records = query.order_by(FileRecord.created_at.desc()).offset(offset).limit(limit).all()
return [_document_from_record(r) for r in records]
@strawberry.field(description="Fetch a single document by ID.")
def document(self, info: strawberry.types.Info, id: int) -> DocumentType | None:
"""Return one document by its primary key, or *null* if not found."""
db, user = _get_db_and_user(info)
_require_auth(user)
rec = db.query(FileRecord).filter(FileRecord.id == id).first()
if rec is None:
return None
if settings.auth_enabled and user:
is_admin = user.get("is_admin", False)
current_user_id = _get_current_user_id(user)
if not is_admin and rec.owner_id != current_user_id:
return None
return _document_from_record(rec)
@strawberry.field(description="List processing pipelines.")
def pipelines(
self,
info: strawberry.types.Info,
owner_id: str | None = None,
limit: int = 20,
offset: int = 0,
) -> list[PipelineType]:
"""Return a paginated list of pipelines."""
db, user = _get_db_and_user(info)
_require_auth(user)
limit = max(1, min(limit, 100))
offset = max(0, offset)
query = db.query(Pipeline)
if settings.auth_enabled and user:
is_admin = user.get("is_admin", False)
current_user_id = _get_current_user_id(user)
if not is_admin:
query = query.filter((Pipeline.owner_id == current_user_id) | (Pipeline.owner_id.is_(None)))
elif owner_id:
query = query.filter(Pipeline.owner_id == owner_id)
elif owner_id:
query = query.filter(Pipeline.owner_id == owner_id)
rows = query.order_by(Pipeline.id).offset(offset).limit(limit).all()
return [_pipeline_from_record(p, db) for p in rows]
@strawberry.field(description="Fetch a single pipeline by ID.")
def pipeline(self, info: strawberry.types.Info, id: int) -> PipelineType | None:
"""Return one pipeline by its primary key, or *null* if not found."""
db, user = _get_db_and_user(info)
_require_auth(user)
row = db.query(Pipeline).filter(Pipeline.id == id).first()
if row is None:
return None
if settings.auth_enabled and user:
is_admin = user.get("is_admin", False)
current_user_id = _get_current_user_id(user)
if not is_admin and row.owner_id is not None and row.owner_id != current_user_id:
return None
return _pipeline_from_record(row, db)
@strawberry.field(description="List non-sensitive application settings (admin only).")
def settings(
self,
info: strawberry.types.Info,
limit: int = 50,
offset: int = 0,
) -> list[SettingType]:
"""Return application settings stored in the database.
Sensitive keys (API secrets, passwords, etc.) are automatically
excluded. Requires admin privileges when auth is enabled.
"""
db, user = _get_db_and_user(info)
_require_admin(user)
limit = max(1, min(limit, 200))
offset = max(0, offset)
rows = (
db.query(ApplicationSettings)
.filter(ApplicationSettings.key.notin_(_SENSITIVE_SETTING_KEYS))
.order_by(ApplicationSettings.key)
.offset(offset)
.limit(limit)
.all()
)
return [_setting_from_record(r) for r in rows]
@strawberry.field(description="List user profiles (admin only).")
def users(
self,
info: strawberry.types.Info,
limit: int = 20,
offset: int = 0,
) -> list[UserType]:
"""Return a paginated list of user profiles. Requires admin privileges."""
db, user = _get_db_and_user(info)
_require_admin(user)
limit = max(1, min(limit, 100))
offset = max(0, offset)
rows = db.query(UserProfile).order_by(UserProfile.user_id).offset(offset).limit(limit).all()
return [_user_from_profile(r) for r in rows]
@strawberry.field(description="Fetch a user profile by user_id (admin only).")
def user(self, info: strawberry.types.Info, user_id: str) -> UserType | None:
"""Return one user profile by *user_id*, or *null* if not found."""
db, user = _get_db_and_user(info)
_require_admin(user)
row = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
return _user_from_profile(row) if row else None
# ---------------------------------------------------------------------------
# Schema and router
# ---------------------------------------------------------------------------
schema = strawberry.Schema(query=Query)
async def get_graphql_context(
request: Request,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
"""Build the per-request context injected into every resolver."""
try:
user = get_current_user(request)
except Exception:
logger.debug("Could not resolve current user for GraphQL context", exc_info=True)
user = None
return {"request": request, "db": db, "user": user}
graphql_router = GraphQLRouter(
schema,
context_getter=get_graphql_context,
graphql_ide="graphiql",
)
app/api/i18n.py
+136
View File
@@ -0,0 +1,136 @@
"""API endpoints for internationalization (i18n).
Provides endpoints for:
* Listing available languages
* Getting/setting user language preference (persisted in session + cookie + DB)
"""
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request, Response
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import UserProfile
from app.utils.i18n import (
DEFAULT_LANGUAGE,
SUPPORTED_LANGUAGE_CODES,
SUPPORTED_LANGUAGES,
detect_language,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/i18n", tags=["i18n"])
class LanguageInfo(BaseModel):
"""Schema for a supported language."""
code: str
name: str
native: str
flag: str
class LanguageListResponse(BaseModel):
"""Response for the list-languages endpoint."""
languages: list[LanguageInfo]
current: str
default: str
class SetLanguageRequest(BaseModel):
"""Request body for setting the preferred language."""
language: str
class SetLanguageResponse(BaseModel):
"""Response after changing the language."""
language: str
message: str
@router.get("/languages", response_model=LanguageListResponse)
async def list_languages(request: Request) -> LanguageListResponse:
"""Return all supported UI languages and the current active language."""
current = detect_language(request)
return LanguageListResponse(
languages=[LanguageInfo(**lang) for lang in SUPPORTED_LANGUAGES],
current=current,
default=DEFAULT_LANGUAGE,
)
@router.post("/language", response_model=SetLanguageResponse)
async def set_language(
body: SetLanguageRequest,
request: Request,
response: Response,
db: Session = Depends(get_db),
) -> SetLanguageResponse:
"""Set the preferred UI language.
Persists the choice in:
1. The server-side session
2. A ``docuelevate_lang`` cookie (30-day expiry)
3. The ``UserProfile.preferred_language`` column (if authenticated)
"""
lang = body.language.lower().strip()
if lang not in SUPPORTED_LANGUAGE_CODES:
lang = DEFAULT_LANGUAGE
# 1. Session
if hasattr(request, "session"):
request.session["preferred_language"] = lang
# 2. Cookie (30 days)
response.set_cookie(
key="docuelevate_lang",
value=lang,
max_age=30 * 24 * 60 * 60,
httponly=False,
samesite="lax",
)
# 3. Database (if user is authenticated)
_persist_language_to_profile(request, db, lang)
language_name = next(
(entry["native"] for entry in SUPPORTED_LANGUAGES if entry["code"] == lang),
lang,
)
logger.info("Language preference set to '%s'", lang)
return SetLanguageResponse(
language=lang,
message=f"Language changed to {language_name}",
)
def _persist_language_to_profile(request: Request, db: Session, lang: str) -> None:
"""Write language preference to the UserProfile row, if the user is logged in."""
user_id: str | None = None
if hasattr(request, "session"):
user = request.session.get("user")
if isinstance(user, dict):
user_id = user.get("preferred_username") or user.get("email") or user.get("id")
elif isinstance(user, str):
user_id = user
if not user_id:
return
try:
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
if profile:
profile.preferred_language = lang # type: ignore[attr-defined]
db.commit()
except Exception:
db.rollback()
logger.debug("Could not persist language preference for user_id=%s", user_id)
app/api/imap_accounts.py
+20
View File
@@ -110,6 +110,13 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
profile_id: int | None = Field(
default=None,
description=(
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
"Null inherits the global imap_attachment_filter setting."
),
)
class ImapAccountUpdate(BaseModel):
@@ -123,6 +130,13 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
profile_id: int | None = Field(
default=None,
description=(
"ID of the ImapIngestionProfile to use. "
"Explicitly sending null clears the override (falls back to global setting)."
),
)
class ImapTestRequest(BaseModel):
@@ -155,6 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
"profile_id": acct.profile_id,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -222,6 +237,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
profile_id=body.profile_id,
)
try:
db.add(acct)
@@ -277,6 +293,10 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
# profile_id: update whenever the field is explicitly present in the request payload
# (including sending null to clear the override).
if "profile_id" in body.model_fields_set:
acct.profile_id = body.profile_id
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
app/api/imap_profiles.py
+257
View File
@@ -0,0 +1,257 @@
"""API endpoints for managing IMAP ingestion profiles.
Ingestion profiles allow fine-grained control over which attachment types are
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
canonical set defined in :mod:`app.utils.allowed_types`.
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
deleted or modified. Users may create their own profiles which are private to
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
to all users but can only be created by administrators.
"""
import json
import logging
from datetime import datetime, timezone
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ImapIngestionProfile
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helpers
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if owner_id is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
class ImapProfileCreate(BaseModel):
"""Schema for creating a new ingestion profile."""
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
description: str | None = Field(default=None, description="Optional description")
allowed_categories: list[str] = Field(
...,
min_length=1,
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
)
class ImapProfileUpdate(BaseModel):
"""Schema for updating an existing profile (all fields optional)."""
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
allowed_categories: list[str] | None = Field(default=None, min_length=1)
# ---------------------------------------------------------------------------
# Validation helpers
# ---------------------------------------------------------------------------
def _validate_categories(categories: list[str]) -> list[str]:
"""Raise 422 if any category key is unknown; return the cleaned list."""
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
if unknown:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
)
# Deduplicate while preserving order
seen: set[str] = set()
result: list[str] = []
for cat in categories:
if cat not in seen:
seen.add(cat)
result.append(cat)
return result
# ---------------------------------------------------------------------------
# Serialisation
# ---------------------------------------------------------------------------
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
"""Serialize a profile row to a response dict."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
# Enrich categories with display metadata
categories_detail = [
{
"key": cat,
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
}
for cat in categories
]
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"categories_detail": categories_detail,
"is_builtin": profile.is_builtin,
"created_at": profile.created_at.isoformat() if profile.created_at else None,
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
}
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/categories", summary="List available file-type categories")
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return the full list of file-type categories that can be used in profiles."""
return [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
@router.get("/", summary="List ingestion profiles visible to the current user")
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
profiles = (
db.query(ImapIngestionProfile)
.filter(
# SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
)
return [_to_response(p) for p in profiles]
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Create a new ingestion profile owned by the current user."""
categories = _validate_categories(body.allowed_categories)
profile = ImapIngestionProfile(
name=body.name,
description=body.description,
owner_id=owner_id,
allowed_categories=json.dumps(categories),
is_builtin=False,
)
try:
db.add(profile)
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
return _to_response(profile)
@router.get("/{profile_id}", summary="Get a single ingestion profile")
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
return _to_response(profile)
@router.put("/{profile_id}", summary="Update an ingestion profile")
def update_profile(
profile_id: int,
request: Request,
body: ImapProfileUpdate,
db: DbSession,
owner_id: CurrentOwner,
) -> dict[str, Any]:
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be modified.",
)
if body.name is not None:
profile.name = body.name
if "description" in body.model_fields_set:
profile.description = body.description
if body.allowed_categories is not None:
categories = _validate_categories(body.allowed_categories)
profile.allowed_categories = json.dumps(categories)
profile.updated_at = datetime.now(timezone.utc)
try:
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
return _to_response(profile)
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be deleted.",
)
try:
db.delete(profile)
db.commit()
except Exception:
db.rollback()
raise
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
app/api/integrations.py
+4 -9
View File
@@ -564,7 +564,6 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str
return {"success": False, "message": "Missing required field: url"}
# Only allow http/https to prevent file:// or other custom scheme attacks
import ipaddress
from urllib.parse import urlparse
parsed = urlparse(url)
@@ -574,14 +573,10 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str
# Block requests to private/internal IPs to prevent SSRF
hostname = parsed.hostname or ""
if hostname:
try:
addr = ipaddress.ip_address(hostname)
if addr.is_private or addr.is_loopback or addr.is_link_local:
return {"success": False, "message": "URLs pointing to internal or private networks are not allowed"}
except ValueError:
# Hostname is not an IP literal — allow DNS names through
if hostname in ("localhost", "localhost.localdomain"):
return {"success": False, "message": "URLs pointing to localhost are not allowed"}
from app.utils.network import is_private_ip
if is_private_ip(hostname):
return {"success": False, "message": "URLs pointing to internal or private networks are not allowed"}
try:
import base64
app/api/local_auth.py
+2
View File
@@ -26,6 +26,7 @@ from starlette.responses import RedirectResponse
from app.config import settings
from app.database import get_db
from app.models import LocalUser, UserProfile
from app.utils.i18n import translate as _translate
from app.utils.local_auth import (
build_session_user,
generate_token,
@@ -41,6 +42,7 @@ router = APIRouter(tags=["local-auth"])
_templates_dir = pathlib.Path(__file__).parents[2] / "frontend" / "templates"
templates = Jinja2Templates(directory=str(_templates_dir))
templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
DbSession = Annotated[Session, Depends(get_db)]
app/api/mobile.py
+347
View File
@@ -0,0 +1,347 @@
"""Mobile app API endpoints.
Provides endpoints specifically designed for the DocuElevate native mobile
app (iOS / Android via React Native / Expo):
* ``POST /mobile/generate-token`` – exchange an active session for a
long-lived API token that the mobile app stores securely. The token is
auto-named "Mobile App – <device_name>" and is identical to regular API
tokens (Bearer auth works everywhere).
* ``POST /mobile/register-device`` – register a push-notification device
token (Expo push token) so the user receives push notifications when
documents finish processing.
* ``GET /mobile/devices`` – list registered devices for the current user.
* ``DELETE /mobile/devices/{device_id}`` – deactivate a device.
* ``GET /mobile/whoami`` – lightweight profile endpoint for the mobile app
to verify authentication state.
"""
import logging
from datetime import datetime, timezone
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.api.api_tokens import generate_api_token, hash_token
from app.auth import require_login
from app.database import get_db
from app.models import ApiToken, MobileDevice
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/mobile", tags=["mobile"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helper
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if not owner_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Request / Response schemas
# ---------------------------------------------------------------------------
class GenerateTokenRequest(BaseModel):
"""Request body for auto-generating a mobile app token."""
device_name: str = Field(
default="Mobile App",
min_length=1,
max_length=120,
description="Human-readable device name used to label the token.",
)
class GenerateTokenResponse(BaseModel):
"""Response containing the one-time-visible API token."""
token: str
token_id: int
name: str
created_at: datetime
class RegisterDeviceRequest(BaseModel):
"""Request body for registering a push-notification device token."""
push_token: str = Field(
min_length=1,
max_length=512,
description="Expo push token (ExponentPushToken[…]) obtained from the mobile app.",
)
device_name: str | None = Field(
default=None,
max_length=255,
description="Optional human-readable device name (e.g. 'John's iPhone').",
)
platform: str = Field(
default="ios",
description="Device platform: 'ios', 'android', or 'web'.",
)
class DeviceResponse(BaseModel):
"""Serialised MobileDevice record."""
id: int
device_name: str | None
platform: str
push_token_preview: str
is_active: bool
created_at: datetime
last_seen_at: datetime | None
class WhoAmIResponse(BaseModel):
"""Lightweight profile response for the mobile app."""
owner_id: str
display_name: str | None
email: str | None
avatar_url: str | None
is_admin: bool
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _device_to_response(device: MobileDevice) -> dict[str, Any]:
"""Convert a MobileDevice ORM object to a serialisable dict."""
# Show only first 20 chars of the push token for security.
token_preview = device.push_token[:20] + "…" if len(device.push_token) > 20 else device.push_token
return {
"id": device.id,
"device_name": device.device_name,
"platform": device.platform,
"push_token_preview": token_preview,
"is_active": device.is_active,
"created_at": device.created_at,
"last_seen_at": device.last_seen_at,
}
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.post("/generate-token", status_code=status.HTTP_201_CREATED, response_model=GenerateTokenResponse)
@require_login
async def generate_mobile_token(
request: Request,
body: GenerateTokenRequest,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Generate a long-lived API token for the mobile app.
The mobile app calls this endpoint immediately after SSO login to obtain
a Bearer token it can store in the secure keychain. The returned token
is functionally identical to manually-created API tokens and works with
every authenticated endpoint.
The token is shown **exactly once** in the response; subsequent requests
show only the prefix for identification.
"""
token_name = f"Mobile App – {body.device_name}"
plaintext = generate_api_token()
token_hash_value = hash_token(plaintext)
prefix = plaintext[:12]
db_token = ApiToken(
owner_id=owner_id,
name=token_name,
token_hash=token_hash_value,
token_prefix=prefix,
)
try:
db.add(db_token)
db.commit()
db.refresh(db_token)
except Exception:
db.rollback()
logger.exception("Failed to create mobile API token for owner_id=%s", owner_id)
raise
logger.info("Mobile API token created: id=%s owner=%s device=%r", db_token.id, owner_id, body.device_name)
return {
"token": plaintext,
"token_id": db_token.id,
"name": token_name,
"created_at": db_token.created_at,
}
@router.post("/register-device", status_code=status.HTTP_201_CREATED, response_model=DeviceResponse)
@require_login
async def register_device(
request: Request,
body: RegisterDeviceRequest,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Register or refresh a push-notification device token.
If the same ``push_token`` is already registered for this user the
record is reactivated and ``last_seen_at`` is updated rather than
creating a duplicate.
"""
platform = body.platform.lower()
if platform not in {"ios", "android", "web"}:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="platform must be one of: ios, android, web",
)
now = datetime.now(timezone.utc)
# Upsert: reuse existing record if the token is already known.
existing = (
db.query(MobileDevice)
.filter(MobileDevice.owner_id == owner_id, MobileDevice.push_token == body.push_token)
.first()
)
if existing:
existing.is_active = True
existing.last_seen_at = now
if body.device_name:
existing.device_name = body.device_name
try:
db.commit()
db.refresh(existing)
except Exception:
db.rollback()
raise
logger.info("Mobile device refreshed: id=%s owner=%s", existing.id, owner_id)
return _device_to_response(existing)
device = MobileDevice(
owner_id=owner_id,
device_name=body.device_name,
platform=platform,
push_token=body.push_token,
is_active=True,
last_seen_at=now,
)
try:
db.add(device)
db.commit()
db.refresh(device)
except Exception:
db.rollback()
logger.exception("Failed to register mobile device for owner_id=%s", owner_id)
raise
logger.info("Mobile device registered: id=%s owner=%s platform=%s", device.id, owner_id, platform)
return _device_to_response(device)
@router.get("/devices", response_model=list[DeviceResponse])
@require_login
async def list_devices(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> list[dict[str, Any]]:
"""List all registered push-notification devices for the current user."""
devices = (
db.query(MobileDevice).filter(MobileDevice.owner_id == owner_id).order_by(MobileDevice.created_at.desc()).all()
)
return [_device_to_response(d) for d in devices]
@router.delete("/devices/{device_id}", status_code=status.HTTP_204_NO_CONTENT)
@require_login
async def deactivate_device(
request: Request,
device_id: int,
owner_id: CurrentOwner,
db: DbSession,
) -> None:
"""Deactivate a push-notification device registration.
The device record is kept for audit purposes but will no longer receive
push notifications.
"""
device = db.get(MobileDevice, device_id)
if not device or device.owner_id != owner_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Device not found")
device.is_active = False
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id)
@router.get("/whoami", response_model=WhoAmIResponse)
@require_login
async def whoami(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Return basic profile information for the authenticated user.
The mobile app calls this after token exchange to populate the user
profile screen and verify that the stored token is still valid.
"""
from app.auth import get_gravatar_url
from app.models import LocalUser, UserProfile
profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first()
local_user = db.query(LocalUser).filter(LocalUser.email == owner_id).first()
display_name: str | None = None
email: str | None = None
avatar_url: str | None = None
is_admin = False
if profile:
display_name = profile.display_name
if local_user:
email = local_user.email
is_admin = bool(local_user.is_admin)
if not display_name and local_user.display_name:
display_name = local_user.display_name
elif "@" in owner_id:
# SSO users commonly have their email as owner_id
email = owner_id
if email:
avatar_url = get_gravatar_url(email)
return {
"owner_id": owner_id,
"display_name": display_name,
"email": email,
"avatar_url": avatar_url,
"is_admin": is_admin,
}
app/api/notifications.py
+9 -10
View File
@@ -452,17 +452,16 @@ async def update_preferences(
)
try:
# Pre-fetch existing preferences for this user to avoid N+1 queries
existing_prefs = (
db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
)
# Build a fast lookup dictionary keyed by (event_type, channel_type, target_id)
prefs_dict = {(pref.event_type, pref.channel_type, pref.target_id): pref for pref in existing_prefs}
for item in body.preferences:
existing = (
db.query(UserNotificationPreference)
.filter(
UserNotificationPreference.owner_id == owner_id,
UserNotificationPreference.event_type == item.event_type,
UserNotificationPreference.channel_type == item.channel_type,
UserNotificationPreference.target_id == item.target_id,
)
.first()
)
existing = prefs_dict.get((item.event_type, item.channel_type, item.target_id))
if existing:
existing.is_enabled = item.is_enabled
else:
app/api/onedrive.py
+40 -112
View File
@@ -3,17 +3,17 @@ OneDrive API endpoints
"""
import logging
import os
from datetime import datetime, timedelta
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.utils.env_utils import update_env_file
from app.utils.oauth_helper import exchange_oauth_token
from app.utils.settings_service import save_setting_to_db
from app.utils.settings_sync import notify_settings_updated
@@ -92,17 +92,18 @@ async def test_onedrive_token(request: Request):
"scope": "offline_access Files.ReadWrite",
}
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
response = await client.post(token_url, data=refresh_data)
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_data = response.json()
token_data = response.json()
access_token = token_data.get("access_token")
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
@@ -115,32 +116,7 @@ async def test_onedrive_token(request: Request):
settings.onedrive_refresh_token = new_refresh_token
# Also try to update .env file if it exists
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if os.path.exists(env_path):
with open(env_path, "r") as f:
env_lines = f.readlines()
updated_lines = []
updated = False
for line in env_lines:
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
updated = True
else:
updated_lines.append(line)
if not updated:
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
with open(env_path, "w") as f:
f.writelines(updated_lines)
logger.info("Updated refresh token in .env file")
except Exception as e:
logger.warning(f"Failed to update refresh token in .env file: {e}")
update_env_file({"ONEDRIVE_REFRESH_TOKEN": new_refresh_token})
# Persist the rotated refresh token to the database
try:
@@ -164,17 +140,18 @@ async def test_onedrive_token(request: Request):
user_info_url = "https://graph.microsoft.com/v1.0/me"
headers = {"Authorization": f"Bearer {access_token}"}
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
user_response = await client.get(user_info_url, headers=headers)
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
# Get user info
user_info = user_response.json()
# Get user info
user_info = user_response.json()
display_name = user_info.get("displayName", "Unknown user")
email = user_info.get("userPrincipalName", "Unknown email")
@@ -246,75 +223,26 @@ async def save_onedrive_settings(
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
)
# Best-effort .env file write
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if not os.path.exists(env_path):
logger.warning(f".env file not found at {env_path}, skipping file write")
else:
logger.info(f"Updating OneDrive settings in {env_path}")
# Build settings dictionary mapped to database/memory keys
onedrive_settings = {
"onedrive_refresh_token": refresh_token,
"onedrive_client_id": client_id,
"onedrive_client_secret": client_secret,
"onedrive_tenant_id": tenant_id,
"onedrive_folder_path": folder_path,
}
with open(env_path, "r") as f:
env_lines = f.readlines()
# Filter out None values
onedrive_settings = {k: v for k, v in onedrive_settings.items() if v is not None}
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
# Best-effort .env file write using the new utility
env_settings = {k.upper(): v for k, v in onedrive_settings.items()}
update_env_file(env_settings)
updated = set()
new_env_lines = []
for line in env_lines:
stripped_line = line.rstrip()
is_updated = False
for key, value in onedrive_settings.items():
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
new_env_lines.append(f"{key}={value}")
updated.add(key)
is_updated = True
break
if not is_updated:
new_env_lines.append(stripped_line)
for key, value in onedrive_settings.items():
if key not in updated:
new_env_lines.append(f"{key}={value}")
with open(env_path, "w") as f:
f.write("\n".join(new_env_lines) + "\n")
logger.info("Successfully updated OneDrive settings in .env file")
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
# Update the settings in memory
if refresh_token:
settings.onedrive_refresh_token = refresh_token
if client_id:
settings.onedrive_client_id = client_id
if client_secret:
settings.onedrive_client_secret = client_secret
if tenant_id:
settings.onedrive_tenant_id = tenant_id
if folder_path:
settings.onedrive_folder_path = folder_path
# Persist to database (primary)
if refresh_token:
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
if client_id:
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
if client_secret:
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
if tenant_id:
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
if folder_path:
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
# Update in-memory settings and persist to database dynamically
for key, value in onedrive_settings.items():
setattr(settings, key, value)
save_setting_to_db(db, key, value, changed_by=changed_by)
notify_settings_updated()
app/api/plans.py
+11 -2
View File
@@ -196,11 +196,20 @@ def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
updated = 0
for sort_order, plan_id in enumerate(body.order):
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
# Fetch all requested plans in a single query to avoid N+1
plan_ids = body.order
plans = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id.in_(plan_ids)).all()
# Build a map for fast O(1) lookup
plan_map = {p.plan_id: p for p in plans}
for sort_order, plan_id in enumerate(plan_ids):
plan = plan_map.get(plan_id)
if plan:
plan.sort_order = sort_order
updated += 1
try:
db.commit()
except Exception:
app/api/profile.py
+358
View File
@@ -0,0 +1,358 @@
"""User self-service profile API.
Provides endpoints for the authenticated user to view and update their own
profile settings without requiring admin access.
Routes:
GET /api/profile — read current user's profile
PATCH /api/profile — update display name, language, theme
POST /api/profile/avatar — upload a new profile picture (JPEG/PNG/GIF/WebP, max 2 MB)
DELETE /api/profile/avatar — remove custom avatar (reverts to Gravatar)
POST /api/profile/change-password — change password (local-auth users only)
"""
from __future__ import annotations
import base64
import logging
from hashlib import md5
from typing import Annotated
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.models import LocalUser, UserProfile
from app.utils.i18n import SUPPORTED_LANGUAGE_CODES
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/profile", tags=["profile"])
DbSession = Annotated[Session, Depends(get_db)]
# Maximum avatar upload size: 2 MB
_MAX_AVATAR_BYTES = 2 * 1024 * 1024
# Allowed MIME types for avatar uploads
_ALLOWED_AVATAR_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
# Valid theme values
_VALID_THEMES = {"light", "dark", "system"}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _get_user_id(request: Request) -> str:
"""Return the stable user identifier from the session.
Raises HTTP 401 if no user is logged in.
"""
user = request.session.get("user")
if not user or not isinstance(user, dict):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
uid = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
if not uid:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Cannot determine user identity")
return uid
def _gravatar_url(email: str | None) -> str:
"""Generate a Gravatar URL for *email*, falling back to identicon."""
if not email:
return "https://www.gravatar.com/avatar/?d=identicon"
# MD5 used for Gravatar URL generation only — not for security
h = md5(email.strip().lower().encode(), usedforsecurity=False).hexdigest()
return f"https://www.gravatar.com/avatar/{h}?d=identicon"
def _get_or_create_profile(db: Session, user_id: str) -> UserProfile:
"""Return the UserProfile for *user_id*, creating a stub if one doesn't exist."""
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
if profile is None:
profile = UserProfile(user_id=user_id)
db.add(profile)
try:
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
return profile
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
class ProfileResponse(BaseModel):
"""Response body for GET /api/profile."""
user_id: str
display_name: str | None
contact_email: str | None
preferred_language: str | None
preferred_theme: str | None
default_document_language: str | None
"""ISO 639-1 code for the user's preferred document translation target language."""
avatar_url: str
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
is_local_user: bool
"""True when the account was created via local email/password sign-up."""
class ProfileUpdateRequest(BaseModel):
"""Request body for PATCH /api/profile."""
display_name: str | None = Field(default=None, max_length=255, description="Human-readable display name")
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
default_document_language: str | None = Field(
default=None,
description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'",
)
class ChangePasswordRequest(BaseModel):
"""Request body for POST /api/profile/change-password."""
current_password: str = Field(..., min_length=1, max_length=128)
new_password: str = Field(..., min_length=8, max_length=128)
new_password_confirm: str = Field(..., min_length=8, max_length=128)
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("", response_model=ProfileResponse)
@require_login
async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
"""Return the current user's profile settings."""
user_id = _get_user_id(request)
profile = _get_or_create_profile(db, user_id)
session_user = request.session.get("user", {})
email = session_user.get("email") if isinstance(session_user, dict) else None
# Determine avatar: prefer stored data, fall back to Gravatar
avatar_url = profile.avatar_data if profile.avatar_data else _gravatar_url(email) # type: ignore[attr-defined]
# Check whether this is a local (email/password) account
is_local = db.query(LocalUser).filter(LocalUser.username == user_id).first() is not None
return ProfileResponse(
user_id=user_id,
display_name=profile.display_name, # type: ignore[arg-type]
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
@router.patch("", response_model=ProfileResponse)
@require_login
async def update_profile(
body: ProfileUpdateRequest, request: Request, response: Response, db: DbSession
) -> ProfileResponse:
"""Update the current user's editable profile settings."""
user_id = _get_user_id(request)
profile = _get_or_create_profile(db, user_id)
# Validate language code
if body.preferred_language is not None:
lang = body.preferred_language.lower().strip()
if lang and lang not in SUPPORTED_LANGUAGE_CODES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported language code: {lang}",
)
profile.preferred_language = lang or None # type: ignore[assignment]
# Keep session and cookie in sync so detect_language() picks up
# the new preference immediately (without a DB round-trip).
if hasattr(request, "session"):
if lang:
request.session["preferred_language"] = lang
else:
request.session.pop("preferred_language", None)
if lang:
response.set_cookie(
key="docuelevate_lang",
value=lang,
max_age=30 * 24 * 60 * 60,
httponly=False,
samesite="lax",
)
else:
response.delete_cookie(key="docuelevate_lang")
# Validate theme
if body.preferred_theme is not None:
theme = body.preferred_theme.lower().strip()
if theme and theme not in _VALID_THEMES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Invalid theme: {theme}. Must be one of: {', '.join(sorted(_VALID_THEMES))}",
)
profile.preferred_theme = theme or None # type: ignore[assignment]
# Validate default document language
if body.default_document_language is not None:
doc_lang = body.default_document_language.lower().strip()
if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported language code: {doc_lang}",
)
profile.default_document_language = doc_lang or None # type: ignore[assignment]
if body.display_name is not None:
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
if body.contact_email is not None:
profile.contact_email = body.contact_email.strip() or None # type: ignore[assignment]
try:
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
session_user = request.session.get("user", {})
email = session_user.get("email") if isinstance(session_user, dict) else None
avatar_url = profile.avatar_data if profile.avatar_data else _gravatar_url(email) # type: ignore[attr-defined]
is_local = db.query(LocalUser).filter(LocalUser.username == user_id).first() is not None
return ProfileResponse(
user_id=user_id,
display_name=profile.display_name, # type: ignore[arg-type]
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
@router.post("/avatar", status_code=status.HTTP_200_OK)
@require_login
async def upload_avatar(
request: Request,
db: DbSession,
file: UploadFile = File(..., description="Profile picture (JPEG, PNG, GIF or WebP; max 2 MB)"),
) -> dict:
"""Upload a new profile picture.
The image is stored as a base64-encoded data URL in ``UserProfile.avatar_data``.
Accepts JPEG, PNG, GIF, or WebP files up to 2 MB.
"""
user_id = _get_user_id(request)
content_type = (file.content_type or "").lower()
if content_type not in _ALLOWED_AVATAR_TYPES:
raise HTTPException(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail=f"Unsupported image type '{content_type}'. Allowed: JPEG, PNG, GIF, WebP.",
)
# Check declared size first (available when the client sends a Content-Length header)
if file.size is not None and file.size > _MAX_AVATAR_BYTES:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail="Avatar image must be 2 MB or smaller.",
)
# Read up to one byte past the limit so we can detect oversized uploads
raw = await file.read(_MAX_AVATAR_BYTES + 1)
if len(raw) > _MAX_AVATAR_BYTES:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail="Avatar image must be 2 MB or smaller.",
)
b64 = base64.b64encode(raw).decode("ascii")
data_url = f"data:{content_type};base64,{b64}"
profile = _get_or_create_profile(db, user_id)
profile.avatar_data = data_url # type: ignore[assignment]
try:
db.commit()
except Exception:
db.rollback()
raise
return {"avatar_url": data_url}
@router.delete("/avatar", status_code=status.HTTP_200_OK)
@require_login
async def delete_avatar(request: Request, db: DbSession) -> dict:
"""Remove the custom avatar and revert to the Gravatar fallback."""
user_id = _get_user_id(request)
profile = _get_or_create_profile(db, user_id)
profile.avatar_data = None # type: ignore[assignment]
try:
db.commit()
except Exception:
db.rollback()
raise
session_user = request.session.get("user", {})
email = session_user.get("email") if isinstance(session_user, dict) else None
return {"avatar_url": _gravatar_url(email)}
@router.post("/change-password", status_code=status.HTTP_200_OK)
@require_login
async def change_password(body: ChangePasswordRequest, request: Request, db: DbSession) -> dict:
"""Change the password for local (email/password) accounts.
Raises 403 if the account is not a local account or the current password is wrong.
Raises 422 if the new passwords do not match.
"""
from app.utils.local_auth import hash_password, verify_password
user_id = _get_user_id(request)
local_user = db.query(LocalUser).filter(LocalUser.username == user_id).first()
if local_user is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Password change is only available for local accounts.",
)
if not verify_password(body.current_password, local_user.hashed_password):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Current password is incorrect.",
)
if body.new_password != body.new_password_confirm:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="New passwords do not match.",
)
local_user.hashed_password = hash_password(body.new_password)
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info("Password changed for local user: %s", user_id)
return {"detail": "Password changed successfully."}
app/api/routing_rules.py
+468
View File
@@ -0,0 +1,468 @@
"""Routing rules API endpoints.
Provides full CRUD for pipeline routing rules that conditionally assign
documents to pipelines based on document properties (file type, category,
metadata fields, size, etc.).
Rules are evaluated in ascending ``position`` order. The first rule whose
condition matches wins and routes the document to the specified target
pipeline. If no rule matches, the caller falls back to the owner's (or
system) default pipeline.
"""
import logging
from typing import Annotated, Any
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.models import Pipeline, PipelineRoutingRule
from app.utils.routing_engine import (
BUILTIN_FIELDS,
VALID_OPERATORS,
_evaluate_condition,
_resolve_field,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/routing-rules", tags=["routing-rules"])
DbSession = Annotated[Session, Depends(get_db)]
MAX_RULES_PER_OWNER = 100
MAX_NAME_LENGTH = 255
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _get_user_id(request: Request) -> str:
"""Return the authenticated user identifier."""
user = getattr(request.state, "user", None)
if user:
if isinstance(user, dict):
return user.get("sub", user.get("email", "anonymous"))
return getattr(user, "sub", getattr(user, "email", "anonymous"))
return "anonymous"
def _is_admin(request: Request) -> bool:
"""Return ``True`` when the current user has admin privileges."""
user = getattr(request.state, "user", None)
if not user:
return False
groups = user.get("groups", []) if isinstance(user, dict) else getattr(user, "groups", [])
return "admin" in groups
def _can_access_rule(rule: PipelineRoutingRule, user_id: str, admin: bool) -> bool:
"""Check whether the user is allowed to read this rule."""
if admin:
return True
return rule.owner_id == user_id
def _can_write_rule(rule: PipelineRoutingRule, user_id: str, admin: bool) -> bool:
"""Check whether the user is allowed to modify this rule."""
if rule.owner_id is None:
return admin
return rule.owner_id == user_id
def _validate_field(field: str) -> None:
"""Raise 422 if the field name is invalid."""
if field in BUILTIN_FIELDS:
return
if field.startswith("metadata.") and len(field) > len("metadata."):
return
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=(
f"Invalid field '{field}'. "
f"Valid built-in fields: {sorted(BUILTIN_FIELDS)}. "
"For AI metadata, use 'metadata.<key>'."
),
)
def _validate_operator(operator: str) -> None:
"""Raise 422 if the operator is not recognised."""
if operator not in VALID_OPERATORS:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Invalid operator '{operator}'. Valid operators: {sorted(VALID_OPERATORS)}",
)
def _serialize_rule(rule: PipelineRoutingRule) -> dict[str, Any]:
"""Serialize a routing rule to a JSON-compatible dict."""
return {
"id": rule.id,
"owner_id": rule.owner_id,
"name": rule.name,
"position": rule.position,
"field": rule.field,
"operator": rule.operator,
"value": rule.value,
"target_pipeline_id": rule.target_pipeline_id,
"is_active": rule.is_active,
"created_at": rule.created_at.isoformat() if rule.created_at else None,
"updated_at": rule.updated_at.isoformat() if rule.updated_at else None,
}
# ---------------------------------------------------------------------------
# Pydantic request models
# ---------------------------------------------------------------------------
class RoutingRuleCreate(BaseModel):
"""Request body for creating a routing rule."""
name: str = Field(..., min_length=1, max_length=MAX_NAME_LENGTH)
field: str = Field(..., min_length=1, max_length=255)
operator: str = Field(..., min_length=1, max_length=50)
value: str = Field(..., max_length=1024)
target_pipeline_id: int
position: int | None = None
is_active: bool = True
class RoutingRuleUpdate(BaseModel):
"""Request body for updating a routing rule."""
name: str | None = Field(None, min_length=1, max_length=MAX_NAME_LENGTH)
field: str | None = Field(None, min_length=1, max_length=255)
operator: str | None = Field(None, min_length=1, max_length=50)
value: str | None = Field(None, max_length=1024)
target_pipeline_id: int | None = None
position: int | None = None
is_active: bool | None = None
class RoutingRuleEvaluateRequest(BaseModel):
"""Request body for dry-run rule evaluation."""
file_type: str | None = None
filename: str | None = None
size: int | None = None
document_type: str | None = None
metadata: dict[str, Any] | None = None
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("")
@require_login
def list_routing_rules(request: Request, db: DbSession) -> list[dict[str, Any]]:
"""List all routing rules accessible by the current user.
Returns the user's own rules plus any system-wide rules (``owner_id=NULL``).
Rules are sorted by position.
"""
user_id = _get_user_id(request)
rules = (
db.query(PipelineRoutingRule)
.filter((PipelineRoutingRule.owner_id == user_id) | (PipelineRoutingRule.owner_id.is_(None)))
.order_by(
PipelineRoutingRule.owner_id.is_(None).asc(),
PipelineRoutingRule.position.asc(),
)
.all()
)
return [_serialize_rule(r) for r in rules]
@router.post("", status_code=status.HTTP_201_CREATED)
@require_login
def create_routing_rule(request: Request, db: DbSession, body: RoutingRuleCreate) -> dict[str, Any]:
"""Create a new routing rule for the current user.
Returns:
The created routing rule.
Raises:
HTTPException 422: If the field or operator is invalid.
HTTPException 404: If the target pipeline does not exist.
HTTPException 409: If the maximum number of rules is reached.
"""
user_id = _get_user_id(request)
_validate_field(body.field)
_validate_operator(body.operator)
# Verify target pipeline exists and is accessible.
pipeline = db.query(Pipeline).filter(Pipeline.id == body.target_pipeline_id).first()
if not pipeline:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Target pipeline {body.target_pipeline_id} not found",
)
# Enforce per-owner limit.
count = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.owner_id == user_id).count()
if count >= MAX_RULES_PER_OWNER:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Maximum of {MAX_RULES_PER_OWNER} routing rules per user reached",
)
# Auto-assign position if not specified.
position = body.position
if position is None:
max_pos = (
db.query(PipelineRoutingRule.position)
.filter(PipelineRoutingRule.owner_id == user_id)
.order_by(PipelineRoutingRule.position.desc())
.first()
)
position = (max_pos[0] + 1) if max_pos else 0
rule = PipelineRoutingRule(
owner_id=user_id,
name=body.name.strip(),
position=position,
field=body.field,
operator=body.operator,
value=body.value,
target_pipeline_id=body.target_pipeline_id,
is_active=body.is_active,
)
try:
db.add(rule)
db.commit()
db.refresh(rule)
except Exception:
db.rollback()
logger.exception("Failed to create routing rule for user=%s", user_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to create routing rule",
)
logger.info("Routing rule created: id=%s, user=%s", rule.id, user_id)
return _serialize_rule(rule)
@router.get("/operators")
def list_operators() -> dict[str, Any]:
"""Return the list of supported operators and fields.
This is a public endpoint (no auth required) so that UIs can populate
dropdowns without hard-coding the catalogue.
"""
return {
"operators": sorted(VALID_OPERATORS),
"builtin_fields": sorted(BUILTIN_FIELDS),
"metadata_prefix": "metadata.",
}
@router.post("/evaluate")
@require_login
def evaluate_rules(request: Request, db: DbSession, body: RoutingRuleEvaluateRequest) -> dict[str, Any]:
"""Dry-run rule evaluation against the provided document properties.
Returns the first matching rule and target pipeline (if any), or
indicates that no rule matched (default pipeline will be used).
"""
user_id = _get_user_id(request)
doc_props: dict[str, Any] = {
"file_type": body.file_type,
"filename": body.filename,
"size": body.size,
"document_type": body.document_type,
"metadata": body.metadata or {},
}
rules = (
db.query(PipelineRoutingRule)
.filter(
PipelineRoutingRule.is_active.is_(True),
(PipelineRoutingRule.owner_id == user_id) | (PipelineRoutingRule.owner_id.is_(None)),
)
.order_by(
PipelineRoutingRule.owner_id.is_(None).asc(),
PipelineRoutingRule.position.asc(),
)
.all()
)
for rule in rules:
actual = _resolve_field(rule.field, doc_props)
if _evaluate_condition(actual, rule.operator, rule.value):
pipeline = db.query(Pipeline).filter(Pipeline.id == rule.target_pipeline_id).first()
return {
"matched": True,
"rule": _serialize_rule(rule),
"target_pipeline": {
"id": pipeline.id,
"name": pipeline.name,
"is_active": pipeline.is_active,
}
if pipeline
else None,
}
return {"matched": False, "rule": None, "target_pipeline": None}
@router.put("/reorder")
@require_login
def reorder_routing_rules(
request: Request,
db: DbSession,
rule_ids: list[int] = Body(..., embed=True),
) -> list[dict[str, Any]]:
"""Reorder the caller's routing rules.
Expects a JSON body ``{"rule_ids": [3, 1, 2]}`` where the list
contains the IDs of the caller's rules in the desired order.
"""
user_id = _get_user_id(request)
rules = (
db.query(PipelineRoutingRule)
.filter(PipelineRoutingRule.owner_id == user_id, PipelineRoutingRule.id.in_(rule_ids))
.all()
)
rule_map = {r.id: r for r in rules}
if len(rule_map) != len(rule_ids) or set(rule_map.keys()) != set(rule_ids):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="rule_ids must contain exactly the IDs of your routing rules",
)
for pos, rid in enumerate(rule_ids):
rule_map[rid].position = pos
try:
db.commit()
except Exception:
db.rollback()
logger.exception("Failed to reorder routing rules for user=%s", user_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to reorder routing rules",
)
ordered = sorted(rules, key=lambda r: r.position)
return [_serialize_rule(r) for r in ordered]
@router.get("/{rule_id}")
@require_login
def get_routing_rule(rule_id: int, request: Request, db: DbSession) -> dict[str, Any]:
"""Return a single routing rule by ID."""
user_id = _get_user_id(request)
admin = _is_admin(request)
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
if not rule or not _can_access_rule(rule, user_id, admin):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
return _serialize_rule(rule)
@router.put("/{rule_id}")
@require_login
def update_routing_rule(rule_id: int, request: Request, db: DbSession, body: RoutingRuleUpdate) -> dict[str, Any]:
"""Update a routing rule.
Only the fields present in the request body are updated.
"""
user_id = _get_user_id(request)
admin = _is_admin(request)
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
if not rule or not _can_access_rule(rule, user_id, admin):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
if not _can_write_rule(rule, user_id, admin):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this rule")
if body.field is not None:
_validate_field(body.field)
rule.field = body.field
if body.operator is not None:
_validate_operator(body.operator)
rule.operator = body.operator
if body.value is not None:
rule.value = body.value
if body.name is not None:
rule.name = body.name.strip()
if body.target_pipeline_id is not None:
pipeline = db.query(Pipeline).filter(Pipeline.id == body.target_pipeline_id).first()
if not pipeline:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Target pipeline {body.target_pipeline_id} not found",
)
rule.target_pipeline_id = body.target_pipeline_id
if body.position is not None:
rule.position = body.position
if body.is_active is not None:
rule.is_active = body.is_active
try:
db.commit()
db.refresh(rule)
except Exception:
db.rollback()
logger.exception("Failed to update routing rule id=%s", rule_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to update routing rule",
)
logger.info("Routing rule updated: id=%s, user=%s", rule_id, user_id)
return _serialize_rule(rule)
@router.delete("/{rule_id}", status_code=status.HTTP_204_NO_CONTENT)
@require_login
def delete_routing_rule(rule_id: int, request: Request, db: DbSession) -> None:
"""Delete a routing rule."""
user_id = _get_user_id(request)
admin = _is_admin(request)
rule = db.query(PipelineRoutingRule).filter(PipelineRoutingRule.id == rule_id).first()
if not rule or not _can_access_rule(rule, user_id, admin):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Routing rule not found")
if not _can_write_rule(rule, user_id, admin):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this rule")
try:
db.delete(rule)
db.commit()
except Exception:
db.rollback()
logger.exception("Failed to delete routing rule id=%s", rule_id)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to delete routing rule",
)
logger.info("Routing rule deleted: id=%s, user=%s", rule_id, user_id)
app/api/shared_links.py
+7 -5
View File
@@ -313,16 +313,18 @@ async def list_shared_links(
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
) -> list[dict[str, Any]]:
"""List all shared links created by the authenticated user."""
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
q = (
db.query(SharedLink, FileRecord.original_filename)
.outerjoin(FileRecord, SharedLink.file_id == FileRecord.id)
.filter(SharedLink.owner_id == owner_id)
)
if active_only:
q = q.filter(SharedLink.is_active.is_(True))
links = q.order_by(SharedLink.created_at.desc()).all()
links_with_filenames = q.order_by(SharedLink.created_at.desc()).all()
base_url = str(request.base_url).rstrip("/")
result = []
for link in links:
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
filename = file_record.original_filename if file_record else None
for link, filename in links_with_filenames:
result.append(_link_to_dict(link, base_url, filename))
return result
app/api/translation.py
+156
View File
@@ -0,0 +1,156 @@
"""
API endpoints for document translation.
Provides on-the-fly translation via the AI provider and access to the
persisted default-language translation.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord
from app.utils.ai_provider import get_ai_provider
from app.utils.user_scope import apply_owner_filter
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
# Maximum characters sent to the AI provider for a single translation request.
_MAX_TRANSLATION_INPUT = 50_000
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
"""Fetch a FileRecord visible to the current user or raise 404."""
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
record = query.first()
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
return record
@router.get("/files/{file_id}/translation/default")
@require_login
def get_default_translation(
request: Request,
file_id: int,
db: DbSession,
) -> JSONResponse:
"""Return the persisted default-language translation for a document.
Returns 404 if no default-language translation has been generated yet
(e.g. because the document is already in the default language).
"""
record = _get_file_or_404(db, file_id, request)
if not record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available for this file",
)
return JSONResponse(
content={
"file_id": record.id,
"detected_language": record.detected_language,
"default_language_code": record.default_language_code,
"text": record.default_language_text,
}
)
@router.get("/files/{file_id}/translate")
@require_login
def translate_on_the_fly(
request: Request,
file_id: int,
db: DbSession,
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
) -> JSONResponse:
"""Translate a document's extracted text into an arbitrary language on the fly.
The translation is generated via the configured AI provider and is **not**
persisted. For the default-language translation, use the
``/files/{file_id}/translation/default`` endpoint instead.
"""
record = _get_file_or_404(db, file_id, request)
source_text = record.ocr_text
if not source_text:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No extracted text available for this file — translation requires OCR text",
)
# If the requested language matches what is already stored, return it directly.
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
return JSONResponse(
content={
"file_id": record.id,
"source_language": record.detected_language,
"target_language": lang,
"text": record.default_language_text,
"cached": True,
}
)
# If the detected language already matches, return the original text.
detected = record.detected_language
if detected and detected == lang:
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected,
"target_language": lang,
"text": source_text,
"cached": True,
}
)
# Truncate to keep AI costs bounded.
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"into {lang}. Preserve the original formatting, paragraph structure, "
f"and meaning. Do not add any commentary — output ONLY the translated text."
),
},
{"role": "user", "content": text_to_translate},
],
model=model,
temperature=0.3,
)
except Exception as exc:
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Translation failed — the AI provider returned an error",
)
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected or "unknown",
"target_language": lang,
"text": translated,
"cached": False,
}
)
app/api/url_upload.py
+64 -88
View File
@@ -2,7 +2,6 @@
API endpoint for processing files from URLs
"""
import ipaddress
import logging
import mimetypes
import os
@@ -10,7 +9,8 @@ import urllib.parse
import uuid
from typing import Optional
import requests
import aiofiles
import httpx
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, HttpUrl, field_validator
@@ -19,6 +19,7 @@ from app.config import settings
from app.tasks.process_document import process_document
from app.utils.allowed_types import ALLOWED_MIME_TYPES
from app.utils.filename_utils import sanitize_filename
from app.utils.network import is_private_ip
# Set up logging
logger = logging.getLogger(__name__)
@@ -42,37 +43,6 @@ class URLUploadRequest(BaseModel):
return v
def is_private_ip(hostname: str) -> bool:
"""
Check if a hostname resolves to a private/internal IP address.
Protects against SSRF attacks by blocking access to internal networks.
"""
try:
# Try to parse as IP address directly
ip = ipaddress.ip_address(hostname)
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
except ValueError:
# Not a direct IP, try to resolve hostname
try:
import socket
# Get all IP addresses for this hostname
addr_info = socket.getaddrinfo(hostname, None)
for info in addr_info:
ip_str = info[4][0]
ip = ipaddress.ip_address(ip_str)
# Block if ANY resolved IP is private/internal
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
return True
return False
except (socket.gaierror, socket.error):
# Cannot resolve - allow for testing/development
# In production, DNS should work properly
# Log this for debugging
logger.warning(f"Could not resolve hostname: {hostname}")
return False # Changed from True to False to allow external domains in tests
def validate_url_safety(url: str) -> None:
"""
Validate that URL is safe to fetch (SSRF protection).
@@ -184,67 +154,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
logger.info(f"Downloading file from URL: {url}")
# Use configured timeout to prevent hanging
response = requests.get(
url,
async with httpx.AsyncClient(
timeout=settings.http_request_timeout,
stream=True, # Stream to handle large files
allow_redirects=True, # Follow redirects
follow_redirects=True,
headers={
"User-Agent": "DocuElevate/1.0", # Identify ourselves
},
)
response.raise_for_status()
) as client:
async with client.stream("GET", url) as response:
response.raise_for_status()
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
if "." in safe_filename:
file_extension = safe_filename.rsplit(".", 1)[1]
target_filename = f"{unique_id}.{file_extension}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
with open(target_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
f.close()
os.remove(target_path)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
# Check for extension using original_filename to avoid any CodeQL issues
# with safe_filename which is derived from the URL directly.
if "." in original_filename:
_, ext = os.path.splitext(original_filename)
# Strip out the leading dot and any non-alphanumeric chars
clean_ext = "".join(c for c in ext if c.isalnum())
if not clean_ext:
clean_ext = "bin"
target_filename = f"{unique_id}.{clean_ext}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
async with aiofiles.open(target_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=8192):
if chunk:
await f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
)
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
# Enqueue for processing
@@ -258,19 +234,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
"size": downloaded_size,
}
except requests.exceptions.Timeout:
except httpx.TimeoutException:
logger.error(f"Timeout while downloading file from URL: {url}")
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
except requests.exceptions.ConnectionError as e:
except httpx.ConnectError as e:
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
except requests.exceptions.HTTPError as e:
except httpx.HTTPStatusError as e:
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
except requests.exceptions.RequestException as e:
except httpx.RequestError as e:
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
app/api/user.py
+20 -7
View File
@@ -12,7 +12,7 @@ from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.models import FileRecord
from app.models import FileRecord, UserProfile
# Set up logging
logger = logging.getLogger(__name__)
@@ -22,7 +22,7 @@ router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
async def whoami_handler(request: Request):
async def whoami_handler(request: Request, db: Session):
"""
Returns user info if logged in, else 401.
"""
@@ -41,20 +41,33 @@ async def whoami_handler(request: Request):
# Add the gravatar URL to the user object instead of creating a new response
user_response = user.copy() # Create a copy to avoid modifying the session
user_response["picture"] = gravatar_url
# Check if the user has a custom avatar stored in their profile
user_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
if user_id:
try:
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
if profile and profile.avatar_data:
user_response["picture"] = profile.avatar_data
else:
user_response["picture"] = gravatar_url
except Exception:
user_response["picture"] = gravatar_url
else:
user_response["picture"] = gravatar_url
return user_response
# Register the same handler under two different paths
@router.get("/whoami")
async def whoami(request: Request):
return await whoami_handler(request)
async def whoami(request: Request, db: DbSession):
return await whoami_handler(request, db)
@router.get("/auth/whoami")
async def auth_whoami(request: Request):
return await whoami_handler(request)
async def auth_whoami(request: Request, db: DbSession):
return await whoami_handler(request, db)
@router.get("/users/search")
app/auth.py
+538 -7
View File
@@ -4,23 +4,26 @@ import logging
import pathlib
from datetime import datetime, timezone
from functools import wraps
from urllib.parse import urlparse
from urllib.parse import urlencode, urlparse
from authlib.integrations.starlette_client import OAuth
from fastapi import APIRouter, Depends, Request, status
from fastapi.responses import JSONResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy import func
from sqlalchemy.orm import Session
from starlette.responses import RedirectResponse
from app.config import settings
from app.database import get_db
from app.middleware.audit_log import get_client_ip
# Conditional imports: only used when multi_user_enabled=True. Imported here at
# module level (not inside auth()) so they don't incur repeated import overhead.
# Guards at call-sites ensure they are never *called* in single-user mode.
from app.models import LocalUser as _LocalUser
from app.models import UserProfile as _UserProfile
from app.utils.i18n import translate as _translate
from app.utils.local_auth import build_session_user as _build_session_user
from app.utils.local_auth import verify_password as _verify_password
@@ -33,11 +36,15 @@ AUTH_ENABLED = settings.auth_enabled
# Set up templates for authentication
templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
templates = Jinja2Templates(directory=str(templates_dir))
templates.env.globals["_"] = lambda key, **kwargs: _translate(key, "en", **kwargs)
# Configure OAuth provider if credentials are provided
OAUTH_CONFIGURED = False
OAUTH_PROVIDER_NAME = "Single Sign-On"
# Social login providers that are enabled and registered
SOCIAL_PROVIDERS: dict[str, dict[str, str]] = {}
if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_secret:
oauth.register(
name="authentik",
@@ -49,6 +56,68 @@ if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_s
OAUTH_CONFIGURED = True
OAUTH_PROVIDER_NAME = settings.oauth_provider_name or "Authentik SSO"
# --- Social Login Providers ---------------------------------------------------
if AUTH_ENABLED and settings.social_auth_google_enabled:
if settings.social_auth_google_client_id and settings.social_auth_google_client_secret:
oauth.register(
name="google",
client_id=settings.social_auth_google_client_id,
client_secret=settings.social_auth_google_client_secret,
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
client_kwargs={"scope": "openid profile email"},
)
SOCIAL_PROVIDERS["google"] = {"name": "Google", "icon": "fab fa-google", "color": "red"}
logger.info("Social login provider registered: Google")
else:
logger.warning("SOCIAL_AUTH_GOOGLE_ENABLED=true but client ID/secret not configured")
if AUTH_ENABLED and settings.social_auth_microsoft_enabled:
if settings.social_auth_microsoft_client_id and settings.social_auth_microsoft_client_secret:
tenant = settings.social_auth_microsoft_tenant or "common"
oauth.register(
name="microsoft",
client_id=settings.social_auth_microsoft_client_id,
client_secret=settings.social_auth_microsoft_client_secret,
server_metadata_url=f"https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration",
client_kwargs={"scope": "openid profile email"},
)
SOCIAL_PROVIDERS["microsoft"] = {"name": "Microsoft", "icon": "fab fa-microsoft", "color": "blue"}
logger.info("Social login provider registered: Microsoft (tenant=%s)", tenant)
else:
logger.warning("SOCIAL_AUTH_MICROSOFT_ENABLED=true but client ID/secret not configured")
if AUTH_ENABLED and settings.social_auth_apple_enabled:
if settings.social_auth_apple_client_id and settings.social_auth_apple_team_id:
oauth.register(
name="apple",
client_id=settings.social_auth_apple_client_id,
server_metadata_url="https://appleid.apple.com/.well-known/openid-configuration",
client_kwargs={
"scope": "openid name email",
"response_mode": "form_post",
},
)
SOCIAL_PROVIDERS["apple"] = {"name": "Apple", "icon": "fab fa-apple", "color": "gray"}
logger.info("Social login provider registered: Apple")
else:
logger.warning("SOCIAL_AUTH_APPLE_ENABLED=true but client ID/team ID not configured")
if AUTH_ENABLED and settings.social_auth_dropbox_enabled:
if settings.social_auth_dropbox_client_id and settings.social_auth_dropbox_client_secret:
oauth.register(
name="dropbox",
client_id=settings.social_auth_dropbox_client_id,
client_secret=settings.social_auth_dropbox_client_secret,
authorize_url="https://www.dropbox.com/oauth2/authorize",
access_token_url="https://api.dropboxapi.com/oauth2/token",
userinfo_endpoint="https://api.dropboxapi.com/2/users/get_current_account",
client_kwargs={"token_endpoint_auth_method": "client_secret_post"},
)
SOCIAL_PROVIDERS["dropbox"] = {"name": "Dropbox", "icon": "fab fa-dropbox", "color": "blue"}
logger.info("Social login provider registered: Dropbox")
else:
logger.warning("SOCIAL_AUTH_DROPBOX_ENABLED=true but client ID/secret not configured")
router = APIRouter()
@@ -56,8 +125,17 @@ def get_current_user(request: Request):
# Check for Bearer token auth first (API tokens)
api_user = getattr(request.state, "api_token_user", None)
if isinstance(api_user, dict):
logger.debug("[AUTH] get_current_user: resolved from API token (user_id=%s)", api_user.get("id"))
return api_user
return request.session.get("user")
session_user = request.session.get("user")
if session_user:
logger.debug(
"[AUTH] get_current_user: resolved from session (user=%s)",
session_user.get("preferred_username") or session_user.get("email") or session_user.get("id"),
)
else:
logger.debug("[AUTH] get_current_user: no user in session or API token")
return session_user
def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
@@ -72,10 +150,12 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
"""
auth_header = request.headers.get("authorization", "")
if not isinstance(auth_header, str) or not auth_header.startswith("Bearer "):
logger.debug("[AUTH] _resolve_bearer_user: no Bearer token in Authorization header")
return None
raw_token = auth_header[7:]
if not raw_token or not isinstance(raw_token, str):
logger.debug("[AUTH] _resolve_bearer_user: empty or invalid token after 'Bearer ' prefix")
return None
from app.api.api_tokens import hash_token
@@ -84,8 +164,15 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
token_hash = hash_token(raw_token)
db_token = db.query(ApiToken).filter(ApiToken.token_hash == token_hash, ApiToken.is_active.is_(True)).first()
if db_token is None:
logger.debug("[AUTH] _resolve_bearer_user: no active API token matched the provided hash")
return None
logger.debug(
"[AUTH] _resolve_bearer_user: matched API token id=%s owner=%s",
db_token.id,
db_token.owner_id,
)
# Update usage tracking
try:
db_token.last_used_at = datetime.now(timezone.utc)
@@ -136,16 +223,18 @@ def require_login(func):
@wraps(func)
async def wrapper(request: Request, *args, **kwargs):
url_path = urlparse(str(request.url)).path
# Check session auth first
if request.session.get("user"):
logger.debug("[AUTH] require_login: session auth OK for %s", url_path)
if inspect.iscoroutinefunction(func):
return await func(*args, request=request, **kwargs)
else:
return func(*args, request=request, **kwargs)
# Fall back to Bearer token auth for API endpoints
url_path = urlparse(str(request.url)).path
if url_path.startswith("/api/"):
logger.debug("[AUTH] require_login: no session, trying Bearer token for %s", url_path)
try:
from app.database import SessionLocal
@@ -159,17 +248,22 @@ def require_login(func):
if api_user:
request.state.api_token_user = api_user
logger.debug(
"[AUTH] require_login: Bearer token auth OK for %s (user=%s)", url_path, api_user.get("id")
)
if inspect.iscoroutinefunction(func):
return await func(*args, request=request, **kwargs)
else:
return func(*args, request=request, **kwargs)
logger.debug("[AUTH] require_login: no valid auth for API endpoint %s — returning 401", url_path)
return JSONResponse(
status_code=status.HTTP_401_UNAUTHORIZED,
content={"error": "Not authenticated"},
)
# Non-API endpoint with no session — redirect to login
logger.debug("[AUTH] require_login: no session for %s — redirecting to /login", url_path)
request.session["redirect_after_login"] = str(request.url)
return RedirectResponse(url="/login", status_code=status.HTTP_302_FOUND)
@@ -186,6 +280,38 @@ def get_gravatar_url(email):
async def login(request: Request):
"""Show login page with appropriate authentication options."""
# Persist the mobile deep-link redirect URI in the session so it survives
# the OAuth provider round-trip and is available when auth completes.
# Accepted schemes:
# • "docuelevate://" — production / EAS builds (custom app scheme)
# • "exp://" — Expo Go development client
# Only custom (non-HTTP) schemes are accepted to prevent open-redirect abuse.
_MOBILE_ALLOWED_SCHEMES = ("docuelevate://", "exp://")
if request.query_params.get("mobile") == "1":
redirect_uri = request.query_params.get("redirect_uri", "")
logger.debug(
"[MOBILE] Login page opened with mobile=1: redirect_uri=%r client_ip=%s",
redirect_uri,
get_client_ip(request),
)
if any(redirect_uri.startswith(s) for s in _MOBILE_ALLOWED_SCHEMES):
request.session["mobile_redirect_uri"] = redirect_uri
logger.info(
"[MOBILE] Mobile redirect URI stored in session: %r",
redirect_uri,
)
else:
logger.warning(
"[MOBILE] Rejected redirect_uri with disallowed scheme: %r (allowed: %s)",
redirect_uri,
", ".join(_MOBILE_ALLOWED_SCHEMES),
)
else:
logger.debug(
"[MOBILE] Login page opened without mobile=1 (standard browser flow) client_ip=%s",
get_client_ip(request),
)
return templates.TemplateResponse(
"login.html",
{
@@ -194,6 +320,7 @@ async def login(request: Request):
"message": request.query_params.get("message"),
"show_oauth": OAUTH_CONFIGURED,
"oauth_provider_name": OAUTH_PROVIDER_NAME,
"social_providers": SOCIAL_PROVIDERS,
"app_version": settings.version,
"csrf_token": getattr(request.state, "csrf_token", ""),
# "Create account" link is only shown when multi-user mode AND local signup are both enabled
@@ -205,12 +332,197 @@ async def login(request: Request):
async def oauth_login(request: Request):
"""Handle OAuth login flow"""
if not OAUTH_CONFIGURED:
logger.debug("[AUTH] oauth_login: OAuth not configured — redirecting to /login")
return RedirectResponse(url="/login?error=OAuth+not+configured", status_code=status.HTTP_302_FOUND)
redirect_uri = request.url_for("oauth_callback")
logger.debug(
"[AUTH] oauth_login: initiating Authentik OAuth redirect_uri=%s session_keys=%s",
redirect_uri,
list(request.session.keys()),
)
return await oauth.authentik.authorize_redirect(request, redirect_uri)
async def social_login(request: Request, provider: str):
"""Initiate a social login flow for the given provider.
Args:
request: The current FastAPI request.
provider: One of the registered social provider keys (google, microsoft, apple, dropbox).
Returns:
A redirect to the provider's authorization page, or back to /login on error.
"""
if provider not in SOCIAL_PROVIDERS:
logger.debug(
"[AUTH] social_login: unknown provider=%r (registered=%s)", provider, list(SOCIAL_PROVIDERS.keys())
)
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
redirect_uri = request.url_for("social_callback", provider=provider)
oauth_client = getattr(oauth, provider, None)
if oauth_client is None:
logger.debug("[AUTH] social_login: provider=%r registered but OAuth client not configured", provider)
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
logger.debug(
"[AUTH] social_login: initiating %s OAuth, redirect_uri=%s session_keys=%s",
provider,
redirect_uri,
list(request.session.keys()),
)
return await oauth_client.authorize_redirect(request, redirect_uri)
def _normalize_social_userinfo(provider: str, token: dict, raw_userinfo: dict | None) -> dict:
"""Normalize the userinfo payload from different social providers into a common format.
Returns a dict with keys: sub, email, name, preferred_username, picture.
Args:
provider: The social provider key (google, microsoft, apple, dropbox).
token: The OAuth token response from the provider. Included for future
provider-specific claim extraction (e.g. ``id_token`` claims).
raw_userinfo: The raw userinfo dict (may be None for providers without standard OIDC userinfo).
Returns:
A normalized user-data dict compatible with the session user format.
"""
userinfo: dict = raw_userinfo or {}
if provider == "dropbox":
# Dropbox returns a non-standard userinfo response
email = userinfo.get("email", "")
name_info = userinfo.get("name", {})
display_name = name_info.get("display_name", "") if isinstance(name_info, dict) else str(name_info)
return {
"sub": userinfo.get("account_id", email),
"email": email,
"name": display_name,
"preferred_username": email,
"picture": userinfo.get("profile_photo_url", ""),
}
# Standard OIDC providers (Google, Microsoft, Apple)
return {
"sub": userinfo.get("sub", ""),
"email": userinfo.get("email", ""),
"name": userinfo.get("name", ""),
"preferred_username": userinfo.get("email", ""),
"picture": userinfo.get("picture", ""),
}
async def social_callback(request: Request, provider: str, db: Session = Depends(get_db)):
"""Handle the OAuth callback from a social login provider.
After the user authorizes with the social provider, this endpoint exchanges
the authorization code for tokens, extracts user information, creates or
updates the user profile, and establishes a session.
Args:
request: The current FastAPI request.
provider: One of the registered social provider keys.
db: Database session (injected).
Returns:
A redirect to the user's original destination or the upload page.
"""
if provider not in SOCIAL_PROVIDERS:
logger.debug("[AUTH] social_callback: unknown provider=%r", provider)
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
oauth_client = getattr(oauth, provider, None)
if oauth_client is None:
logger.debug("[AUTH] social_callback: provider=%r not configured", provider)
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
try:
logger.debug("[AUTH] social_callback: exchanging auth code for provider=%s", provider)
token = await oauth_client.authorize_access_token(request)
# Try standard OIDC userinfo first, fall back to token-embedded userinfo
raw_userinfo = token.get("userinfo")
if not raw_userinfo:
logger.debug("[AUTH] social_callback: no userinfo in token, fetching from userinfo endpoint")
try:
resp = await oauth_client.userinfo(token=token)
raw_userinfo = resp if isinstance(resp, dict) else resp.json() if hasattr(resp, "json") else {}
except Exception:
logger.debug("[AUTH] social_callback: userinfo endpoint failed, using empty dict", exc_info=True)
raw_userinfo = {}
user_data = _normalize_social_userinfo(provider, token, raw_userinfo)
logger.debug(
"[AUTH] social_callback: normalized user_data email=%s sub=%s provider=%s",
user_data.get("email"),
user_data.get("sub"),
provider,
)
if not user_data.get("email"):
logger.debug("[AUTH] social_callback: no email in user_data — aborting")
return RedirectResponse(
url="/login?error=Could+not+retrieve+email+from+provider",
status_code=status.HTTP_302_FOUND,
)
# Add Gravatar if no picture provided
if not user_data.get("picture") and user_data.get("email"):
user_data["picture"] = get_gravatar_url(user_data["email"])
# Tag the login source for audit/debugging
user_data["auth_provider"] = provider
# Social login users are never admin by default (admin must be granted
# via the Authentik/OIDC admin group or manually in the admin panel)
user_data["is_admin"] = False
request.session["user"] = user_data
# Auto-create or update UserProfile
_ensure_user_profile(db, user_data, is_admin=False)
provider_name = SOCIAL_PROVIDERS[provider]["name"]
logger.info(
"[SECURITY] SOCIAL_LOGIN_SUCCESS provider=%s user=%s", provider_name, user_data.get("email", "unknown")
)
# Redirect first-time users to onboarding
user_id = (
user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id")
)
# Mobile app flow: issue an inline API token and redirect back to the app.
logger.debug(
"[MOBILE] social_callback: checking for mobile redirect (session has mobile_redirect_uri=%s)",
"mobile_redirect_uri" in request.session,
)
mobile_resp = _create_mobile_redirect(request, db)
if mobile_resp:
logger.info("[MOBILE] social_callback: returning mobile redirect response for provider=%s", provider)
return mobile_resp
if user_id:
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
if profile and not profile.onboarding_completed:
logger.debug("[AUTH] social_callback: user=%s needs onboarding, redirecting", user_id)
post_onboarding = request.session.pop("redirect_after_login", "/upload")
request.session["post_onboarding_redirect"] = post_onboarding
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
redirect_url = request.session.pop("redirect_after_login", "/upload")
logger.debug("[AUTH] social_callback: login complete, redirecting to %s", redirect_url)
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
except Exception as e:
logger.warning("[SECURITY] SOCIAL_LOGIN_FAILURE provider=%s error=%s", provider, type(e).__name__)
logger.debug("[AUTH] social_callback: full exception for provider=%s", provider, exc_info=True)
return RedirectResponse(
url="/login?error=Social+login+failed.+Please+try+again.", status_code=status.HTTP_302_FOUND
)
def _ensure_user_profile(db: Session, user_data: dict, is_admin: bool = False) -> None:
"""Create or update a UserProfile row for *user_data*.
@@ -310,15 +622,23 @@ def _ensure_user_profile(db: Session, user_data: dict, is_admin: bool = False) -
async def oauth_callback(request: Request, db: Session = Depends(get_db)):
"""Handle OAuth callback from provider"""
try:
logger.debug("[AUTH] oauth_callback: exchanging authorization code for token")
token = await oauth.authentik.authorize_access_token(request)
userinfo = token.get("userinfo")
if not userinfo:
logger.debug("[AUTH] oauth_callback: no userinfo in token response — aborting")
return RedirectResponse(
url="/login?error=Failed+to+retrieve+user+information", status_code=status.HTTP_302_FOUND
)
# Store user info in session
user_data = dict(userinfo)
logger.debug(
"[AUTH] oauth_callback: received userinfo email=%s sub=%s groups=%s",
user_data.get("email"),
user_data.get("sub"),
user_data.get("groups", []),
)
# Add Gravatar picture if no picture is provided
if not user_data.get("picture") and user_data.get("email"):
@@ -333,6 +653,12 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)):
groups = user_data.get("groups", [])
admin_group = (settings.admin_group_name or "admin").strip().lower()
is_admin = admin_group in [group.lower() for group in groups]
logger.debug(
"[AUTH] oauth_callback: admin group check — looking for %r in %s → is_admin=%s",
admin_group,
[g.lower() for g in groups],
is_admin,
)
# Set is_admin flag (defaults to False for OAuth users unless they're in admin group)
user_data["is_admin"] = is_admin
@@ -344,26 +670,183 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)):
# Log the successful authentication
logger.info("[SECURITY] OAUTH_LOGIN_SUCCESS user=%s admin=%s", user_data.get("email", "unknown"), is_admin)
_record_login_event(
db,
request,
user_data.get("email") or user_data.get("preferred_username") or "unknown",
success=True,
method="oauth",
)
# Redirect first-time users to onboarding
user_id = (
user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id")
)
# Mobile app flow: issue an inline API token and redirect back to the app.
# This check runs before onboarding so native-app users are never sent
# to the web-based onboarding wizard.
logger.debug(
"[MOBILE] oauth_callback: checking for mobile redirect (session has mobile_redirect_uri=%s)",
"mobile_redirect_uri" in request.session,
)
mobile_resp = _create_mobile_redirect(request, db)
if mobile_resp:
logger.info("[MOBILE] oauth_callback: returning mobile redirect response")
return mobile_resp
if user_id:
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
if profile and not profile.onboarding_completed:
logger.debug("[AUTH] oauth_callback: user=%s needs onboarding, redirecting", user_id)
post_onboarding = request.session.pop("redirect_after_login", "/upload")
request.session["post_onboarding_redirect"] = post_onboarding
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
# Redirect to original destination or default
redirect_url = request.session.pop("redirect_after_login", "/upload")
logger.debug("[AUTH] oauth_callback: login complete, redirecting to %s", redirect_url)
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
except Exception as e:
logger.warning(f"[SECURITY] OAUTH_LOGIN_FAILURE error={type(e).__name__}")
logger.debug("[AUTH] oauth_callback: full exception details", exc_info=True)
return RedirectResponse(url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND)
def _record_login_event(
db: Session,
request: Request,
username: str,
*,
success: bool,
method: str = "local",
detail: str | None = None,
) -> None:
"""Write a login or login-failure audit event to the database.
Failures are silently swallowed so that an audit-service error never
prevents a legitimate login or surfaces an unrelated 500 error to the user.
Args:
db: Active database session.
request: The current HTTP request (used to extract the client IP).
username: The username that attempted authentication.
success: ``True`` for a successful login, ``False`` for a failure.
method: Authentication method, e.g. ``"local"`` or ``"oauth"``.
detail: Optional extra context for failures (e.g. ``"wrong_password"``).
"""
try:
from app.utils.audit_service import record_event
action = "login" if success else "login.failure"
details: dict = {"method": method}
if detail:
details["reason"] = detail
record_event(
db,
action=action,
user=username,
resource_type="session",
ip_address=get_client_ip(request),
details=details,
severity="info" if success else "warning",
)
except Exception:
logger.debug("Failed to write login audit event for user=%s", username, exc_info=True)
def _create_mobile_redirect(request: Request, db: Session) -> RedirectResponse | None:
"""Generate a mobile API token and return a redirect to the mobile app.
If ``mobile_redirect_uri`` is stored in the session (set when the login
page was opened with ``?mobile=1&redirect_uri=docuelevate://...``), this
function creates a long-lived API token, appends it as a ``?token=``
query parameter to the redirect URI, and returns the redirect so that
``WebBrowser.openAuthSessionAsync`` in the Expo app intercepts the
deep link and stores the token.
Returns ``None`` when the request is not part of a mobile SSO flow.
Args:
request: The current FastAPI request. The ``user`` dict must already
be stored in ``request.session`` before calling this function.
db: Active database session used to persist the new API token.
Returns:
A ``RedirectResponse`` to the deep-link URI with ``?token=<plaintext>``,
or ``None`` if no mobile redirect URI is pending.
"""
mobile_redirect_uri = request.session.pop("mobile_redirect_uri", None)
if not mobile_redirect_uri:
logger.debug("[MOBILE] _create_mobile_redirect: no mobile_redirect_uri in session — skipping mobile flow")
return None
logger.info(
"[MOBILE] _create_mobile_redirect: mobile flow detected, redirect_uri=%r",
mobile_redirect_uri,
)
user = request.session.get("user") or {}
owner_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
logger.debug(
"[MOBILE] Resolving owner_id from session user: sub=%r preferred_username=%r email=%r id=%r → owner_id=%r",
user.get("sub"),
user.get("preferred_username"),
user.get("email"),
user.get("id"),
owner_id,
)
if not owner_id:
logger.warning("Mobile SSO redirect requested but no owner_id could be resolved from session")
return None
# Lazy imports to avoid circular dependency via app.api.__init__
from app.api.api_tokens import generate_api_token, hash_token
from app.models import ApiToken as _ApiToken
plaintext = generate_api_token()
token_hash_value = hash_token(plaintext)
prefix = plaintext[:12]
db_token = _ApiToken(
owner_id=owner_id,
name="Mobile App",
token_hash=token_hash_value,
token_prefix=prefix,
)
try:
db.add(db_token)
db.commit()
except Exception:
db.rollback()
logger.exception("Failed to create mobile API token for owner_id=%s", owner_id)
return None
# Safely append the token as a query parameter, preserving any existing params.
separator = "&" if "?" in mobile_redirect_uri else "?"
redirect_url = f"{mobile_redirect_uri}{separator}{urlencode({'token': plaintext})}"
# Log the full redirect URL at DEBUG so it's visible when debug logging is enabled.
# At INFO level, log a sanitised version (scheme + host only, token prefix only)
# so the plaintext token is never written to persistent info logs.
parsed = urlparse(redirect_url)
existing_params = f"&{parsed.query.replace(f'token={plaintext}', '')}" if parsed.query else ""
sanitised_url = (
f"{parsed.scheme}://{parsed.netloc}{parsed.path}?token={prefix}…[redacted]{existing_params.rstrip('&')}"
)
logger.info(
"[MOBILE] MOBILE_SSO_TOKEN_ISSUED owner=%s token_id=%s redirect_target=%s",
owner_id,
db_token.id,
sanitised_url,
)
logger.debug(
"[MOBILE] Full redirect URL being sent to client: %s",
redirect_url,
)
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
async def auth(request: Request, db: Session = Depends(get_db)):
"""Handle local username/password authentication.
@@ -398,8 +881,13 @@ async def auth(request: Request, db: Session = Depends(get_db)):
)
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
username_lower = username.lower()
local_user = (
db.query(_LocalUser).filter((_LocalUser.username == username) | (_LocalUser.email == username)).first()
db.query(_LocalUser)
.filter(
(func.lower(_LocalUser.username) == username_lower) | (func.lower(_LocalUser.email) == username_lower)
)
.first()
)
logger.debug(
"[AUTH] LocalUser lookup: username=%r found=%s",
@@ -413,6 +901,7 @@ async def auth(request: Request, db: Session = Depends(get_db)):
username,
local_user.is_active,
)
_record_login_event(db, request, username, success=False, detail="account_not_verified")
return RedirectResponse(
url="/login?error=Please+verify+your+email+address+before+logging+in",
status_code=302,
@@ -425,11 +914,24 @@ async def auth(request: Request, db: Session = Depends(get_db)):
)
if not pw_ok:
logger.warning("[SECURITY] LOCAL_LOGIN_FAILURE reason=wrong_password user=%s", username)
_record_login_event(db, request, username, success=False, detail="wrong_password")
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
user_data = _build_session_user(local_user)
request.session["user"] = user_data
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", local_user.email)
_record_login_event(db, request, local_user.email, success=True)
_ensure_user_profile(db, user_data, is_admin=bool(local_user.is_admin))
# Mobile app flow: issue an inline API token and redirect back to the app.
logger.debug(
"[MOBILE] local auth: checking for mobile redirect (session has mobile_redirect_uri=%s)",
"mobile_redirect_uri" in request.session,
)
mobile_resp = _create_mobile_redirect(request, db)
if mobile_resp:
logger.info("[MOBILE] local auth: returning mobile redirect response")
return mobile_resp
profile = db.query(_UserProfile).filter(_UserProfile.user_id == local_user.email).first()
if profile and not profile.onboarding_completed:
post_onboarding = request.session.pop("redirect_after_login", "/upload")
@@ -454,13 +956,13 @@ async def auth(request: Request, db: Session = Depends(get_db)):
logger.debug(
"[AUTH] Admin credential check: admin_configured=%s username_match=%s multi_user_enabled=%s",
admin_configured,
username == settings.admin_username if admin_configured else False,
(username or "").lower() == settings.admin_username.lower() if admin_configured else False,
settings.multi_user_enabled,
)
if (
settings.admin_username
and settings.admin_password
and username == settings.admin_username
and (username or "").lower() == settings.admin_username.lower()
and password == settings.admin_password
):
admin_user_data = {
@@ -473,7 +975,19 @@ async def auth(request: Request, db: Session = Depends(get_db)):
}
request.session["user"] = admin_user_data
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", username)
_record_login_event(db, request, username, success=True)
_ensure_user_profile(db, admin_user_data, is_admin=True)
# Mobile app flow: issue an inline API token and redirect back to the app.
logger.debug(
"[MOBILE] admin auth: checking for mobile redirect (session has mobile_redirect_uri=%s)",
"mobile_redirect_uri" in request.session,
)
mobile_resp = _create_mobile_redirect(request, db)
if mobile_resp:
logger.info("[MOBILE] admin auth: returning mobile redirect response")
return mobile_resp
redirect_url = request.session.pop("redirect_after_login", "/upload")
return RedirectResponse(url=redirect_url, status_code=302)
else:
@@ -485,16 +999,31 @@ async def auth(request: Request, db: Session = Depends(get_db)):
admin_configured,
not username and not password,
)
_record_login_event(db, request, username or "anonymous", success=False, detail="invalid_credentials")
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
async def logout(request: Request):
async def logout(request: Request, db: Session = Depends(get_db)):
"""Handle user logout"""
user = request.session.get("user")
username = "unknown"
if isinstance(user, dict):
username = user.get("preferred_username") or user.get("email") or "unknown"
logger.debug("[AUTH] logout: clearing session for user=%s client_ip=%s", username, get_client_ip(request))
logger.info(f"[SECURITY] LOGOUT user={username}")
try:
from app.utils.audit_service import record_event
record_event(
db,
action="logout",
user=username,
resource_type="session",
ip_address=get_client_ip(request),
severity="info",
)
except Exception:
logger.debug("Failed to write logout audit event for user=%s", username, exc_info=True)
request.session.pop("user", None)
return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302)
@@ -503,6 +1032,8 @@ if AUTH_ENABLED:
router.add_api_route("/login", login, methods=["GET"])
router.add_api_route("/oauth-login", oauth_login, methods=["GET"])
router.add_api_route("/oauth-callback", oauth_callback, methods=["GET"])
router.add_api_route("/social-login/{provider}", social_login, methods=["GET"])
router.add_api_route("/social-callback/{provider}", social_callback, methods=["GET"])
router.add_api_route("/auth", auth, methods=["POST"])
router.add_api_route("/logout", logout, methods=["GET"])
app/celery_app.py
+69 -2
View File
@@ -1,10 +1,15 @@
# app/celery_app.py
import logging
import os
from celery import Celery
from celery.signals import task_failure, worker_ready
from app.config import settings
logger = logging.getLogger(__name__)
celery = Celery(
"document_processor",
broker=settings.redis_url,
@@ -21,6 +26,64 @@ celery.conf.task_routes = {
"app.tasks.*": {"queue": "document_processor"},
}
# Mapping of document pipeline task names to the positional index of ``file_id``
# in their ``args`` tuple. These indices correspond to the task signatures:
# process_with_ocr(filename, file_id, ...) → index 1
# extract_metadata_with_gpt(filename, text, file_id) → index 2
# embed_metadata_into_pdf(path, text, metadata, file_id) → index 3
# Tasks that always pass ``file_id`` as a keyword argument
# (e.g. ``process_document``, ``finalize_document_storage``) are not listed
# here — their ``file_id`` is found via ``kwargs`` instead.
_FILE_ID_ARG_INDEX: dict[str, int] = {
"app.tasks.process_with_ocr.process_with_ocr": 1,
"app.tasks.extract_metadata_with_gpt.extract_metadata_with_gpt": 2,
"app.tasks.embed_metadata_into_pdf.embed_metadata_into_pdf": 3,
}
def _dispatch_user_failure_notification(sender, exception, args: list | None, kwargs: dict | None) -> None:
"""Best-effort per-user failure notification for document pipeline tasks.
Extracts ``file_id`` from the failed task's arguments, looks up the owning
user from the database, and dispatches a ``document.failed`` notification.
"""
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.user_notification import notify_user_document_failed
task_name = sender.name if sender else ""
if not task_name.startswith("app.tasks."):
return
# 1. Resolve file_id from kwargs or positional args
file_id = (kwargs or {}).get("file_id")
if file_id is None:
idx = _FILE_ID_ARG_INDEX.get(task_name)
if idx is not None and args and len(args) > idx:
val = args[idx]
if isinstance(val, int):
file_id = val
if file_id is None:
return
# 2. Look up owner from the database
with SessionLocal() as db:
record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not record or not record.owner_id:
return
owner_id = record.owner_id
filename = record.original_filename or record.local_filename or "unknown"
# 3. Dispatch per-user notification
error_msg = f"{type(exception).__name__}: {exception}" if exception else "Unknown error"
notify_user_document_failed(
owner_id=owner_id,
filename=os.path.basename(filename),
error=error_msg,
file_id=file_id,
)
@worker_ready.connect
def init_sentry_on_worker_ready(**kwargs):
@@ -48,6 +111,10 @@ def task_failure_handler(
kwargs=kwargs or {},
)
except Exception as e:
import logging
logger.exception(f"Failed to send task failure notification: {e}")
logging.exception(f"Failed to send task failure notification: {e}")
# Also dispatch a per-user failure notification for document pipeline tasks
try:
_dispatch_user_failure_notification(sender, exception, args, kwargs)
except Exception:
logger.warning("Could not dispatch per-user failure notification", exc_info=True)
app/celery_worker.py
+2
View File
@@ -40,12 +40,14 @@ from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
from app.tasks.subscription_tasks import apply_pending_subscription_changes_all # noqa: F401
from app.tasks.translate_to_default_language import translate_to_default_language # noqa: F401
# Import new send tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
from app.tasks.upload_to_email import upload_to_email # noqa: F401
from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401
from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401
from app.tasks.upload_to_icloud import upload_to_icloud # noqa: F401
from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401
from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
app/config.py
+209 -1
View File
@@ -48,19 +48,76 @@ class Settings(BaseSettings):
workdir: str
debug: bool = False # Default to False
# Logging level for the application. Accepts standard Python level names:
# DEBUG, INFO, WARNING, ERROR, CRITICAL. When *debug* is True and
# *log_level* has not been explicitly set, the effective level is forced to
# DEBUG so that all ``logger.debug()`` calls produce output.
log_level: str = Field(
default="INFO",
description=(
"Python logging level for the application root logger. "
"Accepts: DEBUG, INFO, WARNING, ERROR, CRITICAL. "
"When DEBUG=True and LOG_LEVEL is not explicitly set, "
"the effective level is automatically lowered to DEBUG."
),
)
# Log output format. ``text`` is the human-readable default.
# ``json`` emits one JSON object per line, ideal for log collectors
# (Promtail, Fluentd, Filebeat, Datadog agent) and SIEM ingestion.
log_format: str = Field(
default="text",
description=(
"Log output format: 'text' (human-readable, default) or "
"'json' (structured JSON lines for SIEM / log aggregation)."
),
)
# Optional syslog forwarding for application logs (not just audit events).
# When enabled, a Python SysLogHandler is added to the root logger so that
# every log message is also sent to the configured syslog receiver.
log_syslog_enabled: bool = Field(
default=False,
description="Forward application logs to a syslog receiver in addition to stdout.",
)
log_syslog_host: str = Field(
default="localhost",
description="Hostname or IP of the syslog receiver for application logs.",
)
log_syslog_port: int = Field(
default=514,
description="Port of the syslog receiver for application logs.",
)
log_syslog_protocol: str = Field(
default="udp",
description="Protocol for syslog transport: 'udp' or 'tcp'.",
)
# Making Dropbox optional
dropbox_enabled: bool = Field(
default=True,
description="Enable Dropbox as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
dropbox_app_key: Optional[str] = None
dropbox_app_secret: Optional[str] = None
dropbox_folder: Optional[str] = None
dropbox_refresh_token: Optional[str] = None
# Making Nextcloud optional
nextcloud_enabled: bool = Field(
default=True,
description="Enable Nextcloud as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
nextcloud_upload_url: Optional[str] = None
nextcloud_username: Optional[str] = None
nextcloud_password: Optional[str] = None
nextcloud_folder: Optional[str] = None
# Making Paperless optional
paperless_enabled: bool = Field(
default=True,
description="Enable Paperless-ngx as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
paperless_ngx_api_token: Optional[str] = None
paperless_host: Optional[str] = None
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
@@ -109,6 +166,25 @@ class Settings(BaseSettings):
google_docai_location: str = "us" # Processor location, e.g. "us" or "eu"
external_hostname: str = "localhost" # Default to localhost
# ---------------------------------------------------------------------------
# Document Translation Settings
# ---------------------------------------------------------------------------
# Default target language for automatic document translation (ISO 639-1 code).
# After OCR / metadata extraction, if the detected document language differs
# from this value the system translates the extracted text into this language
# and stores it alongside the original. Other language translations are
# generated on the fly via the AI provider and are NOT persisted.
# Per-user overrides are stored in UserProfile.default_document_language.
default_document_language: str = Field(
default="en",
description=(
"ISO 639-1 language code for the default translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language "
"differs are automatically translated into this language after "
"processing. Default: 'en' (English)."
),
)
# Authentication settings
auth_enabled: bool = True # Default to enabled
admin_username: Optional[str] = None
@@ -166,12 +242,44 @@ class Settings(BaseSettings):
),
)
# Authentik
# Authentik / Generic OIDC
authentik_client_id: Optional[str] = None
authentik_client_secret: Optional[str] = None
authentik_config_url: Optional[str] = None
oauth_provider_name: Optional[str] = None # Name to display for the OAuth provider
# Social Login Providers
# Google OAuth2
social_auth_google_enabled: bool = False
social_auth_google_client_id: Optional[str] = None
social_auth_google_client_secret: Optional[str] = None
# Microsoft OAuth2 (Azure AD / Microsoft Entra ID)
social_auth_microsoft_enabled: bool = False
social_auth_microsoft_client_id: Optional[str] = None
social_auth_microsoft_client_secret: Optional[str] = None
social_auth_microsoft_tenant: str = Field(
default="common",
description=(
"Azure AD tenant ID or one of 'common', 'organizations', 'consumers'. "
"Use 'common' to allow any Microsoft account and any Azure AD org. "
"Use a specific tenant ID (GUID) to restrict to a single organization. "
"Default: common."
),
)
# Apple Sign-In
social_auth_apple_enabled: bool = False
social_auth_apple_client_id: Optional[str] = None
social_auth_apple_team_id: Optional[str] = None
social_auth_apple_key_id: Optional[str] = None
social_auth_apple_private_key: Optional[str] = None
# Dropbox OAuth2
social_auth_dropbox_enabled: bool = False
social_auth_dropbox_client_id: Optional[str] = None
social_auth_dropbox_client_secret: Optional[str] = None
# Local user signup
allow_local_signup: bool = Field(
default=False,
@@ -394,6 +502,10 @@ class Settings(BaseSettings):
imap2_delete_after_process: bool = False
# Google Drive settings
google_drive_enabled: bool = Field(
default=True,
description="Enable Google Drive as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
google_drive_credentials_json: Optional[str] = ""
google_drive_folder_id: Optional[str] = ""
google_drive_delegate_to: Optional[str] = "" # Optional delegated user email
@@ -405,6 +517,10 @@ class Settings(BaseSettings):
google_drive_refresh_token: Optional[str] = ""
# WebDAV settings
webdav_enabled: bool = Field(
default=True,
description="Enable WebDAV as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
webdav_url: Optional[str] = None
webdav_username: Optional[str] = None
webdav_password: Optional[str] = None
@@ -412,6 +528,10 @@ class Settings(BaseSettings):
webdav_verify_ssl: bool = True
# FTP settings
ftp_enabled: bool = Field(
default=True,
description="Enable FTP as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
ftp_host: Optional[str] = None
ftp_port: Optional[int] = 21
ftp_username: Optional[str] = None
@@ -421,6 +541,10 @@ class Settings(BaseSettings):
ftp_allow_plaintext: bool = True # Default to allowing plaintext fallback
# SFTP settings
sftp_enabled: bool = Field(
default=True,
description="Enable SFTP as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
sftp_host: Optional[str] = None
sftp_port: Optional[int] = 22
sftp_username: Optional[str] = None
@@ -442,6 +566,10 @@ class Settings(BaseSettings):
email_default_recipient: Optional[str] = None
# Email destination settings (dedicated SMTP for document delivery – decoupled from shared email above)
dest_email_enabled: bool = Field(
default=True,
description="Enable Email as an upload destination. Set to False to disable document delivery via email even when credentials are configured.",
)
dest_email_host: Optional[str] = None
dest_email_port: Optional[int] = 587
dest_email_username: Optional[str] = None
@@ -451,6 +579,10 @@ class Settings(BaseSettings):
dest_email_default_recipient: Optional[str] = None # Fallback recipient for document delivery
# OneDrive settings
onedrive_enabled: bool = Field(
default=True,
description="Enable OneDrive as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
onedrive_client_id: Optional[str] = None
onedrive_client_secret: Optional[str] = None
onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts
@@ -458,6 +590,10 @@ class Settings(BaseSettings):
onedrive_folder_path: Optional[str] = None
# AWS S3 settings
s3_enabled: bool = Field(
default=True,
description="Enable Amazon S3 as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
aws_access_key_id: Optional[str] = None
aws_secret_access_key: Optional[str] = None
aws_region: Optional[str] = "us-east-1" # Default region
@@ -466,6 +602,16 @@ class Settings(BaseSettings):
s3_storage_class: Optional[str] = "STANDARD" # Default storage class
s3_acl: Optional[str] = "private" # Default ACL
# iCloud Drive settings
icloud_enabled: bool = Field(
default=True,
description="Enable iCloud Drive as an upload destination. Set to False to disable uploads even when credentials are configured.",
)
icloud_username: Optional[str] = None # Apple ID email address
icloud_password: Optional[str] = None # App-specific password (required for 2FA accounts)
icloud_folder: Optional[str] = None # Target folder path in iCloud Drive (e.g. "Documents/Uploads")
icloud_cookie_directory: Optional[str] = None # Directory for session cookies (default: ~/.pyicloud)
# Uptime Kuma settings
uptime_kuma_url: Optional[str] = None
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
@@ -484,6 +630,14 @@ class Settings(BaseSettings):
# Feature flags
allow_file_delete: bool = True # Default to allowing file deletion from database
compliance_enabled: bool = Field(
default=True,
description=(
"Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). "
"When enabled, admins can view compliance status and apply "
"pre-built regulatory configurations. Default: True."
),
)
# PDF/A archival conversion settings
enable_pdfa_conversion: bool = Field(
@@ -557,6 +711,17 @@ class Settings(BaseSettings):
),
)
imap_attachment_filter: str = Field(
default="documents_only",
description=(
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: "
"'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); "
"'all' – ingest all supported file types including images. "
"This is the global default; individual user IMAP accounts can override it."
),
)
# Batch processing settings
processall_throttle_threshold: int = Field(
default=20,
@@ -855,6 +1020,49 @@ class Settings(BaseSettings):
),
)
# SIEM / External Audit Log Forwarding
# Forward audit events to external SIEM systems for centralised monitoring.
audit_siem_enabled: bool = Field(
default=False,
description="Enable forwarding of audit events to an external SIEM system.",
)
audit_siem_transport: str = Field(
default="syslog",
description=(
"Transport used to forward audit events. "
"Options: 'syslog' (RFC 5424 over UDP/TCP), 'http' (JSON POST to a webhook URL, "
"compatible with Splunk HEC, Logstash HTTP input, Grafana Loki, etc.)."
),
)
audit_siem_syslog_host: str = Field(
default="localhost",
description="Hostname or IP of the syslog receiver.",
)
audit_siem_syslog_port: int = Field(
default=514,
description="Port of the syslog receiver.",
)
audit_siem_syslog_protocol: str = Field(
default="udp",
description="Protocol for syslog transport: 'udp' or 'tcp'.",
)
audit_siem_http_url: str = Field(
default="",
description=(
"HTTP endpoint URL for SIEM webhook delivery. "
"Supports Splunk HEC (https://splunk:8088/services/collector/event), "
"Logstash HTTP input, Grafana Loki push API, or any JSON-accepting endpoint."
),
)
audit_siem_http_token: str = Field(
default="",
description="Bearer / HEC token included in the Authorization header of SIEM HTTP requests.",
)
audit_siem_http_custom_headers: str = Field(
default="",
description="Comma-separated 'Key:Value' pairs of extra headers for SIEM HTTP requests.",
)
# UI / Appearance
ui_default_color_scheme: str = Field(
default="system",
app/database.py
+9 -2
View File
@@ -210,8 +210,10 @@ def _run_schema_migrations(engine: Any) -> None:
if unique_filehash_indexes:
logger.info("Migrating files: dropping unique index on 'filehash'")
with engine.begin() as conn:
preparer = conn.dialect.identifier_preparer
for index in unique_filehash_indexes:
conn.execute(text(f"DROP INDEX IF EXISTS {index['name']}"))
quoted_idx = preparer.quote(index["name"])
conn.execute(text(f"DROP INDEX IF EXISTS {quoted_idx}"))
logger.info("Migration complete: unique index on 'filehash' removed")
except Exception as exc:
logger.warning(f"Skipping filehash unique index drop: {exc}")
@@ -263,12 +265,17 @@ def _ensure_indexes(engine: Any, inspector: Any) -> None:
table_names = inspector.get_table_names()
columns_by_table: dict[str, set[str]] = {}
with engine.begin() as conn:
preparer = conn.dialect.identifier_preparer
for idx_name, table, column in _PERF_INDEXES:
if table in table_names:
if table not in columns_by_table:
columns_by_table[table] = {col["name"] for col in inspector.get_columns(table)}
if column in columns_by_table[table]:
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table} ({column})"))
# SECURITY: Quoted identifiers to prevent SQL injection during index creation
quoted_idx = preparer.quote(idx_name)
quoted_table = preparer.quote(table)
quoted_col = preparer.quote(column)
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {quoted_idx} ON {quoted_table} ({quoted_col})"))
logger.info("Performance indexes ensured")
app/main.py
+149 -6
View File
@@ -1,8 +1,11 @@
#!/usr/bin/env python3
import json as _json_mod
import logging
import os
import pathlib
from contextlib import asynccontextmanager
from datetime import datetime as _dt
from datetime import timezone as _tz
from fastapi import FastAPI, HTTPException, Request, status
from fastapi.middleware.cors import CORSMiddleware
@@ -16,6 +19,7 @@ from starlette.middleware.trustedhost import TrustedHostMiddleware
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
from app.api import router as api_router
from app.api.graphql_api import graphql_router
from app.api.local_auth import router as local_auth_router
from app.auth import router as auth_router
from app.config import settings
@@ -35,6 +39,114 @@ from app.views import router as frontend_router
# Explicitly include the files router
from app.views.files import router as files_router
# ---------------------------------------------------------------------------
# Configure Python root logging level early so that *all* loggers (including
# those already created via ``logging.getLogger(__name__)`` in other modules)
# respect the configured level.
#
# Standard behaviour (matches Django, Flask, 12-factor conventions):
# • ``LOG_LEVEL`` env var takes precedence when explicitly set.
# • When ``DEBUG=True`` and ``LOG_LEVEL`` is **not** set, the effective
# level is automatically lowered to ``DEBUG``.
# • Default (neither flag set): ``INFO``.
#
# ``LOG_FORMAT=json`` enables structured JSON lines on stdout, suitable for
# Promtail, Fluentd, Filebeat, Datadog, Splunk UF, or any log collector.
#
# ``LOG_SYSLOG_ENABLED=true`` adds a Python SysLogHandler so that every log
# message is also forwarded to the configured syslog receiver — useful for
# traditional (non-container) deployments and centralised SIEM ingestion.
#
# Noisy third-party loggers (httpx, httpcore, authlib, etc.) are pinned to
# WARNING when the app-level is DEBUG to keep output useful.
# ---------------------------------------------------------------------------
_explicit_log_level = os.environ.get("LOG_LEVEL")
if settings.debug and _explicit_log_level is None:
_effective_level = "DEBUG"
else:
_effective_level = settings.log_level.upper()
_effective_level_int = getattr(logging, _effective_level, logging.INFO)
class _JsonFormatter(logging.Formatter):
"""Emit one JSON object per log line for machine consumption.
Fields emitted: ``timestamp``, ``level``, ``logger``, ``message``,
``module``, ``funcName``, ``lineno``, and — when present — ``exc_info``.
Compatible with Grafana Loki, Splunk, ELK, Datadog, and most SIEM tools.
"""
def format(self, record: logging.LogRecord) -> str:
log_entry: dict = {
"timestamp": _dt.fromtimestamp(record.created, tz=_tz.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
"module": record.module,
"funcName": record.funcName,
"lineno": record.lineno,
}
if record.exc_info and record.exc_info[1] is not None:
log_entry["exc_info"] = self.formatException(record.exc_info)
return _json_mod.dumps(log_entry, default=str)
# Choose formatter based on LOG_FORMAT setting
if settings.log_format.lower() == "json":
_handler = logging.StreamHandler()
_handler.setFormatter(_JsonFormatter())
logging.root.handlers = [_handler]
logging.root.setLevel(_effective_level_int)
else:
logging.basicConfig(
level=_effective_level_int,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
# Optional: forward application logs to a syslog receiver
if settings.log_syslog_enabled:
import logging.handlers as _lh
import socket as _socket
_proto = settings.log_syslog_protocol.lower()
_socktype = _socket.SOCK_STREAM if _proto == "tcp" else _socket.SOCK_DGRAM
_syslog_handler = _lh.SysLogHandler(
address=(settings.log_syslog_host, settings.log_syslog_port),
socktype=_socktype,
)
_syslog_handler.setLevel(_effective_level_int)
# Use the same formatter as stdout (text or JSON)
if settings.log_format.lower() == "json":
_syslog_handler.setFormatter(_JsonFormatter())
else:
_syslog_handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s"))
logging.root.addHandler(_syslog_handler)
# Keep noisy third-party loggers quiet at DEBUG level
if _effective_level_int <= logging.DEBUG:
for _noisy in (
"httpx",
"httpcore",
"authlib",
"urllib3",
"hpack",
"multipart",
"watchfiles",
):
logging.getLogger(_noisy).setLevel(logging.WARNING)
_startup_logger = logging.getLogger(__name__)
_startup_logger.info(
"Root logging level set to %s (debug=%s, format=%s, syslog=%s)",
_effective_level,
settings.debug,
settings.log_format,
settings.log_syslog_enabled,
)
# Load configuration from .env for the session key
config = Config(".env")
# Use settings.session_secret which has proper validation
@@ -146,6 +258,20 @@ async def lifespan(app: FastAPI):
except Exception:
logging.debug("Scheduled jobs seeding skipped — DB may not be ready yet") # noqa: S110
# Seed the built-in compliance templates (GDPR, HIPAA, SOC2) so they
# are available in the admin compliance dashboard on first startup.
try:
from app.database import SessionLocal as _SessionLocal # noqa: F811
from app.utils.compliance_service import seed_compliance_templates as _seed_compliance
_db_compliance = _SessionLocal()
try:
_seed_compliance(_db_compliance)
finally:
_db_compliance.close()
except Exception:
logging.debug("Compliance template seeding skipped — DB may not be ready yet") # noqa: S110
# Application is now running
yield
@@ -239,6 +365,23 @@ else:
# Custom exception handlers that return JSON for API routes and HTML for frontend routes
# These use their own separate templates instance so that patches in tests on individual
# view modules do not affect the error handler rendering.
_error_templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
_error_templates = Jinja2Templates(directory=str(_error_templates_dir))
# Register the i18n translate helper as a global so error templates can use {{ _("key") }}.
# Error pages use the default language (English); request-specific locale is not needed here.
from app.utils.i18n import SUPPORTED_LANGUAGES as _SUPPORTED_LANGUAGES # noqa: E402
from app.utils.i18n import get_suggested_languages as _get_suggested_languages # noqa: E402
from app.utils.i18n import translate as _translate_fn # noqa: E402
_error_templates.env.globals["_"] = lambda key, **kwargs: _translate_fn(key, "en", **kwargs)
_error_templates.env.globals["min"] = min
_error_templates.env.globals["max"] = max
_error_templates.env.globals["supported_languages"] = _SUPPORTED_LANGUAGES
_error_templates.env.globals["suggested_languages"] = _get_suggested_languages("en", "")
@app.exception_handler(HTTPException)
async def http_exception_handler(request: Request, exc: HTTPException):
"""
@@ -250,15 +393,15 @@ async def http_exception_handler(request: Request, exc: HTTPException):
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
# For frontend routes, return appropriate HTML templates
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
# Handle 404 errors with a custom template
if exc.status_code == 404:
return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND)
return _error_templates.TemplateResponse(
"404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND
)
# For other HTTP errors, we could create specific templates or use a generic one
# For now, return a simple error page
return templates.TemplateResponse(
return _error_templates.TemplateResponse(
"404.html", # Reuse 404 template for other errors, or create a generic error template
{"request": request},
status_code=exc.status_code,
@@ -279,8 +422,7 @@ async def custom_500_handler(request: Request, exc: Exception):
)
# Serve the 500 template for non-API routes
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
return templates.TemplateResponse(
return _error_templates.TemplateResponse(
"500.html",
{"request": request, "exc": exc},
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -298,3 +440,4 @@ app.include_router(files_router) # Explicitly include the files router
app.include_router(auth_router)
app.include_router(local_auth_router)
app.include_router(api_router, prefix="/api")
app.include_router(graphql_router, prefix="/graphql")
app/models.py
+207
View File
@@ -7,6 +7,7 @@ from app.database import Base
# Foreign key constants
_FILES_ID_FK = "files.id"
_PIPELINES_ID_FK = "pipelines.id"
_ROUTING_RULES_TABLE = "pipeline_routing_rules"
class DocumentMetadata(Base):
@@ -83,6 +84,19 @@ class FileRecord(Base):
# Processing pipeline assigned to this file (NULL = use system default)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True)
# Detected document language (ISO 639-1 code, e.g. "de", "en", "fr")
# Extracted from AI metadata during processing; cached here for fast access.
detected_language = Column(String(10), nullable=True)
# Default-language translation of the extracted text.
# Stored when the detected language differs from the user's/system default
# document language. Only the original text and this translation are persisted;
# other languages are translated on the fly via the AI provider.
default_language_text = Column(Text, nullable=True)
# ISO 639-1 code of the default-language translation stored above (e.g. "en").
default_language_code = Column(String(10), nullable=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
@@ -146,6 +160,27 @@ class SettingsAuditLog(Base):
action = Column(String, nullable=False) # "update" or "delete"
class AuditLog(Base):
"""Comprehensive audit log for compliance tracking.
Records all significant actions: login/logout, document CRUD, settings
changes, and administrative operations. Rows are append-only; the API
and service layer never update or delete entries.
"""
__tablename__ = "audit_logs"
id = Column(Integer, primary_key=True, index=True)
timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
user = Column(String, nullable=False, index=True) # Username or "anonymous" / "system"
action = Column(String, nullable=False, index=True) # e.g. "login", "document.create", "settings.update"
resource_type = Column(String, nullable=True, index=True) # e.g. "document", "user", "settings"
resource_id = Column(String, nullable=True) # ID of the affected resource
ip_address = Column(String, nullable=True) # Client IP address
details = Column(Text, nullable=True) # JSON-encoded extra context
severity = Column(String(16), nullable=False, server_default="info") # info / warning / error / critical
class SavedSearch(Base):
"""User-defined saved search filters for quick access to frequently used filter combinations."""
@@ -278,6 +313,23 @@ class UserProfile(Base):
preferred_destination = Column(String(50), nullable=True)
stripe_customer_id = Column(String(64), nullable=True)
# UI language preference for i18n (ISO 639-1 code, e.g. "en", "de", "fr")
# NULL means "auto-detect from browser Accept-Language header"
preferred_language = Column(String(10), nullable=True)
# Default document language for translated versions (ISO 639-1 code).
# When a document's detected language differs from this value, the system
# automatically generates and stores a translation into this language.
# NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting".
default_document_language = Column(String(10), nullable=True)
# UI colour scheme preference: "light" | "dark" | "system" (NULL = "system")
preferred_theme = Column(String(10), nullable=True)
# Custom profile avatar stored as a base64 data-URL (e.g. "data:image/png;base64,...")
# NULL means use the Gravatar fallback derived from the user's e-mail address.
avatar_data = Column(Text, nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -398,6 +450,48 @@ class PipelineStep(Base):
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ImapIngestionProfile(Base):
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
can precisely control what gets ingested from each mailbox.
System-provided built-in profiles (``is_builtin=True``) are seeded by the
migration and cannot be deleted or renamed. Users may create their own profiles
(``owner_id`` set to their identifier) or rely on the global system profiles
(``owner_id=None``).
``allowed_categories`` stores a JSON list of category strings, e.g.::
'["pdf", "office", "opendocument", "text", "web"]'
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
"""
__tablename__ = "imap_ingestion_profiles"
id = Column(Integer, primary_key=True, index=True)
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
name = Column(String(255), nullable=False)
# Optional description shown in the UI
description = Column(Text, nullable=True)
# Owner of this profile. NULL = global/system profile available to all users.
owner_id = Column(String, nullable=True, index=True)
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
# Built-in system profiles that cannot be deleted or modified via the API.
is_builtin = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class UserImapAccount(Base):
"""Per-user IMAP ingestion account.
@@ -436,6 +530,10 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
# Optional reference to an ImapIngestionProfile.
# NULL means "use the global imap_attachment_filter setting" (system default).
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
@@ -527,6 +625,7 @@ class IntegrationType:
EMAIL = "EMAIL"
PAPERLESS = "PAPERLESS"
RCLONE = "RCLONE"
ICLOUD = "ICLOUD"
ALL = {
IMAP,
@@ -543,6 +642,7 @@ class IntegrationType:
EMAIL,
PAPERLESS,
RCLONE,
ICLOUD,
}
@@ -855,3 +955,110 @@ class ScheduledJob(Base):
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class MobileDevice(Base):
"""Registered mobile device for push notifications.
Stores the push token (Expo push token, FCM token, or APNs token) for a
specific user device so that document-processing events can be forwarded
as push notifications to the native mobile app.
"""
__tablename__ = "mobile_devices"
id = Column(Integer, primary_key=True, index=True)
# User that owns this device registration.
owner_id = Column(String, nullable=False, index=True)
# Human-readable name the user gave this device (e.g. "John's iPhone").
device_name = Column(String(255), nullable=True)
# Platform: "ios", "android", or "web".
platform = Column(String(20), nullable=False, default="ios")
# Expo push token (ExponentPushToken[…]) or raw FCM/APNs token.
push_token = Column(String(512), nullable=False)
# Whether push notifications are enabled for this device.
is_active = Column(Boolean, nullable=False, default=True)
# Timestamps.
created_at = Column(DateTime(timezone=True), server_default=func.now())
last_seen_at = Column(DateTime(timezone=True), nullable=True)
__table_args__ = (UniqueConstraint("owner_id", "push_token", name="uq_mobile_device_owner_token"),)
class ComplianceTemplate(Base):
"""Pre-built compliance configuration templates (GDPR, HIPAA, SOC2).
Each row represents an applied compliance template. The ``settings_json``
column stores the concrete setting key/value pairs that were written when
the template was applied. ``status`` tracks the current compliance posture.
"""
__tablename__ = "compliance_templates"
id = Column(Integer, primary_key=True, index=True)
name = Column(String(50), unique=True, nullable=False, index=True) # GDPR, HIPAA, SOC2
display_name = Column(String(100), nullable=False)
description = Column(Text, nullable=True)
settings_json = Column(Text, nullable=False, default="{}") # JSON of applied settings
enabled = Column(Boolean, nullable=False, default=False)
status = Column(String(20), nullable=False, default="not_applied") # not_applied, compliant, partial, non_compliant
applied_at = Column(DateTime(timezone=True), nullable=True)
applied_by = Column(String(255), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class PipelineRoutingRule(Base):
"""Conditional routing rule that assigns documents to pipelines.
Rules are evaluated in ascending ``position`` order for a given owner.
The first rule whose condition matches the document properties wins and
the document is routed to ``target_pipeline_id``. If no rule matches,
the caller falls back to the owner's (or system) default pipeline.
Supported fields:
file_type, document_type, category, filename, size, and any key
inside the AI-extracted metadata JSON (prefixed ``metadata.``).
Supported operators:
equals, not_equals, contains, not_contains, regex, gt, lt, gte, lte.
"""
__tablename__ = _ROUTING_RULES_TABLE
id = Column(Integer, primary_key=True, index=True)
# Owner of this rule. NULL = system-wide rule (admin only).
owner_id = Column(String, nullable=True, index=True)
# Human-readable label for the rule.
name = Column(String(255), nullable=False)
# Evaluation order (lower = earlier). First matching rule wins.
position = Column(Integer, nullable=False, default=0)
# The document property to evaluate.
# Built-in: file_type, document_type, category, filename, size.
# For AI metadata fields, use the "metadata.<key>" prefix.
field = Column(String(255), nullable=False)
# Comparison operator.
operator = Column(String(50), nullable=False)
# Value to compare against (always stored as text; cast as needed).
value = Column(String(1024), nullable=False)
# Target pipeline when the condition matches.
target_pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=False, index=True)
# Soft-disable without deleting.
is_active = Column(Boolean, nullable=False, default=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
app/tasks/embed_metadata_into_pdf.py
+24
View File
@@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as search_exc:
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
# Cache the detected language on the FileRecord and trigger
# default-language translation when the document is in a
# different language.
detected_lang = metadata.get("language") if metadata else None
if detected_lang and extracted_text:
try:
file_record.detected_language = detected_lang
db.commit()
from app.tasks.translate_to_default_language import translate_to_default_language
translate_to_default_language.delay(
file_id,
extracted_text,
detected_lang,
owner_id=file_record.owner_id,
)
logger.info(
f"[{task_id}] Queued default-language translation for file {file_id} "
f"(detected: {detected_lang})"
)
except Exception as trans_exc:
logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}")
# Persist the metadata into a JSON file with the same base name.
# Include file path references for traceability
logger.info(f"[{task_id}] Persisting metadata to JSON")
app/tasks/extract_metadata_with_gpt.py
+192 -191
View File
@@ -1,191 +1,192 @@
#!/usr/bin/env python3
import json
import logging
import os
import re
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
logger = logging.getLogger(__name__)
def extract_json_from_text(text):
"""
Try to extract a JSON object from the text.
- First, check for a JSON block inside triple backticks.
- If not found, try to extract text from the first '{' to the last '}'.
"""
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
match = re.search(pattern, text, re.DOTALL)
if match:
return match.group(1)
else:
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
return text[start : end + 1]
return None
@celery.task(base=BaseTaskWithRetry, bind=True)
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
"""
Uses OpenAI to classify document metadata.
Args:
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
cleaned_text: The extracted text from the document
file_id: Optional file ID for tracking
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"in_progress",
f"Extracting metadata for {os.path.basename(filename)}",
file_id=file_id,
)
# Get file_id from database if not provided
if file_id is None:
tmp_dir = os.path.join(settings.workdir, "tmp")
# Handle both basename and full path
if os.path.isabs(filename):
file_path = filename
else:
file_path = os.path.join(tmp_dir, filename)
if os.path.exists(file_path):
with SessionLocal() as db:
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
if file_record:
file_id = file_record.id
prompt = (
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
"Extract and return the following fields:\n"
"1. **filename**: Machine-readable filename "
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
'3. **absender**: The sender, or "Unknown" if not found.\n'
"4. **correspondent**: The entity or company that issued the document "
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
"Private_Korrespondenz, Sonstige_Informationen].\n"
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
"10. **title**: A human-readable title summarizing the document content.\n"
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
"of the extracted metadata.\n"
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
"### Important Rules:\n"
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
"- **Output Language**: Maintain the document's original language.\n\n"
f"Extracted text:\n{cleaned_text}\n\n"
"Return only valid JSON with no additional commentary.\n"
)
try:
logger.info(f"[{task_id}] Sending classification request for {filename}...")
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider API", file_id=file_id)
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
content = provider.chat_completion(
messages=[
{"role": "system", "content": "You are an intelligent document classifier."},
{"role": "user", "content": prompt},
],
model=model,
temperature=0,
)
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
log_task_progress(
task_id,
"call_ai_provider",
"success",
"Received AI provider response",
file_id=file_id,
detail=f"Raw classification response:\n{content}",
)
json_text = extract_json_from_text(content)
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"failure",
"Invalid JSON in response",
file_id=file_id,
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
)
return {}
metadata = json.loads(json_text)
# SECURITY: Validate filename format from GPT to prevent path traversal
# The prompt requests filenames with only letters, numbers, periods, and underscores
# Enforce this constraint to prevent malicious filenames
suggested_filename = metadata.get("filename", "")
if suggested_filename:
# Check if filename contains only safe characters AND explicitly check for ".."
# Defense in depth: While the regex [\w\-\. ]+ already excludes / and \,
# we explicitly reject ".." to guard against:
# 1. Potential locale-specific \w behavior
# 2. Files literally named ".." which are valid but problematic
# 3. Future code changes that might relax the regex
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
# Reset to empty to trigger fallback to original filename
metadata["filename"] = ""
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(
task_id,
"parse_metadata",
"success",
f"Parsed metadata: {list(metadata.keys())}",
file_id=file_id,
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
)
# Trigger the next step: embedding metadata into the PDF
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
logger.info(f"[{task_id}] Queueing metadata embedding task")
log_task_progress(
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
)
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
return {"s3_file": os.path.basename(filename), "metadata": metadata}
except Exception as e:
logger.exception(f"[{task_id}] AI provider classification failed for {filename}: {e}")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=f"AI provider classification failed for {filename}.\nException: {str(e)}",
)
return {}
#!/usr/bin/env python3
import json
import logging
import os
import re
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
from app.utils.filename_utils import VALID_FILENAME_RE
logger = logging.getLogger(__name__)
def extract_json_from_text(text):
"""
Try to extract a JSON object from the text.
- First, check for a JSON block inside triple backticks.
- If not found, try to extract text from the first '{' to the last '}'.
"""
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
match = re.search(pattern, text, re.DOTALL)
if match:
return match.group(1)
else:
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
return text[start : end + 1]
return None
@celery.task(base=BaseTaskWithRetry, bind=True)
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
"""
Uses OpenAI to classify document metadata.
Args:
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
cleaned_text: The extracted text from the document
file_id: Optional file ID for tracking
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"in_progress",
f"Extracting metadata for {os.path.basename(filename)}",
file_id=file_id,
)
# Get file_id from database if not provided
if file_id is None:
tmp_dir = os.path.join(settings.workdir, "tmp")
# Handle both basename and full path
if os.path.isabs(filename):
file_path = filename
else:
file_path = os.path.join(tmp_dir, filename)
if os.path.exists(file_path):
with SessionLocal() as db:
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
if file_record:
file_id = file_record.id
prompt = (
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
"Extract and return the following fields:\n"
"1. **filename**: Machine-readable filename "
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, spaces, dashes, periods, and underscores).\n"
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
'3. **absender**: The sender, or "Unknown" if not found.\n'
"4. **correspondent**: The entity or company that issued the document "
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
"Private_Korrespondenz, Sonstige_Informationen].\n"
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
"10. **title**: A human-readable title summarizing the document content.\n"
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
"of the extracted metadata.\n"
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
"### Important Rules:\n"
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
"- **Output Language**: Maintain the document's original language.\n\n"
f"Extracted text:\n{cleaned_text}\n\n"
"Return only valid JSON with no additional commentary.\n"
)
try:
logger.info(f"[{task_id}] Sending classification request for {filename}...")
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider API", file_id=file_id)
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
content = provider.chat_completion(
messages=[
{"role": "system", "content": "You are an intelligent document classifier."},
{"role": "user", "content": prompt},
],
model=model,
temperature=0,
)
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
log_task_progress(
task_id,
"call_ai_provider",
"success",
"Received AI provider response",
file_id=file_id,
detail=f"Raw classification response:\n{content}",
)
json_text = extract_json_from_text(content)
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"failure",
"Invalid JSON in response",
file_id=file_id,
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
)
return {}
metadata = json.loads(json_text)
# SECURITY: Validate filename format from GPT to prevent path traversal
# The prompt requests filenames with only letters, numbers, periods, and underscores
# Enforce this constraint to prevent malicious filenames
suggested_filename = metadata.get("filename", "")
if suggested_filename:
# Check if filename contains only safe characters AND explicitly check for ".."
# Defense in depth: While the regex VALID_FILENAME_PATTERN already excludes / and \,
# we explicitly reject ".." to guard against:
# 1. Potential locale-specific \w behavior
# 2. Files literally named ".." which are valid but problematic
# 3. Future code changes that might relax the regex
if not VALID_FILENAME_RE.match(suggested_filename) or ".." in suggested_filename:
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
# Reset to empty to trigger fallback to original filename
metadata["filename"] = ""
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(
task_id,
"parse_metadata",
"success",
f"Parsed metadata: {list(metadata.keys())}",
file_id=file_id,
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
)
# Trigger the next step: embedding metadata into the PDF
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
logger.info(f"[{task_id}] Queueing metadata embedding task")
log_task_progress(
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
)
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
return {"s3_file": os.path.basename(filename), "metadata": metadata}
except Exception as e:
logger.exception(f"[{task_id}] AI provider classification failed for {filename}: {e}")
log_task_progress(
task_id,
"extract_metadata_with_gpt",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=f"AI provider classification failed for {filename}.\nException: {str(e)}",
)
return {}
app/tasks/finalize_document_storage.py
+13 -1
View File
@@ -21,8 +21,9 @@ from app.tasks.send_to_all import (
# Import database and logging utils from main
from app.utils import log_task_progress
# Import notification utility
# Import notification utilities
from app.utils.notification import notify_file_processed
from app.utils.user_notification import notify_user_document_processed
logger = logging.getLogger(__name__)
@@ -139,4 +140,15 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
except Exception as e:
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
# 6. Send per-user notification
if owner_id:
try:
notify_user_document_processed(
owner_id=owner_id,
filename=os.path.basename(processed_file),
file_id=file_id,
)
except Exception as e:
logger.warning(f"[WARNING] Failed to send per-user processed notification: {e}")
return {"status": "Completed", "file": processed_file}
app/tasks/imap_tasks.py
+92 -25
View File
@@ -13,7 +13,11 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
from app.utils.allowed_types import (
ALL_CATEGORIES,
DEFAULT_CATEGORIES,
get_allowed_types_for_categories,
)
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -50,6 +54,38 @@ def _decrypt_imap_password(password: str | None) -> str | None:
return decrypt_value(password)
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
"""Return the list of allowed categories for a profile ID.
Loads the profile from the database. If ``profile_id`` is ``None`` or the
profile is not found, falls back to the global ``settings.imap_attachment_filter``
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
"""
if profile_id is not None:
try:
from app.models import ImapIngestionProfile
db = _get_db_session()
try:
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if profile:
return json.loads(profile.allowed_categories)
finally:
db.close()
except Exception as exc: # noqa: BLE001
logger.warning(
"Could not load IMAP ingestion profile %d (%s: %s) — using global default",
profile_id,
type(exc).__name__,
exc,
)
# Fall back to global setting
if settings.imap_attachment_filter == "all":
return ALL_CATEGORIES
return DEFAULT_CATEGORIES
LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
@@ -180,6 +216,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -250,6 +287,9 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
# Integrations can store a profile_id in config; fall back to global default
profile_id = cfg.get("profile_id")
allowed_categories = _resolve_categories_for_profile(profile_id)
if not (host and username and password):
logger.warning(
@@ -269,6 +309,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
allowed_categories=allowed_categories,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -329,6 +370,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
allowed_categories=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -345,8 +387,22 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
allowed_categories: List of file-type category keys to ingest (e.g.
``["pdf", "office", "images"]``). ``None`` falls back to the
global ``settings.imap_attachment_filter`` mapping.
"""
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
if allowed_categories is None:
allowed_categories = _resolve_categories_for_profile(None)
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
logger.info(
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
mailbox_key,
host,
port,
use_ssl,
allowed_categories,
)
processed_emails = load_processed_emails()
try:
@@ -405,9 +461,13 @@ def pull_inbox(
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue
# Process attachments (and convert non-PDF files).
# We call the function without assigning its return value since it is not used.
fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
# Process attachments using the resolved mime types / extensions.
fetch_attachments_and_enqueue(
email_message,
owner_id=owner_id,
effective_mime_types=effective_mime_types,
effective_extensions=effective_extensions,
)
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -436,27 +496,23 @@ def pull_inbox(
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
effective_mime_types: frozenset[str] | None = None,
effective_extensions: frozenset[str] | None = None,
):
"""
Extracts attachments from the email and processes only allowed file types.
Files are accepted if either:
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
2. They have a '.pdf' file extension (regardless of MIME type)
The caller is responsible for computing ``effective_mime_types`` and
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
before calling this function. ``pull_inbox`` does this automatically.
Allowed file types include:
- PDF: application/pdf or *.pdf extension
- Microsoft Office files:
- Word: application/msword,
application/vnd.openxmlformats-officedocument.wordprocessingml.document
- Excel: application/vnd.ms-excel,
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- PowerPoint: application/vnd.ms-powerpoint,
application/vnd.openxmlformats-officedocument.presentationml.presentation
- Other meaningful attachments:
- Plain text: text/plain
- CSV: text/csv
- Rich Text Format: application/rtf, text/rtf
If either set is ``None`` the function falls back to the default category list
so the function still works correctly when called directly in tests or from
other contexts.
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -465,9 +521,14 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
effective_extensions: Pre-computed frozenset of allowed file extension strings.
Returns True if at least one allowed attachment was processed.
"""
if effective_mime_types is None or effective_extensions is None:
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -482,9 +543,15 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info(
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
filename,
mime_type,
file_ext,
)
continue
file_path = os.path.join(settings.workdir, filename)
@@ -495,7 +562,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
if mime_type == "application/pdf" or is_pdf_by_extension:
process_document.delay(file_path, owner_id=owner_id)
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
elif mime_type in ALLOWED_MIME_TYPES:
elif mime_type in effective_mime_types:
# Other allowed files are sent for conversion
convert_to_pdf.delay(file_path, owner_id=owner_id)
logger.info("Enqueued file for conversion to PDF: %s", filename)
app/tasks/send_to_all.py
+61 -12
View File
@@ -12,6 +12,7 @@ from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_icloud import upload_to_icloud
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
@@ -25,18 +26,32 @@ logger = logging.getLogger(__name__)
def _should_upload_to_dropbox():
return bool(settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token)
return bool(
getattr(settings, "dropbox_enabled", True)
and settings.dropbox_app_key
and settings.dropbox_app_secret
and settings.dropbox_refresh_token
)
def _should_upload_to_nextcloud():
return bool(settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password)
return bool(
getattr(settings, "nextcloud_enabled", True)
and settings.nextcloud_upload_url
and settings.nextcloud_username
and settings.nextcloud_password
)
def _should_upload_to_paperless():
return bool(settings.paperless_ngx_api_token and settings.paperless_host)
return bool(
getattr(settings, "paperless_enabled", True) and settings.paperless_ngx_api_token and settings.paperless_host
)
def _should_upload_to_google_drive():
if not getattr(settings, "google_drive_enabled", True):
return False
# Check for OAuth configuration
if getattr(settings, "google_drive_use_oauth", False):
return bool(
@@ -51,20 +66,33 @@ def _should_upload_to_google_drive():
def _should_upload_to_webdav():
return bool(settings.webdav_url and settings.webdav_username and settings.webdav_password)
return bool(
getattr(settings, "webdav_enabled", True)
and settings.webdav_url
and settings.webdav_username
and settings.webdav_password
)
def _should_upload_to_ftp():
return bool(settings.ftp_host and settings.ftp_username and settings.ftp_password)
return bool(
getattr(settings, "ftp_enabled", True) and settings.ftp_host and settings.ftp_username and settings.ftp_password
)
def _should_upload_to_sftp():
return bool(settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key))
return bool(
getattr(settings, "sftp_enabled", True)
and settings.sftp_host
and settings.sftp_username
and (settings.sftp_password or settings.sftp_private_key)
)
def _should_upload_to_email():
return bool(
settings.dest_email_host
getattr(settings, "dest_email_enabled", True)
and settings.dest_email_host
and settings.dest_email_username
and settings.dest_email_password
and settings.dest_email_default_recipient
@@ -72,18 +100,32 @@ def _should_upload_to_email():
def _should_upload_to_onedrive():
return bool(settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token)
return bool(
getattr(settings, "onedrive_enabled", True)
and settings.onedrive_client_id
and settings.onedrive_client_secret
and settings.onedrive_refresh_token
)
def _should_upload_to_s3():
return bool(settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key)
return bool(
getattr(settings, "s3_enabled", True)
and settings.s3_bucket_name
and settings.aws_access_key_id
and settings.aws_secret_access_key
)
def _should_upload_to_icloud():
return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password)
def get_configured_services_from_validator():
"""
Use the config validator to determine which services are configured properly.
Use the config validator to determine which services are configured and enabled.
Returns a dictionary with service names as keys and boolean values indicating
whether they're properly configured.
whether they're properly configured AND explicitly enabled.
"""
providers = get_provider_status()
@@ -98,12 +140,14 @@ def get_configured_services_from_validator():
"Email": "email",
"OneDrive": "onedrive",
"S3 Storage": "s3",
"iCloud Drive": "icloud",
}
result = {}
for provider_name, internal_name in service_map.items():
if provider_name in providers:
result[internal_name] = providers[provider_name].get("configured", False)
provider = providers[provider_name]
result[internal_name] = provider.get("configured", False) and provider.get("enabled", True)
return result
@@ -206,6 +250,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
"should_upload": _should_upload_to_s3,
"upload_func": upload_to_s3,
},
{
"name": "icloud",
"should_upload": _should_upload_to_icloud,
"upload_func": upload_to_icloud,
},
]
# Optionally get configuration status from validator
app/tasks/translate_to_default_language.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Celery task to translate extracted document text into the default target language.
This task is triggered after metadata extraction when the detected document
language differs from the user's (or system) default document language. The
translated text is persisted in ``FileRecord.default_language_text`` so that
users can always read a reference copy in their preferred language.
Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate``
endpoint and are NOT persisted.
"""
import logging
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord, UserProfile
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
logger = logging.getLogger(__name__)
def _resolve_default_language(owner_id: str | None) -> str:
"""Return the default document language for the given owner.
Resolution order:
1. ``UserProfile.default_document_language`` (per-user override)
2. ``settings.default_document_language`` (global setting)
"""
if owner_id:
with SessionLocal() as db:
profile = db.query(UserProfile).filter_by(user_id=owner_id).first()
if profile and profile.default_document_language:
return profile.default_document_language
return settings.default_document_language
@celery.task(base=BaseTaskWithRetry, bind=True)
def translate_to_default_language(
self,
file_id: int,
extracted_text: str,
detected_language: str,
owner_id: str | None = None,
) -> dict:
"""Translate *extracted_text* into the default document language and persist the result.
Args:
file_id: Primary key of the :class:`FileRecord`.
extracted_text: The OCR / refined text in the document's original language.
detected_language: ISO 639-1 code of the document's detected language.
owner_id: Owner identifier used to resolve per-user language preference.
Returns:
A dict with ``status``, ``target_language``, and the translated text length.
"""
task_id = self.request.id
target_language = _resolve_default_language(owner_id)
# Nothing to do when the document is already in the target language.
if detected_language == target_language:
logger.info(
f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation"
)
log_task_progress(
task_id,
"translate_to_default_language",
"skipped",
f"Document already in {target_language}",
file_id=file_id,
)
return {"status": "skipped", "reason": "already_in_target_language"}
logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'")
log_task_progress(
task_id,
"translate_to_default_language",
"in_progress",
f"Translating from {detected_language} to {target_language}",
file_id=file_id,
)
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated_text = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"from {detected_language} to {target_language}. "
f"Preserve the original formatting, paragraph structure, and meaning. "
f"Do not add any commentary or explanation — output ONLY the translated text."
),
},
{"role": "user", "content": extracted_text},
],
model=model,
temperature=0.3,
)
# Persist the translation.
with SessionLocal() as db:
record = db.query(FileRecord).filter_by(id=file_id).first()
if record:
record.default_language_text = translated_text
record.default_language_code = target_language
record.detected_language = detected_language
db.commit()
logger.info(
f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}"
)
log_task_progress(
task_id,
"translate_to_default_language",
"success",
f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})",
file_id=file_id,
)
return {
"status": "success",
"target_language": target_language,
"translated_length": len(translated_text),
}
except Exception as exc:
logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}")
log_task_progress(
task_id,
"translate_to_default_language",
"failure",
f"Exception: {exc}",
file_id=file_id,
)
raise
app/tasks/upload_to_email.py
+34 -5
View File
@@ -11,6 +11,7 @@ from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import pypdf
from jinja2 import Environment, FileSystemLoader, select_autoescape
from app.celery_app import celery
@@ -23,6 +24,15 @@ logger = logging.getLogger(__name__)
# Constants
_LOGO_FILENAME = "logo.png"
# Mapping from PDF metadata keys (with leading slash stripped) to application-specific names.
# This mirrors the inverse of the mapping used in app/tasks/embed_metadata_into_pdf.py.
_PDF_METADATA_KEY_MAP = {
"Title": "filename",
"Author": "absender",
"Subject": "document_type",
"Keywords": "tags",
}
def get_email_template(template_name="default.html"):
"""
@@ -63,9 +73,12 @@ def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
2. Extract metadata from PDF if it's embedded
2. Extract embedded metadata from PDF using pypdf
Returns a dictionary of metadata or None if not found
JSON metadata takes precedence; embedded PDF metadata fills in any missing
fields using the application's standard key mapping (e.g., /Title → filename).
Returns a dictionary of metadata (may be empty if none found).
"""
metadata = {}
@@ -76,12 +89,28 @@ def extract_metadata_from_file(file_path):
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
# This would require additional dependencies, so for now we'll just check for external JSON
# Try to extract embedded metadata from PDF
if file_path.lower().endswith(".pdf") and os.path.exists(file_path):
try:
with open(file_path, "rb") as f:
pdf_reader = pypdf.PdfReader(f)
pdf_metadata = pdf_reader.metadata
if pdf_metadata:
for key, value in pdf_metadata.items():
# Remove the leading slash from PDF metadata keys (e.g., '/Title' -> 'Title')
clean_key = key[1:] if key.startswith("/") else key
# Map to application-specific key names where possible
mapped_key = _PDF_METADATA_KEY_MAP.get(clean_key, clean_key)
# Only set if not already present (JSON metadata takes precedence)
if mapped_key not in metadata:
metadata[mapped_key] = str(value)
logger.info(f"Extracted embedded metadata from PDF: {file_path}")
except Exception as e:
logger.warning(f"Failed to extract metadata from PDF {file_path}: {str(e)}")
return metadata
app/tasks/upload_to_icloud.py
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Upload files to Apple iCloud Drive via the pyicloud library.
This module uses the ``pyicloud`` library to authenticate with Apple's iCloud
service and upload files to iCloud Drive. Because Apple does not offer a public
REST API for iCloud Drive, this integration relies on the *unofficial*
reverse-engineered protocol implemented by ``pyicloud``.
Requirements
~~~~~~~~~~~~
* An Apple ID with iCloud Drive enabled.
* An **app-specific password** generated at https://appleid.apple.com (required
when two-factor authentication is active – which is the default for all modern
Apple IDs).
* The ``pyicloud`` Python package (``pip install pyicloud``).
Configuration
~~~~~~~~~~~~~
Set the following environment variables (or ``app/config.py`` fields):
* ``ICLOUD_USERNAME`` – Apple ID email address.
* ``ICLOUD_PASSWORD`` – App-specific password.
* ``ICLOUD_FOLDER`` – Target folder path inside iCloud Drive, using ``/`` as
the separator (e.g. ``Documents/Uploads``). The folder is created
automatically if it does not exist.
* ``ICLOUD_COOKIE_DIRECTORY`` – (Optional) Directory for persisting session
cookies so that re-authentication is avoided between task runs. Defaults to
``~/.pyicloud``.
"""
import logging
import os
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import UploadTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def _get_icloud_api(
username: str,
password: str,
cookie_directory: str | None = None,
):
"""Return an authenticated ``PyiCloudService`` instance.
Args:
username: Apple ID email address.
password: App-specific password.
cookie_directory: Optional directory for session cookies.
Returns:
An authenticated ``PyiCloudService`` instance.
Raises:
ImportError: If ``pyicloud`` is not installed.
ValueError: If authentication fails or 2FA is required interactively.
"""
from pyicloud import PyiCloudService # noqa: S404 – unofficial third-party iCloud client
kwargs: dict = {}
if cookie_directory:
kwargs["cookie_directory"] = cookie_directory
api = PyiCloudService(username, password, **kwargs)
# If 2SA/2FA is required the user must use an app-specific password instead.
if api.requires_2sa or api.requires_2fa:
raise ValueError(
"iCloud account requires two-factor authentication. "
"Please generate an app-specific password at https://appleid.apple.com "
"and use it as ICLOUD_PASSWORD."
)
return api
def _navigate_to_folder(drive_root, folder_path: str):
"""Navigate into (or create) the folder hierarchy described by *folder_path*.
Args:
drive_root: The iCloud Drive root node (``api.drive``).
folder_path: ``/``-separated path such as ``Documents/Uploads``.
Returns:
The drive node representing the target folder.
"""
node = drive_root
if not folder_path:
return node
parts = [p for p in folder_path.strip("/").split("/") if p]
for part in parts:
children = {child.name: child for child in node.dir()}
if part in children:
node = children[part]
else:
# Create the missing folder
node = node.mkdir(part)
return node
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_icloud(self, file_path: str, file_id: int = None, folder_override: str = None):
"""Upload a file to Apple iCloud Drive.
Args:
file_path: Local path to the file to upload.
file_id: Optional ``FileRecord.id`` for progress logging.
folder_override: If provided, overrides the default ``ICLOUD_FOLDER``
setting for this upload.
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting iCloud Drive upload: {file_path}")
log_task_progress(
task_id,
"upload_to_icloud",
"in_progress",
f"Uploading to iCloud Drive: {os.path.basename(file_path)}",
file_id=file_id,
)
# ------------------------------------------------------------------
# Validate inputs
# ------------------------------------------------------------------
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
if not settings.icloud_username or not settings.icloud_password:
error_msg = "iCloud credentials are not configured (ICLOUD_USERNAME / ICLOUD_PASSWORD)"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
filename = os.path.basename(file_path)
target_folder = folder_override if folder_override is not None else (settings.icloud_folder or "")
# ------------------------------------------------------------------
# Authenticate & upload
# ------------------------------------------------------------------
try:
api = _get_icloud_api(
settings.icloud_username,
settings.icloud_password,
settings.icloud_cookie_directory,
)
folder_node = _navigate_to_folder(api.drive, target_folder)
with open(file_path, "rb") as fh:
folder_node.upload(fh)
logger.info(f"[{task_id}] Successfully uploaded {filename} to iCloud Drive folder '{target_folder}'")
log_task_progress(
task_id,
"upload_to_icloud",
"success",
f"Uploaded to iCloud Drive: {filename}",
file_id=file_id,
)
return {
"status": "Completed",
"file": file_path,
"icloud_folder": target_folder or "/",
}
except Exception as e:
error_msg = f"Error uploading {filename} to iCloud Drive: {e}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
raise RuntimeError(error_msg) from e
app/tasks/upload_to_user_integration.py
+32
View File
@@ -571,6 +571,37 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t
return {"status": "Completed", "rclone_dest": dest}
def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]:
"""Upload *file_path* to iCloud Drive using per-user credentials.
Expected *cfg* keys:
* ``folder`` – target folder path inside iCloud Drive (e.g. ``Documents/Uploads``).
* ``cookie_directory`` – (optional) path for session cookie persistence.
Expected *creds* keys:
* ``username`` – Apple ID email address.
* ``password`` – app-specific password.
"""
from app.tasks.upload_to_icloud import _get_icloud_api, _navigate_to_folder
username = creds.get("username") or ""
password = creds.get("password") or ""
folder = cfg.get("folder") or ""
cookie_directory = cfg.get("cookie_directory") or None
if not username or not password:
raise ValueError("iCloud integration is missing username or password in credentials")
api = _get_icloud_api(username, password, cookie_directory)
folder_node = _navigate_to_folder(api.drive, folder)
with open(file_path, "rb") as fh:
folder_node.upload(fh)
logger.info("[%s] iCloud Drive upload complete: folder=%s", task_id, folder or "/")
return {"status": "Completed", "icloud_folder": folder or "/"}
# Map IntegrationType → upload helper
_UPLOAD_HANDLERS = {
IntegrationType.DROPBOX: _upload_dropbox,
@@ -584,6 +615,7 @@ _UPLOAD_HANDLERS = {
IntegrationType.PAPERLESS: _upload_paperless,
IntegrationType.EMAIL: _upload_email,
IntegrationType.RCLONE: _upload_rclone,
IntegrationType.ICLOUD: _upload_icloud,
}
app/tasks/upload_with_rclone.py
+3 -3
View File
@@ -55,12 +55,12 @@ def upload_with_rclone(self, file_path: str, destination: str):
try:
# Ensure the remote path exists (create folders if needed)
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, destination]
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, "--", destination]
subprocess.run(mkdir_cmd, check=True, capture_output=True) # noqa: S603
# Construct the upload command
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, file_path, destination, "--progress"]
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, "--progress", "--", file_path, destination]
log_task_progress(task_id, "rclone_upload", "in_progress", f"Executing rclone copy to {destination}")
@@ -71,7 +71,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
if result.returncode == 0:
# Try to get a public link if possible
try:
link_cmd = ["rclone", "link", "--config", rclone_config_path, f"{destination}/{filename}"]
link_cmd = ["rclone", "link", "--config", rclone_config_path, "--", f"{destination}/{filename}"]
link_result = subprocess.run(link_cmd, capture_output=True, text=True, check=False) # noqa: S603
public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
except (subprocess.SubprocessError, OSError) as e:
app/utils/allowed_types.py
+154
View File
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
".md",
".markdown",
}
# ---------------------------------------------------------------------------
# Fine-grained file-type categories used by IMAP ingestion profiles.
# Each category groups related MIME types and extensions so that users can
# enable/disable a logical collection of formats (e.g. "images") rather than
# having to manage individual MIME strings.
# ---------------------------------------------------------------------------
FILE_TYPE_CATEGORIES: dict[str, dict] = {
"pdf": {
"label": "PDF",
"description": "PDF documents (.pdf)",
"mime_types": frozenset({"application/pdf"}),
"extensions": frozenset({".pdf"}),
},
"office": {
"label": "Microsoft Office",
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
"mime_types": frozenset(
{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"application/vnd.ms-word.document.macroEnabled.12",
"application/vnd.ms-word.template.macroEnabled.12",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"application/vnd.ms-excel.sheet.macroEnabled.12",
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.presentationml.template",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
}
),
"extensions": frozenset(
{
".doc",
".docx",
".docm",
".dot",
".dotx",
".dotm",
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw",
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx",
}
),
},
"opendocument": {
"label": "OpenDocument (LibreOffice)",
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
"mime_types": frozenset(
{
"application/vnd.oasis.opendocument.text",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.graphics",
"application/vnd.oasis.opendocument.formula",
}
),
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
},
"text": {
"label": "Text & Data",
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
"mime_types": frozenset(
{
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
),
"extensions": frozenset({".txt", ".csv", ".rtf"}),
},
"web": {
"label": "Web & Markup",
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
"mime_types": frozenset(
{
"text/html",
"text/markdown",
"text/x-markdown",
}
),
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"mime_types": frozenset(
{
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
),
"extensions": frozenset(
{
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
),
},
}
# Default categories for the "documents only" built-in profile (no images)
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
# All categories including images
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
def get_allowed_types_for_categories(
categories: list[str],
) -> tuple[frozenset[str], frozenset[str]]:
"""Return ``(mime_types, extensions)`` for the given category list.
Unknown category names are silently ignored so that future categories
don't break existing profiles.
"""
mime_types: set[str] = set()
extensions: set[str] = set()
for cat in categories:
info = FILE_TYPE_CATEGORIES.get(cat)
if info:
mime_types |= info["mime_types"]
extensions |= info["extensions"]
return frozenset(mime_types), frozenset(extensions)
app/utils/audit_service.py
+331
View File
@@ -0,0 +1,331 @@
"""
Comprehensive audit-event service for DocuElevate.
Provides helpers to **record** audit events (append-only database writes)
and to optionally **forward** them to external SIEM systems.
Supported SIEM transports:
* **Syslog** – RFC 5424 structured-data messages over UDP or TCP.
* **HTTP** – JSON POST payloads compatible with Splunk HEC, Logstash
HTTP input, Grafana Loki push API, and any generic webhook endpoint.
"""
import json
import logging
import re
import socket
import threading
from datetime import datetime, timezone
from typing import Any
import httpx
from fastapi import Request
from sqlalchemy.orm import Session
from app.config import settings
from app.middleware.audit_log import get_client_ip, get_username
from app.models import AuditLog
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Public helpers
# ---------------------------------------------------------------------------
def record_event(
db: Session,
*,
action: str,
user: str = "system",
resource_type: str | None = None,
resource_id: str | None = None,
ip_address: str | None = None,
details: dict[str, Any] | None = None,
severity: str = "info",
) -> AuditLog:
"""Persist an audit event and optionally forward it to SIEM.
Args:
db: Active SQLAlchemy session.
action: Short action identifier (e.g. ``"login"``, ``"document.create"``).
user: Username performing the action.
resource_type: Category of the affected resource (``"document"``, ``"user"`` …).
resource_id: Identifier of the affected resource.
ip_address: Client IP address (``None`` when not applicable).
details: Arbitrary key/value context serialised as JSON.
severity: One of ``info``, ``warning``, ``error``, ``critical``.
Returns:
The newly created :class:`AuditLog` row.
"""
details_json = json.dumps(details, default=str) if details else None
entry = AuditLog(
user=user,
action=action,
resource_type=resource_type,
resource_id=str(resource_id) if resource_id is not None else None,
ip_address=ip_address,
details=details_json,
severity=severity,
)
db.add(entry)
db.commit()
db.refresh(entry)
# Fire-and-forget SIEM forwarding in a background thread so we never
# block the request path.
if settings.audit_siem_enabled:
payload = _build_siem_payload(entry)
thread = threading.Thread(target=_forward_to_siem, args=(payload,), daemon=True)
thread.start()
return entry
def record_event_from_request(
db: Session,
request: Request,
*,
action: str,
resource_type: str | None = None,
resource_id: str | None = None,
details: dict[str, Any] | None = None,
severity: str = "info",
) -> AuditLog:
"""Convenience wrapper that extracts user and IP from a :class:`Request`.
Args:
db: Active SQLAlchemy session.
request: The current HTTP request.
action: Short action identifier.
resource_type: Category of the affected resource.
resource_id: Identifier of the affected resource.
details: Arbitrary key/value context serialised as JSON.
severity: One of ``info``, ``warning``, ``error``, ``critical``.
Returns:
The newly created :class:`AuditLog` row.
"""
return record_event(
db,
action=action,
user=get_username(request),
resource_type=resource_type,
resource_id=resource_id,
ip_address=get_client_ip(request),
details=details,
severity=severity,
)
def query_events(
db: Session,
*,
action: str | None = None,
user: str | None = None,
resource_type: str | None = None,
severity: str | None = None,
since: datetime | None = None,
until: datetime | None = None,
limit: int = 200,
offset: int = 0,
) -> list[AuditLog]:
"""Query audit log entries with optional filtering.
Args:
db: Active SQLAlchemy session.
action: Filter by action string (exact match).
user: Filter by username (exact match).
resource_type: Filter by resource type (exact match).
severity: Filter by severity level (exact match).
since: Only events at or after this timestamp.
until: Only events at or before this timestamp.
limit: Maximum number of rows to return.
offset: Number of rows to skip (for pagination).
Returns:
List of :class:`AuditLog` rows ordered by *timestamp descending*.
"""
q = db.query(AuditLog)
if action:
q = q.filter(AuditLog.action == action)
if user:
q = q.filter(AuditLog.user == user)
if resource_type:
q = q.filter(AuditLog.resource_type == resource_type)
if severity:
q = q.filter(AuditLog.severity == severity)
if since:
q = q.filter(AuditLog.timestamp >= since)
if until:
q = q.filter(AuditLog.timestamp <= until)
return q.order_by(AuditLog.timestamp.desc()).offset(offset).limit(limit).all()
def count_events(
db: Session,
*,
action: str | None = None,
user: str | None = None,
resource_type: str | None = None,
severity: str | None = None,
since: datetime | None = None,
until: datetime | None = None,
) -> int:
"""Return the total count of events matching the given filters.
Args:
db: Active SQLAlchemy session.
action: Filter by action string.
user: Filter by username.
resource_type: Filter by resource type.
severity: Filter by severity level.
since: Only events at or after this timestamp.
until: Only events at or before this timestamp.
Returns:
Integer count.
"""
q = db.query(AuditLog)
if action:
q = q.filter(AuditLog.action == action)
if user:
q = q.filter(AuditLog.user == user)
if resource_type:
q = q.filter(AuditLog.resource_type == resource_type)
if severity:
q = q.filter(AuditLog.severity == severity)
if since:
q = q.filter(AuditLog.timestamp >= since)
if until:
q = q.filter(AuditLog.timestamp <= until)
return q.count()
# ---------------------------------------------------------------------------
# SIEM forwarding internals
# ---------------------------------------------------------------------------
_SYSLOG_FACILITY_LOCAL0 = 16
_SYSLOG_SEVERITY_MAP = {
"info": 6,
"warning": 4,
"error": 3,
"critical": 2,
}
def _build_siem_payload(entry: AuditLog) -> dict[str, Any]:
"""Convert an :class:`AuditLog` row into a plain dict for SIEM delivery."""
ts = entry.timestamp if entry.timestamp else datetime.now(timezone.utc)
return {
"id": entry.id,
"timestamp": ts.isoformat(),
"user": entry.user,
"action": entry.action,
"resource_type": entry.resource_type,
"resource_id": entry.resource_id,
"ip_address": entry.ip_address,
"details": entry.details,
"severity": entry.severity,
"source": "docuelevate",
}
def _forward_to_siem(payload: dict[str, Any]) -> None:
"""Route a SIEM payload to the configured transport."""
transport = settings.audit_siem_transport.lower()
try:
if transport == "syslog":
_send_syslog(payload)
elif transport == "http":
_send_http(payload)
else:
logger.warning("Unknown SIEM transport %r; skipping forwarding", transport)
except Exception:
logger.exception("Failed to forward audit event to SIEM (%s)", transport)
def _send_syslog(payload: dict[str, Any]) -> None:
"""Send a RFC 5424 syslog message to the configured receiver."""
severity_num = _SYSLOG_SEVERITY_MAP.get(payload.get("severity", "info"), 6)
priority = _SYSLOG_FACILITY_LOCAL0 * 8 + severity_num
ts = payload.get("timestamp", datetime.now(timezone.utc).isoformat())
hostname = socket.gethostname()
app_name = "docuelevate"
msg_id = payload.get("action", "-")
# Structured data (SD) element with key event fields.
sd = (
f'[docuelevate@0 user="{payload.get("user", "-")}" '
f'action="{payload.get("action", "-")}" '
f'resource_type="{payload.get("resource_type", "-")}" '
f'resource_id="{payload.get("resource_id", "-")}" '
f'ip="{payload.get("ip_address", "-")}"]'
)
message = json.dumps(payload, default=str)
syslog_msg = f"<{priority}>1 {ts} {hostname} {app_name} - {msg_id} {sd} {message}"
proto = settings.audit_siem_syslog_protocol.lower()
host = settings.audit_siem_syslog_host
port = settings.audit_siem_syslog_port
if proto == "tcp":
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(5)
sock.connect((host, port))
sock.sendall(syslog_msg.encode("utf-8"))
else:
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
sock.settimeout(5)
sock.sendto(syslog_msg.encode("utf-8"), (host, port))
logger.debug("Syslog audit event sent to %s:%s (%s)", host, port, proto)
def _send_http(payload: dict[str, Any]) -> None:
"""POST a JSON audit event to the configured HTTP endpoint."""
url = settings.audit_siem_http_url
if not url:
logger.warning("SIEM HTTP URL not configured; skipping HTTP forwarding")
return
headers: dict[str, str] = {"Content-Type": "application/json"}
token = settings.audit_siem_http_token
if token:
headers["Authorization"] = f"Bearer {token}"
# Parse custom headers (comma-separated "Key:Value" pairs).
# Reject headers that could override security-critical ones already set,
# and validate that header names contain only RFC 7230 token characters.
_PROTECTED_HEADERS = {"authorization", "content-type", "host"}
_VALID_HEADER_NAME = re.compile(r"^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$")
raw_custom = settings.audit_siem_http_custom_headers
if raw_custom:
for raw_pair in raw_custom.split(","):
pair = raw_pair.strip()
if ":" in pair:
k, _, v = pair.partition(":")
name = k.strip()
if not name or not _VALID_HEADER_NAME.match(name):
logger.warning("Skipping invalid SIEM custom header name: %r", name)
continue
if name.lower() in _PROTECTED_HEADERS:
logger.warning("Skipping protected SIEM custom header: %r", name)
continue
headers[name] = v.strip()
# Wrap in Splunk HEC-style envelope when URL contains ``/services/collector``.
body: dict[str, Any]
if "/services/collector" in url:
body = {"event": payload, "sourcetype": "docuelevate:audit", "source": "docuelevate"}
else:
body = payload
with httpx.Client(timeout=10) as client:
resp = client.post(url, json=body, headers=headers)
resp.raise_for_status()
logger.debug("HTTP audit event forwarded to %s (status %s)", url, resp.status_code)
app/utils/compliance_service.py
+433
View File
@@ -0,0 +1,433 @@
"""Compliance service for managing GDPR, HIPAA, and SOC2 compliance templates.
Provides pre-built compliance configurations that can be applied with one click
to ensure the DocuElevate instance meets regulatory requirements.
"""
import json
import logging
from datetime import datetime, timezone
from typing import Any
from sqlalchemy.orm import Session
from app.models import ComplianceTemplate
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Pre-built compliance template definitions
# ---------------------------------------------------------------------------
COMPLIANCE_TEMPLATES: dict[str, dict[str, Any]] = {
"gdpr": {
"display_name": "GDPR (General Data Protection Regulation)",
"description": (
"European Union regulation for data protection and privacy. "
"Enforces data minimisation, encryption at rest, audit logging, "
"and limits PII exposure in telemetry."
),
"settings": {
"auth_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "User authentication must be enabled to control access to personal data.",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Personally identifiable information must not be sent to external monitoring services.",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "HTTP security headers protect against common web vulnerabilities.",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "HTTP Strict Transport Security ensures encrypted connections.",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "CSP headers prevent cross-site scripting and data injection attacks.",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "X-Frame-Options header prevents clickjacking attacks.",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Data minimisation: avoid storing duplicate documents.",
},
],
},
"hipaa": {
"display_name": "HIPAA (Health Insurance Portability and Accountability Act)",
"description": (
"United States regulation for protecting health information. "
"Requires strong access controls, audit trails, encryption, "
"and strict session management."
),
"settings": {
"auth_enabled": "True",
"multi_user_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "Access controls are required to protect electronic Protected Health Information (ePHI).",
},
{
"key": "multi_user_enabled",
"expected": "True",
"label": "Multi-user mode enabled",
"description": "Individual user accounts required for access accountability.",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Protected Health Information must not be sent to external services.",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "Security headers protect ePHI during transmission.",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "Encrypted transport required for all ePHI transmissions.",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "CSP prevents injection attacks that could expose ePHI.",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "Prevents embedding the application in unauthorized frames.",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Minimise data footprint for ePHI.",
},
],
},
"soc2": {
"display_name": "SOC 2 (Service Organization Control 2)",
"description": (
"Trust Service Criteria framework for service organisations. "
"Focuses on security, availability, processing integrity, "
"confidentiality, and privacy."
),
"settings": {
"auth_enabled": "True",
"multi_user_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "Logical access controls required (CC6.1).",
},
{
"key": "multi_user_enabled",
"expected": "True",
"label": "Multi-user mode enabled",
"description": "Individual user accounts for access management (CC6.2).",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Confidential information must not leak to external services (CC6.7).",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "Protection against common web threats (CC6.6).",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "Encrypted transport in transit (CC6.7).",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "Application-level security controls (CC6.6).",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "UI redress attack prevention (CC6.6).",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Data integrity through deduplication (PI1.1).",
},
],
},
}
def seed_compliance_templates(db: Session) -> None:
"""Create or update the built-in compliance template rows.
Called once at application startup to ensure the ``compliance_templates``
table always contains the latest definitions.
"""
for name, defn in COMPLIANCE_TEMPLATES.items():
existing = db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first()
if existing is None:
template = ComplianceTemplate(
name=name,
display_name=defn["display_name"],
description=defn["description"],
settings_json=json.dumps(defn["settings"]),
enabled=False,
status="not_applied",
)
db.add(template)
logger.info(f"Seeded compliance template: {name}")
else:
# Update display_name and description if changed, but preserve user state
existing.display_name = defn["display_name"]
existing.description = defn["description"]
try:
db.commit()
except Exception:
db.rollback()
logger.exception("Failed to seed compliance templates")
def get_all_templates(db: Session) -> list[dict[str, Any]]:
"""Return all compliance templates with their current status."""
templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all()
result = []
for t in templates:
defn = COMPLIANCE_TEMPLATES.get(t.name, {})
checks = defn.get("checks", [])
result.append(
{
"id": t.id,
"name": t.name,
"display_name": t.display_name,
"description": t.description,
"enabled": t.enabled,
"status": t.status,
"applied_at": t.applied_at.isoformat() if t.applied_at else None,
"applied_by": t.applied_by,
"settings": json.loads(t.settings_json) if t.settings_json else {},
"checks": checks,
"check_count": len(checks),
}
)
return result
def get_template_by_name(db: Session, name: str) -> ComplianceTemplate | None:
"""Retrieve a single compliance template by name."""
return db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first()
def evaluate_template_status(db: Session, name: str) -> dict[str, Any]:
"""Evaluate the compliance status of a template against live settings.
Returns a dict with ``status``, ``total``, ``passed``, ``failed``, and
a list of individual ``check_results``.
"""
from app.config import settings as app_settings
from app.utils.settings_service import get_all_settings_from_db
defn = COMPLIANCE_TEMPLATES.get(name)
if defn is None:
return {"status": "unknown", "total": 0, "passed": 0, "failed": 0, "check_results": []}
db_settings = get_all_settings_from_db(db)
checks = defn.get("checks", [])
results: list[dict[str, Any]] = []
passed = 0
for check in checks:
key = check["key"]
expected = check["expected"]
# Resolve effective value: DB > config object
if key in db_settings and db_settings[key] is not None:
actual = str(db_settings[key])
else:
actual = str(getattr(app_settings, key, ""))
is_passing = actual.lower() == expected.lower()
if is_passing:
passed += 1
results.append(
{
"key": key,
"label": check["label"],
"description": check["description"],
"expected": expected,
"actual": actual,
"passing": is_passing,
}
)
total = len(checks)
if passed == total:
status = "compliant"
elif passed > 0:
status = "partial"
else:
status = "non_compliant"
return {
"status": status,
"total": total,
"passed": passed,
"failed": total - passed,
"check_results": results,
}
def apply_template(db: Session, name: str, applied_by: str = "admin") -> dict[str, Any]:
"""Apply a compliance template by writing its settings to the database.
Returns a summary of what was applied.
"""
from app.utils.settings_service import save_setting_to_db
defn = COMPLIANCE_TEMPLATES.get(name)
if defn is None:
return {"success": False, "error": f"Unknown template: {name}"}
template = get_template_by_name(db, name)
if template is None:
return {"success": False, "error": f"Template not found in database: {name}"}
applied_settings: dict[str, str] = {}
errors: list[str] = []
for key, value in defn["settings"].items():
try:
save_setting_to_db(db, key, value, changed_by=f"compliance:{name}")
applied_settings[key] = value
except Exception as e:
errors.append(f"{key}: {e}")
logger.error(f"Failed to apply compliance setting {key}={value}: {e}")
# Update the template record
now = datetime.now(timezone.utc)
template.enabled = True
template.settings_json = json.dumps(applied_settings)
template.applied_at = now
template.applied_by = applied_by
# Evaluate and store status
eval_result = evaluate_template_status(db, name)
template.status = eval_result["status"]
try:
db.commit()
except Exception:
db.rollback()
logger.exception(f"Failed to update compliance template record: {name}")
return {"success": False, "error": "Database commit failed"}
logger.info(f"Applied compliance template '{name}' by {applied_by}: {len(applied_settings)} settings written")
return {
"success": len(errors) == 0,
"template": name,
"applied_settings": applied_settings,
"errors": errors,
"status": eval_result,
}
def get_compliance_summary(db: Session) -> dict[str, Any]:
"""Return a high-level compliance dashboard summary across all templates."""
templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all()
summary: list[dict[str, Any]] = []
total_checks = 0
total_passed = 0
for t in templates:
eval_result = evaluate_template_status(db, t.name)
total_checks += eval_result["total"]
total_passed += eval_result["passed"]
summary.append(
{
"name": t.name,
"display_name": t.display_name,
"enabled": t.enabled,
"status": eval_result["status"],
"total": eval_result["total"],
"passed": eval_result["passed"],
"failed": eval_result["failed"],
"applied_at": t.applied_at.isoformat() if t.applied_at else None,
"applied_by": t.applied_by,
}
)
overall = "compliant" if total_checks > 0 and total_passed == total_checks else "non_compliant"
if 0 < total_passed < total_checks:
overall = "partial"
return {
"overall_status": overall,
"total_checks": total_checks,
"total_passed": total_passed,
"total_failed": total_checks - total_passed,
"templates": summary,
}
app/utils/config_validator/providers.py
+25 -10
View File
@@ -144,7 +144,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
),
"enabled": True,
"enabled": getattr(settings, "dropbox_enabled", True),
"description": "Upload files to Dropbox cloud storage",
"details": {
"folder": getattr(settings, "dropbox_folder", "Not set"),
@@ -161,7 +161,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"configured": bool(
getattr(settings, "dest_email_host", None) and getattr(settings, "dest_email_default_recipient", None)
),
"enabled": True,
"enabled": getattr(settings, "dest_email_enabled", True),
"description": "Send documents via email",
"details": {
"host": getattr(settings, "dest_email_host", "Not set"),
@@ -183,7 +183,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "ftp_username", None)
and getattr(settings, "ftp_password", None)
),
"enabled": True,
"enabled": getattr(settings, "ftp_enabled", True),
"description": "Upload files to FTP server",
"details": {
"host": getattr(settings, "ftp_host", "Not set"),
@@ -214,7 +214,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"name": "Google Drive",
"icon": "fa-brands fa-google-drive",
"configured": is_configured and bool(getattr(settings, "google_drive_folder_id", None)),
"enabled": True,
"enabled": getattr(settings, "google_drive_enabled", True),
"description": "Store documents in Google Drive",
"details": {
"auth_type": "OAuth" if use_oauth else "Service Account",
@@ -250,7 +250,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
),
"enabled": True,
"enabled": getattr(settings, "nextcloud_enabled", True),
"description": "Store documents in NextCloud",
"details": {
"url": getattr(settings, "nextcloud_upload_url", "Not set"),
@@ -270,7 +270,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
),
"enabled": True,
"enabled": getattr(settings, "onedrive_enabled", True),
"description": "Store documents in Microsoft OneDrive",
"details": {
"client_id": getattr(settings, "onedrive_client_id", "Not set"),
@@ -288,7 +288,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"configured": bool(
getattr(settings, "paperless_host", None) and getattr(settings, "paperless_ngx_api_token", None)
),
"enabled": True,
"enabled": getattr(settings, "paperless_enabled", True),
"description": "Document management system for digital archives",
"details": {
"host": getattr(settings, "paperless_host", "Not set"),
@@ -305,7 +305,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "aws_access_key_id", None)
and getattr(settings, "aws_secret_access_key", None)
),
"enabled": True,
"enabled": getattr(settings, "s3_enabled", True),
"description": "Store documents in S3-compatible object storage",
"details": {
"bucket": getattr(settings, "s3_bucket_name", "Not set"),
@@ -327,7 +327,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "sftp_username", None)
and (getattr(settings, "sftp_password", None) or getattr(settings, "sftp_private_key", None))
),
"enabled": True,
"enabled": getattr(settings, "sftp_enabled", True),
"description": "Upload files to SFTP server",
"details": {
"host": getattr(settings, "sftp_host", "Not set"),
@@ -362,7 +362,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "webdav_username", None)
and getattr(settings, "webdav_password", None)
),
"enabled": True,
"enabled": getattr(settings, "webdav_enabled", True),
"description": "Store documents on WebDAV servers",
"details": {
"url": getattr(settings, "webdav_url", "Not set"),
@@ -373,4 +373,19 @@ def get_provider_status() -> dict[str, dict[str, object]]:
},
}
# Check iCloud Drive configuration
providers["iCloud Drive"] = {
"name": "iCloud Drive",
"icon": "fa-brands fa-apple",
"configured": bool(getattr(settings, "icloud_username", None) and getattr(settings, "icloud_password", None)),
"enabled": getattr(settings, "icloud_enabled", True),
"description": "Store documents in Apple iCloud Drive",
"details": {
"username": getattr(settings, "icloud_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "icloud_password", None)),
"folder": getattr(settings, "icloud_folder", "Not set"),
"cookie_directory": getattr(settings, "icloud_cookie_directory", "Not set"),
},
}
return providers
app/utils/config_validator/validators.py
+32 -2
View File
@@ -59,13 +59,43 @@ def validate_auth_config() -> list[str]:
and getattr(settings, "authentik_config_url", None)
)
if not using_simple_auth and not using_oidc:
issues.append("Neither simple authentication nor OIDC are properly configured")
# Check if any social login provider is enabled
using_social_login = any(
getattr(settings, f"social_auth_{p}_enabled", False) for p in ("google", "microsoft", "apple", "dropbox")
)
if not using_simple_auth and not using_oidc and not using_social_login:
issues.append("Neither simple authentication, OIDC, nor social login are properly configured")
# If using OIDC, check for provider name
if using_oidc and not getattr(settings, "oauth_provider_name", None):
issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled")
# Validate individual social login provider configs
if getattr(settings, "social_auth_google_enabled", False):
if not getattr(settings, "social_auth_google_client_id", None):
issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_ID is required when Google login is enabled")
if not getattr(settings, "social_auth_google_client_secret", None):
issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_SECRET is required when Google login is enabled")
if getattr(settings, "social_auth_microsoft_enabled", False):
if not getattr(settings, "social_auth_microsoft_client_id", None):
issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_ID is required when Microsoft login is enabled")
if not getattr(settings, "social_auth_microsoft_client_secret", None):
issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET is required when Microsoft login is enabled")
if getattr(settings, "social_auth_apple_enabled", False):
if not getattr(settings, "social_auth_apple_client_id", None):
issues.append("SOCIAL_AUTH_APPLE_CLIENT_ID is required when Apple login is enabled")
if not getattr(settings, "social_auth_apple_team_id", None):
issues.append("SOCIAL_AUTH_APPLE_TEAM_ID is required when Apple login is enabled")
if getattr(settings, "social_auth_dropbox_enabled", False):
if not getattr(settings, "social_auth_dropbox_client_id", None):
issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_ID is required when Dropbox login is enabled")
if not getattr(settings, "social_auth_dropbox_client_secret", None):
issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_SECRET is required when Dropbox login is enabled")
return issues
app/utils/db_migrate.py
+8 -1
View File
@@ -12,6 +12,7 @@ The utility:
"""
import logging
import re
from typing import Any
from sqlalchemy import MetaData, create_engine, inspect, text
@@ -32,8 +33,10 @@ _TABLE_ORDER = [
"processing_logs",
"application_settings",
"settings_audit_log",
"audit_logs",
"saved_searches",
"webhook_configs",
"shared_links",
]
@@ -82,8 +85,12 @@ def preview_migration(source_url: str) -> dict[str, Any]:
total = 0
with src_engine.connect() as conn:
for table_name in tables:
if not re.match(r"^[a-zA-Z0-9_]+$", table_name):
logger.warning(f"Skipping table with invalid name format: {table_name}")
continue
# table_name is safe — sourced from inspect().get_table_names(), not user input
row = conn.execute(text(f'SELECT COUNT(*) FROM "{table_name}"')).fetchone() # noqa: S608
quoted_table = conn.dialect.identifier_preparer.quote(table_name)
row = conn.execute(text(f"SELECT COUNT(*) FROM {quoted_table}")).fetchone() # noqa: S608
count = row[0] if row else 0
result.append({"name": table_name, "row_count": count})
total += count
app/utils/env_utils.py
+55
View File
@@ -0,0 +1,55 @@
import logging
import os
from typing import Dict
logger = logging.getLogger(__name__)
def update_env_file(settings_to_update: Dict[str, str]) -> bool:
"""
Updates the .env file with the given settings (best-effort).
Creates or modifies existing keys.
Args:
settings_to_update: A dictionary mapping uppercase env var names to their new string values.
Returns:
True if the file was successfully updated, False otherwise.
"""
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if not os.path.exists(env_path):
logger.warning(f".env file not found at {env_path}, skipping file write")
return False
logger.info(f"Updating settings in {env_path}")
with open(env_path, "r") as f:
env_lines = f.readlines()
updated = set()
new_env_lines = []
for line in env_lines:
stripped_line = line.rstrip()
is_updated = False
for key, value in settings_to_update.items():
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
new_env_lines.append(f"{key}={value}")
updated.add(key)
is_updated = True
break
if not is_updated:
new_env_lines.append(stripped_line)
for key, value in settings_to_update.items():
if key not in updated:
new_env_lines.append(f"{key}={value}")
with open(env_path, "w") as f:
f.write("\n".join(new_env_lines) + "\n")
logger.info("Successfully updated settings in .env file")
return True
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
return False
app/utils/filename_utils.py
+5
View File
@@ -8,6 +8,11 @@ from pathlib import Path
logger = logging.getLogger(__name__)
# Pattern for valid filenames (alphanumeric, dash, underscore, period, and space)
# Used for validating GPT-provided filenames and other inputs
VALID_FILENAME_PATTERN = r"^[\w\-\. ]+$"
VALID_FILENAME_RE = re.compile(VALID_FILENAME_PATTERN)
def get_unique_filename(original_path: str, check_exists_func: Callable[[str], bool] | None = None) -> str:
"""
app/utils/i18n.py
+767
View File
@@ -0,0 +1,767 @@
"""Internationalization (i18n) and localization (l10n) utilities.
Provides a JSON-based translation system for the DocuElevate UI with:
* **77 supported languages** covering European, Asian, Middle-Eastern, African, and other languages
* Browser ``Accept-Language`` detection with cookie & user-profile persistence
* AI-powered fallback translation via the configured LLM provider
* Locale-aware date, number, and file-size formatting helpers
* Jinja2 integration via a ``_()`` global function
Language resolution order:
1. User profile ``preferred_language`` (persisted in DB)
2. ``docuelevate_lang`` cookie
3. ``Accept-Language`` HTTP header
4. Default (``en``)
"""
from __future__ import annotations
import json
import logging
from datetime import date, datetime
from functools import lru_cache
from pathlib import Path
from typing import Any
from starlette.requests import Request
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Supported languages (ordered by priority)
# ---------------------------------------------------------------------------
SUPPORTED_LANGUAGES: list[dict[str, str]] = [
# --- Tier 1: Primary European languages ---
# flag: lowercase ISO 3166-1 alpha-2 country code used with the flag-icons CSS library
# (e.g. "gb" → <span class="fi fi-gb">). Regional codes like "gb-wls" are also supported.
{"code": "en", "name": "English", "native": "English", "flag": "gb"},
{"code": "de", "name": "German", "native": "Deutsch", "flag": "de"},
{"code": "fr", "name": "French", "native": "Français", "flag": "fr"},
{"code": "es", "name": "Spanish", "native": "Español", "flag": "es"},
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "it"},
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "pt"},
# --- Tier 2: Western & Northern European ---
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "nl"},
{"code": "nb", "name": "Norwegian Bokmål", "native": "Norsk bokmål", "flag": "no"},
{"code": "no", "name": "Norwegian", "native": "Norsk", "flag": "no"},
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "dk"},
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "se"},
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "fi"},
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "is"},
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "ie"},
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "lu"},
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "es"}, # no dedicated ISO flag; use Spain
{"code": "cy", "name": "Welsh", "native": "Cymraeg", "flag": "gb-wls"}, # flag-icons GB region code
{"code": "fy", "name": "Western Frisian", "native": "Frysk", "flag": "nl"},
{"code": "gl", "name": "Galician", "native": "Galego", "flag": "es"},
{"code": "li", "name": "Limburgish", "native": "Limburgs", "flag": "nl"},
{"code": "vls", "name": "Flemish", "native": "West-Vlams", "flag": "be"},
{"code": "nds", "name": "Low German", "native": "Plattdüütsch", "flag": "de"},
# --- Tier 3: Central & Eastern European ---
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "pl"},
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "cz"},
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "sk"},
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "hu"},
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "si"},
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "hr"},
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "ro"},
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "bg"},
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "gr"},
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "ee"},
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "lv"},
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "lt"},
{"code": "sr", "name": "Serbian", "native": "Српски", "flag": "rs"},
# --- Tier 4: Non-EU European, Middle Eastern & African ---
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "tr"},
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "ua"},
{"code": "he", "name": "Hebrew", "native": "עברית", "flag": "il"},
{"code": "ar", "name": "Arabic", "native": "العربية", "flag": "sa"},
{"code": "fa", "name": "Persian", "native": "فارسی", "flag": "ir"},
{"code": "af", "name": "Afrikaans", "native": "Afrikaans", "flag": "za"},
# --- Tier 5: Asian languages ---
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "cn"},
{"code": "zh-TW", "name": "Traditional Chinese", "native": "繁體中文", "flag": "tw"},
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "jp"},
{"code": "ko", "name": "Korean", "native": "한국어", "flag": "kr"},
{"code": "vi", "name": "Vietnamese", "native": "Tiếng Việt", "flag": "vn"},
{"code": "pa", "name": "Punjabi", "native": "ਪੰਜਾਬੀ", "flag": "in"},
{"code": "kn", "name": "Kannada", "native": "ಕನ್ನಡ", "flag": "in"},
{"code": "hi", "name": "Hindi", "native": "हिन्दी", "flag": "in"},
{"code": "bn", "name": "Bengali", "native": "বাংলা", "flag": "bd"},
{"code": "gu", "name": "Gujarati", "native": "ગુજરાતી", "flag": "in"},
{"code": "ml", "name": "Malayalam", "native": "മലയാളം", "flag": "in"},
{"code": "mr", "name": "Marathi", "native": "मराठी", "flag": "in"},
{"code": "ta", "name": "Tamil", "native": "தமிழ்", "flag": "in"},
{"code": "te", "name": "Telugu", "native": "తెలుగు", "flag": "in"},
{"code": "ur", "name": "Urdu", "native": "اردو", "flag": "pk"},
{"code": "si", "name": "Sinhala", "native": "සිංහල", "flag": "lk"},
{"code": "ne", "name": "Nepali", "native": "नेपाली", "flag": "np"},
{"code": "th", "name": "Thai", "native": "ไทย", "flag": "th"},
{"code": "km", "name": "Khmer", "native": "ខ្មែរ", "flag": "kh"},
{"code": "id", "name": "Indonesian", "native": "Bahasa Indonesia", "flag": "id"},
{"code": "ms", "name": "Malay", "native": "Bahasa Melayu", "flag": "my"},
{"code": "jv", "name": "Javanese", "native": "Basa Jawa", "flag": "id"},
{"code": "tl", "name": "Tagalog", "native": "Filipino", "flag": "ph"},
{"code": "mn", "name": "Mongolian", "native": "Монгол", "flag": "mn"},
{"code": "kk", "name": "Kazakh", "native": "Қазақ тілі", "flag": "kz"},
{"code": "uz", "name": "Uzbek", "native": "Oʻzbekcha", "flag": "uz"},
{"code": "az", "name": "Azerbaijani", "native": "Azərbaycan dili", "flag": "az"},
{"code": "hy", "name": "Armenian", "native": "Հայերեն", "flag": "am"},
{"code": "ka", "name": "Georgian", "native": "ქართული", "flag": "ge"},
# --- Tier 6: African languages ---
{"code": "sw", "name": "Swahili", "native": "Kiswahili", "flag": "ke"},
{"code": "am", "name": "Amharic", "native": "አማርኛ", "flag": "et"},
{"code": "ha", "name": "Hausa", "native": "Hausa", "flag": "ng"},
{"code": "yo", "name": "Yoruba", "native": "Yorùbá", "flag": "ng"},
{"code": "ig", "name": "Igbo", "native": "Igbo", "flag": "ng"},
{"code": "zu", "name": "Zulu", "native": "isiZulu", "flag": "za"},
# --- Tier 7: Constructed & other languages ---
{"code": "eo", "name": "Esperanto", "native": "Esperanto", "flag": "un"}, # UN flag for international language
]
SUPPORTED_LANGUAGE_CODES: set[str] = {lang["code"] for lang in SUPPORTED_LANGUAGES}
DEFAULT_LANGUAGE = "en"
# Lookup map for fast code → language-dict resolution
_LANG_CODE_MAP: dict[str, dict[str, str]] = {lang["code"]: lang for lang in SUPPORTED_LANGUAGES}
# Global-usage order used to fill remaining slots in the smart suggestions list
_POPULAR_LANGUAGE_CODES: list[str] = ["en", "zh", "es", "ar", "fr", "de", "ja", "pt", "hi", "ko"]
# ---------------------------------------------------------------------------
# Translation file loading
# ---------------------------------------------------------------------------
_TRANSLATIONS_DIR = Path(__file__).resolve().parent.parent.parent / "frontend" / "translations"
_translation_cache: dict[str, dict[str, str]] = {}
def _load_translations(locale: str) -> dict[str, str]:
"""Load the translation JSON file for *locale*, with caching."""
if locale in _translation_cache:
return _translation_cache[locale]
filepath = _TRANSLATIONS_DIR / f"{locale}.json"
if not filepath.is_file():
logger.warning("Translation file not found for locale '%s'", locale)
_translation_cache[locale] = {}
return {}
try:
data: dict[str, str] = json.loads(filepath.read_text(encoding="utf-8"))
_translation_cache[locale] = data
return data
except (json.JSONDecodeError, OSError):
logger.exception("Failed to load translations for '%s'", locale)
_translation_cache[locale] = {}
return {}
def reload_translations() -> None:
"""Clear the translation cache so files are re-read on next access."""
_translation_cache.clear()
# ---------------------------------------------------------------------------
# Core translation function
# ---------------------------------------------------------------------------
def translate(key: str, locale: str | None = None, **kwargs: Any) -> str:
"""Return the translated string for *key* in *locale*.
Falls back through:
1. Requested *locale*
2. English (``en``)
3. The raw key itself (to keep the UI functional)
Positional placeholders ``{0}``, ``{1}`` or named placeholders
``{name}`` in the translated string are interpolated via *kwargs*.
"""
locale = locale if locale and locale in SUPPORTED_LANGUAGE_CODES else DEFAULT_LANGUAGE
translations = _load_translations(locale)
value = translations.get(key)
# Fallback to English
if value is None and locale != DEFAULT_LANGUAGE:
en_translations = _load_translations(DEFAULT_LANGUAGE)
value = en_translations.get(key)
# Fallback to key itself
if value is None:
value = key
if kwargs:
try:
value = value.format(**kwargs)
except (KeyError, IndexError):
pass # Return unformatted string rather than crash
return value
# ---------------------------------------------------------------------------
# AI fallback translation (best-effort, non-blocking)
# ---------------------------------------------------------------------------
_ai_translation_cache: dict[tuple[str, str], str] = {}
def translate_with_ai_fallback(text: str, target_locale: str) -> str:
"""Translate *text* using the configured AI provider as a fallback.
Returns the original *text* unchanged when:
* The target locale is English (source language)
* The AI provider is unavailable or returns an error
* The translation has already been cached
Results are cached in-memory for the lifetime of the process.
"""
if target_locale == DEFAULT_LANGUAGE or target_locale not in SUPPORTED_LANGUAGE_CODES:
return text
cache_key = (text, target_locale)
if cache_key in _ai_translation_cache:
return _ai_translation_cache[cache_key]
target_name = next(
(lang["name"] for lang in SUPPORTED_LANGUAGES if lang["code"] == target_locale),
target_locale,
)
try:
from litellm import completion # type: ignore[import-untyped]
from app.config import settings
model = getattr(settings, "ai_model", None) or getattr(settings, "openai_model", "gpt-4o-mini")
response = completion(
model=model,
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following UI text "
f"from English to {target_name}. Return ONLY the translated text, "
f"nothing else. Keep any HTML tags, placeholders like {{name}}, "
f"and special characters intact."
),
},
{"role": "user", "content": text},
],
max_tokens=256,
temperature=0.1,
)
translated = response.choices[0].message.content.strip()
_ai_translation_cache[cache_key] = translated
return translated
except Exception:
logger.debug("AI fallback translation failed for '%s' → %s", text[:50], target_locale)
return text
# ---------------------------------------------------------------------------
# Language detection
# ---------------------------------------------------------------------------
def detect_language(request: Request) -> str:
"""Determine the preferred UI language from the request context.
Resolution order:
1. ``preferred_language`` stored in the user session
2. ``docuelevate_lang`` cookie
3. ``Accept-Language`` HTTP header (best match)
4. Default → ``en``
"""
# 1. User session preference
if hasattr(request, "session"):
session_lang = request.session.get("preferred_language")
if isinstance(session_lang, str) and session_lang in SUPPORTED_LANGUAGE_CODES:
return session_lang
# 2. Cookie
if hasattr(request, "cookies"):
cookie_lang = request.cookies.get("docuelevate_lang")
if isinstance(cookie_lang, str) and cookie_lang in SUPPORTED_LANGUAGE_CODES:
return cookie_lang
# 3. Accept-Language header
accept = ""
if hasattr(request, "headers"):
accept = request.headers.get("accept-language", "")
lang = _parse_accept_language(accept)
if lang:
return lang
return DEFAULT_LANGUAGE
def _parse_accept_language_entries(header: str) -> list[tuple[float, str]]:
"""Parse an ``Accept-Language`` header into quality-sorted ``(q, tag)`` pairs."""
if not header:
return []
entries: list[tuple[float, str]] = []
for raw_part in header.split(","):
part = raw_part.strip()
if not part:
continue
if ";q=" in part:
lang_tag, _, q_str = part.partition(";q=")
try:
quality = float(q_str.strip())
except ValueError:
quality = 0.0
else:
lang_tag = part
quality = 1.0
entries.append((quality, lang_tag.strip().lower()))
entries.sort(key=lambda e: e[0], reverse=True)
return entries
def _parse_accept_language(header: str) -> str | None:
"""Extract the best matching language from an ``Accept-Language`` header.
Parses quality values and returns the highest-priority match among
:data:`SUPPORTED_LANGUAGE_CODES`, or ``None`` if nothing matches.
"""
for _quality, tag in _parse_accept_language_entries(header):
code = tag.split("-")[0]
if code in SUPPORTED_LANGUAGE_CODES:
return code
return None
# Maximum number of languages shown in the compact nav-bar dropdown
_SUGGESTED_LANGUAGES_MAX = 6
def get_suggested_languages(current_locale: str, accept_language_header: str = "") -> list[dict[str, str]]:
"""Return up to :data:`_SUGGESTED_LANGUAGES_MAX` suggested languages for the compact picker.
Selection priority:
1. The currently active language (always included first).
2. Languages listed in the browser's ``Accept-Language`` header.
3. Popular global languages (by estimated speaker count) as fillers.
The resulting list is de-duplicated and capped at
:data:`_SUGGESTED_LANGUAGES_MAX` entries.
"""
candidates: list[str] = []
# 1. Active locale first
if current_locale in SUPPORTED_LANGUAGE_CODES:
candidates.append(current_locale)
# 2. Browser preferences
for _quality, tag in _parse_accept_language_entries(accept_language_header):
if len(candidates) >= _SUGGESTED_LANGUAGES_MAX:
break
code = tag.split("-")[0]
if code in SUPPORTED_LANGUAGE_CODES and code not in candidates:
candidates.append(code)
# 3. Popular language fillers
for code in _POPULAR_LANGUAGE_CODES:
if len(candidates) >= _SUGGESTED_LANGUAGES_MAX:
break
if code not in candidates and code in SUPPORTED_LANGUAGE_CODES:
candidates.append(code)
return [_LANG_CODE_MAP[c] for c in candidates[:_SUGGESTED_LANGUAGES_MAX] if c in _LANG_CODE_MAP]
# ---------------------------------------------------------------------------
# Localization helpers (l10n)
# ---------------------------------------------------------------------------
# Locale-specific formatting rules for date/number display
_LOCALE_FORMATS: dict[str, dict[str, Any]] = {
"en": {
"date": "%B %d, %Y",
"date_short": "%m/%d/%Y",
"datetime": "%B %d, %Y %I:%M %p",
"thousands_sep": ",",
"decimal_sep": ".",
},
"de": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"fr": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u202f",
"decimal_sep": ",",
},
"es": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"it": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"pt": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"nl": {
"date": "%d %B %Y",
"date_short": "%d-%m-%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"nb": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"da": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"sv": {
"date": "%d %B %Y",
"date_short": "%Y-%m-%d",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"fi": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"is": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ga": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"lb": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ca": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"pl": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"cs": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"sk": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"hu": {
"date": "%Y. %B %d.",
"date_short": "%Y.%m.%d.",
"datetime": "%Y. %B %d. %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"sl": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"hr": {
"date": "%d. %B %Y.",
"date_short": "%d.%m.%Y.",
"datetime": "%d. %B %Y. %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ro": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"bg": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"el": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"et": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"lv": {
"date": "%Y. gada %d. %B",
"date_short": "%d.%m.%Y.",
"datetime": "%Y. gada %d. %B %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"lt": {
"date": "%Y m. %B %d d.",
"date_short": "%Y-%m-%d",
"datetime": "%Y m. %B %d d. %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"tr": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"uk": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"zh": {
"date": "%Y年%m月%d日",
"date_short": "%Y/%m/%d",
"datetime": "%Y年%m月%d日 %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"ru": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
# --- New languages ---
"no": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"cy": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"fy": {
"date": "%d %B %Y",
"date_short": "%d-%m-%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"gl": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"li": {
"date": "%d %B %Y",
"date_short": "%d-%m-%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"vls": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"nds": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"sr": {
"date": "%d. %B %Y.",
"date_short": "%d.%m.%Y.",
"datetime": "%d. %B %Y. %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"he": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"ar": {
"date": "%d %B %Y",
"date_short": "%Y/%m/%d",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"fa": {
"date": "%d %B %Y",
"date_short": "%Y/%m/%d",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"af": {
"date": "%d %B %Y",
"date_short": "%Y/%m/%d",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"ja": {
"date": "%Y年%m月%d日",
"date_short": "%Y/%m/%d",
"datetime": "%Y年%m月%d日 %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"ko": {
"date": "%Y년 %m월 %d일",
"date_short": "%Y.%m.%d",
"datetime": "%Y년 %m월 %d일 %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"vi": {
"date": "ngày %d tháng %m năm %Y",
"date_short": "%d/%m/%Y",
"datetime": "ngày %d tháng %m năm %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"pa": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"kn": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"eo": {
"date": "%d-a de %B %Y",
"date_short": "%Y-%m-%d",
"datetime": "%d-a de %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
}
def format_date(value: date | datetime | None, locale: str = DEFAULT_LANGUAGE, short: bool = False) -> str:
"""Format a date/datetime value according to the locale conventions."""
if value is None:
return ""
fmt_key = "date_short" if short else "date"
fmt = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])[fmt_key]
return value.strftime(fmt)
def format_datetime(value: datetime | None, locale: str = DEFAULT_LANGUAGE) -> str:
"""Format a datetime value according to the locale conventions."""
if value is None:
return ""
fmt = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])["datetime"]
return value.strftime(fmt)
def format_number(value: int | float, locale: str = DEFAULT_LANGUAGE) -> str:
"""Format a number with locale-appropriate thousand separators."""
lf = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])
if isinstance(value, float):
int_part, _, dec_part = f"{value:,.2f}".partition(".")
formatted_int = int_part.replace(",", lf["thousands_sep"])
return f"{formatted_int}{lf['decimal_sep']}{dec_part}"
return f"{value:,}".replace(",", lf["thousands_sep"])
@lru_cache(maxsize=32)
def get_language_info(code: str) -> dict[str, str] | None:
"""Return the metadata dict for a supported language code, or ``None``."""
for lang in SUPPORTED_LANGUAGES:
if lang["code"] == code:
return lang
return None
app/utils/network.py
+34
View File
@@ -0,0 +1,34 @@
import ipaddress
import logging
import socket
logger = logging.getLogger(__name__)
def is_private_ip(hostname: str) -> bool:
"""
Check if a hostname resolves to a private/internal IP address.
Protects against SSRF attacks by blocking access to internal networks.
"""
try:
# Try to parse as IP address directly
ip = ipaddress.ip_address(hostname)
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
except ValueError:
# Not a direct IP, try to resolve hostname
try:
# Get all IP addresses for this hostname
addr_info = socket.getaddrinfo(hostname, None)
for info in addr_info:
ip_str = info[4][0]
ip = ipaddress.ip_address(ip_str)
# Block if ANY resolved IP is private/internal
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
return True
return False
except (socket.gaierror, socket.error):
# Cannot resolve - allow for testing/development
# In production, DNS should work properly
# Log this for debugging
logger.warning(f"Could not resolve hostname: {hostname}")
return False # Changed from True to False to allow external domains in tests
app/utils/push_notification.py
+130
View File
@@ -0,0 +1,130 @@
"""Push notification sender for the DocuElevate mobile app.
Uses the **Expo Push Notification** service to deliver notifications to both
iOS (via APNs) and Android (via FCM) without requiring server-side APNs keys
or FCM credentials. The mobile app obtains an ``ExponentPushToken[…]`` at
startup and registers it with the backend via the mobile API.
Reference: https://docs.expo.dev/push-notifications/sending-notifications/
"""
import logging
from typing import Any
import httpx
from app.database import SessionLocal
from app.models import MobileDevice
logger = logging.getLogger(__name__)
EXPO_PUSH_URL = "https://exp.host/--/api/v2/push/send"
# Maximum tokens per batch request (Expo limit).
_EXPO_BATCH_LIMIT = 100
def send_expo_push_notification(
tokens: list[str],
title: str,
body: str,
data: dict[str, Any] | None = None,
sound: str = "default",
badge: int | None = None,
) -> list[dict[str, Any]]:
"""Send a push notification to one or more Expo push tokens.
Args:
tokens: List of Expo push tokens (``ExponentPushToken[…]``).
title: Notification title shown in the system tray.
body: Notification body text.
data: Optional JSON-serialisable dict attached to the notification
(available in the app via ``notification.request.content.data``).
sound: Notification sound. Use ``"default"`` or ``None`` for silent.
badge: iOS badge count. Pass ``0`` to clear.
Returns:
List of Expo push receipt dicts (one per token).
"""
if not tokens:
return []
results: list[dict[str, Any]] = []
# Send in batches to stay within Expo's per-request limit.
for i in range(0, len(tokens), _EXPO_BATCH_LIMIT):
batch = tokens[i : i + _EXPO_BATCH_LIMIT]
messages = []
for token in batch:
msg: dict[str, Any] = {
"to": token,
"title": title,
"body": body,
"sound": sound,
}
if data:
msg["data"] = data
if badge is not None:
msg["badge"] = badge
messages.append(msg)
try:
resp = httpx.post(
EXPO_PUSH_URL,
json=messages,
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate",
"Content-Type": "application/json",
},
timeout=15,
)
resp.raise_for_status()
payload = resp.json()
batch_results = payload.get("data", [])
results.extend(batch_results)
logger.debug("Expo push batch sent: %d tokens, %d results", len(batch), len(batch_results))
except httpx.HTTPStatusError as exc:
logger.error("Expo push HTTP error: %s – %s", exc.response.status_code, exc.response.text)
except Exception:
logger.exception("Expo push notification failed for batch starting at index %d", i)
return results
def send_push_to_owner(
owner_id: str,
title: str,
body: str,
data: dict[str, Any] | None = None,
) -> None:
"""Look up all active push tokens for *owner_id* and send them a notification.
This function is safe to call from Celery task workers. Database errors
and push failures are logged but never raised so that the caller task is
not retried due to a notification failure.
"""
db = SessionLocal()
try:
devices = (
db.query(MobileDevice)
.filter(
MobileDevice.owner_id == owner_id,
MobileDevice.is_active.is_(True),
MobileDevice.push_token.isnot(None),
)
.all()
)
tokens = [d.push_token for d in devices if d.push_token]
except Exception:
logger.exception("Failed to query mobile devices for owner_id=%s", owner_id)
return
finally:
db.close()
if not tokens:
logger.debug("No active push tokens for owner_id=%s", owner_id)
return
logger.info("Sending push notification to %d device(s) for owner_id=%s", len(tokens), owner_id)
send_expo_push_notification(tokens=tokens, title=title, body=body, data=data)
app/utils/routing_engine.py
+223
View File
@@ -0,0 +1,223 @@
"""Routing engine for conditional document-to-pipeline assignment.
Evaluates a set of :class:`PipelineRoutingRule` rows against document
properties and returns the first matching target pipeline (if any).
Supported document fields
-------------------------
* ``file_type`` – MIME type of the file (e.g. ``application/pdf``)
* ``filename`` – original filename
* ``size`` – file size in bytes (numeric comparison)
* ``document_type`` – AI-classified document type (e.g. ``Invoice``)
* ``category`` – alias for ``document_type``
* ``metadata.<key>`` – arbitrary key inside the AI-extracted JSON metadata
Supported comparison operators
------------------------------
* ``equals`` / ``not_equals``
* ``contains`` / ``not_contains`` (substring match, case-insensitive)
* ``regex`` (Python ``re`` full-match, case-insensitive)
* ``gt`` / ``lt`` / ``gte`` / ``lte`` (numeric comparison)
"""
import json
import logging
import re
from typing import Any
from sqlalchemy.orm import Session
from app.models import Pipeline, PipelineRoutingRule
logger = logging.getLogger(__name__)
# Operators recognised by the engine.
VALID_OPERATORS: frozenset[str] = frozenset(
{
"equals",
"not_equals",
"contains",
"not_contains",
"regex",
"gt",
"lt",
"gte",
"lte",
}
)
# Fields that are resolved directly from the FileRecord.
BUILTIN_FIELDS: frozenset[str] = frozenset(
{
"file_type",
"filename",
"size",
"document_type",
"category",
}
)
def _resolve_field(field: str, doc_props: dict[str, Any]) -> Any:
"""Resolve a *field* name to its actual value from *doc_props*.
``doc_props`` is expected to contain top-level keys that mirror the
built-in field names **plus** a ``metadata`` dict with the parsed
AI metadata JSON.
"""
if field == "category":
# ``category`` is an alias for ``document_type``.
field = "document_type"
if field.startswith("metadata."):
meta_key = field[len("metadata.") :]
metadata = doc_props.get("metadata") or {}
return metadata.get(meta_key)
return doc_props.get(field)
def _to_float(value: Any) -> float | None:
"""Try to convert *value* to a float for numeric comparison."""
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _evaluate_condition(actual: Any, operator: str, expected: str) -> bool:
"""Return ``True`` when *actual* satisfies *operator* against *expected*.
All string comparisons are case-insensitive. Numeric operators (``gt``,
``lt``, ``gte``, ``lte``) attempt to cast both sides to ``float``.
"""
if actual is None:
# If the document property is missing, the rule cannot match
# (except for ``not_equals`` / ``not_contains`` which should match).
if operator == "not_equals":
return True
if operator == "not_contains":
return True
return False
actual_str = str(actual).lower()
expected_lower = expected.lower()
if operator == "equals":
return actual_str == expected_lower
if operator == "not_equals":
return actual_str != expected_lower
if operator == "contains":
return expected_lower in actual_str
if operator == "not_contains":
return expected_lower not in actual_str
if operator == "regex":
try:
return bool(re.fullmatch(expected, str(actual), flags=re.IGNORECASE))
except re.error:
logger.warning("Invalid regex in routing rule: %s", expected)
return False
# Numeric operators
actual_num = _to_float(actual)
expected_num = _to_float(expected)
if actual_num is None or expected_num is None:
return False
if operator == "gt":
return actual_num > expected_num
if operator == "lt":
return actual_num < expected_num
if operator == "gte":
return actual_num >= expected_num
if operator == "lte":
return actual_num <= expected_num
return False
def build_document_properties(file_record: Any) -> dict[str, Any]:
"""Build the property dict that the engine evaluates against.
Args:
file_record: A :class:`FileRecord` instance (or any object with the
same attributes).
Returns:
A dict with ``file_type``, ``filename``, ``size``, ``document_type``,
and ``metadata`` keys.
"""
metadata: dict[str, Any] = {}
raw_meta = getattr(file_record, "ai_metadata", None)
if raw_meta:
try:
metadata = json.loads(raw_meta) if isinstance(raw_meta, str) else raw_meta
except (json.JSONDecodeError, TypeError):
metadata = {}
return {
"file_type": getattr(file_record, "mime_type", None),
"filename": getattr(file_record, "original_filename", None),
"size": getattr(file_record, "file_size", None),
"document_type": metadata.get("document_type"),
"metadata": metadata,
}
def evaluate_routing_rules(
db: Session,
owner_id: str | None,
doc_props: dict[str, Any],
) -> Pipeline | None:
"""Evaluate routing rules and return the first matching pipeline.
Rules are fetched for the given *owner_id* **plus** any system-wide rules
(``owner_id IS NULL``). Owner rules are evaluated first (by position),
then system rules.
Args:
db: Active database session.
owner_id: The document owner's identifier (may be ``None``).
doc_props: Document property dict as produced by
:func:`build_document_properties`.
Returns:
The first matching :class:`Pipeline`, or ``None`` when no rule
matches (caller should fall back to the default pipeline).
"""
# Fetch active rules for the owner + system rules, ordered by position.
rules = (
db.query(PipelineRoutingRule)
.filter(
PipelineRoutingRule.is_active.is_(True),
(PipelineRoutingRule.owner_id == owner_id) | (PipelineRoutingRule.owner_id.is_(None)),
)
.order_by(
# Owner-specific rules take priority over system rules.
PipelineRoutingRule.owner_id.is_(None).asc(),
PipelineRoutingRule.position.asc(),
)
.all()
)
for rule in rules:
actual = _resolve_field(rule.field, doc_props)
if _evaluate_condition(actual, rule.operator, rule.value):
pipeline = db.query(Pipeline).filter(Pipeline.id == rule.target_pipeline_id).first()
if pipeline and pipeline.is_active:
logger.info(
"Routing rule matched: rule_id=%s, name=%s, target_pipeline=%s",
rule.id,
rule.name,
rule.target_pipeline_id,
)
return pipeline
logger.warning(
"Routing rule %s matched but target pipeline %s is inactive or missing",
rule.id,
rule.target_pipeline_id,
)
return None
app/utils/settings_service.py
+434
View File
@@ -182,6 +182,153 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
# Social Login Providers
"social_auth_google_enabled": {
"category": "Social Login",
"description": (
"Enable Google Sign-In. Requires SOCIAL_AUTH_GOOGLE_CLIENT_ID and "
"SOCIAL_AUTH_GOOGLE_CLIENT_SECRET from the Google Cloud Console."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://console.cloud.google.com/apis/credentials",
"help_link_label": "Google Cloud Console",
},
"social_auth_google_client_id": {
"category": "Social Login",
"description": "Google OAuth2 client ID from the Google Cloud Console.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_google_client_secret": {
"category": "Social Login",
"description": "Google OAuth2 client secret from the Google Cloud Console.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_enabled": {
"category": "Social Login",
"description": (
"Enable Microsoft Sign-In (Azure AD / Microsoft Entra ID). Requires "
"SOCIAL_AUTH_MICROSOFT_CLIENT_ID and SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET "
"from Azure App Registrations."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade",
"help_link_label": "Azure Portal",
},
"social_auth_microsoft_client_id": {
"category": "Social Login",
"description": "Microsoft OAuth2 application (client) ID from Azure App Registrations.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_client_secret": {
"category": "Social Login",
"description": "Microsoft OAuth2 client secret from Azure App Registrations.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_tenant": {
"category": "Social Login",
"description": (
"Azure AD tenant ID or one of 'common', 'organizations', 'consumers'. "
"Use 'common' to allow any Microsoft account. Use a specific GUID to "
"restrict to a single organization."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_enabled": {
"category": "Social Login",
"description": (
"Enable Sign in with Apple. Requires an Apple Developer account with "
"a Services ID configured for Sign in with Apple."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://developer.apple.com/account/resources/identifiers/list/serviceId",
"help_link_label": "Apple Developer Portal",
},
"social_auth_apple_client_id": {
"category": "Social Login",
"description": "Apple Services ID (e.g. com.example.docuelevate).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_team_id": {
"category": "Social Login",
"description": "Apple Developer Team ID (10-character alphanumeric string).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_key_id": {
"category": "Social Login",
"description": "Apple Sign-In private key ID from the Apple Developer Portal.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_private_key": {
"category": "Social Login",
"description": (
"Apple Sign-In private key (PEM format). Generate this in the Apple Developer Portal. "
"Paste the entire key content including BEGIN/END headers."
),
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_enabled": {
"category": "Social Login",
"description": (
"Enable Dropbox Sign-In. Uses the same Dropbox App you may already have "
"configured for storage, or a separate one."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_client_id": {
"category": "Social Login",
"description": "Dropbox OAuth2 App Key from the Dropbox App Console.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_client_secret": {
"category": "Social Login",
"description": "Dropbox OAuth2 App Secret from the Dropbox App Console.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
# AI Services
"openai_api_key": {
"category": "AI Services",
@@ -375,6 +522,19 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Document Translation
"default_document_language": {
"category": "AI Services",
"description": (
"ISO 639-1 language code for the default document translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language differs "
"are automatically translated into this language after processing."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# OCR Engine Configuration
"ocr_providers": {
"category": "OCR Engines",
@@ -495,6 +655,14 @@ SETTING_METADATA = {
"options": ["us", "eu"],
},
# Storage Providers - Dropbox
"dropbox_enabled": {
"category": "Storage Providers",
"description": "Enable Dropbox as an upload destination. When disabled, no documents will be sent to Dropbox even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"dropbox_app_key": {
"category": "Storage Providers",
"description": "Dropbox app key for OAuth authentication",
@@ -528,6 +696,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_enabled": {
"category": "Storage Providers",
"description": "Enable Nextcloud as an upload destination. When disabled, no documents will be sent to Nextcloud even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"nextcloud_upload_url": {
"category": "Storage Providers",
"description": "Nextcloud WebDAV upload URL",
@@ -561,6 +737,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Paperless-ngx
"paperless_enabled": {
"category": "Storage Providers",
"description": "Enable Paperless-ngx as an upload destination. When disabled, no documents will be sent to Paperless-ngx even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"paperless_ngx_api_token": {
"category": "Storage Providers",
"description": "Paperless-ngx API authentication token",
@@ -578,6 +762,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Google Drive
"google_drive_enabled": {
"category": "Storage Providers",
"description": "Enable Google Drive as an upload destination. When disabled, no documents will be sent to Google Drive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"google_drive_credentials_json": {
"category": "Storage Providers",
"description": "Google Drive service account credentials JSON",
@@ -635,6 +827,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - OneDrive
"onedrive_enabled": {
"category": "Storage Providers",
"description": "Enable OneDrive as an upload destination. When disabled, no documents will be sent to OneDrive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"onedrive_client_id": {
"category": "Storage Providers",
"description": "OneDrive OAuth client ID",
@@ -676,6 +876,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_enabled": {
"category": "Storage Providers",
"description": "Enable WebDAV as an upload destination. When disabled, no documents will be sent to WebDAV even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"webdav_url": {
"category": "Storage Providers",
"description": "WebDAV server URL",
@@ -717,6 +925,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - FTP
"ftp_enabled": {
"category": "Storage Providers",
"description": "Enable FTP as an upload destination. When disabled, no documents will be sent to FTP even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"ftp_host": {
"category": "Storage Providers",
"description": "FTP server hostname or IP address",
@@ -774,6 +990,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - SFTP
"sftp_enabled": {
"category": "Storage Providers",
"description": "Enable SFTP as an upload destination. When disabled, no documents will be sent to SFTP even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sftp_host": {
"category": "Storage Providers",
"description": "SFTP server hostname or IP address",
@@ -838,7 +1062,56 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - iCloud Drive
"icloud_enabled": {
"category": "Storage Providers",
"description": "Enable iCloud Drive as an upload destination. When disabled, no documents will be sent to iCloud Drive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_username": {
"category": "Storage Providers",
"description": "Apple ID email address for iCloud Drive authentication",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_password": {
"category": "Storage Providers",
"description": "App-specific password for iCloud Drive (generate at https://appleid.apple.com)",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"icloud_folder": {
"category": "Storage Providers",
"description": "Target folder path in iCloud Drive (e.g. Documents/Uploads)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_cookie_directory": {
"category": "Storage Providers",
"description": "Directory for persisting iCloud session cookies (default: ~/.pyicloud)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - AWS S3
"s3_enabled": {
"category": "Storage Providers",
"description": "Enable Amazon S3 as an upload destination. When disabled, no documents will be sent to S3 even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"aws_access_key_id": {
"category": "Storage Providers",
"description": "AWS access key ID for S3",
@@ -972,6 +1245,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Email Destination Settings (dedicated SMTP for document delivery)
"dest_email_enabled": {
"category": "Email Destination",
"description": "Enable Email as an upload destination. When disabled, no documents will be delivered via email even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"dest_email_host": {
"category": "Email Destination",
"description": "SMTP server hostname for document delivery (separate from shared email settings)",
@@ -1392,6 +1673,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"imap_attachment_filter": {
"category": "IMAP",
"description": (
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
"Per-user IMAP accounts can override this global default."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
@@ -1607,6 +1900,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"compliance_enabled": {
"category": "Feature Flags",
"description": (
"Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). "
"When enabled, admins can view compliance status and apply "
"pre-built regulatory configurations. Default: True."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Backup / Restore
"backup_enabled": {
"category": "Backup",
@@ -2077,6 +2382,78 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"audit_siem_enabled": {
"category": "Security",
"description": "Enable forwarding of audit events to an external SIEM system (Syslog, Splunk, Logstash, etc.).",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_transport": {
"category": "Security",
"description": (
"Transport used to forward audit events. 'syslog' sends RFC 5424 messages over UDP/TCP. "
"'http' sends JSON POST payloads to a webhook URL (Splunk HEC, Logstash, Grafana Loki, etc.)."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["syslog", "http"],
},
"audit_siem_syslog_host": {
"category": "Security",
"description": "Hostname or IP of the syslog receiver.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_syslog_port": {
"category": "Security",
"description": "Port of the syslog receiver. Default: 514.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_syslog_protocol": {
"category": "Security",
"description": "Protocol for syslog transport: 'udp' or 'tcp'. Default: udp.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["udp", "tcp"],
},
"audit_siem_http_url": {
"category": "Security",
"description": (
"HTTP endpoint URL for SIEM webhook delivery. Supports Splunk HEC, "
"Logstash HTTP input, Grafana Loki push API, or any JSON-accepting endpoint."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_http_token": {
"category": "Security",
"description": "Bearer / HEC token included in the Authorization header of SIEM HTTP requests.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"audit_siem_http_custom_headers": {
"category": "Security",
"description": "Comma-separated 'Key:Value' pairs of extra headers for SIEM HTTP requests.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Rate Limiting
"rate_limiting_enabled": {
"category": "Security",
@@ -2276,6 +2653,63 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Logging
"log_level": {
"category": "Observability",
"description": (
"Python logging level for the application root logger. "
"Accepts: DEBUG, INFO, WARNING, ERROR, CRITICAL. "
"When DEBUG=True and LOG_LEVEL is not explicitly set, "
"the effective level is automatically lowered to DEBUG."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_format": {
"category": "Observability",
"description": (
"Log output format: 'text' (human-readable, default) or "
"'json' (structured JSON lines for SIEM / log aggregation)."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_enabled": {
"category": "Observability",
"description": "Forward application logs to a syslog receiver in addition to stdout.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_host": {
"category": "Observability",
"description": "Hostname or IP of the syslog receiver for application logs.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_port": {
"category": "Observability",
"description": "Port of the syslog receiver for application logs.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_protocol": {
"category": "Observability",
"description": "Protocol for syslog transport: 'udp' or 'tcp'.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
# Observability – Sentry
"sentry_dsn": {
"category": "Observability",
app/utils/user_notification.py
+13
View File
@@ -210,6 +210,19 @@ def dispatch_user_notification(
finally:
db.close()
# 3. Send push notifications to registered mobile devices
try:
from app.utils.push_notification import send_push_to_owner
send_push_to_owner(
owner_id=owner_id,
title=title,
body=message,
data={"event_type": event_type, "file_id": file_id},
)
except Exception:
logger.exception("Error sending push notification for owner_id=%s event=%s", owner_id, event_type)
def notify_user_document_processed(owner_id: str, filename: str, file_id: int | None = None) -> None:
"""Notify a user that their document was successfully processed."""
app/utils/user_scope.py
+56 -8
View File
@@ -20,13 +20,31 @@ from app.models import FileRecord
logger = logging.getLogger(__name__)
def _owner_id_from_user(user: dict) -> str | None:
"""Extract the owner identifier from a user dict.
Priority: ``sub`` (OAuth subject) → ``preferred_username`` → ``email`` → ``id``.
"""
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
def get_current_owner_id(request: Request) -> str | None:
"""Extract the owner identifier for the current authenticated user.
The owner ID is derived from the user's session data. It uses the
``sub`` claim (OAuth subject) when available, falling back to
``preferred_username`` or ``email``. Returns ``None`` when no user
is authenticated.
The owner ID is derived from the user's session data or, when no session
is present, from a valid Bearer API token in the ``Authorization`` header.
This ensures that both browser-based (session cookie) and mobile/API
(Bearer token) requests are correctly identified.
Priority for user resolution:
1. Session ``user`` dict (set by OAuth or local login).
2. ``request.state.api_token_user`` (set by ``require_login`` or an
earlier call to this function during the same request).
3. Direct Bearer token look-up against the database.
Within the resolved user dict the owner ID is chosen as:
``sub`` → ``preferred_username`` → ``email`` → ``id``.
Args:
request: The current FastAPI request with session data.
@@ -34,11 +52,41 @@ def get_current_owner_id(request: Request) -> str | None:
Returns:
A stable string identifier for the user, or ``None``.
"""
# 1. Session-based auth (most common for web UI)
user = request.session.get("user")
if not user or not isinstance(user, dict):
return None
# Prefer 'sub' (OAuth subject), then 'preferred_username', then 'email', then 'id'
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
if user and isinstance(user, dict):
return _owner_id_from_user(user)
# 2. Already-resolved API token user (cached by require_login or a
# prior dependency call during this request)
api_user = getattr(request.state, "api_token_user", None)
if isinstance(api_user, dict):
return _owner_id_from_user(api_user)
# 3. Direct Bearer token resolution – necessary when this function is
# invoked as a FastAPI dependency (via Depends) which runs *before*
# the @require_login decorator wrapper has had a chance to resolve
# the token and populate request.state.api_token_user.
auth_header = request.headers.get("authorization", "")
if isinstance(auth_header, str) and auth_header.startswith("Bearer "):
try:
from app.auth import _resolve_bearer_user
from app.database import SessionLocal
db = SessionLocal()
try:
resolved = _resolve_bearer_user(request, db)
finally:
db.close()
if resolved:
# Cache so subsequent calls (and require_login) skip the DB
request.state.api_token_user = resolved
return _owner_id_from_user(resolved)
except Exception:
logger.debug("Bearer token resolution failed in get_current_owner_id", exc_info=True)
return None
def apply_owner_filter(query: Query, request: Request) -> Query:
app/views/__init__.py
+6
View File
@@ -6,7 +6,9 @@ from fastapi import APIRouter
from app.views.admin_users import router as admin_users_router
from app.views.api_tokens import router as api_tokens_router
from app.views.audit_logs import router as audit_logs_router
from app.views.backup import router as backup_router
from app.views.compliance import router as compliance_router
from app.views.db_wizard import router as db_wizard_router
from app.views.dropbox import router as dropbox_router
from app.views.filemanager import router as filemanager_router
@@ -23,6 +25,7 @@ from app.views.onboarding import router as onboarding_router
from app.views.onedrive import router as onedrive_router
from app.views.pipelines import router as pipelines_router # Processing pipelines
from app.views.plans import router as plans_router # Admin Plan Designer
from app.views.profile import router as profile_router # User self-service profile
from app.views.queue import router as queue_router
from app.views.scheduled_jobs import router as scheduled_jobs_router # Scheduled batch jobs
from app.views.search import router as search_router
@@ -56,8 +59,11 @@ router.include_router(subscriptions_router) # Pricing + subscription pages
router.include_router(plans_router) # Admin Plan Designer
router.include_router(onboarding_router) # User onboarding wizard
router.include_router(pipelines_router) # Processing pipelines
router.include_router(profile_router) # User self-service profile settings
router.include_router(imap_accounts_router) # Per-user IMAP ingestion accounts
router.include_router(integrations_router) # Unified integrations dashboard
router.include_router(notifications_router) # User notification dashboard
router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs
router.include_router(audit_logs_router) # Comprehensive audit log viewer
router.include_router(help_router) # Built-in help / How-To docs
router.include_router(compliance_router) # Compliance templates dashboard
app/views/audit_logs.py
+46
View File
@@ -0,0 +1,46 @@
"""
Audit log viewer UI — admin-only page with filtering and SIEM status.
"""
import logging
from fastapi import Depends, HTTPException, Request, status
from sqlalchemy.orm import Session
from app.views.base import APIRouter, get_db, require_login, settings, templates
from app.views.settings import require_admin_access
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/admin/audit-logs")
@require_login
@require_admin_access
async def audit_logs_page(request: Request, db: Session = Depends(get_db)):
"""Comprehensive audit log viewer with filtering controls.
Displays a chronological log of all significant actions: logins,
document operations, settings changes, and admin actions. The
actual data is fetched client-side via the ``/api/audit-logs`` JSON
endpoint so that filters, pagination, and live refresh work without
full-page reloads.
"""
try:
siem_enabled = settings.audit_siem_enabled
siem_transport = settings.audit_siem_transport if siem_enabled else None
return templates.TemplateResponse(
"audit_logs.html",
{
"request": request,
"app_version": settings.version,
"siem_enabled": siem_enabled,
"siem_transport": siem_transport,
},
)
except Exception as e:
logger.error("Error loading audit logs page: %s", e)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to load audit logs page",
)
app/views/base.py
+94 -1
View File
@@ -11,7 +11,17 @@ from sqlalchemy.orm import Session # noqa: F401
from app.auth import require_login # noqa: F401
from app.config import settings
from app.database import get_db # noqa: F401
from app.database import SessionLocal, get_db # noqa: F401
from app.models import UserProfile
from app.utils.i18n import (
SUPPORTED_LANGUAGES,
detect_language,
format_date,
format_datetime,
format_number,
get_suggested_languages,
translate,
)
# Set up Jinja2 templates
templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
@@ -21,10 +31,58 @@ templates = Jinja2Templates(directory=str(templates_dir))
templates.env.globals["min"] = min
templates.env.globals["max"] = max
# ---------------------------------------------------------------------------
# i18n Jinja2 integration
# ---------------------------------------------------------------------------
# The _() function is available in every template to translate UI strings.
# Usage: {{ _("nav.dashboard") }} or {{ _("upload.max_size", size="10 MB") }}
# The locale is automatically resolved from the request context.
# A default English implementation is registered as a global so error handlers
# that don't go through _inject_global_context still have the function available.
# ---------------------------------------------------------------------------
templates.env.globals["supported_languages"] = SUPPORTED_LANGUAGES
templates.env.globals["_"] = lambda key, **kwargs: translate(key, "en", **kwargs)
# Customize Jinja2Templates to include app_version in all templates
original_template_response = templates.TemplateResponse
def _hydrate_language_from_db(request: Request, session_user: object) -> None:
"""Load the user's preferred language from the DB into the session.
Called once per session when ``preferred_language`` is not yet in the
session. A lightweight DB query fetches the stored preference so that
:func:`detect_language` picks it up from the session on all subsequent
requests without further DB access.
"""
from app.utils.i18n import SUPPORTED_LANGUAGE_CODES
user_id: str | None = None
if isinstance(session_user, dict):
user_id = (
session_user.get("sub")
or session_user.get("preferred_username")
or session_user.get("email")
or session_user.get("id")
)
elif isinstance(session_user, str):
user_id = session_user
if not user_id:
return
db = SessionLocal()
try:
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
if profile and profile.preferred_language and profile.preferred_language in SUPPORTED_LANGUAGE_CODES:
request.session["preferred_language"] = profile.preferred_language
except Exception: # noqa: BLE001 — intentionally broad; DB may be temporarily unavailable
logger.debug("Could not hydrate language preference for user_id=%s", user_id)
finally:
db.close()
def _inject_global_context(ctx: dict) -> None:
"""Inject shared global variables into every template context dict."""
ctx.setdefault("version", settings.version)
@@ -48,8 +106,43 @@ def _inject_global_context(ctx: dict) -> None:
session_user = req.session.get("user")
# When auth is disabled every visitor is effectively "logged in"
ctx.setdefault("is_logged_in", not getattr(settings, "auth_enabled", True) or session_user is not None)
# --- Hydrate session language from DB (once per session) ---
# If the session doesn't have a preferred_language yet but the user
# is logged in, load the stored preference from the database so that
# detect_language() picks it up from the session on this and all
# subsequent requests.
if hasattr(req, "session") and "preferred_language" not in req.session and session_user is not None:
_hydrate_language_from_db(req, session_user)
# --- i18n: detect language and register template helpers ---
current_locale = detect_language(req)
ctx.setdefault("current_locale", current_locale)
# Smart language suggestions for the compact nav-bar dropdown (5-7 languages)
accept_header = req.headers.get("accept-language", "") if hasattr(req, "headers") else ""
ctx.setdefault("suggested_languages", get_suggested_languages(current_locale, accept_header))
def _translate(key: str, **kwargs: object) -> str:
return translate(key, current_locale, **kwargs)
def _format_date(value: object, short: bool = False) -> str:
return format_date(value, current_locale, short=short) # type: ignore[arg-type]
def _format_datetime(value: object) -> str:
return format_datetime(value, current_locale) # type: ignore[arg-type]
def _format_number(value: object) -> str:
return format_number(value, current_locale) # type: ignore[arg-type]
ctx.setdefault("_", _translate)
ctx.setdefault("format_date_l10n", _format_date)
ctx.setdefault("format_datetime_l10n", _format_datetime)
ctx.setdefault("format_number_l10n", _format_number)
else:
ctx.setdefault("is_logged_in", not getattr(settings, "auth_enabled", True))
ctx.setdefault("current_locale", "en")
ctx.setdefault("_", lambda key, **kw: translate(key, "en", **kw))
def template_response_with_version(*args, **kwargs):
app/views/compliance.py
+48
View File
@@ -0,0 +1,48 @@
"""Admin view: compliance templates dashboard page."""
import logging
from fastapi import HTTPException, Request, status
from fastapi.responses import RedirectResponse
from app.views.base import APIRouter, require_login, settings, templates
logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request):
"""Return the session user if they are an admin, else None."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
logger.warning("Non-admin user attempted to access /admin/compliance")
return None
return user
@router.get("/admin/compliance")
@require_login
async def compliance_page(request: Request):
"""Admin compliance templates dashboard page.
Displays GDPR, HIPAA, and SOC2 compliance templates with their current
status and one-click apply functionality.
"""
user = _require_admin(request)
if user is None:
return RedirectResponse(url="/", status_code=status.HTTP_302_FOUND)
try:
return templates.TemplateResponse(
"compliance.html",
{
"request": request,
"app_version": settings.version,
},
)
except Exception as e:
logger.error(f"Error loading compliance page: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to load compliance page",
)
app/views/files.py
+28
View File
@@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
)
@router.get("/files/{file_id}/text/default-language")
@require_login
def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)):
"""Return the persisted default-language translation for the file view."""
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND)
if not file_record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available",
)
return JSONResponse(
content={
"text": file_record.default_language_text,
"language_code": file_record.default_language_code,
"detected_language": file_record.detected_language,
}
)
@router.get("/duplicates")
@require_login
def duplicates_page(
app/views/imap_accounts.py
+43 -1
View File
@@ -1,11 +1,13 @@
"""User-facing view for the per-user IMAP ingestion dashboard."""
import json
import logging
from fastapi import Request
from sqlalchemy.orm import Session
from app.models import UserImapAccount
from app.models import ImapIngestionProfile, UserImapAccount
from app.utils.allowed_types import DEFAULT_CATEGORIES, FILE_TYPE_CATEGORIES
from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id
from app.views.base import APIRouter, Depends, get_db, require_login, templates
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
return max_mb
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
"""Serialize a profile for JSON embedding in the template."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"is_builtin": profile.is_builtin,
}
@router.get("/imap-accounts")
@require_login
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
@@ -49,11 +67,35 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
max_mailboxes = _get_max_mailboxes(tier)
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
# Load ingestion profiles: system-global + user's own
profiles = (
db.query(ImapIngestionProfile)
.filter(
# SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
)
# Category definitions for the UI checkbox builder
categories = [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
return templates.TemplateResponse(
"imap_accounts.html",
{
"request": request,
"accounts": accounts,
"profiles": [_serialize_profile(p) for p in profiles],
"categories": categories,
"default_categories": DEFAULT_CATEGORIES,
"current_count": current_count,
"max_mailboxes": max_mailboxes,
"can_add": can_add,
app/views/onboarding.py
+2
View File
@@ -27,6 +27,7 @@ _DESTINATION_META: list[dict] = [
{"id": "webdav", "name": "WebDAV", "icon": "fas fa-server"},
{"id": "sftp", "name": "SFTP", "icon": "fas fa-terminal"},
{"id": "ftp", "name": "FTP", "icon": "fas fa-server"},
{"id": "icloud", "name": "iCloud Drive", "icon": "fab fa-apple"},
]
@@ -51,6 +52,7 @@ def _get_configured_destinations(cfg: Settings) -> list[dict]:
"webdav": bool(cfg.webdav_url and cfg.webdav_username),
"sftp": bool(cfg.sftp_host and cfg.sftp_username),
"ftp": bool(cfg.ftp_host and cfg.ftp_username),
"icloud": bool(cfg.icloud_username and cfg.icloud_password),
}
return [meta for meta in _DESTINATION_META if checks.get(meta["id"], False)]
app/views/plans.py
+1 -2
View File
@@ -3,12 +3,11 @@
from fastapi import Request
from fastapi.responses import HTMLResponse
from fastapi.routing import APIRouter
from fastapi.templating import Jinja2Templates
from app.auth import require_login
from app.views.base import templates
router = APIRouter()
templates = Jinja2Templates(directory="frontend/templates")
@router.get("/admin/plans", response_class=HTMLResponse)
app/views/profile.py
+40
View File
@@ -0,0 +1,40 @@
"""View route for the user self-service profile settings page.
Route:
GET /profile — renders the profile settings HTML page (requires login)
"""
from __future__ import annotations
import logging
from fastapi import Depends, Request
from sqlalchemy.orm import Session
from app.models import UserProfile
from app.utils.i18n import SUPPORTED_LANGUAGES
from app.views.base import APIRouter, get_db, require_login, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/profile", include_in_schema=False)
@require_login
async def profile_page(request: Request, db: Session = Depends(get_db)):
"""Serve the user profile settings page."""
user = request.session.get("user") or {}
user_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
profile = None
if user_id:
profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
return templates.TemplateResponse(
"profile.html",
{
"request": request,
"profile": profile,
"supported_languages": SUPPORTED_LANGUAGES,
},
)
benchmark_duplicates.py
+101
View File
@@ -0,0 +1,101 @@
import time
import os
import sys
import asyncio
# Mock settings before app imports to bypass validation
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
os.environ["REDIS_URL"] = "redis://localhost:6379"
os.environ["OPENAI_API_KEY"] = "mock_key"
os.environ["WORKDIR"] = "/tmp/workdir"
os.environ["AZURE_AI_KEY"] = "mock"
os.environ["AZURE_REGION"] = "mock"
os.environ["AZURE_ENDPOINT"] = "http://mock"
os.environ["GOTENBERG_URL"] = "http://mock"
os.environ["SESSION_SECRET"] = "mock_secret_mock_secret_mock_secret_mock_secret"
# Ensure app package is accessible
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.database import Base
from app.models import FileRecord
from app.api.duplicates import list_duplicate_groups
# Mocking Request object
class MockRequest:
def __init__(self):
self.session = {"user": {"username": "testuser"}}
self.state = type('State', (), {'user': {"username": "testuser"}})()
class MockURL:
def include_query_params(self, **kwargs):
return f"http://testserver/api/duplicates?page={kwargs.get('page')}"
url = MockURL()
def setup_db():
engine = create_engine('sqlite:///:memory:')
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
db = Session()
return db
def populate_data(db, num_groups, duplicates_per_group):
for i in range(num_groups):
filehash = f"hash_{i}"
# Original
original = FileRecord(
filehash=filehash,
local_filename=f"orig_{i}.txt",
file_size=100,
is_duplicate=False
)
db.add(original)
# Duplicates
for j in range(duplicates_per_group):
dup = FileRecord(
filehash=filehash,
local_filename=f"dup_{i}_{j}.txt",
file_size=100,
is_duplicate=True
)
db.add(dup)
db.commit()
async def run_benchmark(db):
request = MockRequest()
start_time = time.time()
# Run the function we want to benchmark
result = list_duplicate_groups(request=request, db=db, page=1, per_page=500)
if asyncio.iscoroutine(result):
result = await result
end_time = time.time()
return end_time - start_time, result
async def main():
db = setup_db()
# 500 groups, each with 20 duplicates = 10500 records total
print("Populating data...")
populate_data(db, 500, 20)
print("Data populated. Running baseline benchmark...")
# Warmup
result = list_duplicate_groups(request=MockRequest(), db=db, page=1, per_page=500)
if asyncio.iscoroutine(result):
await result
# Benchmark
total_time = 0
iterations = 10
for _ in range(iterations):
time_taken, _ = await run_benchmark(db)
total_time += time_taken
avg_time = total_time / iterations
print(f"Average time over {iterations} iterations: {avg_time:.4f} seconds")
if __name__ == "__main__":
asyncio.run(main())
benchmark_notifications.py
+50
View File
@@ -0,0 +1,50 @@
import json
import time
import pytest
from app.database import get_db
from app.models import UserNotificationTarget, UserNotificationPreference
from app.main import app
from tests.test_notifications_api import _make_client, _OWNER, _cleanup
import statistics
def run_benchmark(notif_engine, notif_session, client, items_count, iterations=5):
# Setup
target = UserNotificationTarget(
owner_id=_OWNER,
channel_type="webhook",
name="My Webhook",
config=json.dumps({"url": "https://x.com"}),
)
notif_session.add(target)
notif_session.commit()
notif_session.refresh(target)
# Generate big payload
preferences = []
for i in range(items_count):
preferences.append({
"event_type": f"event.type.{i}",
"channel_type": "webhook",
"is_enabled": True,
"target_id": target.id,
})
payload = {"preferences": preferences}
# Warm up
client.put("/api/user-notifications/preferences", json=payload)
times = []
for _ in range(iterations):
# Alter the values a bit so it's a real update
for p in payload["preferences"]:
p["is_enabled"] = not p["is_enabled"]
start = time.time()
resp = client.put("/api/user-notifications/preferences", json=payload)
end = time.time()
assert resp.status_code == 200
times.append(end - start)
return statistics.mean(times)
benchmark_onedrive.py
+69
View File
@@ -0,0 +1,69 @@
import asyncio
import time
import httpx
from unittest.mock import patch, MagicMock, AsyncMock
from app.api.onedrive import test_onedrive_token
from app.config import settings
settings.onedrive_refresh_token = "dummy"
settings.onedrive_client_id = "dummy"
settings.onedrive_client_secret = "dummy"
class DummyRequest:
def __init__(self):
self.session = {"user": "dummy"}
async def run_benchmark(func_name, mock_post, mock_get):
mock_post_resp = MagicMock()
mock_post_resp.status_code = 200
mock_post_resp.json.return_value = {
"access_token": "dummy_access",
"expires_in": 3600
}
mock_post.return_value = mock_post_resp
mock_get_resp = MagicMock()
mock_get_resp.status_code = 200
mock_get_resp.json.return_value = {
"displayName": "Test User",
"userPrincipalName": "test@example.com"
}
mock_get.return_value = mock_get_resp
start_time = time.time()
for _ in range(100):
await test_onedrive_token(DummyRequest())
end_time = time.time()
print(f"{func_name} took {end_time - start_time:.4f} seconds")
async def run_benchmark_async(func_name, mock_post, mock_get):
mock_post_resp = MagicMock()
mock_post_resp.status_code = 200
mock_post_resp.json = MagicMock(return_value={
"access_token": "dummy_access",
"expires_in": 3600
})
mock_post.return_value = mock_post_resp
mock_get_resp = MagicMock()
mock_get_resp.status_code = 200
mock_get_resp.json = MagicMock(return_value={
"displayName": "Test User",
"userPrincipalName": "test@example.com"
})
mock_get.return_value = mock_get_resp
start_time = time.time()
for _ in range(100):
await test_onedrive_token(DummyRequest())
end_time = time.time()
print(f"{func_name} took {end_time - start_time:.4f} seconds")
@patch('app.api.onedrive.requests.get')
@patch('app.api.onedrive.requests.post')
def benchmark_sync(mock_post, mock_get):
asyncio.run(run_benchmark("Sync requests (baseline)", mock_post, mock_get))
if __name__ == "__main__":
benchmark_sync()
benchmark_url_upload.py
+49
View File
@@ -0,0 +1,49 @@
import asyncio
import time
from unittest.mock import Mock, patch
from app.api.url_upload import process_url, URLUploadRequest
from app.config import settings
async def main():
# Mock request and URLUploadRequest
request = Mock()
url_request = URLUploadRequest(url="https://example.com/file.pdf")
# Generate a large chunk
large_chunk = b"A" * 8192
num_chunks = 10000 # 8192 * 10000 = ~80MB
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = {"Content-Type": "application/pdf"}
mock_response.iter_content = Mock(return_value=[large_chunk] * num_chunks)
# For async client later
class AsyncMockResponse:
def __init__(self):
self.status_code = 200
self.headers = {"Content-Type": "application/pdf"}
def raise_for_status(self):
pass
async def aiter_bytes(self, chunk_size):
for _ in range(num_chunks):
yield large_chunk
async_mock_response = AsyncMockResponse()
# We will mock requests.get for synchronous, httpx.AsyncClient.get for asynchronous
# Test sync
start_time = time.time()
with patch("app.api.url_upload.requests.get", return_value=mock_response), \
patch("app.api.url_upload.process_document"):
try:
await process_url(request=request, url_request=url_request)
except Exception as e:
print(f"Error: {e}")
end_time = time.time()
print(f"Original execution time (sync writing): {end_time - start_time:.4f} seconds")
if __name__ == "__main__":
asyncio.run(main())
benchmark_url_upload2.py
+75
View File
@@ -0,0 +1,75 @@
import asyncio
import time
import os
import shutil
import tempfile
from unittest.mock import Mock, patch
from fastapi import HTTPException
from app.api.url_upload import process_url, URLUploadRequest
from app.config import settings
async def main():
# Setup test dir
test_dir = tempfile.mkdtemp()
settings.workdir = test_dir
# Mock request and URLUploadRequest
request = Mock()
url_request = URLUploadRequest(url="https://example.com/file.pdf")
# Generate a large chunk
chunk_size = 8192
num_chunks = 20000 # 20000 * 8192 = ~160MB
large_chunk = b"A" * chunk_size
class SyncMockResponse:
def __init__(self):
self.status_code = 200
self.headers = {"Content-Type": "application/pdf"}
def raise_for_status(self):
pass
def iter_content(self, chunk_size):
for _ in range(num_chunks):
# sleep slightly to simulate network latency, otherwise OS file cache obscures the difference
time.sleep(0.0001)
yield large_chunk
sync_mock_response = SyncMockResponse()
class AsyncMockResponse:
def __init__(self):
self.status_code = 200
self.headers = {"Content-Type": "application/pdf"}
self.is_success = True
self.status_code = 200
def raise_for_status(self):
pass
async def aiter_bytes(self, chunk_size=8192):
for _ in range(num_chunks):
await asyncio.sleep(0.0001)
yield large_chunk
class AsyncMockContext:
async def __aenter__(self):
return AsyncMockResponse()
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
async_mock_response = AsyncMockResponse()
# Test sync
start_time = time.time()
with patch("app.api.url_upload.requests.get", return_value=sync_mock_response), \
patch("app.api.url_upload.process_document"):
try:
await process_url(request=request, url_request=url_request)
except Exception as e:
print(f"Error (sync): {e}")
end_time = time.time()
print(f"Original execution time (sync writing): {end_time - start_time:.4f} seconds")
shutil.rmtree(test_dir)
if __name__ == "__main__":
asyncio.run(main())
crowdin.yml
+11
View File
@@ -0,0 +1,11 @@
"project_id_env": "CROWDIN_PROJECT_ID"
"api_token_env": "CROWDIN_PERSONAL_TOKEN"
"base_path": "."
"base_url": "https://api.crowdin.com"
"preserve_hierarchy": true
files:
- source: /frontend/translations/en.json
translation: /frontend/translations/%two_letters_code%.json
type: json
docs/API.md
+342
View File
@@ -1960,6 +1960,160 @@ Pass no `pipeline_id` query parameter (or omit it) to clear the assignment.
```
## Routing Rules
Routing rules let you conditionally assign documents to different pipelines
based on file properties such as type, size, filename, or AI-extracted
metadata. Rules are evaluated in **position order** (lowest first); the first
rule that matches wins. If no rule matches, the system falls back to the
owner's (or global) default pipeline.
### Supported operators and fields
```bash
GET /api/routing-rules/operators
```
Returns the catalogue of valid operators and built-in fields so UIs can
populate dropdowns without hard-coding values.
**Response (200):**
```json
{
"operators": ["contains", "equals", "gt", "gte", "lt", "lte", "not_contains", "not_equals", "regex"],
"builtin_fields": ["category", "document_type", "file_type", "filename", "size"],
"metadata_prefix": "metadata."
}
```
> **Tip:** For AI metadata fields use the `metadata.` prefix, e.g.
> `metadata.sender`, `metadata.amount`.
### List routing rules
```bash
GET /api/routing-rules
```
Returns the current user's rules **plus** any system-wide rules
(`owner_id = null`), ordered by position.
**Response (200):**
```json
[
{
"id": 1,
"owner_id": "alice",
"name": "Route invoices",
"position": 0,
"field": "document_type",
"operator": "equals",
"value": "Invoice",
"target_pipeline_id": 3,
"is_active": true,
"created_at": "2026-03-09T12:00:00+00:00",
"updated_at": "2026-03-09T12:00:00+00:00"
}
]
```
### Create routing rule
```bash
POST /api/routing-rules
Content-Type: application/json
{
"name": "Route invoices",
"field": "document_type",
"operator": "equals",
"value": "Invoice",
"target_pipeline_id": 3
}
```
Optional fields: `position` (auto-assigned if omitted), `is_active` (default `true`).
**Response (201 Created):** The created rule object.
### Get routing rule
```bash
GET /api/routing-rules/{rule_id}
```
**Response (200):** A single rule object.
### Update routing rule
```bash
PUT /api/routing-rules/{rule_id}
Content-Type: application/json
{ "name": "Renamed rule", "operator": "contains", "is_active": false }
```
Only the supplied fields are updated.
**Response (200):** The updated rule object.
### Delete routing rule
```bash
DELETE /api/routing-rules/{rule_id}
```
Returns **204 No Content**.
### Reorder routing rules
```bash
PUT /api/routing-rules/reorder
Content-Type: application/json
{ "rule_ids": [3, 1, 2] }
```
Provide the complete ordered list of your rule IDs. Positions are reassigned
0, 1, 2, … in the given order.
### Evaluate rules (dry run)
```bash
POST /api/routing-rules/evaluate
Content-Type: application/json
{
"file_type": "application/pdf",
"filename": "invoice_2024.pdf",
"size": 204800,
"document_type": "Invoice",
"metadata": { "sender": "Acme Corp" }
}
```
Tests which rule (if any) would match the given properties **without**
actually routing a document.
**Response (200) – match found:**
```json
{
"matched": true,
"rule": { "id": 1, "name": "Route invoices", "..." : "..." },
"target_pipeline": { "id": 3, "name": "Invoice Pipeline", "is_active": true }
}
```
**Response (200) – no match:**
```json
{
"matched": false,
"rule": null,
"target_pipeline": null
}
```
## API Tokens
Personal API tokens allow programmatic access to the DocuElevate API without
@@ -2215,3 +2369,191 @@ Automation hook deliveries follow the same retry policy as regular webhooks: up
## Further Assistance
For additional help with the API, please contact our support team or refer to the [Development Guide](../CONTRIBUTING.md).
## Mobile App API
The mobile API provides endpoints used by the native iOS and Android app. All endpoints require authentication (Bearer token or active session cookie).
For full mobile app documentation see [MobileApp.md](./MobileApp.md).
### POST /api/mobile/generate-token
Exchange an active web session for a long-lived API token scoped to the mobile app.
**Request:**
```json
{ "device_name": "John's iPhone" }
```
**Response (201 Created):**
```json
{
"token": "de_AbCdEfGhIjKl...",
"token_id": 42,
"name": "Mobile App – John's iPhone",
"created_at": "2026-03-10T09:30:00Z"
}
```
> The `token` is shown **once only**.
### POST /api/mobile/register-device
Register an Expo push token to receive push notifications.
**Request:**
```json
{
"push_token": "ExponentPushToken[xxxxxx]",
"device_name": "John's iPhone",
"platform": "ios"
}
```
**Response (201 Created):** Device record with `id`, `platform`, `is_active`, `created_at`.
### GET /api/mobile/devices
List all registered push-notification devices for the current user.
**Response (200 OK):** Array of device records.
### DELETE /api/mobile/devices/{device_id}
Deactivate a push-notification device. The device will no longer receive push notifications.
**Response (204 No Content)**
### GET /api/mobile/whoami
Return basic profile information for the authenticated user.
**Response (200 OK):**
```json
{
"owner_id": "john@example.com",
"display_name": "John Doe",
"email": "john@example.com",
"avatar_url": "https://www.gravatar.com/avatar/...",
"is_admin": false
}
```
---
## GraphQL API
DocuElevate exposes a GraphQL API at `/graphql` alongside the REST API. It
supports flexible queries with field selection, making it ideal for dashboards
and integrations that only need a subset of the available data.
### Endpoint
| Method | URL | Description |
|--------|-----|-------------|
| `POST` | `/graphql` | Execute a GraphQL query or mutation |
| `GET` | `/graphql` | Open the GraphiQL interactive playground |
### Authentication
The GraphQL endpoint honours the same authentication rules as the REST API:
- **`AUTH_ENABLED=False`** (default, single-user mode): all queries are
allowed without credentials.
- **`AUTH_ENABLED=True`** (multi-user mode): a valid session cookie **or**
an `Authorization: Bearer <token>` API token is required. Admin-only
queries (settings, users) additionally require the `is_admin` flag.
### Available Queries
| Field | Returns | Notes |
|-------|---------|-------|
| `documents(ownerId, limit, offset)` | `[DocumentType]` | Paginated list of documents |
| `document(id)` | `DocumentType` | Single document by primary key |
| `pipelines(ownerId, limit, offset)` | `[PipelineType]` | Paginated list of pipelines with steps |
| `pipeline(id)` | `PipelineType` | Single pipeline by primary key |
| `settings(limit, offset)` | `[SettingType]` | Non-sensitive app settings (**admin only**) |
| `users(limit, offset)` | `[UserType]` | User profiles (**admin only**) |
| `user(userId)` | `UserType` | Single user profile (**admin only**) |
> **Note:** Sensitive configuration keys (API secrets, passwords, tokens) are
> automatically excluded from the `settings` query regardless of the caller's
> privilege level.
### GraphiQL Playground
Navigate to `http://<your-instance>/graphql` in a browser to open the
interactive GraphiQL IDE, which provides schema documentation, auto-complete,
and the ability to run queries directly.
### Example Queries
**List recent documents:**
```graphql
{
documents(limit: 5) {
id
originalFilename
mimeType
fileSize
documentTitle
createdAt
}
}
```
**Fetch a pipeline with its steps:**
```graphql
{
pipeline(id: 1) {
id
name
description
isDefault
isActive
steps {
position
stepType
label
enabled
}
}
}
```
**List application settings (admin only):**
```graphql
{
settings {
key
value
updatedAt
}
}
```
**List user profiles (admin only):**
```graphql
{
users(limit: 10) {
userId
displayName
subscriptionTier
isBlocked
}
}
```
**Using variables:**
```graphql
query GetDocument($id: Int!) {
document(id: $id) {
id
originalFilename
documentTitle
isDuplicate
ocrQualityScore
}
}
```
Variables: `{ "id": 42 }`
docs/AuthenticationSetup.md
+10 -1
View File
@@ -20,10 +20,11 @@ For a complete list of configuration options, see the [Configuration Guide](Conf
## Authentication Methods
DocuElevate supports two primary authentication methods:
DocuElevate supports multiple authentication methods that can be used independently or together:
1. **Simple Authentication** - Basic username/password authentication managed by DocuElevate
2. **OpenID Connect** - Integration with identity providers like Authentik, Keycloak, or Auth0
3. **Social Login** - Sign in with Google, Microsoft, Apple, or Dropbox accounts (see [Social Login Setup Guide](SocialLoginSetup.md))
## Session Security
@@ -189,3 +190,11 @@ If you encounter issues with authentication:
- For most providers, you can visit the `/.well-known/openid-configuration` endpoint to verify their settings
For more general configuration issues, see the [Configuration Troubleshooting Guide](ConfigurationTroubleshooting.md).
## Social Login
DocuElevate supports social login with Google, Microsoft, Apple, and Dropbox. Social login allows users to authenticate using their existing accounts with these providers, without needing a separate DocuElevate password.
Social login can be used alongside any other authentication method (simple auth, OIDC, local signup). Each social provider is independently configured.
For detailed setup instructions, prerequisites, and provider-specific configuration, see the **[Social Login Setup Guide](SocialLoginSetup.md)**.
docs/ComplianceGuide.md
+190
View File
@@ -0,0 +1,190 @@
# Compliance Templates Guide
DocuElevate includes pre-built compliance templates for **GDPR**, **HIPAA**, and **SOC 2** that help you configure your instance to meet regulatory requirements. This guide covers how to use the compliance dashboard, apply templates, and monitor your compliance status.
## Overview
The compliance templates feature provides:
- **Pre-built configurations** for GDPR, HIPAA, and SOC 2
- **One-click apply** to configure all required settings at once
- **Compliance status dashboard** to monitor your regulatory posture
- **Individual check results** showing which settings are compliant and which need attention
## Accessing the Dashboard
The compliance dashboard is available to **admin users only**.
1. Log in as an administrator
2. Click **Admin** in the navigation bar
3. Select **Compliance** from the dropdown menu
Or navigate directly to: `/admin/compliance`
## Available Templates
### GDPR (General Data Protection Regulation)
The European Union regulation for data protection and privacy. The GDPR template enforces:
| Setting | Value | Purpose |
|---------|-------|---------|
| `AUTH_ENABLED` | `True` | Controls access to personal data |
| `SENTRY_SEND_DEFAULT_PII` | `False` | Prevents PII leaking to external services |
| `SECURITY_HEADERS_ENABLED` | `True` | Protects against common web vulnerabilities |
| `SECURITY_HEADER_HSTS_ENABLED` | `True` | Ensures encrypted connections |
| `SECURITY_HEADER_CSP_ENABLED` | `True` | Prevents XSS and injection attacks |
| `SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED` | `True` | Prevents clickjacking |
| `ENABLE_DEDUPLICATION` | `True` | Data minimisation — avoids duplicate storage |
### HIPAA (Health Insurance Portability and Accountability Act)
United States regulation for protecting health information. The HIPAA template includes all GDPR settings plus:
| Setting | Value | Purpose |
|---------|-------|---------|
| `MULTI_USER_ENABLED` | `True` | Individual accounts for access accountability |
### SOC 2 (Service Organization Control 2)
Trust Service Criteria framework for service organisations. The SOC 2 template includes the same settings as HIPAA, mapped to SOC 2 Trust Service Criteria (CC6.x, PI1.x).
## Applying a Template
1. Navigate to the **Compliance** dashboard (`/admin/compliance`)
2. Find the template you want to apply (GDPR, HIPAA, or SOC 2)
3. Click **Apply Template**
4. Confirm the action in the dialog
5. The template settings are written to the database immediately
> **Note:** Applying a template writes configuration values to the database. Some settings (e.g., security headers) may require a restart to take effect. Check the Settings page for restart indicators.
## Understanding Compliance Status
Each template shows one of four statuses:
| Status | Badge | Meaning |
|--------|-------|---------|
| **Compliant** | Green | All checks are passing |
| **Partial** | Yellow | Some checks are passing, others are not |
| **Non-Compliant** | Red | No checks are passing |
| **Not Applied** | Grey | Template has never been applied |
### Individual Checks
Click **Show Details** on any template card to see individual check results:
- ✅ **Passing** — The setting matches the expected compliance value
- ❌ **Failing** — The setting does not match; the current and expected values are shown
## API Endpoints
The compliance feature exposes the following API endpoints under `/api/compliance/`:
### List Templates
```bash
GET /api/compliance/templates
```
Returns all compliance templates with their current status.
### Get Single Template
```bash
GET /api/compliance/templates/{name}
```
Returns a single template by name (`gdpr`, `hipaa`, or `soc2`).
### Apply Template
```bash
POST /api/compliance/templates/{name}/apply
```
Applies a compliance template, writing all its settings to the database.
### Get Template Status
```bash
GET /api/compliance/templates/{name}/status
```
Evaluates the live compliance status of a template against current settings.
**Response example:**
```json
{
"status": "partial",
"total": 7,
"passed": 5,
"failed": 2,
"check_results": [
{
"key": "auth_enabled",
"label": "Authentication enabled",
"description": "User authentication must be enabled to control access to personal data.",
"expected": "True",
"actual": "True",
"passing": true
}
]
}
```
### Compliance Summary
```bash
GET /api/compliance/summary
```
Returns an overall compliance summary across all templates.
**Response example:**
```json
{
"overall_status": "partial",
"total_checks": 22,
"total_passed": 18,
"total_failed": 4,
"templates": [
{
"name": "gdpr",
"display_name": "GDPR (General Data Protection Regulation)",
"enabled": true,
"status": "compliant",
"total": 7,
"passed": 7,
"failed": 0,
"applied_at": "2026-03-09T12:00:00+00:00",
"applied_by": "admin@example.com"
}
]
}
```
> **Note:** All API endpoints require admin authentication.
## Configuration
| Variable | Default | Description |
|----------|---------|-------------|
| `COMPLIANCE_ENABLED` | `True` | Enable the compliance templates dashboard. Set to `False` to hide the feature. |
## Best Practices
1. **Apply templates before going live** — Set up compliance before processing real documents
2. **Monitor status regularly** — Check the compliance dashboard after configuration changes
3. **Use the refresh button** — After changing settings elsewhere, refresh the compliance page to see updated status
4. **Combine templates** — You can apply multiple templates; settings overlap is handled automatically
5. **Review after updates** — After upgrading DocuElevate, review your compliance status as new checks may be added
## Related Documentation
- [Configuration Guide](./ConfigurationGuide.md) — Full list of configuration options
- [Privacy & Compliance Guide](./PrivacyCompliance.md) — Privacy notice and GDPR compliance details
- [Deployment Guide](./DeploymentGuide.md) — Production deployment with security best practices
- [Security Audit](../SECURITY_AUDIT.md) — Security findings and mitigations
docs/ConfigurationGuide.md
+264
View File
@@ -16,6 +16,7 @@ Configuration is primarily done through environment variables specified in a `.e
| `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` |
| `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` |
| `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` |
| `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` |
### Batch Processing Settings
@@ -303,6 +304,42 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes —
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
| `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` |
| `IMAP_ATTACHMENT_FILTER` | System-wide fallback for which attachment types are ingested when no ingestion profile is assigned to a mailbox. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual IMAP accounts can override this using ingestion profiles. | `documents_only` |
#### IMAP Ingestion Profiles
For fine-grained control, DocuElevate supports **Ingestion Profiles** — named configurations that let you choose exactly which file-type categories to accept from each mailbox.
Each profile contains a list of enabled **categories**:
| Category | Description |
|----------|-------------|
| `pdf` | PDF documents (`.pdf`) |
| `office` | Microsoft Office files (Word, Excel, PowerPoint — `.docx`, `.xlsx`, `.pptx`, …) |
| `opendocument` | LibreOffice/OpenOffice files (`.odt`, `.ods`, `.odp`, …) |
| `text` | Plain text, CSV and RTF files (`.txt`, `.csv`, `.rtf`) |
| `web` | HTML and Markdown files (`.html`, `.htm`, `.md`, `.markdown`) |
| `images` | Image files (`.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg`) |
Two built-in system profiles are seeded automatically:
| Profile | Categories |
|---------|------------|
| **Documents Only** | pdf, office, opendocument, text, web (no images) |
| **All Files** | All categories, including images |
Users can create their own custom profiles via the **Email Ingestion** dashboard (`/imap-accounts`) by clicking the **Manage profiles** link or the **+** button next to the profile dropdown. Custom profiles are private to the creating user and can be freely edited or deleted.
**API endpoints for ingestion profiles:**
| Method | Endpoint | Description |
|--------|----------|-------------|
| `GET` | `/api/imap-profiles/` | List all visible profiles (system + user's own) |
| `POST` | `/api/imap-profiles/` | Create a new profile |
| `GET` | `/api/imap-profiles/categories` | List available file-type categories |
| `GET` | `/api/imap-profiles/{id}` | Get a single profile |
| `PUT` | `/api/imap-profiles/{id}` | Update a profile (not built-in) |
| `DELETE` | `/api/imap-profiles/{id}` | Delete a profile (not built-in) |
#### Per-User IMAP Integrations
@@ -335,6 +372,28 @@ Credentials are encrypted at rest using Fernet encryption.
| `AUTHENTIK_CONFIG_URL` | Configuration URL for Authentik OpenID Connect. |
| `OAUTH_PROVIDER_NAME` | Display name for the OAuth provider button. |
### Social Login Providers
Social login lets users sign in with their existing Google, Microsoft, Apple, or Dropbox accounts. Each provider is independently enabled and configured. For detailed setup instructions see the [Social Login Setup Guide](SocialLoginSetup.md).
| **Variable** | **Description** | **Default** |
|---|---|---|
| `SOCIAL_AUTH_GOOGLE_ENABLED` | Enable Google Sign-In. | `false` |
| `SOCIAL_AUTH_GOOGLE_CLIENT_ID` | Google OAuth2 client ID from the Google Cloud Console. | *(empty)* |
| `SOCIAL_AUTH_GOOGLE_CLIENT_SECRET` | Google OAuth2 client secret. | *(empty)* |
| `SOCIAL_AUTH_MICROSOFT_ENABLED` | Enable Microsoft Sign-In (Azure AD / Microsoft Entra ID). | `false` |
| `SOCIAL_AUTH_MICROSOFT_CLIENT_ID` | Microsoft application (client) ID from Azure App Registrations. | *(empty)* |
| `SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET` | Microsoft client secret. | *(empty)* |
| `SOCIAL_AUTH_MICROSOFT_TENANT` | Azure AD tenant: `common`, `organizations`, `consumers`, or a tenant GUID. | `common` |
| `SOCIAL_AUTH_APPLE_ENABLED` | Enable Sign in with Apple. | `false` |
| `SOCIAL_AUTH_APPLE_CLIENT_ID` | Apple Services ID (e.g. `com.example.docuelevate`). | *(empty)* |
| `SOCIAL_AUTH_APPLE_TEAM_ID` | Apple Developer Team ID. | *(empty)* |
| `SOCIAL_AUTH_APPLE_KEY_ID` | Apple Sign-In private key ID. | *(empty)* |
| `SOCIAL_AUTH_APPLE_PRIVATE_KEY` | Apple Sign-In private key (PEM format). | *(empty)* |
| `SOCIAL_AUTH_DROPBOX_ENABLED` | Enable Dropbox Sign-In. | `false` |
| `SOCIAL_AUTH_DROPBOX_CLIENT_ID` | Dropbox OAuth2 App Key. | *(empty)* |
| `SOCIAL_AUTH_DROPBOX_CLIENT_SECRET` | Dropbox OAuth2 App Secret. | *(empty)* |
### Multi-User Mode
When multi-user mode is enabled, each authenticated user gets their own isolated document space.
@@ -398,6 +457,142 @@ default overage buffer applied across all plans.
DocuElevate supports HTTP security headers to improve browser-side security. **These headers are disabled by default** since most deployments use a reverse proxy (Traefik, Nginx, etc.) that already adds them. Enable only if deploying directly without a reverse proxy. See [Deployment Guide - Security Headers](DeploymentGuide.md#security-headers) for detailed configuration examples.
### Application Logging
DocuElevate uses Python's standard `logging` module. Two environment variables control log verbosity:
| **Variable** | **Description** | **Default** |
|-------------|----------------|-------------|
| `LOG_LEVEL` | Root logger level. Accepts standard Python level names: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. | `INFO` |
| `DEBUG` | Enable debug mode. When `true` **and** `LOG_LEVEL` is **not** explicitly set, the effective log level is automatically lowered to `DEBUG`. | `false` |
**Precedence rules (standard behaviour):**
1. If `LOG_LEVEL` is explicitly set, it always wins — regardless of `DEBUG`.
2. If only `DEBUG=true` is set (no `LOG_LEVEL`), the effective level becomes `DEBUG`.
3. If neither is set, the default level is `INFO`.
```bash
# Typical production (default)
# LOG_LEVEL=INFO
# Quick debug mode — sets level to DEBUG automatically
DEBUG=true
# Explicit level override (DEBUG flag is ignored for level selection)
LOG_LEVEL=WARNING
```
> **Tip:** At `DEBUG` level, noisy third-party libraries (httpx, authlib, urllib3, etc.) are automatically pinned to `WARNING` so that application debug output remains readable.
#### Structured JSON Logging
Set `LOG_FORMAT=json` to emit structured JSON lines on stdout — one JSON object per log message. This is the standard format for log collectors and SIEM tools:
| **Variable** | **Description** | **Default** |
|-------------|----------------|-------------|
| `LOG_FORMAT` | Log output format: `text` (human-readable) or `json` (structured JSON lines). | `text` |
Each JSON log line contains: `timestamp` (ISO 8601), `level`, `logger`, `message`, `module`, `funcName`, `lineno`, and `exc_info` (when an exception is logged).
```bash
# Enable JSON logging for SIEM / log aggregation
LOG_FORMAT=json
```
**Example JSON output:**
```json
{"timestamp": "2025-03-16T09:18:05.192000+00:00", "level": "INFO", "logger": "app.auth", "message": "[SECURITY] OAUTH_LOGIN_SUCCESS user=alice@example.com admin=False", "module": "auth", "funcName": "oauth_callback", "lineno": 654}
```
**Compatible with:**
- **Grafana Loki** — Promtail scrapes JSON from Docker stdout
- **Splunk** — Universal Forwarder or HEC with JSON sourcetype
- **ELK / OpenSearch** — Filebeat with JSON codec
- **Datadog** — Agent auto-parses JSON logs
- **Fluentd / Vector** — JSON input plugin
- **Docker log drivers** — `--log-driver=json-file` (default) preserves structure
#### Syslog Forwarding (Application Logs)
For traditional (non-container) deployments, application logs can be forwarded directly to a syslog receiver. This is **separate** from audit-log SIEM forwarding (see below) — it sends _every_ Python log message, not just audit events.
| **Variable** | **Description** | **Default** |
|-------------|----------------|-------------|
| `LOG_SYSLOG_ENABLED` | Forward application logs to a syslog receiver in addition to stdout. | `false` |
| `LOG_SYSLOG_HOST` | Hostname or IP of the syslog receiver. | `localhost` |
| `LOG_SYSLOG_PORT` | Port of the syslog receiver. | `514` |
| `LOG_SYSLOG_PROTOCOL` | Protocol: `udp` or `tcp`. | `udp` |
```bash
# Forward all application logs to syslog
LOG_SYSLOG_ENABLED=true
LOG_SYSLOG_HOST=syslog.internal.example.com
LOG_SYSLOG_PORT=514
LOG_SYSLOG_PROTOCOL=udp
# Combine with JSON format for structured syslog messages
LOG_FORMAT=json
LOG_SYSLOG_ENABLED=true
```
> **Note:** When `LOG_FORMAT=json`, syslog messages are also sent as JSON. When `LOG_FORMAT=text`, syslog messages use the standard `name - level - message` format.
### Audit Logging
DocuElevate provides comprehensive audit logging that records significant actions (logins, document CRUD, settings changes) to an append-only database table. Every entry captures the timestamp, user, action, resource, client IP, and optional JSON details.
| **Variable** | **Description** | **Default** |
|--------------------------------|---------------------------------------------------------------------------------------------------|-------------|
| `AUDIT_LOGGING_ENABLED` | Enable the HTTP request audit-logging middleware. | `true` |
| `AUDIT_LOG_INCLUDE_CLIENT_IP` | Include the client IP address in audit log entries. Disable for GDPR-sensitive deployments. | `true` |
#### SIEM Integration
Audit events can be forwarded in real time to external SIEM systems for centralised monitoring, alerting, and long-term retention. Two transports are supported:
* **Syslog** – RFC 5424 structured-data messages over UDP or TCP. Works with rsyslog, syslog-ng, Graylog, Datadog, etc.
* **HTTP** – JSON POST payloads compatible with Splunk HEC, Logstash HTTP input, Grafana Loki push API, and any generic webhook.
| **Variable** | **Description** | **Default** |
|-------------------------------------|---------------------------------------------------------------------------------------------------|---------------|
| `AUDIT_SIEM_ENABLED` | Enable forwarding of audit events to an external SIEM system. | `false` |
| `AUDIT_SIEM_TRANSPORT` | Transport: `syslog` or `http`. | `syslog` |
| `AUDIT_SIEM_SYSLOG_HOST` | Hostname or IP of the syslog receiver. | `localhost` |
| `AUDIT_SIEM_SYSLOG_PORT` | Port of the syslog receiver. | `514` |
| `AUDIT_SIEM_SYSLOG_PROTOCOL` | Protocol for syslog: `udp` or `tcp`. | `udp` |
| `AUDIT_SIEM_HTTP_URL` | HTTP endpoint URL for SIEM delivery (e.g. Splunk HEC, Logstash, Loki). | *(empty)* |
| `AUDIT_SIEM_HTTP_TOKEN` | Bearer / HEC token for the SIEM HTTP endpoint. | *(empty)* |
| `AUDIT_SIEM_HTTP_CUSTOM_HEADERS` | Comma-separated `Key:Value` extra headers for SIEM HTTP requests. | *(empty)* |
**Example – Syslog to rsyslog:**
```bash
AUDIT_SIEM_ENABLED=true
AUDIT_SIEM_TRANSPORT=syslog
AUDIT_SIEM_SYSLOG_HOST=syslog.internal.example.com
AUDIT_SIEM_SYSLOG_PORT=514
AUDIT_SIEM_SYSLOG_PROTOCOL=udp
```
**Example – Splunk HEC:**
```bash
AUDIT_SIEM_ENABLED=true
AUDIT_SIEM_TRANSPORT=http
AUDIT_SIEM_HTTP_URL=https://splunk.example.com:8088/services/collector/event
AUDIT_SIEM_HTTP_TOKEN=your-hec-token
```
**Example – Logstash HTTP input:**
```bash
AUDIT_SIEM_ENABLED=true
AUDIT_SIEM_TRANSPORT=http
AUDIT_SIEM_HTTP_URL=https://logstash.example.com:8080
AUDIT_SIEM_HTTP_TOKEN=
```
### Rate Limiting
DocuElevate implements rate limiting to protect against DoS attacks and API abuse. **Rate limiting is enabled by default** and uses Redis for distributed rate limiting across multiple workers.
@@ -739,6 +934,48 @@ OPENAI_API_KEY=sk-ant-... # passed as the api_key to LiteLLM
---
### Document Translation
After processing, DocuElevate can automatically translate a document's extracted text into a configurable *default language* (e.g. English). This reference translation is stored alongside the original text so users always have a version in a language they understand.
Other languages are translated **on the fly** via the AI provider and are not persisted.
#### Settings
| **Variable** | **Description** | **Default** |
|------------------------------|-----------------------------------------------------------------------------------------------------------|-------------|
| `DEFAULT_DOCUMENT_LANGUAGE` | ISO 639-1 code for the default translation target (e.g. `en`, `de`, `fr`). Documents whose detected language differs are automatically translated into this language after processing. | `en` |
Each user can override this global default in their profile (`UserProfile.default_document_language`).
#### How It Works
1. During metadata extraction the AI detects the document language (stored as `detected_language` on the file record).
2. If the detected language differs from the default target language, a background Celery task (`translate_to_default_language`) translates the extracted text.
3. The translated text is persisted in `default_language_text` and the target code in `default_language_code`.
4. The file detail view shows both the original text and the default-language version.
5. Users can also request on-the-fly translations to any language via the **Translate** dropdown.
#### API Endpoints
| **Endpoint** | **Method** | **Description** |
|-----------------------------------------------|------------|------------------------------------------------------------------------|
| `/api/files/{id}/translation/default` | GET | Returns the persisted default-language translation (404 if unavailable)|
| `/api/files/{id}/translate?lang=xx` | GET | On-the-fly translation to any ISO 639-1 language code |
| `/files/{id}/text/default-language` | GET | View endpoint returning the default-language text as JSON |
#### Example
```bash
# Get the stored English translation of a German document
curl http://localhost:8000/api/files/42/translation/default
# Translate on the fly to French
curl "http://localhost:8000/api/files/42/translate?lang=fr"
```
---
### OCR Providers
DocuElevate supports multiple OCR engines that can be used individually or in combination. Configure the list of active providers with `OCR_PROVIDERS` and tune each provider with the settings below.
@@ -887,6 +1124,7 @@ TESSERACT_LANGUAGE=eng+deu
| **Variable** | **Description** |
|-------------------------------------|-----------------------------------------------------------------------------------------------------|
| `PAPERLESS_ENABLED` | Set to `false` to disable Paperless-ngx uploads without removing credentials. Default: `true` |
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. |
| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
| `PAPERLESS_CUSTOM_FIELD_ABSENDER` | (Optional, Legacy) Name of the custom field in Paperless-ngx to store the sender ("absender") information. If set, the extracted sender will be automatically set as a custom field after document upload. Example: `Absender` or `Sender` |
@@ -929,6 +1167,7 @@ PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient
| **Variable** | **Description** |
|-------------------------|--------------------------------------------------|
| `DROPBOX_ENABLED` | Set to `false` to disable Dropbox uploads without removing credentials. Default: `true` |
| `DROPBOX_APP_KEY` | Dropbox API app key. |
| `DROPBOX_APP_SECRET` | Dropbox API app secret. |
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. |
@@ -940,6 +1179,7 @@ For detailed setup instructions, see the [Dropbox Setup Guide](DropboxSetup.md).
| **Variable** | **Description** |
|-------------------------|---------------------------------------------------------------|
| `NEXTCLOUD_ENABLED` | Set to `false` to disable Nextcloud uploads without removing credentials. Default: `true` |
| `NEXTCLOUD_UPLOAD_URL` | Nextcloud WebDAV URL (e.g. `https://nc.example.com/remote.php/dav/files/<USERNAME>`). |
| `NEXTCLOUD_USERNAME` | Nextcloud login username. |
| `NEXTCLOUD_PASSWORD` | Nextcloud login password. |
@@ -949,6 +1189,7 @@ For detailed setup instructions, see the [Dropbox Setup Guide](DropboxSetup.md).
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `GOOGLE_DRIVE_ENABLED` | Set to `false` to disable Google Drive uploads without removing credentials. Default: `true` |
| `GOOGLE_DRIVE_USE_OAUTH` | Set to `true` to use OAuth flow (recommended) |
| `GOOGLE_DRIVE_CLIENT_ID` | OAuth Client ID (required if using OAuth flow) |
| `GOOGLE_DRIVE_CLIENT_SECRET` | OAuth Client Secret (required if using OAuth flow) |
@@ -965,6 +1206,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
| **Variable** | **Description** |
|-------------------------|---------------------------------------------------------------|
| `WEBDAV_ENABLED` | Set to `false` to disable WebDAV uploads without removing credentials. Default: `true` |
| `WEBDAV_URL` | WebDAV server URL (e.g. `https://webdav.example.com/path`). |
| `WEBDAV_USERNAME` | WebDAV authentication username. |
| `WEBDAV_PASSWORD` | WebDAV authentication password. |
@@ -975,6 +1217,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
| **Variable** | **Description** |
|-------------------------|---------------------------------------------------------------|
| `FTP_ENABLED` | Set to `false` to disable FTP uploads without removing credentials. Default: `true` |
| `FTP_HOST` | FTP server hostname or IP address. |
| `FTP_PORT` | FTP port (default: `21`). |
| `FTP_USERNAME` | FTP authentication username. |
@@ -987,6 +1230,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
| **Variable** | **Description** |
|------------------------------|-------------------------------------------------------|
| `SFTP_ENABLED` | Set to `false` to disable SFTP uploads without removing credentials. Default: `true` |
| `SFTP_HOST` | SFTP server hostname or IP address. |
| `SFTP_PORT` | SFTP port (default: `22`). |
| `SFTP_USERNAME` | SFTP authentication username. |
@@ -1018,6 +1262,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
| **Variable** | **Description** |
|----------------------------------|---------------------------------------------------------------------|
| `DEST_EMAIL_ENABLED` | Set to `false` to disable email delivery without removing credentials. Default: `true` |
| `DEST_EMAIL_HOST` | SMTP server hostname for document delivery. |
| `DEST_EMAIL_PORT` | SMTP port for document delivery (default: `587`). |
| `DEST_EMAIL_USERNAME` | SMTP authentication username for document delivery. |
@@ -1030,6 +1275,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `ONEDRIVE_ENABLED` | Set to `false` to disable OneDrive uploads without removing credentials. Default: `true` |
| `ONEDRIVE_CLIENT_ID` | Azure AD application client ID |
| `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret |
| `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts |
@@ -1042,6 +1288,7 @@ For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `S3_ENABLED` | Set to `false` to disable S3 uploads without removing credentials. Default: `true` |
| `AWS_ACCESS_KEY_ID` | AWS IAM access key ID |
| `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key |
| `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) |
@@ -1052,6 +1299,23 @@ For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md
For detailed setup instructions, see the [Amazon S3 Setup Guide](AmazonS3Setup.md).
### iCloud Drive (Apple)
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `ICLOUD_ENABLED` | Set to `false` to disable iCloud uploads without removing credentials. Default: `true` |
| `ICLOUD_USERNAME` | Apple ID email address |
| `ICLOUD_PASSWORD` | App-specific password (generate at [appleid.apple.com](https://appleid.apple.com/account/manage)) |
| `ICLOUD_FOLDER` | Target folder path in iCloud Drive (e.g. `Documents/Uploads`) |
| `ICLOUD_COOKIE_DIRECTORY` | Optional directory for session cookie persistence (default: `~/.pyicloud`) |
> **Note:** Apple does not provide a public REST API for iCloud Drive. This
> integration uses the [pyicloud](https://github.com/picklepete/pyicloud)
> library which relies on an unofficial, reverse-engineered protocol. Because
> most Apple IDs have two-factor authentication enabled, you **must** generate
> an [app-specific password](https://support.apple.com/en-us/102654) and use
> it as `ICLOUD_PASSWORD`.
### Notification System
| **Variable** | **Description** |
docs/InternationalizationGuide.md
+343
View File
@@ -0,0 +1,343 @@
# Internationalization (i18n) & Localization (l10n) Guide
DocuElevate supports **77 languages** for its web UI, with automatic browser
language detection, user-preference persistence, and an AI-powered fallback
translator for strings that haven't been manually translated yet.
## Supported Languages
| Code | Language | Native Name | Flag | Priority |
|---------|--------------------|--------------------|------|----------|
| `en` | English | English | 🇬🇧 | Tier 1 |
| `de` | German | Deutsch | 🇩🇪 | Tier 1 |
| `fr` | French | Français | 🇫🇷 | Tier 1 |
| `es` | Spanish | Español | 🇪🇸 | Tier 1 |
| `it` | Italian | Italiano | 🇮🇹 | Tier 1 |
| `pt` | Portuguese | Português | 🇵🇹 | Tier 1 |
| `nl` | Dutch | Nederlands | 🇳🇱 | Tier 2 |
| `nb` | Norwegian Bokmål | Norsk bokmål | 🇳🇴 | Tier 2 |
| `no` | Norwegian | Norsk | 🇳🇴 | Tier 2 |
| `da` | Danish | Dansk | 🇩🇰 | Tier 2 |
| `sv` | Swedish | Svenska | 🇸🇪 | Tier 2 |
| `fi` | Finnish | Suomi | 🇫🇮 | Tier 2 |
| `is` | Icelandic | Íslenska | 🇮🇸 | Tier 2 |
| `ga` | Irish | Gaeilge | 🇮🇪 | Tier 2 |
| `lb` | Luxembourgish | Lëtzebuergesch | 🇱🇺 | Tier 2 |
| `ca` | Catalan | Català | 🏴 | Tier 2 |
| `cy` | Welsh | Cymraeg | 🏴󠁧󠁢󠁷󠁬󠁳󠁿 | Tier 2 |
| `fy` | Western Frisian | Frysk | 🇳🇱 | Tier 2 |
| `gl` | Galician | Galego | 🇪🇸 | Tier 2 |
| `li` | Limburgish | Limburgs | 🇳🇱 | Tier 2 |
| `vls` | Flemish | West-Vlams | 🇧🇪 | Tier 2 |
| `nds` | Low German | Plattdüütsch | 🇩🇪 | Tier 2 |
| `pl` | Polish | Polski | 🇵🇱 | Tier 3 |
| `cs` | Czech | Čeština | 🇨🇿 | Tier 3 |
| `sk` | Slovak | Slovenčina | 🇸🇰 | Tier 3 |
| `hu` | Hungarian | Magyar | 🇭🇺 | Tier 3 |
| `sl` | Slovenian | Slovenščina | 🇸🇮 | Tier 3 |
| `hr` | Croatian | Hrvatski | 🇭🇷 | Tier 3 |
| `ro` | Romanian | Română | 🇷🇴 | Tier 3 |
| `bg` | Bulgarian | Български | 🇧🇬 | Tier 3 |
| `el` | Greek | Ελληνικά | 🇬🇷 | Tier 3 |
| `et` | Estonian | Eesti | 🇪🇪 | Tier 3 |
| `lv` | Latvian | Latviešu | 🇱🇻 | Tier 3 |
| `lt` | Lithuanian | Lietuvių | 🇱🇹 | Tier 3 |
| `sr` | Serbian | Српски | 🇷🇸 | Tier 3 |
| `tr` | Turkish | Türkçe | 🇹🇷 | Tier 4 |
| `uk` | Ukrainian | Українська | 🇺🇦 | Tier 4 |
| `he` | Hebrew | עברית | 🇮🇱 | Tier 4 |
| `ar` | Arabic | العربية | 🇸🇦 | Tier 4 |
| `fa` | Persian | فارسی | 🇮🇷 | Tier 4 |
| `af` | Afrikaans | Afrikaans | 🇿🇦 | Tier 4 |
| `zh` | Chinese | 中文 | 🇨🇳 | Tier 5 |
| `zh-TW` | Traditional Chinese | 繁體中文 | 🇹🇼 | Tier 5 |
| `ja` | Japanese | 日本語 | 🇯🇵 | Tier 5 |
| `ko` | Korean | 한국어 | 🇰🇷 | Tier 5 |
| `vi` | Vietnamese | Tiếng Việt | 🇻🇳 | Tier 5 |
| `pa` | Punjabi | ਪੰਜਾਬੀ | 🇮🇳 | Tier 5 |
| `kn` | Kannada | ಕನ್ನಡ | 🇮🇳 | Tier 5 |
| `hi` | Hindi | हिन्दी | 🇮🇳 | Tier 5 |
| `bn` | Bengali | বাংলা | 🇧🇩 | Tier 5 |
| `gu` | Gujarati | ગુજરાતી | 🇮🇳 | Tier 5 |
| `ml` | Malayalam | മലയാളം | 🇮🇳 | Tier 5 |
| `mr` | Marathi | मराठी | 🇮🇳 | Tier 5 |
| `ta` | Tamil | தமிழ் | 🇮🇳 | Tier 5 |
| `te` | Telugu | తెలుగు | 🇮🇳 | Tier 5 |
| `ur` | Urdu | اردو | 🇵🇰 | Tier 5 |
| `si` | Sinhala | සිංහල | 🇱🇰 | Tier 5 |
| `ne` | Nepali | नेपाली | 🇳🇵 | Tier 5 |
| `th` | Thai | ไทย | 🇹🇭 | Tier 5 |
| `km` | Khmer | ខ្មែរ | 🇰🇭 | Tier 5 |
| `id` | Indonesian | Bahasa Indonesia | 🇮🇩 | Tier 5 |
| `ms` | Malay | Bahasa Melayu | 🇲🇾 | Tier 5 |
| `jv` | Javanese | Basa Jawa | 🇮🇩 | Tier 5 |
| `tl` | Tagalog | Filipino | 🇵🇭 | Tier 5 |
| `mn` | Mongolian | Монгол | 🇲🇳 | Tier 5 |
| `kk` | Kazakh | Қазақ тілі | 🇰🇿 | Tier 5 |
| `uz` | Uzbek | Oʻzbekcha | 🇺🇿 | Tier 5 |
| `az` | Azerbaijani | Azərbaycan dili | 🇦🇿 | Tier 5 |
| `hy` | Armenian | Հայերեն | 🇦🇲 | Tier 5 |
| `ka` | Georgian | ქართული | 🇬🇪 | Tier 5 |
| `sw` | Swahili | Kiswahili | 🇰🇪 | Tier 6 |
| `am` | Amharic | አማርኛ | 🇪🇹 | Tier 6 |
| `ha` | Hausa | Hausa | 🇳🇬 | Tier 6 |
| `yo` | Yoruba | Yorùbá | 🇳🇬 | Tier 6 |
| `ig` | Igbo | Igbo | 🇳🇬 | Tier 6 |
| `zu` | Zulu | isiZulu | 🇿🇦 | Tier 6 |
| `eo` | Esperanto | Esperanto | 🌍 | Tier 7 |
> **Tier 1** languages (major European) have complete, manually-reviewed
> translations. **Tier 2–3** languages have complete translations but may
> receive less frequent updates. **Tier 4** covers Non-EU European, Middle
> Eastern, and South African (Afrikaans) languages. **Tier 5** covers Asian
> and Central Asian languages. **Tier 6** covers African languages. **Tier 7**
> covers constructed languages.
## How Language Is Detected
DocuElevate resolves the display language in the following priority order:
1. **User profile preference** — stored in the database (`UserProfile.preferred_language`)
and loaded into the session on login
2. **Cookie** — `docuelevate_lang` cookie (30-day expiry, set when user selects a language)
3. **Browser `Accept-Language` header** — the highest-priority match among supported languages
4. **Default** — English (`en`)
## Selecting Your Language
### Via the Navigation Bar
Click the 🌐 **globe icon** in the top navigation bar. A dropdown menu shows all
available languages with their native names and flag emoji. The current language
is highlighted with a blue checkmark.
### Via the API
```bash
# Set language to German
curl -X POST http://localhost:8000/api/i18n/language \
-H "Content-Type: application/json" \
-d '{"language": "de"}'
# List all available languages
curl http://localhost:8000/api/i18n/languages
```
### Via Cookie (Programmatic)
Set the `docuelevate_lang` cookie to any supported language code:
```javascript
document.cookie = "docuelevate_lang=fr; max-age=2592000; path=/";
location.reload();
```
## For Developers
### Translation File Structure
Translations are stored as flat JSON files in `frontend/translations/`:
```
frontend/translations/
├── en.json # English (base / reference)
├── af.json # Afrikaans
├── ar.json # Arabic
├── bg.json # Bulgarian
├── ca.json # Catalan
├── cs.json # Czech
├── cy.json # Welsh
├── da.json # Danish
├── de.json # German
├── el.json # Greek
├── eo.json # Esperanto
├── es.json # Spanish
├── et.json # Estonian
├── fa.json # Persian
├── fi.json # Finnish
├── fr.json # French
├── fy.json # Frisian
├── ga.json # Irish
├── gl.json # Galician
├── he.json # Hebrew
├── hr.json # Croatian
├── hu.json # Hungarian
├── is.json # Icelandic
├── it.json # Italian
├── ja.json # Japanese
├── kn.json # Kannada
├── ko.json # Korean
├── lb.json # Luxembourgish
├── li.json # Limburgish
├── lt.json # Lithuanian
├── lv.json # Latvian
├── nb.json # Norwegian Bokmål
├── nds.json # Low German (Plattdeutsch)
├── nl.json # Dutch
├── no.json # Norwegian
├── pa.json # Punjabi
├── pl.json # Polish
├── pt.json # Portuguese
├── ro.json # Romanian
├── ru.json # Russian
├── sk.json # Slovak
├── sl.json # Slovenian
├── sr.json # Serbian
├── sv.json # Swedish
├── tr.json # Turkish
├── uk.json # Ukrainian
├── vi.json # Vietnamese
├── vls.json # West Flemish
└── zh.json # Chinese
```
Each file is a flat key-value dictionary with dot-notation namespacing:
```json
{
"nav.dashboard": "Dashboard",
"nav.upload": "Upload",
"upload.max_size": "Maximum file size: {size}",
"footer.copyright": "DocuElevate {year}"
}
```
### Using Translations in Templates
The `_()` function is available globally in all Jinja2 templates:
```jinja2
{# Simple translation #}
<h1>{{ _("dashboard.title") }}</h1>
{# Translation with placeholders #}
<p>{{ _("upload.max_size", size="50 MB") }}</p>
{# Translation in attributes #}
<button aria-label="{{ _('common.save') }}">{{ _("common.save") }}</button>
```
### Using Translations in Python
```python
from app.utils.i18n import translate
# Basic translation
text = translate("nav.dashboard", "de") # → "Übersicht"
# With placeholders
text = translate("footer.copyright", "fr", year="2025") # → "DocuElevate 2025"
```
### Localization Helpers
Format dates, times, and numbers according to locale conventions:
```jinja2
{# In templates — locale is automatically detected #}
<span>{{ format_date_l10n(document.created_at) }}</span>
<span>{{ format_number_l10n(file_count) }}</span>
```
```python
# In Python
from app.utils.i18n import format_date, format_number
format_date(date(2025, 3, 15), "de") # → "15. March 2025"
format_date(date(2025, 3, 15), "de", short=True) # → "15.03.2025"
format_number(1234567, "de") # → "1.234.567"
format_number(1234.56, "en") # → "1,234.56"
```
### Adding a New Translation Key
1. Add the key and English text to `frontend/translations/en.json`
2. Use `{{ _("your.new.key") }}` in templates or `translate("your.new.key", locale)` in Python
That's it. An external automation script picks up new keys in `en.json` and propagates
translations to all other language files. You never need to touch the non-English JSON
files manually — the translate-and-sync pipeline takes care of it.
### AI Fallback Translation
When a translation key exists in English but not in the target language,
DocuElevate can use the configured AI provider (OpenAI, Anthropic, etc.)
to translate the string on-the-fly:
```python
from app.utils.i18n import translate_with_ai_fallback
# Falls back to AI if no manual translation exists
translated = translate_with_ai_fallback("Welcome to our platform", "de")
```
The AI fallback:
- Uses the `AI_MODEL` or `OPENAI_MODEL` setting
- Caches results in memory for the process lifetime
- Returns the original English text if the AI call fails
- Is designed for graceful degradation — the UI never breaks
### Adding a New Language
1. Create a new JSON file in `frontend/translations/` (e.g., `ja.json`)
2. Copy the structure from `en.json` and translate all values
3. Add the language to `SUPPORTED_LANGUAGES` in `app/utils/i18n.py`:
```python
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "🇯🇵"},
```
4. Add locale formatting rules to `_LOCALE_FORMATS` in the same file
5. Create a database migration if needed (the `preferred_language` column
already accepts any string up to 10 characters)
### Database Migration
Migration `027_add_user_language_preference` adds a `preferred_language`
column to the `user_profiles` table. This column stores the user's chosen
UI language as an ISO 639-1 code (e.g., `"de"`, `"fr"`). A `NULL` value
means "auto-detect from browser settings."
### API Reference
#### `GET /api/i18n/languages`
Returns all supported languages and the current active language.
**Response:**
```json
{
"languages": [
{"code": "en", "name": "English", "native": "English", "flag": "🇬🇧"},
{"code": "de", "name": "German", "native": "Deutsch", "flag": "🇩🇪"}
],
"current": "en",
"default": "en"
}
```
#### `POST /api/i18n/language`
Set the preferred UI language. Persists in session, cookie, and database.
**Request:**
```json
{"language": "de"}
```
**Response:**
```json
{
"language": "de",
"message": "Language changed to Deutsch"
}
```
## Configuration
No additional configuration is required. The i18n system works out of the box
with the default English language and automatically detects browser preferences.
| Setting | Default | Description |
|---------|---------|-------------|
| Browser `Accept-Language` | Auto-detected | Used when no explicit preference is set |
| `docuelevate_lang` cookie | Not set | Set when user selects a language via the UI |
| `UserProfile.preferred_language` | `NULL` | Stored in DB for authenticated users |
docs/MobileApp.md
+364
View File
@@ -0,0 +1,364 @@
# Mobile App
DocuElevate includes a native mobile application for iOS and Android built with **React Native** and **Expo**. The app allows users to capture documents with the device camera, pick files from the device storage, and receive push notifications when documents finish processing.
## Features
| Feature | iOS | Android |
|---------|-----|---------|
| SSO login (OAuth2) | ✅ | ✅ |
| Local / basic auth login | ✅ | ✅ |
| Auto-generated API token | ✅ | ✅ |
| Camera capture → upload | ✅ | ✅ |
| File picker upload | ✅ | ✅ |
| Share Sheet / Share Intent | ✅ | ✅ |
| Push notifications | ✅ | ✅ |
| Document list | ✅ | ✅ |
| Dark mode | ✅ | ✅ |
## Getting Started (Development)
### Prerequisites
- Node.js 20.19.4 or later (use [nvm](https://github.com/nvm-sh/nvm): `nvm install` inside `mobile/` reads `.nvmrc` automatically)
- [Expo CLI](https://docs.expo.dev/get-started/installation/): `npm install -g @expo/cli`
- [Expo Go](https://expo.dev/client) app on your iOS or Android device (for development)
- A running DocuElevate server reachable from your device
### Run in development mode
```bash
cd mobile
npm install
npx expo start
```
Scan the QR code with **Expo Go** on your device. On iOS you can also use the Camera app.
## Building for Production
DocuElevate uses **Expo Application Services (EAS)** to produce App Store / Play Store binaries.
```bash
# Install EAS CLI globally
npm install -g eas-cli
# Authenticate with Expo
eas login
# Build for iOS (requires Apple Developer account)
eas build --platform ios
# Build for Android
eas build --platform android
```
> **Note:** The mobile app uses `expo-build-properties` with `buildReactNativeFromSource: true` for iOS builds. This is required for Expo SDK 54 (React Native 0.81) compatibility — some native modules still use legacy bridge APIs (`RCTBridge`, `RCTViewManager`, etc.) that are no longer included in the default precompiled XCFrameworks. Building React Native from source makes these headers available, at the cost of slightly longer iOS build times.
See the [EAS Build documentation](https://docs.expo.dev/build/introduction/) for full setup instructions.
## Automated CI/CD
An **EAS Cloud Workflow** (`mobile/.eas/workflows/create-builds.yml`) automates production builds and iOS submission:
- **Path filtering:** The workflow only triggers on pushes to `main` that include changes inside the `mobile/` directory. Backend-only or documentation-only changes do not trigger a mobile build.
- **Build:** Both iOS and Android production builds run in parallel on EAS Build.
- **Auto-submit (iOS):** After a successful iOS build, the workflow automatically submits the binary to **App Store Connect** using the credentials configured in `eas.json` (`submit.production.ios`). The build then appears in **TestFlight** for internal testing and can be promoted to the App Store from App Store Connect.
> **Prerequisite:** An App Store Connect API Key must be configured in EAS for non-interactive submission. See [Troubleshooting → "Session expired"](#session-expired-local-session-during-ios-build) below for setup instructions.
### Version Management
Build numbers (iOS `buildNumber` / Android `versionCode`) are managed **remotely** by EAS. The `eas.json` configuration uses:
```json
{
"cli": { "appVersionSource": "remote" },
"build": { "production": { "autoIncrement": true } }
}
```
- **`appVersionSource: "remote"`** — EAS stores the current build number on its servers instead of reading it from `app.json`. This ensures every CI build gets a unique, ever-increasing number without needing to commit version bumps back to the repository.
- **`autoIncrement: true`** — EAS automatically increments the build number before each production build.
The `ios.buildNumber` and `android.versionCode` values in `app.json` serve as the **initial seed** when the remote version is first created; after that they are informational only. Do not rely on them for the actual version submitted to the stores.
> **Tip:** To check or manually set the remote version, use `eas build:version:get` and `eas build:version:set`.
## Authentication
### SSO Login Flow
The mobile app uses the server's existing OAuth2/SSO setup:
1. User enters the DocuElevate server URL on the login screen.
2. The app opens `<server>/login?mobile=1&redirect_uri=docuelevate://callback` in the **system browser** (Safari / Chrome Custom Tabs).
3. The server stores `docuelevate://callback` in the browser session and presents the login page.
4. The user authenticates via SSO or local credentials.
5. After successful authentication the server mints a long-lived API token and redirects the browser to `docuelevate://callback?token=<token>`.
6. `WebBrowser.openAuthSessionAsync` intercepts the `docuelevate://` deep link and returns the URL to the app.
7. The app extracts the token from the URL and stores it securely in the device's keychain (`expo-secure-store`).
> **Security note:** The `redirect_uri` is validated server-side; only URIs with the `docuelevate://` custom scheme (production) or the `exp://` scheme (Expo Go development) are accepted, preventing open-redirect attacks.
### Testing in Expo Go
When developing with **Expo Go** the app does not have the `docuelevate://` custom URL scheme registered. The auth flow adapts automatically:
1. `Linking.createURL('callback')` returns an `exp://` URI pointing at the local dev server (e.g. `exp://192.168.1.5:8081/--/callback`).
2. This URI is sent to the server as `redirect_uri`; the server accepts it alongside the production `docuelevate://` scheme.
3. After successful authentication the server redirects back to the `exp://` URI.
4. `WebBrowser.openAuthSessionAsync` intercepts the deep link and the Expo Go app receives the token.
No extra configuration is needed — just run `npx expo start` and scan the QR code with the **Expo Go** app.
### Auto-generated Mobile Token
When the mobile app completes login it automatically creates a named API token (`"Mobile App – <device name>"`) via `POST /api/mobile/generate-token`. This token:
- Works identically to tokens created manually in the web UI.
- Is shown in the **API Tokens** page (`/api-tokens`) and can be revoked there.
- Is stored in the device's secure keychain, never in plain storage.
## Push Notifications
Push notifications are delivered via the **Expo Push Notification** service, which routes through Apple Push Notification service (APNs) for iOS and Firebase Cloud Messaging (FCM) for Android.
**No server-side APNs/FCM credentials are required** – Expo's servers handle the provider integration.
### How it works
1. After login, the app requests notification permission from the operating system.
2. If granted, the app obtains an **Expo Push Token** (`ExponentPushToken[…]`).
3. The token is registered with the backend via `POST /api/mobile/register-device`.
4. When a document finishes processing, the server sends a push notification to all registered devices for that user.
### Managing registered devices
Users can see and remove their registered devices from the **Profile** tab in the app, or via the API:
```bash
# List registered devices
curl -H "Authorization: Bearer <token>" https://your-server/api/mobile/devices
# Remove a device
curl -X DELETE -H "Authorization: Bearer <token>" https://your-server/api/mobile/devices/<id>
```
## Uploading Documents
### Camera Capture
1. Open the **Upload** tab.
2. Tap **Camera**.
3. Point the camera at the document and take a photo.
4. The image is immediately uploaded and queued for processing.
### Photo Library
1. Open the **Upload** tab.
2. Tap **Photos**.
3. Select an existing photo from the device's photo library.
4. The image is uploaded and queued for processing.
### File Picker
1. Open the **Upload** tab.
2. Tap **Files**.
3. Browse to and select one or more files (PDF, DOCX, images, etc.).
4. Files are uploaded and queued for processing.
### Share Sheet (iOS) / Share Intent (Android)
The app registers itself as a share target so any file can be sent directly to DocuElevate from another app:
1. Open a file in Files, Mail, Safari, or any other app.
2. Tap the **Share** button (iOS) or **Share** (Android).
3. Find and tap **DocuElevate** in the share sheet.
4. The file is immediately uploaded and queued for processing.
> **Note:** The app must be installed on the device for it to appear in the share sheet.
#### iOS implementation
`app.json` declares `CFBundleDocumentTypes` (with `LSHandlerRank: Alternate`) inside the iOS `infoPlist`. This tells iOS that DocuElevate can open common document types, making it visible in the share sheet without overriding system defaults. When the user selects DocuElevate, iOS opens the app with a URL via `application:openURL:options:`.
The URL may arrive as a standard `file://` path **or** under the app's custom `docuelevate://` scheme (e.g. `docuelevate://private/var/mobile/Library/…/file.pdf`). The root layout detects the custom-scheme form and rewrites it to a `file://` URL before forwarding it to the Upload screen through `ShareContext`.
#### Android implementation
`app.json` declares `ACTION_SEND` and `ACTION_SEND_MULTIPLE` intent filters for `mimeType: "*/*"` in the `android.intentFilters` section. Incoming content URIs are received the same way as on iOS.
#### Upload status polling
After a file is uploaded the app polls `/api/files?search=<filename>` every 5 seconds to find the corresponding `FileRecord`, then polls `/api/files/{id}` to track the processing status in real time. Polling stops automatically once the status reaches a terminal state (`completed`, `failed`, or `duplicate`).
#### Retrying failed uploads
If a file upload fails (e.g. due to network issues or a server error), the failed item stays visible in the upload list with an error message and a **"Tap to retry"** hint. Users can retry the upload in two ways:
- **Tap** the failed item to immediately retry the upload.
- **Long-press** the failed item to see a confirmation dialog with a **Retry** option.
The retry re-uses the original file URI so no re-selection is needed.
## Mobile API Endpoints
The backend exposes a dedicated `/api/mobile/` namespace:
| Method | Endpoint | Auth | Description |
|--------|----------|------|-------------|
| `POST` | `/api/mobile/generate-token` | Session | Exchange SSO session for API token |
| `POST` | `/api/mobile/register-device` | Bearer | Register Expo push token |
| `GET` | `/api/mobile/devices` | Bearer | List registered devices |
| `DELETE` | `/api/mobile/devices/{id}` | Bearer | Deactivate a device |
| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile |
All other API endpoints (file upload, file listing, etc.) work with Bearer token authentication.
### POST /api/mobile/generate-token
Exchanges an active web session (cookie) for a permanent API token suitable for use in the mobile app.
**Request:**
```json
{ "device_name": "John's iPhone" }
```
**Response (201):**
```json
{
"token": "de_AbCdEfGhIjKl...",
"token_id": 42,
"name": "Mobile App – John's iPhone",
"created_at": "2026-03-10T09:30:00Z"
}
```
> ⚠️ The `token` value is returned **once only**. Store it in the device's secure keychain immediately.
### POST /api/mobile/register-device
Registers an Expo push token for the authenticated user.
**Request:**
```json
{
"push_token": "ExponentPushToken[xxxxxxxxxxxxxxxxxxxxxx]",
"device_name": "John's iPhone",
"platform": "ios"
}
```
Supported platforms: `ios`, `android`, `web`.
Re-registering the same token is safe (idempotent).
### GET /api/mobile/whoami
Returns the current user's profile.
**Response (200):**
```json
{
"owner_id": "john@example.com",
"display_name": "John Doe",
"email": "john@example.com",
"avatar_url": "https://www.gravatar.com/avatar/...",
"is_admin": false
}
```
## Configuration
No server-side configuration is required to enable the mobile app. The Expo push notification routing does not need FCM or APNs credentials on the server.
If you wish to use **direct FCM/APNs** without Expo's relay, replace the `send_expo_push_notification` function in `app/utils/push_notification.py` with your own implementation.
## Project Structure (mobile/)
```
mobile/
├── App.tsx # Root component
├── app.json # Expo/EAS configuration
├── eas.json # EAS Build profiles
├── package.json
├── tsconfig.json
└── src/
├── context/
│ ├── AuthContext.tsx # Auth state + SSO login flow
│ └── ShareContext.tsx # Shared-file queue (iOS Share Sheet / Android Intent)
├── hooks/
│ └── usePushNotifications.ts # Push token registration
├── screens/
│ ├── LoginScreen.tsx # Server URL + SSO button
│ ├── UploadScreen.tsx # Camera capture + photo library + file picker
│ ├── FilesScreen.tsx # Processed document list
│ └── ProfileScreen.tsx # User profile + sign out
└── services/
└── api.ts # DocuElevate REST API client
```
## Troubleshooting
### "Session expired Local session" during iOS build
EAS stores an Apple ID session locally (in `~/.expo/`) to manage code-signing certificates and provisioning profiles. This session expires after a few weeks.
**To fix:**
1. **Refresh the session** by running `eas credentials` and re-authenticating with your Apple ID.
2. **Recommended for automation:** Replace the Apple ID session with an [App Store Connect API Key](https://docs.expo.dev/app-signing/app-credentials/#app-store-connect-api-key). API keys do not expire and work fully non-interactively:
- Create a key at [appstoreconnect.apple.com → Users → Integrations → Keys](https://appstoreconnect.apple.com/access/integrations/api)
- Download the `.p8` file and note the **Key ID** and **Issuer ID**
- Run `eas credentials` → iOS → *Add an App Store Connect API key*
- Upload the `.p8` file when prompted
Once an API key is configured in EAS, automated builds (including CI and EAS Cloud Workflows) will no longer prompt for a password.
### Node.js deprecation warning `[DEP0169]` during EAS build
```
(node:XXXXX) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized…
```
This warning is emitted by **EAS CLI** (an external tool) when it runs on **Node.js 22 or later**, which deprecates `url.parse()`. It does not indicate a problem in the DocuElevate mobile app itself and will not cause a build failure on its own.
The `eas.json` build profiles already include `"NODE_NO_WARNINGS": "1"` in their `env` sections to suppress this warning during EAS Cloud builds. For local builds with a system Node.js ≥ 22, suppress it by running:
```bash
NODE_NO_WARNINGS=1 eas build --platform ios
```
or by activating the project's pinned Node.js version first:
```bash
cd mobile
nvm use # reads .nvmrc → Node 20.19.4 (no deprecation warning)
eas build --platform ios
```
### "Authentication was cancelled or failed"
- Ensure the server URL is correct (including `https://`).
- Verify the server is reachable from your device's network.
- Confirm that `AUTH_ENABLED=True` on the server.
### Push notifications not arriving
1. Check that the app has notification permission (Settings → DocuElevate → Notifications).
2. Verify the device is registered: `GET /api/mobile/devices`.
3. Ensure the server can reach `https://exp.host` (outbound HTTPS on port 443).
4. On Android, add `google-services.json` to the `mobile/` directory if you are building your own binary.
### "Connection refused" or timeout
- Verify that the DocuElevate server is running and accessible.
- Ensure the server's `EXTERNAL_HOSTNAME` or reverse proxy is configured correctly.
- Check that the server accepts CORS requests from `docuelevate://`.
## Related Documentation
- [API Documentation](./API.md)
- [Configuration Guide](./ConfigurationGuide.md)
- [Deployment Guide](./DeploymentGuide.md)
docs/SocialLoginSetup.md
+375
View File
@@ -0,0 +1,375 @@
# Social Login Setup Guide
This guide explains how to configure social login providers (Google, Microsoft, Apple, Dropbox) for DocuElevate. Social login lets your users sign in with their existing accounts, reducing friction and eliminating the need for separate passwords.
## Overview
DocuElevate supports four social login providers:
| Provider | Protocol | Best For |
|----------|----------|----------|
| **Google** | OAuth2 / OpenID Connect | Consumers and Google Workspace organizations |
| **Microsoft** | OAuth2 / OpenID Connect | Microsoft 365 / Azure AD organizations and personal Microsoft accounts |
| **Apple** | OAuth2 / OpenID Connect | iOS/macOS users, privacy-focused users |
| **Dropbox** | OAuth2 | Teams already using Dropbox as a storage destination |
Each provider is **independently enabled** — you can use one, several, or all of them at the same time. Social login works alongside any other DocuElevate authentication method (simple auth, OIDC/Authentik, local signup).
## Prerequisites
Before configuring any social login provider, ensure:
1. **Authentication is enabled**: `AUTH_ENABLED=true` in your `.env` file
2. **Session secret is set**: `SESSION_SECRET` must be a random string of at least 32 characters
3. **HTTPS is configured**: All social login providers require HTTPS redirect URIs in production. Use a reverse proxy (Traefik, Nginx, Caddy) with a valid TLS certificate
4. **External hostname is set**: `EXTERNAL_HOSTNAME` must match your public domain (e.g., `docuelevate.example.com`)
> **Note:** Social login users are regular (non-admin) users by default. To grant admin access, use the Admin Panel (**Settings → User Management**) after the user's first login, or configure admin groups via Authentik/OIDC.
## Callback URLs
Each social login provider uses a callback URL to redirect users back to DocuElevate after authentication. The callback URL pattern is:
```
https://<EXTERNAL_HOSTNAME>/social-callback/<provider>
```
For example, if your DocuElevate instance is at `https://docuelevate.example.com`:
| Provider | Callback URL |
|----------|-------------|
| Google | `https://docuelevate.example.com/social-callback/google` |
| Microsoft | `https://docuelevate.example.com/social-callback/microsoft` |
| Apple | `https://docuelevate.example.com/social-callback/apple` |
| Dropbox | `https://docuelevate.example.com/social-callback/dropbox` |
---
## Google Sign-In
### 1. Create OAuth Credentials in Google Cloud Console
1. Go to the [Google Cloud Console](https://console.cloud.google.com/)
2. Create a new project (or select an existing one)
3. Navigate to **APIs & Services → Credentials**
4. Click **Create Credentials → OAuth client ID**
5. If prompted, configure the **OAuth consent screen** first:
- **User Type**: External (or Internal for Google Workspace)
- **App name**: DocuElevate
- **User support email**: Your email
- **Authorized domains**: Your domain (e.g., `example.com`)
- **Scopes**: Add `email`, `profile`, and `openid`
6. Back on the Credentials page, create an **OAuth 2.0 Client ID**:
- **Application type**: Web application
- **Name**: DocuElevate
- **Authorized redirect URIs**: `https://docuelevate.example.com/social-callback/google`
7. Note the **Client ID** and **Client Secret**
### 2. Configure DocuElevate
Add to your `.env` file:
```bash
SOCIAL_AUTH_GOOGLE_ENABLED=true
SOCIAL_AUTH_GOOGLE_CLIENT_ID=123456789-abcdefg.apps.googleusercontent.com
SOCIAL_AUTH_GOOGLE_CLIENT_SECRET=GOCSPX-your-secret-here
```
### 3. Restart DocuElevate
```bash
docker compose restart api worker
```
### Google-Specific Notes
- **Google Workspace**: If you want to restrict sign-in to users in your Google Workspace organization, set the OAuth consent screen to "Internal"
- **Verification**: Google may require app verification if you're using External user type and requesting sensitive scopes. For small teams (<100 users), you can add test users instead
- **Unified Auth**: If you also use Google Drive as a storage destination, users who sign in with Google will already be authenticated with a Google identity — simplifying the Google Drive integration experience
---
## Microsoft Sign-In (Azure AD / Microsoft Entra ID)
### 1. Register an Application in Azure
1. Go to the [Azure Portal](https://portal.azure.com/)
2. Navigate to **Microsoft Entra ID → App registrations**
3. Click **New registration**
4. Fill in:
- **Name**: DocuElevate
- **Supported account types**: Choose based on your needs:
- *Accounts in this organizational directory only* — single-tenant (your org only)
- *Accounts in any organizational directory* — multi-tenant
- *Accounts in any organizational directory and personal Microsoft accounts* — broadest reach
- **Redirect URI**: Select **Web** and enter `https://docuelevate.example.com/social-callback/microsoft`
5. Click **Register**
6. Note the **Application (client) ID**
7. Navigate to **Certificates & secrets → New client secret**
8. Add a description and expiration, then click **Add**
9. Note the **Value** (this is your client secret — it's only shown once!)
### 2. Configure API Permissions
1. In your app registration, go to **API permissions**
2. Ensure these permissions are present (they're usually added by default):
- `openid`
- `profile`
- `email`
3. Click **Grant admin consent** if you're a tenant admin
### 3. Configure DocuElevate
Add to your `.env` file:
```bash
SOCIAL_AUTH_MICROSOFT_ENABLED=true
SOCIAL_AUTH_MICROSOFT_CLIENT_ID=12345678-abcd-efgh-ijkl-123456789012
SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=your~client~secret~value
SOCIAL_AUTH_MICROSOFT_TENANT=common
```
**Tenant options:**
| Value | Who Can Sign In |
|-------|----------------|
| `common` | Any Microsoft account (personal + any Azure AD organization) |
| `organizations` | Any Azure AD organization (work/school accounts only) |
| `consumers` | Personal Microsoft accounts only (outlook.com, hotmail.com, etc.) |
| `<tenant-id>` | Only users in a specific Azure AD tenant (use the GUID from Azure Portal) |
### 4. Restart DocuElevate
```bash
docker compose restart api worker
```
### Microsoft-Specific Notes
- **Client secret expiration**: Azure AD client secrets expire (max 2 years). Set a calendar reminder to rotate them before they expire
- **Conditional Access**: If your organization uses Azure AD Conditional Access policies, social login will respect them
- **Unified Auth**: If you also use OneDrive as a storage destination, users who sign in with Microsoft will already have a Microsoft identity — potentially simplifying OneDrive integration
---
## Apple Sign-In
Apple Sign-In requires an Apple Developer account ($99/year) and more setup than other providers.
### 1. Configure in Apple Developer Portal
1. Go to the [Apple Developer Portal](https://developer.apple.com/account/)
2. Navigate to **Certificates, Identifiers & Profiles → Identifiers**
3. Click **+** and select **App IDs** → Register an App ID:
- **Description**: DocuElevate
- **Bundle ID**: e.g., `com.example.docuelevate`
- Enable **Sign In with Apple** capability
4. Click **+** again and select **Services IDs**:
- **Description**: DocuElevate Web
- **Identifier**: e.g., `com.example.docuelevate.web` (this is your Client ID)
- Enable **Sign In with Apple**
- Click **Configure** next to Sign In with Apple:
- **Primary App ID**: Select the App ID created above
- **Domains**: `docuelevate.example.com`
- **Return URLs**: `https://docuelevate.example.com/social-callback/apple`
5. Click **Save** and **Continue** → **Register**
6. Navigate to **Keys** → Click **+** to create a new key:
- **Key Name**: DocuElevate Sign-In
- Enable **Sign In with Apple**
- Click **Configure** and select the App ID created above
- Click **Continue** → **Register**
- **Download the private key file** (`.p8`) — you can only download it once!
- Note the **Key ID**
7. Note your **Team ID** (shown in the top-right corner of the Developer Portal)
### 2. Configure DocuElevate
Add to your `.env` file:
```bash
SOCIAL_AUTH_APPLE_ENABLED=true
SOCIAL_AUTH_APPLE_CLIENT_ID=com.example.docuelevate.web
SOCIAL_AUTH_APPLE_TEAM_ID=ABCDE12345
SOCIAL_AUTH_APPLE_KEY_ID=FGHIJ67890
SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----
MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQg...
...your key content here...
-----END PRIVATE KEY-----"
```
> **Tip:** You can also store the private key as a single line with `\n` for line breaks:
> ```bash
> SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIGTAgEAMBMG...\n-----END PRIVATE KEY-----"
> ```
### 3. Restart DocuElevate
```bash
docker compose restart api worker
```
### Apple-Specific Notes
- **Email relay**: Apple offers a "Hide My Email" feature that provides a relay email address (e.g., `abc123@privaterelay.appleid.com`). DocuElevate accepts these addresses
- **First login only**: Apple sends the user's name only on the very first authorization. If the user revokes and re-authorizes, their name may not be sent again
- **Developer account required**: You need an Apple Developer account ($99/year) to use Sign In with Apple
- **Key rotation**: Apple private keys don't expire, but if you suspect compromise, revoke the key in the Developer Portal and create a new one
---
## Dropbox Sign-In
### 1. Create a Dropbox App
1. Go to the [Dropbox App Console](https://www.dropbox.com/developers/apps)
2. Click **Create app**
3. Choose:
- **API**: Scoped access
- **Access type**: Full Dropbox (or App folder, depending on your needs)
- **Name**: DocuElevate Auth (or reuse your existing Dropbox storage app)
4. In the app settings, go to the **OAuth 2** section:
- Add **Redirect URI**: `https://docuelevate.example.com/social-callback/dropbox`
5. Note the **App key** (this is your Client ID) and **App secret** (this is your Client Secret)
> **Tip:** If you already have a Dropbox app configured for DocuElevate's storage integration, you can reuse the same app — just add the social login redirect URI. Alternatively, create a separate app for authentication to keep concerns separated.
### 2. Configure DocuElevate
Add to your `.env` file:
```bash
SOCIAL_AUTH_DROPBOX_ENABLED=true
SOCIAL_AUTH_DROPBOX_CLIENT_ID=your_dropbox_app_key
SOCIAL_AUTH_DROPBOX_CLIENT_SECRET=your_dropbox_app_secret
```
### 3. Restart DocuElevate
```bash
docker compose restart api worker
```
### Dropbox-Specific Notes
- **Unified Auth**: If you also use Dropbox as a storage destination, authenticating via Dropbox establishes the user's Dropbox identity — making it easier to manage Dropbox storage integration
- **App review**: Dropbox may require app review for production apps with more than 50 users. See [Dropbox App Review](https://www.dropbox.com/developers/reference/developer-guide#app-review)
- **Personal vs. Business**: The same app works for both personal Dropbox and Dropbox Business accounts
---
## Unified Authentication and Storage
One of the key advantages of social login in DocuElevate is the potential for **unified authentication** — using the same identity for both signing in and accessing cloud storage destinations:
| Social Login Provider | Related Storage Destination | Benefit |
|---|---|---|
| Google | Google Drive | User already has a Google identity for Drive integration |
| Microsoft | OneDrive | User already has a Microsoft identity for OneDrive integration |
| Dropbox | Dropbox | User already has a Dropbox identity for Dropbox integration |
| Apple | *(none)* | Provides a familiar, privacy-respecting login option |
When a user signs in with a social provider that matches a configured storage destination, the administrator can leverage the same OAuth credentials or simplify the integration setup. Note that the storage integration credentials are configured separately in the admin settings — social login establishes the user's identity, not their storage permissions.
## Combining Multiple Auth Methods
DocuElevate supports running multiple authentication methods simultaneously:
```
┌──────────────────────────────────────────────────┐
│ Login Page │
├──────────────────────────────────────────────────┤
│ Username / Password form (always shown) │
│ │
│ ─── Or continue with ─── │
│ │
│ [Authentik SSO] (if OIDC configured) │
│ [Sign in with Google] (if Google enabled) │
│ [Sign in with Microsoft] (if Microsoft enabled) │
│ [Sign in with Apple] (if Apple enabled) │
│ [Sign in with Dropbox] (if Dropbox enabled) │
│ │
│ [Create account] (if local signup enabled) │
└──────────────────────────────────────────────────┘
```
All methods create or update the same `UserProfile` record, so a user is consistently identified regardless of how they sign in.
## Admin Management
Social login users appear in the **Admin → User Management** panel like any other user. Admins can:
- View which provider a user authenticated with
- Block or unblock social login users
- Set upload limits and subscription tiers
- Grant admin privileges (social login users are never automatically admin)
## Security Considerations
1. **HTTPS is required**: All social login providers require HTTPS callback URLs in production
2. **Credentials are sensitive**: Store client secrets securely — use environment variables, never commit them to source control
3. **Least privilege**: Only request the scopes you need (DocuElevate requests `openid`, `profile`, and `email`)
4. **Rotate secrets**: Set calendar reminders to rotate OAuth client secrets before they expire (especially Microsoft, which has a max 2-year expiration)
5. **Monitor logins**: Check the DocuElevate audit log for unusual login patterns
6. **Social login users are not admins**: Admin access must be explicitly granted by an existing admin
## Troubleshooting
### Common Issues
1. **"Unknown social provider" error**
- The provider is not enabled or credentials are missing
- Check that `SOCIAL_AUTH_<PROVIDER>_ENABLED=true` is set
- Verify client ID and secret are configured
2. **"Could not retrieve email from provider" error**
- The provider didn't return an email address
- For Google: Ensure `email` scope is included (it is by default)
- For Apple: User may have chosen "Hide My Email" — this is expected and should still work
- For Dropbox: Ensure the app has permission to read the user's email
3. **Redirect URI mismatch**
- The callback URL registered with the provider must exactly match what DocuElevate generates
- Check your `EXTERNAL_HOSTNAME` setting
- Ensure you're using HTTPS in production
- The callback URL format is: `https://<EXTERNAL_HOSTNAME>/social-callback/<provider>`
4. **"Social login failed" error**
- Check DocuElevate logs (`docker compose logs api`) for detailed error messages
- Verify the provider's OAuth app is not suspended or in development mode
- For Google: Check if the OAuth consent screen needs verification
- For Microsoft: Ensure admin consent was granted for the required permissions
5. **User can't log in after changing provider settings**
- After changing social login configuration, restart DocuElevate: `docker compose restart api worker`
- Social login settings require a restart to take effect (`restart_required: true`)
### Debug Checklist
- [ ] `AUTH_ENABLED=true` is set
- [ ] `SESSION_SECRET` is at least 32 characters
- [ ] `EXTERNAL_HOSTNAME` matches your public domain
- [ ] Provider-specific `_ENABLED=true` is set
- [ ] Client ID and secret are correctly configured (no extra spaces)
- [ ] Callback URL is registered with the provider
- [ ] HTTPS is working on your domain
- [ ] DocuElevate has been restarted after configuration changes
## Environment Variable Reference
| Variable | Required | Description |
|---|---|---|
| `SOCIAL_AUTH_GOOGLE_ENABLED` | No | Enable Google Sign-In (`true`/`false`). Default: `false` |
| `SOCIAL_AUTH_GOOGLE_CLIENT_ID` | When Google enabled | Google OAuth2 client ID |
| `SOCIAL_AUTH_GOOGLE_CLIENT_SECRET` | When Google enabled | Google OAuth2 client secret |
| `SOCIAL_AUTH_MICROSOFT_ENABLED` | No | Enable Microsoft Sign-In (`true`/`false`). Default: `false` |
| `SOCIAL_AUTH_MICROSOFT_CLIENT_ID` | When Microsoft enabled | Azure AD application (client) ID |
| `SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET` | When Microsoft enabled | Azure AD client secret |
| `SOCIAL_AUTH_MICROSOFT_TENANT` | No | Azure AD tenant. Default: `common` |
| `SOCIAL_AUTH_APPLE_ENABLED` | No | Enable Apple Sign-In (`true`/`false`). Default: `false` |
| `SOCIAL_AUTH_APPLE_CLIENT_ID` | When Apple enabled | Apple Services ID |
| `SOCIAL_AUTH_APPLE_TEAM_ID` | When Apple enabled | Apple Developer Team ID |
| `SOCIAL_AUTH_APPLE_KEY_ID` | When Apple enabled | Apple Sign-In key ID |
| `SOCIAL_AUTH_APPLE_PRIVATE_KEY` | When Apple enabled | Apple Sign-In private key (PEM) |
| `SOCIAL_AUTH_DROPBOX_ENABLED` | No | Enable Dropbox Sign-In (`true`/`false`). Default: `false` |
| `SOCIAL_AUTH_DROPBOX_CLIENT_ID` | When Dropbox enabled | Dropbox App Key |
| `SOCIAL_AUTH_DROPBOX_CLIENT_SECRET` | When Dropbox enabled | Dropbox App Secret |
docs/StorageArchitecture.md
+1
View File
@@ -348,6 +348,7 @@ in task messages or logs.
| `PAPERLESS` | Paperless-ngx REST API, API token |
| `EMAIL` | SMTP/STARTTLS, file as attachment |
| `RCLONE` | `rclone copyto` subprocess, per-user rclone config |
| `ICLOUD` | pyicloud library, Apple ID + app-specific password |
### Multiple Destinations
docs/Troubleshooting.md
+197 -39
View File
@@ -2,6 +2,8 @@
This document provides solutions to common problems encountered when using DocuElevate.
> **Tip:** For configuration-specific issues, see also the [Configuration Troubleshooting](ConfigurationTroubleshooting.md) guide.
## Common Issues
### Application Won't Start
@@ -14,7 +16,7 @@ This document provides solutions to common problems encountered when using DocuE
#### Possible Solutions
1. **Check environment variables**
```bash
docker-compose config
docker compose config
```
Ensure all required variables are set properly in your `.env` file.
@@ -30,6 +32,18 @@ This document provides solutions to common problems encountered when using DocuE
```
Ensure the port isn't already in use by another application.
4. **Check Redis connectivity**
```bash
docker compose logs redis
```
Ensure Redis is running — both the API server and Celery worker depend on it.
5. **Check database migrations**
```bash
docker compose exec api alembic upgrade head
```
Ensure the database schema is up-to-date.
### Document Upload Fails
#### Symptoms
@@ -39,41 +53,51 @@ This document provides solutions to common problems encountered when using DocuE
#### Possible Solutions
1. **Check file size limits**
- Default maximum file size is 100MB
- Adjust `client_max_body_size` in your reverse proxy configuration
- Default maximum file size is 1 GB (`MAX_UPLOAD_SIZE`)
- Individual file limit: `MAX_SINGLE_FILE_SIZE` (default: same as `MAX_UPLOAD_SIZE`)
- If using a reverse proxy, adjust `client_max_body_size` (Nginx) or equivalent
2. **Verify storage space**
```bash
df -h
```
Ensure there's sufficient disk space available.
Ensure there's sufficient disk space on the workdir volume.
3. **Check worker process**
```bash
docker-compose logs worker
docker compose logs worker
```
Verify the Celery worker is running and processing tasks.
4. **Check upload quota**
If multi-user mode and subscriptions are enabled, verify the user hasn't exceeded their daily upload limit (`DEFAULT_DAILY_UPLOAD_LIMIT`).
### OCR or Text Extraction Issues
#### Symptoms
- Documents upload but text isn't extracted
- Poor quality text extraction
- API errors related to Azure services
- API errors related to OCR services
#### Possible Solutions
1. **Verify API credentials**
Check the Azure Document Intelligence API key and endpoint in your `.env` file.
1. **Verify the configured OCR provider**
Check which provider is set via the `OCR_PROVIDER` environment variable (defaults to Azure Document Intelligence).
2. **Check document quality**
2. **Verify API credentials**
Check the credentials for your configured OCR provider in your `.env` file:
- **Azure**: `AZURE_DI_KEY` and `AZURE_DI_ENDPOINT`
- **Tesseract**: No credentials required (local), but ensure `TESSERACT_LANGUAGES` is set
- **EasyOCR**: No credentials required (local)
- **Mistral**: `MISTRAL_OCR_API_KEY`
- **Google Document AI**: `GOOGLE_DOCAI_PROJECT_ID`, `GOOGLE_DOCAI_LOCATION`, `GOOGLE_DOCAI_PROCESSOR_ID`
- **AWS Textract**: `AWS_TEXTRACT_ACCESS_KEY_ID`, `AWS_TEXTRACT_SECRET_ACCESS_KEY`, `AWS_TEXTRACT_REGION`
3. **Check document quality**
- Ensure documents are clearly scanned
- Try preprocessing images to improve quality before upload
3. **Test API connectivity**
```bash
curl -X GET -H "Ocp-Apim-Subscription-Key: YOUR_KEY" "YOUR_ENDPOINT"
```
Ensure the API is accessible from your server.
4. **Try multi-provider OCR**
Configure `OCR_PROVIDERS` (comma-separated list) with a merge strategy (`OCR_MERGE_STRATEGY`: `ai_merge`, `longest`, or `primary`) for better results.
### Email Integration Problems
@@ -84,92 +108,226 @@ This document provides solutions to common problems encountered when using DocuE
#### Possible Solutions
1. **Verify IMAP settings**
Check host, port, username, and password in your configuration.
Check host, port, username, and password for `IMAP1_*` / `IMAP2_*` in your configuration.
2. **Test IMAP connectivity**
```bash
telnet mail.example.com 993
docker compose exec api python -c "import imaplib; m = imaplib.IMAP4_SSL('mail.example.com', 993); print('OK')"
```
Ensure the IMAP server is accessible.
Ensure the IMAP server is accessible from the container.
3. **Enable less secure apps**
For Gmail and some providers, you may need to enable access for less secure apps or use app-specific passwords.
3. **Check for app-specific passwords**
For Gmail and some providers, you must use app-specific passwords instead of your account password.
4. **Check firewall settings**
Ensure your server can make outbound connections to the mail server.
Ensure your server can make outbound connections to the mail server on port 993 (IMAP SSL).
5. **Check attachment filter**
If only certain attachments are expected, verify `IMAP_ATTACHMENT_FILTER` is set correctly (`documents_only` or `all`).
### Storage Integration Issues
#### Symptoms
- Files aren't appearing in Dropbox/Nextcloud/Paperless
- Files aren't appearing in configured storage destinations
- Authentication errors in logs
- API rate limiting errors
#### Possible Solutions
1. **Verify API credentials**
Double-check all API keys, tokens, and secrets.
Double-check all API keys, tokens, and secrets for the relevant service.
2. **Check access permissions**
Ensure the application has write permissions to the specified folders.
Ensure the application has write permissions to the specified folders/buckets.
3. **Refresh tokens**
For OAuth-based services like Dropbox, try generating new refresh tokens.
For OAuth-based services like Dropbox, Google Drive, and OneDrive, try re-authorizing through the integration setup pages.
4. **Examine detailed logs**
```bash
docker-compose logs worker | grep -i dropbox
docker compose logs worker | grep -i "upload_to"
```
Look for specific error messages related to the service.
Look for specific error messages related to the storage service.
5. **Check integration status**
Visit the **Integrations** page in the web UI to verify the connection status of each configured storage backend.
## Search Issues
### Symptoms
- Search returns no results or incomplete results
- Search page shows an error
### Possible Solutions
1. **Check Meilisearch is running**
```bash
docker compose logs meilisearch
```
Ensure the Meilisearch container is healthy and accepting connections.
2. **Verify Meilisearch URL**
Check `MEILISEARCH_URL` in your `.env` file (default: `http://meilisearch:7700`).
3. **Rebuild the search index**
If documents are missing from search results, reprocessing them will re-index their content.
## Pipeline & Routing Issues
### Symptoms
- Documents are not processed according to pipeline steps
- Routing rules don't match expected documents
### Possible Solutions
1. **Verify pipeline assignment**
On the file detail page, check which pipeline (if any) is assigned. The system pipeline applies to all documents by default.
2. **Test routing rules**
Use the **Evaluate** button on the Routing Rules page to test whether a rule matches a specific document.
3. **Check step ordering**
Pipeline steps execute in order — ensure OCR comes before metadata extraction if the AI step depends on extracted text.
## Database Issues
#### Symptoms
### Symptoms
- Application errors related to database connections
- Missing or corrupt data
- Slow performance
#### Possible Solutions
### Possible Solutions
1. **Check database connection string**
Verify the `DATABASE_URL` variable in your `.env` file.
2. **Inspect database integrity**
For SQLite:
```bash
sqlite3 database.db "PRAGMA integrity_check;"
```
(For SQLite databases)
For PostgreSQL (recommended for production):
```bash
docker compose exec api python -c "from app.database import engine; print(engine.url)"
```
3. **Perform database migrations**
```bash
docker-compose exec api alembic upgrade head
docker compose exec api alembic upgrade head
```
Ensure the database schema is up-to-date.
4. **Consider PostgreSQL for production**
SQLite is suitable for small deployments, but PostgreSQL is recommended for multi-user production environments. See the [Database Configuration Guide](DatabaseConfiguration.md).
## Authentication Problems
#### Symptoms
### Symptoms
- Unable to log in
- Redirect loops during authentication
- OAuth errors
#### Possible Solutions
1. **Verify Authentik configuration**
Check client ID, client secret, and configuration URL.
### Possible Solutions
1. **Verify OAuth/OIDC configuration**
Check client ID, client secret, and configuration URL for your identity provider.
2. **Check callback URLs**
Ensure the redirect URIs are correctly configured in your OAuth provider.
Ensure the redirect URIs are correctly configured in your OAuth provider. The callback URL is typically `https://your-domain/auth/callback`.
3. **Clear browser cookies and cache**
Authentication issues can sometimes be resolved by clearing browser data.
4. **Check social login credentials**
If using social login (Google, Microsoft, Apple, Dropbox), verify the corresponding `SOCIAL_AUTH_*` environment variables.
5. **Verify `EXTERNAL_HOSTNAME`**
The `EXTERNAL_HOSTNAME` setting must match the domain users access DocuElevate from — OAuth redirect URLs depend on it.
## Mobile App Issues
### Symptoms
- Can't connect to DocuElevate from the mobile app
- Push notifications not received
- Login fails
### Possible Solutions
1. **Verify the server URL**
Ensure the mobile app is configured with the correct DocuElevate server URL (including `https://`).
2. **Check API token**
Generate a fresh API token from the web UI (Profile → API Tokens) and enter it in the mobile app settings.
3. **Check network connectivity**
The mobile device must be able to reach your DocuElevate server. If using a private network, ensure VPN is connected.
4. **Push notifications**
Push notifications require a valid Expo push token. Check the app settings and ensure notifications are enabled at the OS level.
See the [Mobile App Guide](MobileApp.md) for detailed setup instructions.
## CLI Issues
### Symptoms
- CLI commands fail with connection errors
- Authentication rejected
### Possible Solutions
1. **Verify URL and token**
```bash
docuelevate --url https://your-instance --token de_xxx list
```
Ensure the URL is correct (include the scheme) and the API token is valid.
2. **Check environment variables**
The CLI reads `DOCUELEVATE_URL` and `DOCUELEVATE_API_TOKEN` from the environment. Verify they are exported.
3. **Test API directly**
```bash
curl -H "Authorization: Bearer de_xxx" https://your-instance/api/files
```
If this fails, the issue is with the server, not the CLI.
See the [CLI Guide](CLIGuide.md) for detailed usage.
## Performance Issues
### Symptoms
- Slow document processing
- High memory usage
- Queue backing up
### Possible Solutions
1. **Check worker concurrency**
The Celery worker processes tasks in parallel. If the queue is backing up, consider scaling workers or adjusting concurrency.
2. **Enable batch throttling**
Set `PROCESSALL_THROTTLE_THRESHOLD` and `PROCESSALL_THROTTLE_DELAY` to prevent overwhelming external APIs.
3. **Monitor the queue**
Visit the **Admin → Queue** page to see pending, active, and failed tasks.
4. **Use PostgreSQL**
SQLite can become a bottleneck under load. Migrate to PostgreSQL for better concurrent performance. See the [Database Configuration Guide](DatabaseConfiguration.md).
5. **Check Redis memory**
```bash
docker compose exec redis redis-cli info memory
```
Ensure Redis has sufficient memory for the task queue and cache.
## Getting Additional Help
If you continue to experience issues after trying these solutions:
1. **Check the logs** for detailed error messages
1. **Check the logs** for detailed error messages:
```bash
docker-compose logs --tail=100
docker compose logs --tail=200
```
2. **Open an issue** on the [GitHub repository](https://github.com/christianlouis/document-processor/issues)
2. **Check the status page** at `/status` in the web UI for an overview of all service connections.
3. **Contact the developer** via the information provided on the About page
3. **Open an issue** on the [GitHub repository](https://github.com/christianlouis/DocuElevate/issues) with:
- A description of the problem
- Relevant log output
- Your DocuElevate version (shown on the About page or in the `VERSION` file)
4. **Consult additional documentation**:
- [Configuration Guide](ConfigurationGuide.md) — All environment variables
- [Configuration Troubleshooting](ConfigurationTroubleshooting.md) — Configuration-specific issues
- [Deployment Guide](DeploymentGuide.md) — Infrastructure and deployment
docs/UserGuide.md
+61
View File
@@ -63,6 +63,15 @@ DocuElevate features a simple navigation system with the following main sections
- **Search**: Dedicated full-text search across all document content
- **About**: Information about DocuElevate
### Other Ways to Use DocuElevate
Beyond the web interface, DocuElevate is available through several additional clients:
- **Mobile App (iOS & Android)** — Capture documents with your phone camera or upload from your photo library. See the [Mobile App Guide](MobileApp.md) for setup and usage.
- **Browser Extension** — Clip web pages or send files to DocuElevate directly from Chrome, Firefox, or Edge. See the [Browser Extension Guide](BrowserExtension.md) for installation.
- **CLI Tool** — Upload, download, search, and manage documents from the command line or scripts. See the [CLI Guide](CLIGuide.md) for details.
- **REST & GraphQL API** — Full programmatic access for automation and integrations. See the [API Documentation](API.md).
## Uploading Documents
DocuElevate provides multiple convenient ways to upload documents to the system.
@@ -626,6 +635,58 @@ Pass no `pipeline_id` to clear the assignment and fall back to the system defaul
Admins can create **system pipelines** that appear in every user's pipeline list. These can be set as the global default so all users benefit from a consistent processing baseline. Navigate to **Pipelines** and check the **System pipeline** box when creating a new one (admin only).
### Conditional routing rules
Routing rules automatically assign incoming documents to the right pipeline
based on their properties — no manual pipeline selection required.
**How it works:**
1. Define one or more routing rules via the API
(`POST /api/routing-rules`).
2. Each rule specifies a **field** to inspect, an **operator** (condition),
a **value** to compare against, and a **target pipeline**.
3. When a document is processed, rules are evaluated **in position order**
(lowest first). The first matching rule wins and the document is routed
to that pipeline.
4. If no rule matches, the document is processed by the default pipeline.
**Available fields:**
| Field | Description |
|-------|-------------|
| `file_type` | MIME type, e.g. `application/pdf` |
| `filename` | Original filename |
| `size` | File size in bytes |
| `document_type` | AI-classified type (Invoice, Contract, …) |
| `category` | Alias for `document_type` |
| `metadata.<key>` | Any key from the AI-extracted metadata JSON |
**Available operators:**
| Operator | Description |
|----------|-------------|
| `equals` / `not_equals` | Exact match (case-insensitive) |
| `contains` / `not_contains` | Substring match (case-insensitive) |
| `regex` | Full Python regex match (case-insensitive) |
| `gt` / `lt` / `gte` / `lte` | Numeric comparison (greater/less than) |
**Example:** Route invoices to one pipeline and large files to another:
```
Rule 1: field=document_type, operator=equals, value=Invoice, target_pipeline=3
Rule 2: field=size, operator=gt, value=1048576, target_pipeline=5
```
With first-match-wins logic, an invoice of any size matches Rule 1 and is
routed to pipeline 3. A non-invoice file larger than 1 MB matches Rule 2
and is routed to pipeline 5. Everything else falls back to the default
pipeline.
You can test your rules without actually routing a document using the
**evaluate** endpoint (`POST /api/routing-rules/evaluate`). For the full
API reference, see [API Documentation](API.md#routing-rules).
## API Access
For programmatic access, DocuElevate provides a comprehensive REST API:
docs/howto/EmailIngestion.md
+48
View File
@@ -63,6 +63,54 @@ DocuElevate will process the following attachment types from emails:
| TIFF | `.tif`, `.tiff` | Common format from older scanners/fax |
| Multi-page TIFF | `.tif` | Full multi-page support |
### Controlling Which Attachment Types Are Ingested
By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments.
#### Global Default (Admin Setting)
Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide fallback when no ingestion profile is assigned to a mailbox:
| Value | Behaviour |
|-------|-----------|
| `documents_only` | **(Default)** Only PDFs and office/document files. Images are skipped. |
| `all` | All supported file types, including images. |
```env
IMAP_ATTACHMENT_FILTER=documents_only
```
#### Ingestion Profiles (Fine-Grained Per-Mailbox Control)
For precise control, you can create **Ingestion Profiles** that let you pick exactly which file-type categories to accept from each mailbox. This is more powerful than the binary global toggle and works independently per mailbox.
**Available categories:**
| Category | File types included |
|----------|---------------------|
| PDF | `.pdf` |
| Microsoft Office | `.doc`, `.docx`, `.xls`, `.xlsx`, `.ppt`, `.pptx`, and macro-enabled variants |
| OpenDocument | `.odt`, `.ods`, `.odp`, `.odg`, `.odf` (LibreOffice / OpenOffice) |
| Text & Data | `.txt`, `.csv`, `.rtf` |
| Web & Markup | `.html`, `.htm`, `.md`, `.markdown` |
| Images | `.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg` |
**Managing profiles:**
1. Go to **Email Ingestion** (`/imap-accounts`)
2. Click **Manage profiles** (or the **+** icon next to the profile dropdown)
3. Create a new profile, give it a name, and tick the categories you want
4. When adding or editing a mailbox, select your profile from the dropdown
Two built-in profiles are always available and cannot be deleted:
- **Documents Only** — PDF, Office, OpenDocument, Text, Web (no images)
- **All Files** — all categories including images
Users can also create unlimited **custom profiles** to mix and match exactly the categories they need per mailbox (e.g. a scanner mailbox that only accepts PDFs, or a finance mailbox that accepts Office and CSV but not images).
Custom profiles are created via the UI or the `/api/imap-profiles/` API.
---
## Setting Up Your Scanner/Device
fix_test7.py
+53
View File
@@ -0,0 +1,53 @@
import re
with open("tests/test_api_saved_searches.py", "r") as f:
content = f.read()
# We need to mock get_current_user in app.api.saved_searches (which is imported from app.auth)
# because saved searches uses `_get_user_id` which calls `get_current_user(request)`.
# But `_get_user_id` is NOT a dependency injected via `Depends`!
# Let's verify `app/api/saved_searches.py` uses `Depends` or just calls it.
# In `app/api/saved_searches.py`:
# def _get_user_id(request: Request) -> str:
# user = get_current_user(request)
# if user:
# return user.get("preferred_username") ...
# It's called directly inside the routes: `user_id = _get_user_id(request)`
# It doesn't use `Depends(_get_user_id)`.
# Ah! But earlier I saw `_get_user_id` wasn't mocked properly. Let's use patch to mock `_get_user_id`.
# Wait, `TestClient` can be given an active session, but `app.auth.get_current_user` uses `request.session.get("user")` or Bearer token.
# Is `AUTH_ENABLED` false? The test env has `os.environ["AUTH_ENABLED"] = "False"` in `tests/conftest.py`.
# If `AUTH_ENABLED` is false, `require_login` is a no-op, and `_get_user_id` falls back to "anonymous".
# Actually, `_get_user_id` returns "anonymous" if `get_current_user(request)` is None.
# If `_OWNER` is "test_user@example.com", we should probably just patch `_get_user_id`.
replacement = """def _make_client(int_engine, owner_id: str = _OWNER):
\"\"\"Return a TestClient with *owner_id* injected as the authenticated user.\"\"\"
from app.main import app
from unittest.mock import patch
def override_db():
Session = sessionmaker(bind=int_engine)
session = Session()
try:
yield session
finally:
session.close()
app.dependency_overrides[get_db] = override_db
with patch("app.api.saved_searches._get_user_id", return_value=owner_id):
with TestClient(app, base_url="http://localhost", raise_server_exceptions=False) as client:
yield client
app.dependency_overrides.clear()"""
content = re.sub(
r"def _make_client\(int_engine, owner_id: str = _OWNER\):.*?(?=@pytest\.fixture\(\)\ndef int_client\(int_engine\):)",
replacement + "\n\n\n",
content,
flags=re.DOTALL
)
with open("tests/test_api_saved_searches.py", "w") as f:
f.write(content)
frontend/static/js/common.js
+31 -16
View File
@@ -165,11 +165,11 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
'flex items-center gap-2 px-2 py-1 rounded-md text-gray-700 hover:text-gray-900 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-blue-500';
btn.setAttribute('aria-haspopup', 'true');
btn.setAttribute('aria-expanded', 'false');
btn.setAttribute('aria-label', `Account menu for ${displayName}`);
btn.setAttribute('aria-label', `${((window.__i18n?.accountMenuFor) || 'Account menu for {name}').replace('{name}', displayName)}`);
const avatar = document.createElement('img');
avatar.src = data.picture;
avatar.alt = 'Avatar';
avatar.alt = window.__i18n.avatar || 'Avatar';
avatar.className = 'w-8 h-8 rounded-full';
const nameSpan = document.createElement('span');
@@ -189,7 +189,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
menu.className =
'absolute right-0 mt-2 w-52 bg-white rounded-md shadow-lg ring-1 ring-black ring-opacity-5 z-50 hidden';
menu.setAttribute('role', 'menu');
menu.setAttribute('aria-label', 'Account menu');
menu.setAttribute('aria-label', window.__i18n.accountMenu || 'Account menu');
// User info header
const header = document.createElement('div');
@@ -207,13 +207,16 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
const linksDiv = document.createElement('div');
linksDiv.className = 'py-1';
linksDiv.appendChild(
_makeMenuLink('/subscription', 'fas fa-layer-group text-indigo-400', 'My Subscription', 'text-gray-700')
_makeMenuLink('/profile', 'fas fa-user-circle text-blue-400', window.__i18n.profileSettings || 'Profile Settings', 'text-gray-700')
);
linksDiv.appendChild(
_makeMenuLink('/api-tokens', 'fas fa-key text-yellow-500', 'API Tokens', 'text-gray-700')
_makeMenuLink('/subscription', 'fas fa-layer-group text-indigo-400', window.__i18n.mySubscription || 'My Subscription', 'text-gray-700')
);
linksDiv.appendChild(
_makeMenuLink('/shared-links', 'fas fa-share-alt text-blue-400', 'Shared Links', 'text-gray-700')
_makeMenuLink('/api-tokens', 'fas fa-key text-yellow-500', window.__i18n.apiTokens || 'API Tokens', 'text-gray-700')
);
linksDiv.appendChild(
_makeMenuLink('/shared-links', 'fas fa-share-alt text-blue-400', window.__i18n.sharedLinks || 'Shared Links', 'text-gray-700')
);
// Divider + Sign Out
@@ -222,7 +225,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
const signOutDiv = document.createElement('div');
signOutDiv.className = 'py-1';
signOutDiv.appendChild(
_makeMenuLink('/logout', 'fas fa-sign-out-alt text-red-400', 'Sign Out', 'text-red-600')
_makeMenuLink('/logout', 'fas fa-sign-out-alt text-red-400', window.__i18n.signOut || 'Sign Out', 'text-red-600')
);
menu.appendChild(header);
@@ -256,7 +259,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
userRow.className = 'flex items-center gap-3 px-3 py-3';
const mAvatar = document.createElement('img');
mAvatar.src = data.picture;
mAvatar.alt = 'Avatar';
mAvatar.alt = window.__i18n.avatar || 'Avatar';
mAvatar.className = 'w-8 h-8 rounded-full flex-shrink-0';
const mUserInfo = document.createElement('div');
mUserInfo.className = 'min-w-0';
@@ -272,6 +275,18 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
userRow.appendChild(mUserInfo);
mobileAuthSection.appendChild(userRow);
// Profile Settings link
const profileLink = document.createElement('a');
profileLink.href = '/profile';
profileLink.className =
'flex items-center px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50';
const profileIcon = document.createElement('i');
profileIcon.className = 'fas fa-user-circle mr-2 text-blue-400';
profileIcon.setAttribute('aria-hidden', 'true');
profileLink.appendChild(profileIcon);
profileLink.appendChild(document.createTextNode(window.__i18n.profileSettings || 'Profile Settings'));
mobileAuthSection.appendChild(profileLink);
// Subscription link
const subLink = document.createElement('a');
subLink.href = '/subscription';
@@ -281,7 +296,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
subIcon.className = 'fas fa-layer-group mr-2 text-indigo-400';
subIcon.setAttribute('aria-hidden', 'true');
subLink.appendChild(subIcon);
subLink.appendChild(document.createTextNode('My Subscription'));
subLink.appendChild(document.createTextNode(window.__i18n.mySubscription || 'My Subscription'));
mobileAuthSection.appendChild(subLink);
// API Tokens link
@@ -293,7 +308,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
tokensIcon.className = 'fas fa-key mr-2 text-yellow-500';
tokensIcon.setAttribute('aria-hidden', 'true');
tokensLink.appendChild(tokensIcon);
tokensLink.appendChild(document.createTextNode('API Tokens'));
tokensLink.appendChild(document.createTextNode(window.__i18n.apiTokens || 'API Tokens'));
mobileAuthSection.appendChild(tokensLink);
// Shared Links link
@@ -305,7 +320,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
sharedLinksIcon.className = 'fas fa-share-alt mr-2 text-blue-400';
sharedLinksIcon.setAttribute('aria-hidden', 'true');
sharedLinksLink.appendChild(sharedLinksIcon);
sharedLinksLink.appendChild(document.createTextNode('Shared Links'));
sharedLinksLink.appendChild(document.createTextNode(window.__i18n.sharedLinks || 'Shared Links'));
mobileAuthSection.appendChild(sharedLinksLink);
// Logout link
@@ -317,7 +332,7 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') {
logoutIcon.className = 'fas fa-sign-out-alt mr-2';
logoutIcon.setAttribute('aria-hidden', 'true');
logoutLink.appendChild(logoutIcon);
logoutLink.appendChild(document.createTextNode('Sign Out'));
logoutLink.appendChild(document.createTextNode(window.__i18n.signOut || 'Sign Out'));
mobileAuthSection.appendChild(logoutLink);
}
} else {
@@ -352,7 +367,7 @@ function _renderLoggedOutAuth(authSection, mobileAuthSection) {
loginLink.href = '/login';
loginLink.className =
'px-3 py-1.5 rounded-md text-sm font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-100 border border-gray-300';
loginLink.textContent = 'Log In';
loginLink.textContent = window.__i18n.logIn || 'Log In';
row.appendChild(loginLink);
if (multiUser) {
@@ -360,7 +375,7 @@ function _renderLoggedOutAuth(authSection, mobileAuthSection) {
startLink.href = startHref;
startLink.className =
'px-3 py-1.5 rounded-md text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500';
startLink.textContent = allowSignup ? 'Sign Up' : 'Get Started';
startLink.textContent = allowSignup ? (window.__i18n.signUp || 'Sign Up') : (window.__i18n.getStarted || 'Get Started');
row.appendChild(startLink);
}
@@ -378,7 +393,7 @@ function _renderLoggedOutAuth(authSection, mobileAuthSection) {
loginIcon.className = 'fas fa-sign-in-alt mr-2';
loginIcon.setAttribute('aria-hidden', 'true');
loginLink.appendChild(loginIcon);
loginLink.appendChild(document.createTextNode('Log In'));
loginLink.appendChild(document.createTextNode(window.__i18n.logIn || 'Log In'));
mobileAuthSection.appendChild(loginLink);
if (multiUser) {
@@ -390,7 +405,7 @@ function _renderLoggedOutAuth(authSection, mobileAuthSection) {
startIcon.className = 'fas fa-arrow-right mr-2';
startIcon.setAttribute('aria-hidden', 'true');
startLink.appendChild(startIcon);
startLink.appendChild(document.createTextNode(allowSignup ? 'Sign Up' : 'Get Started'));
startLink.appendChild(document.createTextNode(allowSignup ? (window.__i18n.signUp || 'Sign Up') : (window.__i18n.getStarted || 'Get Started')));
mobileAuthSection.appendChild(startLink);
}
}

Some files were not shown because too many files have changed in this diff Show More