diff --git a/.env.demo b/.env.demo index f8a9a888..20c6b926 100644 --- a/.env.demo +++ b/.env.demo @@ -5,6 +5,7 @@ REDIS_URL=redis://redis:6379/0 EXTERNAL_HOSTNAME=docuelevate.example.com GOTENBERG_URL=http://gotenberg:3000 ALLOW_FILE_DELETE=true # Allow deletion of file records +COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2) # **UI / Appearance** # Default colour scheme: system (follow OS), light, or dark @@ -174,6 +175,33 @@ AUTHENTIK_CLIENT_SECRET= AUTHENTIK_CONFIG_URL= OAUTH_PROVIDER_NAME="Authentik SSO" +# **Social Login Providers** +# Enable one or more social login providers to let users sign in with existing accounts. +# Each provider requires separate OAuth credentials. See docs/SocialLoginSetup.md for details. + +# Google Sign-In (https://console.cloud.google.com/apis/credentials) +# SOCIAL_AUTH_GOOGLE_ENABLED=false +# SOCIAL_AUTH_GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com +# SOCIAL_AUTH_GOOGLE_CLIENT_SECRET=your-google-client-secret + +# Microsoft Sign-In / Azure AD (https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps) +# SOCIAL_AUTH_MICROSOFT_ENABLED=false +# SOCIAL_AUTH_MICROSOFT_CLIENT_ID=your-microsoft-application-id +# SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=your-microsoft-client-secret +# SOCIAL_AUTH_MICROSOFT_TENANT=common # common | organizations | consumers | + +# Apple Sign-In (https://developer.apple.com/account/resources) +# SOCIAL_AUTH_APPLE_ENABLED=false +# SOCIAL_AUTH_APPLE_CLIENT_ID=com.example.docuelevate +# SOCIAL_AUTH_APPLE_TEAM_ID=ABCDE12345 +# SOCIAL_AUTH_APPLE_KEY_ID=FGHIJ67890 +# SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" + +# Dropbox Sign-In (https://www.dropbox.com/developers/apps) +# SOCIAL_AUTH_DROPBOX_ENABLED=false +# SOCIAL_AUTH_DROPBOX_CLIENT_ID=your-dropbox-app-key +# SOCIAL_AUTH_DROPBOX_CLIENT_SECRET=your-dropbox-app-secret + # **AI/ML Services** # Select your AI provider: openai | azure | anthropic | gemini | ollama | openrouter | portkey | litellm AI_PROVIDER=openai @@ -226,6 +254,7 @@ EMAIL_SENDER=DocuElevate System # These settings are intentionally separate from the shared EMAIL_* settings above. # Configuring EMAIL_HOST for password reset / notifications does NOT automatically # enable the email destination – you must set DEST_EMAIL_HOST to activate it. +# DEST_EMAIL_ENABLED=true # Set to false to disable email delivery without removing credentials DEST_EMAIL_HOST=smtp.example.com DEST_EMAIL_PORT=587 DEST_EMAIL_USERNAME=docuelevate@example.com @@ -310,8 +339,15 @@ IMAP2_DELETE_AFTER_PROCESS=false # Use for pre-production instances that share a mailbox with production. IMAP_READONLY_MODE=false +# Controls which attachment types are ingested from IMAP emails. +# 'documents_only' (default) – PDFs and office files only; images are skipped. +# 'all' – all supported file types including images. +# Per-user IMAP accounts can override this global default. +IMAP_ATTACHMENT_FILTER=documents_only + # **Storage/Document Services** # Amazon S3 +# S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials AWS_REGION=us-east-1 AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -321,12 +357,14 @@ S3_STORAGE_CLASS=STANDARD S3_ACL=private # NextCloud +# NEXTCLOUD_ENABLED=true # Set to false to disable NextCloud uploads without removing credentials NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/ NEXTCLOUD_FOLDER="" NEXTCLOUD_USERNAME= NEXTCLOUD_PASSWORD= # Paperless-ngx +# PAPERLESS_ENABLED=true # Set to false to disable Paperless uploads without removing credentials PAPERLESS_HOST=https://paperless.example.com PAPERLESS_NGX_API_TOKEN= # Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value @@ -343,12 +381,14 @@ PAPERLESS_NGX_API_TOKEN= # PAPERLESS_CUSTOM_FIELDS_MAPPING= # Dropbox +# DROPBOX_ENABLED=true # Set to false to disable Dropbox uploads without removing credentials DROPBOX_APP_KEY= DROPBOX_APP_SECRET= DROPBOX_REFRESH_TOKEN= DROPBOX_FOLDER="/Documents/Uploads" # Google Drive +# GOOGLE_DRIVE_ENABLED=true # Set to false to disable Google Drive uploads without removing credentials # Service Account Method: GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"} GOOGLE_DRIVE_FOLDER_ID= @@ -361,6 +401,7 @@ GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret # Required for OAuth method GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token # Required for OAuth method # OneDrive +# ONEDRIVE_ENABLED=true # Set to false to disable OneDrive uploads without removing credentials ONEDRIVE_CLIENT_ID=your-client-id ONEDRIVE_CLIENT_SECRET=your-client-secret ONEDRIVE_TENANT_ID=common @@ -368,6 +409,7 @@ ONEDRIVE_REFRESH_TOKEN=your-refresh-token ONEDRIVE_FOLDER_PATH=Documents/Uploads # WebDAV +# WEBDAV_ENABLED=true # Set to false to disable WebDAV uploads without removing credentials WEBDAV_URL=https://webdav.example.com/path WEBDAV_USERNAME=webdav_user WEBDAV_PASSWORD=your_secure_webdav_password @@ -375,6 +417,7 @@ WEBDAV_FOLDER=/Documents/Uploads WEBDAV_VERIFY_SSL=True # FTP +# FTP_ENABLED=true # Set to false to disable FTP uploads without removing credentials # Security Note: FTP_USE_TLS=True is strongly recommended for secure connections # Set FTP_ALLOW_PLAINTEXT=False in production to prevent unencrypted FTP FTP_HOST=ftp.example.com @@ -386,6 +429,7 @@ FTP_USE_TLS=True FTP_ALLOW_PLAINTEXT=True # SFTP +# SFTP_ENABLED=true # Set to false to disable SFTP uploads without removing credentials # Security Note: Host key verification is enabled by default (False) # Only set to True in development/testing environments if needed # When false, configure SSH known_hosts for proper host key verification @@ -398,6 +442,16 @@ SFTP_PASSWORD=your_secure_sftp_password SFTP_FOLDER=/Documents/Uploads SFTP_DISABLE_HOST_KEY_VERIFICATION=False # Default is False (secure); set to True only for testing +# iCloud Drive +# ICLOUD_ENABLED=true # Set to false to disable iCloud uploads without removing credentials +# Requires an Apple ID with iCloud Drive enabled. +# For accounts with two-factor authentication (most accounts), generate an +# app-specific password at https://appleid.apple.com/account/manage +ICLOUD_USERNAME=your_apple_id@example.com +ICLOUD_PASSWORD=your-app-specific-password +ICLOUD_FOLDER=Documents/Uploads +# ICLOUD_COOKIE_DIRECTORY=/path/to/cookie/dir # Optional: defaults to ~/.pyicloud + # **HTTP Request Settings** # Timeout for HTTP requests - set higher to handle large PDF files (up to 1GB) HTTP_REQUEST_TIMEOUT=120 # Timeout in seconds (default: 120 for large file operations) @@ -526,3 +580,12 @@ EMBEDDING_MAX_TOKENS=8000 # Attach PII (IP addresses, user agents) to Sentry events. # Disable (default) to stay GDPR/CCPA compliant. # SENTRY_SEND_DEFAULT_PII=false + +# **Mobile App – Push Notifications** +# Push notifications are delivered via Expo's push notification service +# (https://expo.dev/notifications) which routes to APNs (iOS) and FCM (Android). +# No additional credentials are required on the server side. +# The mobile app registers its Expo push token via POST /api/mobile/register-device. +# +# To use native FCM/APNs directly (without Expo relay), replace the +# send_expo_push_notification function in app/utils/push_notification.py. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 677150c2..0c206c46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ concurrency: env: IMAGE_NAME: christianlouis/docuelevate + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: # ══════════════════════════════════════════════════════════════════════════ diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d1f3781c..a7879042 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,6 +8,9 @@ on: schedule: - cron: '37 1 * * 1' +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: analyze: name: Analyze (${{ matrix.language }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56466fcb..275e0565 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,9 @@ permissions: pull-requests: write packages: write +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: release: name: Semantic Release diff --git a/.github/workflows/ruff-auto-fix.yml b/.github/workflows/ruff-auto-fix.yml index a834b523..6e1a6748 100644 --- a/.github/workflows/ruff-auto-fix.yml +++ b/.github/workflows/ruff-auto-fix.yml @@ -16,6 +16,9 @@ permissions: contents: write pull-requests: write +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: ruff-auto-fix: name: Auto-fix Ruff Issues diff --git a/BUILD_DATE b/BUILD_DATE index 68ca454f..db87d3bf 100644 --- a/BUILD_DATE +++ b/BUILD_DATE @@ -1 +1 @@ -2026-03-11T11:44:20Z +2026-03-12T11:43:30Z diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c46f88f..107f76f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,213 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## v0.123.1 (2026-03-12) + +### Bug Fixes + +- **tests**: Fix two failing tests - missing DB table and MagicMock IP address + ([`be6b49c`](https://github.com/christianlouis/DocuElevate/commit/be6b49c8721445dc8ea870afda87f311acb192e3)) + +### Code Style + +- Fix ruff formatting in tests/test_imap_profiles.py + ([`8060a79`](https://github.com/christianlouis/DocuElevate/commit/8060a79b9c5172c40edff87bdcf0e095e177560e)) + + +## v0.123.0 (2026-03-12) + +### Bug Fixes + +- **imap**: Address code review feedback on ingestion profiles + ([`2e08773`](https://github.com/christianlouis/DocuElevate/commit/2e087731b92fe554e54955c97166d61fe644bafc)) + +### Features + +- **imap**: Add attachment type filter for IMAP ingestion + ([`554bb21`](https://github.com/christianlouis/DocuElevate/commit/554bb21d329ca2cca868a61adf337e29e4284ca4)) + +- **imap**: Add ImapIngestionProfile model, API, migration and UI + ([`c9f5544`](https://github.com/christianlouis/DocuElevate/commit/c9f554465d5bf027a92bb4aa56db244aad4a6ff6)) + + +## v0.122.0 (2026-03-12) + +### Documentation + +- **storage**: Document explicit enable/disable flags in ConfigurationGuide and .env.demo + ([`fd75961`](https://github.com/christianlouis/DocuElevate/commit/fd7596158064b68a23df4a89201f456df8f28b38)) + +### Features + +- **storage**: Add explicit enable/disable flag for each global storage destination + ([`b72ab3b`](https://github.com/christianlouis/DocuElevate/commit/b72ab3b31835ab6ab574067499544427c4101231)) + + +## v0.121.1 (2026-03-12) + +### Bug Fixes + +- Remove duplicate Audit Logs nav entry and add login/logout audit log events + ([`1e8f433`](https://github.com/christianlouis/DocuElevate/commit/1e8f433419c5c7b0c551dffe05f107d2bb5a35a1)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`50ef607`](https://github.com/christianlouis/DocuElevate/commit/50ef6072935083554dd52a11cc6438891f177a8b)) + +### Testing + +- **notifications**: Improve coverage for user_notification.py to 100% + ([`78204c2`](https://github.com/christianlouis/DocuElevate/commit/78204c2490ffd9ed0bcb62154c3da54fa9614450)) + + +## Unreleased + +### Testing + +- **notifications**: Improve coverage for user_notification.py to 100% + ([`78204c2`](https://github.com/christianlouis/DocuElevate/commit/78204c2490ffd9ed0bcb62154c3da54fa9614450)) + + +## v0.121.0 (2026-03-12) + +### Bug Fixes + +- **auth**: Address code review feedback - sanitize error messages, remove unused import + ([`5d716ad`](https://github.com/christianlouis/DocuElevate/commit/5d716ad78fdcd92cafa0a7765580ef290cc842fc)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`04cde33`](https://github.com/christianlouis/DocuElevate/commit/04cde33d01d53877fc34a60b887c6978d167a97e)) + +### Features + +- **auth**: Add social login support for Google, Microsoft, Apple, and Dropbox + ([`ac6e052`](https://github.com/christianlouis/DocuElevate/commit/ac6e05278896986ca234e6109e225c599eb982c7)) + +### Testing + +- Improve coverage for app/utils/audit_service.py from 59% to 100% + ([`7438551`](https://github.com/christianlouis/DocuElevate/commit/7438551080dfec645a9e9c9ea8ee059764430e90)) + +- **auth**: Add tests for social login and fix existing config validator tests + ([`9c26d41`](https://github.com/christianlouis/DocuElevate/commit/9c26d412d7710dce47d06969270f184411cd6898)) + + +## Unreleased + +### Testing + +- Improve coverage for app/utils/audit_service.py from 59% to 100% + ([`7438551`](https://github.com/christianlouis/DocuElevate/commit/7438551080dfec645a9e9c9ea8ee059764430e90)) + + +## v0.120.0 (2026-03-12) + +### Bug Fixes + +- Remove accidental pip artifact file and update docs for iCloud Drive + ([`ca84a11`](https://github.com/christianlouis/DocuElevate/commit/ca84a11284a979409db186b20b0381f5025d4fdf)) + +- Remove accidental pip artifact file and update docs for iCloud Drive + ([`528f0a6`](https://github.com/christianlouis/DocuElevate/commit/528f0a624de335b21125b8b86700eb4d85dfed86)) + +- **tests**: Add iCloud mocks to all test files and address code review feedback + ([`50af0ea`](https://github.com/christianlouis/DocuElevate/commit/50af0ea67942e545849bbd745258cbed7822af45)) + +### Features + +- **storage**: Add Apple iCloud Drive storage provider + ([`82d67c5`](https://github.com/christianlouis/DocuElevate/commit/82d67c56b34ec5a849bb3fa46aba0182c7f51dcd)) + +- **storage**: Add Apple iCloud Drive storage provider + ([`30f06e0`](https://github.com/christianlouis/DocuElevate/commit/30f06e0b3292d9b01428fc945913ab91888644a3)) + +### Testing + +- **tasks**: Add _should_upload_to_icloud mock to send_to_all tests + ([`6e50c61`](https://github.com/christianlouis/DocuElevate/commit/6e50c6197082bbf2de2935cca1b74602789383f4)) + +- **tasks**: Add _should_upload_to_icloud mock to send_to_all tests + ([`9bc23aa`](https://github.com/christianlouis/DocuElevate/commit/9bc23aa40b9c8a9c943cf75f1051fac9e38c24fc)) + + +## v0.119.0 (2026-03-12) + + +## v0.118.0 (2026-03-11) + +### Features + +- **api**: Add GraphQL endpoint at /graphql with Strawberry + ([`a41ded5`](https://github.com/christianlouis/DocuElevate/commit/a41ded535f32d4892199208e1cab54cc189fde13)) + + +## v0.117.1 (2026-03-11) + +### Bug Fixes + +- Resolve all 47 failing tests in main + ([`df4c91a`](https://github.com/christianlouis/DocuElevate/commit/df4c91a58661e2ecf9fda15500e4ba1259674168)) + + +## v0.117.0 (2026-03-11) + +### Continuous Integration + +- Opt into Node.js 24 for all GitHub Actions workflows + ([`c306d80`](https://github.com/christianlouis/DocuElevate/commit/c306d80755193e4b12222c73ed4da442a2d5e23c)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`ab19ae5`](https://github.com/christianlouis/DocuElevate/commit/ab19ae57060ff2be8e9172714c5e3b308c45334d)) + +- **changelog**: Update changelog [skip ci] + ([`c5b67fe`](https://github.com/christianlouis/DocuElevate/commit/c5b67fe36464b0756553cc54614d0f77273772ec)) + +- **changelog**: Update changelog [skip ci] + ([`eb22a58`](https://github.com/christianlouis/DocuElevate/commit/eb22a580651b4de477b3b5cc945581776a6775fa)) + + +## Unreleased + +### Continuous Integration + +- Opt into Node.js 24 for all GitHub Actions workflows + ([`c306d80`](https://github.com/christianlouis/DocuElevate/commit/c306d80755193e4b12222c73ed4da442a2d5e23c)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`c5b67fe`](https://github.com/christianlouis/DocuElevate/commit/c5b67fe36464b0756553cc54614d0f77273772ec)) + +- **changelog**: Update changelog [skip ci] + ([`eb22a58`](https://github.com/christianlouis/DocuElevate/commit/eb22a580651b4de477b3b5cc945581776a6775fa)) + + +## Unreleased + +### Continuous Integration + +- Opt into Node.js 24 for all GitHub Actions workflows + ([`c306d80`](https://github.com/christianlouis/DocuElevate/commit/c306d80755193e4b12222c73ed4da442a2d5e23c)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`eb22a58`](https://github.com/christianlouis/DocuElevate/commit/eb22a580651b4de477b3b5cc945581776a6775fa)) + + +## Unreleased + +### Continuous Integration + +- Opt into Node.js 24 for all GitHub Actions workflows + ([`c306d80`](https://github.com/christianlouis/DocuElevate/commit/c306d80755193e4b12222c73ed4da442a2d5e23c)) + + ## v0.116.0 (2026-03-11) ### Documentation diff --git a/GIT_SHA b/GIT_SHA index 40d0eb2d..c827e02b 100644 --- a/GIT_SHA +++ b/GIT_SHA @@ -1 +1 @@ -9f7d6c8 +5d239e9 diff --git a/RUNTIME_INFO b/RUNTIME_INFO index 2c455bf9..79167e94 100644 --- a/RUNTIME_INFO +++ b/RUNTIME_INFO @@ -1,10 +1,10 @@ DocuElevate Build Information ============================== -Version: 0.116.0 -Build Date: 2026-03-11T11:44:20Z -Git Commit: 9f7d6c85488081f546e7c392bc46b2827bb1caa6 -Git Short SHA: 9f7d6c8 +Version: 0.123.1 +Build Date: 2026-03-12T11:43:30Z +Git Commit: 5d239e904bba584a5c02c9ec8d9da110d37d548f +Git Short SHA: 5d239e9 Git Branch: main -Commit Date: 2026-03-11T12:43:59+01:00 -Build Timestamp: 2026-03-11T11:44:20Z +Commit Date: 2026-03-12T12:43:08+01:00 +Build Timestamp: 2026-03-12T11:43:30Z ============================== diff --git a/VERSION b/VERSION index 4c08787e..3b1cb767 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.116.0 +0.123.1 diff --git a/app/api/__init__.py b/app/api/__init__.py index 19aefb3c..28b9fbba 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -12,6 +12,7 @@ from app.api.audit_logs import router as audit_logs_router from app.api.azure import router as azure_router from app.api.backup import router as backup_router from app.api.billing import router as billing_router +from app.api.compliance import router as compliance_router from app.api.database import router as database_router from app.api.diagnostic import router as diagnostic_router from app.api.dropbox import router as dropbox_router @@ -20,8 +21,10 @@ from app.api.files import router as files_router from app.api.google_drive import router as google_drive_router from app.api.i18n import router as i18n_router from app.api.imap_accounts import router as imap_accounts_router +from app.api.imap_profiles import router as imap_profiles_router from app.api.integrations import router as integrations_router from app.api.logs import router as logs_router +from app.api.mobile import router as mobile_router from app.api.notifications import router as notifications_router from app.api.onboarding import router as onboarding_router from app.api.onedrive import router as onedrive_router @@ -81,8 +84,11 @@ router.include_router(onboarding_router) router.include_router(billing_router) router.include_router(pipelines_router) router.include_router(imap_accounts_router) +router.include_router(imap_profiles_router) router.include_router(integrations_router) router.include_router(notifications_router) router.include_router(scheduled_jobs_router) router.include_router(audit_logs_router) router.include_router(i18n_router) +router.include_router(mobile_router) +router.include_router(compliance_router) diff --git a/app/api/compliance.py b/app/api/compliance.py new file mode 100644 index 00000000..811faeaa --- /dev/null +++ b/app/api/compliance.py @@ -0,0 +1,183 @@ +"""API endpoints for managing compliance templates (GDPR, HIPAA, SOC2). + +All endpoints require admin privileges. + +Available routes: + GET /api/compliance/templates – list all compliance templates + GET /api/compliance/templates/{name} – get a single template with checks + POST /api/compliance/templates/{name}/apply – one-click apply a template + GET /api/compliance/templates/{name}/status – evaluate compliance status + GET /api/compliance/summary – overall compliance dashboard data +""" + +import logging +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.database import get_db +from app.utils.compliance_service import ( + COMPLIANCE_TEMPLATES, + apply_template, + evaluate_template_status, + get_all_templates, + get_compliance_summary, + get_template_by_name, +) + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/compliance", tags=["compliance"]) + +DbSession = Annotated[Session, Depends(get_db)] + + +# --------------------------------------------------------------------------- +# Authorisation helper +# --------------------------------------------------------------------------- + + +def _require_admin(request: Request) -> dict: + """Ensure the caller is an admin; raises HTTP 403 otherwise.""" + user = request.session.get("user") + if not user or not user.get("is_admin"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + return user + + +AdminUser = Annotated[dict, Depends(_require_admin)] + + +# --------------------------------------------------------------------------- +# Pydantic response models +# --------------------------------------------------------------------------- + + +class CheckResult(BaseModel): + """Individual compliance check result.""" + + key: str + label: str + description: str + expected: str + actual: str + passing: bool + + +class TemplateStatusResponse(BaseModel): + """Status evaluation for a compliance template.""" + + status: str + total: int + passed: int + failed: int + check_results: list[CheckResult] + + +class TemplateResponse(BaseModel): + """Full compliance template representation.""" + + id: int + name: str + display_name: str + description: str | None + enabled: bool + status: str + applied_at: str | None + applied_by: str | None + settings: dict[str, str] + checks: list[dict[str, Any]] + check_count: int + + +class ApplyResponse(BaseModel): + """Result of applying a compliance template.""" + + success: bool + template: str | None = None + applied_settings: dict[str, str] | None = None + errors: list[str] | None = None + error: str | None = None + status: TemplateStatusResponse | None = None + + +class SummaryTemplateResponse(BaseModel): + """Per-template summary for the compliance dashboard.""" + + name: str + display_name: str + enabled: bool + status: str + total: int + passed: int + failed: int + applied_at: str | None + applied_by: str | None + + +class ComplianceSummaryResponse(BaseModel): + """Overall compliance dashboard summary.""" + + overall_status: str + total_checks: int + total_passed: int + total_failed: int + templates: list[SummaryTemplateResponse] + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/templates", response_model=list[TemplateResponse]) +async def list_templates(db: DbSession, admin: AdminUser) -> list[dict[str, Any]]: + """List all compliance templates with their current status.""" + return get_all_templates(db) + + +@router.get("/templates/{name}", response_model=TemplateResponse) +async def get_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]: + """Get a single compliance template by name.""" + templates = get_all_templates(db) + for t in templates: + if t["name"] == name: + return t + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found") + + +@router.post("/templates/{name}/apply", response_model=ApplyResponse) +async def apply_compliance_template(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]: + """Apply a compliance template (one-click). + + Writes all template settings to the database and evaluates the resulting + compliance status. + """ + if name not in COMPLIANCE_TEMPLATES: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found") + + template = get_template_by_name(db, name) + if template is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found") + + admin_email = admin.get("email", "admin") + result = apply_template(db, name, applied_by=admin_email) + if not result.get("success") and result.get("error"): + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=result["error"]) + return result + + +@router.get("/templates/{name}/status", response_model=TemplateStatusResponse) +async def get_template_status(name: str, db: DbSession, admin: AdminUser) -> dict[str, Any]: + """Evaluate the live compliance status of a template.""" + template = get_template_by_name(db, name) + if template is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Template '{name}' not found") + return evaluate_template_status(db, name) + + +@router.get("/summary", response_model=ComplianceSummaryResponse) +async def compliance_summary(db: DbSession, admin: AdminUser) -> dict[str, Any]: + """Overall compliance dashboard summary across all templates.""" + return get_compliance_summary(db) diff --git a/app/api/graphql_api.py b/app/api/graphql_api.py new file mode 100644 index 00000000..5d4530d5 --- /dev/null +++ b/app/api/graphql_api.py @@ -0,0 +1,431 @@ +""" +GraphQL API endpoint for DocuElevate. + +Provides a flexible query interface alongside the existing REST API. +Schema covers: documents, pipelines, settings, and users. + +Endpoint: /graphql +GraphiQL playground: /graphql (via browser) +""" + +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Annotated, Any + +import strawberry +from fastapi import Depends, Request +from sqlalchemy.orm import Session +from strawberry.fastapi import GraphQLRouter + +from app.auth import get_current_user +from app.config import settings +from app.database import get_db +from app.models import ApplicationSettings, FileRecord, Pipeline, PipelineStep, UserProfile + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Strawberry types +# --------------------------------------------------------------------------- + + +@strawberry.type +class DocumentType: + """A processed document stored in the system.""" + + id: int + owner_id: str | None + original_filename: str | None + local_filename: str + file_size: int + mime_type: str | None + document_title: str | None + is_duplicate: bool + ocr_quality_score: int | None + pipeline_id: int | None + created_at: datetime | None + + +@strawberry.type +class PipelineStepType: + """A single step within a processing pipeline.""" + + id: int + pipeline_id: int + position: int + step_type: str + label: str | None + enabled: bool + created_at: datetime | None + + +@strawberry.type +class PipelineType: + """A processing pipeline with its ordered steps.""" + + id: int + owner_id: str | None + name: str + description: str | None + is_default: bool + is_active: bool + steps: list[PipelineStepType] + created_at: datetime | None + updated_at: datetime | None + + +@strawberry.type +class SettingType: + """An application configuration setting stored in the database.""" + + id: int + key: str + value: str | None + created_at: datetime | None + updated_at: datetime | None + + +@strawberry.type +class UserType: + """A user profile in the system.""" + + id: int + user_id: str + display_name: str | None + is_blocked: bool + subscription_tier: str | None + onboarding_completed: bool + created_at: datetime | None + + +# --------------------------------------------------------------------------- +# Conversion helpers +# --------------------------------------------------------------------------- + + +def _document_from_record(rec: FileRecord) -> DocumentType: + return DocumentType( + id=rec.id, + owner_id=rec.owner_id, + original_filename=rec.original_filename, + local_filename=rec.local_filename, + file_size=rec.file_size, + mime_type=rec.mime_type, + document_title=rec.document_title, + is_duplicate=rec.is_duplicate, + ocr_quality_score=rec.ocr_quality_score, + pipeline_id=rec.pipeline_id, + created_at=rec.created_at, + ) + + +def _pipeline_step_from_record(step: PipelineStep) -> PipelineStepType: + return PipelineStepType( + id=step.id, + pipeline_id=step.pipeline_id, + position=step.position, + step_type=step.step_type, + label=step.label, + enabled=step.enabled, + created_at=step.created_at, + ) + + +def _pipeline_from_record(pipeline: Pipeline, db: Session) -> PipelineType: + steps = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline.id).order_by(PipelineStep.position).all() + return PipelineType( + id=pipeline.id, + owner_id=pipeline.owner_id, + name=pipeline.name, + description=pipeline.description, + is_default=pipeline.is_default, + is_active=pipeline.is_active, + steps=[_pipeline_step_from_record(s) for s in steps], + created_at=pipeline.created_at, + updated_at=pipeline.updated_at, + ) + + +def _setting_from_record(setting: ApplicationSettings) -> SettingType: + return SettingType( + id=setting.id, + key=setting.key, + value=setting.value, + created_at=setting.created_at, + updated_at=setting.updated_at, + ) + + +def _user_from_profile(profile: UserProfile) -> UserType: + return UserType( + id=profile.id, + user_id=profile.user_id, + display_name=profile.display_name, + is_blocked=profile.is_blocked, + subscription_tier=profile.subscription_tier, + onboarding_completed=profile.onboarding_completed, + created_at=profile.created_at, + ) + + +# --------------------------------------------------------------------------- +# Context helpers +# --------------------------------------------------------------------------- + +# Keys that contain sensitive data and must never be returned via GraphQL +_SENSITIVE_SETTING_KEYS: frozenset[str] = frozenset( + { + "openai_api_key", + "azure_ai_key", + "session_secret", + "database_url", + "redis_url", + "dropbox_app_secret", + "dropbox_refresh_token", + "google_drive_credentials_json", + "onedrive_client_secret", + "onedrive_refresh_token", + "smtp_password", + "nextcloud_password", + "s3_secret_access_key", + "ftp_password", + "sftp_password", + "webdav_password", + "stripe_secret_key", + "stripe_webhook_secret", + "sentry_dsn", + "social_auth_google_client_secret", + "social_auth_microsoft_client_secret", + "social_auth_apple_private_key", + "social_auth_dropbox_app_secret", + } +) + + +def _get_current_user_id(user: dict[str, Any] | None) -> str | None: + """Extract the stable user identifier from the user dict.""" + if not user: + return None + return user.get("preferred_username") or user.get("email") or user.get("id") or None + + +def _get_db_and_user(info: strawberry.types.Info) -> tuple[Session, dict[str, Any] | None]: + """Extract the database session and current user from the Strawberry context.""" + db: Session = info.context["db"] + user: dict[str, Any] | None = info.context.get("user") + return db, user + + +def _require_auth(user: dict[str, Any] | None) -> None: + """Raise an error when authentication is enabled and no valid user is present.""" + if settings.auth_enabled and not user: + raise strawberry.exceptions.StrawberryGraphQLError("Authentication required") + + +def _require_admin(user: dict[str, Any] | None) -> None: + """Raise an error when the current user is not an admin. + + When ``auth_enabled`` is *False* (single-user / development mode) all + callers are implicitly treated as administrators. + """ + if not settings.auth_enabled: + # Single-user mode: no auth, treat caller as admin + return + _require_auth(user) + if not (user and user.get("is_admin")): + raise strawberry.exceptions.StrawberryGraphQLError("Admin access required") + + +# --------------------------------------------------------------------------- +# Query resolvers +# --------------------------------------------------------------------------- + + +@strawberry.type +class Query: + """Root query type for the DocuElevate GraphQL API.""" + + @strawberry.field(description="List documents, optionally filtered by owner.") + def documents( + self, + info: strawberry.types.Info, + owner_id: str | None = None, + limit: int = 20, + offset: int = 0, + ) -> list[DocumentType]: + """Return a paginated list of documents. + + When *auth_enabled* the caller must be authenticated. Non-admin users + receive only their own documents; admins may query any *owner_id*. + """ + db, user = _get_db_and_user(info) + _require_auth(user) + + limit = max(1, min(limit, 100)) + offset = max(0, offset) + + query = db.query(FileRecord) + + if settings.auth_enabled and user: + is_admin = user.get("is_admin", False) + current_user_id = _get_current_user_id(user) + if not is_admin: + # Non-admins can only see their own documents + query = query.filter(FileRecord.owner_id == current_user_id) + elif owner_id: + query = query.filter(FileRecord.owner_id == owner_id) + elif owner_id: + query = query.filter(FileRecord.owner_id == owner_id) + + records = query.order_by(FileRecord.created_at.desc()).offset(offset).limit(limit).all() + return [_document_from_record(r) for r in records] + + @strawberry.field(description="Fetch a single document by ID.") + def document(self, info: strawberry.types.Info, id: int) -> DocumentType | None: + """Return one document by its primary key, or *null* if not found.""" + db, user = _get_db_and_user(info) + _require_auth(user) + + rec = db.query(FileRecord).filter(FileRecord.id == id).first() + if rec is None: + return None + + if settings.auth_enabled and user: + is_admin = user.get("is_admin", False) + current_user_id = _get_current_user_id(user) + if not is_admin and rec.owner_id != current_user_id: + return None + + return _document_from_record(rec) + + @strawberry.field(description="List processing pipelines.") + def pipelines( + self, + info: strawberry.types.Info, + owner_id: str | None = None, + limit: int = 20, + offset: int = 0, + ) -> list[PipelineType]: + """Return a paginated list of pipelines.""" + db, user = _get_db_and_user(info) + _require_auth(user) + + limit = max(1, min(limit, 100)) + offset = max(0, offset) + + query = db.query(Pipeline) + + if settings.auth_enabled and user: + is_admin = user.get("is_admin", False) + current_user_id = _get_current_user_id(user) + if not is_admin: + query = query.filter((Pipeline.owner_id == current_user_id) | (Pipeline.owner_id.is_(None))) + elif owner_id: + query = query.filter(Pipeline.owner_id == owner_id) + elif owner_id: + query = query.filter(Pipeline.owner_id == owner_id) + + rows = query.order_by(Pipeline.id).offset(offset).limit(limit).all() + return [_pipeline_from_record(p, db) for p in rows] + + @strawberry.field(description="Fetch a single pipeline by ID.") + def pipeline(self, info: strawberry.types.Info, id: int) -> PipelineType | None: + """Return one pipeline by its primary key, or *null* if not found.""" + db, user = _get_db_and_user(info) + _require_auth(user) + + row = db.query(Pipeline).filter(Pipeline.id == id).first() + if row is None: + return None + + if settings.auth_enabled and user: + is_admin = user.get("is_admin", False) + current_user_id = _get_current_user_id(user) + if not is_admin and row.owner_id is not None and row.owner_id != current_user_id: + return None + + return _pipeline_from_record(row, db) + + @strawberry.field(description="List non-sensitive application settings (admin only).") + def settings( + self, + info: strawberry.types.Info, + limit: int = 50, + offset: int = 0, + ) -> list[SettingType]: + """Return application settings stored in the database. + + Sensitive keys (API secrets, passwords, etc.) are automatically + excluded. Requires admin privileges when auth is enabled. + """ + db, user = _get_db_and_user(info) + _require_admin(user) + + limit = max(1, min(limit, 200)) + offset = max(0, offset) + + rows = ( + db.query(ApplicationSettings) + .filter(ApplicationSettings.key.notin_(_SENSITIVE_SETTING_KEYS)) + .order_by(ApplicationSettings.key) + .offset(offset) + .limit(limit) + .all() + ) + return [_setting_from_record(r) for r in rows] + + @strawberry.field(description="List user profiles (admin only).") + def users( + self, + info: strawberry.types.Info, + limit: int = 20, + offset: int = 0, + ) -> list[UserType]: + """Return a paginated list of user profiles. Requires admin privileges.""" + db, user = _get_db_and_user(info) + _require_admin(user) + + limit = max(1, min(limit, 100)) + offset = max(0, offset) + + rows = db.query(UserProfile).order_by(UserProfile.user_id).offset(offset).limit(limit).all() + return [_user_from_profile(r) for r in rows] + + @strawberry.field(description="Fetch a user profile by user_id (admin only).") + def user(self, info: strawberry.types.Info, user_id: str) -> UserType | None: + """Return one user profile by *user_id*, or *null* if not found.""" + db, user = _get_db_and_user(info) + _require_admin(user) + + row = db.query(UserProfile).filter(UserProfile.user_id == user_id).first() + return _user_from_profile(row) if row else None + + +# --------------------------------------------------------------------------- +# Schema and router +# --------------------------------------------------------------------------- + +schema = strawberry.Schema(query=Query) + + +async def get_graphql_context( + request: Request, + db: Annotated[Session, Depends(get_db)], +) -> dict[str, Any]: + """Build the per-request context injected into every resolver.""" + try: + user = get_current_user(request) + except Exception: + logger.debug("Could not resolve current user for GraphQL context", exc_info=True) + user = None + return {"request": request, "db": db, "user": user} + + +graphql_router = GraphQLRouter( + schema, + context_getter=get_graphql_context, + graphql_ide="graphiql", +) diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py index 69b26c72..2aa9b039 100644 --- a/app/api/imap_accounts.py +++ b/app/api/imap_accounts.py @@ -110,6 +110,13 @@ class ImapAccountCreate(BaseModel): use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") is_active: bool = Field(default=True, description="Whether to poll this mailbox") + profile_id: int | None = Field( + default=None, + description=( + "ID of the ImapIngestionProfile that controls which attachment types to ingest. " + "Null inherits the global imap_attachment_filter setting." + ), + ) class ImapAccountUpdate(BaseModel): @@ -123,6 +130,13 @@ class ImapAccountUpdate(BaseModel): use_ssl: bool | None = None delete_after_process: bool | None = None is_active: bool | None = None + profile_id: int | None = Field( + default=None, + description=( + "ID of the ImapIngestionProfile to use. " + "Explicitly sending null clears the override (falls back to global setting)." + ), + ) class ImapTestRequest(BaseModel): @@ -155,6 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]: "use_ssl": acct.use_ssl, "delete_after_process": acct.delete_after_process, "is_active": acct.is_active, + "profile_id": acct.profile_id, "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, "last_error": acct.last_error, "created_at": acct.created_at.isoformat() if acct.created_at else None, @@ -222,6 +237,7 @@ def create_imap_account( use_ssl=body.use_ssl, delete_after_process=body.delete_after_process, is_active=body.is_active, + profile_id=body.profile_id, ) try: db.add(acct) @@ -277,6 +293,10 @@ def update_imap_account( acct.delete_after_process = body.delete_after_process if body.is_active is not None: acct.is_active = body.is_active + # profile_id: update whenever the field is explicitly present in the request payload + # (including sending null to clear the override). + if "profile_id" in body.model_fields_set: + acct.profile_id = body.profile_id # Reset last_error so the next poll gives a fresh result acct.last_error = None diff --git a/app/api/imap_profiles.py b/app/api/imap_profiles.py new file mode 100644 index 00000000..90502686 --- /dev/null +++ b/app/api/imap_profiles.py @@ -0,0 +1,257 @@ +"""API endpoints for managing IMAP ingestion profiles. + +Ingestion profiles allow fine-grained control over which attachment types are +accepted when ingesting emails via IMAP. Each profile carries a list of enabled +file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the +canonical set defined in :mod:`app.utils.allowed_types`. + +Built-in system profiles (``is_builtin=True``) are read-only and cannot be +deleted or modified. Users may create their own profiles which are private to +their ``owner_id``. System-level global profiles (``owner_id=None``) are visible +to all users but can only be created by administrators. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models import ImapIngestionProfile +from app.utils.allowed_types import FILE_TYPE_CATEGORIES +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"]) + +DbSession = Annotated[Session, Depends(get_db)] + +# --------------------------------------------------------------------------- +# Auth helpers +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if owner_id is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + +_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys()) + + +class ImapProfileCreate(BaseModel): + """Schema for creating a new ingestion profile.""" + + name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name") + description: str | None = Field(default=None, description="Optional description") + allowed_categories: list[str] = Field( + ..., + min_length=1, + description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"), + ) + + +class ImapProfileUpdate(BaseModel): + """Schema for updating an existing profile (all fields optional).""" + + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + allowed_categories: list[str] | None = Field(default=None, min_length=1) + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + + +def _validate_categories(categories: list[str]) -> list[str]: + """Raise 422 if any category key is unknown; return the cleaned list.""" + unknown = [c for c in categories if c not in _VALID_CATEGORIES] + if unknown: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}", + ) + # Deduplicate while preserving order + seen: set[str] = set() + result: list[str] = [] + for cat in categories: + if cat not in seen: + seen.add(cat) + result.append(cat) + return result + + +# --------------------------------------------------------------------------- +# Serialisation +# --------------------------------------------------------------------------- + + +def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]: + """Serialize a profile row to a response dict.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + + # Enrich categories with display metadata + categories_detail = [ + { + "key": cat, + "label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat, + "description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "", + } + for cat in categories + ] + + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "categories_detail": categories_detail, + "is_builtin": profile.is_builtin, + "created_at": profile.created_at.isoformat() if profile.created_at else None, + "updated_at": profile.updated_at.isoformat() if profile.updated_at else None, + } + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/categories", summary="List available file-type categories") +def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return the full list of file-type categories that can be used in profiles.""" + return [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + + +@router.get("/", summary="List ingestion profiles visible to the current user") +def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return all profiles: system-global (owner_id=NULL) and the user's own profiles.""" + profiles = ( + db.query(ImapIngestionProfile) + .filter( + # SQLAlchemy requires `== None` for IS NULL comparison in ORM filters + (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711 + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + ) + return [_to_response(p) for p in profiles] + + +@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile") +def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Create a new ingestion profile owned by the current user.""" + categories = _validate_categories(body.allowed_categories) + + profile = ImapIngestionProfile( + name=body.name, + description=body.description, + owner_id=owner_id, + allowed_categories=json.dumps(categories), + is_builtin=False, + ) + try: + db.add(profile) + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name) + return _to_response(profile) + + +@router.get("/{profile_id}", summary="Get a single ingestion profile") +def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Return a single profile by ID. Only the owner or system profiles are accessible.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + return _to_response(profile) + + +@router.put("/{profile_id}", summary="Update an ingestion profile") +def update_profile( + profile_id: int, + request: Request, + body: ImapProfileUpdate, + db: DbSession, + owner_id: CurrentOwner, +) -> dict[str, Any]: + """Update an existing ingestion profile. Built-in profiles cannot be modified.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be modified.", + ) + + if body.name is not None: + profile.name = body.name + if "description" in body.model_fields_set: + profile.description = body.description + if body.allowed_categories is not None: + categories = _validate_categories(body.allowed_categories) + profile.allowed_categories = json.dumps(categories) + + profile.updated_at = datetime.now(timezone.utc) + + try: + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id) + return _to_response(profile) + + +@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile") +def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None: + """Delete an ingestion profile. Built-in profiles cannot be deleted.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be deleted.", + ) + + try: + db.delete(profile) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id) diff --git a/app/api/mobile.py b/app/api/mobile.py new file mode 100644 index 00000000..465872ab --- /dev/null +++ b/app/api/mobile.py @@ -0,0 +1,347 @@ +"""Mobile app API endpoints. + +Provides endpoints specifically designed for the DocuElevate native mobile +app (iOS / Android via React Native / Expo): + +* ``POST /mobile/generate-token`` – exchange an active session for a + long-lived API token that the mobile app stores securely. The token is + auto-named "Mobile App – " and is identical to regular API + tokens (Bearer auth works everywhere). + +* ``POST /mobile/register-device`` – register a push-notification device + token (Expo push token) so the user receives push notifications when + documents finish processing. + +* ``GET /mobile/devices`` – list registered devices for the current user. + +* ``DELETE /mobile/devices/{device_id}`` – deactivate a device. + +* ``GET /mobile/whoami`` – lightweight profile endpoint for the mobile app + to verify authentication state. +""" + +import logging +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.api.api_tokens import generate_api_token, hash_token +from app.auth import require_login +from app.database import get_db +from app.models import ApiToken, MobileDevice +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/mobile", tags=["mobile"]) + +DbSession = Annotated[Session, Depends(get_db)] + + +# --------------------------------------------------------------------------- +# Auth helper +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if not owner_id: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + + +# --------------------------------------------------------------------------- +# Request / Response schemas +# --------------------------------------------------------------------------- + + +class GenerateTokenRequest(BaseModel): + """Request body for auto-generating a mobile app token.""" + + device_name: str = Field( + default="Mobile App", + min_length=1, + max_length=120, + description="Human-readable device name used to label the token.", + ) + + +class GenerateTokenResponse(BaseModel): + """Response containing the one-time-visible API token.""" + + token: str + token_id: int + name: str + created_at: datetime + + +class RegisterDeviceRequest(BaseModel): + """Request body for registering a push-notification device token.""" + + push_token: str = Field( + min_length=1, + max_length=512, + description="Expo push token (ExponentPushToken[…]) obtained from the mobile app.", + ) + device_name: str | None = Field( + default=None, + max_length=255, + description="Optional human-readable device name (e.g. 'John's iPhone').", + ) + platform: str = Field( + default="ios", + description="Device platform: 'ios', 'android', or 'web'.", + ) + + +class DeviceResponse(BaseModel): + """Serialised MobileDevice record.""" + + id: int + device_name: str | None + platform: str + push_token_preview: str + is_active: bool + created_at: datetime + last_seen_at: datetime | None + + +class WhoAmIResponse(BaseModel): + """Lightweight profile response for the mobile app.""" + + owner_id: str + display_name: str | None + email: str | None + avatar_url: str | None + is_admin: bool + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _device_to_response(device: MobileDevice) -> dict[str, Any]: + """Convert a MobileDevice ORM object to a serialisable dict.""" + # Show only first 20 chars of the push token for security. + token_preview = device.push_token[:20] + "…" if len(device.push_token) > 20 else device.push_token + return { + "id": device.id, + "device_name": device.device_name, + "platform": device.platform, + "push_token_preview": token_preview, + "is_active": device.is_active, + "created_at": device.created_at, + "last_seen_at": device.last_seen_at, + } + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.post("/generate-token", status_code=status.HTTP_201_CREATED, response_model=GenerateTokenResponse) +@require_login +async def generate_mobile_token( + request: Request, + body: GenerateTokenRequest, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Generate a long-lived API token for the mobile app. + + The mobile app calls this endpoint immediately after SSO login to obtain + a Bearer token it can store in the secure keychain. The returned token + is functionally identical to manually-created API tokens and works with + every authenticated endpoint. + + The token is shown **exactly once** in the response; subsequent requests + show only the prefix for identification. + """ + token_name = f"Mobile App – {body.device_name}" + plaintext = generate_api_token() + token_hash_value = hash_token(plaintext) + prefix = plaintext[:12] + + db_token = ApiToken( + owner_id=owner_id, + name=token_name, + token_hash=token_hash_value, + token_prefix=prefix, + ) + try: + db.add(db_token) + db.commit() + db.refresh(db_token) + except Exception: + db.rollback() + logger.exception("Failed to create mobile API token for owner_id=%s", owner_id) + raise + + logger.info("Mobile API token created: id=%s owner=%s device=%r", db_token.id, owner_id, body.device_name) + + return { + "token": plaintext, + "token_id": db_token.id, + "name": token_name, + "created_at": db_token.created_at, + } + + +@router.post("/register-device", status_code=status.HTTP_201_CREATED, response_model=DeviceResponse) +@require_login +async def register_device( + request: Request, + body: RegisterDeviceRequest, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Register or refresh a push-notification device token. + + If the same ``push_token`` is already registered for this user the + record is reactivated and ``last_seen_at`` is updated rather than + creating a duplicate. + """ + platform = body.platform.lower() + if platform not in {"ios", "android", "web"}: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="platform must be one of: ios, android, web", + ) + + now = datetime.now(timezone.utc) + + # Upsert: reuse existing record if the token is already known. + existing = ( + db.query(MobileDevice) + .filter(MobileDevice.owner_id == owner_id, MobileDevice.push_token == body.push_token) + .first() + ) + if existing: + existing.is_active = True + existing.last_seen_at = now + if body.device_name: + existing.device_name = body.device_name + try: + db.commit() + db.refresh(existing) + except Exception: + db.rollback() + raise + logger.info("Mobile device refreshed: id=%s owner=%s", existing.id, owner_id) + return _device_to_response(existing) + + device = MobileDevice( + owner_id=owner_id, + device_name=body.device_name, + platform=platform, + push_token=body.push_token, + is_active=True, + last_seen_at=now, + ) + try: + db.add(device) + db.commit() + db.refresh(device) + except Exception: + db.rollback() + logger.exception("Failed to register mobile device for owner_id=%s", owner_id) + raise + + logger.info("Mobile device registered: id=%s owner=%s platform=%s", device.id, owner_id, platform) + return _device_to_response(device) + + +@router.get("/devices", response_model=list[DeviceResponse]) +@require_login +async def list_devices( + request: Request, + owner_id: CurrentOwner, + db: DbSession, +) -> list[dict[str, Any]]: + """List all registered push-notification devices for the current user.""" + devices = ( + db.query(MobileDevice).filter(MobileDevice.owner_id == owner_id).order_by(MobileDevice.created_at.desc()).all() + ) + return [_device_to_response(d) for d in devices] + + +@router.delete("/devices/{device_id}", status_code=status.HTTP_204_NO_CONTENT) +@require_login +async def deactivate_device( + request: Request, + device_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> None: + """Deactivate a push-notification device registration. + + The device record is kept for audit purposes but will no longer receive + push notifications. + """ + device = db.get(MobileDevice, device_id) + if not device or device.owner_id != owner_id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Device not found") + + device.is_active = False + try: + db.commit() + except Exception: + db.rollback() + raise + + logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id) + + +@router.get("/whoami", response_model=WhoAmIResponse) +@require_login +async def whoami( + request: Request, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Return basic profile information for the authenticated user. + + The mobile app calls this after token exchange to populate the user + profile screen and verify that the stored token is still valid. + """ + from app.auth import get_gravatar_url + from app.models import LocalUser, UserProfile + + profile = db.query(UserProfile).filter(UserProfile.user_id == owner_id).first() + local_user = db.query(LocalUser).filter(LocalUser.email == owner_id).first() + + display_name: str | None = None + email: str | None = None + avatar_url: str | None = None + is_admin = False + + if profile: + display_name = profile.display_name + + if local_user: + email = local_user.email + is_admin = bool(local_user.is_admin) + if not display_name and local_user.display_name: + display_name = local_user.display_name + elif "@" in owner_id: + # SSO users commonly have their email as owner_id + email = owner_id + + if email: + avatar_url = get_gravatar_url(email) + + return { + "owner_id": owner_id, + "display_name": display_name, + "email": email, + "avatar_url": avatar_url, + "is_admin": is_admin, + } diff --git a/app/auth.py b/app/auth.py index 8811df94..3cb06465 100644 --- a/app/auth.py +++ b/app/auth.py @@ -15,6 +15,7 @@ from starlette.responses import RedirectResponse from app.config import settings from app.database import get_db +from app.middleware.audit_log import get_client_ip # Conditional imports: only used when multi_user_enabled=True. Imported here at # module level (not inside auth()) so they don't incur repeated import overhead. @@ -38,6 +39,9 @@ templates = Jinja2Templates(directory=str(templates_dir)) OAUTH_CONFIGURED = False OAUTH_PROVIDER_NAME = "Single Sign-On" +# Social login providers that are enabled and registered +SOCIAL_PROVIDERS: dict[str, dict[str, str]] = {} + if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_secret: oauth.register( name="authentik", @@ -49,6 +53,68 @@ if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_s OAUTH_CONFIGURED = True OAUTH_PROVIDER_NAME = settings.oauth_provider_name or "Authentik SSO" +# --- Social Login Providers --------------------------------------------------- +if AUTH_ENABLED and settings.social_auth_google_enabled: + if settings.social_auth_google_client_id and settings.social_auth_google_client_secret: + oauth.register( + name="google", + client_id=settings.social_auth_google_client_id, + client_secret=settings.social_auth_google_client_secret, + server_metadata_url="https://accounts.google.com/.well-known/openid-configuration", + client_kwargs={"scope": "openid profile email"}, + ) + SOCIAL_PROVIDERS["google"] = {"name": "Google", "icon": "fab fa-google", "color": "red"} + logger.info("Social login provider registered: Google") + else: + logger.warning("SOCIAL_AUTH_GOOGLE_ENABLED=true but client ID/secret not configured") + +if AUTH_ENABLED and settings.social_auth_microsoft_enabled: + if settings.social_auth_microsoft_client_id and settings.social_auth_microsoft_client_secret: + tenant = settings.social_auth_microsoft_tenant or "common" + oauth.register( + name="microsoft", + client_id=settings.social_auth_microsoft_client_id, + client_secret=settings.social_auth_microsoft_client_secret, + server_metadata_url=f"https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration", + client_kwargs={"scope": "openid profile email"}, + ) + SOCIAL_PROVIDERS["microsoft"] = {"name": "Microsoft", "icon": "fab fa-microsoft", "color": "blue"} + logger.info("Social login provider registered: Microsoft (tenant=%s)", tenant) + else: + logger.warning("SOCIAL_AUTH_MICROSOFT_ENABLED=true but client ID/secret not configured") + +if AUTH_ENABLED and settings.social_auth_apple_enabled: + if settings.social_auth_apple_client_id and settings.social_auth_apple_team_id: + oauth.register( + name="apple", + client_id=settings.social_auth_apple_client_id, + server_metadata_url="https://appleid.apple.com/.well-known/openid-configuration", + client_kwargs={ + "scope": "openid name email", + "response_mode": "form_post", + }, + ) + SOCIAL_PROVIDERS["apple"] = {"name": "Apple", "icon": "fab fa-apple", "color": "gray"} + logger.info("Social login provider registered: Apple") + else: + logger.warning("SOCIAL_AUTH_APPLE_ENABLED=true but client ID/team ID not configured") + +if AUTH_ENABLED and settings.social_auth_dropbox_enabled: + if settings.social_auth_dropbox_client_id and settings.social_auth_dropbox_client_secret: + oauth.register( + name="dropbox", + client_id=settings.social_auth_dropbox_client_id, + client_secret=settings.social_auth_dropbox_client_secret, + authorize_url="https://www.dropbox.com/oauth2/authorize", + access_token_url="https://api.dropboxapi.com/oauth2/token", + userinfo_endpoint="https://api.dropboxapi.com/2/users/get_current_account", + client_kwargs={"token_endpoint_auth_method": "client_secret_post"}, + ) + SOCIAL_PROVIDERS["dropbox"] = {"name": "Dropbox", "icon": "fab fa-dropbox", "color": "blue"} + logger.info("Social login provider registered: Dropbox") + else: + logger.warning("SOCIAL_AUTH_DROPBOX_ENABLED=true but client ID/secret not configured") + router = APIRouter() @@ -194,6 +260,7 @@ async def login(request: Request): "message": request.query_params.get("message"), "show_oauth": OAUTH_CONFIGURED, "oauth_provider_name": OAUTH_PROVIDER_NAME, + "social_providers": SOCIAL_PROVIDERS, "app_version": settings.version, "csrf_token": getattr(request.state, "csrf_token", ""), # "Create account" link is only shown when multi-user mode AND local signup are both enabled @@ -211,6 +278,149 @@ async def oauth_login(request: Request): return await oauth.authentik.authorize_redirect(request, redirect_uri) +async def social_login(request: Request, provider: str): + """Initiate a social login flow for the given provider. + + Args: + request: The current FastAPI request. + provider: One of the registered social provider keys (google, microsoft, apple, dropbox). + + Returns: + A redirect to the provider's authorization page, or back to /login on error. + """ + if provider not in SOCIAL_PROVIDERS: + return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND) + + redirect_uri = request.url_for("social_callback", provider=provider) + oauth_client = getattr(oauth, provider, None) + if oauth_client is None: + return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND) + + return await oauth_client.authorize_redirect(request, redirect_uri) + + +def _normalize_social_userinfo(provider: str, token: dict, raw_userinfo: dict | None) -> dict: + """Normalize the userinfo payload from different social providers into a common format. + + Returns a dict with keys: sub, email, name, preferred_username, picture. + + Args: + provider: The social provider key (google, microsoft, apple, dropbox). + token: The OAuth token response from the provider. Included for future + provider-specific claim extraction (e.g. ``id_token`` claims). + raw_userinfo: The raw userinfo dict (may be None for providers without standard OIDC userinfo). + + Returns: + A normalized user-data dict compatible with the session user format. + """ + userinfo: dict = raw_userinfo or {} + + if provider == "dropbox": + # Dropbox returns a non-standard userinfo response + email = userinfo.get("email", "") + name_info = userinfo.get("name", {}) + display_name = name_info.get("display_name", "") if isinstance(name_info, dict) else str(name_info) + return { + "sub": userinfo.get("account_id", email), + "email": email, + "name": display_name, + "preferred_username": email, + "picture": userinfo.get("profile_photo_url", ""), + } + + # Standard OIDC providers (Google, Microsoft, Apple) + return { + "sub": userinfo.get("sub", ""), + "email": userinfo.get("email", ""), + "name": userinfo.get("name", ""), + "preferred_username": userinfo.get("email", ""), + "picture": userinfo.get("picture", ""), + } + + +async def social_callback(request: Request, provider: str, db: Session = Depends(get_db)): + """Handle the OAuth callback from a social login provider. + + After the user authorizes with the social provider, this endpoint exchanges + the authorization code for tokens, extracts user information, creates or + updates the user profile, and establishes a session. + + Args: + request: The current FastAPI request. + provider: One of the registered social provider keys. + db: Database session (injected). + + Returns: + A redirect to the user's original destination or the upload page. + """ + if provider not in SOCIAL_PROVIDERS: + return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND) + + oauth_client = getattr(oauth, provider, None) + if oauth_client is None: + return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND) + + try: + token = await oauth_client.authorize_access_token(request) + + # Try standard OIDC userinfo first, fall back to token-embedded userinfo + raw_userinfo = token.get("userinfo") + if not raw_userinfo: + try: + resp = await oauth_client.userinfo(token=token) + raw_userinfo = resp if isinstance(resp, dict) else resp.json() if hasattr(resp, "json") else {} + except Exception: + raw_userinfo = {} + + user_data = _normalize_social_userinfo(provider, token, raw_userinfo) + + if not user_data.get("email"): + return RedirectResponse( + url="/login?error=Could+not+retrieve+email+from+provider", + status_code=status.HTTP_302_FOUND, + ) + + # Add Gravatar if no picture provided + if not user_data.get("picture") and user_data.get("email"): + user_data["picture"] = get_gravatar_url(user_data["email"]) + + # Tag the login source for audit/debugging + user_data["auth_provider"] = provider + + # Social login users are never admin by default (admin must be granted + # via the Authentik/OIDC admin group or manually in the admin panel) + user_data["is_admin"] = False + + request.session["user"] = user_data + + # Auto-create or update UserProfile + _ensure_user_profile(db, user_data, is_admin=False) + + provider_name = SOCIAL_PROVIDERS[provider]["name"] + logger.info( + "[SECURITY] SOCIAL_LOGIN_SUCCESS provider=%s user=%s", provider_name, user_data.get("email", "unknown") + ) + + # Redirect first-time users to onboarding + user_id = ( + user_data.get("sub") or user_data.get("preferred_username") or user_data.get("email") or user_data.get("id") + ) + if user_id: + profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first() + if profile and not profile.onboarding_completed: + post_onboarding = request.session.pop("redirect_after_login", "/upload") + request.session["post_onboarding_redirect"] = post_onboarding + return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND) + + redirect_url = request.session.pop("redirect_after_login", "/upload") + return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND) + except Exception as e: + logger.warning("[SECURITY] SOCIAL_LOGIN_FAILURE provider=%s error=%s", provider, type(e).__name__) + return RedirectResponse( + url="/login?error=Social+login+failed.+Please+try+again.", status_code=status.HTTP_302_FOUND + ) + + def _ensure_user_profile(db: Session, user_data: dict, is_admin: bool = False) -> None: """Create or update a UserProfile row for *user_data*. @@ -344,6 +554,13 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)): # Log the successful authentication logger.info("[SECURITY] OAUTH_LOGIN_SUCCESS user=%s admin=%s", user_data.get("email", "unknown"), is_admin) + _record_login_event( + db, + request, + user_data.get("email") or user_data.get("preferred_username") or "unknown", + success=True, + method="oauth", + ) # Redirect first-time users to onboarding user_id = ( @@ -364,6 +581,48 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)): return RedirectResponse(url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND) +def _record_login_event( + db: Session, + request: Request, + username: str, + *, + success: bool, + method: str = "local", + detail: str | None = None, +) -> None: + """Write a login or login-failure audit event to the database. + + Failures are silently swallowed so that an audit-service error never + prevents a legitimate login or surfaces an unrelated 500 error to the user. + + Args: + db: Active database session. + request: The current HTTP request (used to extract the client IP). + username: The username that attempted authentication. + success: ``True`` for a successful login, ``False`` for a failure. + method: Authentication method, e.g. ``"local"`` or ``"oauth"``. + detail: Optional extra context for failures (e.g. ``"wrong_password"``). + """ + try: + from app.utils.audit_service import record_event + + action = "login" if success else "login.failure" + details: dict = {"method": method} + if detail: + details["reason"] = detail + record_event( + db, + action=action, + user=username, + resource_type="session", + ip_address=get_client_ip(request), + details=details, + severity="info" if success else "warning", + ) + except Exception: + logger.debug("Failed to write login audit event for user=%s", username, exc_info=True) + + async def auth(request: Request, db: Session = Depends(get_db)): """Handle local username/password authentication. @@ -413,6 +672,7 @@ async def auth(request: Request, db: Session = Depends(get_db)): username, local_user.is_active, ) + _record_login_event(db, request, username, success=False, detail="account_not_verified") return RedirectResponse( url="/login?error=Please+verify+your+email+address+before+logging+in", status_code=302, @@ -425,10 +685,12 @@ async def auth(request: Request, db: Session = Depends(get_db)): ) if not pw_ok: logger.warning("[SECURITY] LOCAL_LOGIN_FAILURE reason=wrong_password user=%s", username) + _record_login_event(db, request, username, success=False, detail="wrong_password") return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302) user_data = _build_session_user(local_user) request.session["user"] = user_data logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", local_user.email) + _record_login_event(db, request, local_user.email, success=True) _ensure_user_profile(db, user_data, is_admin=bool(local_user.is_admin)) profile = db.query(_UserProfile).filter(_UserProfile.user_id == local_user.email).first() if profile and not profile.onboarding_completed: @@ -473,6 +735,7 @@ async def auth(request: Request, db: Session = Depends(get_db)): } request.session["user"] = admin_user_data logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", username) + _record_login_event(db, request, username, success=True) _ensure_user_profile(db, admin_user_data, is_admin=True) redirect_url = request.session.pop("redirect_after_login", "/upload") return RedirectResponse(url=redirect_url, status_code=302) @@ -485,16 +748,30 @@ async def auth(request: Request, db: Session = Depends(get_db)): admin_configured, not username and not password, ) + _record_login_event(db, request, username or "anonymous", success=False, detail="invalid_credentials") return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302) -async def logout(request: Request): +async def logout(request: Request, db: Session = Depends(get_db)): """Handle user logout""" user = request.session.get("user") username = "unknown" if isinstance(user, dict): username = user.get("preferred_username") or user.get("email") or "unknown" logger.info(f"[SECURITY] LOGOUT user={username}") + try: + from app.utils.audit_service import record_event + + record_event( + db, + action="logout", + user=username, + resource_type="session", + ip_address=get_client_ip(request), + severity="info", + ) + except Exception: + logger.debug("Failed to write logout audit event for user=%s", username, exc_info=True) request.session.pop("user", None) return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302) @@ -503,6 +780,8 @@ if AUTH_ENABLED: router.add_api_route("/login", login, methods=["GET"]) router.add_api_route("/oauth-login", oauth_login, methods=["GET"]) router.add_api_route("/oauth-callback", oauth_callback, methods=["GET"]) + router.add_api_route("/social-login/{provider}", social_login, methods=["GET"]) + router.add_api_route("/social-callback/{provider}", social_callback, methods=["GET"]) router.add_api_route("/auth", auth, methods=["POST"]) router.add_api_route("/logout", logout, methods=["GET"]) diff --git a/app/celery_worker.py b/app/celery_worker.py index 4881f8a9..fd68c332 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -45,6 +45,7 @@ from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401 from app.tasks.upload_to_email import upload_to_email # noqa: F401 from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401 from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401 +from app.tasks.upload_to_icloud import upload_to_icloud # noqa: F401 from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401 from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401 from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401 diff --git a/app/config.py b/app/config.py index 51df35e0..d3a2080c 100644 --- a/app/config.py +++ b/app/config.py @@ -49,18 +49,30 @@ class Settings(BaseSettings): debug: bool = False # Default to False # Making Dropbox optional + dropbox_enabled: bool = Field( + default=True, + description="Enable Dropbox as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) dropbox_app_key: Optional[str] = None dropbox_app_secret: Optional[str] = None dropbox_folder: Optional[str] = None dropbox_refresh_token: Optional[str] = None # Making Nextcloud optional + nextcloud_enabled: bool = Field( + default=True, + description="Enable Nextcloud as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) nextcloud_upload_url: Optional[str] = None nextcloud_username: Optional[str] = None nextcloud_password: Optional[str] = None nextcloud_folder: Optional[str] = None # Making Paperless optional + paperless_enabled: bool = Field( + default=True, + description="Enable Paperless-ngx as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) paperless_ngx_api_token: Optional[str] = None paperless_host: Optional[str] = None paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless @@ -166,12 +178,44 @@ class Settings(BaseSettings): ), ) - # Authentik + # Authentik / Generic OIDC authentik_client_id: Optional[str] = None authentik_client_secret: Optional[str] = None authentik_config_url: Optional[str] = None oauth_provider_name: Optional[str] = None # Name to display for the OAuth provider + # Social Login Providers + # Google OAuth2 + social_auth_google_enabled: bool = False + social_auth_google_client_id: Optional[str] = None + social_auth_google_client_secret: Optional[str] = None + + # Microsoft OAuth2 (Azure AD / Microsoft Entra ID) + social_auth_microsoft_enabled: bool = False + social_auth_microsoft_client_id: Optional[str] = None + social_auth_microsoft_client_secret: Optional[str] = None + social_auth_microsoft_tenant: str = Field( + default="common", + description=( + "Azure AD tenant ID or one of 'common', 'organizations', 'consumers'. " + "Use 'common' to allow any Microsoft account and any Azure AD org. " + "Use a specific tenant ID (GUID) to restrict to a single organization. " + "Default: common." + ), + ) + + # Apple Sign-In + social_auth_apple_enabled: bool = False + social_auth_apple_client_id: Optional[str] = None + social_auth_apple_team_id: Optional[str] = None + social_auth_apple_key_id: Optional[str] = None + social_auth_apple_private_key: Optional[str] = None + + # Dropbox OAuth2 + social_auth_dropbox_enabled: bool = False + social_auth_dropbox_client_id: Optional[str] = None + social_auth_dropbox_client_secret: Optional[str] = None + # Local user signup allow_local_signup: bool = Field( default=False, @@ -394,6 +438,10 @@ class Settings(BaseSettings): imap2_delete_after_process: bool = False # Google Drive settings + google_drive_enabled: bool = Field( + default=True, + description="Enable Google Drive as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) google_drive_credentials_json: Optional[str] = "" google_drive_folder_id: Optional[str] = "" google_drive_delegate_to: Optional[str] = "" # Optional delegated user email @@ -405,6 +453,10 @@ class Settings(BaseSettings): google_drive_refresh_token: Optional[str] = "" # WebDAV settings + webdav_enabled: bool = Field( + default=True, + description="Enable WebDAV as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) webdav_url: Optional[str] = None webdav_username: Optional[str] = None webdav_password: Optional[str] = None @@ -412,6 +464,10 @@ class Settings(BaseSettings): webdav_verify_ssl: bool = True # FTP settings + ftp_enabled: bool = Field( + default=True, + description="Enable FTP as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) ftp_host: Optional[str] = None ftp_port: Optional[int] = 21 ftp_username: Optional[str] = None @@ -421,6 +477,10 @@ class Settings(BaseSettings): ftp_allow_plaintext: bool = True # Default to allowing plaintext fallback # SFTP settings + sftp_enabled: bool = Field( + default=True, + description="Enable SFTP as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) sftp_host: Optional[str] = None sftp_port: Optional[int] = 22 sftp_username: Optional[str] = None @@ -442,6 +502,10 @@ class Settings(BaseSettings): email_default_recipient: Optional[str] = None # Email destination settings (dedicated SMTP for document delivery – decoupled from shared email above) + dest_email_enabled: bool = Field( + default=True, + description="Enable Email as an upload destination. Set to False to disable document delivery via email even when credentials are configured.", + ) dest_email_host: Optional[str] = None dest_email_port: Optional[int] = 587 dest_email_username: Optional[str] = None @@ -451,6 +515,10 @@ class Settings(BaseSettings): dest_email_default_recipient: Optional[str] = None # Fallback recipient for document delivery # OneDrive settings + onedrive_enabled: bool = Field( + default=True, + description="Enable OneDrive as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) onedrive_client_id: Optional[str] = None onedrive_client_secret: Optional[str] = None onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts @@ -458,6 +526,10 @@ class Settings(BaseSettings): onedrive_folder_path: Optional[str] = None # AWS S3 settings + s3_enabled: bool = Field( + default=True, + description="Enable Amazon S3 as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) aws_access_key_id: Optional[str] = None aws_secret_access_key: Optional[str] = None aws_region: Optional[str] = "us-east-1" # Default region @@ -466,6 +538,16 @@ class Settings(BaseSettings): s3_storage_class: Optional[str] = "STANDARD" # Default storage class s3_acl: Optional[str] = "private" # Default ACL + # iCloud Drive settings + icloud_enabled: bool = Field( + default=True, + description="Enable iCloud Drive as an upload destination. Set to False to disable uploads even when credentials are configured.", + ) + icloud_username: Optional[str] = None # Apple ID email address + icloud_password: Optional[str] = None # App-specific password (required for 2FA accounts) + icloud_folder: Optional[str] = None # Target folder path in iCloud Drive (e.g. "Documents/Uploads") + icloud_cookie_directory: Optional[str] = None # Directory for session cookies (default: ~/.pyicloud) + # Uptime Kuma settings uptime_kuma_url: Optional[str] = None uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes @@ -484,6 +566,14 @@ class Settings(BaseSettings): # Feature flags allow_file_delete: bool = True # Default to allowing file deletion from database + compliance_enabled: bool = Field( + default=True, + description=( + "Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). " + "When enabled, admins can view compliance status and apply " + "pre-built regulatory configurations. Default: True." + ), + ) # PDF/A archival conversion settings enable_pdfa_conversion: bool = Field( @@ -557,6 +647,17 @@ class Settings(BaseSettings): ), ) + imap_attachment_filter: str = Field( + default="documents_only", + description=( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: " + "'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); " + "'all' – ingest all supported file types including images. " + "This is the global default; individual user IMAP accounts can override it." + ), + ) + # Batch processing settings processall_throttle_threshold: int = Field( default=20, diff --git a/app/main.py b/app/main.py index 228e8ff0..6089699b 100644 --- a/app/main.py +++ b/app/main.py @@ -16,6 +16,7 @@ from starlette.middleware.trustedhost import TrustedHostMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware from app.api import router as api_router +from app.api.graphql_api import graphql_router from app.api.local_auth import router as local_auth_router from app.auth import router as auth_router from app.config import settings @@ -146,6 +147,20 @@ async def lifespan(app: FastAPI): except Exception: logging.debug("Scheduled jobs seeding skipped — DB may not be ready yet") # noqa: S110 + # Seed the built-in compliance templates (GDPR, HIPAA, SOC2) so they + # are available in the admin compliance dashboard on first startup. + try: + from app.database import SessionLocal as _SessionLocal # noqa: F811 + from app.utils.compliance_service import seed_compliance_templates as _seed_compliance + + _db_compliance = _SessionLocal() + try: + _seed_compliance(_db_compliance) + finally: + _db_compliance.close() + except Exception: + logging.debug("Compliance template seeding skipped — DB may not be ready yet") # noqa: S110 + # Application is now running yield @@ -239,6 +254,21 @@ else: # Custom exception handlers that return JSON for API routes and HTML for frontend routes +# These use their own separate templates instance so that patches in tests on individual +# view modules do not affect the error handler rendering. +_error_templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates" +_error_templates = Jinja2Templates(directory=str(_error_templates_dir)) +# Register the i18n translate helper as a global so error templates can use {{ _("key") }}. +# Error pages use the default language (English); request-specific locale is not needed here. +from app.utils.i18n import SUPPORTED_LANGUAGES as _SUPPORTED_LANGUAGES # noqa: E402 +from app.utils.i18n import translate as _translate_fn # noqa: E402 + +_error_templates.env.globals["_"] = lambda key, **kwargs: _translate_fn(key, "en", **kwargs) +_error_templates.env.globals["min"] = min +_error_templates.env.globals["max"] = max +_error_templates.env.globals["supported_languages"] = _SUPPORTED_LANGUAGES + + @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): """ @@ -250,15 +280,15 @@ async def http_exception_handler(request: Request, exc: HTTPException): return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) # For frontend routes, return appropriate HTML templates - templates = Jinja2Templates(directory=str(static_dir.parent / "templates")) - # Handle 404 errors with a custom template if exc.status_code == 404: - return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND) + return _error_templates.TemplateResponse( + "404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND + ) # For other HTTP errors, we could create specific templates or use a generic one # For now, return a simple error page - return templates.TemplateResponse( + return _error_templates.TemplateResponse( "404.html", # Reuse 404 template for other errors, or create a generic error template {"request": request}, status_code=exc.status_code, @@ -279,8 +309,7 @@ async def custom_500_handler(request: Request, exc: Exception): ) # Serve the 500 template for non-API routes - templates = Jinja2Templates(directory=str(static_dir.parent / "templates")) - return templates.TemplateResponse( + return _error_templates.TemplateResponse( "500.html", {"request": request, "exc": exc}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -298,3 +327,4 @@ app.include_router(files_router) # Explicitly include the files router app.include_router(auth_router) app.include_router(local_auth_router) app.include_router(api_router, prefix="/api") +app.include_router(graphql_router, prefix="/graphql") diff --git a/app/models.py b/app/models.py index 4522d0b4..5ba06e81 100644 --- a/app/models.py +++ b/app/models.py @@ -401,6 +401,48 @@ class PipelineStep(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class ImapIngestionProfile(Base): + """Named ingestion profile controlling which attachment types are accepted from IMAP emails. + + Profiles group file-type categories (e.g. "pdf", "office", "images") so users + can precisely control what gets ingested from each mailbox. + + System-provided built-in profiles (``is_builtin=True``) are seeded by the + migration and cannot be deleted or renamed. Users may create their own profiles + (``owner_id`` set to their identifier) or rely on the global system profiles + (``owner_id=None``). + + ``allowed_categories`` stores a JSON list of category strings, e.g.:: + + '["pdf", "office", "opendocument", "text", "web"]' + + Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``. + """ + + __tablename__ = "imap_ingestion_profiles" + + id = Column(Integer, primary_key=True, index=True) + + # Human-readable profile name (e.g. "Documents Only", "Documents + Images") + name = Column(String(255), nullable=False) + + # Optional description shown in the UI + description = Column(Text, nullable=True) + + # Owner of this profile. NULL = global/system profile available to all users. + owner_id = Column(String, nullable=True, index=True) + + # JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]' + # See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values. + allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]') + + # Built-in system profiles that cannot be deleted or modified via the API. + is_builtin = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class UserImapAccount(Base): """Per-user IMAP ingestion account. @@ -439,6 +481,10 @@ class UserImapAccount(Base): # When True, emails are deleted from the mailbox after their attachments are processed delete_after_process = Column(Boolean, nullable=False, default=False) + # Optional reference to an ImapIngestionProfile. + # NULL means "use the global imap_attachment_filter setting" (system default). + profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True) + # When False the account is not polled by the periodic task (but not deleted) is_active = Column(Boolean, nullable=False, default=True) @@ -530,6 +576,7 @@ class IntegrationType: EMAIL = "EMAIL" PAPERLESS = "PAPERLESS" RCLONE = "RCLONE" + ICLOUD = "ICLOUD" ALL = { IMAP, @@ -546,6 +593,7 @@ class IntegrationType: EMAIL, PAPERLESS, RCLONE, + ICLOUD, } @@ -858,3 +906,60 @@ class ScheduledJob(Base): created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class MobileDevice(Base): + """Registered mobile device for push notifications. + + Stores the push token (Expo push token, FCM token, or APNs token) for a + specific user device so that document-processing events can be forwarded + as push notifications to the native mobile app. + """ + + __tablename__ = "mobile_devices" + + id = Column(Integer, primary_key=True, index=True) + + # User that owns this device registration. + owner_id = Column(String, nullable=False, index=True) + + # Human-readable name the user gave this device (e.g. "John's iPhone"). + device_name = Column(String(255), nullable=True) + + # Platform: "ios", "android", or "web". + platform = Column(String(20), nullable=False, default="ios") + + # Expo push token (ExponentPushToken[…]) or raw FCM/APNs token. + push_token = Column(String(512), nullable=False) + + # Whether push notifications are enabled for this device. + is_active = Column(Boolean, nullable=False, default=True) + + # Timestamps. + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_seen_at = Column(DateTime(timezone=True), nullable=True) + + __table_args__ = (UniqueConstraint("owner_id", "push_token", name="uq_mobile_device_owner_token"),) + + +class ComplianceTemplate(Base): + """Pre-built compliance configuration templates (GDPR, HIPAA, SOC2). + + Each row represents an applied compliance template. The ``settings_json`` + column stores the concrete setting key/value pairs that were written when + the template was applied. ``status`` tracks the current compliance posture. + """ + + __tablename__ = "compliance_templates" + + id = Column(Integer, primary_key=True, index=True) + name = Column(String(50), unique=True, nullable=False, index=True) # GDPR, HIPAA, SOC2 + display_name = Column(String(100), nullable=False) + description = Column(Text, nullable=True) + settings_json = Column(Text, nullable=False, default="{}") # JSON of applied settings + enabled = Column(Boolean, nullable=False, default=False) + status = Column(String(20), nullable=False, default="not_applied") # not_applied, compliant, partial, non_compliant + applied_at = Column(DateTime(timezone=True), nullable=True) + applied_by = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index 064ccf06..d47ab6c2 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -13,7 +13,11 @@ from celery import shared_task from app.config import settings from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import -from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES +from app.utils.allowed_types import ( + ALL_CATEGORIES, + DEFAULT_CATEGORIES, + get_allowed_types_for_categories, +) # Database session for per-user IMAP accounts (imported lazily to avoid circular imports) _db_session_factory = None @@ -50,6 +54,38 @@ def _decrypt_imap_password(password: str | None) -> str | None: return decrypt_value(password) +def _resolve_categories_for_profile(profile_id: int | None) -> list[str]: + """Return the list of allowed categories for a profile ID. + + Loads the profile from the database. If ``profile_id`` is ``None`` or the + profile is not found, falls back to the global ``settings.imap_attachment_filter`` + string (``'documents_only'`` → default categories; ``'all'`` → all categories). + """ + if profile_id is not None: + try: + from app.models import ImapIngestionProfile + + db = _get_db_session() + try: + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if profile: + return json.loads(profile.allowed_categories) + finally: + db.close() + except Exception as exc: # noqa: BLE001 + logger.warning( + "Could not load IMAP ingestion profile %d (%s: %s) — using global default", + profile_id, + type(exc).__name__, + exc, + ) + + # Fall back to global setting + if settings.imap_attachment_filter == "all": + return ALL_CATEGORIES + return DEFAULT_CATEGORIES + + LOCK_KEY = "imap_lock" # Unique key for locking LOCK_EXPIRE = 300 # Lock expires in 5 minutes @@ -180,6 +216,7 @@ def _pull_user_imap_accounts() -> None: use_ssl=acct.use_ssl, delete_after_process=acct.delete_after_process, owner_id=acct.owner_id, + allowed_categories=_resolve_categories_for_profile(acct.profile_id), ) # Record successful poll acct.last_checked_at = datetime.now(timezone.utc) @@ -250,6 +287,9 @@ def _pull_user_integration_imap() -> None: use_ssl = cfg.get("use_ssl", True) delete_after = cfg.get("delete_after_process", False) gmail_labels = cfg.get("gmail_apply_labels", True) + # Integrations can store a profile_id in config; fall back to global default + profile_id = cfg.get("profile_id") + allowed_categories = _resolve_categories_for_profile(profile_id) if not (host and username and password): logger.warning( @@ -269,6 +309,7 @@ def _pull_user_integration_imap() -> None: delete_after_process=delete_after, owner_id=integ.owner_id, gmail_apply_labels=gmail_labels, + allowed_categories=allowed_categories, ) integ.last_used_at = datetime.now(timezone.utc) integ.last_error = None @@ -329,6 +370,7 @@ def pull_inbox( delete_after_process, owner_id=None, gmail_apply_labels=True, + allowed_categories=None, ): """ Connects to the IMAP inbox, fetches new unread emails from the last 3 days, @@ -345,8 +387,22 @@ def pull_inbox( attributed to this user via ``process_document`` / ``convert_to_pdf``. gmail_apply_labels: Whether to apply Gmail-specific labels and stars to processed emails. Only relevant for Gmail hosts. Defaults to True. + allowed_categories: List of file-type category keys to ingest (e.g. + ``["pdf", "office", "images"]``). ``None`` falls back to the + global ``settings.imap_attachment_filter`` mapping. """ - logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl) + if allowed_categories is None: + allowed_categories = _resolve_categories_for_profile(None) + + effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories) + logger.info( + "Connecting to %s at %s:%s (SSL=%s) — categories: %s", + mailbox_key, + host, + port, + use_ssl, + allowed_categories, + ) processed_emails = load_processed_emails() try: @@ -405,9 +461,13 @@ def pull_inbox( logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key) continue - # Process attachments (and convert non-PDF files). - # We call the function without assigning its return value since it is not used. - fetch_attachments_and_enqueue(email_message, owner_id=owner_id) + # Process attachments using the resolved mime types / extensions. + fetch_attachments_and_enqueue( + email_message, + owner_id=owner_id, + effective_mime_types=effective_mime_types, + effective_extensions=effective_extensions, + ) if settings.imap_readonly_mode: logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key) @@ -436,27 +496,23 @@ def pull_inbox( logger.exception("Error pulling mailbox %s: %s", mailbox_key, e) -def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): +def fetch_attachments_and_enqueue( + email_message, + owner_id: str | None = None, + effective_mime_types: frozenset[str] | None = None, + effective_extensions: frozenset[str] | None = None, +): """ Extracts attachments from the email and processes only allowed file types. - Files are accepted if either: - 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR - 2. They have a '.pdf' file extension (regardless of MIME type) + The caller is responsible for computing ``effective_mime_types`` and + ``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or + the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories` + before calling this function. ``pull_inbox`` does this automatically. - Allowed file types include: - - PDF: application/pdf or *.pdf extension - - Microsoft Office files: - - Word: application/msword, - application/vnd.openxmlformats-officedocument.wordprocessingml.document - - Excel: application/vnd.ms-excel, - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - - PowerPoint: application/vnd.ms-powerpoint, - application/vnd.openxmlformats-officedocument.presentationml.presentation - - Other meaningful attachments: - - Plain text: text/plain - - CSV: text/csv - - Rich Text Format: application/rtf, text/rtf + If either set is ``None`` the function falls back to the default category list + so the function still works correctly when called directly in tests or from + other contexts. If the attachment is a PDF (by extension or MIME type), it is enqueued for upload; any other allowed file is enqueued for conversion to PDF. @@ -465,9 +521,14 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): email_message: The parsed email message to extract attachments from. owner_id: Optional user identifier forwarded to ``process_document`` / ``convert_to_pdf`` for multi-tenant attribution. + effective_mime_types: Pre-computed frozenset of allowed MIME type strings. + effective_extensions: Pre-computed frozenset of allowed file extension strings. Returns True if at least one allowed attachment was processed. """ + if effective_mime_types is None or effective_extensions is None: + effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES) + has_attachment = False for part in email_message.walk(): if part.get_content_maintype() == "multipart": @@ -482,9 +543,15 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): mime_type = part.get_content_type() file_ext = os.path.splitext(filename)[1].lower() + # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension - if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension: - logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) + if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension: + logger.info( + "Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set", + filename, + mime_type, + file_ext, + ) continue file_path = os.path.join(settings.workdir, filename) @@ -495,7 +562,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): if mime_type == "application/pdf" or is_pdf_by_extension: process_document.delay(file_path, owner_id=owner_id) logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type) - elif mime_type in ALLOWED_MIME_TYPES: + elif mime_type in effective_mime_types: # Other allowed files are sent for conversion convert_to_pdf.delay(file_path, owner_id=owner_id) logger.info("Enqueued file for conversion to PDF: %s", filename) diff --git a/app/tasks/send_to_all.py b/app/tasks/send_to_all.py index 9a9de6f3..214e0225 100644 --- a/app/tasks/send_to_all.py +++ b/app/tasks/send_to_all.py @@ -12,6 +12,7 @@ from app.tasks.upload_to_dropbox import upload_to_dropbox from app.tasks.upload_to_email import upload_to_email from app.tasks.upload_to_ftp import upload_to_ftp from app.tasks.upload_to_google_drive import upload_to_google_drive +from app.tasks.upload_to_icloud import upload_to_icloud from app.tasks.upload_to_nextcloud import upload_to_nextcloud from app.tasks.upload_to_onedrive import upload_to_onedrive from app.tasks.upload_to_paperless import upload_to_paperless @@ -25,18 +26,32 @@ logger = logging.getLogger(__name__) def _should_upload_to_dropbox(): - return bool(settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token) + return bool( + getattr(settings, "dropbox_enabled", True) + and settings.dropbox_app_key + and settings.dropbox_app_secret + and settings.dropbox_refresh_token + ) def _should_upload_to_nextcloud(): - return bool(settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password) + return bool( + getattr(settings, "nextcloud_enabled", True) + and settings.nextcloud_upload_url + and settings.nextcloud_username + and settings.nextcloud_password + ) def _should_upload_to_paperless(): - return bool(settings.paperless_ngx_api_token and settings.paperless_host) + return bool( + getattr(settings, "paperless_enabled", True) and settings.paperless_ngx_api_token and settings.paperless_host + ) def _should_upload_to_google_drive(): + if not getattr(settings, "google_drive_enabled", True): + return False # Check for OAuth configuration if getattr(settings, "google_drive_use_oauth", False): return bool( @@ -51,20 +66,33 @@ def _should_upload_to_google_drive(): def _should_upload_to_webdav(): - return bool(settings.webdav_url and settings.webdav_username and settings.webdav_password) + return bool( + getattr(settings, "webdav_enabled", True) + and settings.webdav_url + and settings.webdav_username + and settings.webdav_password + ) def _should_upload_to_ftp(): - return bool(settings.ftp_host and settings.ftp_username and settings.ftp_password) + return bool( + getattr(settings, "ftp_enabled", True) and settings.ftp_host and settings.ftp_username and settings.ftp_password + ) def _should_upload_to_sftp(): - return bool(settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key)) + return bool( + getattr(settings, "sftp_enabled", True) + and settings.sftp_host + and settings.sftp_username + and (settings.sftp_password or settings.sftp_private_key) + ) def _should_upload_to_email(): return bool( - settings.dest_email_host + getattr(settings, "dest_email_enabled", True) + and settings.dest_email_host and settings.dest_email_username and settings.dest_email_password and settings.dest_email_default_recipient @@ -72,18 +100,32 @@ def _should_upload_to_email(): def _should_upload_to_onedrive(): - return bool(settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token) + return bool( + getattr(settings, "onedrive_enabled", True) + and settings.onedrive_client_id + and settings.onedrive_client_secret + and settings.onedrive_refresh_token + ) def _should_upload_to_s3(): - return bool(settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key) + return bool( + getattr(settings, "s3_enabled", True) + and settings.s3_bucket_name + and settings.aws_access_key_id + and settings.aws_secret_access_key + ) + + +def _should_upload_to_icloud(): + return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password) def get_configured_services_from_validator(): """ - Use the config validator to determine which services are configured properly. + Use the config validator to determine which services are configured and enabled. Returns a dictionary with service names as keys and boolean values indicating - whether they're properly configured. + whether they're properly configured AND explicitly enabled. """ providers = get_provider_status() @@ -98,12 +140,14 @@ def get_configured_services_from_validator(): "Email": "email", "OneDrive": "onedrive", "S3 Storage": "s3", + "iCloud Drive": "icloud", } result = {} for provider_name, internal_name in service_map.items(): if provider_name in providers: - result[internal_name] = providers[provider_name].get("configured", False) + provider = providers[provider_name] + result[internal_name] = provider.get("configured", False) and provider.get("enabled", True) return result @@ -206,6 +250,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: "should_upload": _should_upload_to_s3, "upload_func": upload_to_s3, }, + { + "name": "icloud", + "should_upload": _should_upload_to_icloud, + "upload_func": upload_to_icloud, + }, ] # Optionally get configuration status from validator diff --git a/app/tasks/upload_to_icloud.py b/app/tasks/upload_to_icloud.py new file mode 100644 index 00000000..f9eff5f7 --- /dev/null +++ b/app/tasks/upload_to_icloud.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +"""Upload files to Apple iCloud Drive via the pyicloud library. + +This module uses the ``pyicloud`` library to authenticate with Apple's iCloud +service and upload files to iCloud Drive. Because Apple does not offer a public +REST API for iCloud Drive, this integration relies on the *unofficial* +reverse-engineered protocol implemented by ``pyicloud``. + +Requirements +~~~~~~~~~~~~ +* An Apple ID with iCloud Drive enabled. +* An **app-specific password** generated at https://appleid.apple.com (required + when two-factor authentication is active – which is the default for all modern + Apple IDs). +* The ``pyicloud`` Python package (``pip install pyicloud``). + +Configuration +~~~~~~~~~~~~~ +Set the following environment variables (or ``app/config.py`` fields): + +* ``ICLOUD_USERNAME`` – Apple ID email address. +* ``ICLOUD_PASSWORD`` – App-specific password. +* ``ICLOUD_FOLDER`` – Target folder path inside iCloud Drive, using ``/`` as + the separator (e.g. ``Documents/Uploads``). The folder is created + automatically if it does not exist. +* ``ICLOUD_COOKIE_DIRECTORY`` – (Optional) Directory for persisting session + cookies so that re-authentication is avoided between task runs. Defaults to + ``~/.pyicloud``. +""" + +import logging +import os + +from app.celery_app import celery +from app.config import settings +from app.tasks.retry_config import UploadTaskWithRetry +from app.utils import log_task_progress + +logger = logging.getLogger(__name__) + + +def _get_icloud_api( + username: str, + password: str, + cookie_directory: str | None = None, +): + """Return an authenticated ``PyiCloudService`` instance. + + Args: + username: Apple ID email address. + password: App-specific password. + cookie_directory: Optional directory for session cookies. + + Returns: + An authenticated ``PyiCloudService`` instance. + + Raises: + ImportError: If ``pyicloud`` is not installed. + ValueError: If authentication fails or 2FA is required interactively. + """ + from pyicloud import PyiCloudService # noqa: S404 – unofficial third-party iCloud client + + kwargs: dict = {} + if cookie_directory: + kwargs["cookie_directory"] = cookie_directory + + api = PyiCloudService(username, password, **kwargs) + + # If 2SA/2FA is required the user must use an app-specific password instead. + if api.requires_2sa or api.requires_2fa: + raise ValueError( + "iCloud account requires two-factor authentication. " + "Please generate an app-specific password at https://appleid.apple.com " + "and use it as ICLOUD_PASSWORD." + ) + + return api + + +def _navigate_to_folder(drive_root, folder_path: str): + """Navigate into (or create) the folder hierarchy described by *folder_path*. + + Args: + drive_root: The iCloud Drive root node (``api.drive``). + folder_path: ``/``-separated path such as ``Documents/Uploads``. + + Returns: + The drive node representing the target folder. + """ + node = drive_root + if not folder_path: + return node + + parts = [p for p in folder_path.strip("/").split("/") if p] + for part in parts: + children = {child.name: child for child in node.dir()} + if part in children: + node = children[part] + else: + # Create the missing folder + node = node.mkdir(part) + return node + + +@celery.task(base=UploadTaskWithRetry, bind=True) +def upload_to_icloud(self, file_path: str, file_id: int = None, folder_override: str = None): + """Upload a file to Apple iCloud Drive. + + Args: + file_path: Local path to the file to upload. + file_id: Optional ``FileRecord.id`` for progress logging. + folder_override: If provided, overrides the default ``ICLOUD_FOLDER`` + setting for this upload. + """ + task_id = self.request.id + logger.info(f"[{task_id}] Starting iCloud Drive upload: {file_path}") + log_task_progress( + task_id, + "upload_to_icloud", + "in_progress", + f"Uploading to iCloud Drive: {os.path.basename(file_path)}", + file_id=file_id, + ) + + # ------------------------------------------------------------------ + # Validate inputs + # ------------------------------------------------------------------ + if not os.path.exists(file_path): + error_msg = f"File not found: {file_path}" + logger.error(f"[{task_id}] {error_msg}") + log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id) + raise FileNotFoundError(error_msg) + + if not settings.icloud_username or not settings.icloud_password: + error_msg = "iCloud credentials are not configured (ICLOUD_USERNAME / ICLOUD_PASSWORD)" + logger.error(f"[{task_id}] {error_msg}") + log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id) + raise ValueError(error_msg) + + filename = os.path.basename(file_path) + target_folder = folder_override if folder_override is not None else (settings.icloud_folder or "") + + # ------------------------------------------------------------------ + # Authenticate & upload + # ------------------------------------------------------------------ + try: + api = _get_icloud_api( + settings.icloud_username, + settings.icloud_password, + settings.icloud_cookie_directory, + ) + + folder_node = _navigate_to_folder(api.drive, target_folder) + + with open(file_path, "rb") as fh: + folder_node.upload(fh) + + logger.info(f"[{task_id}] Successfully uploaded {filename} to iCloud Drive folder '{target_folder}'") + log_task_progress( + task_id, + "upload_to_icloud", + "success", + f"Uploaded to iCloud Drive: {filename}", + file_id=file_id, + ) + return { + "status": "Completed", + "file": file_path, + "icloud_folder": target_folder or "/", + } + + except Exception as e: + error_msg = f"Error uploading {filename} to iCloud Drive: {e}" + logger.error(f"[{task_id}] {error_msg}") + log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id) + raise RuntimeError(error_msg) from e diff --git a/app/tasks/upload_to_user_integration.py b/app/tasks/upload_to_user_integration.py index 1295f6ce..db21701d 100644 --- a/app/tasks/upload_to_user_integration.py +++ b/app/tasks/upload_to_user_integration.py @@ -571,6 +571,37 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t return {"status": "Completed", "rclone_dest": dest} +def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]: + """Upload *file_path* to iCloud Drive using per-user credentials. + + Expected *cfg* keys: + * ``folder`` – target folder path inside iCloud Drive (e.g. ``Documents/Uploads``). + * ``cookie_directory`` – (optional) path for session cookie persistence. + + Expected *creds* keys: + * ``username`` – Apple ID email address. + * ``password`` – app-specific password. + """ + from app.tasks.upload_to_icloud import _get_icloud_api, _navigate_to_folder + + username = creds.get("username") or "" + password = creds.get("password") or "" + folder = cfg.get("folder") or "" + cookie_directory = cfg.get("cookie_directory") or None + + if not username or not password: + raise ValueError("iCloud integration is missing username or password in credentials") + + api = _get_icloud_api(username, password, cookie_directory) + folder_node = _navigate_to_folder(api.drive, folder) + + with open(file_path, "rb") as fh: + folder_node.upload(fh) + + logger.info("[%s] iCloud Drive upload complete: folder=%s", task_id, folder or "/") + return {"status": "Completed", "icloud_folder": folder or "/"} + + # Map IntegrationType → upload helper _UPLOAD_HANDLERS = { IntegrationType.DROPBOX: _upload_dropbox, @@ -584,6 +615,7 @@ _UPLOAD_HANDLERS = { IntegrationType.PAPERLESS: _upload_paperless, IntegrationType.EMAIL: _upload_email, IntegrationType.RCLONE: _upload_rclone, + IntegrationType.ICLOUD: _upload_icloud, } diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py index 733ff10e..eaa37e6e 100644 --- a/app/utils/allowed_types.py +++ b/app/utils/allowed_types.py @@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = { ".md", ".markdown", } + +# --------------------------------------------------------------------------- +# Fine-grained file-type categories used by IMAP ingestion profiles. +# Each category groups related MIME types and extensions so that users can +# enable/disable a logical collection of formats (e.g. "images") rather than +# having to manage individual MIME strings. +# --------------------------------------------------------------------------- + +FILE_TYPE_CATEGORIES: dict[str, dict] = { + "pdf": { + "label": "PDF", + "description": "PDF documents (.pdf)", + "mime_types": frozenset({"application/pdf"}), + "extensions": frozenset({".pdf"}), + }, + "office": { + "label": "Microsoft Office", + "description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)", + "mime_types": frozenset( + { + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-word.template.macroEnabled.12", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.presentationml.template", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + } + ), + "extensions": frozenset( + { + ".doc", + ".docx", + ".docm", + ".dot", + ".dotx", + ".dotm", + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", + ".xlt", + ".xltx", + ".xlw", + ".ppt", + ".pptx", + ".pptm", + ".pps", + ".ppsx", + ".pot", + ".potx", + } + ), + }, + "opendocument": { + "label": "OpenDocument (LibreOffice)", + "description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)", + "mime_types": frozenset( + { + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.graphics", + "application/vnd.oasis.opendocument.formula", + } + ), + "extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}), + }, + "text": { + "label": "Text & Data", + "description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)", + "mime_types": frozenset( + { + "text/plain", + "text/csv", + "application/rtf", + "text/rtf", + } + ), + "extensions": frozenset({".txt", ".csv", ".rtf"}), + }, + "web": { + "label": "Web & Markup", + "description": "HTML and Markdown files (.html, .htm, .md, .markdown)", + "mime_types": frozenset( + { + "text/html", + "text/markdown", + "text/x-markdown", + } + ), + "extensions": frozenset({".html", ".htm", ".md", ".markdown"}), + }, + "images": { + "label": "Images", + "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)", + "mime_types": frozenset( + { + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/bmp", + "image/tiff", + "image/webp", + "image/svg+xml", + } + ), + "extensions": frozenset( + { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + } + ), + }, +} + +# Default categories for the "documents only" built-in profile (no images) +DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"] +# All categories including images +ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"] + + +def get_allowed_types_for_categories( + categories: list[str], +) -> tuple[frozenset[str], frozenset[str]]: + """Return ``(mime_types, extensions)`` for the given category list. + + Unknown category names are silently ignored so that future categories + don't break existing profiles. + """ + mime_types: set[str] = set() + extensions: set[str] = set() + for cat in categories: + info = FILE_TYPE_CATEGORIES.get(cat) + if info: + mime_types |= info["mime_types"] + extensions |= info["extensions"] + return frozenset(mime_types), frozenset(extensions) diff --git a/app/utils/compliance_service.py b/app/utils/compliance_service.py new file mode 100644 index 00000000..0b52c92c --- /dev/null +++ b/app/utils/compliance_service.py @@ -0,0 +1,433 @@ +"""Compliance service for managing GDPR, HIPAA, and SOC2 compliance templates. + +Provides pre-built compliance configurations that can be applied with one click +to ensure the DocuElevate instance meets regulatory requirements. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy.orm import Session + +from app.models import ComplianceTemplate + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Pre-built compliance template definitions +# --------------------------------------------------------------------------- + +COMPLIANCE_TEMPLATES: dict[str, dict[str, Any]] = { + "gdpr": { + "display_name": "GDPR (General Data Protection Regulation)", + "description": ( + "European Union regulation for data protection and privacy. " + "Enforces data minimisation, encryption at rest, audit logging, " + "and limits PII exposure in telemetry." + ), + "settings": { + "auth_enabled": "True", + "sentry_send_default_pii": "False", + "security_headers_enabled": "True", + "security_header_hsts_enabled": "True", + "security_header_csp_enabled": "True", + "security_header_x_frame_options_enabled": "True", + "enable_deduplication": "True", + }, + "checks": [ + { + "key": "auth_enabled", + "expected": "True", + "label": "Authentication enabled", + "description": "User authentication must be enabled to control access to personal data.", + }, + { + "key": "sentry_send_default_pii", + "expected": "False", + "label": "PII excluded from telemetry", + "description": "Personally identifiable information must not be sent to external monitoring services.", + }, + { + "key": "security_headers_enabled", + "expected": "True", + "label": "Security headers enabled", + "description": "HTTP security headers protect against common web vulnerabilities.", + }, + { + "key": "security_header_hsts_enabled", + "expected": "True", + "label": "HSTS enabled", + "description": "HTTP Strict Transport Security ensures encrypted connections.", + }, + { + "key": "security_header_csp_enabled", + "expected": "True", + "label": "Content Security Policy enabled", + "description": "CSP headers prevent cross-site scripting and data injection attacks.", + }, + { + "key": "security_header_x_frame_options_enabled", + "expected": "True", + "label": "Clickjacking protection enabled", + "description": "X-Frame-Options header prevents clickjacking attacks.", + }, + { + "key": "enable_deduplication", + "expected": "True", + "label": "Deduplication enabled", + "description": "Data minimisation: avoid storing duplicate documents.", + }, + ], + }, + "hipaa": { + "display_name": "HIPAA (Health Insurance Portability and Accountability Act)", + "description": ( + "United States regulation for protecting health information. " + "Requires strong access controls, audit trails, encryption, " + "and strict session management." + ), + "settings": { + "auth_enabled": "True", + "multi_user_enabled": "True", + "sentry_send_default_pii": "False", + "security_headers_enabled": "True", + "security_header_hsts_enabled": "True", + "security_header_csp_enabled": "True", + "security_header_x_frame_options_enabled": "True", + "enable_deduplication": "True", + }, + "checks": [ + { + "key": "auth_enabled", + "expected": "True", + "label": "Authentication enabled", + "description": "Access controls are required to protect electronic Protected Health Information (ePHI).", + }, + { + "key": "multi_user_enabled", + "expected": "True", + "label": "Multi-user mode enabled", + "description": "Individual user accounts required for access accountability.", + }, + { + "key": "sentry_send_default_pii", + "expected": "False", + "label": "PII excluded from telemetry", + "description": "Protected Health Information must not be sent to external services.", + }, + { + "key": "security_headers_enabled", + "expected": "True", + "label": "Security headers enabled", + "description": "Security headers protect ePHI during transmission.", + }, + { + "key": "security_header_hsts_enabled", + "expected": "True", + "label": "HSTS enabled", + "description": "Encrypted transport required for all ePHI transmissions.", + }, + { + "key": "security_header_csp_enabled", + "expected": "True", + "label": "Content Security Policy enabled", + "description": "CSP prevents injection attacks that could expose ePHI.", + }, + { + "key": "security_header_x_frame_options_enabled", + "expected": "True", + "label": "Clickjacking protection enabled", + "description": "Prevents embedding the application in unauthorized frames.", + }, + { + "key": "enable_deduplication", + "expected": "True", + "label": "Deduplication enabled", + "description": "Minimise data footprint for ePHI.", + }, + ], + }, + "soc2": { + "display_name": "SOC 2 (Service Organization Control 2)", + "description": ( + "Trust Service Criteria framework for service organisations. " + "Focuses on security, availability, processing integrity, " + "confidentiality, and privacy." + ), + "settings": { + "auth_enabled": "True", + "multi_user_enabled": "True", + "sentry_send_default_pii": "False", + "security_headers_enabled": "True", + "security_header_hsts_enabled": "True", + "security_header_csp_enabled": "True", + "security_header_x_frame_options_enabled": "True", + "enable_deduplication": "True", + }, + "checks": [ + { + "key": "auth_enabled", + "expected": "True", + "label": "Authentication enabled", + "description": "Logical access controls required (CC6.1).", + }, + { + "key": "multi_user_enabled", + "expected": "True", + "label": "Multi-user mode enabled", + "description": "Individual user accounts for access management (CC6.2).", + }, + { + "key": "sentry_send_default_pii", + "expected": "False", + "label": "PII excluded from telemetry", + "description": "Confidential information must not leak to external services (CC6.7).", + }, + { + "key": "security_headers_enabled", + "expected": "True", + "label": "Security headers enabled", + "description": "Protection against common web threats (CC6.6).", + }, + { + "key": "security_header_hsts_enabled", + "expected": "True", + "label": "HSTS enabled", + "description": "Encrypted transport in transit (CC6.7).", + }, + { + "key": "security_header_csp_enabled", + "expected": "True", + "label": "Content Security Policy enabled", + "description": "Application-level security controls (CC6.6).", + }, + { + "key": "security_header_x_frame_options_enabled", + "expected": "True", + "label": "Clickjacking protection enabled", + "description": "UI redress attack prevention (CC6.6).", + }, + { + "key": "enable_deduplication", + "expected": "True", + "label": "Deduplication enabled", + "description": "Data integrity through deduplication (PI1.1).", + }, + ], + }, +} + + +def seed_compliance_templates(db: Session) -> None: + """Create or update the built-in compliance template rows. + + Called once at application startup to ensure the ``compliance_templates`` + table always contains the latest definitions. + """ + for name, defn in COMPLIANCE_TEMPLATES.items(): + existing = db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first() + if existing is None: + template = ComplianceTemplate( + name=name, + display_name=defn["display_name"], + description=defn["description"], + settings_json=json.dumps(defn["settings"]), + enabled=False, + status="not_applied", + ) + db.add(template) + logger.info(f"Seeded compliance template: {name}") + else: + # Update display_name and description if changed, but preserve user state + existing.display_name = defn["display_name"] + existing.description = defn["description"] + try: + db.commit() + except Exception: + db.rollback() + logger.exception("Failed to seed compliance templates") + + +def get_all_templates(db: Session) -> list[dict[str, Any]]: + """Return all compliance templates with their current status.""" + templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all() + result = [] + for t in templates: + defn = COMPLIANCE_TEMPLATES.get(t.name, {}) + checks = defn.get("checks", []) + result.append( + { + "id": t.id, + "name": t.name, + "display_name": t.display_name, + "description": t.description, + "enabled": t.enabled, + "status": t.status, + "applied_at": t.applied_at.isoformat() if t.applied_at else None, + "applied_by": t.applied_by, + "settings": json.loads(t.settings_json) if t.settings_json else {}, + "checks": checks, + "check_count": len(checks), + } + ) + return result + + +def get_template_by_name(db: Session, name: str) -> ComplianceTemplate | None: + """Retrieve a single compliance template by name.""" + return db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first() + + +def evaluate_template_status(db: Session, name: str) -> dict[str, Any]: + """Evaluate the compliance status of a template against live settings. + + Returns a dict with ``status``, ``total``, ``passed``, ``failed``, and + a list of individual ``check_results``. + """ + from app.config import settings as app_settings + from app.utils.settings_service import get_all_settings_from_db + + defn = COMPLIANCE_TEMPLATES.get(name) + if defn is None: + return {"status": "unknown", "total": 0, "passed": 0, "failed": 0, "check_results": []} + + db_settings = get_all_settings_from_db(db) + checks = defn.get("checks", []) + results: list[dict[str, Any]] = [] + passed = 0 + + for check in checks: + key = check["key"] + expected = check["expected"] + + # Resolve effective value: DB > config object + if key in db_settings and db_settings[key] is not None: + actual = str(db_settings[key]) + else: + actual = str(getattr(app_settings, key, "")) + + is_passing = actual.lower() == expected.lower() + if is_passing: + passed += 1 + + results.append( + { + "key": key, + "label": check["label"], + "description": check["description"], + "expected": expected, + "actual": actual, + "passing": is_passing, + } + ) + + total = len(checks) + if passed == total: + status = "compliant" + elif passed > 0: + status = "partial" + else: + status = "non_compliant" + + return { + "status": status, + "total": total, + "passed": passed, + "failed": total - passed, + "check_results": results, + } + + +def apply_template(db: Session, name: str, applied_by: str = "admin") -> dict[str, Any]: + """Apply a compliance template by writing its settings to the database. + + Returns a summary of what was applied. + """ + from app.utils.settings_service import save_setting_to_db + + defn = COMPLIANCE_TEMPLATES.get(name) + if defn is None: + return {"success": False, "error": f"Unknown template: {name}"} + + template = get_template_by_name(db, name) + if template is None: + return {"success": False, "error": f"Template not found in database: {name}"} + + applied_settings: dict[str, str] = {} + errors: list[str] = [] + + for key, value in defn["settings"].items(): + try: + save_setting_to_db(db, key, value, changed_by=f"compliance:{name}") + applied_settings[key] = value + except Exception as e: + errors.append(f"{key}: {e}") + logger.error(f"Failed to apply compliance setting {key}={value}: {e}") + + # Update the template record + now = datetime.now(timezone.utc) + template.enabled = True + template.settings_json = json.dumps(applied_settings) + template.applied_at = now + template.applied_by = applied_by + + # Evaluate and store status + eval_result = evaluate_template_status(db, name) + template.status = eval_result["status"] + + try: + db.commit() + except Exception: + db.rollback() + logger.exception(f"Failed to update compliance template record: {name}") + return {"success": False, "error": "Database commit failed"} + + logger.info(f"Applied compliance template '{name}' by {applied_by}: {len(applied_settings)} settings written") + + return { + "success": len(errors) == 0, + "template": name, + "applied_settings": applied_settings, + "errors": errors, + "status": eval_result, + } + + +def get_compliance_summary(db: Session) -> dict[str, Any]: + """Return a high-level compliance dashboard summary across all templates.""" + templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all() + summary: list[dict[str, Any]] = [] + total_checks = 0 + total_passed = 0 + + for t in templates: + eval_result = evaluate_template_status(db, t.name) + total_checks += eval_result["total"] + total_passed += eval_result["passed"] + summary.append( + { + "name": t.name, + "display_name": t.display_name, + "enabled": t.enabled, + "status": eval_result["status"], + "total": eval_result["total"], + "passed": eval_result["passed"], + "failed": eval_result["failed"], + "applied_at": t.applied_at.isoformat() if t.applied_at else None, + "applied_by": t.applied_by, + } + ) + + overall = "compliant" if total_checks > 0 and total_passed == total_checks else "non_compliant" + if 0 < total_passed < total_checks: + overall = "partial" + + return { + "overall_status": overall, + "total_checks": total_checks, + "total_passed": total_passed, + "total_failed": total_checks - total_passed, + "templates": summary, + } diff --git a/app/utils/config_validator/providers.py b/app/utils/config_validator/providers.py index db278d0c..9a1e7069 100644 --- a/app/utils/config_validator/providers.py +++ b/app/utils/config_validator/providers.py @@ -144,7 +144,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "dropbox_app_secret", None) and getattr(settings, "dropbox_refresh_token", None) ), - "enabled": True, + "enabled": getattr(settings, "dropbox_enabled", True), "description": "Upload files to Dropbox cloud storage", "details": { "folder": getattr(settings, "dropbox_folder", "Not set"), @@ -161,7 +161,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: "configured": bool( getattr(settings, "dest_email_host", None) and getattr(settings, "dest_email_default_recipient", None) ), - "enabled": True, + "enabled": getattr(settings, "dest_email_enabled", True), "description": "Send documents via email", "details": { "host": getattr(settings, "dest_email_host", "Not set"), @@ -183,7 +183,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "ftp_username", None) and getattr(settings, "ftp_password", None) ), - "enabled": True, + "enabled": getattr(settings, "ftp_enabled", True), "description": "Upload files to FTP server", "details": { "host": getattr(settings, "ftp_host", "Not set"), @@ -214,7 +214,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: "name": "Google Drive", "icon": "fa-brands fa-google-drive", "configured": is_configured and bool(getattr(settings, "google_drive_folder_id", None)), - "enabled": True, + "enabled": getattr(settings, "google_drive_enabled", True), "description": "Store documents in Google Drive", "details": { "auth_type": "OAuth" if use_oauth else "Service Account", @@ -250,7 +250,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "nextcloud_username", None) and getattr(settings, "nextcloud_password", None) ), - "enabled": True, + "enabled": getattr(settings, "nextcloud_enabled", True), "description": "Store documents in NextCloud", "details": { "url": getattr(settings, "nextcloud_upload_url", "Not set"), @@ -270,7 +270,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "onedrive_client_secret", None) and getattr(settings, "onedrive_refresh_token", None) ), - "enabled": True, + "enabled": getattr(settings, "onedrive_enabled", True), "description": "Store documents in Microsoft OneDrive", "details": { "client_id": getattr(settings, "onedrive_client_id", "Not set"), @@ -288,7 +288,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: "configured": bool( getattr(settings, "paperless_host", None) and getattr(settings, "paperless_ngx_api_token", None) ), - "enabled": True, + "enabled": getattr(settings, "paperless_enabled", True), "description": "Document management system for digital archives", "details": { "host": getattr(settings, "paperless_host", "Not set"), @@ -305,7 +305,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "aws_access_key_id", None) and getattr(settings, "aws_secret_access_key", None) ), - "enabled": True, + "enabled": getattr(settings, "s3_enabled", True), "description": "Store documents in S3-compatible object storage", "details": { "bucket": getattr(settings, "s3_bucket_name", "Not set"), @@ -327,7 +327,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "sftp_username", None) and (getattr(settings, "sftp_password", None) or getattr(settings, "sftp_private_key", None)) ), - "enabled": True, + "enabled": getattr(settings, "sftp_enabled", True), "description": "Upload files to SFTP server", "details": { "host": getattr(settings, "sftp_host", "Not set"), @@ -362,7 +362,7 @@ def get_provider_status() -> dict[str, dict[str, object]]: and getattr(settings, "webdav_username", None) and getattr(settings, "webdav_password", None) ), - "enabled": True, + "enabled": getattr(settings, "webdav_enabled", True), "description": "Store documents on WebDAV servers", "details": { "url": getattr(settings, "webdav_url", "Not set"), @@ -373,4 +373,19 @@ def get_provider_status() -> dict[str, dict[str, object]]: }, } + # Check iCloud Drive configuration + providers["iCloud Drive"] = { + "name": "iCloud Drive", + "icon": "fa-brands fa-apple", + "configured": bool(getattr(settings, "icloud_username", None) and getattr(settings, "icloud_password", None)), + "enabled": getattr(settings, "icloud_enabled", True), + "description": "Store documents in Apple iCloud Drive", + "details": { + "username": getattr(settings, "icloud_username", "Not set"), + "password": mask_sensitive_value(getattr(settings, "icloud_password", None)), + "folder": getattr(settings, "icloud_folder", "Not set"), + "cookie_directory": getattr(settings, "icloud_cookie_directory", "Not set"), + }, + } + return providers diff --git a/app/utils/config_validator/validators.py b/app/utils/config_validator/validators.py index 36fa02e6..6ef8d52b 100644 --- a/app/utils/config_validator/validators.py +++ b/app/utils/config_validator/validators.py @@ -59,13 +59,43 @@ def validate_auth_config() -> list[str]: and getattr(settings, "authentik_config_url", None) ) - if not using_simple_auth and not using_oidc: - issues.append("Neither simple authentication nor OIDC are properly configured") + # Check if any social login provider is enabled + using_social_login = any( + getattr(settings, f"social_auth_{p}_enabled", False) for p in ("google", "microsoft", "apple", "dropbox") + ) + + if not using_simple_auth and not using_oidc and not using_social_login: + issues.append("Neither simple authentication, OIDC, nor social login are properly configured") # If using OIDC, check for provider name if using_oidc and not getattr(settings, "oauth_provider_name", None): issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled") + # Validate individual social login provider configs + if getattr(settings, "social_auth_google_enabled", False): + if not getattr(settings, "social_auth_google_client_id", None): + issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_ID is required when Google login is enabled") + if not getattr(settings, "social_auth_google_client_secret", None): + issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_SECRET is required when Google login is enabled") + + if getattr(settings, "social_auth_microsoft_enabled", False): + if not getattr(settings, "social_auth_microsoft_client_id", None): + issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_ID is required when Microsoft login is enabled") + if not getattr(settings, "social_auth_microsoft_client_secret", None): + issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET is required when Microsoft login is enabled") + + if getattr(settings, "social_auth_apple_enabled", False): + if not getattr(settings, "social_auth_apple_client_id", None): + issues.append("SOCIAL_AUTH_APPLE_CLIENT_ID is required when Apple login is enabled") + if not getattr(settings, "social_auth_apple_team_id", None): + issues.append("SOCIAL_AUTH_APPLE_TEAM_ID is required when Apple login is enabled") + + if getattr(settings, "social_auth_dropbox_enabled", False): + if not getattr(settings, "social_auth_dropbox_client_id", None): + issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_ID is required when Dropbox login is enabled") + if not getattr(settings, "social_auth_dropbox_client_secret", None): + issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_SECRET is required when Dropbox login is enabled") + return issues diff --git a/app/utils/i18n.py b/app/utils/i18n.py index 04055adc..ab94a908 100644 --- a/app/utils/i18n.py +++ b/app/utils/i18n.py @@ -223,16 +223,19 @@ def detect_language(request: Request) -> str: # 1. User session preference if hasattr(request, "session"): session_lang = request.session.get("preferred_language") - if session_lang and session_lang in SUPPORTED_LANGUAGE_CODES: + if isinstance(session_lang, str) and session_lang in SUPPORTED_LANGUAGE_CODES: return session_lang # 2. Cookie - cookie_lang = request.cookies.get("docuelevate_lang") - if cookie_lang and cookie_lang in SUPPORTED_LANGUAGE_CODES: - return cookie_lang + if hasattr(request, "cookies"): + cookie_lang = request.cookies.get("docuelevate_lang") + if isinstance(cookie_lang, str) and cookie_lang in SUPPORTED_LANGUAGE_CODES: + return cookie_lang # 3. Accept-Language header - accept = request.headers.get("accept-language", "") + accept = "" + if hasattr(request, "headers"): + accept = request.headers.get("accept-language", "") lang = _parse_accept_language(accept) if lang: return lang diff --git a/app/utils/push_notification.py b/app/utils/push_notification.py new file mode 100644 index 00000000..f5f0e521 --- /dev/null +++ b/app/utils/push_notification.py @@ -0,0 +1,130 @@ +"""Push notification sender for the DocuElevate mobile app. + +Uses the **Expo Push Notification** service to deliver notifications to both +iOS (via APNs) and Android (via FCM) without requiring server-side APNs keys +or FCM credentials. The mobile app obtains an ``ExponentPushToken[…]`` at +startup and registers it with the backend via the mobile API. + +Reference: https://docs.expo.dev/push-notifications/sending-notifications/ +""" + +import logging +from typing import Any + +import httpx + +from app.database import SessionLocal +from app.models import MobileDevice + +logger = logging.getLogger(__name__) + +EXPO_PUSH_URL = "https://exp.host/--/api/v2/push/send" + +# Maximum tokens per batch request (Expo limit). +_EXPO_BATCH_LIMIT = 100 + + +def send_expo_push_notification( + tokens: list[str], + title: str, + body: str, + data: dict[str, Any] | None = None, + sound: str = "default", + badge: int | None = None, +) -> list[dict[str, Any]]: + """Send a push notification to one or more Expo push tokens. + + Args: + tokens: List of Expo push tokens (``ExponentPushToken[…]``). + title: Notification title shown in the system tray. + body: Notification body text. + data: Optional JSON-serialisable dict attached to the notification + (available in the app via ``notification.request.content.data``). + sound: Notification sound. Use ``"default"`` or ``None`` for silent. + badge: iOS badge count. Pass ``0`` to clear. + + Returns: + List of Expo push receipt dicts (one per token). + """ + if not tokens: + return [] + + results: list[dict[str, Any]] = [] + + # Send in batches to stay within Expo's per-request limit. + for i in range(0, len(tokens), _EXPO_BATCH_LIMIT): + batch = tokens[i : i + _EXPO_BATCH_LIMIT] + messages = [] + for token in batch: + msg: dict[str, Any] = { + "to": token, + "title": title, + "body": body, + "sound": sound, + } + if data: + msg["data"] = data + if badge is not None: + msg["badge"] = badge + messages.append(msg) + + try: + resp = httpx.post( + EXPO_PUSH_URL, + json=messages, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Content-Type": "application/json", + }, + timeout=15, + ) + resp.raise_for_status() + payload = resp.json() + batch_results = payload.get("data", []) + results.extend(batch_results) + logger.debug("Expo push batch sent: %d tokens, %d results", len(batch), len(batch_results)) + except httpx.HTTPStatusError as exc: + logger.error("Expo push HTTP error: %s – %s", exc.response.status_code, exc.response.text) + except Exception: + logger.exception("Expo push notification failed for batch starting at index %d", i) + + return results + + +def send_push_to_owner( + owner_id: str, + title: str, + body: str, + data: dict[str, Any] | None = None, +) -> None: + """Look up all active push tokens for *owner_id* and send them a notification. + + This function is safe to call from Celery task workers. Database errors + and push failures are logged but never raised so that the caller task is + not retried due to a notification failure. + """ + db = SessionLocal() + try: + devices = ( + db.query(MobileDevice) + .filter( + MobileDevice.owner_id == owner_id, + MobileDevice.is_active.is_(True), + MobileDevice.push_token.isnot(None), + ) + .all() + ) + tokens = [d.push_token for d in devices if d.push_token] + except Exception: + logger.exception("Failed to query mobile devices for owner_id=%s", owner_id) + return + finally: + db.close() + + if not tokens: + logger.debug("No active push tokens for owner_id=%s", owner_id) + return + + logger.info("Sending push notification to %d device(s) for owner_id=%s", len(tokens), owner_id) + send_expo_push_notification(tokens=tokens, title=title, body=body, data=data) diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 39ed812e..dceaa943 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -182,6 +182,153 @@ SETTING_METADATA = { "required": False, "restart_required": True, }, + # Social Login Providers + "social_auth_google_enabled": { + "category": "Social Login", + "description": ( + "Enable Google Sign-In. Requires SOCIAL_AUTH_GOOGLE_CLIENT_ID and " + "SOCIAL_AUTH_GOOGLE_CLIENT_SECRET from the Google Cloud Console." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + "help_link": "https://console.cloud.google.com/apis/credentials", + "help_link_label": "Google Cloud Console", + }, + "social_auth_google_client_id": { + "category": "Social Login", + "description": "Google OAuth2 client ID from the Google Cloud Console.", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_google_client_secret": { + "category": "Social Login", + "description": "Google OAuth2 client secret from the Google Cloud Console.", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": True, + }, + "social_auth_microsoft_enabled": { + "category": "Social Login", + "description": ( + "Enable Microsoft Sign-In (Azure AD / Microsoft Entra ID). Requires " + "SOCIAL_AUTH_MICROSOFT_CLIENT_ID and SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET " + "from Azure App Registrations." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + "help_link": "https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade", + "help_link_label": "Azure Portal", + }, + "social_auth_microsoft_client_id": { + "category": "Social Login", + "description": "Microsoft OAuth2 application (client) ID from Azure App Registrations.", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_microsoft_client_secret": { + "category": "Social Login", + "description": "Microsoft OAuth2 client secret from Azure App Registrations.", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": True, + }, + "social_auth_microsoft_tenant": { + "category": "Social Login", + "description": ( + "Azure AD tenant ID or one of 'common', 'organizations', 'consumers'. " + "Use 'common' to allow any Microsoft account. Use a specific GUID to " + "restrict to a single organization." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_apple_enabled": { + "category": "Social Login", + "description": ( + "Enable Sign in with Apple. Requires an Apple Developer account with " + "a Services ID configured for Sign in with Apple." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + "help_link": "https://developer.apple.com/account/resources/identifiers/list/serviceId", + "help_link_label": "Apple Developer Portal", + }, + "social_auth_apple_client_id": { + "category": "Social Login", + "description": "Apple Services ID (e.g. com.example.docuelevate).", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_apple_team_id": { + "category": "Social Login", + "description": "Apple Developer Team ID (10-character alphanumeric string).", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_apple_key_id": { + "category": "Social Login", + "description": "Apple Sign-In private key ID from the Apple Developer Portal.", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_apple_private_key": { + "category": "Social Login", + "description": ( + "Apple Sign-In private key (PEM format). Generate this in the Apple Developer Portal. " + "Paste the entire key content including BEGIN/END headers." + ), + "type": "string", + "sensitive": True, + "required": False, + "restart_required": True, + }, + "social_auth_dropbox_enabled": { + "category": "Social Login", + "description": ( + "Enable Dropbox Sign-In. Uses the same Dropbox App you may already have " + "configured for storage, or a separate one." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_dropbox_client_id": { + "category": "Social Login", + "description": "Dropbox OAuth2 App Key from the Dropbox App Console.", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "social_auth_dropbox_client_secret": { + "category": "Social Login", + "description": "Dropbox OAuth2 App Secret from the Dropbox App Console.", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": True, + }, # AI Services "openai_api_key": { "category": "AI Services", @@ -495,6 +642,14 @@ SETTING_METADATA = { "options": ["us", "eu"], }, # Storage Providers - Dropbox + "dropbox_enabled": { + "category": "Storage Providers", + "description": "Enable Dropbox as an upload destination. When disabled, no documents will be sent to Dropbox even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "dropbox_app_key": { "category": "Storage Providers", "description": "Dropbox app key for OAuth authentication", @@ -528,6 +683,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - Nextcloud + "nextcloud_enabled": { + "category": "Storage Providers", + "description": "Enable Nextcloud as an upload destination. When disabled, no documents will be sent to Nextcloud even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "nextcloud_upload_url": { "category": "Storage Providers", "description": "Nextcloud WebDAV upload URL", @@ -561,6 +724,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - Paperless-ngx + "paperless_enabled": { + "category": "Storage Providers", + "description": "Enable Paperless-ngx as an upload destination. When disabled, no documents will be sent to Paperless-ngx even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "paperless_ngx_api_token": { "category": "Storage Providers", "description": "Paperless-ngx API authentication token", @@ -578,6 +749,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - Google Drive + "google_drive_enabled": { + "category": "Storage Providers", + "description": "Enable Google Drive as an upload destination. When disabled, no documents will be sent to Google Drive even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "google_drive_credentials_json": { "category": "Storage Providers", "description": "Google Drive service account credentials JSON", @@ -635,6 +814,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - OneDrive + "onedrive_enabled": { + "category": "Storage Providers", + "description": "Enable OneDrive as an upload destination. When disabled, no documents will be sent to OneDrive even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "onedrive_client_id": { "category": "Storage Providers", "description": "OneDrive OAuth client ID", @@ -676,6 +863,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - WebDAV + "webdav_enabled": { + "category": "Storage Providers", + "description": "Enable WebDAV as an upload destination. When disabled, no documents will be sent to WebDAV even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "webdav_url": { "category": "Storage Providers", "description": "WebDAV server URL", @@ -717,6 +912,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - FTP + "ftp_enabled": { + "category": "Storage Providers", + "description": "Enable FTP as an upload destination. When disabled, no documents will be sent to FTP even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "ftp_host": { "category": "Storage Providers", "description": "FTP server hostname or IP address", @@ -774,6 +977,14 @@ SETTING_METADATA = { "restart_required": False, }, # Storage Providers - SFTP + "sftp_enabled": { + "category": "Storage Providers", + "description": "Enable SFTP as an upload destination. When disabled, no documents will be sent to SFTP even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "sftp_host": { "category": "Storage Providers", "description": "SFTP server hostname or IP address", @@ -838,7 +1049,56 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Storage Providers - iCloud Drive + "icloud_enabled": { + "category": "Storage Providers", + "description": "Enable iCloud Drive as an upload destination. When disabled, no documents will be sent to iCloud Drive even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "icloud_username": { + "category": "Storage Providers", + "description": "Apple ID email address for iCloud Drive authentication", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "icloud_password": { + "category": "Storage Providers", + "description": "App-specific password for iCloud Drive (generate at https://appleid.apple.com)", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": False, + }, + "icloud_folder": { + "category": "Storage Providers", + "description": "Target folder path in iCloud Drive (e.g. Documents/Uploads)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "icloud_cookie_directory": { + "category": "Storage Providers", + "description": "Directory for persisting iCloud session cookies (default: ~/.pyicloud)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Storage Providers - AWS S3 + "s3_enabled": { + "category": "Storage Providers", + "description": "Enable Amazon S3 as an upload destination. When disabled, no documents will be sent to S3 even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "aws_access_key_id": { "category": "Storage Providers", "description": "AWS access key ID for S3", @@ -972,6 +1232,14 @@ SETTING_METADATA = { "restart_required": False, }, # Email Destination Settings (dedicated SMTP for document delivery) + "dest_email_enabled": { + "category": "Email Destination", + "description": "Enable Email as an upload destination. When disabled, no documents will be delivered via email even if credentials are configured.", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, "dest_email_host": { "category": "Email Destination", "description": "SMTP server hostname for document delivery (separate from shared email settings)", @@ -1392,6 +1660,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "imap_attachment_filter": { + "category": "IMAP", + "description": ( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). " + "Per-user IMAP accounts can override this global default." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Monitoring - Uptime Kuma "uptime_kuma_url": { "category": "Monitoring", @@ -1595,6 +1875,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "compliance_enabled": { + "category": "Feature Flags", + "description": ( + "Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). " + "When enabled, admins can view compliance status and apply " + "pre-built regulatory configurations. Default: True." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Backup / Restore "backup_enabled": { "category": "Backup", diff --git a/app/utils/user_notification.py b/app/utils/user_notification.py index f6a277cd..791b2219 100644 --- a/app/utils/user_notification.py +++ b/app/utils/user_notification.py @@ -210,6 +210,19 @@ def dispatch_user_notification( finally: db.close() + # 3. Send push notifications to registered mobile devices + try: + from app.utils.push_notification import send_push_to_owner + + send_push_to_owner( + owner_id=owner_id, + title=title, + body=message, + data={"event_type": event_type, "file_id": file_id}, + ) + except Exception: + logger.exception("Error sending push notification for owner_id=%s event=%s", owner_id, event_type) + def notify_user_document_processed(owner_id: str, filename: str, file_id: int | None = None) -> None: """Notify a user that their document was successfully processed.""" diff --git a/app/views/__init__.py b/app/views/__init__.py index 1b2b0443..b2814ff5 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -8,6 +8,7 @@ from app.views.admin_users import router as admin_users_router from app.views.api_tokens import router as api_tokens_router from app.views.audit_logs import router as audit_logs_router from app.views.backup import router as backup_router +from app.views.compliance import router as compliance_router from app.views.db_wizard import router as db_wizard_router from app.views.dropbox import router as dropbox_router from app.views.filemanager import router as filemanager_router @@ -63,3 +64,4 @@ router.include_router(notifications_router) # User notification dashboard router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs router.include_router(audit_logs_router) # Comprehensive audit log viewer router.include_router(help_router) # Built-in help / How-To docs +router.include_router(compliance_router) # Compliance templates dashboard diff --git a/app/views/base.py b/app/views/base.py index fc9d5e0d..21c8b5dd 100644 --- a/app/views/base.py +++ b/app/views/base.py @@ -35,9 +35,12 @@ templates.env.globals["max"] = max # The _() function is available in every template to translate UI strings. # Usage: {{ _("nav.dashboard") }} or {{ _("upload.max_size", size="10 MB") }} # The locale is automatically resolved from the request context. +# A default English implementation is registered as a global so error handlers +# that don't go through _inject_global_context still have the function available. # --------------------------------------------------------------------------- templates.env.globals["supported_languages"] = SUPPORTED_LANGUAGES +templates.env.globals["_"] = lambda key, **kwargs: translate(key, "en", **kwargs) # Customize Jinja2Templates to include app_version in all templates original_template_response = templates.TemplateResponse diff --git a/app/views/compliance.py b/app/views/compliance.py new file mode 100644 index 00000000..e9888f97 --- /dev/null +++ b/app/views/compliance.py @@ -0,0 +1,48 @@ +"""Admin view: compliance templates dashboard page.""" + +import logging + +from fastapi import HTTPException, Request, status +from fastapi.responses import RedirectResponse + +from app.views.base import APIRouter, require_login, settings, templates + +logger = logging.getLogger(__name__) +router = APIRouter() + + +def _require_admin(request: Request): + """Return the session user if they are an admin, else None.""" + user = request.session.get("user") + if not user or not user.get("is_admin"): + logger.warning("Non-admin user attempted to access /admin/compliance") + return None + return user + + +@router.get("/admin/compliance") +@require_login +async def compliance_page(request: Request): + """Admin compliance templates dashboard page. + + Displays GDPR, HIPAA, and SOC2 compliance templates with their current + status and one-click apply functionality. + """ + user = _require_admin(request) + if user is None: + return RedirectResponse(url="/", status_code=status.HTTP_302_FOUND) + + try: + return templates.TemplateResponse( + "compliance.html", + { + "request": request, + "app_version": settings.version, + }, + ) + except Exception as e: + logger.error(f"Error loading compliance page: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load compliance page", + ) diff --git a/app/views/imap_accounts.py b/app/views/imap_accounts.py index 460c83ba..c952a8cd 100644 --- a/app/views/imap_accounts.py +++ b/app/views/imap_accounts.py @@ -1,11 +1,13 @@ """User-facing view for the per-user IMAP ingestion dashboard.""" +import json import logging from fastapi import Request from sqlalchemy.orm import Session -from app.models import UserImapAccount +from app.models import ImapIngestionProfile, UserImapAccount +from app.utils.allowed_types import DEFAULT_CATEGORIES, FILE_TYPE_CATEGORIES from app.utils.subscription import get_tier, get_user_tier_id from app.utils.user_scope import get_current_owner_id from app.views.base import APIRouter, Depends, get_db, require_login, templates @@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None: return max_mb +def _serialize_profile(profile: ImapIngestionProfile) -> dict: + """Serialize a profile for JSON embedding in the template.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "is_builtin": profile.is_builtin, + } + + @router.get("/imap-accounts") @require_login async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): @@ -49,11 +67,35 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): max_mailboxes = _get_max_mailboxes(tier) can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes) + # Load ingestion profiles: system-global + user's own + profiles = ( + db.query(ImapIngestionProfile) + .filter( + # SQLAlchemy requires `== None` for IS NULL comparison in ORM filters + (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711 + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + ) + + # Category definitions for the UI checkbox builder + categories = [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + return templates.TemplateResponse( "imap_accounts.html", { "request": request, "accounts": accounts, + "profiles": [_serialize_profile(p) for p in profiles], + "categories": categories, + "default_categories": DEFAULT_CATEGORIES, "current_count": current_count, "max_mailboxes": max_mailboxes, "can_add": can_add, diff --git a/app/views/onboarding.py b/app/views/onboarding.py index 3e551dbf..2b5fa2b2 100644 --- a/app/views/onboarding.py +++ b/app/views/onboarding.py @@ -27,6 +27,7 @@ _DESTINATION_META: list[dict] = [ {"id": "webdav", "name": "WebDAV", "icon": "fas fa-server"}, {"id": "sftp", "name": "SFTP", "icon": "fas fa-terminal"}, {"id": "ftp", "name": "FTP", "icon": "fas fa-server"}, + {"id": "icloud", "name": "iCloud Drive", "icon": "fab fa-apple"}, ] @@ -51,6 +52,7 @@ def _get_configured_destinations(cfg: Settings) -> list[dict]: "webdav": bool(cfg.webdav_url and cfg.webdav_username), "sftp": bool(cfg.sftp_host and cfg.sftp_username), "ftp": bool(cfg.ftp_host and cfg.ftp_username), + "icloud": bool(cfg.icloud_username and cfg.icloud_password), } return [meta for meta in _DESTINATION_META if checks.get(meta["id"], False)] diff --git a/app/views/plans.py b/app/views/plans.py index 6f01c40a..5eb059d6 100644 --- a/app/views/plans.py +++ b/app/views/plans.py @@ -3,12 +3,11 @@ from fastapi import Request from fastapi.responses import HTMLResponse from fastapi.routing import APIRouter -from fastapi.templating import Jinja2Templates from app.auth import require_login +from app.views.base import templates router = APIRouter() -templates = Jinja2Templates(directory="frontend/templates") @router.get("/admin/plans", response_class=HTMLResponse) diff --git a/docs/API.md b/docs/API.md index d5a0bf40..9515df4c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2063,3 +2063,191 @@ print(response.json()) ## Further Assistance For additional help with the API, please contact our support team or refer to the [Development Guide](../CONTRIBUTING.md). + +## Mobile App API + +The mobile API provides endpoints used by the native iOS and Android app. All endpoints require authentication (Bearer token or active session cookie). + +For full mobile app documentation see [MobileApp.md](./MobileApp.md). + +### POST /api/mobile/generate-token + +Exchange an active web session for a long-lived API token scoped to the mobile app. + +**Request:** +```json +{ "device_name": "John's iPhone" } +``` + +**Response (201 Created):** +```json +{ + "token": "de_AbCdEfGhIjKl...", + "token_id": 42, + "name": "Mobile App – John's iPhone", + "created_at": "2026-03-10T09:30:00Z" +} +``` + +> The `token` is shown **once only**. + +### POST /api/mobile/register-device + +Register an Expo push token to receive push notifications. + +**Request:** +```json +{ + "push_token": "ExponentPushToken[xxxxxx]", + "device_name": "John's iPhone", + "platform": "ios" +} +``` + +**Response (201 Created):** Device record with `id`, `platform`, `is_active`, `created_at`. + +### GET /api/mobile/devices + +List all registered push-notification devices for the current user. + +**Response (200 OK):** Array of device records. + +### DELETE /api/mobile/devices/{device_id} + +Deactivate a push-notification device. The device will no longer receive push notifications. + +**Response (204 No Content)** + +### GET /api/mobile/whoami + +Return basic profile information for the authenticated user. + +**Response (200 OK):** +```json +{ + "owner_id": "john@example.com", + "display_name": "John Doe", + "email": "john@example.com", + "avatar_url": "https://www.gravatar.com/avatar/...", + "is_admin": false +} +``` + +--- + +## GraphQL API + +DocuElevate exposes a GraphQL API at `/graphql` alongside the REST API. It +supports flexible queries with field selection, making it ideal for dashboards +and integrations that only need a subset of the available data. + +### Endpoint + +| Method | URL | Description | +|--------|-----|-------------| +| `POST` | `/graphql` | Execute a GraphQL query or mutation | +| `GET` | `/graphql` | Open the GraphiQL interactive playground | + +### Authentication + +The GraphQL endpoint honours the same authentication rules as the REST API: + +- **`AUTH_ENABLED=False`** (default, single-user mode): all queries are + allowed without credentials. +- **`AUTH_ENABLED=True`** (multi-user mode): a valid session cookie **or** + an `Authorization: Bearer ` API token is required. Admin-only + queries (settings, users) additionally require the `is_admin` flag. + +### Available Queries + +| Field | Returns | Notes | +|-------|---------|-------| +| `documents(ownerId, limit, offset)` | `[DocumentType]` | Paginated list of documents | +| `document(id)` | `DocumentType` | Single document by primary key | +| `pipelines(ownerId, limit, offset)` | `[PipelineType]` | Paginated list of pipelines with steps | +| `pipeline(id)` | `PipelineType` | Single pipeline by primary key | +| `settings(limit, offset)` | `[SettingType]` | Non-sensitive app settings (**admin only**) | +| `users(limit, offset)` | `[UserType]` | User profiles (**admin only**) | +| `user(userId)` | `UserType` | Single user profile (**admin only**) | + +> **Note:** Sensitive configuration keys (API secrets, passwords, tokens) are +> automatically excluded from the `settings` query regardless of the caller's +> privilege level. + +### GraphiQL Playground + +Navigate to `http:///graphql` in a browser to open the +interactive GraphiQL IDE, which provides schema documentation, auto-complete, +and the ability to run queries directly. + +### Example Queries + +**List recent documents:** +```graphql +{ + documents(limit: 5) { + id + originalFilename + mimeType + fileSize + documentTitle + createdAt + } +} +``` + +**Fetch a pipeline with its steps:** +```graphql +{ + pipeline(id: 1) { + id + name + description + isDefault + isActive + steps { + position + stepType + label + enabled + } + } +} +``` + +**List application settings (admin only):** +```graphql +{ + settings { + key + value + updatedAt + } +} +``` + +**List user profiles (admin only):** +```graphql +{ + users(limit: 10) { + userId + displayName + subscriptionTier + isBlocked + } +} +``` + +**Using variables:** +```graphql +query GetDocument($id: Int!) { + document(id: $id) { + id + originalFilename + documentTitle + isDuplicate + ocrQualityScore + } +} +``` +Variables: `{ "id": 42 }` diff --git a/docs/AuthenticationSetup.md b/docs/AuthenticationSetup.md index 5cc9a7b4..320e6191 100644 --- a/docs/AuthenticationSetup.md +++ b/docs/AuthenticationSetup.md @@ -20,10 +20,11 @@ For a complete list of configuration options, see the [Configuration Guide](Conf ## Authentication Methods -DocuElevate supports two primary authentication methods: +DocuElevate supports multiple authentication methods that can be used independently or together: 1. **Simple Authentication** - Basic username/password authentication managed by DocuElevate 2. **OpenID Connect** - Integration with identity providers like Authentik, Keycloak, or Auth0 +3. **Social Login** - Sign in with Google, Microsoft, Apple, or Dropbox accounts (see [Social Login Setup Guide](SocialLoginSetup.md)) ## Session Security @@ -189,3 +190,11 @@ If you encounter issues with authentication: - For most providers, you can visit the `/.well-known/openid-configuration` endpoint to verify their settings For more general configuration issues, see the [Configuration Troubleshooting Guide](ConfigurationTroubleshooting.md). + +## Social Login + +DocuElevate supports social login with Google, Microsoft, Apple, and Dropbox. Social login allows users to authenticate using their existing accounts with these providers, without needing a separate DocuElevate password. + +Social login can be used alongside any other authentication method (simple auth, OIDC, local signup). Each social provider is independently configured. + +For detailed setup instructions, prerequisites, and provider-specific configuration, see the **[Social Login Setup Guide](SocialLoginSetup.md)**. diff --git a/docs/ComplianceGuide.md b/docs/ComplianceGuide.md new file mode 100644 index 00000000..c85150c7 --- /dev/null +++ b/docs/ComplianceGuide.md @@ -0,0 +1,190 @@ +# Compliance Templates Guide + +DocuElevate includes pre-built compliance templates for **GDPR**, **HIPAA**, and **SOC 2** that help you configure your instance to meet regulatory requirements. This guide covers how to use the compliance dashboard, apply templates, and monitor your compliance status. + +## Overview + +The compliance templates feature provides: + +- **Pre-built configurations** for GDPR, HIPAA, and SOC 2 +- **One-click apply** to configure all required settings at once +- **Compliance status dashboard** to monitor your regulatory posture +- **Individual check results** showing which settings are compliant and which need attention + +## Accessing the Dashboard + +The compliance dashboard is available to **admin users only**. + +1. Log in as an administrator +2. Click **Admin** in the navigation bar +3. Select **Compliance** from the dropdown menu + +Or navigate directly to: `/admin/compliance` + +## Available Templates + +### GDPR (General Data Protection Regulation) + +The European Union regulation for data protection and privacy. The GDPR template enforces: + +| Setting | Value | Purpose | +|---------|-------|---------| +| `AUTH_ENABLED` | `True` | Controls access to personal data | +| `SENTRY_SEND_DEFAULT_PII` | `False` | Prevents PII leaking to external services | +| `SECURITY_HEADERS_ENABLED` | `True` | Protects against common web vulnerabilities | +| `SECURITY_HEADER_HSTS_ENABLED` | `True` | Ensures encrypted connections | +| `SECURITY_HEADER_CSP_ENABLED` | `True` | Prevents XSS and injection attacks | +| `SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED` | `True` | Prevents clickjacking | +| `ENABLE_DEDUPLICATION` | `True` | Data minimisation — avoids duplicate storage | + +### HIPAA (Health Insurance Portability and Accountability Act) + +United States regulation for protecting health information. The HIPAA template includes all GDPR settings plus: + +| Setting | Value | Purpose | +|---------|-------|---------| +| `MULTI_USER_ENABLED` | `True` | Individual accounts for access accountability | + +### SOC 2 (Service Organization Control 2) + +Trust Service Criteria framework for service organisations. The SOC 2 template includes the same settings as HIPAA, mapped to SOC 2 Trust Service Criteria (CC6.x, PI1.x). + +## Applying a Template + +1. Navigate to the **Compliance** dashboard (`/admin/compliance`) +2. Find the template you want to apply (GDPR, HIPAA, or SOC 2) +3. Click **Apply Template** +4. Confirm the action in the dialog +5. The template settings are written to the database immediately + +> **Note:** Applying a template writes configuration values to the database. Some settings (e.g., security headers) may require a restart to take effect. Check the Settings page for restart indicators. + +## Understanding Compliance Status + +Each template shows one of four statuses: + +| Status | Badge | Meaning | +|--------|-------|---------| +| **Compliant** | Green | All checks are passing | +| **Partial** | Yellow | Some checks are passing, others are not | +| **Non-Compliant** | Red | No checks are passing | +| **Not Applied** | Grey | Template has never been applied | + +### Individual Checks + +Click **Show Details** on any template card to see individual check results: + +- ✅ **Passing** — The setting matches the expected compliance value +- ❌ **Failing** — The setting does not match; the current and expected values are shown + +## API Endpoints + +The compliance feature exposes the following API endpoints under `/api/compliance/`: + +### List Templates + +```bash +GET /api/compliance/templates +``` + +Returns all compliance templates with their current status. + +### Get Single Template + +```bash +GET /api/compliance/templates/{name} +``` + +Returns a single template by name (`gdpr`, `hipaa`, or `soc2`). + +### Apply Template + +```bash +POST /api/compliance/templates/{name}/apply +``` + +Applies a compliance template, writing all its settings to the database. + +### Get Template Status + +```bash +GET /api/compliance/templates/{name}/status +``` + +Evaluates the live compliance status of a template against current settings. + +**Response example:** + +```json +{ + "status": "partial", + "total": 7, + "passed": 5, + "failed": 2, + "check_results": [ + { + "key": "auth_enabled", + "label": "Authentication enabled", + "description": "User authentication must be enabled to control access to personal data.", + "expected": "True", + "actual": "True", + "passing": true + } + ] +} +``` + +### Compliance Summary + +```bash +GET /api/compliance/summary +``` + +Returns an overall compliance summary across all templates. + +**Response example:** + +```json +{ + "overall_status": "partial", + "total_checks": 22, + "total_passed": 18, + "total_failed": 4, + "templates": [ + { + "name": "gdpr", + "display_name": "GDPR (General Data Protection Regulation)", + "enabled": true, + "status": "compliant", + "total": 7, + "passed": 7, + "failed": 0, + "applied_at": "2026-03-09T12:00:00+00:00", + "applied_by": "admin@example.com" + } + ] +} +``` + +> **Note:** All API endpoints require admin authentication. + +## Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `COMPLIANCE_ENABLED` | `True` | Enable the compliance templates dashboard. Set to `False` to hide the feature. | + +## Best Practices + +1. **Apply templates before going live** — Set up compliance before processing real documents +2. **Monitor status regularly** — Check the compliance dashboard after configuration changes +3. **Use the refresh button** — After changing settings elsewhere, refresh the compliance page to see updated status +4. **Combine templates** — You can apply multiple templates; settings overlap is handled automatically +5. **Review after updates** — After upgrading DocuElevate, review your compliance status as new checks may be added + +## Related Documentation + +- [Configuration Guide](./ConfigurationGuide.md) — Full list of configuration options +- [Privacy & Compliance Guide](./PrivacyCompliance.md) — Privacy notice and GDPR compliance details +- [Deployment Guide](./DeploymentGuide.md) — Production deployment with security best practices +- [Security Audit](../SECURITY_AUDIT.md) — Security findings and mitigations diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 131324e4..643ac930 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -16,6 +16,7 @@ Configuration is primarily done through environment variables specified in a `.e | `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` | | `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` | | `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` | +| `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` | ### Batch Processing Settings @@ -303,6 +304,42 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes — | `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` | | `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` | | `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` | +| `IMAP_ATTACHMENT_FILTER` | System-wide fallback for which attachment types are ingested when no ingestion profile is assigned to a mailbox. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual IMAP accounts can override this using ingestion profiles. | `documents_only` | + +#### IMAP Ingestion Profiles + +For fine-grained control, DocuElevate supports **Ingestion Profiles** — named configurations that let you choose exactly which file-type categories to accept from each mailbox. + +Each profile contains a list of enabled **categories**: + +| Category | Description | +|----------|-------------| +| `pdf` | PDF documents (`.pdf`) | +| `office` | Microsoft Office files (Word, Excel, PowerPoint — `.docx`, `.xlsx`, `.pptx`, …) | +| `opendocument` | LibreOffice/OpenOffice files (`.odt`, `.ods`, `.odp`, …) | +| `text` | Plain text, CSV and RTF files (`.txt`, `.csv`, `.rtf`) | +| `web` | HTML and Markdown files (`.html`, `.htm`, `.md`, `.markdown`) | +| `images` | Image files (`.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg`) | + +Two built-in system profiles are seeded automatically: + +| Profile | Categories | +|---------|------------| +| **Documents Only** | pdf, office, opendocument, text, web (no images) | +| **All Files** | All categories, including images | + +Users can create their own custom profiles via the **Email Ingestion** dashboard (`/imap-accounts`) by clicking the **Manage profiles** link or the **+** button next to the profile dropdown. Custom profiles are private to the creating user and can be freely edited or deleted. + +**API endpoints for ingestion profiles:** + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `GET` | `/api/imap-profiles/` | List all visible profiles (system + user's own) | +| `POST` | `/api/imap-profiles/` | Create a new profile | +| `GET` | `/api/imap-profiles/categories` | List available file-type categories | +| `GET` | `/api/imap-profiles/{id}` | Get a single profile | +| `PUT` | `/api/imap-profiles/{id}` | Update a profile (not built-in) | +| `DELETE` | `/api/imap-profiles/{id}` | Delete a profile (not built-in) | #### Per-User IMAP Integrations @@ -335,6 +372,28 @@ Credentials are encrypted at rest using Fernet encryption. | `AUTHENTIK_CONFIG_URL` | Configuration URL for Authentik OpenID Connect. | | `OAUTH_PROVIDER_NAME` | Display name for the OAuth provider button. | +### Social Login Providers + +Social login lets users sign in with their existing Google, Microsoft, Apple, or Dropbox accounts. Each provider is independently enabled and configured. For detailed setup instructions see the [Social Login Setup Guide](SocialLoginSetup.md). + +| **Variable** | **Description** | **Default** | +|---|---|---| +| `SOCIAL_AUTH_GOOGLE_ENABLED` | Enable Google Sign-In. | `false` | +| `SOCIAL_AUTH_GOOGLE_CLIENT_ID` | Google OAuth2 client ID from the Google Cloud Console. | *(empty)* | +| `SOCIAL_AUTH_GOOGLE_CLIENT_SECRET` | Google OAuth2 client secret. | *(empty)* | +| `SOCIAL_AUTH_MICROSOFT_ENABLED` | Enable Microsoft Sign-In (Azure AD / Microsoft Entra ID). | `false` | +| `SOCIAL_AUTH_MICROSOFT_CLIENT_ID` | Microsoft application (client) ID from Azure App Registrations. | *(empty)* | +| `SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET` | Microsoft client secret. | *(empty)* | +| `SOCIAL_AUTH_MICROSOFT_TENANT` | Azure AD tenant: `common`, `organizations`, `consumers`, or a tenant GUID. | `common` | +| `SOCIAL_AUTH_APPLE_ENABLED` | Enable Sign in with Apple. | `false` | +| `SOCIAL_AUTH_APPLE_CLIENT_ID` | Apple Services ID (e.g. `com.example.docuelevate`). | *(empty)* | +| `SOCIAL_AUTH_APPLE_TEAM_ID` | Apple Developer Team ID. | *(empty)* | +| `SOCIAL_AUTH_APPLE_KEY_ID` | Apple Sign-In private key ID. | *(empty)* | +| `SOCIAL_AUTH_APPLE_PRIVATE_KEY` | Apple Sign-In private key (PEM format). | *(empty)* | +| `SOCIAL_AUTH_DROPBOX_ENABLED` | Enable Dropbox Sign-In. | `false` | +| `SOCIAL_AUTH_DROPBOX_CLIENT_ID` | Dropbox OAuth2 App Key. | *(empty)* | +| `SOCIAL_AUTH_DROPBOX_CLIENT_SECRET` | Dropbox OAuth2 App Secret. | *(empty)* | + ### Multi-User Mode When multi-user mode is enabled, each authenticated user gets their own isolated document space. @@ -942,6 +1001,7 @@ TESSERACT_LANGUAGE=eng+deu | **Variable** | **Description** | |-------------------------------------|-----------------------------------------------------------------------------------------------------| +| `PAPERLESS_ENABLED` | Set to `false` to disable Paperless-ngx uploads without removing credentials. Default: `true` | | `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. | | `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). | | `PAPERLESS_CUSTOM_FIELD_ABSENDER` | (Optional, Legacy) Name of the custom field in Paperless-ngx to store the sender ("absender") information. If set, the extracted sender will be automatically set as a custom field after document upload. Example: `Absender` or `Sender` | @@ -984,6 +1044,7 @@ PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient | **Variable** | **Description** | |-------------------------|--------------------------------------------------| +| `DROPBOX_ENABLED` | Set to `false` to disable Dropbox uploads without removing credentials. Default: `true` | | `DROPBOX_APP_KEY` | Dropbox API app key. | | `DROPBOX_APP_SECRET` | Dropbox API app secret. | | `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | @@ -995,6 +1056,7 @@ For detailed setup instructions, see the [Dropbox Setup Guide](DropboxSetup.md). | **Variable** | **Description** | |-------------------------|---------------------------------------------------------------| +| `NEXTCLOUD_ENABLED` | Set to `false` to disable Nextcloud uploads without removing credentials. Default: `true` | | `NEXTCLOUD_UPLOAD_URL` | Nextcloud WebDAV URL (e.g. `https://nc.example.com/remote.php/dav/files/`). | | `NEXTCLOUD_USERNAME` | Nextcloud login username. | | `NEXTCLOUD_PASSWORD` | Nextcloud login password. | @@ -1004,6 +1066,7 @@ For detailed setup instructions, see the [Dropbox Setup Guide](DropboxSetup.md). | **Variable** | **Description** | |---------------------------------|-------------------------------------------------------| +| `GOOGLE_DRIVE_ENABLED` | Set to `false` to disable Google Drive uploads without removing credentials. Default: `true` | | `GOOGLE_DRIVE_USE_OAUTH` | Set to `true` to use OAuth flow (recommended) | | `GOOGLE_DRIVE_CLIENT_ID` | OAuth Client ID (required if using OAuth flow) | | `GOOGLE_DRIVE_CLIENT_SECRET` | OAuth Client Secret (required if using OAuth flow) | @@ -1020,6 +1083,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS | **Variable** | **Description** | |-------------------------|---------------------------------------------------------------| +| `WEBDAV_ENABLED` | Set to `false` to disable WebDAV uploads without removing credentials. Default: `true` | | `WEBDAV_URL` | WebDAV server URL (e.g. `https://webdav.example.com/path`). | | `WEBDAV_USERNAME` | WebDAV authentication username. | | `WEBDAV_PASSWORD` | WebDAV authentication password. | @@ -1030,6 +1094,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS | **Variable** | **Description** | |-------------------------|---------------------------------------------------------------| +| `FTP_ENABLED` | Set to `false` to disable FTP uploads without removing credentials. Default: `true` | | `FTP_HOST` | FTP server hostname or IP address. | | `FTP_PORT` | FTP port (default: `21`). | | `FTP_USERNAME` | FTP authentication username. | @@ -1042,6 +1107,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS | **Variable** | **Description** | |------------------------------|-------------------------------------------------------| +| `SFTP_ENABLED` | Set to `false` to disable SFTP uploads without removing credentials. Default: `true` | | `SFTP_HOST` | SFTP server hostname or IP address. | | `SFTP_PORT` | SFTP port (default: `22`). | | `SFTP_USERNAME` | SFTP authentication username. | @@ -1073,6 +1139,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS | **Variable** | **Description** | |----------------------------------|---------------------------------------------------------------------| +| `DEST_EMAIL_ENABLED` | Set to `false` to disable email delivery without removing credentials. Default: `true` | | `DEST_EMAIL_HOST` | SMTP server hostname for document delivery. | | `DEST_EMAIL_PORT` | SMTP port for document delivery (default: `587`). | | `DEST_EMAIL_USERNAME` | SMTP authentication username for document delivery. | @@ -1085,6 +1152,7 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS | **Variable** | **Description** | |---------------------------------|-------------------------------------------------------| +| `ONEDRIVE_ENABLED` | Set to `false` to disable OneDrive uploads without removing credentials. Default: `true` | | `ONEDRIVE_CLIENT_ID` | Azure AD application client ID | | `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret | | `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts | @@ -1097,6 +1165,7 @@ For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md | **Variable** | **Description** | |---------------------------------|-------------------------------------------------------| +| `S3_ENABLED` | Set to `false` to disable S3 uploads without removing credentials. Default: `true` | | `AWS_ACCESS_KEY_ID` | AWS IAM access key ID | | `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key | | `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) | @@ -1107,6 +1176,23 @@ For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md For detailed setup instructions, see the [Amazon S3 Setup Guide](AmazonS3Setup.md). +### iCloud Drive (Apple) + +| **Variable** | **Description** | +|---------------------------------|-------------------------------------------------------| +| `ICLOUD_ENABLED` | Set to `false` to disable iCloud uploads without removing credentials. Default: `true` | +| `ICLOUD_USERNAME` | Apple ID email address | +| `ICLOUD_PASSWORD` | App-specific password (generate at [appleid.apple.com](https://appleid.apple.com/account/manage)) | +| `ICLOUD_FOLDER` | Target folder path in iCloud Drive (e.g. `Documents/Uploads`) | +| `ICLOUD_COOKIE_DIRECTORY` | Optional directory for session cookie persistence (default: `~/.pyicloud`) | + +> **Note:** Apple does not provide a public REST API for iCloud Drive. This +> integration uses the [pyicloud](https://github.com/picklepete/pyicloud) +> library which relies on an unofficial, reverse-engineered protocol. Because +> most Apple IDs have two-factor authentication enabled, you **must** generate +> an [app-specific password](https://support.apple.com/en-us/102654) and use +> it as `ICLOUD_PASSWORD`. + ### Notification System | **Variable** | **Description** | diff --git a/docs/MobileApp.md b/docs/MobileApp.md new file mode 100644 index 00000000..92e92383 --- /dev/null +++ b/docs/MobileApp.md @@ -0,0 +1,252 @@ +# Mobile App + +DocuElevate includes a native mobile application for iOS and Android built with **React Native** and **Expo**. The app allows users to capture documents with the device camera, pick files from the device storage, and receive push notifications when documents finish processing. + +## Features + +| Feature | iOS | Android | +|---------|-----|---------| +| SSO login (OAuth2) | ✅ | ✅ | +| Local / basic auth login | ✅ | ✅ | +| Auto-generated API token | ✅ | ✅ | +| Camera capture → upload | ✅ | ✅ | +| File picker upload | ✅ | ✅ | +| Share Sheet / Share Intent | ✅ | ✅ | +| Push notifications | ✅ | ✅ | +| Document list | ✅ | ✅ | +| Dark mode | ✅ | ✅ | + +## Getting Started (Development) + +### Prerequisites + +- Node.js 18 or later +- [Expo CLI](https://docs.expo.dev/get-started/installation/): `npm install -g @expo/cli` +- [Expo Go](https://expo.dev/client) app on your iOS or Android device (for development) +- A running DocuElevate server reachable from your device + +### Run in development mode + +```bash +cd mobile +npm install +npx expo start +``` + +Scan the QR code with **Expo Go** on your device. On iOS you can also use the Camera app. + +## Building for Production + +DocuElevate uses **Expo Application Services (EAS)** to produce App Store / Play Store binaries. + +```bash +# Install EAS CLI globally +npm install -g eas-cli + +# Authenticate with Expo +eas login + +# Build for iOS (requires Apple Developer account) +eas build --platform ios + +# Build for Android +eas build --platform android +``` + +See the [EAS Build documentation](https://docs.expo.dev/build/introduction/) for full setup instructions. + +## Authentication + +### SSO Login Flow + +The mobile app uses the server's existing OAuth2/SSO setup: + +1. User enters the DocuElevate server URL on the login screen. +2. The app opens `/login?mobile=1&redirect_uri=docuelevate://callback` in the **system browser** (Safari / Chrome). +3. The user authenticates via SSO or local credentials. +4. The server redirects back to `docuelevate://callback`. +5. The app calls `POST /api/mobile/generate-token` to exchange the session for a **long-lived API token**. +6. The token is stored securely in the device's keychain (`expo-secure-store`). + +### Auto-generated Mobile Token + +When the mobile app completes login it automatically creates a named API token (`"Mobile App – "`) via `POST /api/mobile/generate-token`. This token: + +- Works identically to tokens created manually in the web UI. +- Is shown in the **API Tokens** page (`/api-tokens`) and can be revoked there. +- Is stored in the device's secure keychain, never in plain storage. + +## Push Notifications + +Push notifications are delivered via the **Expo Push Notification** service, which routes through Apple Push Notification service (APNs) for iOS and Firebase Cloud Messaging (FCM) for Android. + +**No server-side APNs/FCM credentials are required** – Expo's servers handle the provider integration. + +### How it works + +1. After login, the app requests notification permission from the operating system. +2. If granted, the app obtains an **Expo Push Token** (`ExponentPushToken[…]`). +3. The token is registered with the backend via `POST /api/mobile/register-device`. +4. When a document finishes processing, the server sends a push notification to all registered devices for that user. + +### Managing registered devices + +Users can see and remove their registered devices from the **Profile** tab in the app, or via the API: + +```bash +# List registered devices +curl -H "Authorization: Bearer " https://your-server/api/mobile/devices + +# Remove a device +curl -X DELETE -H "Authorization: Bearer " https://your-server/api/mobile/devices/ +``` + +## Uploading Documents + +### Camera Capture + +1. Open the **Upload** tab. +2. Tap **Camera**. +3. Point the camera at the document and take a photo. +4. The image is immediately uploaded and queued for processing. + +### File Picker + +1. Open the **Upload** tab. +2. Tap **File Picker**. +3. Browse to and select one or more files (PDF, DOCX, images, etc.). +4. Files are uploaded and queued for processing. + +### Share Sheet (iOS) / Share Intent (Android) + +The app registers itself as a share target so any file can be sent directly to DocuElevate from another app: + +1. Open a file in Files, Mail, Safari, or any other app. +2. Tap the **Share** button (iOS) or **Share** (Android). +3. Find and tap **DocuElevate** in the share sheet. +4. The file is immediately uploaded. + +> **Note:** The app must be installed on the device for it to appear in the share sheet. + +## Mobile API Endpoints + +The backend exposes a dedicated `/api/mobile/` namespace: + +| Method | Endpoint | Auth | Description | +|--------|----------|------|-------------| +| `POST` | `/api/mobile/generate-token` | Session | Exchange SSO session for API token | +| `POST` | `/api/mobile/register-device` | Bearer | Register Expo push token | +| `GET` | `/api/mobile/devices` | Bearer | List registered devices | +| `DELETE` | `/api/mobile/devices/{id}` | Bearer | Deactivate a device | +| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile | + +All other API endpoints (file upload, file listing, etc.) work with Bearer token authentication. + +### POST /api/mobile/generate-token + +Exchanges an active web session (cookie) for a permanent API token suitable for use in the mobile app. + +**Request:** +```json +{ "device_name": "John's iPhone" } +``` + +**Response (201):** +```json +{ + "token": "de_AbCdEfGhIjKl...", + "token_id": 42, + "name": "Mobile App – John's iPhone", + "created_at": "2026-03-10T09:30:00Z" +} +``` + +> ⚠️ The `token` value is returned **once only**. Store it in the device's secure keychain immediately. + +### POST /api/mobile/register-device + +Registers an Expo push token for the authenticated user. + +**Request:** +```json +{ + "push_token": "ExponentPushToken[xxxxxxxxxxxxxxxxxxxxxx]", + "device_name": "John's iPhone", + "platform": "ios" +} +``` + +Supported platforms: `ios`, `android`, `web`. + +Re-registering the same token is safe (idempotent). + +### GET /api/mobile/whoami + +Returns the current user's profile. + +**Response (200):** +```json +{ + "owner_id": "john@example.com", + "display_name": "John Doe", + "email": "john@example.com", + "avatar_url": "https://www.gravatar.com/avatar/...", + "is_admin": false +} +``` + +## Configuration + +No server-side configuration is required to enable the mobile app. The Expo push notification routing does not need FCM or APNs credentials on the server. + +If you wish to use **direct FCM/APNs** without Expo's relay, replace the `send_expo_push_notification` function in `app/utils/push_notification.py` with your own implementation. + +## Project Structure (mobile/) + +``` +mobile/ +├── App.tsx # Root component +├── app.json # Expo/EAS configuration +├── eas.json # EAS Build profiles +├── package.json +├── tsconfig.json +└── src/ + ├── context/ + │ └── AuthContext.tsx # Auth state + SSO login flow + ├── hooks/ + │ └── usePushNotifications.ts # Push token registration + ├── screens/ + │ ├── LoginScreen.tsx # Server URL + SSO button + │ ├── UploadScreen.tsx # Camera capture + file picker + │ ├── FilesScreen.tsx # Processed document list + │ └── ProfileScreen.tsx # User profile + sign out + └── services/ + └── api.ts # DocuElevate REST API client +``` + +## Troubleshooting + +### "Authentication was cancelled or failed" + +- Ensure the server URL is correct (including `https://`). +- Verify the server is reachable from your device's network. +- Confirm that `AUTH_ENABLED=True` on the server. + +### Push notifications not arriving + +1. Check that the app has notification permission (Settings → DocuElevate → Notifications). +2. Verify the device is registered: `GET /api/mobile/devices`. +3. Ensure the server can reach `https://exp.host` (outbound HTTPS on port 443). +4. On Android, add `google-services.json` to the `mobile/` directory if you are building your own binary. + +### "Connection refused" or timeout + +- Verify that the DocuElevate server is running and accessible. +- Ensure the server's `EXTERNAL_HOSTNAME` or reverse proxy is configured correctly. +- Check that the server accepts CORS requests from `docuelevate://`. + +## Related Documentation + +- [API Documentation](./API.md) +- [Configuration Guide](./ConfigurationGuide.md) +- [Deployment Guide](./DeploymentGuide.md) diff --git a/docs/SocialLoginSetup.md b/docs/SocialLoginSetup.md new file mode 100644 index 00000000..5b4523ba --- /dev/null +++ b/docs/SocialLoginSetup.md @@ -0,0 +1,375 @@ +# Social Login Setup Guide + +This guide explains how to configure social login providers (Google, Microsoft, Apple, Dropbox) for DocuElevate. Social login lets your users sign in with their existing accounts, reducing friction and eliminating the need for separate passwords. + +## Overview + +DocuElevate supports four social login providers: + +| Provider | Protocol | Best For | +|----------|----------|----------| +| **Google** | OAuth2 / OpenID Connect | Consumers and Google Workspace organizations | +| **Microsoft** | OAuth2 / OpenID Connect | Microsoft 365 / Azure AD organizations and personal Microsoft accounts | +| **Apple** | OAuth2 / OpenID Connect | iOS/macOS users, privacy-focused users | +| **Dropbox** | OAuth2 | Teams already using Dropbox as a storage destination | + +Each provider is **independently enabled** — you can use one, several, or all of them at the same time. Social login works alongside any other DocuElevate authentication method (simple auth, OIDC/Authentik, local signup). + +## Prerequisites + +Before configuring any social login provider, ensure: + +1. **Authentication is enabled**: `AUTH_ENABLED=true` in your `.env` file +2. **Session secret is set**: `SESSION_SECRET` must be a random string of at least 32 characters +3. **HTTPS is configured**: All social login providers require HTTPS redirect URIs in production. Use a reverse proxy (Traefik, Nginx, Caddy) with a valid TLS certificate +4. **External hostname is set**: `EXTERNAL_HOSTNAME` must match your public domain (e.g., `docuelevate.example.com`) + +> **Note:** Social login users are regular (non-admin) users by default. To grant admin access, use the Admin Panel (**Settings → User Management**) after the user's first login, or configure admin groups via Authentik/OIDC. + +## Callback URLs + +Each social login provider uses a callback URL to redirect users back to DocuElevate after authentication. The callback URL pattern is: + +``` +https:///social-callback/ +``` + +For example, if your DocuElevate instance is at `https://docuelevate.example.com`: + +| Provider | Callback URL | +|----------|-------------| +| Google | `https://docuelevate.example.com/social-callback/google` | +| Microsoft | `https://docuelevate.example.com/social-callback/microsoft` | +| Apple | `https://docuelevate.example.com/social-callback/apple` | +| Dropbox | `https://docuelevate.example.com/social-callback/dropbox` | + +--- + +## Google Sign-In + +### 1. Create OAuth Credentials in Google Cloud Console + +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) +2. Create a new project (or select an existing one) +3. Navigate to **APIs & Services → Credentials** +4. Click **Create Credentials → OAuth client ID** +5. If prompted, configure the **OAuth consent screen** first: + - **User Type**: External (or Internal for Google Workspace) + - **App name**: DocuElevate + - **User support email**: Your email + - **Authorized domains**: Your domain (e.g., `example.com`) + - **Scopes**: Add `email`, `profile`, and `openid` +6. Back on the Credentials page, create an **OAuth 2.0 Client ID**: + - **Application type**: Web application + - **Name**: DocuElevate + - **Authorized redirect URIs**: `https://docuelevate.example.com/social-callback/google` +7. Note the **Client ID** and **Client Secret** + +### 2. Configure DocuElevate + +Add to your `.env` file: + +```bash +SOCIAL_AUTH_GOOGLE_ENABLED=true +SOCIAL_AUTH_GOOGLE_CLIENT_ID=123456789-abcdefg.apps.googleusercontent.com +SOCIAL_AUTH_GOOGLE_CLIENT_SECRET=GOCSPX-your-secret-here +``` + +### 3. Restart DocuElevate + +```bash +docker compose restart api worker +``` + +### Google-Specific Notes + +- **Google Workspace**: If you want to restrict sign-in to users in your Google Workspace organization, set the OAuth consent screen to "Internal" +- **Verification**: Google may require app verification if you're using External user type and requesting sensitive scopes. For small teams (<100 users), you can add test users instead +- **Unified Auth**: If you also use Google Drive as a storage destination, users who sign in with Google will already be authenticated with a Google identity — simplifying the Google Drive integration experience + +--- + +## Microsoft Sign-In (Azure AD / Microsoft Entra ID) + +### 1. Register an Application in Azure + +1. Go to the [Azure Portal](https://portal.azure.com/) +2. Navigate to **Microsoft Entra ID → App registrations** +3. Click **New registration** +4. Fill in: + - **Name**: DocuElevate + - **Supported account types**: Choose based on your needs: + - *Accounts in this organizational directory only* — single-tenant (your org only) + - *Accounts in any organizational directory* — multi-tenant + - *Accounts in any organizational directory and personal Microsoft accounts* — broadest reach + - **Redirect URI**: Select **Web** and enter `https://docuelevate.example.com/social-callback/microsoft` +5. Click **Register** +6. Note the **Application (client) ID** +7. Navigate to **Certificates & secrets → New client secret** +8. Add a description and expiration, then click **Add** +9. Note the **Value** (this is your client secret — it's only shown once!) + +### 2. Configure API Permissions + +1. In your app registration, go to **API permissions** +2. Ensure these permissions are present (they're usually added by default): + - `openid` + - `profile` + - `email` +3. Click **Grant admin consent** if you're a tenant admin + +### 3. Configure DocuElevate + +Add to your `.env` file: + +```bash +SOCIAL_AUTH_MICROSOFT_ENABLED=true +SOCIAL_AUTH_MICROSOFT_CLIENT_ID=12345678-abcd-efgh-ijkl-123456789012 +SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET=your~client~secret~value +SOCIAL_AUTH_MICROSOFT_TENANT=common +``` + +**Tenant options:** + +| Value | Who Can Sign In | +|-------|----------------| +| `common` | Any Microsoft account (personal + any Azure AD organization) | +| `organizations` | Any Azure AD organization (work/school accounts only) | +| `consumers` | Personal Microsoft accounts only (outlook.com, hotmail.com, etc.) | +| `` | Only users in a specific Azure AD tenant (use the GUID from Azure Portal) | + +### 4. Restart DocuElevate + +```bash +docker compose restart api worker +``` + +### Microsoft-Specific Notes + +- **Client secret expiration**: Azure AD client secrets expire (max 2 years). Set a calendar reminder to rotate them before they expire +- **Conditional Access**: If your organization uses Azure AD Conditional Access policies, social login will respect them +- **Unified Auth**: If you also use OneDrive as a storage destination, users who sign in with Microsoft will already have a Microsoft identity — potentially simplifying OneDrive integration + +--- + +## Apple Sign-In + +Apple Sign-In requires an Apple Developer account ($99/year) and more setup than other providers. + +### 1. Configure in Apple Developer Portal + +1. Go to the [Apple Developer Portal](https://developer.apple.com/account/) +2. Navigate to **Certificates, Identifiers & Profiles → Identifiers** +3. Click **+** and select **App IDs** → Register an App ID: + - **Description**: DocuElevate + - **Bundle ID**: e.g., `com.example.docuelevate` + - Enable **Sign In with Apple** capability +4. Click **+** again and select **Services IDs**: + - **Description**: DocuElevate Web + - **Identifier**: e.g., `com.example.docuelevate.web` (this is your Client ID) + - Enable **Sign In with Apple** + - Click **Configure** next to Sign In with Apple: + - **Primary App ID**: Select the App ID created above + - **Domains**: `docuelevate.example.com` + - **Return URLs**: `https://docuelevate.example.com/social-callback/apple` +5. Click **Save** and **Continue** → **Register** +6. Navigate to **Keys** → Click **+** to create a new key: + - **Key Name**: DocuElevate Sign-In + - Enable **Sign In with Apple** + - Click **Configure** and select the App ID created above + - Click **Continue** → **Register** + - **Download the private key file** (`.p8`) — you can only download it once! + - Note the **Key ID** +7. Note your **Team ID** (shown in the top-right corner of the Developer Portal) + +### 2. Configure DocuElevate + +Add to your `.env` file: + +```bash +SOCIAL_AUTH_APPLE_ENABLED=true +SOCIAL_AUTH_APPLE_CLIENT_ID=com.example.docuelevate.web +SOCIAL_AUTH_APPLE_TEAM_ID=ABCDE12345 +SOCIAL_AUTH_APPLE_KEY_ID=FGHIJ67890 +SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY----- +MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQg... +...your key content here... +-----END PRIVATE KEY-----" +``` + +> **Tip:** You can also store the private key as a single line with `\n` for line breaks: +> ```bash +> SOCIAL_AUTH_APPLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIGTAgEAMBMG...\n-----END PRIVATE KEY-----" +> ``` + +### 3. Restart DocuElevate + +```bash +docker compose restart api worker +``` + +### Apple-Specific Notes + +- **Email relay**: Apple offers a "Hide My Email" feature that provides a relay email address (e.g., `abc123@privaterelay.appleid.com`). DocuElevate accepts these addresses +- **First login only**: Apple sends the user's name only on the very first authorization. If the user revokes and re-authorizes, their name may not be sent again +- **Developer account required**: You need an Apple Developer account ($99/year) to use Sign In with Apple +- **Key rotation**: Apple private keys don't expire, but if you suspect compromise, revoke the key in the Developer Portal and create a new one + +--- + +## Dropbox Sign-In + +### 1. Create a Dropbox App + +1. Go to the [Dropbox App Console](https://www.dropbox.com/developers/apps) +2. Click **Create app** +3. Choose: + - **API**: Scoped access + - **Access type**: Full Dropbox (or App folder, depending on your needs) + - **Name**: DocuElevate Auth (or reuse your existing Dropbox storage app) +4. In the app settings, go to the **OAuth 2** section: + - Add **Redirect URI**: `https://docuelevate.example.com/social-callback/dropbox` +5. Note the **App key** (this is your Client ID) and **App secret** (this is your Client Secret) + +> **Tip:** If you already have a Dropbox app configured for DocuElevate's storage integration, you can reuse the same app — just add the social login redirect URI. Alternatively, create a separate app for authentication to keep concerns separated. + +### 2. Configure DocuElevate + +Add to your `.env` file: + +```bash +SOCIAL_AUTH_DROPBOX_ENABLED=true +SOCIAL_AUTH_DROPBOX_CLIENT_ID=your_dropbox_app_key +SOCIAL_AUTH_DROPBOX_CLIENT_SECRET=your_dropbox_app_secret +``` + +### 3. Restart DocuElevate + +```bash +docker compose restart api worker +``` + +### Dropbox-Specific Notes + +- **Unified Auth**: If you also use Dropbox as a storage destination, authenticating via Dropbox establishes the user's Dropbox identity — making it easier to manage Dropbox storage integration +- **App review**: Dropbox may require app review for production apps with more than 50 users. See [Dropbox App Review](https://www.dropbox.com/developers/reference/developer-guide#app-review) +- **Personal vs. Business**: The same app works for both personal Dropbox and Dropbox Business accounts + +--- + +## Unified Authentication and Storage + +One of the key advantages of social login in DocuElevate is the potential for **unified authentication** — using the same identity for both signing in and accessing cloud storage destinations: + +| Social Login Provider | Related Storage Destination | Benefit | +|---|---|---| +| Google | Google Drive | User already has a Google identity for Drive integration | +| Microsoft | OneDrive | User already has a Microsoft identity for OneDrive integration | +| Dropbox | Dropbox | User already has a Dropbox identity for Dropbox integration | +| Apple | *(none)* | Provides a familiar, privacy-respecting login option | + +When a user signs in with a social provider that matches a configured storage destination, the administrator can leverage the same OAuth credentials or simplify the integration setup. Note that the storage integration credentials are configured separately in the admin settings — social login establishes the user's identity, not their storage permissions. + +## Combining Multiple Auth Methods + +DocuElevate supports running multiple authentication methods simultaneously: + +``` +┌──────────────────────────────────────────────────┐ +│ Login Page │ +├──────────────────────────────────────────────────┤ +│ Username / Password form (always shown) │ +│ │ +│ ─── Or continue with ─── │ +│ │ +│ [Authentik SSO] (if OIDC configured) │ +│ [Sign in with Google] (if Google enabled) │ +│ [Sign in with Microsoft] (if Microsoft enabled) │ +│ [Sign in with Apple] (if Apple enabled) │ +│ [Sign in with Dropbox] (if Dropbox enabled) │ +│ │ +│ [Create account] (if local signup enabled) │ +└──────────────────────────────────────────────────┘ +``` + +All methods create or update the same `UserProfile` record, so a user is consistently identified regardless of how they sign in. + +## Admin Management + +Social login users appear in the **Admin → User Management** panel like any other user. Admins can: + +- View which provider a user authenticated with +- Block or unblock social login users +- Set upload limits and subscription tiers +- Grant admin privileges (social login users are never automatically admin) + +## Security Considerations + +1. **HTTPS is required**: All social login providers require HTTPS callback URLs in production +2. **Credentials are sensitive**: Store client secrets securely — use environment variables, never commit them to source control +3. **Least privilege**: Only request the scopes you need (DocuElevate requests `openid`, `profile`, and `email`) +4. **Rotate secrets**: Set calendar reminders to rotate OAuth client secrets before they expire (especially Microsoft, which has a max 2-year expiration) +5. **Monitor logins**: Check the DocuElevate audit log for unusual login patterns +6. **Social login users are not admins**: Admin access must be explicitly granted by an existing admin + +## Troubleshooting + +### Common Issues + +1. **"Unknown social provider" error** + - The provider is not enabled or credentials are missing + - Check that `SOCIAL_AUTH__ENABLED=true` is set + - Verify client ID and secret are configured + +2. **"Could not retrieve email from provider" error** + - The provider didn't return an email address + - For Google: Ensure `email` scope is included (it is by default) + - For Apple: User may have chosen "Hide My Email" — this is expected and should still work + - For Dropbox: Ensure the app has permission to read the user's email + +3. **Redirect URI mismatch** + - The callback URL registered with the provider must exactly match what DocuElevate generates + - Check your `EXTERNAL_HOSTNAME` setting + - Ensure you're using HTTPS in production + - The callback URL format is: `https:///social-callback/` + +4. **"Social login failed" error** + - Check DocuElevate logs (`docker compose logs api`) for detailed error messages + - Verify the provider's OAuth app is not suspended or in development mode + - For Google: Check if the OAuth consent screen needs verification + - For Microsoft: Ensure admin consent was granted for the required permissions + +5. **User can't log in after changing provider settings** + - After changing social login configuration, restart DocuElevate: `docker compose restart api worker` + - Social login settings require a restart to take effect (`restart_required: true`) + +### Debug Checklist + +- [ ] `AUTH_ENABLED=true` is set +- [ ] `SESSION_SECRET` is at least 32 characters +- [ ] `EXTERNAL_HOSTNAME` matches your public domain +- [ ] Provider-specific `_ENABLED=true` is set +- [ ] Client ID and secret are correctly configured (no extra spaces) +- [ ] Callback URL is registered with the provider +- [ ] HTTPS is working on your domain +- [ ] DocuElevate has been restarted after configuration changes + +## Environment Variable Reference + +| Variable | Required | Description | +|---|---|---| +| `SOCIAL_AUTH_GOOGLE_ENABLED` | No | Enable Google Sign-In (`true`/`false`). Default: `false` | +| `SOCIAL_AUTH_GOOGLE_CLIENT_ID` | When Google enabled | Google OAuth2 client ID | +| `SOCIAL_AUTH_GOOGLE_CLIENT_SECRET` | When Google enabled | Google OAuth2 client secret | +| `SOCIAL_AUTH_MICROSOFT_ENABLED` | No | Enable Microsoft Sign-In (`true`/`false`). Default: `false` | +| `SOCIAL_AUTH_MICROSOFT_CLIENT_ID` | When Microsoft enabled | Azure AD application (client) ID | +| `SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET` | When Microsoft enabled | Azure AD client secret | +| `SOCIAL_AUTH_MICROSOFT_TENANT` | No | Azure AD tenant. Default: `common` | +| `SOCIAL_AUTH_APPLE_ENABLED` | No | Enable Apple Sign-In (`true`/`false`). Default: `false` | +| `SOCIAL_AUTH_APPLE_CLIENT_ID` | When Apple enabled | Apple Services ID | +| `SOCIAL_AUTH_APPLE_TEAM_ID` | When Apple enabled | Apple Developer Team ID | +| `SOCIAL_AUTH_APPLE_KEY_ID` | When Apple enabled | Apple Sign-In key ID | +| `SOCIAL_AUTH_APPLE_PRIVATE_KEY` | When Apple enabled | Apple Sign-In private key (PEM) | +| `SOCIAL_AUTH_DROPBOX_ENABLED` | No | Enable Dropbox Sign-In (`true`/`false`). Default: `false` | +| `SOCIAL_AUTH_DROPBOX_CLIENT_ID` | When Dropbox enabled | Dropbox App Key | +| `SOCIAL_AUTH_DROPBOX_CLIENT_SECRET` | When Dropbox enabled | Dropbox App Secret | diff --git a/docs/StorageArchitecture.md b/docs/StorageArchitecture.md index 0e24bd1b..af055406 100644 --- a/docs/StorageArchitecture.md +++ b/docs/StorageArchitecture.md @@ -348,6 +348,7 @@ in task messages or logs. | `PAPERLESS` | Paperless-ngx REST API, API token | | `EMAIL` | SMTP/STARTTLS, file as attachment | | `RCLONE` | `rclone copyto` subprocess, per-user rclone config | +| `ICLOUD` | pyicloud library, Apple ID + app-specific password | ### Multiple Destinations diff --git a/docs/howto/EmailIngestion.md b/docs/howto/EmailIngestion.md index 1a911f6f..bfb6b735 100644 --- a/docs/howto/EmailIngestion.md +++ b/docs/howto/EmailIngestion.md @@ -63,6 +63,54 @@ DocuElevate will process the following attachment types from emails: | TIFF | `.tif`, `.tiff` | Common format from older scanners/fax | | Multi-page TIFF | `.tif` | Full multi-page support | +### Controlling Which Attachment Types Are Ingested + +By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments. + +#### Global Default (Admin Setting) + +Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide fallback when no ingestion profile is assigned to a mailbox: + +| Value | Behaviour | +|-------|-----------| +| `documents_only` | **(Default)** Only PDFs and office/document files. Images are skipped. | +| `all` | All supported file types, including images. | + +```env +IMAP_ATTACHMENT_FILTER=documents_only +``` + +#### Ingestion Profiles (Fine-Grained Per-Mailbox Control) + +For precise control, you can create **Ingestion Profiles** that let you pick exactly which file-type categories to accept from each mailbox. This is more powerful than the binary global toggle and works independently per mailbox. + +**Available categories:** + +| Category | File types included | +|----------|---------------------| +| PDF | `.pdf` | +| Microsoft Office | `.doc`, `.docx`, `.xls`, `.xlsx`, `.ppt`, `.pptx`, and macro-enabled variants | +| OpenDocument | `.odt`, `.ods`, `.odp`, `.odg`, `.odf` (LibreOffice / OpenOffice) | +| Text & Data | `.txt`, `.csv`, `.rtf` | +| Web & Markup | `.html`, `.htm`, `.md`, `.markdown` | +| Images | `.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg` | + +**Managing profiles:** + +1. Go to **Email Ingestion** (`/imap-accounts`) +2. Click **Manage profiles** (or the **+** icon next to the profile dropdown) +3. Create a new profile, give it a name, and tick the categories you want +4. When adding or editing a mailbox, select your profile from the dropdown + +Two built-in profiles are always available and cannot be deleted: + +- **Documents Only** — PDF, Office, OpenDocument, Text, Web (no images) +- **All Files** — all categories including images + +Users can also create unlimited **custom profiles** to mix and match exactly the categories they need per mailbox (e.g. a scanner mailbox that only accepts PDFs, or a finance mailbox that accepts Office and CSV but not images). + +Custom profiles are created via the UI or the `/api/imap-profiles/` API. + --- ## Setting Up Your Scanner/Device diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 338346dd..25d79498 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -168,15 +168,15 @@ {{ _("nav.scheduled_jobs") }} + + Compliance + {{ _("nav.backup_restore") }} Audit Logs - - Audit Logs - {{ _("nav.status") }} @@ -389,6 +389,9 @@ {{ _("nav.scheduled_jobs") }} + + Compliance + {{ _("nav.backup_restore") }} diff --git a/frontend/templates/compliance.html b/frontend/templates/compliance.html new file mode 100644 index 00000000..06685253 --- /dev/null +++ b/frontend/templates/compliance.html @@ -0,0 +1,302 @@ +{% extends "base.html" %} +{% block title %}Compliance Templates - DocuElevate{% endblock %} + +{% block content %} +
+ +
+
+
+

Compliance Templates

+

+ Pre-built compliance configurations for GDPR, HIPAA, and SOC 2. + Apply templates with one click to align your instance with regulatory requirements. +

+
+ +
+
+ + +
+
+
+ + + +
+

+ +

+

+ of checks passing across all templates +

+
+
+
+
+
+
Passed
+
+
+
+
Failed
+
+
+
+
Total
+
+
+
+
+ + + + + +
+ +
+ + +
+ +

No compliance templates available

+

Templates will appear here after seeding.

+
+ + +
+ +

Loading compliance templates…

+
+
+ + +{% endblock %} diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 801d8186..347ac81b 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -524,6 +524,7 @@ + diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html index e22614e6..226bb190 100644 --- a/frontend/templates/imap_accounts.html +++ b/frontend/templates/imap_accounts.html @@ -172,6 +172,11 @@ Delete after process +