diff --git a/.env.demo b/.env.demo index 5bf0a14d..6920b998 100644 --- a/.env.demo +++ b/.env.demo @@ -3,10 +3,25 @@ WORKDIR=/workdir DATABASE_URL=sqlite:///./app/database.db REDIS_URL=redis://redis:6379/0 EXTERNAL_HOSTNAME=docuelevate.example.com +# PUBLIC_BASE_URL=https://docuelevate.example.com # Full URL with scheme; required when X-Forwarded-Proto is not forwarded by your proxy GOTENBERG_URL=http://gotenberg:3000 ALLOW_FILE_DELETE=true # Allow deletion of file records COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2) +# **Database Connection Pool** (PostgreSQL / MySQL only; ignored for SQLite) +# DB_POOL_SIZE=10 # Persistent connections per worker (default: 10) +# DB_MAX_OVERFLOW=20 # Extra connections under burst (default: 20) +# DB_POOL_TIMEOUT=30 # Seconds to wait for a pool connection (default: 30) +# DB_POOL_RECYCLE=1800 # Recycle connections after N seconds (default: 1800) + +# **Per-User Upload Rate Limiting** (health-aware, Redis-backed) +# UPLOAD_RATE_LIMIT_PER_USER=20 # Max uploads per user per window (default: 20) +# UPLOAD_RATE_LIMIT_WINDOW=60 # Sliding window in seconds (default: 60) + +# **System Reset / Factory Reset** +# FACTORY_RESET_ON_STARTUP=false # Wipe all user data on every startup (demo/testing only) +# ENABLE_FACTORY_RESET=false # Show the System Reset page in admin UI + # **Logging** # LOG_LEVEL controls the Python root-logger level. # Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO). @@ -160,6 +175,16 @@ AUTH_ENABLED=true # Generate a secure random string, for example: # python -c "import secrets; print(secrets.token_hex(32))" SESSION_SECRET=b39fd43f68d0491ca942f28a16e484b1e763fe9accf4445ca2669a5f3b179eb4 + +# Session lifetime in days (default: 30). Common values: 30, 60, 90. +# Determines how long a user stays logged in before needing to re-authenticate. +# SESSION_LIFETIME_DAYS=30 +# Override with a custom value (takes precedence over SESSION_LIFETIME_DAYS): +# SESSION_LIFETIME_CUSTOM_DAYS= + +# Time-to-live in seconds for QR code login challenges (default: 120 = 2 minutes). +# QR_LOGIN_CHALLENGE_TTL_SECONDS=120 + ADMIN_USERNAME=admin ADMIN_PASSWORD=your_secure_password ADMIN_GROUP_NAME=admin @@ -433,6 +458,15 @@ ONEDRIVE_TENANT_ID=common ONEDRIVE_REFRESH_TOKEN=your-refresh-token ONEDRIVE_FOLDER_PATH=Documents/Uploads +# SharePoint +SHAREPOINT_CLIENT_ID=your-client-id +SHAREPOINT_CLIENT_SECRET=your-client-secret +SHAREPOINT_TENANT_ID=common +SHAREPOINT_REFRESH_TOKEN=your-refresh-token +SHAREPOINT_SITE_URL=https://tenant.sharepoint.com/sites/sitename +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads + # WebDAV # WEBDAV_ENABLED=true # Set to false to disable WebDAV uploads without removing credentials WEBDAV_URL=https://webdav.example.com/path diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c206c46..45777958 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,18 @@ jobs: - run: ruff check app/ tests/ - run: ruff format --check app/ tests/ + migration-chain: + name: Alembic Migration Chain Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Validate migration chain + run: python scripts/check_alembic_migrations.py + html-lint: name: HTML Accessibility Lint runs-on: ubuntu-latest @@ -138,7 +150,7 @@ jobs: build: name: Build & Push Docker Image runs-on: ubuntu-latest - needs: [run-tests, mypy, dependency-scan, html-lint] + needs: [run-tests, mypy, dependency-scan, html-lint, migration-chain] if: github.event_name == 'push' steps: - name: Checkout Code diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1416b15c..773edaa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,16 @@ repos: .env.demo )$ + # Alembic migration chain validation + - repo: local + hooks: + - id: check-alembic-migrations + name: Check Alembic migration chain + entry: python scripts/check_alembic_migrations.py + language: python + pass_filenames: false + files: ^migrations/versions/.*\.py$ + # Conventional commits validation - repo: https://github.com/compilerla/conventional-pre-commit rev: v3.0.0 diff --git a/BUILD_DATE b/BUILD_DATE index 07f63328..ab2725eb 100644 --- a/BUILD_DATE +++ b/BUILD_DATE @@ -1 +1 @@ -2026-03-16T21:40:03Z +2026-03-20T23:38:07Z diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8e8606..1d59d41d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,339 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## v0.163.0 (2026-03-20) + +### Bug Fixes + +- **security**: Escape HTML in folder browser to prevent XSS from folder names + ([`b3a2387`](https://github.com/christianlouis/DocuElevate/commit/b3a238744d1618b7f891b7c43f243dd9099ecf49)) + +### Code Style + +- Apply ruff auto-fix + ([`65cf33c`](https://github.com/christianlouis/DocuElevate/commit/65cf33ce89cfcb7f81f31278a2cf6523c32a4754)) + +### Documentation + +- Update setup guides and API docs for folder browser and system credentials + ([`d6c21b8`](https://github.com/christianlouis/DocuElevate/commit/d6c21b8026ccc8fb71ed12394faba08214e0793d)) + +### Features + +- **api**: Add folder browser API endpoints and UI for Dropbox and OneDrive + ([`a842306`](https://github.com/christianlouis/DocuElevate/commit/a8423064ec72bf8014ca37936960c5da9ddcea1f)) + +- **auth**: Default to system-wide app credentials in OAuth wizards for user mode + ([`23c8c76`](https://github.com/christianlouis/DocuElevate/commit/23c8c76b392a95e723f25f8cb3714d4e5f256b30)) + +- **ui**: Replace manual credential fields with OAuth wizard flow for watch folder sources + ([`ed01952`](https://github.com/christianlouis/DocuElevate/commit/ed0195261032fb92df5f655a73f056ac380fe8b6)) + +### Testing + +- Add tests for folder browser APIs and system credentials toggle + ([`1e1e6e6`](https://github.com/christianlouis/DocuElevate/commit/1e1e6e62807a6d4b7bc84b2d8c967dceb97f1cb6)) + + +## v0.162.0 (2026-03-20) + + +## v0.161.0 (2026-03-20) + +### Documentation + +- Update scaling, health probe, and beat scheduler documentation + ([`4d019d5`](https://github.com/christianlouis/DocuElevate/commit/4d019d53d9ad161a68639d6ba2109ba3ac77df34)) + +### Features + +- **scaling**: Enable horizontal scaling for API and worker pods + ([`f75b125`](https://github.com/christianlouis/DocuElevate/commit/f75b12599291050f97aa452d980f17547ddd7bd6)) + + +## v0.160.3 (2026-03-20) + +### Bug Fixes + +- **mobile**: Wire i18n reactivity, translate all screens, sync language with server + ([`3b5ca04`](https://github.com/christianlouis/DocuElevate/commit/3b5ca04ebc8b6dd20f877bc4797e364e0997d840)) + +### Chores + +- **mobile**: Upgrade ESLint to v9 with flat config and fix expo-localization version + ([`0e6a4c5`](https://github.com/christianlouis/DocuElevate/commit/0e6a4c5084b3704653965f0091c36b9c78b8ad60)) + + +## v0.160.2 (2026-03-20) + +### Bug Fixes + +- **dropbox**: Fix Invalid redirect_uri error by adding PUBLIC_BASE_URL config and URL-encoding + ([`5e3e2b1`](https://github.com/christianlouis/DocuElevate/commit/5e3e2b19997d3af6570fbaa1e75a49cbfe6cf78d)) + + +## v0.160.1 (2026-03-20) + +### Bug Fixes + +- **mobile**: Update expo-localization version from ~16.0.6 to ~16.1.0 + ([`78c3717`](https://github.com/christianlouis/DocuElevate/commit/78c3717661923b43f1762fa7728c75e803938fb7)) + + +## v0.160.0 (2026-03-20) + +### Bug Fixes + +- **mobile**: Address code review feedback - error handling, filename collision, hash display + ([`6541529`](https://github.com/christianlouis/DocuElevate/commit/65415292507aa37408428400aef7e87e006abfd0)) + +### Features + +- **mobile**: Add pre-login legal pages, multi-image selection, file detail view, search, i18n, HEIC + support + ([`67c17e7`](https://github.com/christianlouis/DocuElevate/commit/67c17e7baa8edf76be394d0aff42c2adeae351e1)) + + +## v0.159.0 (2026-03-19) + +### Code Style + +- Apply ruff auto-fix + ([`910fb29`](https://github.com/christianlouis/DocuElevate/commit/910fb297ba1122b751250b323a330e15e276daf6)) + +### Features + +- **integrations**: Add Dropbox connection test and global-credential sharing + ([`d1f9819`](https://github.com/christianlouis/DocuElevate/commit/d1f9819f4e12320bb901b76366fa8d4a8cd23a66)) + + +## v0.158.4 (2026-03-19) + +### Bug Fixes + +- **ui**: Show proper error when signup username has invalid characters + ([`689c616`](https://github.com/christianlouis/DocuElevate/commit/689c616e448bf3edfcb91b347ae9f6fac43b1983)) + + +## v0.158.3 (2026-03-19) + +### Bug Fixes + +- **upload**: Reject exact duplicates at upload time and prevent duplicate mobile uploads + ([`d5c18cc`](https://github.com/christianlouis/DocuElevate/commit/d5c18ccf07efa28532724a78e63cc6bd7eab9515)) + +### Refactoring + +- **mobile**: Extract normalizeFileUri to shared utility module + ([`ec88221`](https://github.com/christianlouis/DocuElevate/commit/ec882214e29d22da9d8025ea857ba754fbddae37)) + + +## v0.158.2 (2026-03-19) + +### Bug Fixes + +- **mobile**: Add user feedback when server URL is unavailable + ([`1366317`](https://github.com/christianlouis/DocuElevate/commit/136631762bc81399aecd5766663da7d923cba985)) + +- **mobile**: Apple App Store compliance fixes + ([`5c15a23`](https://github.com/christianlouis/DocuElevate/commit/5c15a2395a0dcfa5fe5cf1a34a98ae743d29eabc)) + +- **mobile**: Fix file sharing deep-link conflicts and add MIME type inference + ([`1559686`](https://github.com/christianlouis/DocuElevate/commit/1559686f903808e2ab4547f21575460aaeca839e)) + +- **mobile**: Fix shared file upload hanging by copying to cache + ([`f549505`](https://github.com/christianlouis/DocuElevate/commit/f549505bfd625100e77a170cfcae372913680835)) + +### Documentation + +- Add Apple App Store Compliance audit report + ([`1572f32`](https://github.com/christianlouis/DocuElevate/commit/1572f322d72583c45c88e55ea51b2456e548de4c)) + +### Refactoring + +- **mobile**: Extract shared MIME type utility and improve error handling + ([`cfe83d7`](https://github.com/christianlouis/DocuElevate/commit/cfe83d7efa7f93ca51bfcac417cfafc650bbc36b)) + + +## v0.158.1 (2026-03-19) + +### Bug Fixes + +- **mobile**: Add shared file to ShareContext directly in +not-found.tsx + ([`71a7a57`](https://github.com/christianlouis/DocuElevate/commit/71a7a57adc1bb6de88c060dba563afa433283e38)) + + +## v0.158.0 (2026-03-19) + +### Bug Fixes + +- Address code review feedback (assertion, exc_info logging) + ([`faa68ad`](https://github.com/christianlouis/DocuElevate/commit/faa68adaa143774a8757fda4fd0248cc3aef551e)) + +- **config**: Add SETTING_METADATA for db pool and upload rate limit settings + ([`1d7286c`](https://github.com/christianlouis/DocuElevate/commit/1d7286c4c68de903951d9819bcb58ae1300dab7b)) + +- **config**: Remove duplicate dictionary keys and class fields from merge + ([`d34b8bc`](https://github.com/christianlouis/DocuElevate/commit/d34b8bceb9f96719bf5c922156d5487a4c6426b4)) + +- **db**: Use NullPool for SQLite and expose pool tuning settings + ([`56bf665`](https://github.com/christianlouis/DocuElevate/commit/56bf66539757cf1d98fe251ad64fb75b481462ef)) + +- **tests**: Add docstring to rate limiter no-op override + ([`c9f9001`](https://github.com/christianlouis/DocuElevate/commit/c9f900124407baa230b38f6e1d0f5ee78e0bedcc)) + +- **tests**: Disable upload rate limiter in test client fixture + ([`6bf121f`](https://github.com/christianlouis/DocuElevate/commit/6bf121f02f2c32f7b73435f42d30b9681710089e)) + +### Features + +- **api**: Add per-user health-aware upload rate limiting + ([`571cc81`](https://github.com/christianlouis/DocuElevate/commit/571cc817893a1f4cbd7cfbee7f1c46ca1a4f14d8)) + + +## v0.157.2 (2026-03-19) + +### Bug Fixes + +- **ui**: Improve devices page table layout to prevent horizontal scrolling + ([`b12e891`](https://github.com/christianlouis/DocuElevate/commit/b12e8916824273eefb85f0bb498d4991e8c66d5e)) + + +## v0.157.1 (2026-03-19) + +### Bug Fixes + +- **mobile**: Resolve iOS "unmatched route docuelevate://" error in Open In share flow + ([`f2b7db8`](https://github.com/christianlouis/DocuElevate/commit/f2b7db88ba86e894a9bf2e4e7d6a54cd7f423194)) + + +## v0.157.0 (2026-03-19) + +### Features + +- **api**: Allow disabled tokens/devices to be deleted & reactivated; add token lifetime + ([`e4749b4`](https://github.com/christianlouis/DocuElevate/commit/e4749b4e7cdd78ab95627736d3fcd84fecc55e53)) + + +## v0.156.3 (2026-03-18) + +### Bug Fixes + +- **auth**: Exempt /api/qr-auth/claim from CSRF to fix mobile QR login + ([`a4aaebf`](https://github.com/christianlouis/DocuElevate/commit/a4aaebfe6649ef051790b6af24e0739bdd9ecfed)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`d8d2016`](https://github.com/christianlouis/DocuElevate/commit/d8d2016f8597f775cdf3848e7baeaae98cb998b0)) + + +## Unreleased + + +## v0.156.2 (2026-03-18) + +### Bug Fixes + +- **qr-login**: Render QR code server-side using segno instead of CDN JS library + ([`a8eb650`](https://github.com/christianlouis/DocuElevate/commit/a8eb6504ac100c14b10f57511fde8f1202deeffe)) + +### Chores + +- Initial plan for server-side QR code rendering + ([`6727253`](https://github.com/christianlouis/DocuElevate/commit/6727253958a6ae8436fc1184736ec43eef2ab820)) + +- Remove accidentally committed =1.6.0 file + ([`ba8c88b`](https://github.com/christianlouis/DocuElevate/commit/ba8c88bc17d3245cba6079ac5ba3be5fddf36366)) + + +## v0.156.1 (2026-03-18) + +### Bug Fixes + +- Add missing SETTING_METADATA entries for db pool and upload rate limit settings + ([`dc0a19b`](https://github.com/christianlouis/DocuElevate/commit/dc0a19bd118d3503ad50608f55fb0bc4ce104948)) + + +## v0.156.0 (2026-03-17) + +### Bug Fixes + +- **storage**: Use RuntimeError instead of bare Exception in SharePoint task + ([`2b698cc`](https://github.com/christianlouis/DocuElevate/commit/2b698cc6940fb731b1ab87300ad0f7fdebc8f024)) + +- **test**: Add missing _should_upload_to_sharepoint mock to send_to_all tests + ([`bcf2d00`](https://github.com/christianlouis/DocuElevate/commit/bcf2d00c3324a3ded3852e16759ea4d0af3af666)) + +### Documentation + +- Add SharePoint setup guide and update all references + ([`13aa14b`](https://github.com/christianlouis/DocuElevate/commit/13aa14b8e4102437f72f6c260b2795a6ee761eb9)) + +### Features + +- **storage**: Add SharePoint integration for document storage + ([`b85fc1d`](https://github.com/christianlouis/DocuElevate/commit/b85fc1d277475c06c1efa100c391b7a3c43e7c25)) + + +## v0.155.1 (2026-03-17) + +### Bug Fixes + +- **api**: Add ttl_seconds to QR challenge response and fix client-side countdown + ([`0f6a1ee`](https://github.com/christianlouis/DocuElevate/commit/0f6a1ee1ec8186d70afc17abe50258c968058c92)) + +- **mobile**: Replace gap with marginLeft for React Native compatibility + ([`70b193e`](https://github.com/christianlouis/DocuElevate/commit/70b193e07d2f85e353b6b12c56bf5bcbf828ee24)) + +### Documentation + +- Update QR code login documentation with scanner and TTL details + ([`723b14e`](https://github.com/christianlouis/DocuElevate/commit/723b14e660737887c454b8e8300ac38bb390841f)) + + +## v0.155.0 (2026-03-17) + + +## v0.154.0 (2026-03-17) + +### Bug Fixes + +- **db**: Address code review feedback - fix comment stripping, type hints, test skip, and docs + ([`a007b4f`](https://github.com/christianlouis/DocuElevate/commit/a007b4fd98ed327a9874082766e0f3b8c32a1a68)) + +### Features + +- **db**: Add migration chain CI validation, pre-commit hook, script template, docs, and tests + ([`aa49fa3`](https://github.com/christianlouis/DocuElevate/commit/aa49fa3ae6ae0bfbb0f9efa6304cf109eca1c402)) + + +## v0.153.1 (2026-03-16) + +### Bug Fixes + +- **mobile**: Add root index.tsx redirect to prevent stale Hello World screen + ([`1350aa6`](https://github.com/christianlouis/DocuElevate/commit/1350aa6a5ede19d924a9a4c277ef8089b940ea3c)) + + +## v0.153.0 (2026-03-16) + +### Bug Fixes + +- **system-reset**: Address code review feedback + ([`7dffdc0`](https://github.com/christianlouis/DocuElevate/commit/7dffdc05542cbfe37a91f86b661790038195fef5)) + +### Features + +- **system-reset**: Add system reset and factory reset feature + ([`a88d790`](https://github.com/christianlouis/DocuElevate/commit/a88d790445e5aae9f8ba04e735501d6c93589dec)) + +### Testing + +- **system-reset**: Add comprehensive tests and documentation + ([`96bfba8`](https://github.com/christianlouis/DocuElevate/commit/96bfba8057e506f561d21318b0205e521cf67109)) + + ## v0.152.0 (2026-03-16) ### Features diff --git a/GIT_SHA b/GIT_SHA index e19a3133..b98bde3f 100644 --- a/GIT_SHA +++ b/GIT_SHA @@ -1 +1 @@ -793b6d3 +6f5a73f diff --git a/RUNTIME_INFO b/RUNTIME_INFO index a17ef9cc..3d9f8b3b 100644 --- a/RUNTIME_INFO +++ b/RUNTIME_INFO @@ -1,10 +1,10 @@ DocuElevate Build Information ============================== -Version: 0.152.0 -Build Date: 2026-03-16T21:40:03Z -Git Commit: 793b6d3e2562518cb404579f656e82a6dc60da02 -Git Short SHA: 793b6d3 +Version: 0.163.0 +Build Date: 2026-03-20T23:38:07Z +Git Commit: 6f5a73f98ae3e890a232e5202521b1a263cfe5d0 +Git Short SHA: 6f5a73f Git Branch: main -Commit Date: 2026-03-16T22:39:40+01:00 -Build Timestamp: 2026-03-16T21:40:03Z +Commit Date: 2026-03-21T00:37:49+01:00 +Build Timestamp: 2026-03-20T23:38:07Z ============================== diff --git a/VERSION b/VERSION index a30a0fd9..599e9b4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.152.0 +0.163.0 diff --git a/app/api/__init__.py b/app/api/__init__.py index 7c5ff2bb..de5c41e8 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -13,6 +13,7 @@ from app.api.automation import router as automation_router from app.api.azure import router as azure_router from app.api.backup import router as backup_router from app.api.billing import router as billing_router +from app.api.classification_rules import router as classification_rules_router from app.api.compliance import router as compliance_router from app.api.database import router as database_router from app.api.diagnostic import router as diagnostic_router @@ -34,16 +35,19 @@ from app.api.pipelines import router as pipelines_router from app.api.plans import router as plans_router from app.api.process import router as process_router from app.api.profile import router as profile_router +from app.api.qr_auth import router as qr_auth_router from app.api.queue import router as queue_router from app.api.routing_rules import router as routing_rules_router from app.api.saved_searches import router as saved_searches_router from app.api.scheduled_jobs import router as scheduled_jobs_router from app.api.search import router as search_router +from app.api.sessions import router as sessions_router from app.api.settings import router as settings_router from app.api.shared_links import public_router as shared_links_public_router from app.api.shared_links import router as shared_links_router from app.api.similarity import router as similarity_router from app.api.subscriptions import router as subscriptions_router +from app.api.system_reset import router as system_reset_router from app.api.translation import router as translation_router from app.api.url_upload import router as url_upload_router @@ -97,6 +101,10 @@ router.include_router(scheduled_jobs_router) router.include_router(audit_logs_router) router.include_router(i18n_router) router.include_router(mobile_router) +router.include_router(sessions_router) +router.include_router(qr_auth_router) router.include_router(compliance_router) +router.include_router(system_reset_router) router.include_router(translation_router) +router.include_router(classification_rules_router) router.include_router(automation_router) diff --git a/app/api/api_tokens.py b/app/api/api_tokens.py index a53ba7f2..62074b8a 100644 --- a/app/api/api_tokens.py +++ b/app/api/api_tokens.py @@ -13,7 +13,7 @@ plaintext is returned exactly once at creation time. import hashlib import logging import secrets -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Annotated, Any from fastapi import APIRouter, Depends, HTTPException, Request, status @@ -42,6 +42,9 @@ TOKEN_HASH_ITERATIONS = 100_000 #: PBKDF2 salt for API token hashing (not secret, but fixed for determinism). TOKEN_HASH_SALT = b"api-token-v1" +#: Name prefix used for tokens created by the mobile app flow. +MOBILE_TOKEN_PREFIX = "Mobile App" + # --------------------------------------------------------------------------- # Auth helper @@ -91,6 +94,21 @@ def hash_token(token: str) -> str: return dk.hex() +def _token_to_dict(t: ApiToken) -> dict[str, Any]: + """Convert an ``ApiToken`` ORM instance to a serialisable dict.""" + return { + "id": t.id, + "name": t.name, + "token_prefix": t.token_prefix, + "is_active": t.is_active, + "last_used_at": t.last_used_at, + "last_used_ip": t.last_used_ip, + "created_at": t.created_at, + "revoked_at": t.revoked_at, + "expires_at": t.expires_at, + } + + # --------------------------------------------------------------------------- # Pydantic schemas # --------------------------------------------------------------------------- @@ -100,6 +118,12 @@ class TokenCreate(BaseModel): """Schema for creating a new API token.""" name: str = Field(..., min_length=1, max_length=255, description="Human-readable label for the token") + expires_in_days: int | None = Field( + default=None, + ge=1, + le=3650, # Maximum 10 years; keeps tokens from being effectively permanent while allowing long-lived CI/CD tokens. + description="Optional lifetime in days. If omitted the token never expires.", + ) class TokenResponse(BaseModel): @@ -113,6 +137,7 @@ class TokenResponse(BaseModel): last_used_ip: str | None created_at: datetime | None revoked_at: datetime | None + expires_at: datetime | None model_config = {"from_attributes": True} @@ -143,11 +168,16 @@ async def create_token( token_hash_value = hash_token(plaintext) prefix = plaintext[:12] # "de_" prefix + 9 random chars = 12 chars total + expires_at = None + if body.expires_in_days is not None: + expires_at = datetime.now(timezone.utc) + timedelta(days=body.expires_in_days) + db_token = ApiToken( owner_id=owner_id, name=body.name, token_hash=token_hash_value, token_prefix=prefix, + expires_at=expires_at, ) try: db.add(db_token) @@ -168,6 +198,7 @@ async def create_token( "last_used_ip": db_token.last_used_ip, "created_at": db_token.created_at, "revoked_at": db_token.revoked_at, + "expires_at": db_token.expires_at, "token": plaintext, } @@ -177,48 +208,114 @@ async def list_tokens( owner_id: CurrentOwner, db: DbSession, ) -> list[dict[str, Any]]: - """List all API tokens for the authenticated user.""" - tokens = db.query(ApiToken).filter(ApiToken.owner_id == owner_id).order_by(ApiToken.created_at.desc()).all() - return [ - { - "id": t.id, - "name": t.name, - "token_prefix": t.token_prefix, - "is_active": t.is_active, - "last_used_at": t.last_used_at, - "last_used_ip": t.last_used_ip, - "created_at": t.created_at, - "revoked_at": t.revoked_at, - } - for t in tokens - ] + """List non-mobile API tokens for the authenticated user. + + Mobile tokens (whose names start with ``"Mobile App"``) are excluded + from this list; they are managed on the dedicated Devices page via + ``GET /api/api-tokens/mobile``. + """ + tokens = ( + db.query(ApiToken) + .filter( + ApiToken.owner_id == owner_id, + ~ApiToken.name.startswith(MOBILE_TOKEN_PREFIX), + ) + .order_by(ApiToken.created_at.desc()) + .all() + ) + return [_token_to_dict(t) for t in tokens] + + +@router.get("/mobile", response_model=list[TokenResponse]) +async def list_mobile_tokens( + owner_id: CurrentOwner, + db: DbSession, +) -> list[dict[str, Any]]: + """List mobile API tokens for the authenticated user. + + Returns tokens whose names start with ``"Mobile App"`` — these are + created via the mobile SSO flow or QR code login. + """ + tokens = ( + db.query(ApiToken) + .filter( + ApiToken.owner_id == owner_id, + ApiToken.name.startswith(MOBILE_TOKEN_PREFIX), + ) + .order_by(ApiToken.created_at.desc()) + .all() + ) + return [_token_to_dict(t) for t in tokens] @router.delete("/{token_id}", status_code=status.HTTP_200_OK) -async def revoke_token( +async def revoke_or_delete_token( token_id: int, owner_id: CurrentOwner, db: DbSession, ) -> dict[str, str]: - """Revoke (soft-delete) an API token. + """Revoke or permanently delete an API token. - The token row is kept for audit purposes but marked inactive with a - ``revoked_at`` timestamp. + * **Active token** – soft-revoked: the row is kept for audit purposes + but marked inactive with a ``revoked_at`` timestamp. + * **Already-revoked token** – hard-deleted: the row is permanently + removed from the database. """ db_token = db.query(ApiToken).filter(ApiToken.id == token_id, ApiToken.owner_id == owner_id).first() if not db_token: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found") - if not db_token.is_active: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Token is already revoked") + if db_token.is_active: + # Soft-revoke the active token. + try: + db_token.is_active = False + db_token.revoked_at = datetime.now(timezone.utc) + db.commit() + except Exception: + db.rollback() + raise + logger.info("API token revoked: id=%s owner=%s", token_id, owner_id) + return {"detail": "Token revoked"} + # Hard-delete an already-revoked token. try: - db_token.is_active = False - db_token.revoked_at = datetime.now(timezone.utc) + db.delete(db_token) db.commit() except Exception: db.rollback() raise + logger.info("API token permanently deleted: id=%s owner=%s", token_id, owner_id) + return {"detail": "Token deleted"} - logger.info("API token revoked: id=%s owner=%s", token_id, owner_id) - return {"detail": "Token revoked"} + +@router.post("/{token_id}/reactivate", status_code=status.HTTP_200_OK, response_model=TokenResponse) +async def reactivate_token( + token_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Reactivate a previously revoked API token. + + Clears the ``revoked_at`` timestamp and sets ``is_active`` back to + ``True``. The token can be used for authentication again immediately. + If the token had an ``expires_at`` in the past the caller should + consider re-creating a new token instead. + """ + db_token = db.query(ApiToken).filter(ApiToken.id == token_id, ApiToken.owner_id == owner_id).first() + if not db_token: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found") + + if db_token.is_active: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Token is already active") + + try: + db_token.is_active = True + db_token.revoked_at = None + db.commit() + db.refresh(db_token) + except Exception: + db.rollback() + raise + + logger.info("API token reactivated: id=%s owner=%s", token_id, owner_id) + return _token_to_dict(db_token) diff --git a/app/api/classification_rules.py b/app/api/classification_rules.py new file mode 100644 index 00000000..594ade24 --- /dev/null +++ b/app/api/classification_rules.py @@ -0,0 +1,325 @@ +"""Classification Rules API endpoints. + +Provides CRUD operations for managing custom document classification rules. +System-wide rules (``owner_id IS NULL``) can only be managed by admins. +""" + +from __future__ import annotations + +import logging +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.database import get_db +from app.models import ClassificationRuleModel +from app.utils.classification_rules import ( + BUILTIN_CATEGORIES, + RULE_TYPE_CONTENT, + RULE_TYPE_FILENAME, + RULE_TYPE_METADATA, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/classification-rules", tags=["classification"]) + +DbSession = Annotated[Session, Depends(get_db)] + +_VALID_RULE_TYPES = {RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _get_user_id(request: Request) -> str: + """Extract the user identifier from the request session.""" + user = getattr(request.state, "user", None) + if user and hasattr(user, "get"): + return user.get("sub") or user.get("email") or "anonymous" + return "anonymous" + + +def _is_admin(request: Request) -> bool: + """Check whether the current user is an admin.""" + user = getattr(request.state, "user", None) + if user and hasattr(user, "get"): + groups = user.get("groups", []) + return "admin" in groups or "Admin" in groups + return False + + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + + +class RuleCreate(BaseModel): + """Schema for creating a classification rule.""" + + name: str = Field(..., min_length=1, max_length=255) + category: str = Field(..., min_length=1, max_length=100) + rule_type: str = Field(..., description="One of: filename_pattern, content_keyword, metadata_match") + pattern: str = Field(..., min_length=1, max_length=1000) + priority: int = Field(default=0, ge=0, le=1000) + case_sensitive: bool = False + enabled: bool = True + + +class RuleUpdate(BaseModel): + """Schema for updating a classification rule.""" + + name: str | None = Field(default=None, min_length=1, max_length=255) + category: str | None = Field(default=None, min_length=1, max_length=100) + rule_type: str | None = Field(default=None) + pattern: str | None = Field(default=None, min_length=1, max_length=1000) + priority: int | None = Field(default=None, ge=0, le=1000) + case_sensitive: bool | None = None + enabled: bool | None = None + + +class RuleResponse(BaseModel): + """Schema for a classification rule response.""" + + id: int + owner_id: str | None + name: str + category: str + rule_type: str + pattern: str + priority: int + case_sensitive: bool + enabled: bool + + model_config = {"from_attributes": True} + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/categories") +@require_login +async def list_categories(request: Request) -> dict[str, str]: + """Return all built-in classification categories. + + Custom categories created via rules are not included here; they are + discovered dynamically when rules are evaluated. + """ + return BUILTIN_CATEGORIES + + +@router.get("/rule-types") +@require_login +async def list_rule_types(request: Request) -> list[dict[str, str]]: + """Return the supported rule types with descriptions.""" + return [ + { + "type": RULE_TYPE_FILENAME, + "label": "Filename Pattern", + "description": "Regex pattern matched against the original filename.", + }, + { + "type": RULE_TYPE_CONTENT, + "label": "Content Keyword", + "description": "Pipe-separated keywords matched against the OCR text.", + }, + { + "type": RULE_TYPE_METADATA, + "label": "Metadata Match", + "description": "field=value pattern matched against existing AI metadata.", + }, + ] + + +@router.get("/") +@require_login +async def list_rules(request: Request, db: DbSession) -> list[dict[str, Any]]: + """List classification rules visible to the current user. + + Returns both system rules (``owner_id IS NULL``) and the user's own rules. + """ + user_id = _get_user_id(request) + rules = ( + db.query(ClassificationRuleModel) + .filter((ClassificationRuleModel.owner_id.is_(None)) | (ClassificationRuleModel.owner_id == user_id)) + .order_by(ClassificationRuleModel.priority.desc(), ClassificationRuleModel.id) + .all() + ) + return [ + { + "id": r.id, + "owner_id": r.owner_id, + "name": r.name, + "category": r.category, + "rule_type": r.rule_type, + "pattern": r.pattern, + "priority": r.priority, + "case_sensitive": r.case_sensitive, + "enabled": r.enabled, + } + for r in rules + ] + + +@router.post("/", status_code=status.HTTP_201_CREATED) +@require_login +async def create_rule(request: Request, body: RuleCreate, db: DbSession) -> dict[str, Any]: + """Create a new custom classification rule. + + The rule is owned by the current user. Admins may create system-wide + rules by setting ``owner_id`` to ``null`` (not yet exposed). + """ + if body.rule_type not in _VALID_RULE_TYPES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid rule_type. Must be one of: {', '.join(sorted(_VALID_RULE_TYPES))}", + ) + + user_id = _get_user_id(request) + + # Check for duplicate name within the user's scope + existing = ( + db.query(ClassificationRuleModel) + .filter(ClassificationRuleModel.owner_id == user_id, ClassificationRuleModel.name == body.name) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"A rule named '{body.name}' already exists.", + ) + + rule = ClassificationRuleModel( + owner_id=user_id, + name=body.name, + category=body.category, + rule_type=body.rule_type, + pattern=body.pattern, + priority=body.priority, + case_sensitive=body.case_sensitive, + enabled=body.enabled, + ) + try: + db.add(rule) + db.commit() + db.refresh(rule) + except Exception: + db.rollback() + raise + + logger.info("Classification rule created: id=%s, user=%s", rule.id, user_id) + return { + "id": rule.id, + "owner_id": rule.owner_id, + "name": rule.name, + "category": rule.category, + "rule_type": rule.rule_type, + "pattern": rule.pattern, + "priority": rule.priority, + "case_sensitive": rule.case_sensitive, + "enabled": rule.enabled, + } + + +@router.get("/{rule_id}") +@require_login +async def get_rule(request: Request, rule_id: int, db: DbSession) -> dict[str, Any]: + """Get a single classification rule by ID.""" + user_id = _get_user_id(request) + rule = db.query(ClassificationRuleModel).filter(ClassificationRuleModel.id == rule_id).first() + if rule is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule not found") + + # Users can see system rules and their own rules + if rule.owner_id is not None and rule.owner_id != user_id and not _is_admin(request): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule not found") + + return { + "id": rule.id, + "owner_id": rule.owner_id, + "name": rule.name, + "category": rule.category, + "rule_type": rule.rule_type, + "pattern": rule.pattern, + "priority": rule.priority, + "case_sensitive": rule.case_sensitive, + "enabled": rule.enabled, + } + + +@router.put("/{rule_id}") +@require_login +async def update_rule(request: Request, rule_id: int, body: RuleUpdate, db: DbSession) -> dict[str, Any]: + """Update an existing classification rule. + + Users can only update their own rules. Admins can update any rule. + """ + user_id = _get_user_id(request) + rule = db.query(ClassificationRuleModel).filter(ClassificationRuleModel.id == rule_id).first() + if rule is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule not found") + + if rule.owner_id != user_id and not _is_admin(request): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot modify this rule") + + if body.rule_type is not None and body.rule_type not in _VALID_RULE_TYPES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid rule_type. Must be one of: {', '.join(sorted(_VALID_RULE_TYPES))}", + ) + + update_data = body.model_dump(exclude_unset=True) + for field_name, value in update_data.items(): + setattr(rule, field_name, value) + + try: + db.commit() + db.refresh(rule) + except Exception: + db.rollback() + raise + + logger.info("Classification rule updated: id=%s, user=%s", rule.id, user_id) + return { + "id": rule.id, + "owner_id": rule.owner_id, + "name": rule.name, + "category": rule.category, + "rule_type": rule.rule_type, + "pattern": rule.pattern, + "priority": rule.priority, + "case_sensitive": rule.case_sensitive, + "enabled": rule.enabled, + } + + +@router.delete("/{rule_id}", status_code=status.HTTP_204_NO_CONTENT) +@require_login +async def delete_rule(request: Request, rule_id: int, db: DbSession) -> None: + """Delete a classification rule. + + Users can only delete their own rules. Admins can delete any rule. + """ + user_id = _get_user_id(request) + rule = db.query(ClassificationRuleModel).filter(ClassificationRuleModel.id == rule_id).first() + if rule is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule not found") + + if rule.owner_id != user_id and not _is_admin(request): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Cannot delete this rule") + + try: + db.delete(rule) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("Classification rule deleted: id=%s, user=%s", rule_id, user_id) diff --git a/app/api/diagnostic.py b/app/api/diagnostic.py index a329da46..d21a3d39 100644 --- a/app/api/diagnostic.py +++ b/app/api/diagnostic.py @@ -21,6 +21,67 @@ _DEFAULT_REDIS_URL = "redis://localhost:6379/0" router = APIRouter() +# --------------------------------------------------------------------------- +# Unauthenticated probe endpoints for Kubernetes liveness / readiness checks. +# These intentionally skip authentication so that kubelet can reach them +# without credentials. They live under /diagnostic/healthz/* so that the +# existing authenticated /diagnostic/health endpoint is unaffected. +# --------------------------------------------------------------------------- + + +@router.get("/diagnostic/healthz/live") +async def liveness_probe() -> JSONResponse: + """Lightweight liveness probe for Kubernetes. + + Returns **200 OK** as long as the process is running. Kubernetes uses + this to decide whether to *restart* the container — it should therefore + be as cheap as possible and **never** check external dependencies. + + **Authentication:** None (designed for kubelet probes). + """ + return JSONResponse(content={"status": "ok"}, status_code=200) + + +@router.get("/diagnostic/healthz/ready") +async def readiness_probe() -> JSONResponse: + """Readiness probe for Kubernetes. + + Verifies that the application can serve traffic by checking the database + and Redis. Kubernetes uses this to decide whether to *route traffic* to + the pod. + + Returns **200 OK** when all critical subsystems are reachable, or + **503 Service Unavailable** when the database is down. + + **Authentication:** None (designed for kubelet probes). + """ + checks: dict[str, dict[str, str]] = {} + db_ok = False + + # ── Database check ───────────────────────────────────────────────── + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + checks["database"] = {"status": "ok"} + db_ok = True + except Exception as exc: + logger.warning("Readiness probe: database check failed: %s", exc) + checks["database"] = {"status": "error", "detail": str(exc)} + + # ── Redis check ──────────────────────────────────────────────────── + try: + redis_url = settings.redis_url or _DEFAULT_REDIS_URL + r = redis_lib.from_url(redis_url, socket_connect_timeout=2, socket_timeout=2) + r.ping() + checks["redis"] = {"status": "ok"} + except Exception as exc: + logger.warning("Readiness probe: Redis check failed: %s", exc) + checks["redis"] = {"status": "error", "detail": str(exc)} + + http_status = 503 if not db_ok else 200 + overall = "ready" if db_ok else "not_ready" + return JSONResponse(content={"status": overall, "checks": checks}, status_code=http_status) + @router.get("/diagnostic/health") @require_login diff --git a/app/api/dropbox.py b/app/api/dropbox.py index da52c758..86f72a25 100644 --- a/app/api/dropbox.py +++ b/app/api/dropbox.py @@ -5,6 +5,7 @@ Dropbox API endpoints import logging import os from typing import Annotated, Optional +from urllib.parse import quote import httpx from fastapi import APIRouter, Depends, Form, HTTPException, Request, status @@ -23,6 +24,93 @@ logger = logging.getLogger(__name__) router = APIRouter() +def _build_dropbox_redirect_uri(request: Request) -> str: + """Build the Dropbox OAuth callback redirect URI. + + Uses ``PUBLIC_BASE_URL`` when configured (recommended for deployments behind + a reverse proxy that doesn't forward ``X-Forwarded-Proto``). Falls back to + deriving the URI from the incoming request's scheme and host headers. + """ + if settings.public_base_url: + return settings.public_base_url.rstrip("/") + "/dropbox-callback" + return f"{request.url.scheme}://{request.url.netloc}/dropbox-callback" + + +@router.get("/dropbox/global-authorize-url") +@require_login +async def dropbox_global_authorize_url(request: Request): + """Return the Dropbox OAuth authorization URL using the global app credentials. + + This endpoint is used when ``DROPBOX_ALLOW_GLOBAL_CREDENTIALS_FOR_INTEGRATIONS`` + is enabled so that users can authorize their personal Dropbox integration without + needing to supply their own app key/secret. Only the public ``app_key`` is + embedded in the URL; the ``app_secret`` is never sent to the browser. + """ + if not settings.dropbox_allow_global_credentials_for_integrations: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Global credentials for integrations are not enabled", + ) + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Global Dropbox credentials are not configured", + ) + redirect_uri = _build_dropbox_redirect_uri(request) + authorize_url = ( + "https://www.dropbox.com/oauth2/authorize" + f"?client_id={settings.dropbox_app_key}" + "&response_type=code" + "&token_access_type=offline" + f"&redirect_uri={quote(redirect_uri, safe='')}" + ) + return {"authorize_url": authorize_url} + + +@router.post("/dropbox/exchange-token-global") +@require_login +async def exchange_dropbox_token_global( + request: Request, + code: Annotated[str, Form(...)], + redirect_uri: Annotated[str, Form(...)], +): + """Exchange an authorization code using the global Dropbox app credentials. + + Used when ``DROPBOX_ALLOW_GLOBAL_CREDENTIALS_FOR_INTEGRATIONS`` is enabled so + that the ``app_secret`` is never exposed to the browser. Only the OAuth code + and redirect URI need to be supplied by the client. + """ + if not settings.dropbox_allow_global_credentials_for_integrations: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Global credentials for integrations are not enabled", + ) + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Global Dropbox credentials are not configured", + ) + + token_url = "https://api.dropboxapi.com/oauth2/token" + payload = { + "client_id": settings.dropbox_app_key, + "client_secret": settings.dropbox_app_secret, + "code": code, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + } + + token_data = exchange_oauth_token(provider_name="Dropbox", token_url=token_url, payload=payload) + + return { + "refresh_token": token_data["refresh_token"], + "access_token": token_data["access_token"], + "expires_in": token_data.get("expires_in", 14400), + # Return the public app_key so the callback can store it in the integration + "app_key": settings.dropbox_app_key, + } + + @router.post("/dropbox/exchange-token") @require_login async def exchange_dropbox_token( @@ -210,6 +298,91 @@ async def test_dropbox_token(request: Request): return {"status": "error", "message": f"Connection error: {str(e)}"} +@router.post("/dropbox/list-folders") +@require_login +async def list_dropbox_folders( + request: Request, + access_token: Annotated[str, Form(...)], + path: Annotated[str, Form()] = "", +): + """ + List folders in a Dropbox account for the directory selector. + + Accepts an OAuth access token (short-lived) and a path to list. + Returns a flat list of folder entries under the given path. + """ + try: + # Normalize path: Dropbox API uses "" for root, otherwise "/path" + folder_path = path.strip() + if folder_path == "/": + folder_path = "" + elif folder_path and not folder_path.startswith("/"): + folder_path = f"/{folder_path}" + + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + payload = { + "path": folder_path, + "recursive": False, + "include_deleted": False, + "include_has_explicit_shared_members": False, + "include_mounted_folders": True, + } + + response = requests.post( + "https://api.dropboxapi.com/2/files/list_folder", + headers=headers, + json=payload, + timeout=settings.http_request_timeout, + ) + + if response.status_code == 401: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Access token is invalid or expired. Please re-authorize.", + ) + + if response.status_code != 200: + logger.error(f"Dropbox list_folder failed: {response.status_code} {response.text}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to list Dropbox folders: {response.text}", + ) + + data = response.json() + folders = [] + for entry in data.get("entries", []): + if entry.get(".tag") == "folder": + folders.append( + { + "name": entry["name"], + "path": entry["path_display"], + "id": entry.get("id", ""), + } + ) + + # Sort folders alphabetically + folders.sort(key=lambda f: f["name"].lower()) + + return { + "folders": folders, + "path": folder_path or "/", + "has_more": data.get("has_more", False), + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Error listing Dropbox folders: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list folders: {str(e)}", + ) + + @router.post("/dropbox/save-settings") @require_login async def save_dropbox_settings( diff --git a/app/api/files.py b/app/api/files.py index 64aa62b0..8eae6e55 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -20,6 +20,7 @@ from sqlalchemy.orm import Session from app.auth import require_login from app.config import settings from app.database import get_db +from app.middleware.upload_rate_limit import require_upload_rate_limit from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.process_document import process_document @@ -346,7 +347,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession): try: # Find all file records - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -385,7 +388,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): """ try: # Find all file records - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -457,7 +462,9 @@ def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: Db Useful for re-running OCR on files with poor text quality or missing OCR text. """ try: - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -537,7 +544,9 @@ def bulk_download_files(request: Request, file_ids: List[int], db: DbSession): Files not found on disk are silently skipped. """ try: - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -619,7 +628,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession): """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -675,7 +686,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession): """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -938,7 +951,9 @@ def retry_subtask( """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1080,7 +1095,9 @@ def get_file_preview( try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1160,7 +1177,9 @@ def download_file( try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1249,7 +1268,12 @@ async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None: - """Check for an exact duplicate of the uploaded file and return a warning if found.""" + """Check for an exact duplicate of the uploaded file. + + Returns a dict with duplicate info when the file's SHA-256 hash matches an + already-processed document, or ``None`` when no duplicate is found (or + deduplication is disabled). + """ if not settings.enable_deduplication: return None @@ -1268,8 +1292,8 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s "original_file_id": existing.id, "original_filename": existing.original_filename, "message": ( - "This file appears to be an exact duplicate of an already-processed document. " - "It will still be queued but will be flagged as a duplicate." + "This file is an exact duplicate of an already-processed document. " + "It has not been queued for processing again." ), } except Exception as e: @@ -1280,7 +1304,12 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s @router.post("/ui-upload") @require_login -async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)): +async def ui_upload( + request: Request, + db: DbSession, + file: UploadFile = File(...), + _rate_ok: None = Depends(require_upload_rate_limit), +): """Endpoint to accept a user-uploaded file and enqueue it for processing.""" workdir = settings.workdir @@ -1366,6 +1395,25 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") file_size = written_size + # ── Early duplicate rejection ────────────────────────────────────────── + # Check for exact duplicates (same SHA-256 hash) BEFORE enqueuing a + # processing task. When deduplication is enabled and the file already + # exists, we skip processing entirely, clean up the temp file, and + # return the existing file's information to the caller. + exact_duplicate = _check_for_exact_duplicate(db, target_path, safe_filename) + if exact_duplicate: + # Remove the just-saved temp file — it's a duplicate. + try: + os.remove(target_path) + except OSError: + pass + return { + "status": "duplicate", + "original_filename": safe_filename, + "stored_filename": target_filename, + "duplicate_of": exact_duplicate, + } + # Determine if the file is a PDF or needs conversion mime_type, _ = mimetypes.guess_type(target_path) file_ext = os.path.splitext(target_path)[1].lower() @@ -1429,6 +1477,8 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... ".tif", ".webp", ".svg", + ".heic", + ".heif", }: # If it's an image, convert to PDF first task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id) @@ -1442,20 +1492,12 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id) - # Check for exact duplicates (same SHA-256 hash) before returning. - # This gives the caller an immediate warning without waiting for the pipeline. - # Only performed when deduplication is enabled in settings. - exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename) - - response: dict = { + return { "task_id": task.id, "status": "queued", "original_filename": safe_filename, "stored_filename": target_filename, } - if exact_duplicate_warning: - response["duplicate_warning"] = exact_duplicate_warning - return response # --------------------------------------------------------------------------- diff --git a/app/api/integrations.py b/app/api/integrations.py index 8d2d9209..c0d58893 100644 --- a/app/api/integrations.py +++ b/app/api/integrations.py @@ -32,6 +32,21 @@ from app.utils.encryption import decrypt_value, encrypt_value from app.utils.subscription import get_tier, get_user_tier_id from app.utils.user_scope import get_current_owner_id +# Optional Dropbox SDK — imported at module level so tests can patch it cleanly. +try: + import dropbox as dbx_lib + from dropbox.exceptions import AuthError as _DropboxAuthError + from dropbox.exceptions import BadInputError as _DropboxBadInputError +except ImportError: # pragma: no cover + dbx_lib = None # type: ignore[assignment] + + class _DropboxAuthError(Exception): # type: ignore[no-redef] + """Stub — only used when the dropbox package is missing.""" + + class _DropboxBadInputError(Exception): # type: ignore[no-redef] + """Stub — only used when the dropbox package is missing.""" + + logger = logging.getLogger(__name__) router = APIRouter(prefix="/integrations", tags=["integrations"]) @@ -550,6 +565,47 @@ def _test_s3_connection(config: dict[str, Any] | None, credentials: dict[str, An return {"success": False, "message": "S3 connection failed"} +def _test_dropbox_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]: + """Test a Dropbox connection by verifying OAuth credentials via the Dropbox API.""" + if dbx_lib is None: + return {"success": False, "message": "dropbox package is not installed"} # pragma: no cover + + creds = credentials or {} + app_key = creds.get("app_key", "") + app_secret = creds.get("app_secret", "") + refresh_token = creds.get("refresh_token", "") + + if not refresh_token: + return {"success": False, "message": "Missing required credential: refresh_token"} + if not app_key or not app_secret: + return {"success": False, "message": "Missing required credentials: app_key and app_secret"} + + try: + dbx = dbx_lib.Dropbox( + app_key=app_key, + app_secret=app_secret, + oauth2_refresh_token=refresh_token, + ) + account = dbx.users_get_current_account() + display_name = getattr(account, "name", None) + name_str = "" + if display_name: + name_str = f" ({getattr(display_name, 'display_name', '') or ''})" + return {"success": True, "message": f"Dropbox connection successful{name_str}"} + except _DropboxAuthError as exc: + logger.warning("Dropbox auth error: %s", exc) + return { + "success": False, + "message": "Dropbox authentication failed — check app_key, app_secret, and refresh_token", + } + except _DropboxBadInputError as exc: + logger.warning("Dropbox bad input error: %s", exc) + return {"success": False, "message": "Dropbox connection failed — invalid credentials format"} + except Exception as exc: # noqa: BLE001 + logger.warning("Dropbox connection error: %s", exc) + return {"success": False, "message": "Dropbox connection failed — check credentials and network connectivity"} + + def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]: """Test a WebDAV/Nextcloud connection by issuing an HTTP PROPFIND.""" import urllib.request @@ -596,6 +652,7 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str _CONNECTION_TESTERS: dict[str, Any] = { + IntegrationType.DROPBOX: _test_dropbox_connection, IntegrationType.IMAP: _test_imap_connection, IntegrationType.S3: _test_s3_connection, IntegrationType.WEBDAV: _test_webdav_connection, diff --git a/app/api/mobile.py b/app/api/mobile.py index 465872ab..5305c5d2 100644 --- a/app/api/mobile.py +++ b/app/api/mobile.py @@ -120,6 +120,7 @@ class WhoAmIResponse(BaseModel): email: str | None avatar_url: str | None is_admin: bool + preferred_language: str | None # --------------------------------------------------------------------------- @@ -273,31 +274,44 @@ async def list_devices( return [_device_to_response(d) for d in devices] -@router.delete("/devices/{device_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/devices/{device_id}", status_code=status.HTTP_200_OK) @require_login async def deactivate_device( request: Request, device_id: int, owner_id: CurrentOwner, db: DbSession, -) -> None: - """Deactivate a push-notification device registration. +) -> dict[str, str]: + """Deactivate or permanently delete a push-notification device registration. - The device record is kept for audit purposes but will no longer receive - push notifications. + * **Active device** – soft-deactivated: the record is kept for audit + purposes but will no longer receive push notifications. + * **Already-inactive device** – hard-deleted: the record is permanently + removed from the database. """ device = db.get(MobileDevice, device_id) if not device or device.owner_id != owner_id: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Device not found") - device.is_active = False + if device.is_active: + device.is_active = False + try: + db.commit() + except Exception: + db.rollback() + raise + logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id) + return {"detail": "Device deactivated"} + + # Hard-delete an already-inactive device. try: + db.delete(device) db.commit() except Exception: db.rollback() raise - - logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id) + logger.info("Mobile device permanently deleted: id=%s owner=%s", device_id, owner_id) + return {"detail": "Device deleted"} @router.get("/whoami", response_model=WhoAmIResponse) @@ -344,4 +358,5 @@ async def whoami( "email": email, "avatar_url": avatar_url, "is_admin": is_admin, + "preferred_language": profile.preferred_language if profile else None, } diff --git a/app/api/onedrive.py b/app/api/onedrive.py index 25429344..9e19c303 100644 --- a/app/api/onedrive.py +++ b/app/api/onedrive.py @@ -56,6 +56,7 @@ async def exchange_onedrive_token( # Return just what's needed by the frontend return { "refresh_token": token_data["refresh_token"], + "access_token": token_data.get("access_token", ""), "expires_in": token_data.get("expires_in", 3600), } @@ -183,6 +184,102 @@ async def test_onedrive_token(request: Request): return {"status": "error", "message": f"Connection error: {str(e)}"} +@router.post("/onedrive/list-folders") +@require_login +async def list_onedrive_folders( + request: Request, + access_token: Annotated[str, Form(...)], + path: Annotated[str, Form()] = "", +): + """ + List folders in a OneDrive account for the directory selector. + + Accepts an OAuth access token (short-lived) and a path to list. + Returns a flat list of folder entries under the given path. + """ + try: + folder_path = path.strip().strip("/") + + headers = { + "Authorization": f"Bearer {access_token}", + } + + # Build the Graph API URL for listing children + if not folder_path or folder_path == "root": + url = "https://graph.microsoft.com/v1.0/me/drive/root/children" + else: + url = f"https://graph.microsoft.com/v1.0/me/drive/root:/{folder_path}:/children" + + # Only request folders and minimal fields + params = { + "$filter": "folder ne null", + "$select": "name,id,parentReference,folder", + "$top": "200", + } + + response = requests.get( + url, + headers=headers, + params=params, + timeout=settings.http_request_timeout, + ) + + if response.status_code == 401: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Access token is invalid or expired. Please re-authorize.", + ) + + if response.status_code != 200: + logger.error(f"OneDrive list children failed: {response.status_code} {response.text}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to list OneDrive folders: {response.text}", + ) + + data = response.json() + folders = [] + for item in data.get("value", []): + if "folder" in item: + parent_path = "" + if item.get("parentReference", {}).get("path"): + # parentReference.path looks like /drive/root:/some/path + raw_parent = item["parentReference"]["path"] + prefix = "/drive/root:" + if raw_parent.startswith(prefix): + parent_path = raw_parent[len(prefix) :] + elif raw_parent == "/drive/root": + parent_path = "" + + item_path = f"{parent_path}/{item['name']}" if parent_path else f"/{item['name']}" + + folders.append( + { + "name": item["name"], + "path": item_path, + "id": item.get("id", ""), + "child_count": item.get("folder", {}).get("childCount", 0), + } + ) + + # Sort folders alphabetically + folders.sort(key=lambda f: f["name"].lower()) + + return { + "folders": folders, + "path": f"/{folder_path}" if folder_path else "/", + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Error listing OneDrive folders: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list folders: {str(e)}", + ) + + def format_time_remaining(time_delta): """Format a timedelta into a human-readable string.""" if time_delta.total_seconds() <= 0: diff --git a/app/api/pipelines.py b/app/api/pipelines.py index 8175832f..6acf273f 100644 --- a/app/api/pipelines.py +++ b/app/api/pipelines.py @@ -117,8 +117,14 @@ PIPELINE_STEP_TYPES: dict[str, dict[str, Any]] = { }, "classify": { "label": "Document Classification", - "description": "Classify the document type using AI without full metadata extraction.", - "config_schema": {}, + "description": "Classify the document type using built-in and custom rules (filename patterns, content keywords, metadata matching).", + "config_schema": { + "use_builtin_rules": { + "type": "boolean", + "default": True, + "description": "Include the pre-built classification rules (invoice, contract, receipt, etc.).", + }, + }, }, } diff --git a/app/api/qr_auth.py b/app/api/qr_auth.py new file mode 100644 index 00000000..8f891e4e --- /dev/null +++ b/app/api/qr_auth.py @@ -0,0 +1,233 @@ +"""QR code login API endpoints for mobile app authentication. + +Provides a secure challenge-response flow for logging into the mobile app +by scanning a QR code displayed in the web interface: + +1. **Web user** calls ``POST /qr-auth/challenge`` → receives a time-limited + challenge token (encoded in the QR code). +2. **Web UI** polls ``GET /qr-auth/challenge/{id}/status`` to detect when + the mobile app has claimed the challenge. +3. **Mobile app** scans the QR code and calls ``POST /qr-auth/claim`` with + the challenge token + device name → receives an API token. + +Security properties: +* Challenges expire after a configurable TTL (default 2 minutes). +* Single-use: once claimed, a challenge cannot be reused (replay-safe). +* Cryptographically random 64-byte tokens. +* IP addresses are logged for audit. +""" + +from __future__ import annotations + +import base64 +import io +import logging +from datetime import datetime +from typing import Annotated, Any + +import segno +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.database import get_db +from app.middleware.audit_log import get_client_ip +from app.utils.session_manager import ( + claim_qr_challenge, + create_qr_challenge, + get_challenge_status, +) +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/qr-auth", tags=["qr-auth"]) + +DbSession = Annotated[Session, Depends(get_db)] + + +# --------------------------------------------------------------------------- +# Auth helper +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if not owner_id: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + + +# --------------------------------------------------------------------------- +# Request / Response schemas +# --------------------------------------------------------------------------- + + +class CreateChallengeResponse(BaseModel): + """Response after creating a QR login challenge.""" + + challenge_id: int + challenge_token: str + expires_at: datetime + ttl_seconds: int = Field(description="Seconds until the challenge expires (use for client-side countdown).") + qr_payload: str = Field(description="The string to encode in the QR code.") + qr_code_svg: str = Field(description="Base64-encoded SVG data URI of the QR code, ready for use in an src.") + + +class ChallengeStatusResponse(BaseModel): + """Response for polling the status of a QR challenge.""" + + id: int + status: str # "pending", "claimed", "expired", "cancelled" + device_name: str | None = None + claimed_at: datetime | None = None + expires_at: datetime + + +class ClaimChallengeRequest(BaseModel): + """Request body for claiming a QR login challenge.""" + + challenge_token: str = Field(min_length=1, max_length=256) + device_name: str = Field( + default="Mobile App", + min_length=1, + max_length=120, + description="Human-readable device name.", + ) + + +class ClaimChallengeResponse(BaseModel): + """Response after successfully claiming a QR challenge.""" + + token: str + token_id: int + name: str + owner_id: str + created_at: datetime + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# QR code rendering parameters +_QR_ERROR_LEVEL = "M" # Medium error correction (~15% recovery); sufficient for on-screen display +_QR_SCALE = 4 # Each QR module is rendered as 4×4 SVG pixels + + +def _generate_qr_svg(payload: str) -> str: + """Generate a QR code for *payload* and return it as a base64 SVG data URI. + + Using ``segno`` (pure-Python, no Pillow dependency) and SVG output so the + QR code scales crisply at any resolution without requiring a canvas or any + client-side JavaScript library. + """ + qr = segno.make(payload, error=_QR_ERROR_LEVEL) + buf = io.BytesIO() + qr.save(buf, kind="svg", scale=_QR_SCALE, xmldecl=False, svgclass=None, lineclass=None, omitsize=True) + svg_bytes = buf.getvalue() + return "data:image/svg+xml;base64," + base64.b64encode(svg_bytes).decode("ascii") + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.post("/challenge", status_code=status.HTTP_201_CREATED, response_model=CreateChallengeResponse) +@require_login +async def create_challenge( + request: Request, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Create a new QR login challenge. + + The returned ``qr_payload`` should be encoded into a QR code and + displayed to the user. The mobile app scans this QR code and + calls the ``/claim`` endpoint. + """ + ip = get_client_ip(request) + challenge = create_qr_challenge(db, owner_id, ip_address=ip) + + # The QR payload is a JSON-like string with enough info for the mobile + # app to know the server URL and challenge token. + base_url = str(request.base_url).rstrip("/") + qr_payload = f"docuelevate://qr-login?token={challenge.challenge_token}&server={base_url}" + + # Compute the TTL in seconds so the client can run a countdown timer + # without comparing absolute timestamps (which breaks when client and + # server clocks are out of sync). + ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds())) + + return { + "challenge_id": challenge.id, + "challenge_token": challenge.challenge_token, + "expires_at": challenge.expires_at, + "ttl_seconds": ttl_seconds, + "qr_payload": qr_payload, + "qr_code_svg": _generate_qr_svg(qr_payload), + } + + +@router.get("/challenge/{challenge_id}/status", response_model=ChallengeStatusResponse) +@require_login +async def poll_challenge_status( + request: Request, + challenge_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Poll the status of a QR login challenge. + + The web UI calls this endpoint every few seconds to check if the + mobile app has scanned the QR code and claimed the challenge. + """ + result = get_challenge_status(db, challenge_id, owner_id) + if not result: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Challenge not found") + return result + + +@router.post("/claim", response_model=ClaimChallengeResponse) +async def claim_challenge( + request: Request, + body: ClaimChallengeRequest, + db: DbSession, +) -> dict[str, Any]: + """Claim a QR login challenge and receive an API token. + + This endpoint is called by the mobile app after scanning a QR code. + It does **not** require authentication — the challenge token itself + serves as proof that the user authorized this login from their web + session. + """ + ip = get_client_ip(request) + result = claim_qr_challenge(db, body.challenge_token, device_name=body.device_name, ip_address=ip) + + if not result: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid, expired, or already claimed challenge.", + ) + + try: + from app.utils.audit_service import record_event + + record_event( + db, + action="qr_login_claimed", + user=result["owner_id"], + resource_type="session", + ip_address=ip, + details={"device_name": body.device_name, "token_id": result["token_id"]}, + severity="info", + ) + except Exception: + logger.debug("Failed to write QR login audit event", exc_info=True) + + return result diff --git a/app/api/sessions.py b/app/api/sessions.py new file mode 100644 index 00000000..5772016c --- /dev/null +++ b/app/api/sessions.py @@ -0,0 +1,196 @@ +"""API endpoints for managing user sessions. + +Provides endpoints for listing active sessions, revoking individual sessions, +and the "log off everywhere" feature that invalidates all sessions and API +tokens across all devices. +""" + +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.database import get_db +from app.middleware.audit_log import get_client_ip +from app.utils.session_manager import ( + get_session_lifetime_days, + list_user_sessions, + revoke_all_sessions, + revoke_session, +) +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/sessions", tags=["sessions"]) + +DbSession = Annotated[Session, Depends(get_db)] + + +# --------------------------------------------------------------------------- +# Auth helper +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if not owner_id: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + + +# --------------------------------------------------------------------------- +# Response schemas +# --------------------------------------------------------------------------- + + +class SessionResponse(BaseModel): + """Serialised user session for the management UI.""" + + id: int + device_info: str | None + ip_address: str | None + created_at: datetime + last_active_at: datetime + expires_at: datetime + is_current: bool = False + + +class SessionListResponse(BaseModel): + """Response for listing active sessions.""" + + sessions: list[SessionResponse] + session_lifetime_days: int + + +class RevokeAllResponse(BaseModel): + """Response after revoking all sessions.""" + + revoked_count: int + message: str + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/", response_model=SessionListResponse) +@require_login +async def list_sessions( + request: Request, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """List all active sessions for the current user.""" + sessions = list_user_sessions(db, owner_id) + + # Determine which session is the current one + current_token = request.session.get("_session_token") + + session_list = [] + for s in sessions: + session_list.append( + { + "id": s.id, + "device_info": s.device_info, + "ip_address": s.ip_address, + "created_at": s.created_at, + "last_active_at": s.last_active_at, + "expires_at": s.expires_at, + "is_current": s.session_token == current_token if current_token else False, + } + ) + + return { + "sessions": session_list, + "session_lifetime_days": get_session_lifetime_days(), + } + + +@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT) +@require_login +async def revoke_single_session( + request: Request, + session_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> None: + """Revoke a specific session by ID.""" + success = revoke_session(db, session_id, owner_id) + if not success: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found") + + try: + from app.utils.audit_service import record_event + + record_event( + db, + action="session_revoked", + user=owner_id, + resource_type="session", + resource_id=str(session_id), + ip_address=get_client_ip(request), + severity="info", + ) + except Exception: + logger.debug("Failed to write session revocation audit event", exc_info=True) + + +@router.post("/revoke-all", response_model=RevokeAllResponse) +@require_login +async def revoke_all( + request: Request, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Revoke all sessions except the current one ("log off everywhere"). + + Also revokes all active API tokens for the user, which invalidates + mobile app sessions and any programmatic access. + """ + # Find current session to preserve it + current_token = request.session.get("_session_token") + current_session_id = None + if current_token: + from app.models import UserSession + + current = db.query(UserSession).filter(UserSession.session_token == current_token).first() + if current: + current_session_id = current.id + + count = revoke_all_sessions( + db, + owner_id, + except_session_id=current_session_id, + revoke_api_tokens=True, + ) + + try: + from app.utils.audit_service import record_event + + record_event( + db, + action="revoke_all_sessions", + user=owner_id, + resource_type="session", + ip_address=get_client_ip(request), + details={"revoked_count": count}, + severity="warning", + ) + except Exception: + logger.debug("Failed to write revoke-all audit event", exc_info=True) + + return { + "revoked_count": count, + "message": f"Successfully revoked {count} session(s) and all API tokens.", + } diff --git a/app/api/system_reset.py b/app/api/system_reset.py new file mode 100644 index 00000000..5c7ff501 --- /dev/null +++ b/app/api/system_reset.py @@ -0,0 +1,124 @@ +""" +System reset API endpoints for DocuElevate. + +Provides admin-only REST endpoints for: +- Full system reset (wipe all user data) +- Reset with re-import (move originals → reimport folder, wipe, re-ingest) + +Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and +admin privileges. +""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.config import settings +from app.database import get_db + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"]) + + +def _require_admin(request: Request) -> dict: + """Ensure the caller is an admin. Raises 403 otherwise.""" + user = request.session.get("user") + if not user or not user.get("is_admin"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + return user + + +AdminUser = Annotated[dict, Depends(_require_admin)] + + +def _require_feature_enabled() -> None: + """Raise 404 when the factory-reset feature flag is off.""" + if not settings.enable_factory_reset: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.", + ) + + +class ResetRequest(BaseModel): + """Body for system reset endpoints. Requires explicit confirmation.""" + + confirmation: str + + +@router.post("/full") +async def full_reset( + body: ResetRequest, + _admin: AdminUser, + db: Session = Depends(get_db), +) -> dict: + """Wipe all user data (database + work-files). + + The caller must send ``{"confirmation": "DELETE"}`` to proceed. + """ + _require_feature_enabled() + + if body.confirmation != "DELETE": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.', + ) + + from app.utils.system_reset import perform_full_reset + + try: + result = perform_full_reset(db) + except Exception as exc: + logger.exception("Full system reset failed") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"System reset failed: {exc}", + ) from exc + + return {"status": "ok", "result": result} + + +@router.post("/reimport") +async def reset_and_reimport( + body: ResetRequest, + _admin: AdminUser, + db: Session = Depends(get_db), +) -> dict: + """Move original files to a reimport folder, wipe everything, and + configure the reimport folder as a watch folder for automatic + re-ingestion. + + The caller must send ``{"confirmation": "REIMPORT"}`` to proceed. + """ + _require_feature_enabled() + + if body.confirmation != "REIMPORT": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.', + ) + + from app.utils.system_reset import perform_reset_and_reimport + + try: + result = perform_reset_and_reimport(db) + except Exception as exc: + logger.exception("Reset-and-reimport failed") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Reset and reimport failed: {exc}", + ) from exc + + return {"status": "ok", "result": result} + + +@router.get("/status") +async def reset_status(_admin: AdminUser) -> dict: + """Return whether the system reset feature is enabled.""" + return { + "enabled": settings.enable_factory_reset, + "factory_reset_on_startup": settings.factory_reset_on_startup, + } diff --git a/app/api/url_upload.py b/app/api/url_upload.py index ae286ad3..e93eaea3 100644 --- a/app/api/url_upload.py +++ b/app/api/url_upload.py @@ -11,11 +11,12 @@ from typing import Optional import aiofiles import httpx -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel, HttpUrl, field_validator from app.auth import require_login from app.config import settings +from app.middleware.upload_rate_limit import require_upload_rate_limit from app.tasks.process_document import process_document from app.utils.allowed_types import ALLOWED_MIME_TYPES from app.utils.filename_utils import sanitize_filename @@ -107,7 +108,11 @@ def validate_file_type(content_type: str, filename: str) -> bool: @router.post("/process-url") @require_login -async def process_url(request: Request, url_request: URLUploadRequest): +async def process_url( + request: Request, + url_request: URLUploadRequest, + _rate_ok: None = Depends(require_upload_rate_limit), +): """ Download a file from a URL and enqueue it for processing. diff --git a/app/auth.py b/app/auth.py index a1a40702..5b92b156 100644 --- a/app/auth.py +++ b/app/auth.py @@ -103,11 +103,18 @@ if AUTH_ENABLED and settings.social_auth_apple_enabled: logger.warning("SOCIAL_AUTH_APPLE_ENABLED=true but client ID/team ID not configured") if AUTH_ENABLED and settings.social_auth_dropbox_enabled: - if settings.social_auth_dropbox_client_id and settings.social_auth_dropbox_client_secret: + # Determine which credentials to use for Dropbox social login + _dropbox_client_id = settings.social_auth_dropbox_client_id + _dropbox_client_secret = settings.social_auth_dropbox_client_secret + if settings.social_auth_dropbox_use_global_credentials and not _dropbox_client_id: + _dropbox_client_id = settings.dropbox_app_key + _dropbox_client_secret = settings.dropbox_app_secret + + if _dropbox_client_id and _dropbox_client_secret: oauth.register( name="dropbox", - client_id=settings.social_auth_dropbox_client_id, - client_secret=settings.social_auth_dropbox_client_secret, + client_id=_dropbox_client_id, + client_secret=_dropbox_client_secret, authorize_url="https://www.dropbox.com/oauth2/authorize", access_token_url="https://api.dropboxapi.com/oauth2/token", userinfo_endpoint="https://api.dropboxapi.com/2/users/get_current_account", @@ -129,6 +136,25 @@ def get_current_user(request: Request): return api_user session_user = request.session.get("user") if session_user: + # Validate server-side session if a session token is present + session_token = request.session.get("_session_token") + if session_token: + try: + from app.database import SessionLocal + from app.utils.session_manager import validate_session + + db = SessionLocal() + try: + valid = validate_session(db, session_token) + if not valid: + logger.debug("[AUTH] get_current_user: server-side session invalid — clearing") + request.session.pop("user", None) + request.session.pop("_session_token", None) + return None + finally: + db.close() + except Exception: + logger.debug("[AUTH] get_current_user: session validation error", exc_info=True) logger.debug( "[AUTH] get_current_user: resolved from session (user=%s)", session_user.get("preferred_username") or session_user.get("email") or session_user.get("id"), @@ -167,6 +193,16 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None: logger.debug("[AUTH] _resolve_bearer_user: no active API token matched the provided hash") return None + # Reject tokens that have passed their optional expiry. + if db_token.expires_at is not None: + now_utc = datetime.now(timezone.utc) + expires_aware = db_token.expires_at + if expires_aware.tzinfo is None: + expires_aware = expires_aware.replace(tzinfo=timezone.utc) + if now_utc > expires_aware: + logger.debug("[AUTH] _resolve_bearer_user: API token id=%s has expired", db_token.id) + return None + logger.debug( "[AUTH] _resolve_bearer_user: matched API token id=%s owner=%s", db_token.id, @@ -481,6 +517,27 @@ async def social_callback(request: Request, provider: str, db: Session = Depends request.session["user"] = user_data + # Create server-side session for tracking and revocation + try: + from app.utils.session_manager import create_session + + _session_user_id = ( + user_data.get("sub") + or user_data.get("preferred_username") + or user_data.get("email") + or user_data.get("id") + ) + if _session_user_id: + user_session = create_session( + db, + user_id=_session_user_id, + ip_address=get_client_ip(request), + user_agent=request.headers.get("user-agent"), + ) + request.session["_session_token"] = user_session.session_token + except Exception: + logger.debug("[AUTH] Failed to create server-side session for social user", exc_info=True) + # Auto-create or update UserProfile _ensure_user_profile(db, user_data, is_admin=False) @@ -665,6 +722,27 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)): request.session["user"] = user_data + # Create server-side session for tracking and revocation + try: + from app.utils.session_manager import create_session + + _session_user_id = ( + user_data.get("sub") + or user_data.get("preferred_username") + or user_data.get("email") + or user_data.get("id") + ) + if _session_user_id: + user_session = create_session( + db, + user_id=_session_user_id, + ip_address=get_client_ip(request), + user_agent=request.headers.get("user-agent"), + ) + request.session["_session_token"] = user_session.session_token + except Exception: + logger.debug("[AUTH] Failed to create server-side session for OAuth user", exc_info=True) + # Auto-create or update UserProfile so the user appears in admin user management _ensure_user_profile(db, user_data, is_admin=is_admin) @@ -918,6 +996,19 @@ async def auth(request: Request, db: Session = Depends(get_db)): return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302) user_data = _build_session_user(local_user) request.session["user"] = user_data + # Create server-side session for tracking and revocation + try: + from app.utils.session_manager import create_session + + user_session = create_session( + db, + user_id=local_user.email, + ip_address=get_client_ip(request), + user_agent=request.headers.get("user-agent"), + ) + request.session["_session_token"] = user_session.session_token + except Exception: + logger.debug("[AUTH] Failed to create server-side session", exc_info=True) logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", local_user.email) _record_login_event(db, request, local_user.email, success=True) _ensure_user_profile(db, user_data, is_admin=bool(local_user.is_admin)) @@ -974,6 +1065,20 @@ async def auth(request: Request, db: Session = Depends(get_db)): "is_admin": True, } request.session["user"] = admin_user_data + # Create server-side session for tracking and revocation + try: + from app.utils.session_manager import create_session + + admin_user_id = settings.admin_username or "admin" + user_session = create_session( + db, + user_id=admin_user_id, + ip_address=get_client_ip(request), + user_agent=request.headers.get("user-agent"), + ) + request.session["_session_token"] = user_session.session_token + except Exception: + logger.debug("[AUTH] Failed to create server-side session for admin", exc_info=True) logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", username) _record_login_event(db, request, username, success=True) _ensure_user_profile(db, admin_user_data, is_admin=True) @@ -1024,6 +1129,20 @@ async def logout(request: Request, db: Session = Depends(get_db)): ) except Exception: logger.debug("Failed to write logout audit event for user=%s", username, exc_info=True) + # Revoke server-side session + session_token = request.session.get("_session_token") + if session_token: + try: + from app.utils.session_manager import validate_session + + user_session = validate_session(db, session_token) + if user_session: + user_session.is_revoked = True + user_session.revoked_at = datetime.now(timezone.utc) + db.commit() + except Exception: + logger.debug("[AUTH] Failed to revoke server-side session", exc_info=True) + request.session.pop("_session_token", None) request.session.pop("user", None) return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302) diff --git a/app/celery_worker.py b/app/celery_worker.py index 17a1635b..e8debec4 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -23,6 +23,7 @@ from app.tasks.batch_tasks import ( # noqa: F401 sync_search_index, ) from app.tasks.check_credentials import check_credentials +from app.tasks.classify_document import classify_document_task # noqa: F401 from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding # noqa: F401 from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401 from app.tasks.convert_to_pdfa import convert_to_pdfa # noqa: F401 @@ -53,6 +54,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401 from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401 from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401 from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401 +from app.tasks.upload_to_sharepoint import upload_to_sharepoint # noqa: F401 from app.tasks.upload_to_user_integration import upload_to_user_integration # noqa: F401 from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401 from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401 diff --git a/app/config.py b/app/config.py index fbbbfd05..7b957f4c 100644 --- a/app/config.py +++ b/app/config.py @@ -13,6 +13,24 @@ class Settings(BaseSettings): database_url: str redis_url: str + + # Database connection-pool tuning (ignored for SQLite, which uses NullPool). + db_pool_size: int = Field( + default=10, + description="Number of persistent connections kept in the pool per worker process.", + ) + db_max_overflow: int = Field( + default=20, + description="Additional connections allowed beyond db_pool_size under burst load.", + ) + db_pool_timeout: int = Field( + default=30, + description="Seconds to wait for a connection from the pool before raising a TimeoutError.", + ) + db_pool_recycle: int = Field( + default=1800, + description="Recycle (close and reopen) connections after this many seconds to avoid stale connections.", + ) openai_api_key: str openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint openai_model: str = "gpt-4o-mini" # Default model @@ -102,6 +120,16 @@ class Settings(BaseSettings): dropbox_app_secret: Optional[str] = None dropbox_folder: Optional[str] = None dropbox_refresh_token: Optional[str] = None + dropbox_allow_global_credentials_for_integrations: bool = Field( + default=False, + description=( + "When True, users may authorize their personal Dropbox integrations using the global " + "DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without " + "needing to create their own Dropbox app. The Dropbox OAuth flow is initiated " + "server-side so the app secret is never exposed to the browser. " + "Default: False (each user must supply their own app credentials)." + ), + ) # Making Nextcloud optional nextcloud_enabled: bool = Field( @@ -165,6 +193,35 @@ class Settings(BaseSettings): google_docai_processor_id: Optional[str] = None google_docai_location: str = "us" # Processor location, e.g. "us" or "eu" external_hostname: str = "localhost" # Default to localhost + public_base_url: Optional[str] = Field( + default=None, + description=( + "The full public base URL of the application, including scheme " + "(e.g., 'https://docuelevate.example.com'). " + "When set, this overrides the auto-detected URL for OAuth redirect URIs. " + "This is required when the application is behind a reverse proxy that does " + "not forward X-Forwarded-Proto headers correctly." + ), + ) + + # --------------------------------------------------------------------------- + # Document Translation Settings + # --------------------------------------------------------------------------- + # Default target language for automatic document translation (ISO 639-1 code). + # After OCR / metadata extraction, if the detected document language differs + # from this value the system translates the extracted text into this language + # and stores it alongside the original. Other language translations are + # generated on the fly via the AI provider and are NOT persisted. + # Per-user overrides are stored in UserProfile.default_document_language. + default_document_language: str = Field( + default="en", + description=( + "ISO 639-1 language code for the default translation target " + "(e.g. 'en', 'de', 'fr'). Documents whose detected language " + "differs are automatically translated into this language after " + "processing. Default: 'en' (English)." + ), + ) # --------------------------------------------------------------------------- # Document Translation Settings @@ -190,6 +247,26 @@ class Settings(BaseSettings): admin_username: Optional[str] = None admin_password: Optional[str] = None session_secret: Optional[str] = None + session_lifetime_days: int = Field( + default=30, + description=( + "Session lifetime in days. Common values: 30, 60, 90. " + "Determines how long a user stays logged in before being required to re-authenticate. " + "Applies to both browser sessions and the session cookie max_age." + ), + ) + session_lifetime_custom_days: int | None = Field( + default=None, + description=( + "Override session_lifetime_days with a custom value. " + "When set, this takes precedence over session_lifetime_days. " + "Useful for admin-configured non-standard durations." + ), + ) + qr_login_challenge_ttl_seconds: int = Field( + default=120, + description="Time-to-live in seconds for QR login challenges (default: 2 minutes).", + ) admin_group_name: str = "admin" # Multi-user settings @@ -279,6 +356,16 @@ class Settings(BaseSettings): social_auth_dropbox_enabled: bool = False social_auth_dropbox_client_id: Optional[str] = None social_auth_dropbox_client_secret: Optional[str] = None + social_auth_dropbox_use_global_credentials: bool = Field( + default=False, + description=( + "When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET " + "credentials (the storage integration credentials) instead of requiring separate " + "SOCIAL_AUTH_DROPBOX_CLIENT_ID / SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. " + "Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and the global Dropbox app credentials to be set. " + "Default: False." + ), + ) # Local user signup allow_local_signup: bool = Field( @@ -589,6 +676,15 @@ class Settings(BaseSettings): onedrive_refresh_token: Optional[str] = None # Required for personal accounts onedrive_folder_path: Optional[str] = None + # SharePoint settings + sharepoint_client_id: Optional[str] = None + sharepoint_client_secret: Optional[str] = None + sharepoint_tenant_id: Optional[str] = "common" + sharepoint_refresh_token: Optional[str] = None + sharepoint_site_url: Optional[str] = None # e.g. https://tenant.sharepoint.com/sites/sitename + sharepoint_document_library: Optional[str] = "Documents" # Document library name + sharepoint_folder_path: Optional[str] = None # Subfolder inside the library + # AWS S3 settings s3_enabled: bool = Field( default=True, @@ -639,6 +735,25 @@ class Settings(BaseSettings): ), ) + # System reset / factory reset settings + factory_reset_on_startup: bool = Field( + default=False, + description=( + "When enabled, DocuElevate wipes all user data (database rows and " + "work-files on disk) on every startup so the instance always comes " + "up in a clean, fresh state. Useful for demo or testing environments. " + "Default: False." + ), + ) + enable_factory_reset: bool = Field( + default=False, + description=( + "Show the 'System Reset' page in the admin UI. When enabled, " + "administrators can trigger a full data wipe or a wipe-and-reimport " + "directly from the web interface. Default: False." + ), + ) + # PDF/A archival conversion settings enable_pdfa_conversion: bool = Field( default=False, @@ -1073,6 +1188,20 @@ class Settings(BaseSettings): ), ) + # Per-user upload rate limiting (health-aware, Redis-backed sliding window) + upload_rate_limit_per_user: int = Field( + default=20, + description=( + "Maximum number of file uploads allowed per user within the sliding window. " + "The effective limit may be reduced dynamically when the system is under heavy load " + "(high queue depth or CPU usage). Set to 0 to disable per-user upload rate limiting." + ), + ) + upload_rate_limit_window: int = Field( + default=60, + description="Sliding window size in seconds for per-user upload rate limiting (default: 60).", + ) + # Rate Limiting Configuration (see SECURITY_AUDIT.md and docs/API.md) # Protects against DoS attacks and API abuse rate_limiting_enabled: bool = Field( diff --git a/app/database.py b/app/database.py index f563931a..701e4673 100644 --- a/app/database.py +++ b/app/database.py @@ -10,6 +10,7 @@ from typing import Any from sqlalchemy import create_engine, exc from sqlalchemy.engine.url import make_url from sqlalchemy.orm import Session, declarative_base, sessionmaker +from sqlalchemy.pool import NullPool, QueuePool from app.config import settings @@ -17,9 +18,37 @@ logger = logging.getLogger(__name__) Base = declarative_base() -# Parse the DATABASE_URL +# --------------------------------------------------------------------------- +# Engine construction +# --------------------------------------------------------------------------- DB_URL = settings.database_url -engine = create_engine(DB_URL, connect_args={"check_same_thread": False}) +_parsed_url = make_url(DB_URL) + +_connect_args: dict[str, Any] = {} +_engine_kwargs: dict[str, Any] = { + "pool_pre_ping": True, # detect stale / dropped connections before use +} + +if _parsed_url.get_backend_name() == "sqlite": + # SQLite does not benefit from connection pooling and is prone to + # QueuePool exhaustion under concurrent access. NullPool opens a fresh + # connection for each request and closes it immediately afterwards, + # completely avoiding the "QueuePool limit reached" TimeoutError. + _connect_args["check_same_thread"] = False + _engine_kwargs["poolclass"] = NullPool +else: + # PostgreSQL / MySQL — use a bounded QueuePool with configurable limits. + _engine_kwargs["poolclass"] = QueuePool + _engine_kwargs.update( + { + "pool_size": settings.db_pool_size, + "max_overflow": settings.db_max_overflow, + "pool_timeout": settings.db_pool_timeout, + "pool_recycle": settings.db_pool_recycle, + } + ) + +engine = create_engine(DB_URL, connect_args=_connect_args, **_engine_kwargs) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/app/main.py b/app/main.py index 97bb1d57..21b50f30 100644 --- a/app/main.py +++ b/app/main.py @@ -170,6 +170,12 @@ async def lifespan(app: FastAPI): # Startup: Initialize database init_db() # Create tables if they don't exist + # Factory reset on startup — wipe all user data before anything else + if settings.factory_reset_on_startup: + from app.utils.system_reset import perform_startup_reset + + perform_startup_reset() + # Load settings from database after DB initialization from app.database import SessionLocal from app.utils.config_loader import load_settings_from_db @@ -318,8 +324,19 @@ app.add_middleware(CSRFMiddleware, config=settings) # See SECURITY_AUDIT.md – Infrastructure Security section app.add_middleware(AuditLogMiddleware, config=settings) + # 3) Session Middleware (for request.session to work) -app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET) +def _get_session_max_age() -> int: + """Compute session max-age at startup time.""" + try: + from app.utils.session_manager import get_session_max_age_seconds + + return get_session_max_age_seconds() + except Exception: + return 30 * 86400 # 30 days default fallback + + +app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET, max_age=_get_session_max_age()) # 3a) CORS Middleware - handles cross-origin requests and preflight (OPTIONS) responses. # Disabled by default: set CORS_ENABLED=True only when NOT using a reverse proxy diff --git a/app/middleware/csrf.py b/app/middleware/csrf.py index 842958af..f21088ad 100644 --- a/app/middleware/csrf.py +++ b/app/middleware/csrf.py @@ -20,6 +20,9 @@ How it works: Exempt paths (CSRF is not checked even for state-changing methods): - ``/oauth-callback`` – OAuth 2.0 callback; protected by the ``state`` parameter. +- ``/api/qr-auth/claim`` – Called by the unauthenticated mobile app; the + cryptographically-random, single-use challenge token provides equivalent + protection. """ import logging @@ -39,6 +42,10 @@ CSRF_PROTECTED_METHODS = {"POST", "PUT", "DELETE", "PATCH"} # their own replay-protection mechanism). CSRF_EXEMPT_PATHS = { "/oauth-callback", + # The mobile app calls this endpoint without a browser session/CSRF token. + # The cryptographically-random, single-use challenge token already provides + # equivalent protection against cross-site request forgery. + "/api/qr-auth/claim", } diff --git a/app/middleware/upload_rate_limit.py b/app/middleware/upload_rate_limit.py new file mode 100644 index 00000000..16b7d907 --- /dev/null +++ b/app/middleware/upload_rate_limit.py @@ -0,0 +1,290 @@ +"""Per-user, health-aware upload rate limiter for DocuElevate. + +This module provides a FastAPI dependency that enforces per-user upload rate +limits using a Redis-backed sliding window counter. The effective limit is +dynamically reduced when the system is under heavy load (high Celery queue +depth or elevated CPU load average), ensuring the server remains responsive +to all users even during bulk-upload scenarios. + +Usage in an endpoint:: + + from app.middleware.upload_rate_limit import require_upload_rate_limit + + @router.post("/ui-upload") + @require_login + async def ui_upload( + request: Request, + _rate_ok: None = Depends(require_upload_rate_limit), + ... + ): + ... + +See ``docs/ConfigurationGuide.md`` for the configuration options +(``UPLOAD_RATE_LIMIT_PER_USER``, ``UPLOAD_RATE_LIMIT_WINDOW``). +""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +import redis +from fastapi import HTTPException, Request, status + +from app.config import settings +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Redis key prefix +# --------------------------------------------------------------------------- +_KEY_PREFIX = "docuelevate:upload_rate" + +# --------------------------------------------------------------------------- +# Health-check queue names (Celery defaults used by DocuElevate) +# --------------------------------------------------------------------------- +_CELERY_QUEUES = ("document_processor", "default", "celery") + +# --------------------------------------------------------------------------- +# Singleton Redis client (lazy-initialised; fail-open when unavailable) +# --------------------------------------------------------------------------- +_redis_client: redis.Redis | None = None + + +def _get_redis() -> redis.Redis | None: + """Return a shared Redis client, or *None* when Redis is unavailable.""" + global _redis_client + if _redis_client is not None: + return _redis_client + try: + _redis_client = redis.Redis.from_url( + settings.redis_url, + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + ) + # Quick connectivity check – raises on failure. + _redis_client.ping() + return _redis_client + except Exception: # noqa: BLE001 + logger.debug("Redis unavailable for upload rate limiter – falling back to allow-all", exc_info=True) + _redis_client = None + return None + + +# --------------------------------------------------------------------------- +# Health metrics helpers +# --------------------------------------------------------------------------- + + +def _get_queue_depth(r: redis.Redis) -> int: + """Return the total number of pending tasks across all Celery queues.""" + total = 0 + for queue_name in _CELERY_QUEUES: + try: + total += r.llen(queue_name) + except Exception: # noqa: BLE001, S110 + logger.debug("Could not read queue length for %r", queue_name, exc_info=True) + return total + + +def _get_cpu_load_ratio() -> float: + """Return the 1-minute load average divided by the number of CPU cores. + + Returns ``0.0`` on platforms that do not support :func:`os.getloadavg` + (e.g. Windows) so that the limiter never penalises on those systems. + """ + try: + load_1m = os.getloadavg()[0] + cpu_count = os.cpu_count() or 1 + return load_1m / cpu_count + except (OSError, AttributeError): + return 0.0 + + +def compute_effective_limit( + base_limit: int, + queue_depth: int = 0, + cpu_load_ratio: float = 0.0, +) -> tuple[int, float, str]: + """Compute the effective upload rate limit based on system health. + + The function applies a *reduction factor* (``0.0 < factor ≤ 1.0``) to the + configured base limit. Both queue depth and CPU load contribute + independently; the lowest factor wins. + + Args: + base_limit: The configured maximum uploads per window. + queue_depth: Total pending tasks in Celery queues. + cpu_load_ratio: 1-minute load average divided by CPU count. + + Returns: + A 3-tuple of ``(effective_limit, factor, reason)`` where *reason* + is a human-readable tag for logging. + """ + factor = 1.0 + reason = "normal" + + # --- Queue-depth thresholds --- + if queue_depth > 200: + factor, reason = min(factor, 0.10), f"critical_queue({queue_depth})" + elif queue_depth > 100: + factor, reason = min(factor, 0.25), f"high_queue({queue_depth})" + elif queue_depth > 50: + factor, reason = min(factor, 0.50), f"moderate_queue({queue_depth})" + + # --- CPU-load thresholds --- + if cpu_load_ratio > 3.0: + new_factor = 0.10 + if new_factor < factor: + factor, reason = new_factor, f"critical_cpu({cpu_load_ratio:.1f})" + elif cpu_load_ratio > 2.0: + new_factor = 0.25 + if new_factor < factor: + factor, reason = new_factor, f"high_cpu({cpu_load_ratio:.1f})" + elif cpu_load_ratio > 1.5: + new_factor = 0.50 + if new_factor < factor: + factor, reason = new_factor, f"moderate_cpu({cpu_load_ratio:.1f})" + + effective = max(1, int(base_limit * factor)) + return effective, factor, reason + + +# --------------------------------------------------------------------------- +# Core sliding-window check (Redis sorted set) +# --------------------------------------------------------------------------- + + +def _check_and_record( + r: redis.Redis, + user_id: str, + window: int, + effective_limit: int, +) -> dict[str, Any] | None: + """Atomically check the user's upload count and record the new upload. + + Uses a Redis sorted set where each member is a unique timestamp-based ID + and the score is the Unix timestamp. Entries older than *window* seconds + are pruned on every call so the set never grows unbounded. + + Returns: + ``None`` if the request is allowed, or a ``dict`` with ``count``, + ``limit``, and ``retry_after`` if the limit is exceeded. + """ + key = f"{_KEY_PREFIX}:{user_id}" + now = time.time() + window_start = now - window + + pipe = r.pipeline(transaction=True) + # 1. Remove entries outside the window + pipe.zremrangebyscore(key, "-inf", window_start) + # 2. Count current entries + pipe.zcard(key) + # 3. Retrieve the oldest entry's score (to compute retry_after) + pipe.zrange(key, 0, 0, withscores=True) + results = pipe.execute() + + current_count: int = results[1] + oldest_entries: list = results[2] + + if current_count >= effective_limit: + # Compute how long until the oldest entry expires from the window. + if oldest_entries: + oldest_score = oldest_entries[0][1] + retry_after = max(1, int((oldest_score + window) - now)) + else: + retry_after = max(1, window // 2) + return { + "count": current_count, + "limit": effective_limit, + "retry_after": retry_after, + } + + # 4. Record this upload (unique member = timestamp with random suffix) + member = f"{now}:{os.urandom(4).hex()}" + pipe2 = r.pipeline(transaction=True) + pipe2.zadd(key, {member: now}) + pipe2.expire(key, window + 60) # TTL slightly longer than window + pipe2.execute() + + return None + + +# --------------------------------------------------------------------------- +# FastAPI dependency +# --------------------------------------------------------------------------- + + +async def require_upload_rate_limit(request: Request) -> None: + """FastAPI dependency that enforces per-user upload rate limits. + + The dependency is designed to **fail open**: if Redis is unavailable the + request is allowed through so that uploads are never blocked by a + monitoring outage. + + Raises: + HTTPException: 429 Too Many Requests when the per-user upload limit + is exceeded. The ``Retry-After`` header indicates how many + seconds the client should wait before retrying. + """ + r = _get_redis() + if r is None: + # Redis unavailable – fail open. + return + + # Identify the user (owner_id for multi-user, IP fallback). + user_id = get_current_owner_id(request) + if not user_id: + user_id = f"ip:{request.client.host}" if request.client else "ip:unknown" + + base_limit: int = settings.upload_rate_limit_per_user + window: int = settings.upload_rate_limit_window + + # Gather health metrics and compute effective limit. + try: + queue_depth = _get_queue_depth(r) + except Exception: # noqa: BLE001 + queue_depth = 0 + + cpu_load_ratio = _get_cpu_load_ratio() + effective_limit, factor, health_reason = compute_effective_limit(base_limit, queue_depth, cpu_load_ratio) + + # Sliding-window check. + try: + rejection = _check_and_record(r, user_id, window, effective_limit) + except Exception as exc: # noqa: BLE001 + logger.warning("Upload rate-limit check failed (allowing request): %s", exc) + return + + if rejection is not None: + retry_after = rejection["retry_after"] + logger.warning( + "Upload rate limit exceeded: user=%s count=%d/%d window=%ds health=%s retry_after=%ds", + user_id, + rejection["count"], + rejection["limit"], + window, + health_reason, + retry_after, + ) + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail=( + f"Upload rate limit exceeded ({rejection['count']}/{rejection['limit']} " + f"in {window}s). Retry after {retry_after}s." + ), + headers={"Retry-After": str(retry_after)}, + ) + + if factor < 1.0: + logger.info( + "Upload allowed with reduced limit: user=%s effective=%d/%d health=%s", + user_id, + effective_limit, + base_limit, + health_reason, + ) diff --git a/app/models.py b/app/models.py index 2152eed4..94697fc4 100644 --- a/app/models.py +++ b/app/models.py @@ -625,6 +625,7 @@ class IntegrationType: EMAIL = "EMAIL" PAPERLESS = "PAPERLESS" RCLONE = "RCLONE" + SHAREPOINT = "SHAREPOINT" ICLOUD = "ICLOUD" ALL = { @@ -642,6 +643,7 @@ class IntegrationType: EMAIL, PAPERLESS, RCLONE, + SHAREPOINT, ICLOUD, } @@ -806,6 +808,9 @@ class ApiToken(Base): created_at = Column(DateTime(timezone=True), server_default=func.now()) revoked_at = Column(DateTime(timezone=True), nullable=True) + # Optional expiry: if set, the token is rejected after this timestamp. + expires_at = Column(DateTime(timezone=True), nullable=True) + class SharedLink(Base): """Shareable, time-limited or view-limited document link. @@ -957,6 +962,51 @@ class ScheduledJob(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class ClassificationRuleModel(Base): + """Custom document classification rule. + + Rules are evaluated during the ``classify`` pipeline step to assign a + category to a document. System-wide rules have ``owner_id IS NULL``; + user-specific rules belong to a single owner. + """ + + __tablename__ = "classification_rules" + + id = Column(Integer, primary_key=True, index=True) + + # NULL = system-wide rule visible to all users. + owner_id = Column(String, nullable=True, index=True) + + # Human-readable rule name (unique per owner). + name = Column(String(255), nullable=False) + + # Target category (e.g. "invoice", "contract", "receipt"). + category = Column(String(100), nullable=False, index=True) + + # Rule type: "filename_pattern", "content_keyword", or "metadata_match". + rule_type = Column(String(50), nullable=False) + + # The matching pattern: + # - filename_pattern: a regex + # - content_keyword: pipe-separated keywords + # - metadata_match: "field=value" + pattern = Column(String(1000), nullable=False) + + # Higher priority rules are evaluated first (default 0). + priority = Column(Integer, nullable=False, default=0) + + # Whether pattern matching is case-sensitive. + case_sensitive = Column(Boolean, nullable=False, default=False) + + # Disabled rules are skipped during classification. + enabled = Column(Boolean, nullable=False, default=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + __table_args__ = (UniqueConstraint("owner_id", "name", name="uq_classification_rules_owner_name"),) + + class MobileDevice(Base): """Registered mobile device for push notifications. @@ -991,6 +1041,86 @@ class MobileDevice(Base): __table_args__ = (UniqueConstraint("owner_id", "push_token", name="uq_mobile_device_owner_token"),) +class UserSession(Base): + """Server-side session tracking for invalidation and device management. + + Each row represents an active browser or app session. The ``session_token`` + is stored in the user's cookie and validated on every authenticated request. + Revoking a row (``is_revoked=True``) immediately terminates that session + on the next request. + """ + + __tablename__ = "user_sessions" + + id = Column(Integer, primary_key=True, index=True) + + # Cryptographically random token stored in the session cookie. + session_token = Column(String(128), unique=True, nullable=False, index=True) + + # Stable owner identifier — matches FileRecord.owner_id. + user_id = Column(String, nullable=False, index=True) + + # Client metadata for display in the session management UI. + ip_address = Column(String(45), nullable=True) + user_agent = Column(String(512), nullable=True) + device_info = Column(String(255), nullable=True) + + is_revoked = Column(Boolean, nullable=False, default=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_active_at = Column(DateTime(timezone=True), server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + + +class QRLoginChallenge(Base): + """Time-limited QR code login challenge for mobile app authentication. + + A logged-in web user generates a challenge that produces a QR code. The + mobile app scans the QR code and calls the claim endpoint with the + ``challenge_token``. The server verifies the challenge is still valid, + unclaimed, and unexpired, then issues an API token for the mobile app. + + Security properties: + * Time-bound (default 2 minutes). + * Single-use (``is_claimed`` prevents replay). + * Cryptographically random 64-byte token. + * Bound to the creating user — only that user's mobile device receives a + token. + """ + + __tablename__ = "qr_login_challenges" + + id = Column(Integer, primary_key=True, index=True) + + # Cryptographically random token encoded in the QR code. + challenge_token = Column(String(128), unique=True, nullable=False, index=True) + + # The user who created this challenge (from the web session). + user_id = Column(String, nullable=False, index=True) + + # Whether the challenge has been successfully claimed by a mobile app. + is_claimed = Column(Boolean, nullable=False, default=False) + + # Whether the challenge has been explicitly cancelled or expired. + is_cancelled = Column(Boolean, nullable=False, default=False) + + # IP address of the web client that created the challenge. + created_by_ip = Column(String(45), nullable=True) + + # IP address of the mobile client that claimed the challenge. + claimed_by_ip = Column(String(45), nullable=True) + + # Device name provided by the mobile app when claiming. + device_name = Column(String(255), nullable=True) + + # The API token ID that was issued to the mobile app (for audit trail). + issued_token_id = Column(Integer, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=False) + claimed_at = Column(DateTime(timezone=True), nullable=True) + + class ComplianceTemplate(Base): """Pre-built compliance configuration templates (GDPR, HIPAA, SOC2). diff --git a/app/tasks/classify_document.py b/app/tasks/classify_document.py new file mode 100644 index 00000000..d87f9f13 --- /dev/null +++ b/app/tasks/classify_document.py @@ -0,0 +1,174 @@ +"""Celery task for rule-based document classification. + +This task is executed as a pipeline step (``step_type="classify"``). It +applies built-in and user-defined classification rules against the document's +filename, OCR text, and existing AI metadata to assign a ``document_type`` +category. + +The result is stored in the ``ai_metadata`` JSON blob on the +:class:`~app.models.FileRecord` (field ``classification``). +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from app.celery_app import celery +from app.database import SessionLocal +from app.models import ClassificationRuleModel, FileRecord +from app.tasks.retry_config import BaseTaskWithRetry +from app.utils import log_task_progress +from app.utils.classification_rules import ( + ClassificationResult, + classify_document, + db_rule_to_engine_rule, +) + +logger = logging.getLogger(__name__) + +STEP_NAME = "classify_document" + + +def _load_custom_rules(owner_id: str | None) -> list[Any]: + """Load enabled custom classification rules from the database. + + Returns engine-level :class:`ClassificationRule` dataclass instances. + Rules are loaded in priority-descending order. System rules + (``owner_id IS NULL``) and the user's own rules are both included. + """ + with SessionLocal() as db: + query = db.query(ClassificationRuleModel).filter(ClassificationRuleModel.enabled.is_(True)) + if owner_id: + query = query.filter( + (ClassificationRuleModel.owner_id.is_(None)) | (ClassificationRuleModel.owner_id == owner_id) + ) + else: + query = query.filter(ClassificationRuleModel.owner_id.is_(None)) + rules = query.order_by(ClassificationRuleModel.priority.desc()).all() + return [db_rule_to_engine_rule(r) for r in rules] + + +@celery.task(base=BaseTaskWithRetry, bind=True) +def classify_document_task( + self: Any, + file_id: int, + owner_id: str | None = None, +) -> dict[str, Any]: + """Classify a document using rule-based matching. + + This task: + 1. Loads the :class:`FileRecord` from the database. + 2. Gathers filename, OCR text, and existing AI metadata. + 3. Loads built-in + user-defined classification rules. + 4. Runs the classification engine. + 5. Persists the result into ``ai_metadata.classification``. + + Args: + file_id: Primary key of the :class:`FileRecord` to classify. + owner_id: Owner identifier for loading user-specific rules. + + Returns: + Dict with ``category``, ``confidence``, and ``matched_rules``. + """ + task_id = self.request.id + + log_task_progress( + task_id, + STEP_NAME, + "in_progress", + f"Starting classification for file {file_id}", + file_id=file_id, + ) + + try: + with SessionLocal() as db: + file_record: FileRecord | None = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if file_record is None: + log_task_progress( + task_id, + STEP_NAME, + "failure", + f"FileRecord {file_id} not found", + file_id=file_id, + ) + return {"status": "error", "detail": "File not found"} + + # Gather inputs + filename = file_record.original_filename or "" + text = file_record.ocr_text or "" + existing_metadata: dict[str, Any] = {} + if file_record.ai_metadata: + try: + existing_metadata = json.loads(file_record.ai_metadata) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to parse ai_metadata for file %s, starting fresh", file_id) + existing_metadata = {} + + # Load custom rules + effective_owner = owner_id or file_record.owner_id + custom_rules = _load_custom_rules(effective_owner) + + # Run classification engine + result: ClassificationResult = classify_document( + filename=filename, + text=text, + metadata=existing_metadata, + custom_rules=custom_rules, + ) + + # Persist result into ai_metadata + classification_data = { + "category": result.category, + "confidence": result.confidence, + "matched_rules": [ + { + "rule_name": m.rule_name, + "rule_type": m.rule_type, + "category": m.category, + "confidence": m.confidence, + } + for m in result.matched_rules + ], + } + + existing_metadata["classification"] = classification_data + + # If no document_type was set yet, populate it from the classification + if not existing_metadata.get("document_type"): + from app.utils.classification_rules import BUILTIN_CATEGORIES + + existing_metadata["document_type"] = BUILTIN_CATEGORIES.get( + result.category, result.category.replace("_", " ").title() + ) + + file_record.ai_metadata = json.dumps(existing_metadata, ensure_ascii=False) + db.commit() + + log_task_progress( + task_id, + STEP_NAME, + "success", + f"Classified as '{result.category}' with confidence {result.confidence}", + file_id=file_id, + detail=f"Matched {len(result.matched_rules)} rule(s)", + ) + + return { + "status": "success", + "category": result.category, + "confidence": result.confidence, + "matched_rules": len(result.matched_rules), + } + + except Exception as e: + logger.exception("Classification failed for file %s: %s", file_id, e) + log_task_progress( + task_id, + STEP_NAME, + "failure", + f"Classification failed: {e}", + file_id=file_id, + ) + raise diff --git a/app/tasks/convert_to_pdf.py b/app/tasks/convert_to_pdf.py index 2a0ff4ac..08320f55 100644 --- a/app/tasks/convert_to_pdf.py +++ b/app/tasks/convert_to_pdf.py @@ -205,7 +205,7 @@ def convert_to_pdf( ".pdf", # PDF (already in PDF format but can be processed) } - IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg"} + IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg", ".heic", ".heif"} HTML_EXTENSIONS = {".html", ".htm"} diff --git a/app/tasks/send_to_all.py b/app/tasks/send_to_all.py index 214e0225..17f60c84 100644 --- a/app/tasks/send_to_all.py +++ b/app/tasks/send_to_all.py @@ -18,6 +18,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive from app.tasks.upload_to_paperless import upload_to_paperless from app.tasks.upload_to_s3 import upload_to_s3 from app.tasks.upload_to_sftp import upload_to_sftp +from app.tasks.upload_to_sharepoint import upload_to_sharepoint from app.tasks.upload_to_webdav import upload_to_webdav from app.utils.config_validator import get_provider_status from app.utils.logging import log_task_progress @@ -121,6 +122,18 @@ def _should_upload_to_icloud(): return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password) +def _should_upload_to_sharepoint(): + return bool( + settings.sharepoint_client_id + and settings.sharepoint_client_secret + and settings.sharepoint_site_url + and ( + settings.sharepoint_refresh_token + or (settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common") + ) + ) + + def get_configured_services_from_validator(): """ Use the config validator to determine which services are configured and enabled. @@ -140,6 +153,7 @@ def get_configured_services_from_validator(): "Email": "email", "OneDrive": "onedrive", "S3 Storage": "s3", + "SharePoint": "sharepoint", "iCloud Drive": "icloud", } @@ -250,6 +264,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: "should_upload": _should_upload_to_s3, "upload_func": upload_to_s3, }, + { + "name": "sharepoint", + "should_upload": _should_upload_to_sharepoint, + "upload_func": upload_to_sharepoint, + }, { "name": "icloud", "should_upload": _should_upload_to_icloud, diff --git a/app/tasks/upload_to_sharepoint.py b/app/tasks/upload_to_sharepoint.py new file mode 100644 index 00000000..0e2593c1 --- /dev/null +++ b/app/tasks/upload_to_sharepoint.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Upload documents to Microsoft SharePoint via the Microsoft Graph API. + +This module authenticates using MSAL (same OAuth2 flow as OneDrive) and +uploads files to a configurable SharePoint Online document library using +the chunked upload session approach for reliability with large files. + +Key differences from the OneDrive provider: +- Uses ``/sites/{siteId}/drives/{driveId}`` instead of ``/me/drive`` +- Requires a SharePoint site URL to resolve the site and drive IDs +- Targets a named document library (default: ``Documents``) +""" + +import logging +import os +import time +import urllib.parse + +import msal +import requests + +from app.celery_app import celery +from app.config import settings +from app.tasks.retry_config import UploadTaskWithRetry +from app.utils import log_task_progress + +logger = logging.getLogger(__name__) + + +def get_sharepoint_token() -> str: + """Acquire a Microsoft Graph API access token for SharePoint. + + Uses MSAL ``ConfidentialClientApplication`` with the refresh-token flow + (delegated permissions) or the client-credentials flow (application + permissions) depending on configuration. + + Returns: + A valid access token string. + + Raises: + ValueError: When required settings are missing or token acquisition fails. + """ + if not settings.sharepoint_client_id or not settings.sharepoint_client_secret: + raise ValueError("SharePoint client ID and client secret must be configured") + + tenant = settings.sharepoint_tenant_id or "common" + logger.info("Using SharePoint tenant: %s", tenant) + + scopes = ["https://graph.microsoft.com/.default"] + + if settings.sharepoint_refresh_token: + app = msal.ConfidentialClientApplication( + client_id=settings.sharepoint_client_id, + client_credential=settings.sharepoint_client_secret, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + + logger.info("Attempting to acquire SharePoint token using refresh token") + token_response = app.acquire_token_by_refresh_token( + refresh_token=settings.sharepoint_refresh_token, scopes=scopes + ) + + if "access_token" not in token_response: + error = token_response.get("error", "") + error_desc = token_response.get("error_description", "Unknown error") + logger.error("Failed to get SharePoint access token: %s - %s", error, error_desc) + raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}") + + if "refresh_token" in token_response: + settings.sharepoint_refresh_token = token_response["refresh_token"] + logger.info("Updated SharePoint refresh token in memory") + + return token_response["access_token"] + + elif settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common": + authority = f"https://login.microsoftonline.com/{settings.sharepoint_tenant_id}" + app = msal.ConfidentialClientApplication( + client_id=settings.sharepoint_client_id, + client_credential=settings.sharepoint_client_secret, + authority=authority, + ) + + token_response = app.acquire_token_for_client(scopes=scopes) + + if "access_token" not in token_response: + error = token_response.get("error", "") + error_desc = token_response.get("error_description", "Unknown error") + raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}") + + return token_response["access_token"] + + else: + raise ValueError("For SharePoint, either a refresh token or a non-'common' tenant ID is required") + + +def resolve_sharepoint_drive(access_token: str, site_url: str, library_name: str) -> tuple[str, str]: + """Resolve the Graph API site ID and drive ID for a SharePoint site. + + Args: + access_token: Valid Microsoft Graph API token. + site_url: Full SharePoint site URL, e.g. + ``https://tenant.sharepoint.com/sites/sitename``. + library_name: Display name of the document library (e.g. ``Documents``). + + Returns: + A ``(site_id, drive_id)`` tuple. + + Raises: + ValueError: When the site URL cannot be parsed. + RuntimeError: When the Graph API call fails. + """ + parsed = urllib.parse.urlparse(site_url) + hostname = parsed.hostname + site_path = parsed.path.rstrip("/") + + if not hostname or not site_path: + raise ValueError( + f"Invalid SharePoint site URL '{site_url}'. Expected format: https://tenant.sharepoint.com/sites/sitename" + ) + + headers = {"Authorization": f"Bearer {access_token}"} + + # Resolve site ID + site_api_url = f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}" + logger.info("Resolving SharePoint site: %s", site_api_url) + resp = requests.get(site_api_url, headers=headers, timeout=settings.http_request_timeout) + + if resp.status_code != 200: + raise RuntimeError(f"Failed to resolve SharePoint site: {resp.status_code} - {resp.text}") + + site_id = resp.json()["id"] + logger.info("Resolved SharePoint site ID: %s", site_id) + + # Resolve drive ID from the document library name + drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives" + resp = requests.get(drives_url, headers=headers, timeout=settings.http_request_timeout) + + if resp.status_code != 200: + raise RuntimeError(f"Failed to list SharePoint drives: {resp.status_code} - {resp.text}") + + drives = resp.json().get("value", []) + drive_id = None + for drive in drives: + if drive.get("name", "").lower() == library_name.lower(): + drive_id = drive["id"] + break + + if not drive_id: + available = [d.get("name") for d in drives] + raise RuntimeError(f"Document library '{library_name}' not found on site. Available libraries: {available}") + + logger.info("Resolved SharePoint drive ID: %s (library: %s)", drive_id, library_name) + return site_id, drive_id + + +def create_sharepoint_upload_session( + filename: str, folder_path: str | None, drive_id: str, site_id: str, access_token: str +) -> str: + """Create a resumable upload session on a SharePoint document library. + + Args: + filename: Name of the file to upload. + folder_path: Optional subfolder path inside the library. + drive_id: Graph API drive ID of the document library. + site_id: Graph API site ID. + access_token: Valid access token. + + Returns: + The upload session URL for chunked PUT requests. + + Raises: + RuntimeError: When session creation fails. + """ + base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}" + + if folder_path: + folder_path = folder_path.strip("/") + path_components = folder_path.split("/") + encoded_path = "/".join(urllib.parse.quote(component) for component in path_components) + encoded_filename = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession" + else: + encoded_filename = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_filename}:/createUploadSession" + + url = f"{base_url}{item_path}" + request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}} + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + + logger.info("Creating SharePoint upload session for %s at path %s", filename, folder_path) + response = requests.post(url, headers=headers, json=request_body, timeout=settings.http_request_timeout) + + if response.status_code == 200: + upload_url = response.json().get("uploadUrl") + logger.info("SharePoint upload session created for %s", filename) + return upload_url + else: + raise RuntimeError(f"Failed to create SharePoint upload session: {response.status_code} - {response.text}") + + +def upload_large_file_sharepoint(file_path: str, upload_url: str) -> dict: + """Upload a file to SharePoint using a chunked upload session. + + Args: + file_path: Local path to the file. + upload_url: The upload session URL from ``create_sharepoint_upload_session``. + + Returns: + The Graph API response dict containing file metadata. + + Raises: + RuntimeError: When a chunk upload fails after retries. + """ + file_size = os.path.getsize(file_path) + chunk_size = 10 * 1024 * 1024 # 10 MB + + response = None + with open(file_path, "rb") as f: + chunk_number = 0 + while True: + chunk = f.read(chunk_size) + if not chunk: + break + + chunk_start = chunk_number * chunk_size + chunk_end = chunk_start + len(chunk) - 1 + content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}" + + headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range} + + max_retries = 3 + retry_delay = 2 + + for attempt in range(max_retries): + try: + response = requests.put( + upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout + ) + if response.status_code in (201, 202): + break + else: + logger.warning( + "SharePoint chunk upload failed (attempt %d): %d", attempt + 1, response.status_code + ) + if attempt < max_retries - 1: + time.sleep(retry_delay * (attempt + 1)) + except Exception as e: + logger.warning("SharePoint chunk upload error (attempt %d): %s", attempt + 1, str(e)) + if attempt < max_retries - 1: + time.sleep(retry_delay * (attempt + 1)) + + if response is None or response.status_code not in (201, 202): + status = response.status_code if response else "no response" + text = response.text if response else "" + raise RuntimeError(f"Failed to upload chunk after {max_retries} attempts: {status} - {text}") + + chunk_number += 1 + + return response.json() if response else {} + + +@celery.task(base=UploadTaskWithRetry, bind=True) +def upload_to_sharepoint(self, file_path: str, file_id: int = None, folder_override: str = None): + """Upload a file to SharePoint Online. + + Args: + file_path: Path to the file to upload. + file_id: Optional file ID to associate with logs. + folder_override: Optional folder path override. + + Returns: + A dict with upload status and file details. + + Raises: + FileNotFoundError: When the file does not exist. + ValueError: When SharePoint is not configured. + RuntimeError: When the upload fails. + """ + task_id = self.request.id + logger.info("[%s] Starting SharePoint upload: %s", task_id, file_path) + log_task_progress( + task_id, + "upload_to_sharepoint", + "in_progress", + f"Uploading to SharePoint: {os.path.basename(file_path)}", + file_id=file_id, + ) + + if not os.path.exists(file_path): + error_msg = f"File not found: {file_path}" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise FileNotFoundError(error_msg) + + filename = os.path.basename(file_path) + + if not settings.sharepoint_client_id: + error_msg = "SharePoint client ID is not configured" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise ValueError(error_msg) + + if not settings.sharepoint_site_url: + error_msg = "SharePoint site URL is not configured" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise ValueError(error_msg) + + try: + access_token = get_sharepoint_token() + + library_name = settings.sharepoint_document_library or "Documents" + site_id, drive_id = resolve_sharepoint_drive(access_token, settings.sharepoint_site_url, library_name) + + folder_path = folder_override if folder_override is not None else settings.sharepoint_folder_path + + upload_url = create_sharepoint_upload_session(filename, folder_path, drive_id, site_id, access_token) + result = upload_large_file_sharepoint(file_path, upload_url) + + web_url = result.get("webUrl", "Not available") + logger.info("[%s] Successfully uploaded %s to SharePoint", task_id, filename) + logger.info("[%s] File accessible at: %s", task_id, web_url) + log_task_progress( + task_id, "upload_to_sharepoint", "success", f"Uploaded to SharePoint: {filename}", file_id=file_id + ) + + return { + "status": "Completed", + "file_path": file_path, + "sharepoint_path": f"{folder_path or ''}/{filename}", + "web_url": web_url, + } + + except Exception as e: + error_msg = f"Failed to upload {filename} to SharePoint: {str(e)}" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise RuntimeError(error_msg) from e diff --git a/app/tasks/upload_to_user_integration.py b/app/tasks/upload_to_user_integration.py index db21701d..1b23d321 100644 --- a/app/tasks/upload_to_user_integration.py +++ b/app/tasks/upload_to_user_integration.py @@ -571,6 +571,113 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t return {"status": "Completed", "rclone_dest": dest} +def _upload_sharepoint(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]: + """Upload *file_path* to SharePoint using per-user MSAL credentials.""" + import urllib.parse + + import msal + import requests as _requests + + client_id = creds.get("client_id") or "" + client_secret = creds.get("client_secret") or "" + refresh_token = creds.get("refresh_token") or "" + tenant = cfg.get("tenant_id") or "common" + site_url = cfg.get("site_url") or "" + library_name = cfg.get("document_library") or "Documents" + folder_path = cfg.get("folder_path") or "" + + if not (client_id and client_secret): + raise ValueError("SharePoint integration is missing client_id or client_secret in credentials") + if not site_url: + raise ValueError("SharePoint integration is missing site_url in config") + + scopes = ["https://graph.microsoft.com/.default"] + msal_app = msal.ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + + if refresh_token: + token_resp = msal_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=scopes) + else: + token_resp = msal_app.acquire_token_for_client(scopes=scopes) + + if "access_token" not in token_resp: + raise ValueError(f"SharePoint token acquisition failed: {token_resp.get('error_description', 'unknown')}") + + access_token = token_resp["access_token"] + headers = {"Authorization": f"Bearer {access_token}"} + + # Resolve site ID + parsed = urllib.parse.urlparse(site_url) + hostname = parsed.hostname + site_path = parsed.path.rstrip("/") + if not hostname or not site_path: + raise ValueError(f"Invalid SharePoint site URL: {site_url}") + + resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}", headers=headers, timeout=30) + resp.raise_for_status() + site_id = resp.json()["id"] + + # Resolve drive ID + resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives", headers=headers, timeout=30) + resp.raise_for_status() + drive_id = None + for drive in resp.json().get("value", []): + if drive.get("name", "").lower() == library_name.lower(): + drive_id = drive["id"] + break + if not drive_id: + raise RuntimeError(f"Document library '{library_name}' not found on SharePoint site") + + filename = os.path.basename(file_path) + + # Build upload-session URL + base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}" + if folder_path: + folder_path = folder_path.strip("/") + encoded_path = "/".join(urllib.parse.quote(p) for p in folder_path.split("/")) + encoded_file = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_path}/{encoded_file}:/createUploadSession" + else: + encoded_file = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_file}:/createUploadSession" + + session_url = f"{base_url}{item_path}" + session_headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + resp = _requests.post( + session_url, + headers=session_headers, + json={"item": {"@microsoft.graph.conflictBehavior": "replace"}}, + timeout=30, + ) + resp.raise_for_status() + upload_url = resp.json()["uploadUrl"] + + file_size = os.path.getsize(file_path) + chunk_size = 10 * 1024 * 1024 + with open(file_path, "rb") as fh: + chunk_num = 0 + while True: + chunk = fh.read(chunk_size) + if not chunk: + break + start = chunk_num * chunk_size + end = start + len(chunk) - 1 + upload_headers = { + "Content-Length": str(len(chunk)), + "Content-Range": f"bytes {start}-{end}/{file_size}", + } + upload_resp = _requests.put(upload_url, headers=upload_headers, data=chunk, timeout=120) + if upload_resp.status_code not in (201, 202): + raise RuntimeError(f"SharePoint chunk upload failed: {upload_resp.status_code}") + chunk_num += 1 + + logger.info("[%s] SharePoint upload complete: %s/%s", task_id, folder_path, filename) + return {"status": "Completed", "sharepoint_folder": folder_path, "filename": filename} + + def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]: """Upload *file_path* to iCloud Drive using per-user credentials. @@ -615,6 +722,7 @@ _UPLOAD_HANDLERS = { IntegrationType.PAPERLESS: _upload_paperless, IntegrationType.EMAIL: _upload_email, IntegrationType.RCLONE: _upload_rclone, + IntegrationType.SHAREPOINT: _upload_sharepoint, IntegrationType.ICLOUD: _upload_icloud, } diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py index eaa37e6e..7212c0f8 100644 --- a/app/utils/allowed_types.py +++ b/app/utils/allowed_types.py @@ -68,6 +68,8 @@ IMAGE_MIME_TYPES: set[str] = { "image/tiff", "image/webp", "image/svg+xml", + "image/heic", + "image/heif", } # --------------------------------------------------------------------------- @@ -124,6 +126,8 @@ ALLOWED_EXTENSIONS: set[str] = { ".tif", ".webp", ".svg", + ".heic", + ".heif", # Web ".html", ".htm", @@ -234,7 +238,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { }, "images": { "label": "Images", - "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)", + "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)", "mime_types": frozenset( { "image/jpeg", @@ -245,6 +249,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { "image/tiff", "image/webp", "image/svg+xml", + "image/heic", + "image/heif", } ), "extensions": frozenset( @@ -258,6 +264,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { ".tif", ".webp", ".svg", + ".heic", + ".heif", } ), }, diff --git a/app/utils/classification_rules.py b/app/utils/classification_rules.py new file mode 100644 index 00000000..760f03e4 --- /dev/null +++ b/app/utils/classification_rules.py @@ -0,0 +1,378 @@ +""" +Rule-based document classification engine. + +Provides pre-built categories and a rule matcher that classifies documents +using filename patterns, content keywords, and metadata fields. Custom +rules stored in the database are evaluated alongside the built-in defaults. + +Usage:: + + from app.utils.classification_rules import classify_document + + result = classify_document( + filename="2024-03-01_Invoice_Acme.pdf", + text="Invoice total: $1,234.56", + metadata={"absender": "Acme Corp"}, + custom_rules=custom_rules_from_db, + ) + # result -> ClassificationResult(category="invoice", confidence=85, matched_rules=[...]) +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Pre-built categories +# --------------------------------------------------------------------------- + +#: Canonical category names recognized by the system. Users may also define +#: their own categories via custom rules. +BUILTIN_CATEGORIES: dict[str, str] = { + "invoice": "Invoice", + "contract": "Contract", + "receipt": "Receipt", + "letter": "Letter", + "report": "Report", + "bank_statement": "Bank Statement", + "tax_document": "Tax Document", + "insurance": "Insurance Document", + "payslip": "Payslip", + "unknown": "Unknown", +} + +# --------------------------------------------------------------------------- +# Rule type constants +# --------------------------------------------------------------------------- + +RULE_TYPE_FILENAME = "filename_pattern" +RULE_TYPE_CONTENT = "content_keyword" +RULE_TYPE_METADATA = "metadata_match" + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass +class ClassificationRule: + """A single classification rule.""" + + name: str + category: str + rule_type: str # filename_pattern | content_keyword | metadata_match + pattern: str # regex for filename, keyword(s) for content, "field=value" for metadata + priority: int = 0 # higher = evaluated first + case_sensitive: bool = False + + def __post_init__(self) -> None: + if self.rule_type not in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA): + raise ValueError(f"Invalid rule_type: {self.rule_type!r}") + + +@dataclass +class MatchedRule: + """Records which rule matched and why.""" + + rule_name: str + rule_type: str + category: str + confidence: int + + +@dataclass +class ClassificationResult: + """The outcome of running the classification engine on a document.""" + + category: str + confidence: int # 0 – 100 + matched_rules: list[MatchedRule] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Built-in rules +# --------------------------------------------------------------------------- + +BUILTIN_RULES: list[ClassificationRule] = [ + # ── Invoice ─────────────────────────────────────────────────────────── + ClassificationRule("builtin_invoice_filename", "invoice", RULE_TYPE_FILENAME, r"(?i)invoice|rechnung|facture"), + ClassificationRule( + "builtin_invoice_content", + "invoice", + RULE_TYPE_CONTENT, + "invoice number|invoice total|amount due|rechnung|rechnungsnummer|total amount|bill to", + ), + ClassificationRule("builtin_invoice_metadata", "invoice", RULE_TYPE_METADATA, "document_type=Invoice"), + ClassificationRule( + "builtin_invoice_kommunikationsart", "invoice", RULE_TYPE_METADATA, "kommunikationsart=Rechnung" + ), + # ── Contract ────────────────────────────────────────────────────────── + ClassificationRule("builtin_contract_filename", "contract", RULE_TYPE_FILENAME, r"(?i)contract|vertrag|agreement"), + ClassificationRule( + "builtin_contract_content", + "contract", + RULE_TYPE_CONTENT, + "hereby agrees|terms and conditions|vertrag|agreement between|party agrees|effective date", + ), + ClassificationRule("builtin_contract_metadata", "contract", RULE_TYPE_METADATA, "document_type=Contract"), + ClassificationRule( + "builtin_contract_kommunikationsart", "contract", RULE_TYPE_METADATA, "kommunikationsart=Vertrag" + ), + # ── Receipt ─────────────────────────────────────────────────────────── + ClassificationRule("builtin_receipt_filename", "receipt", RULE_TYPE_FILENAME, r"(?i)receipt|quittung|beleg"), + ClassificationRule( + "builtin_receipt_content", + "receipt", + RULE_TYPE_CONTENT, + "receipt|quittung|payment received|thank you for your purchase|transaction id", + ), + ClassificationRule("builtin_receipt_metadata", "receipt", RULE_TYPE_METADATA, "document_type=Receipt"), + ClassificationRule( + "builtin_receipt_kommunikationsart", "receipt", RULE_TYPE_METADATA, "kommunikationsart=Quittung" + ), + # ── Letter ──────────────────────────────────────────────────────────── + ClassificationRule("builtin_letter_filename", "letter", RULE_TYPE_FILENAME, r"(?i)letter|brief|schreiben"), + ClassificationRule( + "builtin_letter_content", + "letter", + RULE_TYPE_CONTENT, + "dear sir|dear madam|sehr geehrte|to whom it may concern|sincerely|mit freundlichen", + ), + # ── Report ──────────────────────────────────────────────────────────── + ClassificationRule("builtin_report_filename", "report", RULE_TYPE_FILENAME, r"(?i)report|bericht"), + ClassificationRule( + "builtin_report_content", + "report", + RULE_TYPE_CONTENT, + "executive summary|table of contents|annual report|quarterly report|findings", + ), + # ── Bank statement ──────────────────────────────────────────────────── + ClassificationRule( + "builtin_bank_filename", + "bank_statement", + RULE_TYPE_FILENAME, + r"(?i)bank.?statement|kontoauszug", + ), + ClassificationRule( + "builtin_bank_content", + "bank_statement", + RULE_TYPE_CONTENT, + "account statement|kontoauszug|opening balance|closing balance|account number", + ), + ClassificationRule( + "builtin_bank_kommunikationsart", "bank_statement", RULE_TYPE_METADATA, "kommunikationsart=Kontoauszug" + ), + # ── Tax document ────────────────────────────────────────────────────── + ClassificationRule("builtin_tax_filename", "tax_document", RULE_TYPE_FILENAME, r"(?i)tax|steuer|steuerbescheid"), + ClassificationRule( + "builtin_tax_content", + "tax_document", + RULE_TYPE_CONTENT, + "tax return|steuerbescheid|taxable income|finanzamt|tax assessment", + ), + # ── Insurance ───────────────────────────────────────────────────────── + ClassificationRule( + "builtin_insurance_filename", "insurance", RULE_TYPE_FILENAME, r"(?i)insurance|versicherung|police" + ), + ClassificationRule( + "builtin_insurance_content", + "insurance", + RULE_TYPE_CONTENT, + "insurance policy|versicherung|policennummer|coverage|premium|deductible", + ), + # ── Payslip ─────────────────────────────────────────────────────────── + ClassificationRule( + "builtin_payslip_filename", "payslip", RULE_TYPE_FILENAME, r"(?i)payslip|gehaltsabrechnung|lohnabrechnung" + ), + ClassificationRule( + "builtin_payslip_content", + "payslip", + RULE_TYPE_CONTENT, + "gross salary|net salary|gehaltsabrechnung|lohnabrechnung|bruttolohn|nettolohn", + ), +] + + +# --------------------------------------------------------------------------- +# Confidence scoring +# --------------------------------------------------------------------------- + +#: Base confidence for each rule type when it matches. +_CONFIDENCE_MAP: dict[str, int] = { + RULE_TYPE_FILENAME: 60, + RULE_TYPE_CONTENT: 70, + RULE_TYPE_METADATA: 90, +} + +#: Extra confidence per additional matching rule of the same category (capped). +_CONFIDENCE_BONUS_PER_EXTRA_RULE = 10 + + +# --------------------------------------------------------------------------- +# Matching helpers +# --------------------------------------------------------------------------- + + +def _match_filename(rule: ClassificationRule, filename: str) -> bool: + """Return True if *rule.pattern* (regex) matches anywhere in *filename*.""" + if not filename: + return False + flags = 0 if rule.case_sensitive else re.IGNORECASE + return bool(re.search(rule.pattern, filename, flags)) + + +def _match_content(rule: ClassificationRule, text: str) -> bool: + """Return True if any keyword in *rule.pattern* appears in *text*. + + Keywords are separated by ``|`` (pipe). + """ + if not text: + return False + keywords = [kw.strip() for kw in rule.pattern.split("|") if kw.strip()] + text_lower = text if rule.case_sensitive else text.lower() + return any((kw if rule.case_sensitive else kw.lower()) in text_lower for kw in keywords) + + +def _match_metadata(rule: ClassificationRule, metadata: dict[str, Any] | None) -> bool: + """Return True if *rule.pattern* (``field=value``) matches *metadata*. + + Pattern format: ``field_name=expected_value``. + """ + if not metadata: + return False + if "=" not in rule.pattern: + return False + field_name, expected_value = rule.pattern.split("=", 1) + actual = metadata.get(field_name.strip()) + if actual is None: + return False + if rule.case_sensitive: + return str(actual) == expected_value.strip() + return str(actual).lower() == expected_value.strip().lower() + + +_MATCHERS: dict[str, tuple] = { + RULE_TYPE_FILENAME: (_match_filename, "filename"), + RULE_TYPE_CONTENT: (_match_content, "text"), + RULE_TYPE_METADATA: (_match_metadata, "metadata"), +} + + +def _evaluate_rule( + rule: ClassificationRule, + filename: str, + text: str, + metadata: dict[str, Any] | None, +) -> MatchedRule | None: + """Evaluate a single rule against the document. Return a :class:`MatchedRule` on match.""" + entry = _MATCHERS.get(rule.rule_type) + if entry is None: + return None + + matcher, arg_key = entry + arg_map = {"filename": filename, "text": text, "metadata": metadata} + matched = matcher(rule, arg_map[arg_key]) + + if matched: + return MatchedRule( + rule_name=rule.name, + rule_type=rule.rule_type, + category=rule.category, + confidence=_CONFIDENCE_MAP.get(rule.rule_type, 50), + ) + return None + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def classify_document( + filename: str = "", + text: str = "", + metadata: dict[str, Any] | None = None, + custom_rules: list[ClassificationRule] | None = None, +) -> ClassificationResult: + """Classify a document by evaluating built-in and custom rules. + + Rules are evaluated in priority order (highest first, then built-in before + custom for the same priority). The category with the most rule matches + wins; ties are broken by cumulative confidence. + + Args: + filename: Original filename of the document. + text: Extracted / OCR text of the document. + metadata: Previously-extracted AI metadata dict (e.g. from ``ai_metadata``). + custom_rules: Optional list of user-defined :class:`ClassificationRule` objects. + + Returns: + A :class:`ClassificationResult` with the best matching category, + overall confidence score, and the list of rules that fired. + """ + all_rules = list(BUILTIN_RULES) + if custom_rules: + all_rules.extend(custom_rules) + + # Sort by priority descending (higher priority first) + all_rules.sort(key=lambda r: r.priority, reverse=True) + + matches: list[MatchedRule] = [] + for rule in all_rules: + result = _evaluate_rule(rule, filename, text, metadata) + if result is not None: + matches.append(result) + + if not matches: + return ClassificationResult(category="unknown", confidence=0, matched_rules=[]) + + # Aggregate by category: pick the one with the most matches, then highest + # cumulative confidence as tiebreaker. + category_scores: dict[str, list[MatchedRule]] = {} + for m in matches: + category_scores.setdefault(m.category, []).append(m) + + best_category = max( + category_scores, + key=lambda cat: (len(category_scores[cat]), sum(m.confidence for m in category_scores[cat])), + ) + + best_matches = category_scores[best_category] + base_confidence = max(m.confidence for m in best_matches) + bonus = min( + (len(best_matches) - 1) * _CONFIDENCE_BONUS_PER_EXTRA_RULE, + 100 - base_confidence, + ) + final_confidence = min(base_confidence + bonus, 100) + + return ClassificationResult( + category=best_category, + confidence=final_confidence, + matched_rules=best_matches, + ) + + +def db_rule_to_engine_rule(db_rule: Any) -> ClassificationRule: + """Convert a database ``ClassificationRuleModel`` row to an engine :class:`ClassificationRule`. + + Args: + db_rule: A SQLAlchemy model instance with ``name``, ``category``, + ``rule_type``, ``pattern``, ``priority``, and ``case_sensitive`` attributes. + + Returns: + A :class:`ClassificationRule` dataclass instance. + """ + return ClassificationRule( + name=db_rule.name, + category=db_rule.category, + rule_type=db_rule.rule_type, + pattern=db_rule.pattern, + priority=db_rule.priority, + case_sensitive=getattr(db_rule, "case_sensitive", False), + ) diff --git a/app/utils/config_validator/providers.py b/app/utils/config_validator/providers.py index 9a1e7069..91383438 100644 --- a/app/utils/config_validator/providers.py +++ b/app/utils/config_validator/providers.py @@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]: }, } + # Check SharePoint configuration + providers["SharePoint"] = { + "name": "SharePoint", + "icon": "fa-brands fa-microsoft", + "configured": bool( + getattr(settings, "sharepoint_client_id", None) + and getattr(settings, "sharepoint_client_secret", None) + and getattr(settings, "sharepoint_site_url", None) + ), + "enabled": True, + "description": "Store documents in Microsoft SharePoint Online", + "details": { + "client_id": getattr(settings, "sharepoint_client_id", "Not set"), + "client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)), + "tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"), + "refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)), + "site_url": getattr(settings, "sharepoint_site_url", "Not set"), + "document_library": getattr(settings, "sharepoint_document_library", "Not set"), + "folder_path": getattr(settings, "sharepoint_folder_path", "Not set"), + }, + } + # Check S3 configuration providers["S3 Storage"] = { "name": "S3 Storage", diff --git a/app/utils/session_manager.py b/app/utils/session_manager.py new file mode 100644 index 00000000..e0de02bc --- /dev/null +++ b/app/utils/session_manager.py @@ -0,0 +1,505 @@ +"""Server-side session management utilities. + +Provides helpers for creating, validating, and revoking user sessions. +Sessions are tracked in the ``user_sessions`` table and referenced by a +cryptographically random token stored in the browser cookie. This enables +the "log off everywhere" feature and per-session revocation. +""" + +from __future__ import annotations + +import logging +import secrets +from datetime import datetime, timedelta, timezone + +from sqlalchemy.orm import Session + +from app.config import settings +from app.models import ApiToken, QRLoginChallenge, UserSession + +logger = logging.getLogger(__name__) + + +def _ensure_tz_aware(dt: datetime | None) -> datetime | None: + """Return *dt* with UTC tzinfo if it is naive, or unchanged if already aware. + + SQLite does not persist timezone information, so datetimes read back from + the database are offset-naive. This helper normalises them for safe + comparison with ``datetime.now(timezone.utc)``. + """ + if dt is not None and dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt + + +def get_session_lifetime_days() -> int: + """Return the effective session lifetime in days. + + If ``session_lifetime_custom_days`` is set it takes precedence over + ``session_lifetime_days``. + """ + custom = getattr(settings, "session_lifetime_custom_days", None) + if custom is not None and isinstance(custom, int) and custom > 0: + return custom + return max(1, getattr(settings, "session_lifetime_days", 30)) + + +def get_session_max_age_seconds() -> int: + """Return the session max-age in seconds for the cookie.""" + return get_session_lifetime_days() * 86400 + + +def create_session( + db: Session, + user_id: str, + ip_address: str | None = None, + user_agent: str | None = None, +) -> UserSession: + """Create a new server-side session record. + + Args: + db: Database session. + user_id: Stable owner identifier. + ip_address: Client IP address. + user_agent: Client User-Agent header. + + Returns: + The newly created ``UserSession`` instance. + """ + session_token = secrets.token_urlsafe(64) + now = datetime.now(timezone.utc) + lifetime_days = get_session_lifetime_days() + expires_at = now + timedelta(days=lifetime_days) + + device_info = _parse_device_info(user_agent) + + user_session = UserSession( + session_token=session_token, + user_id=user_id, + ip_address=ip_address, + user_agent=(user_agent or "")[:512], + device_info=device_info, + created_at=now, + last_active_at=now, + expires_at=expires_at, + ) + try: + db.add(user_session) + db.commit() + db.refresh(user_session) + except Exception: + db.rollback() + logger.exception("Failed to create session for user_id=%s", user_id) + raise + + logger.info( + "[SESSION] Created session id=%s user=%s device=%r expires=%s", + user_session.id, + user_id, + device_info, + expires_at.isoformat(), + ) + return user_session + + +def validate_session(db: Session, session_token: str) -> UserSession | None: + """Validate a session token and return the session if valid. + + A session is valid when: + * It exists in the database. + * ``is_revoked`` is ``False``. + * ``expires_at`` is in the future. + + Side-effect: updates ``last_active_at`` on valid sessions. + + Returns: + The ``UserSession`` if valid, else ``None``. + """ + if not session_token: + return None + + now = datetime.now(timezone.utc) + user_session = db.query(UserSession).filter(UserSession.session_token == session_token).first() + + if not user_session: + logger.debug("[SESSION] Token not found in database") + return None + + if user_session.is_revoked: + logger.debug("[SESSION] Session id=%s is revoked", user_session.id) + return None + + if user_session.expires_at: + expires = _ensure_tz_aware(user_session.expires_at) + if expires < now: + logger.debug("[SESSION] Session id=%s has expired", user_session.id) + return None + + # Update last_active_at (throttled to avoid excessive writes) + last_active = _ensure_tz_aware(user_session.last_active_at) + if not last_active or (now - last_active).total_seconds() > 60: + try: + user_session.last_active_at = now + db.commit() + except Exception: + db.rollback() + logger.debug("[SESSION] Failed to update last_active_at for session id=%s", user_session.id) + + return user_session + + +def revoke_session(db: Session, session_id: int, user_id: str) -> bool: + """Revoke a single session by ID. + + Args: + db: Database session. + session_id: The session record ID to revoke. + user_id: The owner — ensures a user can only revoke their own sessions. + + Returns: + ``True`` if the session was found and revoked, ``False`` otherwise. + """ + user_session = db.get(UserSession, session_id) + if not user_session or user_session.user_id != user_id: + return False + + now = datetime.now(timezone.utc) + user_session.is_revoked = True + user_session.revoked_at = now + try: + db.commit() + except Exception: + db.rollback() + raise + + logger.info("[SESSION] Revoked session id=%s user=%s", session_id, user_id) + return True + + +def revoke_all_sessions( + db: Session, + user_id: str, + *, + except_session_id: int | None = None, + revoke_api_tokens: bool = True, +) -> int: + """Revoke all active sessions for a user ("log off everywhere"). + + Args: + db: Database session. + user_id: The owner whose sessions should be revoked. + except_session_id: If provided, keep this session active (the + current browser session). + revoke_api_tokens: If ``True``, also revoke all active API tokens. + + Returns: + Number of sessions revoked. + """ + now = datetime.now(timezone.utc) + query = db.query(UserSession).filter( + UserSession.user_id == user_id, + UserSession.is_revoked.is_(False), + ) + if except_session_id is not None: + query = query.filter(UserSession.id != except_session_id) + + sessions = query.all() + count = 0 + for s in sessions: + s.is_revoked = True + s.revoked_at = now + count += 1 + + if revoke_api_tokens: + tokens = ( + db.query(ApiToken) + .filter( + ApiToken.owner_id == user_id, + ApiToken.is_active.is_(True), + ) + .all() + ) + for t in tokens: + t.is_active = False + t.revoked_at = now + + try: + db.commit() + except Exception: + db.rollback() + raise + + logger.info( + "[SESSION] Revoked all sessions for user=%s (count=%d, except_session_id=%s, tokens_revoked=%s)", + user_id, + count, + except_session_id, + revoke_api_tokens, + ) + return count + + +def list_user_sessions(db: Session, user_id: str) -> list[UserSession]: + """Return all non-revoked, non-expired sessions for a user. + + Results are ordered by most recently active first. + """ + now = datetime.now(timezone.utc) + sessions = ( + db.query(UserSession) + .filter( + UserSession.user_id == user_id, + UserSession.is_revoked.is_(False), + ) + .order_by(UserSession.last_active_at.desc()) + .all() + ) + # Filter expired sessions in Python to handle timezone-naive datetimes (SQLite) + result = [] + for s in sessions: + expires = _ensure_tz_aware(s.expires_at) + if expires and expires > now: + result.append(s) + return result + + +def cleanup_expired_sessions(db: Session) -> int: + """Delete sessions that expired more than 7 days ago. + + Intended to be called periodically (e.g. via Celery beat) to keep the + table from growing unbounded. + + Returns: + Number of rows deleted. + """ + cutoff = datetime.now(timezone.utc) - timedelta(days=7) + count = db.query(UserSession).filter(UserSession.expires_at < cutoff).delete(synchronize_session=False) + try: + db.commit() + except Exception: + db.rollback() + raise + if count: + logger.info("[SESSION] Cleaned up %d expired sessions", count) + return count + + +# --------------------------------------------------------------------------- +# QR login helpers +# --------------------------------------------------------------------------- + + +def create_qr_challenge(db: Session, user_id: str, ip_address: str | None = None) -> QRLoginChallenge: + """Create a new QR login challenge. + + Args: + db: Database session. + user_id: The authenticated web user creating the challenge. + ip_address: IP address of the web client. + + Returns: + The newly created ``QRLoginChallenge``. + """ + token = secrets.token_urlsafe(64) + ttl = getattr(settings, "qr_login_challenge_ttl_seconds", 120) + now = datetime.now(timezone.utc) + expires_at = now + timedelta(seconds=ttl) + + challenge = QRLoginChallenge( + challenge_token=token, + user_id=user_id, + created_by_ip=ip_address, + created_at=now, + expires_at=expires_at, + ) + try: + db.add(challenge) + db.commit() + db.refresh(challenge) + except Exception: + db.rollback() + logger.exception("Failed to create QR login challenge for user_id=%s", user_id) + raise + + logger.info("[QR_AUTH] Challenge created: id=%s user=%s expires=%s", challenge.id, user_id, expires_at.isoformat()) + return challenge + + +def validate_qr_challenge(db: Session, challenge_token: str) -> QRLoginChallenge | None: + """Validate a QR challenge token without claiming it. + + Returns the challenge if it exists, is not expired, not claimed, + and not cancelled. Returns ``None`` otherwise. + """ + if not challenge_token: + return None + + now = datetime.now(timezone.utc) + challenge = db.query(QRLoginChallenge).filter(QRLoginChallenge.challenge_token == challenge_token).first() + + if not challenge: + return None + if challenge.is_claimed or challenge.is_cancelled: + return None + + expires = _ensure_tz_aware(challenge.expires_at) + if expires and expires < now: + return None + + return challenge + + +def claim_qr_challenge( + db: Session, + challenge_token: str, + device_name: str = "Mobile App", + ip_address: str | None = None, +) -> dict | None: + """Claim a QR challenge and issue an API token. + + This is the critical security path. The challenge is validated, + marked as claimed atomically, and an API token is issued for the + user who created the challenge. + + Args: + db: Database session. + challenge_token: The token from the QR code. + device_name: Name provided by the mobile app. + ip_address: IP address of the claiming mobile device. + + Returns: + Dict with ``token`` (plaintext), ``token_id``, ``name``, ``owner_id`` + and ``created_at`` on success, or ``None`` if the challenge is invalid. + """ + from app.api.api_tokens import generate_api_token, hash_token + + challenge = validate_qr_challenge(db, challenge_token) + if not challenge: + logger.warning("[QR_AUTH] Invalid or expired challenge token attempted") + return None + + now = datetime.now(timezone.utc) + + # Mark as claimed first to prevent race conditions + challenge.is_claimed = True + challenge.claimed_at = now + challenge.claimed_by_ip = ip_address + challenge.device_name = device_name + + # Generate API token for the mobile app + token_name = f"Mobile App (QR) – {device_name}" + plaintext = generate_api_token() + token_hash_value = hash_token(plaintext) + prefix = plaintext[:12] + + db_token = ApiToken( + owner_id=challenge.user_id, + name=token_name, + token_hash=token_hash_value, + token_prefix=prefix, + ) + + try: + db.add(db_token) + db.flush() + challenge.issued_token_id = db_token.id + db.commit() + db.refresh(db_token) + except Exception: + db.rollback() + logger.exception("[QR_AUTH] Failed to issue token for challenge id=%s", challenge.id) + raise + + logger.info( + "[QR_AUTH] Challenge claimed: id=%s user=%s device=%r token_id=%s", + challenge.id, + challenge.user_id, + device_name, + db_token.id, + ) + + return { + "token": plaintext, + "token_id": db_token.id, + "name": token_name, + "owner_id": challenge.user_id, + "created_at": db_token.created_at, + } + + +def get_challenge_status(db: Session, challenge_id: int, user_id: str) -> dict | None: + """Get the current status of a QR challenge (for polling from the web UI). + + Returns: + Dict with ``status`` ("pending", "claimed", "expired", "cancelled") + and metadata, or ``None`` if the challenge doesn't belong to the user. + """ + challenge = db.get(QRLoginChallenge, challenge_id) + if not challenge or challenge.user_id != user_id: + return None + + now = datetime.now(timezone.utc) + expires = _ensure_tz_aware(challenge.expires_at) + + if challenge.is_claimed: + status = "claimed" + elif challenge.is_cancelled: + status = "cancelled" + elif expires and expires < now: + status = "expired" + else: + status = "pending" + + return { + "id": challenge.id, + "status": status, + "device_name": challenge.device_name, + "claimed_at": challenge.claimed_at, + "expires_at": challenge.expires_at, + } + + +def _parse_device_info(user_agent: str | None) -> str | None: + """Extract a human-readable device description from User-Agent. + + This is a lightweight parser — not a full UA library — that covers + the most common browsers and platforms. + """ + if not user_agent: + return None + + ua = user_agent.lower() + + # Platform detection + platform = "Unknown" + if "iphone" in ua: + platform = "iPhone" + elif "ipad" in ua: + platform = "iPad" + elif "android" in ua: + platform = "Android" + elif "macintosh" in ua or "mac os" in ua: + platform = "macOS" + elif "windows" in ua: + platform = "Windows" + elif "linux" in ua: + platform = "Linux" + elif "cros" in ua: + platform = "ChromeOS" + + # Browser detection + browser = "Unknown Browser" + if "edg/" in ua or "edge/" in ua: + browser = "Edge" + elif "opr/" in ua or "opera" in ua: + browser = "Opera" + elif "chrome/" in ua and "safari/" in ua: + browser = "Chrome" + elif "safari/" in ua and "chrome/" not in ua: + browser = "Safari" + elif "firefox/" in ua: + browser = "Firefox" + elif "docuelevate" in ua: + browser = "DocuElevate App" + + return f"{browser} on {platform}" diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 78ff6cca..0c990143 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -39,6 +39,50 @@ SETTING_METADATA = { "required": True, "restart_required": True, }, + "db_pool_size": { + "category": "Core", + "description": ( + "Number of persistent database connections kept in the pool per worker process. " + "Ignored for SQLite (which uses NullPool). Default: 10." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_max_overflow": { + "category": "Core", + "description": ( + "Additional database connections allowed beyond db_pool_size under burst load. " + "Ignored for SQLite. Default: 20." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_pool_timeout": { + "category": "Core", + "description": ( + "Seconds to wait for a database connection from the pool before raising a TimeoutError. " + "Ignored for SQLite. Default: 30." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_pool_recycle": { + "category": "Core", + "description": ( + "Recycle (close and reopen) database connections after this many seconds " + "to avoid stale connections. Ignored for SQLite. Default: 1800." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, "workdir": { "category": "Core", "description": "Working directory for file storage and processing", @@ -55,6 +99,18 @@ SETTING_METADATA = { "required": True, # Required for OAuth redirects and external URLs "restart_required": True, }, + "public_base_url": { + "category": "Core", + "description": ( + "Full public base URL including scheme (e.g., https://docuelevate.example.com). " + "When set, overrides auto-detected URLs for OAuth redirect URIs. " + "Required when behind a reverse proxy that does not forward X-Forwarded-Proto." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, "debug": { "category": "Core", "description": "Enable debug mode for verbose logging", @@ -134,6 +190,30 @@ SETTING_METADATA = { "required": True, # Required when auth_enabled=True (validated in config.py) "restart_required": True, }, + "session_lifetime_days": { + "category": "Authentication", + "description": "Session lifetime in days (default 30). Determines how long a user stays logged in.", + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "session_lifetime_custom_days": { + "category": "Authentication", + "description": "Override session_lifetime_days with a custom value. Takes precedence when set.", + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "qr_login_challenge_ttl_seconds": { + "category": "Authentication", + "description": "Time-to-live in seconds for QR login challenges (default 120).", + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, "admin_username": { "category": "Authentication", "description": "Admin username for local authentication", @@ -302,6 +382,19 @@ SETTING_METADATA = { "required": False, "restart_required": True, }, + "social_auth_dropbox_use_global_credentials": { + "category": "Social Login", + "description": ( + "When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET " + "credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / " + "SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. " + "Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, "social_auth_dropbox_enabled": { "category": "Social Login", "description": ( @@ -695,6 +788,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "dropbox_allow_global_credentials_for_integrations": { + "category": "Storage Providers", + "description": ( + "When True, users may authorize their personal Dropbox integrations using the global " + "DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without " + "needing to create their own Dropbox app." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Storage Providers - Nextcloud "nextcloud_enabled": { "category": "Storage Providers", @@ -875,6 +980,63 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Storage Providers - SharePoint + "sharepoint_client_id": { + "category": "Storage Providers", + "description": "SharePoint Azure AD application (client) ID", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_client_secret": { + "category": "Storage Providers", + "description": "SharePoint Azure AD client secret", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": False, + }, + "sharepoint_tenant_id": { + "category": "Storage Providers", + "description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_refresh_token": { + "category": "Storage Providers", + "description": "SharePoint OAuth refresh token", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": False, + }, + "sharepoint_site_url": { + "category": "Storage Providers", + "description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_document_library": { + "category": "Storage Providers", + "description": "SharePoint document library name (default: 'Documents')", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_folder_path": { + "category": "Storage Providers", + "description": "Subfolder path inside the SharePoint document library", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Storage Providers - WebDAV "webdav_enabled": { "category": "Storage Providers", @@ -1912,6 +2074,28 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "factory_reset_on_startup": { + "category": "Feature Flags", + "description": ( + "Wipe all user data on every startup so the instance always starts fresh. " + "Useful for demo/testing environments. Default: False." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "enable_factory_reset": { + "category": "Feature Flags", + "description": ( + "Show the System Reset page in the admin UI. Allows administrators to " + "trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Backup / Restore "backup_enabled": { "category": "Backup", @@ -1935,14 +2119,26 @@ SETTING_METADATA = { "category": "Backup", "description": ( "Storage provider for remote backup copies. " - "Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. " + "Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. " "Leave empty to keep backups local only." ), "type": "string", "sensitive": False, "required": False, "restart_required": False, - "options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"], + "options": [ + "", + "s3", + "dropbox", + "google_drive", + "onedrive", + "sharepoint", + "nextcloud", + "webdav", + "ftp", + "sftp", + "email", + ], }, "backup_remote_folder": { "category": "Backup", @@ -2454,6 +2650,27 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Per-user upload rate limiting + "upload_rate_limit_per_user": { + "category": "Security", + "description": ( + "Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. " + "The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. " + "Default: 20." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "upload_rate_limit_window": { + "category": "Security", + "description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Rate Limiting "rate_limiting_enabled": { "category": "Security", diff --git a/app/utils/system_reset.py b/app/utils/system_reset.py new file mode 100644 index 00000000..dfeaa5b5 --- /dev/null +++ b/app/utils/system_reset.py @@ -0,0 +1,296 @@ +""" +System reset utilities for DocuElevate. + +Provides functions to: +- Wipe all user data (database rows + work-files on disk) for a fresh start. +- Wipe with re-import: move original files to a dedicated folder, wipe + everything, then let the watch-folder mechanism re-ingest the files. + +Security: All public functions in this module require admin-level access. +They MUST only be invoked from admin-guarded API/view endpoints. +""" + +import logging +import shutil +from pathlib import Path + +from sqlalchemy.orm import Session + +from app.config import settings + +logger = logging.getLogger(__name__) + +# Subdirectories inside *workdir* that contain user-generated data. +# Everything else (app code, static assets, config) is left untouched. +_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups") + +# JSON cache files written by watch-folder / ingest tasks. +_CACHE_FILES = ( + "watch_folder_processed.json", + "ftp_ingest_processed.json", + "sftp_ingest_processed.json", + "dropbox_ingest_processed.json", + "gdrive_ingest_processed.json", + "onedrive_ingest_processed.json", + "nextcloud_ingest_processed.json", + "s3_ingest_processed.json", + "webdav_ingest_processed.json", + "processed_mails.json", + "credential_failures.json", +) + +# The folder name used for storing files prior to re-import. +REIMPORT_FOLDER_NAME = "reimport" + + +def _wipe_workdir_data(workdir: str) -> dict[str, int]: + """Delete user data subdirectories and cache files inside *workdir*. + + Leaves the workdir directory itself intact so the application can + continue to write into it. Also leaves any files that do not belong + to the known data subdirectories or caches. + + Returns: + A dict with counts of deleted directories and files. + """ + workdir_path = Path(workdir) + deleted_dirs = 0 + deleted_files = 0 + + # Remove data subdirectories + for subdir in _USER_DATA_SUBDIRS: + target = workdir_path / subdir + if target.is_dir(): + shutil.rmtree(target) + logger.info("Deleted data directory: %s", target) + deleted_dirs += 1 + + # Remove cache / state JSON files + for cache_file in _CACHE_FILES: + target = workdir_path / cache_file + if target.is_file(): + target.unlink() + logger.info("Deleted cache file: %s", target) + deleted_files += 1 + + # Also remove user_wf_*.json files (per-user watch folder caches) + for f in workdir_path.glob("user_wf_*.json"): + f.unlink() + logger.info("Deleted user watch-folder cache: %s", f) + deleted_files += 1 + + # Remove loose files in workdir root that are user uploads (uuid-named + # files like "a1b2c3d4-…pdf") but NOT application config files. + for entry in workdir_path.iterdir(): + if entry.is_file() and entry.suffix.lower() in { + ".pdf", + ".png", + ".jpg", + ".jpeg", + ".tiff", + ".tif", + ".docx", + ".doc", + ".xlsx", + ".xls", + ".pptx", + ".heic", + ".heif", + ".webp", + ".bmp", + ".gif", + ".txt", + ".rtf", + ".odt", + ".ods", + ".odp", + ".csv", + ".pages", + ".numbers", + ".keynote", + }: + entry.unlink() + logger.info("Deleted loose workdir file: %s", entry) + deleted_files += 1 + + return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files} + + +def _wipe_database(db: Session) -> dict[str, int]: + """Delete all user-generated rows from the database. + + Preserves schema (tables, migrations) and system-seeded rows that will + be re-created on the next startup (subscription plans, default pipeline, + scheduled jobs, compliance templates). + + Returns: + A dict mapping table name → number of rows deleted. + """ + from app.models import ( + AuditLog, + BackupRecord, + DocumentMetadata, + FileProcessingStep, + FileRecord, + InAppNotification, + ProcessingLog, + SavedSearch, + SettingsAuditLog, + SharedLink, + UserImapAccount, + UserIntegration, + UserNotificationPreference, + UserNotificationTarget, + ) + + # Order matters: delete children before parents to respect FK constraints. + tables_to_wipe: list[tuple[str, type]] = [ + ("file_processing_steps", FileProcessingStep), + ("processing_logs", ProcessingLog), + ("shared_links", SharedLink), + ("in_app_notifications", InAppNotification), + ("user_notification_preferences", UserNotificationPreference), + ("user_notification_targets", UserNotificationTarget), + ("user_imap_accounts", UserImapAccount), + ("user_integrations", UserIntegration), + ("saved_searches", SavedSearch), + ("settings_audit_log", SettingsAuditLog), + ("audit_logs", AuditLog), + ("backup_records", BackupRecord), + ("document_metadata", DocumentMetadata), + ("files", FileRecord), + ] + + result: dict[str, int] = {} + for table_name, model in tables_to_wipe: + try: + count = db.query(model).delete() + result[table_name] = count + logger.info("Wiped %d rows from %s", count, table_name) + except Exception: + logger.exception("Failed to wipe table %s during system reset", table_name) + db.rollback() + raise + + db.commit() + return result + + +def perform_full_reset(db: Session) -> dict: + """Perform a complete system reset: wipe database rows + work-files. + + Args: + db: An active SQLAlchemy session. + + Returns: + Summary dict with ``database`` and ``filesystem`` sub-dicts. + """ + logger.warning(">>> SYSTEM RESET: wiping all user data <<<") + + db_result = _wipe_database(db) + fs_result = _wipe_workdir_data(settings.workdir) + + logger.warning(">>> SYSTEM RESET complete <<<") + return {"database": db_result, "filesystem": fs_result} + + +def perform_reset_and_reimport(db: Session) -> dict: + """Move original files to a reimport folder, wipe everything, then + configure the reimport folder as a watch folder for re-ingestion. + + The watch-folder scanner (``scan_all_watch_folders``) will pick up + the files on its next periodic run and process them exactly as if + they had been freshly uploaded — respecting the same backoff + strategy, size limits, and rate limits. + + Args: + db: An active SQLAlchemy session. + + Returns: + Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts. + """ + workdir_path = Path(settings.workdir) + reimport_dir = workdir_path / REIMPORT_FOLDER_NAME + original_dir = workdir_path / "original" + + # 1. Collect original files + files_moved = 0 + reimport_dir.mkdir(parents=True, exist_ok=True) + + if original_dir.is_dir(): + for entry in original_dir.iterdir(): + if entry.is_file(): + # Validate the resolved path stays within original_dir (path traversal guard) + try: + entry.resolve().relative_to(original_dir.resolve()) + except ValueError: + logger.warning("Skipping file outside original dir: %s", entry) + continue + dest = reimport_dir / entry.name + # Avoid overwriting: append counter if name clash + if dest.exists(): + stem = dest.stem + suffix = dest.suffix + counter = 1 + while dest.exists(): + dest = reimport_dir / f"{stem}_{counter}{suffix}" + counter += 1 + shutil.copy2(str(entry), str(dest)) + files_moved += 1 + + logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir) + + # 2. Perform the full reset (wipe DB + other workdir data) + reset_result = perform_full_reset(db) + + # 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS) + # and set up watch folder config to point at it. + _configure_reimport_watch_folder(str(reimport_dir)) + + reset_result["reimport"] = { + "files_moved": files_moved, + "reimport_folder": str(reimport_dir), + } + logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved) + return reset_result + + +def _configure_reimport_watch_folder(reimport_path: str) -> None: + """Append *reimport_path* to the application's watch-folder list. + + The watch-folder scanner uses ``settings.watch_folders`` (a + comma-separated string). We mutate the runtime setting so the + next scan picks up the folder. We also set + ``watch_folder_delete_after_process = True`` so files are cleaned + up after successful processing. + """ + current = getattr(settings, "watch_folders", None) or "" + folders = [f.strip() for f in current.split(",") if f.strip()] + + if reimport_path not in folders: + folders.append(reimport_path) + + # Mutate runtime settings (not persisted to .env — ephemeral) + object.__setattr__(settings, "watch_folders", ",".join(folders)) + object.__setattr__(settings, "watch_folder_delete_after_process", True) + logger.info("Configured reimport watch folder: %s", reimport_path) + + +def perform_startup_reset() -> None: + """Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``. + + Wipes database and filesystem data so the instance starts completely + fresh. Uses its own DB session so it runs before the normal lifespan + seeding logic. + """ + from app.database import SessionLocal + + logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data") + db = SessionLocal() + try: + perform_full_reset(db) + except Exception: + logger.exception("Factory reset on startup failed") + db.rollback() + finally: + db.close() diff --git a/app/views/__init__.py b/app/views/__init__.py index b25a3be8..377a0309 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -10,6 +10,7 @@ from app.views.audit_logs import router as audit_logs_router from app.views.backup import router as backup_router from app.views.compliance import router as compliance_router from app.views.db_wizard import router as db_wizard_router +from app.views.devices import router as devices_router # Mobile devices dashboard from app.views.dropbox import router as dropbox_router from app.views.filemanager import router as filemanager_router @@ -26,6 +27,7 @@ from app.views.onedrive import router as onedrive_router from app.views.pipelines import router as pipelines_router # Processing pipelines from app.views.plans import router as plans_router # Admin Plan Designer from app.views.profile import router as profile_router # User self-service profile +from app.views.qr_login import router as qr_login_router # QR code mobile login from app.views.queue import router as queue_router from app.views.scheduled_jobs import router as scheduled_jobs_router # Scheduled batch jobs from app.views.search import router as search_router @@ -34,6 +36,7 @@ from app.views.share import router as share_router from app.views.shared_links import router as shared_links_router from app.views.status import router as status_router from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages +from app.views.system_reset import router as system_reset_router # System reset / factory reset from app.views.wizard import router as wizard_router # Create a main router that includes all the view routers @@ -60,6 +63,7 @@ router.include_router(plans_router) # Admin Plan Designer router.include_router(onboarding_router) # User onboarding wizard router.include_router(pipelines_router) # Processing pipelines router.include_router(profile_router) # User self-service profile settings +router.include_router(qr_login_router) # QR code mobile login page router.include_router(imap_accounts_router) # Per-user IMAP ingestion accounts router.include_router(integrations_router) # Unified integrations dashboard router.include_router(notifications_router) # User notification dashboard @@ -67,3 +71,5 @@ router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs router.include_router(audit_logs_router) # Comprehensive audit log viewer router.include_router(help_router) # Built-in help / How-To docs router.include_router(compliance_router) # Compliance templates dashboard +router.include_router(devices_router) # Mobile devices dashboard +router.include_router(system_reset_router) # System reset / factory reset diff --git a/app/views/base.py b/app/views/base.py index 609ff9fb..0c00b551 100644 --- a/app/views/base.py +++ b/app/views/base.py @@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None: "allow_signup", getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False), ) + ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False)) req = ctx.get("request") if req is not None: diff --git a/app/views/devices.py b/app/views/devices.py new file mode 100644 index 00000000..e8c346d1 --- /dev/null +++ b/app/views/devices.py @@ -0,0 +1,25 @@ +"""View route for the Devices management page. + +Renders the ``devices.html`` template where users can see their registered +mobile devices, mobile API tokens (created via the mobile SSO flow or QR +code login), and revoke access per-device. +""" + +import logging + +from fastapi import APIRouter, Request + +from app.views.base import require_login, templates + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/devices", include_in_schema=False) +@require_login +async def devices_page(request: Request): + """Render the Devices management page.""" + return templates.TemplateResponse( + "devices.html", + {"request": request, "page_title": "Devices"}, + ) diff --git a/app/views/dropbox.py b/app/views/dropbox.py index f623a7c6..5723650a 100644 --- a/app/views/dropbox.py +++ b/app/views/dropbox.py @@ -14,6 +14,19 @@ from app.views.base import APIRouter, Depends, get_db, require_login, settings, router = APIRouter() +def _get_dropbox_callback_url(request: Request) -> str: + """Return the Dropbox OAuth callback URL. + + Uses ``PUBLIC_BASE_URL`` when configured so that the redirect URI displayed + to the user (and registered in the Dropbox developer console) matches the + one used in the OAuth authorization request. Falls back to deriving the URL + from the incoming request when ``PUBLIC_BASE_URL`` is not set. + """ + if settings.public_base_url: + return settings.public_base_url.rstrip("/") + "/dropbox-callback" + return f"{request.url.scheme}://{request.url.netloc}/dropbox-callback" + + @router.get("/dropbox-setup") @require_login async def dropbox_setup_page( @@ -30,6 +43,8 @@ async def dropbox_setup_page( path from the integration's existing config is pre-populated; global admin credentials are never exposed in this mode. """ + callback_url = _get_dropbox_callback_url(request) + if integration_id is not None: owner_id = get_current_owner_id(request) integration = ( @@ -46,6 +61,12 @@ async def dropbox_setup_page( cfg = {} # Support both "folder" (DROPBOX destination) and "folder_path" (WATCH_FOLDER source) folder_path = cfg.get("folder", cfg.get("folder_path", "")) + # Determine if global credentials are available for users to reuse + global_creds_available = bool( + settings.dropbox_allow_global_credentials_for_integrations + and settings.dropbox_app_key + and settings.dropbox_app_secret + ) return templates.TemplateResponse( "dropbox.html", { @@ -56,9 +77,12 @@ async def dropbox_setup_page( "integration_name": integration.name, "integration_type": integration.integration_type, "folder_path": folder_path, - "app_key_value": "", + # Only expose the public app key (not the secret) when global creds are allowed + "app_key_value": settings.dropbox_app_key if global_creds_available else "", "app_secret_value": "", "refresh_token_value": "", + "global_creds_available": global_creds_available, + "callback_url": callback_url, }, ) @@ -78,6 +102,7 @@ async def dropbox_setup_page( "integration_id": integration_id, "integration_name": None, "integration_type": None, + "callback_url": callback_url, }, ) @@ -108,5 +133,6 @@ async def dropbox_callback(request: Request, code: str = None, error: str = None "app_key_value": "", # The callback will prioritize sessionStorage values "app_secret_value": "", # The callback will prioritize sessionStorage values "folder_path": "", # The callback will prioritize sessionStorage values + "callback_url": _get_dropbox_callback_url(request), }, ) diff --git a/app/views/files.py b/app/views/files.py index aa1ee577..2f9bc3c4 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -396,9 +396,7 @@ _STEP_TYPE_TO_STAGES: dict[str, list[str]] = { "embed_metadata": ["embed_metadata_into_pdf"], "compute_embedding": ["compute_embedding"], "send_to_destinations": ["finalize_document_storage", "send_to_all_destinations"], - # "classify" is defined in PIPELINE_STEP_TYPES but has no Celery log stages yet. - # When a classify task is implemented, add its stage key(s) here. - "classify": [], + "classify": ["classify_document"], } # These internal bookkeeping stages are always shown in the flow regardless of diff --git a/app/views/google_drive.py b/app/views/google_drive.py index f6bde9c0..ec70f92c 100644 --- a/app/views/google_drive.py +++ b/app/views/google_drive.py @@ -45,6 +45,9 @@ async def google_drive_setup_page( except (json.JSONDecodeError, TypeError): cfg = {} folder_id = cfg.get("folder_id", "") + # Provide system-wide OAuth credentials when available so users can + # authorize without registering their own Google Cloud app. + has_system_credentials = bool(settings.google_drive_client_id and settings.google_drive_client_secret) return templates.TemplateResponse( "google_drive.html", { @@ -58,10 +61,13 @@ async def google_drive_setup_page( "use_oauth": True, "oauth_configured": bool(integration.credentials), "sa_configured": False, - "client_id": False, - "client_id_value": "", - "client_secret": False, - "client_secret_value": "", + "has_system_credentials": has_system_credentials, + "client_id": bool(settings.google_drive_client_id) if has_system_credentials else False, + "client_id_value": (settings.google_drive_client_id or "" if has_system_credentials else ""), + "client_secret": bool(settings.google_drive_client_secret) if has_system_credentials else False, + "client_secret_value": ( + settings.google_drive_client_secret or "" if has_system_credentials else "" + ), "refresh_token": False, "refresh_token_value": "", "has_credentials_json": False, @@ -90,6 +96,7 @@ async def google_drive_setup_page( "use_oauth": use_oauth, "oauth_configured": oauth_configured, "sa_configured": sa_configured, + "has_system_credentials": bool(settings.google_drive_client_id and settings.google_drive_client_secret), "client_id": bool(settings.google_drive_client_id), "client_id_value": settings.google_drive_client_id or "", "client_secret": bool(settings.google_drive_client_secret), diff --git a/app/views/onedrive.py b/app/views/onedrive.py index 4b8b3763..d9fd5e6e 100644 --- a/app/views/onedrive.py +++ b/app/views/onedrive.py @@ -44,6 +44,9 @@ async def onedrive_setup_page( cfg = {} # Support both "folder_path" (WATCH_FOLDER / ONEDRIVE destination) folder_path = cfg.get("folder_path", cfg.get("folder", "")) + # Provide system-wide app credentials when available so users can + # authorize without registering their own Azure/OneDrive app. + has_system_credentials = bool(settings.onedrive_client_id and settings.onedrive_client_secret) return templates.TemplateResponse( "onedrive.html", { @@ -54,11 +57,12 @@ async def onedrive_setup_page( "integration_name": integration.name, "integration_type": integration.integration_type, "folder_path": folder_path, - "client_id": False, - "client_id_value": "", - "client_secret": False, - "client_secret_value": "", - "tenant_id": "common", + "has_system_credentials": has_system_credentials, + "client_id": bool(settings.onedrive_client_id) if has_system_credentials else False, + "client_id_value": settings.onedrive_client_id or "" if has_system_credentials else "", + "client_secret": bool(settings.onedrive_client_secret) if has_system_credentials else False, + "client_secret_value": (settings.onedrive_client_secret or "" if has_system_credentials else ""), + "tenant_id": settings.onedrive_tenant_id or "common", "refresh_token": False, "refresh_token_value": "", }, @@ -75,6 +79,7 @@ async def onedrive_setup_page( "request": request, "user_mode": False, "is_configured": is_configured, + "has_system_credentials": bool(settings.onedrive_client_id and settings.onedrive_client_secret), "client_id": bool(settings.onedrive_client_id), "client_id_value": settings.onedrive_client_id or "", "client_secret": bool(settings.onedrive_client_secret), diff --git a/app/views/qr_login.py b/app/views/qr_login.py new file mode 100644 index 00000000..aeb33f91 --- /dev/null +++ b/app/views/qr_login.py @@ -0,0 +1,26 @@ +"""View route for the QR code mobile login page. + +Route: + GET /qr-login — renders the QR login page (requires login) +""" + +from __future__ import annotations + +import logging + +from fastapi import Request + +from app.views.base import APIRouter, require_login, templates + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/qr-login", include_in_schema=False) +@require_login +async def qr_login_page(request: Request): + """Serve the QR code login page for mobile app authentication.""" + return templates.TemplateResponse( + "qr_login.html", + {"request": request}, + ) diff --git a/app/views/system_reset.py b/app/views/system_reset.py new file mode 100644 index 00000000..6ef90fca --- /dev/null +++ b/app/views/system_reset.py @@ -0,0 +1,40 @@ +""" +System reset view — admin-only UI page. + +Renders a confirmation-heavy page that allows administrators to: +1. **Full Reset** — wipe all user data (DB + disk) for a fresh start. +2. **Reset & Re-import** — move originals to a reimport folder, wipe, + and let the watch-folder mechanism re-ingest them. + +Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag. +""" + +import logging + +from fastapi import Depends, Request +from fastapi.responses import RedirectResponse, Response +from sqlalchemy.orm import Session + +from app.config import settings +from app.views.base import APIRouter, get_db, require_login, templates +from app.views.settings import require_admin_access + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/admin/system-reset") +@require_login +@require_admin_access +async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response: + """Render the system reset administration page.""" + if not settings.enable_factory_reset: + return RedirectResponse(url="/settings", status_code=302) + + return templates.TemplateResponse( + "system_reset.html", + { + "request": request, + "factory_reset_on_startup": settings.factory_reset_on_startup, + }, + ) diff --git a/docker-compose.yaml b/docker-compose.yaml index 05e1584c..10c370f6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile - container_name: document_api + # No container_name — allows `docker compose up --scale api=N` restart: always # We'll keep the code in /app, but set working_dir to the shared data directory @@ -24,7 +24,7 @@ services: depends_on: - redis - - worker + - beat # Mount the shared working directory for data volumes: @@ -34,13 +34,14 @@ services: build: context: . dockerfile: Dockerfile - container_name: document_worker + # No container_name — allows `docker compose up --scale worker=N` restart: always # same shared working directory working_dir: /workdir - command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"] + # Workers process tasks only — no -B flag (Beat runs in the dedicated beat service) + command: ["celery", "-A", "app.celery_worker", "worker", "--loglevel=info", "-Q", "document_processor,default,celery"] env_file: - .env environment: @@ -54,6 +55,26 @@ services: volumes: - /var/docparse/workdir:/workdir + # Dedicated Celery Beat scheduler — exactly one instance must run at all times. + # Beat publishes periodic tasks to the Redis broker; workers pick them up. + # Do NOT scale this service (replicas must stay at 1). + beat: + build: + context: . + dockerfile: Dockerfile + container_name: document_beat + restart: always + working_dir: /workdir + command: ["celery", "-A", "app.celery_worker", "beat", "--loglevel=info"] + env_file: + - .env + environment: + - PYTHONPATH=/app + depends_on: + - redis + volumes: + - /var/docparse/workdir:/workdir + gotenberg: image: gotenberg/gotenberg:latest container_name: gotenberg diff --git a/docs/API.md b/docs/API.md index afd57e79..4ecb2cca 100644 --- a/docs/API.md +++ b/docs/API.md @@ -27,9 +27,11 @@ DocuElevate implements rate limiting to protect against abuse and DoS attacks. R ### Default Limits - **Default endpoints**: 100 requests per minute -- **File upload**: 600 requests per minute +- **File upload**: 600 requests per minute (global) + 20 per user per 60 s (per-user, health-aware) - **Authentication**: 10 requests per minute +**Per-user upload rate limiting**: Upload endpoints (`/api/ui-upload`, `/api/process-url`) enforce a per-user sliding-window limit that adapts to system load. Under heavy queue depth or high CPU usage, the effective limit is reduced automatically. See the [Configuration Guide](ConfigurationGuide.md#per-user-upload-rate-limiting) for details. + **Note**: Document processing endpoints (OCR, metadata extraction) use built-in queue throttling to control processing rates and prevent upstream API overloads. No additional API-level rate limit is applied to processing endpoints. ### Rate Limit Headers @@ -53,6 +55,10 @@ RATE_LIMITING_ENABLED=true RATE_LIMIT_DEFAULT=100/minute RATE_LIMIT_UPLOAD=600/minute RATE_LIMIT_AUTH=10/minute + +# Per-user upload rate limiting (health-aware) +UPLOAD_RATE_LIMIT_PER_USER=20 # Max uploads per user per window +UPLOAD_RATE_LIMIT_WINDOW=60 # Sliding window in seconds ``` See [Configuration Guide](ConfigurationGuide.md) for more details. @@ -115,7 +121,8 @@ curl -X GET "http:///api/files" \ |--------|----------|-------------| | `POST` | `/api/api-tokens/` | Create a new token | | `GET` | `/api/api-tokens/` | List all your tokens | -| `DELETE` | `/api/api-tokens/{id}` | Revoke a token | +| `DELETE` | `/api/api-tokens/{id}` | Revoke (active) or permanently delete (revoked) a token | +| `POST` | `/api/api-tokens/{id}/reactivate` | Reactivate a revoked token | ### Session Authentication @@ -235,17 +242,33 @@ The DocuElevate browser extension uses this endpoint to send files directly from **POST** `/api/ui-upload` -Upload one or more files from your computer for processing. +Upload a file from your computer for processing. **Request**: -- Multipart form data with file(s) +- Multipart form data with a single `file` field -**Response**: +**Response** (new file): ```json { - "success": true, - "file_ids": [123, 124], - "message": "Files uploaded and queued for processing" + "task_id": "abc-123", + "status": "queued", + "original_filename": "invoice.pdf", + "stored_filename": "a1b2c3d4.pdf" +} +``` + +**Response** (exact duplicate, when `ENABLE_DEDUPLICATION=True`): +```json +{ + "status": "duplicate", + "original_filename": "invoice.pdf", + "stored_filename": "e5f6a7b8.pdf", + "duplicate_of": { + "duplicate_type": "exact", + "original_file_id": 42, + "original_filename": "invoice.pdf", + "message": "This file is an exact duplicate of an already-processed document. It has not been queued for processing again." + } } ``` @@ -1356,7 +1379,7 @@ Test an integration connection without saving. Useful for "Test connection" UI b {"success": true, "message": "IMAP connection successful"} ``` -Supported connection tests: `IMAP`, `S3`, `WEBDAV`, `NEXTCLOUD`. Other types return a message that testing is not yet supported. +Supported connection tests: `DROPBOX`, `IMAP`, `S3`, `WEBDAV`, `NEXTCLOUD`. Other types return a message that testing is not yet supported. ### GET /api/integrations/quota/ @@ -1381,6 +1404,55 @@ Get the current user's integration quota usage. } ``` +## Cloud Provider Folder Browser + +Browse folders in connected cloud storage providers. These endpoints are used by the OAuth callback pages to let users select a target folder after authorization. + +### POST /api/dropbox/list-folders + +List folders in a Dropbox account. Requires a short-lived OAuth access token obtained during the authorization flow. + +**Request (form-data):** + +| Field | Type | Required | Description | +|----------------|--------|----------|--------------------------------------| +| `access_token` | string | Yes | Dropbox OAuth access token | +| `path` | string | No | Folder path to list (default: root) | + +**Response (200):** +```json +{ + "folders": [ + { "name": "Documents", "path": "/Documents", "id": "id:abc123" }, + { "name": "Photos", "path": "/Photos", "id": "id:def456" } + ], + "path": "/", + "has_more": false +} +``` + +### POST /api/onedrive/list-folders + +List folders in a OneDrive account. Requires a short-lived OAuth access token obtained during the authorization flow. + +**Request (form-data):** + +| Field | Type | Required | Description | +|----------------|--------|----------|--------------------------------------| +| `access_token` | string | Yes | Microsoft Graph access token | +| `path` | string | No | Folder path to list (default: root) | + +**Response (200):** +```json +{ + "folders": [ + { "name": "Documents", "path": "/Documents", "id": "abc123", "child_count": 5 }, + { "name": "Pictures", "path": "/Pictures", "id": "def456", "child_count": 12 } + ], + "path": "/" +} +``` + ## Webhooks Manage webhook configurations for notifying external systems when document events occur. All webhook endpoints require admin access. @@ -1571,6 +1643,47 @@ Lightweight endpoint returning the total number of queued + in-progress items. D ## Diagnostic +### GET /api/diagnostic/healthz/live + +Lightweight liveness probe for Kubernetes. Returns **200 OK** as long as the process is running. This endpoint does **not** check external dependencies and is intentionally cheap. + +**Authentication:** None (designed for kubelet probes) + +**Response (200 OK):** +```json +{ + "status": "ok" +} +``` + +### GET /api/diagnostic/healthz/ready + +Readiness probe for Kubernetes. Verifies that the application can serve traffic by checking database and Redis connectivity. + +**Authentication:** None (designed for kubelet probes) + +**Response (200 OK) – ready to serve traffic:** +```json +{ + "status": "ready", + "checks": { + "database": {"status": "ok"}, + "redis": {"status": "ok"} + } +} +``` + +**Response (503 Service Unavailable) – database unreachable:** +```json +{ + "status": "not_ready", + "checks": { + "database": {"status": "error", "detail": "..."}, + "redis": {"status": "ok"} + } +} +``` + ### GET /api/diagnostic/health System health endpoint designed for monitoring tools such as Grafana, Uptime Kuma, Prometheus blackbox exporter, or any HTTP-based health checker. @@ -2127,12 +2240,14 @@ Usage tracking records when each token was last used and from which IP address. ### POST /api/api-tokens/ -Create a new API token. +Create a new API token. Optionally specify a lifetime in days via +`expires_in_days` (1–3650). If omitted the token never expires. **Request:** ```json { - "name": "CI Pipeline" + "name": "CI Pipeline", + "expires_in_days": 90 } ``` @@ -2147,7 +2262,8 @@ Create a new API token. "last_used_at": null, "last_used_ip": null, "created_at": "2026-03-08T12:00:00Z", - "revoked_at": null + "revoked_at": null, + "expires_at": "2026-06-06T12:00:00Z" } ``` @@ -2169,15 +2285,20 @@ List all tokens for the authenticated user. The full token value is never includ "last_used_at": "2026-03-08T15:30:00Z", "last_used_ip": "203.0.113.42", "created_at": "2026-03-08T12:00:00Z", - "revoked_at": null + "revoked_at": null, + "expires_at": "2026-06-06T12:00:00Z" } ] ``` ### DELETE /api/api-tokens/{token_id} -Revoke a token. The token is soft-deleted (kept for audit purposes) and can no -longer be used for authentication. +Revoke or permanently delete a token: + +* **Active token** – soft-revoked (kept for audit purposes, marked inactive). + Response: `{"detail": "Token revoked"}` +* **Already-revoked token** – permanently deleted from the database. + Response: `{"detail": "Token deleted"}` **Response (200):** ```json @@ -2186,6 +2307,13 @@ longer be used for authentication. } ``` +### POST /api/api-tokens/{token_id}/reactivate + +Reactivate a previously revoked token. Clears `revoked_at` and sets +`is_active` back to `true`. + +**Response (200):** The updated `TokenResponse` object. + ### Using API Tokens Include the token in the `Authorization` header of any API request: @@ -2214,6 +2342,164 @@ print(response.json()) ``` +## Classification Rules + +The classification rules API lets you manage custom document classification rules. Rules are evaluated during the `classify` pipeline step to assign a category to each document based on filename patterns, content keywords, and metadata fields. + +### Built-in Categories + +```bash +GET /api/classification-rules/categories +``` + +Returns the pre-built classification categories. + +**Response (200):** +```json +{ + "invoice": "Invoice", + "contract": "Contract", + "receipt": "Receipt", + "letter": "Letter", + "report": "Report", + "bank_statement": "Bank Statement", + "tax_document": "Tax Document", + "insurance": "Insurance Document", + "payslip": "Payslip", + "unknown": "Unknown" +} +``` + +### Rule Types + +```bash +GET /api/classification-rules/rule-types +``` + +Returns the supported rule types with descriptions. + +**Response (200):** +```json +[ + { + "type": "filename_pattern", + "label": "Filename Pattern", + "description": "Regex pattern matched against the original filename." + }, + { + "type": "content_keyword", + "label": "Content Keyword", + "description": "Pipe-separated keywords matched against the OCR text." + }, + { + "type": "metadata_match", + "label": "Metadata Match", + "description": "field=value pattern matched against existing AI metadata." + } +] +``` + +### List Rules + +```bash +GET /api/classification-rules/ +``` + +List all classification rules visible to the current user (system rules + own rules). + +**Response (200):** +```json +[ + { + "id": 1, + "owner_id": "user@example.com", + "name": "German Invoice Filename", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "(?i)rechnung", + "priority": 10, + "case_sensitive": false, + "enabled": true + } +] +``` + +### Create Rule + +```bash +POST /api/classification-rules/ +``` + +Create a new custom classification rule. + +**Request:** +```json +{ + "name": "German Invoice Filename", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "(?i)rechnung", + "priority": 10, + "case_sensitive": false, + "enabled": true +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique rule name (per user) | +| `category` | string | Yes | Target category (e.g. `invoice`, `contract`, or custom) | +| `rule_type` | string | Yes | One of: `filename_pattern`, `content_keyword`, `metadata_match` | +| `pattern` | string | Yes | Regex (filename), pipe-separated keywords (content), or `field=value` (metadata) | +| `priority` | integer | No | Higher priority rules are evaluated first (default: 0) | +| `case_sensitive` | boolean | No | Case-sensitive matching (default: false) | +| `enabled` | boolean | No | Whether the rule is active (default: true) | + +**Response (201):** +```json +{ + "id": 1, + "owner_id": "user@example.com", + "name": "German Invoice Filename", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "(?i)rechnung", + "priority": 10, + "case_sensitive": false, + "enabled": true +} +``` + +### Get Rule + +```bash +GET /api/classification-rules/{rule_id} +``` + +### Update Rule + +```bash +PUT /api/classification-rules/{rule_id} +``` + +**Request (partial update):** +```json +{ + "priority": 20, + "enabled": false +} +``` + +### Delete Rule + +```bash +DELETE /api/classification-rules/{rule_id} +``` + +**Response:** `204 No Content` + + + ## Automation (Zapier / Make.com) Manage automation hook subscriptions for integrating DocuElevate with external platforms like Zapier and Make.com. All endpoints require API token authentication (`Authorization: Bearer `). @@ -2366,6 +2652,8 @@ The `id` field is unique per event and is used by Zapier for deduplication. If a Automation hook deliveries follow the same retry policy as regular webhooks: up to 3 retries with exponential backoff (60 s, 300 s, 900 s) and ±20% jitter. +## Further Assistance + ## Further Assistance For additional help with the API, please contact our support team or refer to the [Development Guide](../CONTRIBUTING.md). @@ -2420,9 +2708,14 @@ List all registered push-notification devices for the current user. ### DELETE /api/mobile/devices/{device_id} -Deactivate a push-notification device. The device will no longer receive push notifications. +Deactivate or permanently delete a push-notification device: -**Response (204 No Content)** +* **Active device** – soft-deactivated (record kept, will no longer receive push notifications). + Response: `{"detail": "Device deactivated"}` +* **Already-inactive device** – permanently deleted from the database. + Response: `{"detail": "Device deleted"}` + +**Response (200)** ### GET /api/mobile/whoami @@ -2557,3 +2850,64 @@ query GetDocument($id: Int!) { } ``` Variables: `{ "id": 42 }` + +## System Reset + +Admin-only endpoints for resetting the system to a clean state. Requires `ENABLE_FACTORY_RESET=True`. + +### GET /api/admin/system-reset/status + +Check whether the system reset feature is enabled. + +**Response (200):** +```json +{ + "enabled": true, + "factory_reset_on_startup": false +} +``` + +### POST /api/admin/system-reset/full + +Wipe all user data (database + work-files). + +**Request:** +```json +{ + "confirmation": "DELETE" +} +``` + +**Response (200):** +```json +{ + "status": "ok", + "result": { + "database": { "files": 42, "processing_logs": 100 }, + "filesystem": { "deleted_dirs": 5, "deleted_files": 12 } + } +} +``` + +### POST /api/admin/system-reset/reimport + +Move original files to a reimport folder, wipe everything, and configure the reimport folder as a watch folder for re-ingestion. + +**Request:** +```json +{ + "confirmation": "REIMPORT" +} +``` + +**Response (200):** +```json +{ + "status": "ok", + "result": { + "database": { "files": 42 }, + "filesystem": { "deleted_dirs": 5, "deleted_files": 12 }, + "reimport": { "files_moved": 42, "reimport_folder": "/workdir/reimport" } + } +} +``` diff --git a/docs/AppleAppStoreCompliance.md b/docs/AppleAppStoreCompliance.md new file mode 100644 index 00000000..2df05a2a --- /dev/null +++ b/docs/AppleAppStoreCompliance.md @@ -0,0 +1,285 @@ +# Apple App Store Compliance Audit Report + +This document details the findings from a comprehensive audit of the DocuElevate mobile app against Apple's App Store Review Guidelines, Human Interface Guidelines (HIG), and privacy requirements. It covers all areas of compliance, risks for rejection, and recommendations. + +> **Last Audited:** March 2026 +> **App Version:** 1.0.0 +> **Expo SDK:** 54.0.0 +> **Bundle ID:** `org.docuelevate.mobile` + +--- + +## Executive Summary + +The DocuElevate mobile app is broadly compliant with Apple's App Store requirements. The following issues were identified and resolved as part of this audit: + +| Issue | Severity | Status | +|-------|----------|--------| +| Unused `fetch` background mode declared | High | ✅ Fixed | +| Missing privacy manifest for required reason APIs | High | ✅ Fixed | +| No account deletion option (Guideline 5.1.1(v)) | Critical | ✅ Fixed | +| No Privacy Policy / Terms of Service links in-app | High | ✅ Fixed | +| Emoji used as UI icons instead of platform-native icons | Medium | ✅ Fixed | +| Missing app version display | Low | ✅ Fixed | +| Unused `Switch` import in ProfileScreen | Low | ✅ Fixed | + +--- + +## 1. Human Interface Guidelines (HIG) + +### 1.1 Navigation & Tab Bar ✅ + +- The app uses a standard bottom tab bar with three tabs: Upload, Files, and Profile. +- Tab icons use **Ionicons** (an icon set that closely maps to Apple's SF Symbols). +- Active/inactive tab colors follow iOS conventions (`#1e40af` active, `#9ca3af` inactive). +- Header styling uses a solid color background with white text, consistent with iOS navigation bar patterns. + +### 1.2 Icons & Visual Assets ✅ + +- **App icon:** Custom `icon.png` provided at root level; Expo handles generating all required sizes. +- **Splash screen:** Uses branded splash with `contain` resize mode and matching background color. +- **Adaptive icon (Android):** Properly configured with foreground image and background color. +- **Action buttons:** Previously used emoji characters (📷, 🖼️, 📄) which render inconsistently across iOS versions. **Fixed:** Now using Ionicons (`camera-outline`, `images-outline`, `document-outline`). +- **Status indicators:** Previously used emoji (✅, ❌, ⏳, ⚙️). **Fixed:** Now using Ionicons with semantic colors. + +### 1.3 Typography & Colors ✅ + +- Uses system fonts (default React Native text rendering uses San Francisco on iOS). +- Color palette (`#1e40af` primary blue, semantic reds/greens/grays) provides sufficient contrast ratios. +- Text sizes follow iOS recommended minimums (body text ≥ 13pt). + +### 1.4 Touch Targets ✅ + +- All interactive elements have `minHeight: 44` or `minHeight: 48` (meets Apple's 44×44pt minimum). +- Back links, cancel buttons, and retry buttons all meet minimum touch target requirements. + +### 1.5 Safe Areas ✅ + +- The app uses `react-native-safe-area-context` (`SafeAreaProvider`) to respect device notches, Dynamic Island, and home indicator. + +### 1.6 Dark Mode ✅ + +- `userInterfaceStyle: "automatic"` is set in `app.json`, enabling automatic dark mode support. + +--- + +## 2. Privacy & Data Usage + +### 2.1 Permission Descriptions ✅ + +All iOS permission strings (Info.plist keys) are present and provide clear, specific descriptions of why each permission is needed: + +| Permission | Key | Description | +|-----------|-----|-------------| +| Camera | `NSCameraUsageDescription` | "DocuElevate uses the camera to scan QR codes for login and to capture documents for upload." | +| Photo Library (Read) | `NSPhotoLibraryUsageDescription` | "DocuElevate accesses your photo library to select documents for upload." | +| Photo Library (Write) | `NSPhotoLibraryAddUsageDescription` | "DocuElevate saves scanned documents to your photo library." | + +**Assessment:** All descriptions clearly explain the purpose, which is a requirement for App Review approval. + +### 2.2 Push Notifications ✅ + +- Push notification permission is requested at runtime (not at launch) when the user enters the authenticated area. +- The app works gracefully without push notifications if permission is denied. +- Device tokens are registered via a dedicated backend endpoint. + +### 2.3 Background Modes ✅ (Fixed) + +- **Previous state:** `UIBackgroundModes` included `["fetch", "remote-notification"]`. +- **Issue:** The app does not implement background fetch (`application:performFetchWithCompletionHandler:`). Apple may reject apps that declare background modes they don't actively use (Guideline 2.5.4). +- **Fix:** Removed `fetch` from `UIBackgroundModes`. Only `remote-notification` remains, which is required for push notification delivery. + +### 2.4 Privacy Manifest ✅ (Fixed) + +Starting in Spring 2024, Apple requires a privacy manifest (`PrivacyInfo.xcprivacy`) for apps using specific APIs. The following required reason APIs are used by the app's dependencies: + +| API Category | Reason Code | Justification | +|-------------|-------------|---------------| +| `NSPrivacyAccessedAPICategoryUserDefaults` | `CA92.1` | Used by `@react-native-async-storage/async-storage` for user preferences | +| `NSPrivacyAccessedAPICategoryFileTimestamp` | `C617.1` | Used by `expo-file-system` to read file metadata | +| `NSPrivacyAccessedAPICategoryDiskSpace` | `E174.1` | Used by Expo runtime for storage space checks | +| `NSPrivacyAccessedAPICategorySystemBootTime` | `35F9.1` | Used by React Native's timing APIs | + +The privacy manifest is configured via `expo-build-properties` plugin in `app.json`, which ensures it is included in the generated Xcode project during EAS Build. + +### 2.5 Tracking & Analytics ✅ + +- `NSPrivacyTracking: false` — the app does **not** track users. +- `NSPrivacyCollectedDataTypes: []` — no data types are collected for tracking. +- No analytics SDKs (Firebase Analytics, Amplitude, Mixpanel, etc.) are included. +- No App Tracking Transparency (ATT) prompt is needed. + +### 2.6 Encryption Declaration ✅ + +- `ITSAppUsesNonExemptEncryption: false` — the app uses only standard HTTPS/TLS for network communication, which is exempt from export compliance requirements. + +### 2.7 Data Storage Security ✅ + +- API tokens are stored in the device keychain via `expo-secure-store` (uses iOS Keychain Services). +- No sensitive data is stored in `AsyncStorage` or `UserDefaults`. +- Server URL is stored in secure storage, not in plain text files. + +--- + +## 3. App Store Review Guidelines Compliance + +### 3.1 Functionality (Guideline 2.x) ✅ + +- **2.1 App Completeness:** The app provides a complete, functional experience. All advertised features (camera capture, file upload, document list, push notifications) work as described. +- **2.3 Accurate Metadata:** App name ("DocuElevate"), description, and screenshots should accurately reflect the app's functionality. +- **2.5.4 Background Modes:** Only `remote-notification` is declared, which is actively used. ✅ Fixed. + +### 3.2 Content & Intellectual Property (Guideline 3.x) ✅ + +- No third-party trademarked content is used. +- The app does not display user-generated content publicly (documents are private to each user). +- No copyrighted content is bundled with the app. + +### 3.3 Business (Guideline 3.1.x) ✅ + +- The app does not include in-app purchases, subscriptions, or payment processing. +- No physical goods or services are sold through the app. +- Authentication is handled via self-hosted or enterprise SSO — no Apple Sign-In requirement applies (Apple Sign-In is required only when third-party social login options like Google/Facebook are offered as the primary login method; enterprise SSO to a self-hosted server is exempt). + +### 3.4 Safety & Privacy (Guideline 5.x) ✅ + +- **5.1.1 Data Collection and Storage:** The app collects only what is necessary for its functionality (server URL, auth token, push token). +- **5.1.1(v) Account Deletion:** ✅ Fixed. Users can now initiate account deletion from the Profile screen, which opens the server's account deletion page in the browser. +- **5.1.2 Data Use and Sharing:** No data is shared with third parties or used for advertising. + +### 3.5 Privacy Policy ✅ (Fixed) + +- **Requirement:** Apple requires all apps to have an accessible privacy policy. +- **Fix:** Privacy Policy and Terms of Service links are now accessible from the Profile screen, opening the server's hosted policy pages. +- **App Store Connect:** The privacy policy URL must also be provided in App Store Connect during submission. + +### 3.6 Login & Authentication ✅ + +- Two login methods are available: SSO (browser-based OAuth) and QR code scanning. +- Both methods provide clear error messages on failure. +- The app correctly handles authentication cancellation. +- Session restoration on app launch is implemented. +- **Demo Account:** For App Review, a demo account may need to be provided in App Store Connect's review notes. Ensure the review team can access a test server. + +--- + +## 4. Technical Compliance + +### 4.1 API Usage ✅ + +- No private APIs are used (all functionality comes from Expo SDK and React Native public APIs). +- No deprecated APIs are used that would trigger rejection. + +### 4.2 Network Security ✅ + +- The app validates server URLs require `http://` or `https://` scheme. +- All API calls use Bearer token authentication over HTTPS. +- App Transport Security (ATS) is not explicitly disabled — default iOS ATS rules apply. + +### 4.3 Deep Linking ✅ + +- Custom URL scheme `docuelevate://` is properly registered. +- Deep link handling for QR login (`docuelevate://qr-login`) and file sharing is implemented correctly. +- `WebBrowser.openAuthSessionAsync` is used for OAuth, which properly handles the authentication session lifecycle. + +### 4.4 Document Handling ✅ + +- `CFBundleDocumentTypes` properly declares supported file types. +- `LSSupportsOpeningDocumentsInPlace: false` ensures iOS copies shared files to the app's accessible Inbox directory, avoiding security-scoped URL issues. +- The `+not-found.tsx` handler correctly intercepts iOS "Open In…" file paths. +- `UploadScreen` uses `expo-file-system` to copy external files to cache before uploading for reliable file access. + +### 4.5 Crash Resistance ✅ + +- All network calls are wrapped in try/catch blocks. +- Error states are displayed to users with actionable recovery options (retry buttons). +- Permission denials are handled gracefully with explanatory messages. + +--- + +## 5. Onboarding & First-Run Experience + +### 5.1 Welcome Screen ✅ + +- Clean, informative welcome screen with app branding and feature highlights. +- Clear "Get Started" call-to-action leading to the login screen. +- No misleading claims or functionality promises. + +### 5.2 Login Flow ✅ + +- Server URL entry with input validation. +- Two clear authentication options (SSO and QR code). +- Error handling with user-friendly alert dialogs. +- Back navigation available from all auth screens. + +### 5.3 First-Run Permissions ✅ + +- Camera permission is requested at the point of use (when tapping Camera button), not at launch. +- Photo library permission is requested at the point of use. +- Push notification permission is requested after authentication, not before. +- All permission requests include clear usage descriptions. + +--- + +## 6. Remaining Recommendations + +### 6.1 App Store Connect Preparation + +Before submission, ensure the following are configured in App Store Connect: + +- [ ] **Privacy Policy URL** — must point to the server's `/privacy` endpoint +- [ ] **App Store description** — accurate description of features +- [ ] **Screenshots** — for iPhone and iPad (since `supportsTablet: true`) +- [ ] **App category** — "Business" or "Productivity" +- [ ] **Age rating** — complete the questionnaire (likely 4+) +- [ ] **Review notes** — provide demo server URL and test credentials for the Apple review team +- [ ] **Privacy Nutrition Labels** — declare data types collected (device ID for push notifications, authentication tokens) + +### 6.2 Accessibility Enhancements (Recommended) + +While the app includes `accessibilityRole` and `accessibilityLabel` on interactive elements, consider: + +- Adding `accessibilityHint` to buttons where the action isn't immediately obvious. +- Testing with VoiceOver to ensure all screens are fully navigable. +- Ensuring all status changes are announced to screen readers. + +### 6.3 iPad Support + +The app declares `supportsTablet: true`. Ensure: + +- UI scales appropriately on iPad screen sizes. +- Split View and Slide Over multitasking work correctly. +- Touch targets remain accessible on larger screens. + +### 6.4 Localization (Future Enhancement) + +- The app currently uses English-only strings. +- For broader App Store reach, consider localizing the app name, description, and in-app strings. + +--- + +## 7. Compliance Checklist Summary + +| Area | Status | Notes | +|------|--------|-------| +| Human Interface Guidelines | ✅ Pass | Ionicons used for platform-consistent iconography | +| App Icons & Visual Assets | ✅ Pass | All required assets provided | +| Device Data Usage | ✅ Pass | Camera, photos, notifications properly handled | +| Privacy Disclosures | ✅ Pass | Info.plist keys and privacy manifest configured | +| Background Modes | ✅ Pass | Only `remote-notification` declared | +| Restricted APIs | ✅ Pass | No private or deprecated APIs used | +| Content Standards | ✅ Pass | No misleading or inappropriate content | +| Functionality | ✅ Pass | Complete, functional app experience | +| Business Model | ✅ Pass | No IAP conflicts | +| Safety & Privacy | ✅ Pass | Account deletion available, privacy policy linked | +| Onboarding | ✅ Pass | Clear, permission-respectful first-run experience | +| Privacy Manifest | ✅ Pass | Required reason APIs declared | + +--- + +## References + +- [Apple App Store Review Guidelines](https://developer.apple.com/app-store/review/guidelines/) +- [Apple Human Interface Guidelines](https://developer.apple.com/design/human-interface-guidelines/) +- [Apple Privacy Manifest Requirements](https://developer.apple.com/documentation/bundleresources/privacy_manifest_files) +- [App Store Connect Help](https://developer.apple.com/help/app-store-connect/) diff --git a/docs/AuthenticationSetup.md b/docs/AuthenticationSetup.md index 320e6191..c77f48b3 100644 --- a/docs/AuthenticationSetup.md +++ b/docs/AuthenticationSetup.md @@ -154,6 +154,62 @@ DocuElevate can work with any OpenID Connect-compliant provider, not just Authen OAUTH_PROVIDER_NAME=Auth0 ``` +## Server-Side Session Management + +DocuElevate supports server-side session tracking. Every login creates a `UserSession` record that can be listed and revoked individually or all at once ("log off everywhere"). + +### Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `SESSION_LIFETIME_DAYS` | Number of days before a session expires | `30` | +| `SESSION_LIFETIME_CUSTOM_DAYS` | Override for `SESSION_LIFETIME_DAYS` when set | — | + +### Managing Sessions + +Users can manage their active sessions from the **Profile → Security** section: + +- **View active sessions** — see browser, device, IP address, and last activity for each session. +- **Revoke a single session** — immediately invalidate one session. +- **Log off everywhere** — revoke all sessions (optionally keeping the current one) and all API tokens at once. + +Expired sessions are automatically cleaned up by a periodic background task. + +### API Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/api/sessions` | List the current user's active sessions | +| `DELETE` | `/api/sessions/{id}` | Revoke a single session | +| `POST` | `/api/sessions/revoke-all` | Revoke all sessions for the current user | + +## QR Code Login + +QR code login allows users to authenticate a mobile device by scanning a QR code displayed in the web UI, without manually entering credentials on the phone. + +### How It Works + +1. The authenticated web user opens the **QR Login** page and a challenge QR code is displayed. +2. The user opens the DocuElevate mobile app and taps **Scan QR Code to Login**, which opens the device camera. +3. The mobile app scans the QR code. The QR code contains both the challenge token and the server URL (`docuelevate://qr-login?token=...&server=...`), so there is no need to enter the server URL manually. +4. An API token is issued for the mobile device and the web UI is notified via polling. + +> **Note:** The countdown timer on the web page uses server-relative time (TTL in seconds) rather than absolute timestamps, so it works correctly even when the client's clock is not in sync with the server. + +### Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `QR_LOGIN_CHALLENGE_TTL_SECONDS` | How long a QR challenge is valid (seconds) | `120` | + +### API Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| `POST` | `/api/qr-auth/challenge` | Create a new QR login challenge (returns `ttl_seconds` for client countdown) | +| `GET` | `/api/qr-auth/challenge/{id}/status` | Poll the status of a challenge | +| `POST` | `/api/qr-auth/claim` | Claim a challenge from a mobile device | + ## Security Considerations 1. **Always use HTTPS** in production to protect authentication tokens and passwords diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 4fac2f3c..f977b6be 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -11,12 +11,19 @@ Configuration is primarily done through environment variables specified in a `.e | **Variable** | **Description** | **Example** | |------------------------|----------------------------------------------------------|--------------------------------| | `DATABASE_URL` | Path/URL to the SQLite database (or other SQL backend). Use the [Database Wizard](/database-wizard) for guided setup. See [Database Configuration](DatabaseConfiguration.md). | `sqlite:///./app/database.db` | +| `DB_POOL_SIZE` | Number of persistent connections in the pool per worker (PostgreSQL/MySQL only; ignored for SQLite). | `10` | +| `DB_MAX_OVERFLOW` | Additional connections beyond `DB_POOL_SIZE` under burst load (PostgreSQL/MySQL only). | `20` | +| `DB_POOL_TIMEOUT` | Seconds to wait for a pool connection before raising `TimeoutError` (PostgreSQL/MySQL only). | `30` | +| `DB_POOL_RECYCLE` | Recycle connections after this many seconds to avoid stale connections (PostgreSQL/MySQL only). | `1800` | | `REDIS_URL` | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0` | | `WORKDIR` | Working directory for the application. | `/workdir` | | `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` | | `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` | +| `PUBLIC_BASE_URL` | Full public base URL including scheme (e.g., `https://docuelevate.example.com`). When set, overrides auto-detected URLs used for OAuth redirect URIs. **Required when your reverse proxy does not forward `X-Forwarded-Proto` headers.** | *(not set)* | | `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` | | `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` | +| `FACTORY_RESET_ON_STARTUP` | Wipe all user data on every startup (demo/testing). | `false` | +| `ENABLE_FACTORY_RESET` | Show the System Reset page in the admin UI. | `false` | ### Batch Processing Settings @@ -79,6 +86,28 @@ Control how the web UI queues and paces file uploads to avoid overwhelming the b **Example**: With `UPLOAD_CONCURRENCY=3` and `UPLOAD_QUEUE_DELAY_MS=500`, a directory of 5,000 files is uploaded ≈ 3 at a time with 500 ms pacing – the backend processes files at its own rate while the queue drains in the background without triggering API rate limits. +### Per-User Upload Rate Limiting + +Server-side rate limiting that prevents any single user from overwhelming the system with bulk uploads. The limiter uses a Redis-backed sliding window and dynamically adjusts limits based on system health. + +| **Variable** | **Description** | **Default** | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------|-------------| +| `UPLOAD_RATE_LIMIT_PER_USER` | Maximum uploads allowed per user within the sliding window. Effective limit may be reduced under load. | `20` | +| `UPLOAD_RATE_LIMIT_WINDOW` | Sliding window size in seconds. | `60` | + +**Health-aware dynamic limiting**: The effective per-user limit is automatically reduced when the system is under heavy load: + +| **System condition** | **Effective limit** | **Trigger** | +|--------------------------------|---------------------|--------------------------------| +| Normal | 100 % of base | Queue < 50, CPU load normal | +| Moderate load | 50 % of base | Queue 50–100 or CPU > 1.5× | +| High load | 25 % of base | Queue 100–200 or CPU > 2× | +| Critical load | 10 % of base | Queue > 200 or CPU > 3× | + +When a user exceeds the limit, the server returns **HTTP 429 Too Many Requests** with a `Retry-After` header. The browser client (see *Client-Side Upload Throttling* above) automatically pauses and retries. + +> **Note**: The limiter fails open — if Redis is unavailable, all uploads are allowed through so that a monitoring outage never blocks document processing. + ### File Upload Size Limits **Security Feature**: Control file upload sizes to prevent resource exhaustion attacks. See [SECURITY_AUDIT.md](../SECURITY_AUDIT.md#5-file-upload-size-limits) for security details. @@ -364,6 +393,9 @@ Credentials are encrypted at rest using Fernet encryption. |-------------------------|---------------------------------------------------------------| | `AUTH_ENABLED` | Enable or disable authentication (`true`/`false`). | | `SESSION_SECRET` | Secret key used to encrypt sessions and cookies (at least 32 chars). | +| `SESSION_LIFETIME_DAYS` | Number of days before a server-side session expires. Default: `30`. | +| `SESSION_LIFETIME_CUSTOM_DAYS` | Override for `SESSION_LIFETIME_DAYS` when set. | +| `QR_LOGIN_CHALLENGE_TTL_SECONDS` | How long a QR login challenge is valid (seconds). Default: `120`. | | `ADMIN_USERNAME` | Username for basic authentication (when not using OIDC). | | `ADMIN_PASSWORD` | Password for basic authentication (when not using OIDC). | | `ADMIN_GROUP_NAME` | Group name in OIDC claims that grants admin access. Default: `admin`. | @@ -1284,6 +1316,20 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md). +### SharePoint Online + +| **Variable** | **Description** | +|---------------------------------|-------------------------------------------------------| +| `SHAREPOINT_CLIENT_ID` | Azure AD application client ID | +| `SHAREPOINT_CLIENT_SECRET` | Azure AD application client secret | +| `SHAREPOINT_TENANT_ID` | Azure AD tenant ID (use "common" for multi-tenant apps) | +| `SHAREPOINT_REFRESH_TOKEN` | OAuth 2.0 refresh token | +| `SHAREPOINT_SITE_URL` | SharePoint site URL (e.g. `https://tenant.sharepoint.com/sites/sitename`) | +| `SHAREPOINT_DOCUMENT_LIBRARY` | Document library name (default: `Documents`) | +| `SHAREPOINT_FOLDER_PATH` | Subfolder path inside the document library | + +SharePoint uses the same Microsoft Graph API as OneDrive. See the [OneDrive Setup Guide](OneDriveSetup.md) for Azure AD app registration instructions — the same app registration can be reused for SharePoint with the `Sites.ReadWrite.All` permission. + ### Amazon S3 | **Variable** | **Description** | @@ -1556,14 +1602,30 @@ DocuElevate detects and flags documents that share the same content, even if the ### Exact Duplicate Detection (SHA-256) -When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the new document is stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=`) and no further processing is performed. +When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the upload is rejected immediately — no processing task is created, and the temporary file is removed from disk. The `/api/ui-upload` response returns `"status": "duplicate"` together with a `duplicate_of` object that identifies the original file. + +If the same file somehow reaches the Celery worker (e.g. via a watch-folder ingest) it is still caught there and stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=`) with no further processing. | Variable | Description | Default | |---|---|---| | `ENABLE_DEDUPLICATION` | Hash-based exact duplicate detection on ingest. | `True` | | `SHOW_DEDUPLICATION_STEP` | Show the "Check for Duplicates" step in the processing timeline UI. | `True` | -An immediate duplicate warning is also included in the `/api/ui-upload` JSON response so the frontend can alert the user before the pipeline completes. +When the upload is an exact duplicate the `/api/ui-upload` response looks like: + +```json +{ + "status": "duplicate", + "original_filename": "invoice.pdf", + "stored_filename": "abc-123.pdf", + "duplicate_of": { + "duplicate_type": "exact", + "original_file_id": 42, + "original_filename": "invoice.pdf", + "message": "This file is an exact duplicate of an already-processed document. It has not been queued for processing again." + } +} +``` ### Near-Duplicate Detection (Content Similarity) @@ -1643,6 +1705,7 @@ For example: | S3 | `docs/uploads/` | `docs/uploads/pdfa/` | | Nextcloud | `/Files` | `/Files/pdfa` | | OneDrive | `Documents/Uploads` | `Documents/Uploads/pdfa` | +| SharePoint | `Uploads` | `Uploads/pdfa` | | Google Drive | *(folder ID)* | `GOOGLE_DRIVE_PDFA_FOLDER_ID` | Set `PDFA_UPLOAD_FOLDER` to an empty string to upload PDF/A files into the @@ -1849,6 +1912,15 @@ ONEDRIVE_TENANT_ID=common ONEDRIVE_REFRESH_TOKEN=your_refresh_token ONEDRIVE_FOLDER_PATH=Documents/Uploads +# SharePoint Online +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=your-tenant-id +SHAREPOINT_REFRESH_TOKEN=your_refresh_token +SHAREPOINT_SITE_URL=https://tenant.sharepoint.com/sites/sitename +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads + # Amazon S3 AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -1876,6 +1948,52 @@ BACKUP_RETAIN_WEEKLY=13 You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables. +## System Reset / Factory Reset + +DocuElevate provides two mechanisms for resetting the system to a clean state. Both are **disabled by default** and must be explicitly enabled. + +### Automatic Reset on Startup + +Set `FACTORY_RESET_ON_STARTUP=true` to wipe all user data (database rows and work-files) every time the application starts. This is useful for demo, testing, or ephemeral environments where you always want a fresh instance. + +```dotenv +FACTORY_RESET_ON_STARTUP=true +``` + +> **Warning:** This destroys all documents, processing history, audit logs, and backups on every restart. Application settings and configuration are preserved. + +### Admin UI Reset Page + +Set `ENABLE_FACTORY_RESET=true` to display the **System Reset** page in the admin navigation menu. From this page, administrators can: + +| Action | Confirmation | Description | +|--------|-------------|-------------| +| **Full Reset** | Type `DELETE` | Wipes all database rows and work-files. The system returns to its initial state. | +| **Reset & Re-import** | Type `REIMPORT` | Copies original files to a `reimport/` folder inside the workdir, wipes everything, then configures the reimport folder as a watch folder so files are automatically re-ingested with the same processing pipeline, rate limits, and backoff strategy as regular uploads. | + +```dotenv +ENABLE_FACTORY_RESET=true +``` + +### API Endpoints + +When `ENABLE_FACTORY_RESET=true`, two admin-only API endpoints are available: + +- `POST /api/admin/system-reset/full` — body: `{"confirmation": "DELETE"}` +- `POST /api/admin/system-reset/reimport` — body: `{"confirmation": "REIMPORT"}` +- `GET /api/admin/system-reset/status` — returns current feature-flag state + +### What Gets Deleted + +| Deleted | Preserved | +|---------|-----------| +| All document records (`files` table) | Application settings (`application_settings` table) | +| Processing logs and steps | User accounts and profiles | +| Audit logs | Subscription plans | +| Backup records | Pipelines and scheduled jobs | +| Original, processed, and temporary files | The workdir directory itself | +| Watch-folder caches and ingestion state | OAuth and integration configuration | + ## Configuration File Location The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`. diff --git a/docs/CredentialRotationGuide.md b/docs/CredentialRotationGuide.md index bb911752..3dbe3c20 100644 --- a/docs/CredentialRotationGuide.md +++ b/docs/CredentialRotationGuide.md @@ -11,7 +11,7 @@ Credentials fall into two categories: | Category | Examples | |---|---| | **API keys** | OpenAI API key, Azure AI key, Paperless-ngx API token, AWS access keys | -| **OAuth tokens / secrets** | Dropbox, Google Drive, OneDrive, Authentik client secrets and refresh tokens | +| **OAuth tokens / secrets** | Dropbox, Google Drive, OneDrive, SharePoint, Authentik client secrets and refresh tokens | | **Passwords** | Admin password, Nextcloud, Email (SMTP), IMAP, FTP, SFTP, WebDAV | | **Private keys** | SFTP private key and passphrase | @@ -119,6 +119,15 @@ For service-account credentials (`google_drive_credentials_json`): 4. Re-authorize via the OAuth flow to get a fresh `onedrive_refresh_token`. 5. Delete the old client secret in Azure. +### SharePoint (Microsoft OAuth) + +1. SharePoint uses the same Azure AD app registration as OneDrive. +2. In **Azure App Registrations**, navigate to **Certificates & secrets** for your app. +3. Add a new client secret. +4. Update `sharepoint_client_secret` in DocuElevate. +5. Re-authorize via the OAuth flow to get a fresh `sharepoint_refresh_token`. +6. Delete the old client secret in Azure. + ### Authentik (OIDC) 1. In your Authentik admin panel, navigate to the DocuElevate application and regenerate the client secret. diff --git a/docs/DatabaseConfiguration.md b/docs/DatabaseConfiguration.md index 9eb1fece..06fd2916 100644 --- a/docs/DatabaseConfiguration.md +++ b/docs/DatabaseConfiguration.md @@ -287,6 +287,27 @@ alembic revision --autogenerate -m "describe your change" Review the generated file in `migrations/versions/` before applying it. +> **Tip:** For detailed guidance on naming conventions, idempotent patterns, parallel-branch workflows, and resolving merge conflicts, see the [Migration Workflow Guide](MigrationWorkflow.md). + +### Validating the Migration Chain + +A CI check and pre-commit hook validate that the migration chain has no broken +references, duplicate revisions, or diverged heads. Run the check locally: + +```bash +python scripts/check_alembic_migrations.py +python scripts/check_alembic_migrations.py --verbose # extra detail +``` + +If you see **"Multiple migration heads detected"**, two branches added +migrations from the same parent. Create a merge migration: + +```bash +alembic merge heads -m "merge_parallel_branches" +``` + +For a complete walk-through, see the [Migration Workflow Guide](MigrationWorkflow.md). + ### Automating Migrations in Docker Compose Add a short-lived `migrate` service that runs before the API and Worker: @@ -316,18 +337,26 @@ The Helm chart includes a pre-install and pre-upgrade Job hook that runs `alembi ## Connection Pooling -SQLAlchemy manages a connection pool automatically. The defaults are suitable for most deployments. For high-concurrency or Kubernetes deployments you may want to tune: +SQLAlchemy manages a connection pool automatically. DocuElevate selects the pool +strategy based on the database backend: + +- **SQLite** — uses `NullPool` (a fresh connection per request, closed immediately). + This avoids the `QueuePool limit reached` `TimeoutError` that can occur under + concurrent load because SQLite does not benefit from persistent connection pooling. +- **PostgreSQL / MySQL** — uses a bounded `QueuePool` whose size is configurable + via environment variables. ```bash -# Optional — these are set via environment variables if you extend app/database.py -# Typical production values: -DB_POOL_SIZE=10 # Number of persistent connections per worker -DB_MAX_OVERFLOW=20 # Additional connections allowed beyond pool_size -DB_POOL_TIMEOUT=30 # Seconds to wait for a connection from the pool -DB_POOL_RECYCLE=1800 # Recycle connections after 30 minutes (avoids stale connections) +# Tune these for PostgreSQL / MySQL (ignored when using SQLite): +DB_POOL_SIZE=10 # Number of persistent connections per worker (default: 10) +DB_MAX_OVERFLOW=20 # Additional connections allowed beyond pool_size (default: 20) +DB_POOL_TIMEOUT=30 # Seconds to wait for a connection from the pool (default: 30) +DB_POOL_RECYCLE=1800 # Recycle connections after 30 minutes (default: 1800) ``` -> **Note:** These environment variables are not exposed in the default `app/config.py`. If you need to tune them, extend the database engine creation in `app/database.py`. +All backends also enable `pool_pre_ping`, which sends a lightweight health-check +before each connection is handed out. This detects stale or dropped connections +and transparently reconnects. For **PgBouncer** (external connection pooling), point `DATABASE_URL` at your PgBouncer instance and use transaction-mode pooling: @@ -491,4 +520,13 @@ Then retry `alembic upgrade head`. Either increase `max_connections` in `postgresql.conf` or add PgBouncer in front of PostgreSQL. The default PostgreSQL `max_connections` is `100`; reduce `DB_POOL_SIZE` per worker to stay within this limit. +### "QueuePool limit reached" TimeoutError (SQLite) + +If you see `TimeoutError: QueuePool limit of size 5 overflow 10 reached`, your +deployment is still running an older version of DocuElevate that used a bounded +connection pool for SQLite. Upgrade to the latest release — SQLite now uses +`NullPool`, which eliminates this error entirely. If you are already on the +latest version and are still seeing pool exhaustion, ensure you are not +overriding the engine creation manually. + For more help, see the [Troubleshooting Guide](Troubleshooting.md). diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index cb66bfa9..43345465 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -19,7 +19,7 @@ This guide covers all supported deployment methods for DocuElevate. - Access to required external services (if configured): - AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider) - Azure Document Intelligence - - Dropbox, Google Drive, OneDrive, S3, or other storage APIs + - Dropbox, Google Drive, OneDrive, SharePoint, S3, or other storage APIs - SMTP / IMAP server (for email processing) - Notification services (Discord, Telegram, etc.) @@ -349,16 +349,18 @@ workdir: ## Scaling +DocuElevate is designed for horizontal scaling. Both API and worker pods are stateless and can be scaled independently. + ### Docker Compose -Add more worker containers: +Scale workers (task processing) and API pods (request handling) independently: -```yaml -worker: - deploy: - replicas: 3 +```bash +docker compose up -d --scale worker=3 --scale api=2 ``` +> **Note:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. Do not scale it. It publishes periodic tasks to the Redis broker; workers pick them up. + ### Kubernetes / Helm Enable HPA: @@ -377,13 +379,15 @@ worker: maxReplicas: 10 ``` +The Helm chart deploys a separate **beat** pod (always 1 replica, `Recreate` strategy) so that scheduled tasks are never duplicated when workers scale. + --- ## Monitoring - **Docker Compose**: `docker-compose logs -f`, `docker stats` - **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f` -- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed. +- **Prometheus / Grafana**: Scrape the `/api/diagnostic/healthz/ready` endpoint for readiness; add custom metrics as needed. - **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring. --- diff --git a/docs/DropboxSetup.md b/docs/DropboxSetup.md index 8923a595..39989735 100644 --- a/docs/DropboxSetup.md +++ b/docs/DropboxSetup.md @@ -28,9 +28,10 @@ End users authorize their own Dropbox integration from the **Integrations** dash 1. Navigate to `/integrations` and click **+ Add Destination** (or **+ Add Source** for Watch Folder). 2. Create a Dropbox destination integration (or a Watch Folder with `source_type = dropbox`). 3. Click the **Authorize** button next to the integration — it links directly to the OAuth wizard pre-loaded with your integration's configuration. -4. Enter your Dropbox App Key and App Secret in the wizard (or use the global admin credentials if pre-configured). +4. If the administrator has configured system-wide Dropbox app credentials (`DROPBOX_APP_KEY` / `DROPBOX_APP_SECRET`), the wizard defaults to using them — no need to register your own Dropbox app. Uncheck the toggle to use custom credentials if needed. 5. Click **Start Authentication Flow**, authorize access in Dropbox, and the refresh token is automatically saved to your personal integration record. -6. The page redirects back to `/integrations` on success. Re-authorization is available at any time via the **Re-Authorize** button. +6. After authorization, an interactive **folder browser** lets you select the target folder directly from your Dropbox — no need to manually type folder paths. +7. The page redirects to `/integrations` on success. Re-authorization is available at any time via the **Re-Authorize** button. > **Note:** Your credentials are stored encrypted per-integration and are never mixed with other users' data. Each user can have multiple Dropbox integrations with independent tokens. @@ -128,7 +129,31 @@ If you encounter issues with Dropbox integration: 1. **Authentication Errors**: Make sure your App Key and App Secret are correct 2. **Token Expired**: Click "Refresh Token" button on the setup page to obtain a new token 3. **Folder Permissions**: Ensure your app has the correct permissions enabled for file operations -4. **Invalid Redirect URI**: Verify that the redirect URI in your app settings matches the one used in the authentication flow +4. **Invalid Redirect URI**: See section below for the most common cause and fix. 5. **Rate Limiting**: Dropbox API has rate limits; if exceeded, wait and try again +### Fixing "Invalid redirect_uri" Error + +This error appears on the Dropbox authorization page when the redirect URI in the OAuth request does not match any URI registered in your Dropbox app console. + +**Most common cause**: The application is deployed behind a reverse proxy (Traefik, Nginx, Caddy) that does **not** forward the `X-Forwarded-Proto: https` header to DocuElevate. Without this header, the server cannot determine that it is being accessed over HTTPS and may construct an `http://` redirect URI, while the registered URI in Dropbox is `https://`. + +**Fix**: + +Option 1 – Configure your proxy to forward `X-Forwarded-Proto`: + +```nginx +proxy_set_header X-Forwarded-Proto $scheme; +``` + +Option 2 – Set `PUBLIC_BASE_URL` in your environment (recommended for most deployments): + +```bash +PUBLIC_BASE_URL=https://docuelevate.example.com +``` + +When `PUBLIC_BASE_URL` is set, DocuElevate uses it directly for all OAuth redirect URIs instead of trying to infer the scheme from request headers. This is the most reliable option. + +After setting `PUBLIC_BASE_URL`, ensure the Dropbox app console redirect URI matches exactly (e.g., `https://docuelevate.example.com/dropbox-callback`). The setup wizard at `/dropbox-setup` will show you the exact URI to register. + For more general configuration issues, see the [Configuration Troubleshooting Guide](ConfigurationTroubleshooting.md). diff --git a/docs/GoogleDriveSetup.md b/docs/GoogleDriveSetup.md index e85addcc..bd0d59d5 100644 --- a/docs/GoogleDriveSetup.md +++ b/docs/GoogleDriveSetup.md @@ -30,7 +30,7 @@ End users can authorize their own Google Drive integration directly from the **I 1. Navigate to `/integrations` and click **+ Add Destination** (or **+ Add Source** for Watch Folder). 2. Create a Google Drive destination integration (or a Watch Folder with `source_type = google_drive`). 3. Click the **Authorize** button — it opens the OAuth wizard pre-loaded with your integration's configuration. -4. Enter your Google OAuth Client ID and Client Secret in the wizard. +4. If the administrator has configured system-wide Google Drive app credentials (`GOOGLE_DRIVE_CLIENT_ID` / `GOOGLE_DRIVE_CLIENT_SECRET`), the wizard defaults to using them — no need to register your own Google Cloud app. Uncheck the toggle to use custom credentials if needed. 5. Click **Start Authentication Flow** and authorize access in Google. 6. Credentials are saved automatically to your personal integration record; the page redirects back to `/integrations`. 7. Re-authorization is available at any time via the **Re-Authorize** button. diff --git a/docs/KubernetesDeployment.md b/docs/KubernetesDeployment.md index 94247873..c23ddc81 100644 --- a/docs/KubernetesDeployment.md +++ b/docs/KubernetesDeployment.md @@ -373,6 +373,8 @@ worker: replicaCount: 4 ``` +> **Beat scheduler:** The Helm chart deploys a dedicated `beat` pod (always exactly 1 replica with `Recreate` strategy) that publishes periodic tasks to the Redis broker. Workers consume these tasks — scaling workers does **not** duplicate scheduled jobs. + ### Horizontal Pod Autoscaler ```yaml @@ -433,24 +435,30 @@ externalRedis: ### Kubernetes Probes -The Helm chart configures liveness and readiness probes on the API pods via `/api/health`. Default settings: +The Helm chart configures **unauthenticated** liveness and readiness probes on the API pods so kubelet can reach them without credentials. Default settings: ```yaml api: livenessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/live port: 8000 initialDelaySeconds: 30 - periodSeconds: 30 + periodSeconds: 20 readinessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/ready port: 8000 - initialDelaySeconds: 10 + initialDelaySeconds: 15 periodSeconds: 10 ``` +| Endpoint | Auth | Purpose | +|----------|------|---------| +| `/api/diagnostic/healthz/live` | None | Lightweight liveness check — returns 200 if the process is running | +| `/api/diagnostic/healthz/ready` | None | Readiness check — verifies database and Redis connectivity (503 when DB is down) | +| `/api/diagnostic/health` | Required | Full health status for monitoring dashboards (Grafana, Uptime Kuma) | + ### Prometheus Scraping Add annotations to expose metrics (if using a Prometheus-compatible exporter): diff --git a/docs/MigrationWorkflow.md b/docs/MigrationWorkflow.md new file mode 100644 index 00000000..0bb3e862 --- /dev/null +++ b/docs/MigrationWorkflow.md @@ -0,0 +1,360 @@ +# Migration Workflow + +This guide explains how to create, test, and merge Alembic database migrations in DocuElevate — especially when **multiple feature branches** add migrations in parallel. + +## Table of Contents + +- [Quick Reference](#quick-reference) +- [Creating a New Migration](#creating-a-new-migration) +- [Migration Naming Convention](#migration-naming-convention) +- [Idempotent Migration Patterns](#idempotent-migration-patterns) +- [Parallel Branch Development](#parallel-branch-development) +- [Resolving Migration Conflicts](#resolving-migration-conflicts) +- [CI Validation](#ci-validation) +- [Pre-commit Hook](#pre-commit-hook) +- [Troubleshooting](#troubleshooting) + +--- + +## Quick Reference + +```bash +# Create a new migration after editing app/models.py +alembic revision --autogenerate -m "add_foobar_column" + +# Apply all pending migrations +alembic upgrade head + +# Check current database version +alembic current + +# View migration history +alembic history --verbose + +# Detect multiple heads (diverged branches) +alembic heads + +# Create a merge migration to resolve multiple heads +alembic merge heads -m "merge_parallel_branches" + +# Validate migration chain integrity (CI script) +python scripts/check_alembic_migrations.py +python scripts/check_alembic_migrations.py --verbose +``` + +--- + +## Creating a New Migration + +1. **Edit `app/models.py`** — add or modify SQLAlchemy model classes. + +2. **Generate the migration** from the repo root. Use `--rev-id` to set the + revision identifier directly (avoids renaming afterwards): + + ```bash + alembic revision --autogenerate --rev-id 037_add_my_new_table -m "add my new table" + ``` + + This creates `migrations/versions/037_add_my_new_table_add_my_new_table.py` + with `revision = "037_add_my_new_table"`. Rename the file to match: + + ```bash + mv migrations/versions/037_add_my_new_table_add_my_new_table.py \ + migrations/versions/037_add_my_new_table.py + ``` + + Alternatively, generate with the default hash and then rename: + + ```bash + alembic revision --autogenerate -m "add_my_new_table" + # Rename: mv migrations/versions/_add_my_new_table.py migrations/versions/037_add_my_new_table.py + # Update revision inside the file to match the filename stem. + ``` + + Alembic uses the `migrations/script.py.mako` template to generate the file. The template includes inline comments about idempotent patterns — read them. + +3. **Review the generated code** — autogenerate is helpful but not perfect. Check: + - Are new tables and columns detected correctly? + - Does the `downgrade()` reverse all changes? + - Are SQLite-incompatible operations wrapped in `batch_alter_table()`? + +4. **Test the migration** against a fresh database: + + ```bash + # Apply + alembic upgrade head + + # Rollback + alembic downgrade -1 + + # Re-apply + alembic upgrade head + ``` + +5. **Run the chain validation**: + + ```bash + python scripts/check_alembic_migrations.py + ``` + +--- + +## Migration Naming Convention + +All migration files follow a **sequential numeric prefix** scheme: + +``` +NNN_short_description.py +``` + +| Component | Rule | +|-----------|------| +| `NNN` | Three-digit zero-padded number, incrementing from the previous migration | +| `short_description` | Lowercase snake_case summary of the change | + +The **`revision`** variable inside the file **must match the filename stem** exactly: + +```python +# File: migrations/versions/037_add_classification_rules.py +revision: str = "037_add_classification_rules" +down_revision: Union[str, None] = "036_add_document_translation_fields" +``` + +The CI check (`scripts/check_alembic_migrations.py`) enforces this consistency. + +--- + +## Idempotent Migration Patterns + +Migrations should be **idempotent** — safe to run even if the change already exists. This is critical for SQLite compatibility and for recovering from partial failures. + +### Add a Column (only if missing) + +```python +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if "my_table" in inspector.get_table_names(): + existing = {c["name"] for c in inspector.get_columns("my_table")} + if "new_col" not in existing: + with op.batch_alter_table("my_table") as batch_op: + batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True)) +``` + +### Create a Table (only if missing) + +```python +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if "new_table" not in inspector.get_table_names(): + op.create_table( + "new_table", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + ) +``` + +### Drop a Column (only if present) + +```python +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if "my_table" in inspector.get_table_names(): + existing = {c["name"] for c in inspector.get_columns("my_table")} + if "new_col" in existing: + with op.batch_alter_table("my_table") as batch_op: + batch_op.drop_column("new_col") +``` + +### Use `batch_alter_table` for SQLite + +SQLite does not support `ALTER TABLE DROP COLUMN` or `ALTER TABLE RENAME COLUMN` natively. Alembic's `batch_alter_table` context manager works around this by recreating the table: + +```python +with op.batch_alter_table("users") as batch_op: + batch_op.add_column(sa.Column("phone", sa.String(20), nullable=True)) + batch_op.drop_column("fax") +``` + +--- + +## Parallel Branch Development + +When two feature branches both add migrations from the same parent, the migration chain **diverges** into multiple heads. This is normal and expected — Alembic supports it — but the heads must be merged before the code reaches `main`. + +### Example + +``` +main: 001 → 002 → 003 + ↘ Branch A: 004_add_widgets + ↘ Branch B: 004_add_gadgets ← two heads! +``` + +### How to Avoid Conflicts + +1. **Coordinate** — if two developers are both adding migrations, assign different sequence numbers (e.g., `037_` and `038_`). Even if both depend on `036_`, different numbers prevent filename collisions. + +2. **Rebase early** — before opening a PR, rebase your branch onto the latest `main`: + + ```bash + git fetch origin main + git rebase origin/main + ``` + + If `main` now has a new migration `037_*`, renumber yours to `038_*` and update `down_revision` to point at `037_*`. + +3. **Check for multiple heads** locally: + + ```bash + python scripts/check_alembic_migrations.py + # or + alembic heads + ``` + +--- + +## Resolving Migration Conflicts + +If your PR's CI check reports **"Multiple migration heads detected"**, follow these steps: + +### Step 1 — Update Your Branch + +```bash +git fetch origin main +git merge origin/main +# or +git rebase origin/main +``` + +### Step 2 — Check Heads + +```bash +python scripts/check_alembic_migrations.py --verbose +``` + +The output lists the conflicting heads. + +### Step 3 — Create a Merge Migration + +```bash +alembic merge heads -m "merge_parallel_branches" +``` + +This generates a new migration with **two parents** (a merge point): + +```python +down_revision = ("037_add_widgets", "037_add_gadgets") +``` + +### Step 4 — Rename and Validate + +Rename the merge migration to the next sequence number: + +```bash +mv migrations/versions/_merge_parallel_branches.py \ + migrations/versions/038_merge_parallel_branches.py +``` + +Update the `revision` inside to match, then validate: + +```bash +python scripts/check_alembic_migrations.py +``` + +### Step 5 — Test + +```bash +alembic upgrade head +alembic downgrade -1 +alembic upgrade head +``` + +--- + +## CI Validation + +The CI pipeline (`.github/workflows/ci.yml`) includes a **migration-chain** job that runs: + +```bash +python scripts/check_alembic_migrations.py +``` + +This script checks for: + +| Check | Description | +|-------|-------------| +| Multiple heads | Diverged migration chains that need a merge migration | +| Broken references | A `down_revision` that points to a non-existent revision | +| Duplicate revisions | Two files declaring the same `revision` identifier | +| Filename mismatches | The `revision` variable doesn't match the filename stem | + +The job runs in Stage 1 (fast-fail gates) alongside lint checks. If it fails, the build is blocked until the migration chain is fixed. + +--- + +## Pre-commit Hook + +A local pre-commit hook is configured in `.pre-commit-config.yaml` that runs the same check whenever you commit a change to `migrations/versions/`: + +```yaml +- repo: local + hooks: + - id: check-alembic-migrations + name: Check Alembic migration chain + entry: python scripts/check_alembic_migrations.py + language: python + pass_filenames: false + files: ^migrations/versions/.*\.py$ +``` + +Install the hook: + +```bash +pip install pre-commit +pre-commit install +``` + +--- + +## Troubleshooting + +### "Multiple migration heads detected" + +See [Resolving Migration Conflicts](#resolving-migration-conflicts) above. + +### "Broken chain: revision X references down_revision Y which does not exist" + +You removed or renamed a migration that another migration depends on. Either restore the missing file or update the dependent migration's `down_revision`. + +### "Filename mismatch: file declares revision=X but filename stem is Y" + +The `revision` string inside the Python file must match the filename (without `.py`). Rename the file or update the variable. + +### "relation already exists" when running `alembic upgrade head` + +The database has a table that a pending migration tries to create. Stamp the current state: + +```bash +alembic stamp head +``` + +### Autogenerate doesn't detect my changes + +Ensure all models are imported in `migrations/env.py`. The `from app.models import ...` block at the top must include your new model class. + +### SQLite "no such column" after downgrade + +SQLite has limited `ALTER TABLE` support. Always use `op.batch_alter_table()` for column operations on existing tables. + +--- + +## Further Reading + +- [Alembic Tutorial](https://alembic.sqlalchemy.org/en/latest/tutorial.html) +- [Alembic Branch / Merge](https://alembic.sqlalchemy.org/en/latest/branches.html) +- [Database Configuration Guide](DatabaseConfiguration.md) diff --git a/docs/MobileApp.md b/docs/MobileApp.md index 27c0aa80..07ff077b 100644 --- a/docs/MobileApp.md +++ b/docs/MobileApp.md @@ -8,12 +8,18 @@ DocuElevate includes a native mobile application for iOS and Android built with |---------|-----|---------| | SSO login (OAuth2) | ✅ | ✅ | | Local / basic auth login | ✅ | ✅ | +| QR code login (scan from web) | ✅ | ✅ | | Auto-generated API token | ✅ | ✅ | | Camera capture → upload | ✅ | ✅ | | File picker upload | ✅ | ✅ | +| Multi-image selection from library | ✅ | ✅ | | Share Sheet / Share Intent | ✅ | ✅ | | Push notifications | ✅ | ✅ | -| Document list | ✅ | ✅ | +| Document list with search | ✅ | ✅ | +| File detail view with processing logs | ✅ | ✅ | +| Pre-login legal pages (GDPR) | ✅ | ✅ | +| Localization (EN, DE, ES, FR, IT) | ✅ | ✅ | +| Language selection | ✅ | ✅ | | Dark mode | ✅ | ✅ | ## Getting Started (Development) @@ -112,6 +118,18 @@ When developing with **Expo Go** the app does not have the `docuelevate://` cust No extra configuration is needed — just run `npx expo start` and scan the QR code with the **Expo Go** app. +### QR Code Login Flow + +As an alternative to SSO, users can log in by scanning a QR code displayed in the web UI: + +1. The authenticated web user navigates to **Profile → Security & Sessions → Log in on mobile via QR code**. +2. A QR code is displayed containing a deep link: `docuelevate://qr-login?token=&server=`. +3. In the mobile app, the user taps **Scan QR Code to Login**, which opens the device camera. +4. The app scans the QR code, extracts both the server URL and the challenge token, and calls `POST /api/qr-auth/claim`. +5. An API token is issued and stored securely — no need to enter the server URL manually. + +> **Note:** The QR code already contains the server URL, so users do not need to type it in when using QR login. + ### Auto-generated Mobile Token When the mobile app completes login it automatically creates a named API token (`"Mobile App – "`) via `POST /api/mobile/generate-token`. This token: @@ -158,8 +176,8 @@ curl -X DELETE -H "Authorization: Bearer " https://your-server/api/mobile 1. Open the **Upload** tab. 2. Tap **Photos**. -3. Select an existing photo from the device's photo library. -4. The image is uploaded and queued for processing. +3. Select one or more photos from the device's photo library (multi-selection is supported). +4. All selected images are uploaded and queued for processing. ### File Picker @@ -185,6 +203,32 @@ The app registers itself as a share target so any file can be sent directly to D The URL may arrive as a standard `file://` path **or** under the app's custom `docuelevate://` scheme (e.g. `docuelevate://private/var/mobile/Library/…/file.pdf`). The root layout detects the custom-scheme form and rewrites it to a `file://` URL before forwarding it to the Upload screen through `ShareContext`. +##### Handling "unmatched route" errors from "Open In…" + +iOS sometimes delivers the file path under the `docuelevate://` scheme, e.g.: + +``` +docuelevate://private/var/mobile/Library/Mobile Documents/…/Invoice.pdf +``` + +expo-router strips the scheme and tries to match `/private/var/mobile/…` as an in-app route. Because no such route exists, it previously threw an **"unmatched route docuelevate://"** error and the upload never completed. + +The fix is a catch-all `+not-found.tsx` route (see `mobile/app/+not-found.tsx`). When expo-router cannot match the path, it renders this screen instead. The screen detects that the path is a filesystem path rather than a real in-app route, adds the file directly to `ShareContext`, and redirects to the Upload tab. `UploadScreen` picks up the pending file and begins uploading automatically. The `Linking` listener in the root layout may also fire for the same URL; `ShareContext.addPendingFile` deduplicates by URI so the file is only uploaded once. + +##### File accessibility and local caching + +Shared files may reference paths outside the app's sandbox or use security-scoped URLs that React Native's `fetch` cannot read directly. To guarantee reliable uploads: + +- **`LSSupportsOpeningDocumentsInPlace`** is set to `false` in `app.json`, which tells iOS to copy shared files into the app's `Documents/Inbox` directory before handing them to the app. +- **`UploadScreen`** uses `expo-file-system` (`FileSystem.copyAsync`) to copy any `file://` URI that is outside the app's cache/documents directory to a local cache path before uploading. This ensures the file is readable regardless of its origin. +- **MIME type inference**: Both `+not-found.tsx` and the `Linking` handler in `_layout.tsx` infer the MIME type from the file extension (e.g. `.pdf` → `application/pdf`) so the server receives a correct `Content-Type` instead of `application/octet-stream`. + +##### iOS Action / Share Extension (future enhancement) + +Apps like DeepL ("Translate in DeepL") and Microsoft Word ("Convert to Word") appear as **Action Extensions** in the iOS share sheet — a system-level feature that requires a separate Xcode target built with Swift or Objective-C. A proper Action Extension runs in its own process and must share authentication credentials with the main app via an iOS **App Group** (shared keychain / shared container). + +This level of iOS-native integration is a planned future enhancement. Until it is available, the recommended workflow is the current one: tap **Share → DocuElevate** (the app appears in the "Open With" row of the share sheet via `CFBundleDocumentTypes`). + #### Android implementation `app.json` declares `ACTION_SEND` and `ACTION_SEND_MULTIPLE` intent filters for `mimeType: "*/*"` in the `android.intentFilters` section. Incoming content URIs are received the same way as on iOS. @@ -202,6 +246,75 @@ If a file upload fails (e.g. due to network issues or a server error), the faile The retry re-uses the original file URI so no re-selection is needed. +## Document Search + +The **Files** tab includes a search bar at the top that lets users search through their processed documents by filename. Searches are debounced (400ms) to avoid excessive API calls. Clear the search with the ✕ button to return to the full list. + +## File Detail View + +Tapping any document in the **Files** tab opens a detail view showing: + +- **File metadata**: filename, file size, MIME type, upload date, and file hash +- **Processing status**: current status with a colour-coded icon +- **Processing log**: chronological list of processing steps with individual status indicators and timestamps + +Pull-to-refresh updates the detail view. This replicates the web interface at `/files/{id}` and `/files/{id}/detail` in a mobile-friendly layout. + +## Legal & Compliance + +### GDPR & Apple App Store Compliance + +Privacy Policy, Terms of Service, and Imprint links are accessible **before login** from both the **Welcome Screen** and the **Login Screen**. This ensures compliance with: + +- **GDPR** (General Data Protection Regulation) – users must be able to review the privacy policy before providing personal data +- **Apple App Store Review Guidelines** – apps must provide accessible privacy information before account creation + +Post-login, the same links are available in the **Profile** tab under the "Legal" section. + +## Localization (i18n) + +The mobile app supports five languages with automatic device-locale detection: + +| Language | Code | Status | +|----------|------|--------| +| English | `en` | ✅ Complete | +| German (Deutsch) | `de` | ✅ Complete | +| Spanish (Español) | `es` | ✅ Complete | +| French (Français) | `fr` | ✅ Complete | +| Italian (Italiano) | `it` | ✅ Complete | + +### How it works + +Language priority (highest to lowest): + +1. **Server preference** — `preferred_language` returned by `GET /api/mobile/whoami` on login or app resume. Allows a language set on the desktop web interface to propagate to mobile automatically. +2. **AsyncStorage** — the last language explicitly selected on the device, used as an offline fallback when the server is unreachable. +3. **Device locale** — detected via `expo-localization` on first launch. +4. **English** — final fallback when none of the above match a supported locale. + +When a user selects a language on mobile the choice is: +- Applied immediately to all screens (via `LocaleContext`) +- Persisted locally to AsyncStorage +- Synced to the server via `POST /api/i18n/language` (fire-and-forget), so the next desktop login reflects the same preference. + +> **Note**: If the server's preferred language is not supported by the mobile app (e.g. a locale added to the web frontend but not yet translated for mobile), the mobile app falls back to the next priority in the list above. + +### Adding a new language + +1. Create a new translation file in `mobile/src/i18n/` (e.g. `pt.json` for Portuguese) +2. Copy the structure from `en.json` and translate all values +3. Import the new file in `mobile/src/i18n/index.ts` +4. Add it to the `translations` object and `getSupportedLanguages()` array + +## User Settings + +The **Profile** tab includes a **Settings** section where users can: + +- **Change language**: Select from the supported languages (English, German, Spanish, French, Italian) +- View server connection details +- Access legal documents (Privacy Policy, Terms of Service, Imprint) +- Sign out or delete their account + ## Mobile API Endpoints The backend exposes a dedicated `/api/mobile/` namespace: @@ -212,7 +325,8 @@ The backend exposes a dedicated `/api/mobile/` namespace: | `POST` | `/api/mobile/register-device` | Bearer | Register Expo push token | | `GET` | `/api/mobile/devices` | Bearer | List registered devices | | `DELETE` | `/api/mobile/devices/{id}` | Bearer | Deactivate a device | -| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile | +| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile (includes `preferred_language`) | +| `POST` | `/api/i18n/language` | Bearer | Sync language preference to server | All other API endpoints (file upload, file listing, etc.) work with Bearer token authentication. @@ -256,7 +370,7 @@ Re-registering the same token is safe (idempotent). ### GET /api/mobile/whoami -Returns the current user's profile. +Returns the current user's profile, including the server-stored language preference. **Response (200):** ```json @@ -265,10 +379,15 @@ Returns the current user's profile. "display_name": "John Doe", "email": "john@example.com", "avatar_url": "https://www.gravatar.com/avatar/...", - "is_admin": false + "is_admin": false, + "preferred_language": "de" } ``` +`preferred_language` is `null` when no preference has been saved. The mobile +app applies this value on login / app resume, falling back to AsyncStorage and +then the device locale when it is `null` or unsupported. + ## Configuration No server-side configuration is required to enable the mobile app. The Expo push notification routing does not need FCM or APNs credentials on the server. @@ -279,7 +398,20 @@ If you wish to use **direct FCM/APNs** without Expo's relay, replace the `send_e ``` mobile/ -├── App.tsx # Root component +├── App.tsx # Root component (legacy, not used at runtime) +├── app/ # Expo Router file-based routes +│ ├── _layout.tsx # Root layout (AuthGuard + providers) +│ ├── index.tsx # Root redirect → /(auth)/ +│ ├── (auth)/ # Unauthenticated route group +│ │ ├── _layout.tsx # Stack navigator (headerless) +│ │ ├── index.tsx # Welcome screen +│ │ ├── login.tsx # Login screen +│ │ └── qr-scanner.tsx # QR code scanner screen +│ └── (tabs)/ # Authenticated route group +│ ├── _layout.tsx # Tab navigator +│ ├── index.tsx # Upload screen (default tab) +│ ├── files.tsx # Files screen +│ └── profile.tsx # Profile screen ├── app.json # Expo/EAS configuration ├── eas.json # EAS Build profiles ├── package.json @@ -291,16 +423,48 @@ mobile/ ├── hooks/ │ └── usePushNotifications.ts # Push token registration ├── screens/ - │ ├── LoginScreen.tsx # Server URL + SSO button + │ ├── LoginScreen.tsx # Server URL + SSO button + QR code scanner + │ ├── QRScannerScreen.tsx # Camera-based QR code scanner for login │ ├── UploadScreen.tsx # Camera capture + photo library + file picker - │ ├── FilesScreen.tsx # Processed document list - │ └── ProfileScreen.tsx # User profile + sign out + │ ├── FilesScreen.tsx # Processed document list with search + │ ├── FileDetailScreen.tsx # File detail view with processing logs + │ ├── ProfileScreen.tsx # User profile + settings + sign out + │ └── WelcomeScreen.tsx # Pre-login welcome with legal links + ├── i18n/ # Localization (i18n) + │ ├── index.ts # i18n module (locale detection, t() function) + │ ├── en.json # English translations + │ ├── de.json # German translations + │ ├── es.json # Spanish translations + │ ├── fr.json # French translations + │ └── it.json # Italian translations + ├── utils/ + │ ├── mimeTypes.ts # MIME type mapping for file extensions + │ └── normalizeUri.ts # URI normalization for deduplication └── services/ └── api.ts # DocuElevate REST API client ``` ## Troubleshooting +### App shows "Hello World" / default Expo page after update + +If the iOS or Android app shows a generic "Hello World – This is the first page of your app" screen instead of the DocuElevate UI, it means a stale default `index.tsx` file (generated by Expo CLI scaffolding) is being picked up in the `mobile/app/` directory. + +**To fix:** + +1. Delete any leftover default `mobile/app/index.tsx` that is **not** the repository version (the repo version contains a `` to `/(auth)/`). +2. Clear the Metro bundler cache and rebuild: + ```bash + cd mobile + npx expo start --clear + ``` +3. For production builds, run a clean EAS build: + ```bash + eas build --platform ios --clear-cache + ``` + +The repository includes a root `app/index.tsx` that immediately redirects to the authentication flow, so this issue should not recur once the correct file is present. + ### "Session expired Local session" during iOS build EAS stores an Apple ID session locally (in `~/.expo/`) to manage code-signing certificates and provisioning profiles. This session expires after a few weeks. @@ -362,3 +526,4 @@ eas build --platform ios - [API Documentation](./API.md) - [Configuration Guide](./ConfigurationGuide.md) - [Deployment Guide](./DeploymentGuide.md) +- [Apple App Store Compliance Audit](./AppleAppStoreCompliance.md) diff --git a/docs/OneDriveSetup.md b/docs/OneDriveSetup.md index e9a555f4..ba2fc873 100644 --- a/docs/OneDriveSetup.md +++ b/docs/OneDriveSetup.md @@ -29,9 +29,10 @@ End users authorize their own OneDrive integration from the **Integrations** das 1. Navigate to `/integrations` and click **+ Add Destination** (or **+ Add Source** for Watch Folder). 2. Create a OneDrive destination integration (or a Watch Folder with `source_type = onedrive`). 3. Click the **Authorize** button next to the integration — it links directly to the OAuth wizard pre-loaded with your integration's configuration. -4. Enter your Azure AD Client ID and Client Secret in the wizard. +4. If the administrator has configured system-wide OneDrive app credentials (`ONEDRIVE_CLIENT_ID` / `ONEDRIVE_CLIENT_SECRET`), the wizard defaults to using them — no need to register your own Azure AD app. Uncheck the toggle to use custom credentials if needed. 5. Click **Start Authentication Flow**, authorize access via Microsoft, and the refresh token is automatically saved to your personal integration record. -6. The page redirects back to `/integrations` on success. Re-authorization is available at any time via the **Re-Authorize** button. +6. After authorization, an interactive **folder browser** lets you select the target folder directly from your OneDrive — no need to manually type folder paths. +7. The page redirects to `/integrations` on success. Re-authorization is available at any time via the **Re-Authorize** button. > **Note:** Your credentials are stored encrypted per-integration and are never mixed with other users' data. Each user can have multiple OneDrive integrations with independent tokens. diff --git a/docs/ProductionReadiness.md b/docs/ProductionReadiness.md index 2f93c7c5..a938e39e 100644 --- a/docs/ProductionReadiness.md +++ b/docs/ProductionReadiness.md @@ -34,7 +34,7 @@ Use this checklist to track readiness before going live. - [ ] **Redis** — Running and accessible only from internal network - [ ] **Meilisearch** — Running and accessible only from internal network - [ ] **Worker replicas** — At least 2 workers configured for redundancy -- [ ] **Monitoring** — `/api/health` polled by uptime checker +- [ ] **Monitoring** — `/api/diagnostic/health` polled by uptime checker - [ ] **Backups** — Automated backup of database, workdir, and Meilisearch data - [ ] **Log retention** — Logs shipped to a persistent store or aggregator - [ ] **Secrets management** — API keys not committed to source control @@ -285,22 +285,24 @@ For SSO/OIDC (Authentik, Keycloak, Auth0, etc.) see the [Authentication Setup Gu ### Docker Compose -Use the `deploy.replicas` setting (requires Docker Swarm mode) or simply run multiple workers: - -```yaml -worker: - deploy: - replicas: 3 -``` - -Or scale after deployment: +Scale workers independently: ```bash -docker-compose up -d --scale worker=3 +docker compose up -d --scale worker=3 ``` Each worker processes tasks from the Celery queue independently. Ensure the shared `workdir` volume is accessible from all worker containers. +> **Important:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. It is defined as a dedicated service in `docker-compose.yaml` with a fixed `container_name`. Do not scale it. + +### Scaling the API + +API pods are fully stateless (sessions use encrypted cookies, not server-side state) and can be scaled behind a load balancer: + +```bash +docker compose up -d --scale api=3 +``` + ### Kubernetes (Helm) ```yaml @@ -339,11 +341,32 @@ celery -A app.celery_worker worker -Q default,celery --concurrency=2 ### Health Check Endpoint -DocuElevate exposes `/api/health` for readiness probing. Configure your uptime monitor to poll this endpoint: +DocuElevate exposes three health-related endpoints: + +| Endpoint | Auth | Purpose | +|----------|------|---------| +| `GET /api/diagnostic/healthz/live` | None | Lightweight liveness probe — returns 200 if the process is running | +| `GET /api/diagnostic/healthz/ready` | None | Readiness probe — checks database and Redis (503 when DB is down) | +| `GET /api/diagnostic/health` | Required | Full status for monitoring dashboards (Grafana, Uptime Kuma) | + +For **Kubernetes probes**, use the unauthenticated endpoints: + +```yaml +livenessProbe: + httpGet: + path: /api/diagnostic/healthz/live + port: 8000 +readinessProbe: + httpGet: + path: /api/diagnostic/healthz/ready + port: 8000 +``` + +For **uptime monitors** (Uptime Kuma, Grafana, etc.), use the authenticated endpoint: ```bash -curl http://docuelevate.example.com/api/health -# Expected: {"status": "ok", ...} +curl http://docuelevate.example.com/api/diagnostic/health +# Expected: {"status": "healthy", ...} ``` Set `UPTIME_KUMA_URL` to your Uptime Kuma push URL for heartbeat monitoring: @@ -502,4 +525,4 @@ For a dedicated Kubernetes deployment guide, including architecture diagrams, PV - **Image Pull Policy**: Use `IfNotPresent` in production with pinned image tags (not `latest`) for reproducible deployments. -- **Liveness & Readiness Probes**: Already configured in the Helm chart via `/api/health`. Verify they are tuned to your startup time. +- **Liveness & Readiness Probes**: Already configured in the Helm chart via unauthenticated endpoints (`/api/diagnostic/healthz/live` and `/api/diagnostic/healthz/ready`). Verify they are tuned to your startup time. diff --git a/docs/README.md b/docs/README.md index a3a6a96c..bc210343 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ Welcome to the DocuElevate documentation. This directory contains comprehensive - [Google Drive Setup](GoogleDriveSetup.md) - How to set up Google Drive integration - [Dropbox Setup](DropboxSetup.md) - How to set up Dropbox integration - [OneDrive Setup](OneDriveSetup.md) - How to set up Microsoft OneDrive/Graph integration + - [SharePoint Setup](SharePointSetup.md) - How to set up Microsoft SharePoint Online integration - [Amazon S3 Setup](AmazonS3Setup.md) - How to set up Amazon S3 integration - [Authentication Setup](AuthenticationSetup.md) - How to set up user authentication - [Notifications Setup](NotificationsSetup.md) - How to set up system notifications diff --git a/docs/SettingsManagement.md b/docs/SettingsManagement.md index b64c1ef4..288c261e 100644 --- a/docs/SettingsManagement.md +++ b/docs/SettingsManagement.md @@ -28,7 +28,7 @@ Settings are organized into logical categories for easy navigation: - **Authentication**: Login settings, session secrets, OAuth configuration, admin group - **AI Services**: AI provider selection, model configuration, embeddings, and credentials (OpenAI, Azure, Anthropic, Gemini, Ollama, OpenRouter, Portkey, LiteLLM) - **OCR Engines**: OCR provider selection and configuration (Tesseract, EasyOCR, Mistral, Google DocAI, AWS Textract) -- **Storage Providers**: Dropbox, Google Drive, OneDrive, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless +- **Storage Providers**: Dropbox, Google Drive, OneDrive, SharePoint, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless - **Email**: SMTP configuration for sending emails - **IMAP**: Email ingestion configuration (supports two mailbox accounts) - **Monitoring**: Uptime Kuma integration diff --git a/docs/SharePointSetup.md b/docs/SharePointSetup.md new file mode 100644 index 00000000..560f1671 --- /dev/null +++ b/docs/SharePointSetup.md @@ -0,0 +1,185 @@ +# Setting up SharePoint Integration + +This guide explains how to set up the Microsoft SharePoint Online integration for DocuElevate. + +## Required Configuration Parameters + +| **Variable** | **Description** | +|---------------------------------|-------------------------------------------------------| +| `SHAREPOINT_CLIENT_ID` | Azure AD application client ID | +| `SHAREPOINT_CLIENT_SECRET` | Azure AD application client secret | +| `SHAREPOINT_TENANT_ID` | Azure AD tenant ID (use "common" for multi-tenant apps) | +| `SHAREPOINT_REFRESH_TOKEN` | OAuth 2.0 refresh token | +| `SHAREPOINT_SITE_URL` | SharePoint site URL (e.g. `https://tenant.sharepoint.com/sites/sitename`) | +| `SHAREPOINT_DOCUMENT_LIBRARY` | Document library name (default: `Documents`) | +| `SHAREPOINT_FOLDER_PATH` | Subfolder path inside the document library | + +For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md). + +## Overview + +SharePoint Online integration uses the same Microsoft Graph API as OneDrive. The key difference is that SharePoint targets a **site-specific document library** rather than a personal OneDrive. Documents are uploaded via chunked upload sessions for reliability with large files. + +> **Tip:** If you already have an Azure AD app registration for OneDrive, you can reuse it for SharePoint — just add the `Sites.ReadWrite.All` permission. + +## Setup Steps + +### 1. Register an application in Azure Active Directory + +If you don't already have an app registration (e.g. from OneDrive setup): + +1. Go to the [Azure Portal](https://portal.azure.com/) +2. Navigate to **Azure Active Directory** > **App registrations** +3. Click **New registration** +4. Enter a name for your application (e.g., "DocuElevate") +5. For **Supported account types**, select: + - **Single tenant**: "Accounts in this organizational directory only" + - **Multi-tenant**: "Accounts in any organizational directory" +6. For **Redirect URI**, select "Web" and enter your callback URL (e.g., `https://your-domain.com/onedrive-callback`) +7. Click **Register** + +### 2. Get Application (client) ID + +1. After registration, note the **Application (client) ID** from the overview page +2. Set this value as `SHAREPOINT_CLIENT_ID` + +### 3. Create a client secret + +1. In your application page, go to **Certificates & secrets** +2. Under **Client secrets**, click **New client secret** +3. Add a description and select an expiration period +4. Click **Add** and immediately copy the secret value (it will only be shown once) +5. Set this value as `SHAREPOINT_CLIENT_SECRET` + +### 4. Configure API permissions + +1. In your application page, go to **API permissions** +2. Click **Add a permission** +3. Select **Microsoft Graph** +4. For **delegated permissions** (user-context access), add: + - `Sites.ReadWrite.All` — Read and write items in all site collections + - `offline_access` — Required for refresh tokens +5. For **application permissions** (app-only access without a user), add: + - `Sites.ReadWrite.All` — Read and write items in all site collections +6. Click **Add permissions** +7. Click **Grant admin consent** (requires admin privileges) + +> **Important:** SharePoint access requires `Sites.ReadWrite.All` rather than the `Files.ReadWrite` permission used by OneDrive. + +### 5. Get your Tenant ID + +1. In the Azure Portal, find your **Tenant ID** (also called "Directory ID") +2. It is on the **Azure Active Directory** overview page +3. Set this value as `SHAREPOINT_TENANT_ID` + +### 6. Generate a Refresh Token + +#### Using the OneDrive Auth Wizard + +The SharePoint integration reuses the same MSAL token flow as OneDrive: + +1. Navigate to `/onedrive-setup` +2. Enter your SharePoint Client ID and Tenant ID +3. Click **Start Authentication Flow** and follow the prompts +4. Copy the generated refresh token and set it as `SHAREPOINT_REFRESH_TOKEN` + +#### Manual Method + +1. Open the following URL in your browser (replace placeholders): + ``` + https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI&response_mode=query&scope=https://graph.microsoft.com/.default offline_access&prompt=consent + ``` +2. Sign in with your Microsoft work account +3. After authentication, copy the `code` parameter from the redirect URL +4. Exchange the code for tokens: + ```bash + curl -X POST https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "client_id=YOUR_CLIENT_ID&scope=https://graph.microsoft.com/.default offline_access&code=YOUR_AUTH_CODE&redirect_uri=YOUR_REDIRECT_URI&grant_type=authorization_code&client_secret=YOUR_CLIENT_SECRET" + ``` +5. From the response JSON, copy the `refresh_token` value +6. Set this as `SHAREPOINT_REFRESH_TOKEN` + +### 7. Find your SharePoint Site URL + +Your SharePoint site URL follows the pattern: +``` +https://YOUR-TENANT.sharepoint.com/sites/SITE-NAME +``` + +For example: +- `https://contoso.sharepoint.com/sites/documents` +- `https://contoso.sharepoint.com/sites/engineering-team` + +Set this as `SHAREPOINT_SITE_URL`. + +### 8. Choose your Document Library + +Each SharePoint site has one or more document libraries. The default library is usually called `Documents` (or `Shared Documents`). You can find your library names by navigating to your SharePoint site in a browser and looking at the left sidebar. + +Set the library name as `SHAREPOINT_DOCUMENT_LIBRARY` (default: `Documents`). + +### 9. Set the Upload Folder (Optional) + +If you want documents to be uploaded into a subfolder inside the library, set `SHAREPOINT_FOLDER_PATH`. For example, `Uploads` or `DocuElevate/Processed`. + +## App-Only Access (No User Token) + +For fully automated scenarios without user interaction: + +1. Add **Application permissions** (not Delegated) for `Sites.ReadWrite.All` +2. Grant admin consent +3. Set `SHAREPOINT_TENANT_ID` to your organization's tenant ID +4. Leave `SHAREPOINT_REFRESH_TOKEN` empty — the app will use the client credentials flow + +> **Note:** Client credentials flow requires a specific tenant ID (not "common"). + +## Configuration Examples + +**With Refresh Token (Delegated Permissions):** +```dotenv +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=87654321-4321-4321-4321-210987654321 +SHAREPOINT_REFRESH_TOKEN=your_refresh_token +SHAREPOINT_SITE_URL=https://contoso.sharepoint.com/sites/documents +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads +``` + +**App-Only Access (Application Permissions):** +```dotenv +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=87654321-4321-4321-4321-210987654321 +# No refresh token needed for app-only access +SHAREPOINT_SITE_URL=https://contoso.sharepoint.com/sites/documents +SHAREPOINT_DOCUMENT_LIBRARY=Shared Documents +SHAREPOINT_FOLDER_PATH=DocuElevate/Processed +``` + +## Troubleshooting + +### "Failed to resolve SharePoint site" + +- Verify `SHAREPOINT_SITE_URL` is correct and accessible +- Ensure your app has `Sites.ReadWrite.All` permission with admin consent +- Check that the site exists and your account has access to it + +### "Document library not found" + +- Verify the library name in `SHAREPOINT_DOCUMENT_LIBRARY` matches exactly (case-insensitive) +- Navigate to your SharePoint site in a browser to confirm the library name +- Common names: `Documents`, `Shared Documents` + +### Token errors + +- If using a refresh token, try re-authorizing via the OAuth flow +- Ensure `offline_access` scope is included in your permissions +- For app-only access, verify the tenant ID is not set to "common" + +### Permission errors + +- Ensure an admin has granted consent for `Sites.ReadWrite.All` +- Verify the app registration has the correct permissions +- Check that the site's sharing settings allow API access diff --git a/docs/StorageArchitecture.md b/docs/StorageArchitecture.md index af055406..399b89a8 100644 --- a/docs/StorageArchitecture.md +++ b/docs/StorageArchitecture.md @@ -341,6 +341,7 @@ in task messages or logs. | `S3` | boto3 `upload_file`, per-user access key | | `GOOGLE_DRIVE` | Google Drive API v3, OAuth or service account | | `ONEDRIVE` | Microsoft Graph API, MSAL confidential-client | +| `SHAREPOINT` | Microsoft Graph API, site/drive resolution + chunked upload | | `WEBDAV` | HTTP PUT request, Basic Auth | | `NEXTCLOUD` | WebDAV (same as WEBDAV, Nextcloud-compatible path) | | `FTP` | ftplib FTPS (TLS preferred, plaintext configurable) | diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 8d6b0a82..8f791cfb 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -87,7 +87,7 @@ DocuElevate provides multiple convenient ways to upload documents to the system. #### Supported File Types - **Documents**: PDF, Word (.doc, .docx), Excel (.xls, .xlsx), PowerPoint (.ppt, .pptx) -- **Images**: JPEG, PNG, GIF, BMP, TIFF, WebP, SVG +- **Images**: JPEG, PNG, GIF, BMP, TIFF, WebP, SVG, HEIC, HEIF - **Text**: Plain text (.txt), CSV, RTF, HTML, XML, Markdown - **Maximum file size**: 500MB per file @@ -170,7 +170,7 @@ The **Integrations** page (`/integrations`) provides a unified view of all your - **S3** — bucket, region, access key, secret key - **WebDAV / Nextcloud** — URL, folder, username, password - **FTP / SFTP** — host, port, remote path, username, password - - **Dropbox / Google Drive / OneDrive** — folder path, with a link to the OAuth setup page + - **Dropbox / Google Drive / OneDrive / SharePoint** — folder path, with a link to the OAuth setup page - **Email Forward** — recipient email address - **Watch Folder** — source type (Local, S3, Dropbox, Google Drive, OneDrive, Nextcloud, WebDAV), per-type config fields, delete after processing toggle - **Paperless NGX** — URL and API token @@ -582,7 +582,25 @@ Processing pipelines let you define exactly what happens to your documents when | `embed_metadata` | Write extracted metadata into the PDF document properties | | `compute_embedding` | Compute semantic embeddings for similarity search | | `send_to_destinations` | Upload the processed document to all configured storage destinations | -| `classify` | Classify the document type with AI | +| `classify` | Classify the document type using rules (filename patterns, content keywords, metadata) | + +#### Classify step – rule-based document classification + +The `classify` step assigns a category to each document by evaluating **built-in** and **custom** classification rules. Rules are matched against three signals: + +- **Filename patterns** — regex matched against the original filename (e.g. `(?i)invoice` matches filenames containing "invoice"). +- **Content keywords** — pipe-separated keywords matched against the OCR text (e.g. `invoice number|amount due`). +- **Metadata match** — `field=value` matched against existing AI metadata (e.g. `document_type=Invoice`). + +**Pre-built categories** include: Invoice, Contract, Receipt, Letter, Report, Bank Statement, Tax Document, Insurance, and Payslip. You can also define your own custom categories. + +The classification result is stored in the document's `ai_metadata` under the `classification` key with the matched category, confidence score, and list of matched rules. If no `document_type` was previously set by AI metadata extraction, the classify step will also populate it. + +> **Tip:** Manage custom classification rules via **Settings → Classification Rules** or the `/api/classification-rules/` API. See the [API Documentation](./API.md#classification-rules) for details. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `use_builtin_rules` | boolean | `true` | Include the pre-built classification rules | #### OCR step options diff --git a/frontend/static/js/common.js b/frontend/static/js/common.js index 67219a11..1ec9ae6b 100644 --- a/frontend/static/js/common.js +++ b/frontend/static/js/common.js @@ -215,6 +215,9 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') { linksDiv.appendChild( _makeMenuLink('/api-tokens', 'fas fa-key text-yellow-500', window.__i18n.apiTokens || 'API Tokens', 'text-gray-700') ); + linksDiv.appendChild( + _makeMenuLink('/devices', 'fas fa-mobile-alt text-blue-500', window.__i18n.devices || 'Devices', 'text-gray-700') + ); linksDiv.appendChild( _makeMenuLink('/shared-links', 'fas fa-share-alt text-blue-400', window.__i18n.sharedLinks || 'Shared Links', 'text-gray-700') ); @@ -311,6 +314,18 @@ function _makeMenuLink(href, iconClass, label, extraClasses = '') { tokensLink.appendChild(document.createTextNode(window.__i18n.apiTokens || 'API Tokens')); mobileAuthSection.appendChild(tokensLink); + // Devices link + const devicesLink = document.createElement('a'); + devicesLink.href = '/devices'; + devicesLink.className = + 'flex items-center px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50'; + const devicesIcon = document.createElement('i'); + devicesIcon.className = 'fas fa-mobile-alt mr-2 text-blue-500'; + devicesIcon.setAttribute('aria-hidden', 'true'); + devicesLink.appendChild(devicesIcon); + devicesLink.appendChild(document.createTextNode(window.__i18n.devices || 'Devices')); + mobileAuthSection.appendChild(devicesLink); + // Shared Links link const sharedLinksLink = document.createElement('a'); sharedLinksLink.href = '/shared-links'; diff --git a/frontend/static/js/upload.js b/frontend/static/js/upload.js index 0e25ce7b..b8ecd62a 100644 --- a/frontend/static/js/upload.js +++ b/frontend/static/js/upload.js @@ -433,9 +433,17 @@ function _uploadSingleFile(file, progressBar, statusEl, onTerminal) { if (xhr.status === 200) { const result = JSON.parse(xhr.responseText); progressBar.style.width = '100%'; - progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full'; - statusEl.textContent = `Success: Task ID: ${result.task_id}`; - statusEl.className = 'text-xs text-green-600 mt-1'; + + if (result.status === 'duplicate' && result.duplicate_of) { + // Exact duplicate – no processing task was created + progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full'; + statusEl.textContent = `Duplicate – already processed (file #${result.duplicate_of.original_file_id})`; + statusEl.className = 'text-xs text-yellow-600 mt-1'; + } else { + progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full'; + statusEl.textContent = `Success: Task ID: ${result.task_id}`; + statusEl.className = 'text-xs text-green-600 mt-1'; + } _onUploadSuccess(); onTerminal(); resolve({ rateLimited: false, retryAfterSeconds: 0 }); diff --git a/frontend/templates/api_tokens.html b/frontend/templates/api_tokens.html index 5e4633cf..b4facee8 100644 --- a/frontend/templates/api_tokens.html +++ b/frontend/templates/api_tokens.html @@ -33,6 +33,20 @@ aria-required="true" /> +
+ + +
+ + + + @@ -209,9 +258,10 @@ function apiTokens() { tokens: [], loading: true, creating: false, - revoking: null, + acting: null, error: null, newTokenName: '', + newTokenExpiresDays: null, newlyCreatedToken: null, copied: false, baseUrl: window.location.origin, @@ -238,13 +288,15 @@ function apiTokens() { this.error = null; this.newlyCreatedToken = null; try { + const body = { name: this.newTokenName.trim() }; + if (this.newTokenExpiresDays) body.expires_in_days = parseInt(this.newTokenExpiresDays); const res = await fetch('/api/api-tokens/', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken, }, - body: JSON.stringify({ name: this.newTokenName.trim() }), + body: JSON.stringify(body), }); if (!res.ok) { const data = await res.json().catch(() => ({})); @@ -253,6 +305,7 @@ function apiTokens() { const data = await res.json(); this.newlyCreatedToken = data.token; this.newTokenName = ''; + this.newTokenExpiresDays = null; await this.loadTokens(); } catch (e) { this.error = e.message; @@ -263,7 +316,7 @@ function apiTokens() { async revokeToken(token) { if (!confirm(`Revoke token "${token.name}"? This cannot be undone.`)) return; - this.revoking = token.id; + this.acting = token.id; this.error = null; try { const res = await fetch(`/api/api-tokens/${token.id}`, { @@ -278,10 +331,66 @@ function apiTokens() { } catch (e) { this.error = e.message; } finally { - this.revoking = null; + this.acting = null; } }, + async reactivateToken(token) { + if (!confirm({{ _("api_tokens.reactivate_confirm") | tojson }})) return; + this.acting = token.id; + this.error = null; + try { + const res = await fetch(`/api/api-tokens/${token.id}/reactivate`, { + method: 'POST', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to reactivate token'); + } + await this.loadTokens(); + } catch (e) { + this.error = e.message; + } finally { + this.acting = null; + } + }, + + async deleteToken(token) { + if (!confirm({{ _("api_tokens.delete_confirm") | tojson }})) return; + this.acting = token.id; + this.error = null; + try { + const res = await fetch(`/api/api-tokens/${token.id}`, { + method: 'DELETE', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to delete token'); + } + await this.loadTokens(); + } catch (e) { + this.error = e.message; + } finally { + this.acting = null; + } + }, + + tokenStatusClass(token) { + if (!token.is_active) return 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + if (token.expires_at && new Date(token.expires_at) < new Date()) + return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-400'; + return 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400'; + }, + + tokenStatusLabel(token) { + if (!token.is_active) return '{{ _("api_tokens.status_revoked") }}'; + if (token.expires_at && new Date(token.expires_at) < new Date()) + return '{{ _("api_tokens.status_expired") }}'; + return '{{ _("api_tokens.status_active") }}'; + }, + copyToken() { if (this.newlyCreatedToken) { navigator.clipboard.writeText(this.newlyCreatedToken); diff --git a/frontend/templates/base.html b/frontend/templates/base.html index d049cfe2..b7e47e44 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -177,6 +177,11 @@ {{ _("nav.backup_restore") }} + {% if enable_factory_reset %} + + {{ _("nav.system_reset") }} + + {% endif %} Audit Logs @@ -445,6 +450,11 @@ {{ _("nav.backup_restore") }} + {% if enable_factory_reset %} + + {{ _("nav.system_reset") }} + + {% endif %} {{ _("nav.status") }} @@ -560,6 +570,7 @@ profileSettings: {{ _("nav.profile_settings") | tojson }}, mySubscription: {{ _("nav.my_subscription") | tojson }}, apiTokens: {{ _("nav.api_tokens") | tojson }}, + devices: {{ _("nav.devices") | tojson }}, sharedLinks: {{ _("nav.shared_links") | tojson }}, signOut: {{ _("nav.sign_out") | tojson }}, logIn: {{ _("nav.login") | tojson }}, diff --git a/frontend/templates/devices.html b/frontend/templates/devices.html new file mode 100644 index 00000000..b640d950 --- /dev/null +++ b/frontend/templates/devices.html @@ -0,0 +1,459 @@ +{% extends "base.html" %} + +{% block title %}{{ _("devices.page_title") }}{% endblock %} + +{% block content %} + + + +{% endblock %} diff --git a/frontend/templates/dropbox.html b/frontend/templates/dropbox.html index a94a5b96..6c41bd96 100644 --- a/frontend/templates/dropbox.html +++ b/frontend/templates/dropbox.html @@ -104,7 +104,7 @@

Step 3: Set OAuth 2 Redirect URI

  1. In your app's settings page, go to the "OAuth 2" section
  2. -
  3. Add a redirect URI: {{ request.url.scheme }}://{{ request.url.netloc }}/dropbox-callback
  4. +
  5. Add a redirect URI: {{ callback_url }}
  6. Click "Add" to save the redirect URI
@@ -116,6 +116,34 @@
+ {% if user_mode and global_creds_available %} + +
+
+ +
+

Using shared application credentials

+

Your administrator has enabled shared Dropbox app credentials. You can authorize your account without supplying your own App Key and Secret.

+
+
+
+ + {% if folder_path %} +
+

Target folder (from integration settings)

+

{{ folder_path }}

+
+ {% endif %} + +
+ +
+ {% else %}
@@ -146,6 +174,7 @@ Start Authentication Flow
+ {% endif %} {% if not user_mode %} @@ -268,6 +297,10 @@ DROPBOX_FOLDER={{ folder_path|default('/Documents/Uploads', true) }} document.addEventListener('DOMContentLoaded', function() { const userMode = {{ 'true' if user_mode else 'false' }}; + const globalCredsAvailable = {{ 'true' if global_creds_available else 'false' }}; + // Redirect URI for OAuth: prefer server-provided value (respects PUBLIC_BASE_URL), + // fall back to window.location.origin for resilience. + const dropboxCallbackUrl = {{ callback_url | tojson }} || (window.location.origin + "/dropbox-callback"); // Store integration_id if provided (for per-user OAuth flow) const integrationId = "{{ integration_id or '' }}"; @@ -277,6 +310,7 @@ document.addEventListener('DOMContentLoaded', function() { // Elements const startAuthFlowBtn = document.getElementById('start-auth-flow'); + const startAuthFlowGlobalBtn = document.getElementById('start-auth-flow-global'); const testTokenBtn = document.getElementById('test-token'); const refreshTokenBtn = document.getElementById('refresh-token-btn'); const tokenStatus = document.getElementById('token-status'); @@ -328,11 +362,38 @@ document.addEventListener('DOMContentLoaded', function() { } }); + // Global-credentials "Authorize with Dropbox" button (user mode, admin-provided creds) + if (startAuthFlowGlobalBtn) { + startAuthFlowGlobalBtn.addEventListener('click', async function() { + startAuthFlowGlobalBtn.disabled = true; + startAuthFlowGlobalBtn.innerHTML = ' Redirecting…'; + try { + const resp = await fetch('/api/dropbox/global-authorize-url'); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + showModal('error', 'Error', err.detail || 'Could not retrieve authorization URL.'); + startAuthFlowGlobalBtn.disabled = false; + startAuthFlowGlobalBtn.innerHTML = 'Authorize with Dropbox'; + return; + } + const data = await resp.json(); + // Signal to the callback that global credentials should be used for the exchange + sessionStorage.setItem('dropbox_use_global_creds', 'true'); + window.location.href = data.authorize_url; + } catch (err) { + showModal('error', 'Network Error', err.message || 'Unknown error'); + startAuthFlowGlobalBtn.disabled = false; + startAuthFlowGlobalBtn.innerHTML = 'Authorize with Dropbox'; + } + }); + } + // Start Authentication Flow button click - startAuthFlowBtn.addEventListener('click', function() { + if (startAuthFlowBtn) { + startAuthFlowBtn.addEventListener('click', function() { const appKey = document.getElementById('app-key').value.trim(); const appSecret = appSecretInput.value.trim(); - const redirectUri = window.location.origin + "/dropbox-callback"; + const redirectUri = dropboxCallbackUrl; if (!appKey) { showModal('error', 'Validation Error', 'Please enter your App Key'); @@ -362,6 +423,7 @@ document.addEventListener('DOMContentLoaded', function() { // Redirect the user to the Dropbox login page window.location.href = authUrl; }); + } // end if (startAuthFlowBtn) // Test Token button click (admin mode only) if (testTokenBtn) { diff --git a/frontend/templates/dropbox_callback.html b/frontend/templates/dropbox_callback.html index fd7b0f31..bedddc70 100644 --- a/frontend/templates/dropbox_callback.html +++ b/frontend/templates/dropbox_callback.html @@ -59,6 +59,40 @@
+ + +

Configuration for Worker Nodes

@@ -94,6 +128,7 @@ document.addEventListener('DOMContentLoaded', function() { const code = "{{ code }}"; // Get credentials from session storage (these take precedence over server-provided values) + const useGlobalCreds = sessionStorage.getItem('dropbox_use_global_creds') === 'true'; const appKey = sessionStorage.getItem('dropbox_app_key') || "{{ app_key_value }}"; const appSecret = sessionStorage.getItem('dropbox_app_secret') || "{{ app_secret_value }}"; const folderPath = sessionStorage.getItem('dropbox_folder_path') || "{{ folder_path }}" || '/Documents/Uploads'; @@ -107,20 +142,100 @@ document.addEventListener('DOMContentLoaded', function() { } } - const redirectUri = window.location.origin + "/dropbox-callback"; + // Use server-provided callback URL (respects PUBLIC_BASE_URL) with fallback to window.location.origin + const redirectUri = {{ callback_url | tojson }} || (window.location.origin + "/dropbox-callback"); // Automatically exchange the code for a refresh token if (code) { - if (!appKey || !appSecret) { - showError("Missing App Key or App Secret. Please go back to the setup page and try again."); - return; + if (useGlobalCreds) { + // Global credentials: the server handles the exchange using the admin app secret + exchangeCodeGlobal(code, redirectUri); + } else { + if (!appKey || !appSecret) { + showError("Missing App Key or App Secret. Please go back to the setup page and try again."); + return; + } + exchangeCode(code, appKey, appSecret, redirectUri); } - - exchangeCode(code, appKey, appSecret, redirectUri); } else { showError("No authorization code was found in the URL"); } + function exchangeCodeGlobal(code, redirectUri) { + const formData = new FormData(); + formData.append('code', code); + formData.append('redirect_uri', redirectUri); + + document.getElementById('processing-message').innerHTML = + '

Exchanging authorization code using shared credentials…

'; + + fetch('/api/dropbox/exchange-token-global', { + method: 'POST', + body: formData + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to exchange token'); + }); + } + return response.json(); + }) + .then(data => { + if (data.refresh_token) { + const resolvedAppKey = data.app_key || ''; + if (integrationId) { + const creds = { + refresh_token: data.refresh_token, + // Store public app_key with the integration (no secret stored browser-side) + app_key: resolvedAppKey, + // Flag so the backend knows to use global app_secret for future operations + use_global_app_secret: true, + }; + + const body = { credentials: creds }; + + document.getElementById('processing-message').innerHTML = + '

Saving credentials to your integration…

'; + + return fetch(`/api/integrations/${integrationId}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }).then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error('Failed to save credentials: ' + (err.detail || 'Unknown error')); + }); + } + return response.json(); + }).then(() => { + // Clean up + sessionStorage.removeItem('dropbox_use_global_creds'); + sessionStorage.removeItem('oauth_integration_id'); + + document.getElementById('processing-message').innerHTML = + '

✓ Dropbox authorized successfully!

' + + '

Redirecting to Integrations…

'; + document.querySelector('.animate-spin').parentNode.classList.add('hidden'); + setTimeout(() => { window.location.href = '/integrations'; }, 2000); + }); + } + // Global admin flow — the exchange-token-global endpoint is intended for + // per-user integrations, so reaching here without an integrationId is unexpected. + console.warn('dropbox_callback: global creds flow reached without integration_id'); + sessionStorage.removeItem('dropbox_use_global_creds'); + showSuccess(data.refresh_token, resolvedAppKey, '', folderPath); + setTimeout(() => { window.location.href = '/status'; }, 10000); + } else { + throw new Error('No refresh token was received from the server'); + } + }) + .catch(error => { + showError(error.message); + }); + } + function exchangeCode(code, appKey, appSecret, redirectUri) { const formData = new FormData(); formData.append('client_id', appKey); @@ -173,18 +288,21 @@ document.addEventListener('DOMContentLoaded', function() { } return response.json(); }).then(() => { - // Clean up + // Clean up session storage sessionStorage.removeItem('dropbox_app_key'); sessionStorage.removeItem('dropbox_app_secret'); sessionStorage.removeItem('dropbox_folder_path'); sessionStorage.removeItem('oauth_integration_id'); + sessionStorage.removeItem('dropbox_use_system_creds'); - // Show brief success then redirect to integrations - document.getElementById('processing-message').innerHTML = - '

✓ Dropbox authorized successfully!

' + - '

Redirecting to Integrations...

'; + // Hide processing spinner, show success document.querySelector('.animate-spin').parentNode.classList.add('hidden'); - setTimeout(() => { window.location.href = '/integrations'; }, 2000); + document.getElementById('processing-message').innerHTML = + '

✓ Dropbox authorized successfully!

'; + document.getElementById('success-container').classList.remove('hidden'); + + // Show folder browser with the access token + initFolderBrowser(data.access_token, integrationId); }); } @@ -279,6 +397,128 @@ DROPBOX_FOLDER=${folderPath || '/Documents/Uploads'}`; }); } } + + // ── Folder browser ──────────────────────────────────────────────── + function escapeHtml(str) { + const div = document.createElement('div'); + div.appendChild(document.createTextNode(str)); + return div.innerHTML; + } + + function initFolderBrowser(accessToken, integrationId) { + const folderSelector = document.getElementById('folder-selector'); + if (!folderSelector || !integrationId) return; + + folderSelector.classList.remove('hidden'); + let currentPath = ''; + + const folderList = document.getElementById('folder-list'); + const breadcrumb = document.getElementById('folder-breadcrumb'); + const selectedInput = document.getElementById('selected-folder-path'); + const saveBtn = document.getElementById('save-folder-btn'); + const saveStatus = document.getElementById('folder-save-status'); + + function loadFolders(path) { + currentPath = path; + folderList.innerHTML = '

Loading folders…

'; + + const formData = new FormData(); + formData.append('access_token', accessToken); + formData.append('path', path); + + fetch('/api/dropbox/list-folders', { method: 'POST', body: formData }) + .then(r => r.json()) + .then(data => { + if (data.folders && data.folders.length > 0) { + folderList.innerHTML = data.folders.map(f => + `` + ).join(''); + + folderList.querySelectorAll('.folder-item').forEach(btn => { + btn.addEventListener('click', () => { + const p = btn.getAttribute('data-path'); + selectedInput.value = p; + loadFolders(p); + }); + }); + } else { + folderList.innerHTML = '
No subfolders found
'; + } + updateBreadcrumb(path); + }) + .catch(err => { + folderList.innerHTML = `
Failed to load folders: ${escapeHtml(err.message)}
`; + }); + } + + function updateBreadcrumb(path) { + const parts = path.split('/').filter(Boolean); + let html = ''; + let accumulated = ''; + for (const part of parts) { + accumulated += '/' + part; + html += `/`; + html += ``; + } + breadcrumb.innerHTML = html; + breadcrumb.querySelectorAll('.folder-nav-btn').forEach(btn => { + btn.addEventListener('click', () => { + const p = btn.getAttribute('data-path'); + selectedInput.value = p || '/'; + loadFolders(p); + }); + }); + } + + // Save selected folder to integration config + saveBtn.addEventListener('click', () => { + const folderPath = selectedInput.value.trim() || '/'; + saveBtn.disabled = true; + saveBtn.textContent = 'Saving…'; + + // Get the current integration config, update folder_path, then PUT back + fetch(`/api/integrations/${integrationId}`) + .then(r => r.json()) + .then(intg => { + const cfg = intg.config || {}; + // Update the correct folder key based on integration type + if (cfg.source_type) { + cfg.folder_path = folderPath; + } else { + cfg.folder = folderPath; + } + return fetch(`/api/integrations/${integrationId}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ config: cfg }), + }); + }) + .then(r => { + if (!r.ok) throw new Error('Failed to save folder'); + return r.json(); + }) + .then(() => { + saveStatus.textContent = '✓ Folder saved! Redirecting…'; + saveStatus.className = 'mt-2 text-sm text-green-600'; + saveStatus.classList.remove('hidden'); + saveBtn.textContent = 'Saved ✓'; + setTimeout(() => { window.location.href = '/integrations'; }, 1500); + }) + .catch(err => { + saveStatus.textContent = 'Error: ' + err.message; + saveStatus.className = 'mt-2 text-sm text-red-600'; + saveStatus.classList.remove('hidden'); + saveBtn.disabled = false; + saveBtn.textContent = 'Save Folder'; + }); + }); + + // Load root folders initially + loadFolders(''); + } }); {% endblock %} diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 51d5d48a..1d142a03 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -524,6 +524,7 @@ +
diff --git a/frontend/templates/google_drive.html b/frontend/templates/google_drive.html index 10a9aca9..43b46d73 100644 --- a/frontend/templates/google_drive.html +++ b/frontend/templates/google_drive.html @@ -167,14 +167,31 @@

Now enter your Client ID and Client Secret below, and we'll help you complete the OAuth flow. You can set the folder ID after authentication is complete.

-
- - + {% if user_mode and has_system_credentials %} + +
+
+ {% endif %} -
- - +
+
+
+ + +
+ +
+ + +
+
@@ -430,6 +447,9 @@ GOOGLE_DRIVE_FOLDER_ID={{ folder_id|default('YOUR_FOLDER_ID', true) }} {% endblock %} diff --git a/frontend/templates/profile.html b/frontend/templates/profile.html index b3c47219..cc6144d7 100644 --- a/frontend/templates/profile.html +++ b/frontend/templates/profile.html @@ -343,9 +343,203 @@ + +
+

+ {{ _("sessions.security_heading") }} +

+

+ {{ _("sessions.security_subtitle") }} +

+ + +
+ + +
+ + +
+ +

{{ _("sessions.no_other_sessions") }}

+
+ + + + + + +
+
+{% endblock %} diff --git a/frontend/templates/signup.html b/frontend/templates/signup.html index 1cb3a043..6f83d664 100644 --- a/frontend/templates/signup.html +++ b/frontend/templates/signup.html @@ -32,6 +32,14 @@ error: '', async submit() { this.error = ''; + if (this.username.length < 3 || this.username.length > 64) { + this.error = 'Username must be between 3 and 64 characters.'; + return; + } + if (!/^[a-zA-Z0-9_-]+$/.test(this.username)) { + this.error = 'Username may only contain letters, numbers, hyphens, and underscores. Dots and other special characters are not allowed.'; + return; + } if (this.password !== this.password_confirm) { this.error = 'Passwords do not match.'; return; @@ -58,7 +66,12 @@ } } else { const data = await resp.json(); - this.error = data.detail || 'Registration failed. Please try again.'; + const detail = data.detail; + if (Array.isArray(detail)) { + this.error = detail.map(e => e.msg || String(e)).join(' ') || 'Registration failed. Please try again.'; + } else { + this.error = detail || 'Registration failed. Please try again.'; + } } } catch(e) { this.error = 'Network error. Please try again.'; diff --git a/frontend/templates/system_reset.html b/frontend/templates/system_reset.html new file mode 100644 index 00000000..452a8142 --- /dev/null +++ b/frontend/templates/system_reset.html @@ -0,0 +1,261 @@ +{% extends "base.html" %} +{% block title %}{{ _("system_reset.page_title") }} - DocuElevate{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +
+ + +
+
+

+ + {{ _("system_reset.heading") }} +

+
+

+ {{ _("system_reset.subtitle") }} +

+
+ + + {% if factory_reset_on_startup %} + + {% endif %} + + + + +
+ + +
+
+
+ +
+
+

{{ _("system_reset.full_reset_title") }}

+

{{ _("system_reset.full_reset_subtitle") }}

+
+
+ +
+

{{ _("system_reset.full_reset_desc") }}

+
    +
  • {{ _("system_reset.full_reset_item_db") }}
  • +
  • {{ _("system_reset.full_reset_item_files") }}
  • +
  • {{ _("system_reset.full_reset_item_cache") }}
  • +
  • {{ _("system_reset.full_reset_item_settings_kept") }}
  • +
+
+ +
+ + +

{{ _("system_reset.type_delete_help") }}

+ + +
+
+ + +
+
+
+ +
+
+

{{ _("system_reset.reimport_title") }}

+

{{ _("system_reset.reimport_subtitle") }}

+
+
+ +
+

{{ _("system_reset.reimport_desc") }}

+
    +
  1. {{ _("system_reset.reimport_step_1") }}
  2. +
  3. {{ _("system_reset.reimport_step_2") }}
  4. +
  5. {{ _("system_reset.reimport_step_3") }}
  6. +
+

{{ _("system_reset.reimport_note") }}

+
+ +
+ + +

{{ _("system_reset.type_reimport_help") }}

+ + +
+
+
+ + +
+
+ +
+

+

+      
+
+
+
+ + +{% endblock %} diff --git a/frontend/translations/en.json b/frontend/translations/en.json index 19e3a668..dbb9f8fb 100644 --- a/frontend/translations/en.json +++ b/frontend/translations/en.json @@ -316,6 +316,7 @@ "admin_users.total_no_users": "No users", "admin_users.total_one_user": "1 user", "api_tokens.col_created": "Created", + "api_tokens.col_expires": "Expires", "api_tokens.col_last_ip": "Last IP", "api_tokens.col_last_used": "Last Used", "api_tokens.col_name": "Name", @@ -327,6 +328,14 @@ "api_tokens.create_heading": "Create New Token", "api_tokens.create_token": "Create Token", "api_tokens.creating": "Creating…", + "api_tokens.delete": "Delete", + "api_tokens.delete_confirm": "Permanently delete this revoked token? This cannot be undone.", + "api_tokens.delete_prefix": "Permanently delete token", + "api_tokens.expires_at_label": "Expires (optional)", + "api_tokens.expires_at_placeholder": "e.g. 30, 90, 365 days", + "api_tokens.expires_in_days_label": "Token lifetime (days)", + "api_tokens.expires_never": "Never", + "api_tokens.expires_on": "Expires", "api_tokens.heading": "API Tokens", "api_tokens.intro": "Create personal API tokens to interact with the DocuElevate API programmatically. Use tokens for webhook uploads, CI/CD pipelines, or any script that needs to upload or retrieve documents.", "api_tokens.loading_tokens": "Loading tokens…", @@ -334,9 +343,13 @@ "api_tokens.no_tokens_heading": "No API tokens yet", "api_tokens.no_tokens_help": "Create your first token above to get started.", "api_tokens.page_title": "API Tokens – DocuElevate", + "api_tokens.reactivate": "Reactivate", + "api_tokens.reactivate_confirm": "Reactivate this token? It will be usable again immediately.", + "api_tokens.reactivate_prefix": "Reactivate token", "api_tokens.revoke": "Revoke", "api_tokens.revoke_prefix": "Revoke token", "api_tokens.status_active": "Active", + "api_tokens.status_expired": "Expired", "api_tokens.status_revoked": "Revoked", "api_tokens.table_aria": "API Tokens", "api_tokens.token_created": "Token created successfully!", @@ -608,6 +621,46 @@ "dashboard.title": "Dashboard", "dashboard.total_files": "Total Files", "dashboard.welcome": "Welcome to DocuElevate", + "devices.col_created": "Connected", + "devices.col_device": "Device", + "devices.col_last_ip": "Last IP", + "devices.col_last_seen": "Last Seen", + "devices.col_last_used": "Last Used", + "devices.col_platform": "Platform", + "devices.col_push_token": "Push Token", + "devices.col_status": "Status", + "devices.col_token_prefix": "Token Prefix", + "devices.confirm_deactivate_device": "Remove this device? It will stop receiving push notifications.", + "devices.confirm_revoke_token": "Revoke access for this device? It will need to log in again.", + "devices.deactivate_device": "Remove", + "devices.delete_device": "Delete", + "devices.delete_device_confirm": "Permanently delete this inactive device? This cannot be undone.", + "devices.delete_token": "Delete", + "devices.delete_token_confirm": "Permanently delete this revoked token? This cannot be undone.", + "devices.device_deleted_success": "Device permanently deleted.", + "devices.device_removed_success": "Device removed successfully.", + "devices.heading": "Mobile Devices", + "devices.intro": "Manage your mobile app connections and registered devices. You can revoke access for individual devices here.", + "devices.loading": "Loading devices…", + "devices.mobile_tokens_description": "These tokens were created when you logged in via the mobile app or scanned a QR code. Revoking a token will sign the device out.", + "devices.mobile_tokens_heading": "Mobile App Tokens", + "devices.no_devices": "No registered devices", + "devices.no_devices_help": "Install the DocuElevate mobile app and log in to register a device for push notifications.", + "devices.no_mobile_tokens": "No mobile app tokens", + "devices.no_mobile_tokens_help": "Log in via the mobile app or scan a QR code to create a mobile token.", + "devices.page_title": "Devices – DocuElevate", + "devices.qr_login_cta": "Connect a new device via QR code", + "devices.reactivate_token": "Reactivate", + "devices.reactivate_token_confirm": "Reactivate this token? The device will be able to use it again immediately.", + "devices.registered_devices_description": "Devices registered for push notifications from the DocuElevate mobile app.", + "devices.registered_devices_heading": "Registered Devices", + "devices.revoke_token": "Revoke", + "devices.status_active": "Active", + "devices.status_inactive": "Inactive", + "devices.status_revoked": "Revoked", + "devices.token_deleted_success": "Token permanently deleted.", + "devices.token_reactivated_success": "Token reactivated successfully.", + "devices.token_revoked_success": "Device token revoked successfully.", "duplicates.file_id_label": "File ID", "duplicates.file_id_placeholder": "e.g. 42", "duplicates.find_btn": "Find", @@ -1152,6 +1205,7 @@ "nav.dark_mode": "Dark Mode", "nav.dashboard": "Dashboard", "nav.developer_docs": "Developer Docs", + "nav.devices": "Devices", "nav.duplicates": "Duplicates", "nav.file_manager": "File Manager", "nav.files": "Files", @@ -1184,6 +1238,7 @@ "nav.skip_to_content": "Skip to main content", "nav.status": "Status", "nav.subscription": "Subscription", + "nav.system_reset": "System Reset", "nav.toggle_dark_mode": "Toggle dark mode", "nav.toggle_nav": "Toggle navigation menu", "nav.upload": "Upload", @@ -1451,6 +1506,20 @@ "profile.theme_system": "System Default", "profile.update_password": "Update Password", "profile.updating": "Updating…", + "qr_login.claimed_device": "Device: {device_name}", + "qr_login.claimed_message": "QR code login successful! Your mobile device is now connected.", + "qr_login.description": "Scan this QR code with the DocuElevate mobile app to log in instantly.", + "qr_login.expired_message": "This QR code has expired. Please generate a new one.", + "qr_login.generate_new": "Generate New QR Code", + "qr_login.heading": "Mobile App QR Login", + "qr_login.how_it_works": "How it works", + "qr_login.page_title": "QR Code Login – DocuElevate", + "qr_login.pending_message": "Waiting for mobile app to scan…", + "qr_login.step_1": "Open the DocuElevate app on your phone", + "qr_login.step_2": "Tap \"Scan QR Code\" on the login screen", + "qr_login.step_3": "Point your camera at this QR code", + "qr_login.subtitle": "Log in to the mobile app by scanning a QR code from this page.", + "qr_login.time_remaining": "Expires in {seconds} seconds", "queue.active_tasks": "Active Tasks", "queue.auto_refresh_1": "Auto-refreshes every", "queue.auto_refresh_2": "seconds", @@ -1512,6 +1581,25 @@ "search.saved_label": "Saved Searches", "search.saved_loading": "Loading...", "search.title": "Search Documents", + "sessions.active_sessions": "Active Sessions", + "sessions.confirm_revoke_all": "This will log you out of all other devices and browsers, and revoke all API tokens. Continue?", + "sessions.confirm_revoke_one": "Are you sure you want to end this session?", + "sessions.current_session": "This device", + "sessions.device_info": "Device", + "sessions.expires": "Expires", + "sessions.ip_address": "IP Address", + "sessions.last_active": "Last active", + "sessions.log_off_everywhere": "Log Off All Other Sessions", + "sessions.log_off_everywhere_desc": "End all other browser sessions and revoke all API tokens. Your current session will remain active.", + "sessions.no_other_sessions": "No other active sessions found.", + "sessions.qr_login_link": "Log in on mobile via QR code", + "sessions.revoke": "End Session", + "sessions.revoked_all_success": "All other sessions have been ended.", + "sessions.revoked_success": "Session ended successfully.", + "sessions.security_heading": "Security & Sessions", + "sessions.security_subtitle": "Manage your active sessions across devices and browsers.", + "sessions.session_lifetime": "Session lifetime: {days} days", + "sessions.started": "Started", "settings.audit_log_btn": "Audit Log", "settings.autocomplete_hint": "Type to search known values, or enter any custom value.", "settings.autocomplete_no_matches": "No matches — you can still type a custom value", @@ -1723,6 +1811,36 @@ "subscription.upgrade_info": "Upgrades take effect immediately. Downgrades are scheduled for the end of your current billing period.", "subscription.upgrade_to_prefix": "Upgrade to", "subscription.usage_heading": "Usage", + "system_reset.danger_desc": "The actions below will permanently destroy data. They cannot be undone. Application settings and configuration are preserved, but all documents, files, processing history, and audit logs will be deleted.", + "system_reset.danger_zone": "Danger Zone — Irreversible Actions", + "system_reset.full_reset_button": "Wipe All Data", + "system_reset.full_reset_desc": "Permanently deletes all user data from the database and removes all work files from disk. The application will be in its initial state after this operation.", + "system_reset.full_reset_item_cache": "All watch-folder caches and ingestion state", + "system_reset.full_reset_item_db": "All document records, processing logs, and audit history", + "system_reset.full_reset_item_files": "All original, processed, and temporary files on disk", + "system_reset.full_reset_item_settings_kept": "Application settings and configuration are preserved", + "system_reset.full_reset_subtitle": "Wipe everything and start fresh", + "system_reset.full_reset_title": "Full System Reset", + "system_reset.heading": "System Reset", + "system_reset.js_error_generic": "An error occurred. Please check the server logs for details.", + "system_reset.js_success_full": "System reset complete. All user data has been wiped.", + "system_reset.js_success_reimport": "Reset complete. Original files have been staged for re-import via the watch folder.", + "system_reset.page_title": "System Reset", + "system_reset.reimport_button": "Reset & Re-import", + "system_reset.reimport_desc": "Copies your original files to a special reimport folder, wipes everything, then lets the watch-folder mechanism re-process them as if they were freshly uploaded.", + "system_reset.reimport_note": "Re-imported files will go through the full processing pipeline with the same rate limits and backoff strategy as regular uploads.", + "system_reset.reimport_step_1": "Original files are copied to a dedicated reimport folder", + "system_reset.reimport_step_2": "All data (database + work files) is wiped clean", + "system_reset.reimport_step_3": "The reimport folder is configured as a watch folder for automatic re-ingestion", + "system_reset.reimport_subtitle": "Wipe and re-process all original files", + "system_reset.reimport_title": "Reset & Re-import", + "system_reset.startup_reset_active": "Factory Reset on Startup is ACTIVE", + "system_reset.startup_reset_desc": "FACTORY_RESET_ON_STARTUP is enabled. All user data is wiped every time the application starts.", + "system_reset.subtitle": "Reset DocuElevate to a clean, fresh state. All user data will be permanently deleted.", + "system_reset.type_delete": "Type DELETE to confirm", + "system_reset.type_delete_help": "You must type the word DELETE in capital letters to enable the reset button.", + "system_reset.type_reimport": "Type REIMPORT to confirm", + "system_reset.type_reimport_help": "You must type the word REIMPORT in capital letters to enable the button.", "terms.cookie_link": "Cookie Policy", "terms.heading": "Terms of Service", "terms.last_updated": "Last Updated:", diff --git a/helm/docuelevate/templates/beat-deployment.yaml b/helm/docuelevate/templates/beat-deployment.yaml new file mode 100644 index 00000000..3ed52251 --- /dev/null +++ b/helm/docuelevate/templates/beat-deployment.yaml @@ -0,0 +1,83 @@ +{{- /* + Celery Beat scheduler — publishes periodic tasks to the broker. + Exactly ONE replica must run; never scale this deployment. +*/ -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "docuelevate.fullname" . }}-beat + namespace: {{ .Release.Namespace }} + labels: + {{- include "docuelevate.labels" . | nindent 4 }} + app.kubernetes.io/component: beat +spec: + replicas: 1 + strategy: + type: Recreate # Prevent two Beat instances from running simultaneously + selector: + matchLabels: + {{- include "docuelevate.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: beat + template: + metadata: + labels: + {{- include "docuelevate.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: beat + {{- with .Values.beat.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "docuelevate.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.beat.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: beat + image: {{ include "docuelevate.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - celery + - -A + - app.celery_worker + - beat + - --loglevel=info + envFrom: + - configMapRef: + name: {{ include "docuelevate.fullname" . }}-config + - secretRef: + name: {{ include "docuelevate.fullname" . }}-secret + {{- with .Values.beat.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.beat.resources | nindent 12 }} + volumeMounts: + - name: workdir + mountPath: /workdir + volumes: + - name: workdir + {{- if .Values.workdir.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.beat.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.beat.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.beat.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/docuelevate/templates/worker-deployment.yaml b/helm/docuelevate/templates/worker-deployment.yaml index c698b0f0..efb09f1c 100644 --- a/helm/docuelevate/templates/worker-deployment.yaml +++ b/helm/docuelevate/templates/worker-deployment.yaml @@ -42,7 +42,6 @@ spec: - -A - app.celery_worker - worker - - -B - --loglevel=info - -Q - document_processor,default,celery diff --git a/helm/docuelevate/values.yaml b/helm/docuelevate/values.yaml index 7ed056b0..de112df5 100644 --- a/helm/docuelevate/values.yaml +++ b/helm/docuelevate/values.yaml @@ -121,10 +121,10 @@ api: type: ClusterIP port: 8000 - # Liveness / readiness probes + # Liveness / readiness probes (unauthenticated endpoints for kubelet) livenessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/live port: 8000 initialDelaySeconds: 30 periodSeconds: 20 @@ -132,7 +132,7 @@ api: readinessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/ready port: 8000 initialDelaySeconds: 15 periodSeconds: 10 @@ -191,7 +191,36 @@ worker: drop: ["ALL"] # --------------------------------------------------------------------------- -# Shared workdir volume (api + worker mount the same PVC) +# Celery Beat scheduler (singleton — always exactly 1 replica) +# Beat publishes periodic tasks; workers consume them from the broker. +# --------------------------------------------------------------------------- +beat: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + +# --------------------------------------------------------------------------- +# Shared workdir volume (api + worker + beat mount the same PVC) # --------------------------------------------------------------------------- workdir: persistence: diff --git a/migrations/env.py b/migrations/env.py index a0bc2f99..14566b67 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -24,6 +24,7 @@ from app.models import ( # noqa: F401 ApplicationSettings, AuditLog, BackupRecord, + ClassificationRuleModel, ComplianceTemplate, DocumentMetadata, FileProcessingStep, diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 00000000..fb8a9c55 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,40 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """${message}.""" + # Use ``op.batch_alter_table()`` for SQLite compatibility. + # Always check whether the table/column already exists before altering + # to keep migrations idempotent (safe to re-run). + # + # Example – add a column only if it is missing: + # + # conn = op.get_bind() + # inspector = sa.inspect(conn) + # if "my_table" in inspector.get_table_names(): + # existing = {c["name"] for c in inspector.get_columns("my_table")} + # if "new_col" not in existing: + # with op.batch_alter_table("my_table") as batch_op: + # batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True)) + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Reverse ${message}.""" + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/037_add_user_sessions_and_qr_challenges.py b/migrations/versions/037_add_user_sessions_and_qr_challenges.py new file mode 100644 index 00000000..9610f56d --- /dev/null +++ b/migrations/versions/037_add_user_sessions_and_qr_challenges.py @@ -0,0 +1,72 @@ +"""Add user_sessions and qr_login_challenges tables. + +Adds server-side session tracking (user_sessions) for the "log off +everywhere" feature and per-session revocation, and QR login challenges +(qr_login_challenges) for secure mobile app authentication via QR code. + +Revision ID: 037_add_user_sessions_and_qr_challenges +Revises: 036_add_document_translation_fields +Create Date: 2026-03-16 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "037_add_user_sessions_and_qr_challenges" +down_revision: Union[str, None] = "036_add_document_translation_fields" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Create user_sessions and qr_login_challenges tables.""" + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + + if "user_sessions" not in existing_tables: + op.create_table( + "user_sessions", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("session_token", sa.String(128), nullable=False, unique=True, index=True), + sa.Column("user_id", sa.String(), nullable=False, index=True), + sa.Column("ip_address", sa.String(45), nullable=True), + sa.Column("user_agent", sa.String(512), nullable=True), + sa.Column("device_info", sa.String(255), nullable=True), + sa.Column("is_revoked", sa.Boolean(), nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("last_active_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + ) + + if "qr_login_challenges" not in existing_tables: + op.create_table( + "qr_login_challenges", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("challenge_token", sa.String(128), nullable=False, unique=True, index=True), + sa.Column("user_id", sa.String(), nullable=False, index=True), + sa.Column("is_claimed", sa.Boolean(), nullable=False, server_default="0"), + sa.Column("is_cancelled", sa.Boolean(), nullable=False, server_default="0"), + sa.Column("created_by_ip", sa.String(45), nullable=True), + sa.Column("claimed_by_ip", sa.String(45), nullable=True), + sa.Column("device_name", sa.String(255), nullable=True), + sa.Column("issued_token_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + """Drop user_sessions and qr_login_challenges tables.""" + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + + if "qr_login_challenges" in existing_tables: + op.drop_table("qr_login_challenges") + + if "user_sessions" in existing_tables: + op.drop_table("user_sessions") diff --git a/migrations/versions/038_add_api_token_expires_at.py b/migrations/versions/038_add_api_token_expires_at.py new file mode 100644 index 00000000..20d932aa --- /dev/null +++ b/migrations/versions/038_add_api_token_expires_at.py @@ -0,0 +1,43 @@ +"""Add expires_at column to api_tokens table. + +Allows API tokens to be issued with an optional lifetime. If ``expires_at`` +is set, the token is automatically rejected after that timestamp. + +Revision ID: 038_add_api_token_expires_at +Revises: 037_add_user_sessions_and_qr_challenges +Create Date: 2026-03-18 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "038_add_api_token_expires_at" +down_revision: Union[str, None] = "037_add_user_sessions_and_qr_challenges" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Add expires_at column to api_tokens (idempotent).""" + conn = op.get_bind() + inspector = sa.inspect(conn) + if "api_tokens" not in inspector.get_table_names(): + return + existing_columns = {col["name"] for col in inspector.get_columns("api_tokens")} + if "expires_at" not in existing_columns: + op.add_column( + "api_tokens", + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + """Remove expires_at column from api_tokens.""" + conn = op.get_bind() + inspector = sa.inspect(conn) + if "api_tokens" not in inspector.get_table_names(): + return + existing_columns = {col["name"] for col in inspector.get_columns("api_tokens")} + if "expires_at" in existing_columns: + op.drop_column("api_tokens", "expires_at") diff --git a/migrations/versions/039_add_classification_rules.py b/migrations/versions/039_add_classification_rules.py new file mode 100644 index 00000000..66e1aaf2 --- /dev/null +++ b/migrations/versions/039_add_classification_rules.py @@ -0,0 +1,46 @@ +"""Add classification_rules table for custom document classification rules. + +Revision ID: 039_add_classification_rules +Revises: 038_add_api_token_expires_at +Create Date: 2026-03-17 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "039_add_classification_rules" +down_revision: Union[str, None] = "038_add_api_token_expires_at" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Create classification_rules table.""" + op.create_table( + "classification_rules", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("owner_id", sa.String(), nullable=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("category", sa.String(100), nullable=False), + sa.Column("rule_type", sa.String(50), nullable=False), + sa.Column("pattern", sa.String(1000), nullable=False), + sa.Column("priority", sa.Integer(), nullable=False, server_default="0"), + sa.Column("case_sensitive", sa.Boolean(), nullable=False, server_default="0"), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default="1"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("owner_id", "name", name="uq_classification_rules_owner_name"), + ) + op.create_index("ix_classification_rules_id", "classification_rules", ["id"]) + op.create_index("ix_classification_rules_owner_id", "classification_rules", ["owner_id"]) + op.create_index("ix_classification_rules_category", "classification_rules", ["category"]) + + +def downgrade() -> None: + """Drop classification_rules table.""" + op.drop_index("ix_classification_rules_category", "classification_rules") + op.drop_index("ix_classification_rules_owner_id", "classification_rules") + op.drop_index("ix_classification_rules_id", "classification_rules") + op.drop_table("classification_rules") diff --git a/migrations/versions/037_add_automation_hooks.py b/migrations/versions/040_add_automation_hooks.py similarity index 86% rename from migrations/versions/037_add_automation_hooks.py rename to migrations/versions/040_add_automation_hooks.py index 6fbd995f..3d9b778c 100644 --- a/migrations/versions/037_add_automation_hooks.py +++ b/migrations/versions/040_add_automation_hooks.py @@ -1,7 +1,7 @@ """Add automation_hooks table for Zapier / Make.com webhook subscriptions. -Revision ID: 037_add_automation_hooks -Revises: 036_add_document_translation_fields +Revision ID: 040_add_automation_hooks +Revises: 039_add_classification_rules Create Date: 2026-03-09 """ @@ -10,8 +10,8 @@ from typing import Union import sqlalchemy as sa from alembic import op -revision: str = "037_add_automation_hooks" -down_revision: Union[str, None] = "036_add_document_translation_fields" +revision: str = "040_add_automation_hooks" +down_revision: Union[str, None] = "039_add_classification_rules" depends_on: Union[str, None] = None diff --git a/mobile/README.md b/mobile/README.md index 815ab301..601d71c3 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -163,6 +163,16 @@ The app registers itself as a share target so any file can be sent directly to D The root layout (`app/_layout.tsx`) listens for incoming URLs via `Linking.addEventListener` (warm start) and `Linking.getInitialURL()` (cold start). If the URL uses the `docuelevate://` scheme it is automatically rewritten to `file://` before being forwarded. Incoming files are stored in `ShareContext` and automatically uploaded by `UploadScreen`. +#### Handling "unmatched route" errors from "Open In…" + +iOS sometimes delivers the file path under the `docuelevate://` scheme: + +``` +docuelevate://private/var/mobile/Library/Mobile Documents/…/Invoice.pdf +``` + +expo-router strips the scheme and tries to match `/private/var/mobile/…` as an in-app route. The catch-all `app/+not-found.tsx` intercepts this, detects the filesystem-path pattern, adds the file directly to `ShareContext`, and redirects to the Upload tab. `UploadScreen` picks up the pending file and begins uploading automatically. The `Linking` listener in the root layout may also fire for the same URL; `ShareContext` deduplicates by URI to prevent double uploads. + **Supported iOS file types:** PDF, images (JPEG / PNG / GIF / BMP / TIFF / WebP), plain text, Word (`.docx`, `.doc`), Excel (`.xlsx`, `.xls`), PowerPoint (`.pptx`, `.ppt`), and any other file (`public.data`). To use the share sheet: @@ -174,6 +184,10 @@ To use the share sheet: > **Note:** `CFBundleDocumentTypes` with `LSHandlerRank: Alternate` means DocuElevate appears in the share sheet as an option but does **not** become the default app for any file type. +#### iOS Action Extension (future enhancement) + +Apps like DeepL ("Translate in DeepL") appear as **Action Extensions** in the iOS share sheet, which requires a separate Xcode target and native Swift code. This is planned as a future enhancement. The current `CFBundleDocumentTypes` approach places DocuElevate in the "Open With" row of the share sheet. + ### Android – how it works `app.json` declares `intentFilters` for `ACTION_SEND` and `ACTION_SEND_MULTIPLE` with `mimeType: "*/*"`. When a user shares a file from another app and selects DocuElevate, Android delivers the content URI through the share intent, which is captured via `Linking.getInitialURL()` and processed the same way as on iOS. diff --git a/mobile/app.json b/mobile/app.json index 1aa4aef0..0e755fed 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -19,12 +19,12 @@ "bundleIdentifier": "org.docuelevate.mobile", "appleTeamId": "975U2ZESBM", "infoPlist": { - "NSCameraUsageDescription": "DocuElevate uses the camera to capture documents for upload.", + "NSCameraUsageDescription": "DocuElevate uses the camera to scan QR codes for login and to capture documents for upload.", "NSPhotoLibraryUsageDescription": "DocuElevate accesses your photo library to select documents for upload.", "NSPhotoLibraryAddUsageDescription": "DocuElevate saves scanned documents to your photo library.", - "UIBackgroundModes": ["fetch", "remote-notification"], + "UIBackgroundModes": ["remote-notification"], "ITSAppUsesNonExemptEncryption": false, - "LSSupportsOpeningDocumentsInPlace": true, + "LSSupportsOpeningDocumentsInPlace": false, "CFBundleDocumentTypes": [ { "CFBundleTypeName": "All Documents", @@ -86,7 +86,29 @@ "expo-build-properties", { "ios": { - "buildReactNativeFromSource": true + "buildReactNativeFromSource": true, + "privacyManifests": { + "NSPrivacyAccessedAPITypes": [ + { + "NSPrivacyAccessedAPIType": "NSPrivacyAccessedAPICategoryUserDefaults", + "NSPrivacyAccessedAPITypeReasons": ["CA92.1"] + }, + { + "NSPrivacyAccessedAPIType": "NSPrivacyAccessedAPICategoryFileTimestamp", + "NSPrivacyAccessedAPITypeReasons": ["C617.1"] + }, + { + "NSPrivacyAccessedAPIType": "NSPrivacyAccessedAPICategoryDiskSpace", + "NSPrivacyAccessedAPITypeReasons": ["E174.1"] + }, + { + "NSPrivacyAccessedAPIType": "NSPrivacyAccessedAPICategorySystemBootTime", + "NSPrivacyAccessedAPITypeReasons": ["35F9.1"] + } + ], + "NSPrivacyCollectedDataTypes": [], + "NSPrivacyTracking": false + } } } ], @@ -101,7 +123,7 @@ [ "expo-camera", { - "cameraPermission": "DocuElevate uses the camera to capture documents for upload." + "cameraPermission": "DocuElevate uses the camera to scan QR codes for login and to capture documents for upload." } ], "expo-document-picker", diff --git a/mobile/app/(auth)/_layout.tsx b/mobile/app/(auth)/_layout.tsx index 7040faec..e7288998 100644 --- a/mobile/app/(auth)/_layout.tsx +++ b/mobile/app/(auth)/_layout.tsx @@ -12,6 +12,7 @@ export default function AuthLayout() { + ); } diff --git a/mobile/app/(auth)/qr-scanner.tsx b/mobile/app/(auth)/qr-scanner.tsx new file mode 100644 index 00000000..b6c018fb --- /dev/null +++ b/mobile/app/(auth)/qr-scanner.tsx @@ -0,0 +1,4 @@ +/** + * QR scanner route – camera-based QR code scanning for mobile login. + */ +export { default } from "../../src/screens/QRScannerScreen"; diff --git a/mobile/app/(tabs)/_layout.tsx b/mobile/app/(tabs)/_layout.tsx index c115f7de..7068f4d2 100644 --- a/mobile/app/(tabs)/_layout.tsx +++ b/mobile/app/(tabs)/_layout.tsx @@ -11,10 +11,15 @@ import { Ionicons } from "@expo/vector-icons"; import React from "react"; import { usePushNotifications } from "../../src/hooks/usePushNotifications"; import { useAuth } from "../../src/context/AuthContext"; +import { useLocale, t } from "../../src/i18n"; export default function TabLayout() { const { isAuthenticated } = useAuth(); usePushNotifications(isAuthenticated); + // Subscribe to language changes so tab labels re-render when the language + // is switched. The `lang` variable is intentionally unused – its only + // purpose is to make this component a consumer of LocaleContext. + useLocale(); return ( ( ), @@ -48,23 +53,32 @@ export default function TabLayout() { ( ), - headerTitle: "My Documents", + headerTitle: t("files.title"), }} /> ( ), - headerTitle: "Profile", + headerTitle: t("tabs.profile"), + }} + /> + {/* File detail screen – hidden from tab bar, accessed via navigation */} + diff --git a/mobile/app/(tabs)/file-detail.tsx b/mobile/app/(tabs)/file-detail.tsx new file mode 100644 index 00000000..ad30c207 --- /dev/null +++ b/mobile/app/(tabs)/file-detail.tsx @@ -0,0 +1,4 @@ +/** + * File detail route – displays processing status and logs for a single file. + */ +export { default } from "../../src/screens/FileDetailScreen"; diff --git a/mobile/app/+not-found.tsx b/mobile/app/+not-found.tsx new file mode 100644 index 00000000..d88f1999 --- /dev/null +++ b/mobile/app/+not-found.tsx @@ -0,0 +1,154 @@ +/** + * Catch-all "not found" route for expo-router. + * + * This screen intercepts two different situations: + * + * 1. **iOS "Open In…" / share sheet** — iOS delivers files to the app via a + * `docuelevate://` URL. expo-router strips the custom scheme and + * tries to match the raw filesystem path (e.g. + * `/private/var/mobile/Library/…/file.pdf`) as an in-app route. Because + * no such route exists, expo-router previously threw "unmatched route + * docuelevate://…" and the upload never happened. + * + * This screen detects the filesystem-path pattern, adds the file directly + * to `ShareContext`, and redirects to the Upload tab. `UploadScreen` + * picks up the pending file and begins uploading automatically. + * + * The `Linking` listener in `_layout.tsx` may also fire for the same URL; + * `ShareContext.addPendingFile` deduplicates by URI so the file is only + * uploaded once. + * + * 2. **Any other unmatched in-app route** — redirect silently to the root so + * the user isn't left on a blank error page. + */ + +import { usePathname, useRouter } from "expo-router"; +import React, { useEffect } from "react"; +import { ActivityIndicator, StyleSheet, View } from "react-native"; +import { useShare } from "../src/context/ShareContext"; +import { mimeTypeFromFilename } from "../src/utils/mimeTypes"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * First path-segment names that identify iOS/Android sandbox filesystem paths. + * These can never be expo-router route-group names, so their presence is a + * strong positive signal that the URL is a shared file rather than a route. + * + * iOS: /private/var/mobile/… → "private" + * /var/mobile/… → "var" (symlink to /private/var/mobile) + * /tmp/… → "tmp" + * Android: /data/user/0/… → "data" + * /storage/emulated/0/… → "storage" + */ +const FS_PATH_ROOTS = ["private", "var", "tmp", "data", "storage"]; + +/** + * Route-group / special-file prefixes that identify genuine in-app routes + * rather than filesystem path segments. + * + * ⚠️ Keep this list in sync with the top-level entries in the `app/` + * directory. Add an entry here if you add a new top-level route group + * that does **not** use the parentheses convention. + */ +const IN_APP_ROUTE_PREFIXES = [ + "(auth)", // app/(auth)/ + "(tabs)", // app/(tabs)/ + "_", // expo-router special files (_layout, _sitemap, …) + "+", // expo-router special files (+not-found, …) + "--", // Expo Go development proxy prefix +]; + +/** + * Return `true` when `pathname` looks like a filesystem path delivered by iOS + * "Open In…" (e.g. `/private/var/mobile/Library/…/file.pdf`) rather than a + * legitimate in-app route. + * + * Detection strategy: + * 1. **Positive check** – if the first path segment matches a known device + * filesystem root (see `FS_PATH_ROOTS`), it is definitely a file path. + * 2. **Fallback negative check** – if the path does not start with any known + * in-app route prefix (see `IN_APP_ROUTE_PREFIXES`), treat it as a file + * path. This is a heuristic but safe because expo-router route groups + * always use parentheses (e.g. `(auth)`, `(tabs)`). + */ +function looksLikeFilePath(pathname: string): boolean { + const stripped = pathname.replace(/^\/+/, ""); + if (stripped.length === 0) return false; + + // Positive signal: path starts with a known device filesystem root segment. + const firstSegment = stripped.split("/")[0]; + if (FS_PATH_ROOTS.includes(firstSegment)) return true; + + // Fallback: paths that start with a known in-app route prefix are routes. + return !IN_APP_ROUTE_PREFIXES.some((prefix) => stripped.startsWith(prefix)); +} + +/** + * Extract a display filename from a filesystem path. + * Handles URL-encoded characters and strips query strings. + */ +function filenameFromPath(pathname: string): string { + try { + const decoded = decodeURIComponent(pathname); + const segments = decoded.split("/").filter(Boolean); + const last = segments[segments.length - 1] ?? "shared_file"; + return last.split("?")[0] || "shared_file"; + } catch { + return "shared_file"; + } +} + +// --------------------------------------------------------------------------- +// Screen component +// --------------------------------------------------------------------------- + +export default function NotFoundScreen() { + const pathname = usePathname(); + const router = useRouter(); + const { addPendingFile } = useShare(); + + // Guard: track which pathname has been handled so the effect does not + // re-fire when `router` or `addPendingFile` change identity mid-navigation. + const handledRef = React.useRef(null); + + useEffect(() => { + if (handledRef.current === pathname) return; // already handled + handledRef.current = pathname; + + if (looksLikeFilePath(pathname)) { + // Filesystem path from iOS "Open In…" – add the file to ShareContext + // and redirect to the Upload tab. UploadScreen will pick up the + // pending file and begin uploading automatically. + // + // The pathname from expo-router is the raw filesystem path + // (e.g. "/private/var/mobile/Library/…/file.pdf"). Reconstruct a + // file:// URI so the upload logic can read the file. + const fileUri = `file://${pathname}`; + const filename = filenameFromPath(pathname); + addPendingFile({ uri: fileUri, filename, mimeType: mimeTypeFromFilename(filename) }); + router.replace("/(tabs)/"); + } else { + // Truly unknown in-app route – fall back to the root redirect. + router.replace("/"); + } + }, [pathname, router, addPendingFile]); + + // Show a brief spinner while the redirect is in flight. + return ( + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + alignItems: "center", + justifyContent: "center", + backgroundColor: "#f9fafb", + }, +}); diff --git a/mobile/app/_layout.tsx b/mobile/app/_layout.tsx index 3d328673..0feda68f 100644 --- a/mobile/app/_layout.tsx +++ b/mobile/app/_layout.tsx @@ -9,6 +9,11 @@ * sheet (CFBundleDocumentTypes) or Android via a SEND intent, the incoming * file:// / content:// URL is captured and forwarded to UploadScreen via * ShareContext. + * + * The companion `+not-found.tsx` handles the case where expo-router receives + * a `docuelevate://` URL with a filesystem path (from iOS "Open In…") and + * cannot match it to a route. It adds the file directly to ShareContext and + * redirects to the Upload tab so the file is uploaded transparently. */ import * as Linking from "expo-linking"; @@ -18,6 +23,8 @@ import { ActivityIndicator, StyleSheet, Text, View } from "react-native"; import { SafeAreaProvider } from "react-native-safe-area-context"; import { AuthProvider, useAuth } from "../src/context/AuthContext"; import { ShareProvider, useShare } from "../src/context/ShareContext"; +import { LocaleProvider, useLocale, isLanguageSupported } from "../src/i18n"; +import { mimeTypeFromFilename } from "../src/utils/mimeTypes"; // --------------------------------------------------------------------------- // Helpers @@ -26,6 +33,13 @@ import { ShareProvider, useShare } from "../src/context/ShareContext"; /** The custom URL scheme registered in app.json. */ const APP_SCHEME_PREFIX = "docuelevate://"; +/** + * Known deep-link path prefixes that should NOT be treated as shared files. + * These are in-app deep-link routes handled by their respective screens + * (e.g. QR login, OAuth callback). + */ +const DEEP_LINK_PATHS = ["qr-login", "callback"]; + /** Extract a display filename from a file:// or content:// URI. */ function filenameFromUri(uri: string): string { try { @@ -43,12 +57,18 @@ function filenameFromUri(uri: string): string { * URLs to ShareContext. Extracted as a module-level factory so the handler * itself is created once and can be easily unit-tested without a React context. * - * On iOS the Share Sheet / "Open In" action may deliver the file path under + * On iOS the Share Sheet / "Open In…" action may deliver the file path under * the app's custom URL scheme (`docuelevate://…/file.pdf`) instead of a plain * `file://` URL. When that happens we rewrite the URL to `file:///…` so the * upload logic can read the file normally. + * + * Note: expo-router also receives the same URL and will attempt to match it as + * an in-app route. When no route matches it renders `+not-found.tsx`, which + * adds the file to ShareContext directly and redirects to the Upload tab. + * Both this handler and `+not-found.tsx` call `addPendingFile`; + * `ShareContext` deduplicates by URI so the file is only uploaded once. */ -function makeUrlHandler(addPendingFile: (f: { uri: string; filename: string }) => void) { +function makeUrlHandler(addPendingFile: (f: { uri: string; filename: string; mimeType?: string }) => void) { return ({ url }: { url: string }) => { let fileUri = url; @@ -57,13 +77,22 @@ function makeUrlHandler(addPendingFile: (f: { uri: string; filename: string }) = // (expo-router groups always start with "("). if (url.startsWith(APP_SCHEME_PREFIX)) { const path = url.slice(APP_SCHEME_PREFIX.length); - if (path.length > 0 && !path.startsWith("(")) { + + // Skip known in-app deep-link paths (e.g. qr-login, callback). + // These are handled by their respective screens, not the share flow. + const pathBase = path.split("?")[0].replace(/^\/+/, ""); + if (DEEP_LINK_PATHS.includes(pathBase) || path.startsWith("(")) { + return; + } + + if (path.length > 0) { fileUri = "file:///" + path.replace(/^\/+/, ""); } } if (!fileUri.startsWith("file://") && !fileUri.startsWith("content://")) return; - addPendingFile({ uri: fileUri, filename: filenameFromUri(fileUri) }); + const filename = filenameFromUri(fileUri); + addPendingFile({ uri: fileUri, filename, mimeType: mimeTypeFromFilename(filename) }); }; } @@ -72,11 +101,22 @@ function makeUrlHandler(addPendingFile: (f: { uri: string; filename: string }) = // --------------------------------------------------------------------------- function AuthGuard() { - const { isLoading, isAuthenticated } = useAuth(); + const { isLoading, isAuthenticated, user } = useAuth(); const { addPendingFile } = useShare(); + const { setLang } = useLocale(); const segments = useSegments(); const router = useRouter(); + // Apply the server-side language preference whenever the user profile is + // loaded (on login or app resume). This syncs the language set on the + // desktop/web client to the mobile app. If the server language is not + // supported by the mobile app, we leave the current language unchanged. + useEffect(() => { + if (user?.preferred_language && isLanguageSupported(user.preferred_language)) { + void setLang(user.preferred_language); + } + }, [user?.preferred_language, setLang]); + // Listen for files shared from other apps (iOS Share Sheet / Android Intent). // Both cold-start (app was not running) and warm-start (app in background) // cases are handled. @@ -118,8 +158,11 @@ function AuthGuard() { return ( + + {/* +not-found handles unmatched routes such as iOS "Open In…" file paths */} + ); } @@ -131,11 +174,13 @@ function AuthGuard() { export default function RootLayout() { return ( - - - - - + + + + + + + ); } diff --git a/mobile/app/index.tsx b/mobile/app/index.tsx new file mode 100644 index 00000000..2146ea1b --- /dev/null +++ b/mobile/app/index.tsx @@ -0,0 +1,17 @@ +/** + * Root index route – redirects to the auth flow on launch. + * + * expo-router renders this when the "/" route is matched (i.e. on cold start). + * Without this file, a stale default scaffold page ("Hello World") can appear + * if one was left behind by a previous build or Expo CLI scaffolding. + * + * The redirect targets the (auth) group; the AuthGuard in _layout.tsx will + * immediately forward authenticated users to (tabs). + */ + +import { Redirect } from "expo-router"; +import React from "react"; + +export default function RootIndex() { + return ; +} diff --git a/mobile/eslint.config.js b/mobile/eslint.config.js new file mode 100644 index 00000000..7d4d4459 --- /dev/null +++ b/mobile/eslint.config.js @@ -0,0 +1 @@ +module.exports = require("eslint-config-expo/flat"); diff --git a/mobile/package-lock.json b/mobile/package-lock.json index 1073f1ff..fb89fda2 100644 --- a/mobile/package-lock.json +++ b/mobile/package-lock.json @@ -27,6 +27,7 @@ "expo-image-manipulator": "~14.0.8", "expo-image-picker": "~17.0.10", "expo-linking": "~8.0.11", + "expo-localization": "~17.0.8", "expo-notifications": "~0.32.16", "expo-router": "~6.0.23", "expo-secure-store": "~15.0.8", @@ -44,7 +45,7 @@ "devDependencies": { "@babel/core": "^7.24.0", "@types/react": "~19.1.0", - "eslint": "^8.57.0", + "eslint": "^9.0.0", "eslint-config-expo": "~10.0.0", "typescript": "^5.3.0" }, @@ -1633,37 +1634,40 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", "dev": true, "license": "MIT", "dependencies": { - "ajv": "^6.12.4", + "ajv": "^6.14.0", "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", + "espree": "^10.0.1", + "globals": "^14.0.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.5", "strip-json-comments": "^3.1.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/@eslint/js": { - "version": "8.57.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", - "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", + "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", "dev": true, "license": "MIT", "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" } }, "node_modules/@eslint/object-schema": { @@ -2313,22 +2317,6 @@ "node": ">=18.18.0" } }, - "node_modules/@humanwhocodes/config-array": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", - "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", - "deprecated": "Use @eslint/config-array instead", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanwhocodes/object-schema": "^2.0.3", - "debug": "^4.3.1", - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=10.10.0" - } - }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -2343,14 +2331,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", - "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", - "deprecated": "Use @eslint/object-schema instead", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/@humanwhocodes/retry": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", @@ -2667,44 +2647,6 @@ "@tybys/wasm-util": "^0.10.0" } }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/@nolyfill/is-core-module": { "version": "1.0.39", "resolved": "https://registry.npmjs.org/@nolyfill/is-core-module/-/is-core-module-1.0.39.tgz", @@ -5766,19 +5708,6 @@ "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", "license": "MIT" }, - "node_modules/doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "esutils": "^2.0.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/dotenv": { "version": "16.4.7", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", @@ -6068,60 +5997,63 @@ } }, "node_modules/eslint": { - "version": "8.57.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", - "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", - "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", + "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.1", - "@humanwhocodes/config-array": "^0.13.0", + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.5", + "@eslint/js": "9.39.4", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", + "cross-spawn": "^7.0.6", "debug": "^4.3.2", - "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", + "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", + "minimatch": "^3.1.5", "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" + "optionator": "^0.9.3" }, "bin": { "eslint": "bin/eslint.js" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { - "url": "https://opencollective.com/eslint" + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } } }, "node_modules/eslint-config-expo": { @@ -6260,191 +6192,6 @@ "eslint": ">=8.10" } }, - "node_modules/eslint-plugin-expo/node_modules/@eslint/eslintrc": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", - "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.14.0", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", - "minimatch": "^3.1.5", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-plugin-expo/node_modules/@eslint/js": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", - "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - } - }, - "node_modules/eslint-plugin-expo/node_modules/eslint": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", - "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.2", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.5", - "@eslint/js": "9.39.4", - "@eslint/plugin-kit": "^0.4.1", - "@humanfs/node": "^0.16.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@humanwhocodes/retry": "^0.4.2", - "@types/estree": "^1.0.6", - "ajv": "^6.14.0", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.6", - "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^8.0.0", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.5", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "jiti": "*" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - } - } - }, - "node_modules/eslint-plugin-expo/node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-plugin-expo/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-plugin-expo/node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^8.15.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-plugin-expo/node_modules/file-entry-cache": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", - "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "flat-cache": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/eslint-plugin-expo/node_modules/flat-cache": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", - "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", - "dev": true, - "license": "MIT", - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.4" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/eslint-plugin-expo/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/eslint-plugin-import": { "version": "2.32.0", "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", @@ -6586,9 +6333,9 @@ } }, "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -6596,7 +6343,7 @@ "estraverse": "^5.2.0" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" @@ -6615,19 +6362,45 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.9.0", + "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" + "eslint-visitor-keys": "^4.2.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" @@ -7078,6 +6851,19 @@ "react-native": "*" } }, + "node_modules/expo-localization": { + "version": "17.0.8", + "resolved": "https://registry.npmjs.org/expo-localization/-/expo-localization-17.0.8.tgz", + "integrity": "sha512-UrdwklZBDJ+t+ZszMMiE0SXZ2eJxcquCuQcl6EvGHM9K+e6YqKVRQ+w8qE+iIB3H75v2RJy6MHAaLK+Mqeo04g==", + "license": "MIT", + "dependencies": { + "rtl-detect": "^1.0.2" + }, + "peerDependencies": { + "expo": "*", + "react": "*" + } + }, "node_modules/expo-manifests": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/expo-manifests/-/expo-manifests-1.0.10.tgz", @@ -7335,16 +7121,6 @@ ], "license": "BSD-3-Clause" }, - "node_modules/fastq": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", - "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, - "license": "ISC", - "dependencies": { - "reusify": "^1.0.4" - } - }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -7428,16 +7204,16 @@ } }, "node_modules/file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", "dev": true, "license": "MIT", "dependencies": { - "flat-cache": "^3.0.4" + "flat-cache": "^4.0.0" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": ">=16.0.0" } }, "node_modules/fill-range": { @@ -7512,24 +7288,23 @@ } }, "node_modules/flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", "dev": true, "license": "MIT", "dependencies": { "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" + "keyv": "^4.5.4" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": ">=16" } }, "node_modules/flatted": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz", - "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -7827,16 +7602,13 @@ } }, "node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", "dev": true, "license": "MIT", - "dependencies": { - "type-fest": "^0.20.2" - }, "engines": { - "node": ">=8" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -7877,13 +7649,6 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "license": "ISC" }, - "node_modules/graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true, - "license": "MIT" - }, "node_modules/has-bigints": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", @@ -8518,16 +8283,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -10891,27 +10646,6 @@ "inherits": "~2.0.3" } }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -11484,17 +11218,6 @@ "node": ">=4" } }, - "node_modules/reusify": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", - "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -11532,29 +11255,11 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } + "node_modules/rtl-detect": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/rtl-detect/-/rtl-detect-1.1.2.tgz", + "integrity": "sha512-PGMBq03+TTG/p/cRB7HCLKJ1MgDIi07+QU1faSjiYRfmY5UsAttV9Hs08jDAHVwcOwmVLcSJkpwyfXszVjWfIQ==", + "license": "BSD-3-Clause" }, "node_modules/safe-array-concat": { "version": "1.1.3", @@ -12483,13 +12188,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true, - "license": "MIT" - }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -12651,19 +12349,6 @@ "node": ">=4" } }, - "node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", diff --git a/mobile/package.json b/mobile/package.json index bc62426c..6037583d 100644 --- a/mobile/package.json +++ b/mobile/package.json @@ -8,7 +8,7 @@ "android": "expo start --android", "ios": "expo start --ios", "web": "expo start --web", - "lint": "eslint src --ext .ts,.tsx", + "lint": "eslint src", "type-check": "tsc --noEmit", "build:ios": "eas build --platform ios", "build:android": "eas build --platform android", @@ -36,6 +36,7 @@ "expo-image-manipulator": "~14.0.8", "expo-image-picker": "~17.0.10", "expo-linking": "~8.0.11", + "expo-localization": "~17.0.8", "expo-notifications": "~0.32.16", "expo-router": "~6.0.23", "expo-secure-store": "~15.0.8", @@ -53,7 +54,7 @@ "devDependencies": { "@babel/core": "^7.24.0", "@types/react": "~19.1.0", - "eslint": "^8.57.0", + "eslint": "^9.0.0", "eslint-config-expo": "~10.0.0", "typescript": "^5.3.0" }, diff --git a/mobile/src/context/AuthContext.tsx b/mobile/src/context/AuthContext.tsx index 4bbb66e7..e4e1ab84 100644 --- a/mobile/src/context/AuthContext.tsx +++ b/mobile/src/context/AuthContext.tsx @@ -38,6 +38,7 @@ export interface AuthState { user: WhoAmIResponse | null; baseUrl: string; signIn: (serverUrl: string) => Promise; + signInWithQR: (serverUrl: string, challengeToken: string) => Promise; signOut: () => Promise; setToken: (token: string) => Promise; } @@ -52,6 +53,7 @@ const AuthContext = createContext({ user: null, baseUrl: "", signIn: async () => {}, + signInWithQR: async () => {}, signOut: async () => {}, setToken: async () => {}, }); @@ -143,6 +145,19 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { [setToken] ); + const signInWithQR = useCallback( + async (serverUrl: string, challengeToken: string) => { + const cleanUrl = serverUrl.replace(/\/$/, ""); + await api.init(cleanUrl); + setBaseUrl(cleanUrl); + + const deviceInfo = await _getDeviceName(); + const resp = await api.claimQRChallenge(challengeToken, deviceInfo); + await setToken(resp.token); + }, + [setToken] + ); + const signOut = useCallback(async () => { await SecureStore.deleteItemAsync(SECURE_STORE_API_TOKEN_KEY); await SecureStore.deleteItemAsync(SECURE_STORE_OWNER_ID_KEY); @@ -158,6 +173,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { user, baseUrl, signIn, + signInWithQR, signOut, setToken, }} diff --git a/mobile/src/context/ShareContext.tsx b/mobile/src/context/ShareContext.tsx index 43643bdb..caae315c 100644 --- a/mobile/src/context/ShareContext.tsx +++ b/mobile/src/context/ShareContext.tsx @@ -9,6 +9,7 @@ */ import React, { createContext, useCallback, useContext, useState } from "react"; +import { normalizeFileUri } from "../utils/normalizeUri"; export interface SharedFile { uri: string; @@ -32,7 +33,13 @@ export function ShareProvider({ children }: { children: React.ReactNode }) { const [pendingFiles, setPendingFiles] = useState([]); const addPendingFile = useCallback((file: SharedFile) => { - setPendingFiles((prev) => [...prev, file]); + setPendingFiles((prev) => { + // Deduplicate by normalised URI so the same file is not uploaded twice + // when both the Linking handler (_layout.tsx) and +not-found.tsx fire. + const norm = normalizeFileUri(file.uri); + if (prev.some((f) => normalizeFileUri(f.uri) === norm)) return prev; + return [...prev, file]; + }); }, []); const clearPendingFiles = useCallback(() => { diff --git a/mobile/src/i18n/de.json b/mobile/src/i18n/de.json new file mode 100644 index 00000000..79589bf5 --- /dev/null +++ b/mobile/src/i18n/de.json @@ -0,0 +1,116 @@ +{ + "common": { + "retry": "Erneut versuchen", + "cancel": "Abbrechen", + "back": "Zurück", + "error": "Fehler", + "loading": "Laden…", + "search": "Suchen", + "clear_search": "Suche löschen" + }, + "welcome": { + "tagline": "Intelligente Dokumentenverarbeitung", + "description": "Dokumente einlesen, OCR durchführen, Metadaten mit KI extrahieren und Dateien in Ihren Cloud-Speicher leiten – alles in einer nahtlosen Pipeline.", + "get_started": "Loslegen", + "hint": "Verbinden Sie sich mit Ihrem selbst gehosteten oder Cloud-DocuElevate-Server.", + "feature_ocr_title": "OCR & Texterkennung", + "feature_ocr_desc": "Gescannte PDFs und Bilder automatisch in durchsuchbaren Text umwandeln.", + "feature_ai_title": "KI-Metadatenextraktion", + "feature_ai_desc": "KI klassifiziert Dokumente und extrahiert Schlüsselfelder wie Datum, Beträge und Betreff.", + "feature_cloud_title": "Multi-Cloud-Speicher", + "feature_cloud_desc": "Verarbeitete Dateien an Dropbox, Google Drive, OneDrive, S3, Nextcloud und mehr weiterleiten." + }, + "login": { + "server_url": "Server-URL", + "server_url_placeholder": "https://ihr-docuelevate-server.com", + "sign_in_sso": "Mit SSO anmelden", + "scan_qr": "📱 QR-Code scannen zum Anmelden", + "hint": "Melden Sie sich per SSO an oder scannen Sie einen QR-Code aus der Web-App.", + "back": "← Zurück", + "or": "oder", + "server_url_required": "Server-URL erforderlich", + "server_url_required_msg": "Bitte geben Sie die URL Ihres DocuElevate-Servers ein.", + "invalid_url": "Ungültige URL", + "invalid_url_msg": "Die Server-URL muss mit http:// oder https:// beginnen", + "sign_in_failed": "Anmeldung fehlgeschlagen", + "qr_login_failed": "QR-Anmeldung fehlgeschlagen" + }, + "upload": { + "camera": "Kamera", + "photos": "Fotos", + "files": "Dateien", + "camera_access_title": "Kamerazugriff erforderlich", + "camera_access_msg": "Bitte erlauben Sie den Kamerazugriff in den Einstellungen, um Dokumente aufzunehmen.", + "photo_access_title": "Fotobibliothek-Zugriff erforderlich", + "photo_access_msg": "Bitte erlauben Sie den Zugriff auf die Fotobibliothek in den Einstellungen.", + "file_picker_error": "Dateiauswahl-Fehler", + "file_picker_error_msg": "Dateiauswahl konnte nicht geöffnet werden", + "empty_title": "Tippen Sie auf Kamera, Fotos oder Dateien, um ein Dokument hochzuladen.", + "empty_hint": "Sie können auch Dateien aus anderen Apps direkt an DocuElevate senden.", + "sign_in_required": "Bitte melden Sie sich an, um Dokumente hochzuladen.", + "status_queued": "In der Warteschlange…", + "status_processing": "Wird verarbeitet…", + "status_completed": "Verarbeitet", + "status_failed": "Verarbeitung fehlgeschlagen", + "status_duplicate": "Duplikat – bereits verarbeitet", + "tap_retry": "Zum Wiederholen tippen", + "retry_title": "Upload wiederholen", + "retry_msg": "Möchten Sie den Upload von \"{filename}\" wiederholen?", + "capture_label": "Dokument mit Kamera aufnehmen", + "photo_label": "Foto aus der Bibliothek auswählen", + "file_label": "Datei vom Gerät auswählen" + }, + "files": { + "title": "Meine Dokumente", + "search_placeholder": "Dokumente durchsuchen…", + "empty_title": "Noch keine Dokumente.", + "empty_hint": "Laden Sie ein Dokument über den Upload-Tab hoch.", + "search_empty": "Keine Dokumente gefunden.", + "search_empty_hint": "Versuchen Sie einen anderen Suchbegriff.", + "view_details": "Details für {filename} anzeigen" + }, + "file_detail": { + "title": "Dateidetails", + "back": "Zurück zu Dateien", + "file_size": "Dateigröße", + "mime_type": "MIME-Typ", + "uploaded": "Hochgeladen", + "file_hash": "Datei-Hash", + "last_step": "Letzter Schritt", + "total_steps": "Gesamtschritte", + "processing_log": "Verarbeitungsprotokoll", + "no_logs": "Noch keine Verarbeitungsprotokolle.", + "file_not_found": "Datei nicht gefunden" + }, + "profile": { + "title": "Profil", + "not_signed_in": "Nicht angemeldet", + "connection": "Verbindung", + "server": "Server", + "user_id": "Benutzer-ID", + "legal": "Rechtliches", + "privacy_policy": "Datenschutzrichtlinie", + "terms_of_service": "Nutzungsbedingungen", + "imprint": "Impressum", + "sign_out": "Abmelden", + "sign_out_title": "Abmelden", + "sign_out_msg": "Möchten Sie sich wirklich abmelden?", + "delete_account": "Konto löschen", + "delete_account_title": "Konto löschen", + "delete_account_msg": "Dadurch werden Ihr Konto und alle zugehörigen Daten dauerhaft gelöscht. Diese Aktion kann nicht rückgängig gemacht werden.", + "could_not_open": "Konnte {page} nicht öffnen. Bitte versuchen Sie es erneut.", + "admin": "Admin", + "settings": "Einstellungen", + "language": "Sprache" + }, + "legal": { + "privacy_policy": "Datenschutz", + "terms": "AGB", + "imprint": "Impressum" + }, + "tabs": { + "upload": "Hochladen", + "files": "Dateien", + "profile": "Profil" + } +} diff --git a/mobile/src/i18n/en.json b/mobile/src/i18n/en.json new file mode 100644 index 00000000..137503c3 --- /dev/null +++ b/mobile/src/i18n/en.json @@ -0,0 +1,116 @@ +{ + "common": { + "retry": "Retry", + "cancel": "Cancel", + "back": "Back", + "error": "Error", + "loading": "Loading…", + "search": "Search", + "clear_search": "Clear search" + }, + "welcome": { + "tagline": "Intelligent Document Processing", + "description": "Ingest documents, run OCR, extract metadata with AI, and route files to your cloud storage — all in one seamless pipeline.", + "get_started": "Get Started", + "hint": "Connect to your self-hosted or cloud DocuElevate server.", + "feature_ocr_title": "OCR & Text Extraction", + "feature_ocr_desc": "Convert scanned PDFs and images into fully searchable text automatically.", + "feature_ai_title": "AI Metadata Extraction", + "feature_ai_desc": "AI classifies documents and pulls out key fields like dates, amounts, and subjects.", + "feature_cloud_title": "Multi-Cloud Storage", + "feature_cloud_desc": "Route processed files to Dropbox, Google Drive, OneDrive, S3, Nextcloud, and more." + }, + "login": { + "server_url": "Server URL", + "server_url_placeholder": "https://your-docuelevate-server.com", + "sign_in_sso": "Sign in with SSO", + "scan_qr": "📱 Scan QR Code to Login", + "hint": "Sign in via SSO or scan a QR code from the web app.", + "back": "← Back", + "or": "or", + "server_url_required": "Server URL required", + "server_url_required_msg": "Please enter the URL of your DocuElevate server.", + "invalid_url": "Invalid URL", + "invalid_url_msg": "The server URL must start with http:// or https://", + "sign_in_failed": "Sign-in failed", + "qr_login_failed": "QR Login Failed" + }, + "upload": { + "camera": "Camera", + "photos": "Photos", + "files": "Files", + "camera_access_title": "Camera access required", + "camera_access_msg": "Please grant camera access in Settings to capture documents.", + "photo_access_title": "Photo library access required", + "photo_access_msg": "Please grant photo library access in Settings to select images.", + "file_picker_error": "File picker error", + "file_picker_error_msg": "Could not open file picker", + "empty_title": "Tap Camera, Photos, or Files to upload a document.", + "empty_hint": "You can also share files from other apps directly to DocuElevate.", + "sign_in_required": "Please sign in to upload documents.", + "status_queued": "Queued for processing…", + "status_processing": "Processing…", + "status_completed": "Processed", + "status_failed": "Processing failed", + "status_duplicate": "Duplicate – already processed", + "tap_retry": "Tap to retry", + "retry_title": "Retry Upload", + "retry_msg": "Do you want to retry uploading \"{filename}\"?", + "capture_label": "Capture document with camera", + "photo_label": "Select photo from library", + "file_label": "Pick file from device" + }, + "files": { + "title": "My Documents", + "search_placeholder": "Search documents…", + "empty_title": "No documents yet.", + "empty_hint": "Upload a document from the Upload tab to get started.", + "search_empty": "No documents match your search.", + "search_empty_hint": "Try a different search term.", + "view_details": "View details for {filename}" + }, + "file_detail": { + "title": "File Details", + "back": "Back to Files", + "file_size": "File Size", + "mime_type": "MIME Type", + "uploaded": "Uploaded", + "file_hash": "File Hash", + "last_step": "Last Step", + "total_steps": "Total Steps", + "processing_log": "Processing Log", + "no_logs": "No processing logs yet.", + "file_not_found": "File not found" + }, + "profile": { + "title": "Profile", + "not_signed_in": "Not signed in", + "connection": "Connection", + "server": "Server", + "user_id": "User ID", + "legal": "Legal", + "privacy_policy": "Privacy Policy", + "terms_of_service": "Terms of Service", + "imprint": "Imprint", + "sign_out": "Sign out", + "sign_out_title": "Sign out", + "sign_out_msg": "Are you sure you want to sign out?", + "delete_account": "Delete Account", + "delete_account_title": "Delete Account", + "delete_account_msg": "This will permanently delete your account and all associated data. This action cannot be undone.", + "could_not_open": "Could not open the {page}. Please try again.", + "admin": "Admin", + "settings": "Settings", + "language": "Language" + }, + "legal": { + "privacy_policy": "Privacy Policy", + "terms": "Terms", + "imprint": "Imprint" + }, + "tabs": { + "upload": "Upload", + "files": "Files", + "profile": "Profile" + } +} diff --git a/mobile/src/i18n/es.json b/mobile/src/i18n/es.json new file mode 100644 index 00000000..eb434f34 --- /dev/null +++ b/mobile/src/i18n/es.json @@ -0,0 +1,116 @@ +{ + "common": { + "retry": "Reintentar", + "cancel": "Cancelar", + "back": "Atrás", + "error": "Error", + "loading": "Cargando…", + "search": "Buscar", + "clear_search": "Borrar búsqueda" + }, + "welcome": { + "tagline": "Procesamiento Inteligente de Documentos", + "description": "Ingiere documentos, ejecuta OCR, extrae metadatos con IA y envía archivos a tu almacenamiento en la nube — todo en una sola línea de trabajo.", + "get_started": "Comenzar", + "hint": "Conéctate a tu servidor DocuElevate autoalojado o en la nube.", + "feature_ocr_title": "OCR y Extracción de Texto", + "feature_ocr_desc": "Convierte PDFs e imágenes escaneadas en texto completamente buscable automáticamente.", + "feature_ai_title": "Extracción de Metadatos con IA", + "feature_ai_desc": "La IA clasifica documentos y extrae campos clave como fechas, montos y asuntos.", + "feature_cloud_title": "Almacenamiento Multi-Nube", + "feature_cloud_desc": "Envía archivos procesados a Dropbox, Google Drive, OneDrive, S3, Nextcloud y más." + }, + "login": { + "server_url": "URL del Servidor", + "server_url_placeholder": "https://tu-servidor-docuelevate.com", + "sign_in_sso": "Iniciar sesión con SSO", + "scan_qr": "📱 Escanear código QR para iniciar sesión", + "hint": "Inicia sesión mediante SSO o escanea un código QR desde la app web.", + "back": "← Atrás", + "or": "o", + "server_url_required": "URL del servidor requerida", + "server_url_required_msg": "Por favor ingresa la URL de tu servidor DocuElevate.", + "invalid_url": "URL inválida", + "invalid_url_msg": "La URL del servidor debe comenzar con http:// o https://", + "sign_in_failed": "Error al iniciar sesión", + "qr_login_failed": "Error en inicio de sesión QR" + }, + "upload": { + "camera": "Cámara", + "photos": "Fotos", + "files": "Archivos", + "camera_access_title": "Acceso a la cámara requerido", + "camera_access_msg": "Permite el acceso a la cámara en Ajustes para capturar documentos.", + "photo_access_title": "Acceso a la biblioteca de fotos requerido", + "photo_access_msg": "Permite el acceso a la biblioteca de fotos en Ajustes para seleccionar imágenes.", + "file_picker_error": "Error del selector de archivos", + "file_picker_error_msg": "No se pudo abrir el selector de archivos", + "empty_title": "Toca Cámara, Fotos o Archivos para subir un documento.", + "empty_hint": "También puedes compartir archivos desde otras apps directamente a DocuElevate.", + "sign_in_required": "Inicia sesión para subir documentos.", + "status_queued": "En cola para procesamiento…", + "status_processing": "Procesando…", + "status_completed": "Procesado", + "status_failed": "Procesamiento fallido", + "status_duplicate": "Duplicado – ya procesado", + "tap_retry": "Toca para reintentar", + "retry_title": "Reintentar Subida", + "retry_msg": "¿Deseas reintentar la subida de \"{filename}\"?", + "capture_label": "Capturar documento con la cámara", + "photo_label": "Seleccionar foto de la biblioteca", + "file_label": "Seleccionar archivo del dispositivo" + }, + "files": { + "title": "Mis Documentos", + "search_placeholder": "Buscar documentos…", + "empty_title": "Aún no hay documentos.", + "empty_hint": "Sube un documento desde la pestaña Subir para comenzar.", + "search_empty": "Ningún documento coincide con tu búsqueda.", + "search_empty_hint": "Intenta con otro término de búsqueda.", + "view_details": "Ver detalles de {filename}" + }, + "file_detail": { + "title": "Detalles del Archivo", + "back": "Volver a Archivos", + "file_size": "Tamaño", + "mime_type": "Tipo MIME", + "uploaded": "Subido", + "file_hash": "Hash del Archivo", + "last_step": "Último Paso", + "total_steps": "Pasos Totales", + "processing_log": "Registro de Procesamiento", + "no_logs": "Aún no hay registros de procesamiento.", + "file_not_found": "Archivo no encontrado" + }, + "profile": { + "title": "Perfil", + "not_signed_in": "No has iniciado sesión", + "connection": "Conexión", + "server": "Servidor", + "user_id": "ID de Usuario", + "legal": "Legal", + "privacy_policy": "Política de Privacidad", + "terms_of_service": "Términos de Servicio", + "imprint": "Aviso Legal", + "sign_out": "Cerrar sesión", + "sign_out_title": "Cerrar sesión", + "sign_out_msg": "¿Estás seguro de que deseas cerrar sesión?", + "delete_account": "Eliminar Cuenta", + "delete_account_title": "Eliminar Cuenta", + "delete_account_msg": "Esto eliminará permanentemente tu cuenta y todos los datos asociados. Esta acción no se puede deshacer.", + "could_not_open": "No se pudo abrir {page}. Inténtalo de nuevo.", + "admin": "Admin", + "settings": "Configuración", + "language": "Idioma" + }, + "legal": { + "privacy_policy": "Privacidad", + "terms": "Términos", + "imprint": "Aviso Legal" + }, + "tabs": { + "upload": "Subir", + "files": "Archivos", + "profile": "Perfil" + } +} diff --git a/mobile/src/i18n/fr.json b/mobile/src/i18n/fr.json new file mode 100644 index 00000000..3760d4d5 --- /dev/null +++ b/mobile/src/i18n/fr.json @@ -0,0 +1,116 @@ +{ + "common": { + "retry": "Réessayer", + "cancel": "Annuler", + "back": "Retour", + "error": "Erreur", + "loading": "Chargement…", + "search": "Rechercher", + "clear_search": "Effacer la recherche" + }, + "welcome": { + "tagline": "Traitement Intelligent de Documents", + "description": "Ingérez des documents, lancez l'OCR, extrayez les métadonnées avec l'IA et transférez les fichiers vers votre stockage cloud — le tout dans un flux unique.", + "get_started": "Commencer", + "hint": "Connectez-vous à votre serveur DocuElevate auto-hébergé ou cloud.", + "feature_ocr_title": "OCR et Extraction de Texte", + "feature_ocr_desc": "Convertissez automatiquement les PDF scannés et les images en texte entièrement consultable.", + "feature_ai_title": "Extraction de Métadonnées par IA", + "feature_ai_desc": "L'IA classe les documents et extrait les champs clés comme les dates, montants et sujets.", + "feature_cloud_title": "Stockage Multi-Cloud", + "feature_cloud_desc": "Transférez les fichiers traités vers Dropbox, Google Drive, OneDrive, S3, Nextcloud et plus." + }, + "login": { + "server_url": "URL du Serveur", + "server_url_placeholder": "https://votre-serveur-docuelevate.com", + "sign_in_sso": "Se connecter avec SSO", + "scan_qr": "📱 Scanner le code QR pour se connecter", + "hint": "Connectez-vous via SSO ou scannez un code QR depuis l'application web.", + "back": "← Retour", + "or": "ou", + "server_url_required": "URL du serveur requise", + "server_url_required_msg": "Veuillez entrer l'URL de votre serveur DocuElevate.", + "invalid_url": "URL invalide", + "invalid_url_msg": "L'URL du serveur doit commencer par http:// ou https://", + "sign_in_failed": "Échec de la connexion", + "qr_login_failed": "Échec de la connexion QR" + }, + "upload": { + "camera": "Appareil photo", + "photos": "Photos", + "files": "Fichiers", + "camera_access_title": "Accès à l'appareil photo requis", + "camera_access_msg": "Veuillez autoriser l'accès à l'appareil photo dans les Réglages pour capturer des documents.", + "photo_access_title": "Accès à la photothèque requis", + "photo_access_msg": "Veuillez autoriser l'accès à la photothèque dans les Réglages pour sélectionner des images.", + "file_picker_error": "Erreur du sélecteur de fichiers", + "file_picker_error_msg": "Impossible d'ouvrir le sélecteur de fichiers", + "empty_title": "Appuyez sur Appareil photo, Photos ou Fichiers pour télécharger un document.", + "empty_hint": "Vous pouvez aussi partager des fichiers depuis d'autres applications vers DocuElevate.", + "sign_in_required": "Veuillez vous connecter pour télécharger des documents.", + "status_queued": "En file d'attente…", + "status_processing": "En cours de traitement…", + "status_completed": "Traité", + "status_failed": "Échec du traitement", + "status_duplicate": "Doublon – déjà traité", + "tap_retry": "Appuyez pour réessayer", + "retry_title": "Réessayer le téléchargement", + "retry_msg": "Voulez-vous réessayer le téléchargement de \"{filename}\" ?", + "capture_label": "Capturer un document avec l'appareil photo", + "photo_label": "Sélectionner une photo de la bibliothèque", + "file_label": "Choisir un fichier depuis l'appareil" + }, + "files": { + "title": "Mes Documents", + "search_placeholder": "Rechercher des documents…", + "empty_title": "Pas encore de documents.", + "empty_hint": "Téléchargez un document depuis l'onglet Télécharger pour commencer.", + "search_empty": "Aucun document ne correspond à votre recherche.", + "search_empty_hint": "Essayez un autre terme de recherche.", + "view_details": "Voir les détails de {filename}" + }, + "file_detail": { + "title": "Détails du Fichier", + "back": "Retour aux Fichiers", + "file_size": "Taille", + "mime_type": "Type MIME", + "uploaded": "Téléchargé", + "file_hash": "Hash du Fichier", + "last_step": "Dernière Étape", + "total_steps": "Étapes Totales", + "processing_log": "Journal de Traitement", + "no_logs": "Pas encore de journaux de traitement.", + "file_not_found": "Fichier non trouvé" + }, + "profile": { + "title": "Profil", + "not_signed_in": "Non connecté", + "connection": "Connexion", + "server": "Serveur", + "user_id": "ID Utilisateur", + "legal": "Mentions Légales", + "privacy_policy": "Politique de Confidentialité", + "terms_of_service": "Conditions d'Utilisation", + "imprint": "Mentions Légales", + "sign_out": "Se déconnecter", + "sign_out_title": "Se déconnecter", + "sign_out_msg": "Êtes-vous sûr de vouloir vous déconnecter ?", + "delete_account": "Supprimer le Compte", + "delete_account_title": "Supprimer le Compte", + "delete_account_msg": "Cela supprimera définitivement votre compte et toutes les données associées. Cette action est irréversible.", + "could_not_open": "Impossible d'ouvrir {page}. Veuillez réessayer.", + "admin": "Admin", + "settings": "Paramètres", + "language": "Langue" + }, + "legal": { + "privacy_policy": "Confidentialité", + "terms": "Conditions", + "imprint": "Mentions Légales" + }, + "tabs": { + "upload": "Télécharger", + "files": "Fichiers", + "profile": "Profil" + } +} diff --git a/mobile/src/i18n/index.ts b/mobile/src/i18n/index.ts new file mode 100644 index 00000000..84bc1504 --- /dev/null +++ b/mobile/src/i18n/index.ts @@ -0,0 +1,208 @@ +/** + * Lightweight i18n module for the DocuElevate mobile app. + * + * Uses the device locale (via expo-localization) to select the best matching + * translation file. Falls back to English for missing keys or unsupported + * locales. + * + * Supported languages: English, German, Spanish, French, Italian. + * + * ## React integration + * + * Wrap the app root in `` and call `useLocale()` in any + * component that renders translated strings. `useLocale()` returns the + * active language code and a `setLang` setter that: + * 1. Updates the in-memory `currentLanguage` variable (so `t()` picks it up) + * 2. Triggers a React re-render of every consumer + * 3. Persists the choice to AsyncStorage (survives app restarts) + * + * Language priority on startup: + * server preference (from /api/mobile/whoami) > AsyncStorage > device locale > "en" + */ + +import AsyncStorage from "@react-native-async-storage/async-storage"; +import { getLocales } from "expo-localization"; +import React from "react"; + +import de from "./de.json"; +import en from "./en.json"; +import es from "./es.json"; +import fr from "./fr.json"; +import it from "./it.json"; + +// --------------------------------------------------------------------------- +// Translation catalog +// --------------------------------------------------------------------------- + +type TranslationMap = Record>; + +const translations: Record = { en, de, es, fr, it }; + +// --------------------------------------------------------------------------- +// Locale detection +// --------------------------------------------------------------------------- + +const LANG_STORAGE_KEY = "@docuelevate:language"; + +/** Resolve the best-matching language code from the device locale list. */ +function detectLanguage(): string { + try { + const locales = getLocales(); + if (locales.length > 0) { + // Try exact match first (e.g. "de"), then fall back to language prefix + const code = locales[0].languageCode?.toLowerCase(); + if (code && translations[code]) return code; + } + } catch { + // getLocales() can throw on some platforms – default to English + } + return "en"; +} + +let currentLanguage: string = detectLanguage(); + +// --------------------------------------------------------------------------- +// Plain-function public API (framework-agnostic) +// --------------------------------------------------------------------------- + +/** + * Translate a dot-separated key, e.g. `t("upload.camera")`. + * + * Supports simple placeholder interpolation: + * `t("upload.retry_msg", { filename: "doc.pdf" })` + * replaces `{filename}` in the translated string. + * + * Falls back to the English value, then to the raw key if no translation + * exists. + */ +export function t(key: string, params?: Record): string { + const [section, ...rest] = key.split("."); + const subKey = rest.join("."); + + let value = + translations[currentLanguage]?.[section]?.[subKey] ?? + translations.en?.[section]?.[subKey] ?? + key; + + if (params) { + for (const [k, v] of Object.entries(params)) { + value = value.replaceAll(`{${k}}`, v); + } + } + + return value; +} + +/** Return the current language code (e.g. "en", "de"). */ +export function getLanguage(): string { + return currentLanguage; +} + +/** + * Update the active language in memory. + * Prefer `useLocale().setLang` inside React components – it also persists + * the choice and triggers re-renders. + */ +export function setLanguage(lang: string): void { + if (translations[lang]) { + currentLanguage = lang; + } +} + +/** Return true if the given language code is supported by the mobile app. */ +export function isLanguageSupported(lang: string): boolean { + return Object.prototype.hasOwnProperty.call(translations, lang); +} + +/** Return the list of supported language codes. */ +export function getSupportedLanguages(): { code: string; label: string }[] { + return [ + { code: "en", label: "English" }, + { code: "de", label: "Deutsch" }, + { code: "es", label: "Español" }, + { code: "fr", label: "Français" }, + { code: "it", label: "Italiano" }, + ]; +} + +// --------------------------------------------------------------------------- +// React integration – context + provider + hook +// --------------------------------------------------------------------------- + +interface LocaleContextValue { + /** The active language code, e.g. "en" or "de". */ + lang: string; + /** + * Switch to a new language. Persists the choice to AsyncStorage and + * triggers a re-render of every `useLocale()` consumer. + */ + setLang: (code: string) => Promise; +} + +const LocaleContext = React.createContext({ + lang: currentLanguage, + // Default setter used outside of a provider – updates in-memory only. + setLang: async (code: string) => { + setLanguage(code); + }, +}); + +/** + * Wrap the app root in `LocaleProvider` to enable reactive language switching. + * + * On mount it reads the persisted language from AsyncStorage so the user's + * choice survives app restarts. The server-preferred language is applied + * externally (see `AuthGuard` in `app/_layout.tsx`) after the profile is + * fetched from `/api/mobile/whoami`. + */ +export function LocaleProvider({ children }: { children: React.ReactNode }): React.ReactElement { + const [lang, setLangState] = React.useState(currentLanguage); + + // Restore the persisted language preference once on app start. + React.useEffect(() => { + AsyncStorage.getItem(LANG_STORAGE_KEY) + .then((saved) => { + if (saved && isLanguageSupported(saved)) { + setLanguage(saved); + setLangState(saved); + } + }) + .catch(() => { + // Ignore read errors – fall back to device-detected language. + }); + }, []); + + const setLang = React.useCallback(async (code: string): Promise => { + if (!isLanguageSupported(code)) return; + setLanguage(code); + setLangState(code); + try { + await AsyncStorage.setItem(LANG_STORAGE_KEY, code); + } catch { + // Ignore write errors – the in-memory change is still applied. + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // setLangState is a React state setter – its identity is guaranteed stable + + const value = React.useMemo(() => ({ lang, setLang }), [lang, setLang]); + + return React.createElement(LocaleContext.Provider, { value }, children); +} + +/** + * Hook that subscribes to language changes. + * + * Any component calling `useLocale()` re-renders automatically when the + * language changes. Call `t()` freely inside the component body – the + * re-render will pick up the new translations. + * + * ```tsx + * function MyScreen() { + * const { lang, setLang } = useLocale(); // subscribes to changes + * return {t("common.loading")}; + * } + * ``` + */ +export function useLocale(): LocaleContextValue { + return React.useContext(LocaleContext); +} diff --git a/mobile/src/i18n/it.json b/mobile/src/i18n/it.json new file mode 100644 index 00000000..71fc3699 --- /dev/null +++ b/mobile/src/i18n/it.json @@ -0,0 +1,116 @@ +{ + "common": { + "retry": "Riprova", + "cancel": "Annulla", + "back": "Indietro", + "error": "Errore", + "loading": "Caricamento…", + "search": "Cerca", + "clear_search": "Cancella ricerca" + }, + "welcome": { + "tagline": "Elaborazione Intelligente dei Documenti", + "description": "Acquisisci documenti, esegui l'OCR, estrai metadati con l'IA e invia i file al tuo cloud storage — tutto in un unico flusso.", + "get_started": "Inizia", + "hint": "Collegati al tuo server DocuElevate self-hosted o cloud.", + "feature_ocr_title": "OCR ed Estrazione Testo", + "feature_ocr_desc": "Converti automaticamente PDF e immagini scansionate in testo completamente ricercabile.", + "feature_ai_title": "Estrazione Metadati con IA", + "feature_ai_desc": "L'IA classifica i documenti ed estrae campi chiave come date, importi e oggetti.", + "feature_cloud_title": "Archiviazione Multi-Cloud", + "feature_cloud_desc": "Invia i file elaborati a Dropbox, Google Drive, OneDrive, S3, Nextcloud e altro." + }, + "login": { + "server_url": "URL del Server", + "server_url_placeholder": "https://il-tuo-server-docuelevate.com", + "sign_in_sso": "Accedi con SSO", + "scan_qr": "📱 Scansiona il codice QR per accedere", + "hint": "Accedi tramite SSO o scansiona un codice QR dall'app web.", + "back": "← Indietro", + "or": "o", + "server_url_required": "URL del server richiesto", + "server_url_required_msg": "Inserisci l'URL del tuo server DocuElevate.", + "invalid_url": "URL non valido", + "invalid_url_msg": "L'URL del server deve iniziare con http:// o https://", + "sign_in_failed": "Accesso fallito", + "qr_login_failed": "Accesso QR fallito" + }, + "upload": { + "camera": "Fotocamera", + "photos": "Foto", + "files": "File", + "camera_access_title": "Accesso alla fotocamera richiesto", + "camera_access_msg": "Consenti l'accesso alla fotocamera nelle Impostazioni per acquisire documenti.", + "photo_access_title": "Accesso alla libreria foto richiesto", + "photo_access_msg": "Consenti l'accesso alla libreria foto nelle Impostazioni per selezionare immagini.", + "file_picker_error": "Errore nel selettore file", + "file_picker_error_msg": "Impossibile aprire il selettore file", + "empty_title": "Tocca Fotocamera, Foto o File per caricare un documento.", + "empty_hint": "Puoi anche condividere file da altre app direttamente su DocuElevate.", + "sign_in_required": "Accedi per caricare documenti.", + "status_queued": "In coda per l'elaborazione…", + "status_processing": "Elaborazione in corso…", + "status_completed": "Elaborato", + "status_failed": "Elaborazione fallita", + "status_duplicate": "Duplicato – già elaborato", + "tap_retry": "Tocca per riprovare", + "retry_title": "Riprova Caricamento", + "retry_msg": "Vuoi riprovare a caricare \"{filename}\"?", + "capture_label": "Acquisisci documento con la fotocamera", + "photo_label": "Seleziona foto dalla libreria", + "file_label": "Seleziona file dal dispositivo" + }, + "files": { + "title": "I Miei Documenti", + "search_placeholder": "Cerca documenti…", + "empty_title": "Nessun documento ancora.", + "empty_hint": "Carica un documento dalla scheda Carica per iniziare.", + "search_empty": "Nessun documento corrisponde alla tua ricerca.", + "search_empty_hint": "Prova con un altro termine di ricerca.", + "view_details": "Visualizza dettagli per {filename}" + }, + "file_detail": { + "title": "Dettagli File", + "back": "Torna ai File", + "file_size": "Dimensione", + "mime_type": "Tipo MIME", + "uploaded": "Caricato", + "file_hash": "Hash del File", + "last_step": "Ultimo Passaggio", + "total_steps": "Passaggi Totali", + "processing_log": "Registro di Elaborazione", + "no_logs": "Nessun registro di elaborazione ancora.", + "file_not_found": "File non trovato" + }, + "profile": { + "title": "Profilo", + "not_signed_in": "Non connesso", + "connection": "Connessione", + "server": "Server", + "user_id": "ID Utente", + "legal": "Legale", + "privacy_policy": "Informativa sulla Privacy", + "terms_of_service": "Termini di Servizio", + "imprint": "Note Legali", + "sign_out": "Esci", + "sign_out_title": "Esci", + "sign_out_msg": "Sei sicuro di voler uscire?", + "delete_account": "Elimina Account", + "delete_account_title": "Elimina Account", + "delete_account_msg": "Questo eliminerà permanentemente il tuo account e tutti i dati associati. Questa azione non può essere annullata.", + "could_not_open": "Impossibile aprire {page}. Riprova.", + "admin": "Admin", + "settings": "Impostazioni", + "language": "Lingua" + }, + "legal": { + "privacy_policy": "Privacy", + "terms": "Termini", + "imprint": "Note Legali" + }, + "tabs": { + "upload": "Carica", + "files": "File", + "profile": "Profilo" + } +} diff --git a/mobile/src/screens/FileDetailScreen.tsx b/mobile/src/screens/FileDetailScreen.tsx new file mode 100644 index 00000000..8d7cad20 --- /dev/null +++ b/mobile/src/screens/FileDetailScreen.tsx @@ -0,0 +1,332 @@ +/** + * FileDetailScreen – shows detailed status and processing logs for a single file. + * + * Replicates the web /files/:id and /files/:id/detail views in a + * mobile-friendly layout. Displays file metadata, processing status with + * a progress indicator, and a chronological list of processing log entries. + */ + +import { Ionicons } from "@expo/vector-icons"; +import { useLocalSearchParams, useRouter } from "expo-router"; +import React, { useCallback, useEffect, useState } from "react"; +import { + ActivityIndicator, + Pressable, + RefreshControl, + ScrollView, + StyleSheet, + Text, + View, +} from "react-native"; +import type { FileDetail } from "../services/api"; +import api from "../services/api"; +import { useLocale, t } from "../i18n"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function formatBytes(bytes: number | null | undefined): string { + if (bytes === null || bytes === undefined) return "–"; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + +function formatDateTime(iso: string): string { + try { + return new Date(iso).toLocaleString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }); + } catch { + return iso; + } +} + +function statusColor(status: string): string { + const colors: Record = { + completed: "#059669", + processing: "#d97706", + pending: "#6b7280", + failed: "#dc2626", + duplicate: "#6b7280", + }; + return colors[status?.toLowerCase()] ?? "#6b7280"; +} + +function statusIcon(status: string): keyof typeof Ionicons.glyphMap { + const icons: Record = { + completed: "checkmark-circle", + processing: "sync-circle", + pending: "time-outline", + failed: "close-circle", + duplicate: "copy-outline", + }; + return icons[status?.toLowerCase()] ?? "document-outline"; +} + +function logStepIcon(status: string): { name: keyof typeof Ionicons.glyphMap; color: string } { + const lower = status?.toLowerCase(); + if (lower === "completed" || lower === "success") return { name: "checkmark-circle", color: "#059669" }; + if (lower === "failed" || lower === "error") return { name: "close-circle", color: "#dc2626" }; + if (lower === "skipped") return { name: "remove-circle-outline", color: "#9ca3af" }; + if (lower === "processing" || lower === "running") return { name: "sync-circle", color: "#d97706" }; + return { name: "ellipse-outline", color: "#6b7280" }; +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +export default function FileDetailScreen() { + const { id } = useLocalSearchParams<{ id: string }>(); + const router = useRouter(); + const [detail, setDetail] = useState(null); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [error, setError] = useState(null); + // Subscribe to language changes so translated strings re-render. + useLocale(); + + const fileId = parseInt(id ?? "0", 10); + + const fetchDetail = useCallback(async () => { + if (!fileId) return; + try { + const data = await api.getFileDetail(fileId); + setDetail(data); + setError(null); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : "Failed to load file details"); + } + }, [fileId]); + + useEffect(() => { + (async () => { + setLoading(true); + await fetchDetail(); + setLoading(false); + })(); + }, [fetchDetail]); + + const handleRefresh = useCallback(async () => { + setRefreshing(true); + await fetchDetail(); + setRefreshing(false); + }, [fetchDetail]); + + if (loading) { + return ( + + + + ); + } + + if (error || !detail) { + return ( + + {error ?? t("file_detail.file_not_found")} + + {t("common.retry")} + + router.back()}> + {t("common.back")} + + + ); + } + + const file = detail.file; + const status = detail.processing_status; + + return ( + } + > + {/* Header with back button */} + router.back()} + accessibilityRole="button" + accessibilityLabel={t("file_detail.back")} + > + + {t("file_detail.back")} + + + {/* File info card */} + + + + + + {file.original_filename} + + + {status.status.charAt(0).toUpperCase() + status.status.slice(1)} + + + + + + + + + + + + + + + {/* Processing logs */} + + {t("file_detail.processing_log")} + {detail.logs.length === 0 ? ( + {t("file_detail.no_logs")} + ) : ( + detail.logs.map((log, idx) => { + const icon = logStepIcon(log.status); + const isLast = idx === detail.logs.length - 1; + return ( + + + + {log.step_name} + + {log.message} + + {formatDateTime(log.timestamp)} + + + ); + }) + )} + + + ); +} + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +function MetaRow({ label, value }: { label: string; value: string }) { + return ( + + {label} + + {value} + + + ); +} + +// --------------------------------------------------------------------------- +// Styles +// --------------------------------------------------------------------------- + +const styles = StyleSheet.create({ + scroll: { flex: 1, backgroundColor: "#f9fafb" }, + content: { padding: 16, paddingBottom: 40 }, + center: { + flex: 1, + alignItems: "center", + justifyContent: "center", + backgroundColor: "#f9fafb", + padding: 24, + }, + errorText: { color: "#dc2626", fontSize: 15, textAlign: "center", marginBottom: 16 }, + retryButton: { + backgroundColor: "#1e40af", + borderRadius: 8, + paddingHorizontal: 24, + paddingVertical: 10, + marginBottom: 12, + }, + retryText: { color: "#fff", fontWeight: "600" }, + backButton: { paddingVertical: 10 }, + backButtonText: { color: "#6b7280", fontSize: 14 }, + backRow: { + flexDirection: "row", + alignItems: "center", + marginBottom: 16, + minHeight: 44, + }, + backLabel: { + fontSize: 15, + color: "#1e40af", + fontWeight: "600", + marginLeft: 6, + }, + card: { + backgroundColor: "#fff", + borderRadius: 12, + padding: 16, + marginBottom: 16, + shadowColor: "#000", + shadowOpacity: 0.04, + shadowOffset: { width: 0, height: 2 }, + shadowRadius: 6, + elevation: 2, + }, + cardHeader: { + flexDirection: "row", + alignItems: "flex-start", + marginBottom: 16, + }, + filename: { + fontSize: 17, + fontWeight: "700", + color: "#111827", + marginBottom: 4, + }, + statusBadge: { + fontSize: 13, + fontWeight: "600", + textTransform: "capitalize", + }, + metaGrid: {}, + metaRow: { + flexDirection: "row", + justifyContent: "space-between", + alignItems: "center", + paddingVertical: 8, + borderBottomWidth: 1, + borderBottomColor: "#f3f4f6", + }, + metaLabel: { fontSize: 13, color: "#6b7280", fontWeight: "500" }, + metaValue: { fontSize: 13, color: "#374151", maxWidth: "55%", textAlign: "right" }, + sectionTitle: { + fontSize: 15, + fontWeight: "700", + color: "#374151", + marginBottom: 12, + }, + emptyLog: { fontSize: 13, color: "#9ca3af", fontStyle: "italic" }, + logEntry: { + flexDirection: "row", + alignItems: "flex-start", + paddingVertical: 10, + }, + logEntryBorder: { + borderBottomWidth: 1, + borderBottomColor: "#f3f4f6", + }, + logIcon: { marginRight: 10, marginTop: 1 }, + logContent: { flex: 1 }, + logStep: { fontSize: 13, fontWeight: "600", color: "#374151", marginBottom: 2 }, + logMessage: { fontSize: 12, color: "#6b7280", lineHeight: 17, marginBottom: 2 }, + logTimestamp: { fontSize: 11, color: "#9ca3af" }, +}); diff --git a/mobile/src/screens/FilesScreen.tsx b/mobile/src/screens/FilesScreen.tsx index 3a2a21d4..d40b1c65 100644 --- a/mobile/src/screens/FilesScreen.tsx +++ b/mobile/src/screens/FilesScreen.tsx @@ -1,8 +1,10 @@ /** - * FilesScreen – list of documents processed by DocuElevate. + * FilesScreen – list of documents processed by DocuElevate with search. */ -import React, { useCallback, useEffect, useState } from "react"; +import { Ionicons } from "@expo/vector-icons"; +import { useRouter } from "expo-router"; +import React, { useCallback, useEffect, useRef, useState } from "react"; import { ActivityIndicator, FlatList, @@ -10,10 +12,12 @@ import { RefreshControl, StyleSheet, Text, + TextInput, View, } from "react-native"; import type { FileRecord } from "../services/api"; import api from "../services/api"; +import { useLocale, t } from "../i18n"; function formatBytes(bytes: number | null): string { if (bytes === null || bytes === undefined) return "–"; @@ -34,29 +38,34 @@ function formatDate(iso: string): string { } } -function statusEmoji(status: string): string { - const map: Record = { - completed: "✅", - processing: "⚙️", - pending: "⏳", - failed: "❌", - duplicate: "🔁", +function statusIcon(status: string): { name: keyof typeof Ionicons.glyphMap; color: string } { + const map: Record = { + completed: { name: "checkmark-circle", color: "#059669" }, + processing: { name: "sync-circle", color: "#d97706" }, + pending: { name: "time-outline", color: "#6b7280" }, + failed: { name: "close-circle", color: "#dc2626" }, + duplicate: { name: "copy-outline", color: "#6b7280" }, }; - return map[status?.toLowerCase()] ?? "📄"; + return map[status?.toLowerCase()] ?? { name: "document-outline", color: "#6b7280" }; } export default function FilesScreen() { + const router = useRouter(); const [files, setFiles] = useState([]); const [loading, setLoading] = useState(true); const [refreshing, setRefreshing] = useState(false); const [page, setPage] = useState(1); const [hasMore, setHasMore] = useState(true); const [error, setError] = useState(null); + const [searchQuery, setSearchQuery] = useState(""); + const searchTimeoutRef = useRef | null>(null); + // Subscribe to language changes so translated strings re-render. + useLocale(); const fetchFiles = useCallback( - async (pageNum: number, replace: boolean) => { + async (pageNum: number, replace: boolean, search?: string) => { try { - const data = await api.listFiles(pageNum, 20); + const data = await api.listFiles(pageNum, 20, search || undefined); if (replace) { setFiles(data); } else { @@ -82,18 +91,56 @@ export default function FilesScreen() { const handleRefresh = useCallback(async () => { setRefreshing(true); setPage(1); - await fetchFiles(1, true); + await fetchFiles(1, true, searchQuery); setRefreshing(false); - }, [fetchFiles]); + }, [fetchFiles, searchQuery]); const handleLoadMore = useCallback(async () => { if (!hasMore || loading || refreshing) return; const next = page + 1; setPage(next); - await fetchFiles(next, false); - }, [fetchFiles, hasMore, loading, page, refreshing]); + await fetchFiles(next, false, searchQuery); + }, [fetchFiles, hasMore, loading, page, refreshing, searchQuery]); - if (loading) { + const handleSearch = useCallback( + (text: string) => { + setSearchQuery(text); + // Debounce search requests + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + searchTimeoutRef.current = setTimeout(async () => { + setPage(1); + setLoading(true); + try { + await fetchFiles(1, true, text); + } finally { + setLoading(false); + } + }, 400); + }, + [fetchFiles] + ); + + const handleClearSearch = useCallback(async () => { + setSearchQuery(""); + setPage(1); + setLoading(true); + try { + await fetchFiles(1, true); + } finally { + setLoading(false); + } + }, [fetchFiles]); + + const handleFilePress = useCallback( + (file: FileRecord) => { + router.push({ pathname: "/(tabs)/file-detail", params: { id: String(file.id) } }); + }, + [router] + ); + + if (loading && files.length === 0) { return ( @@ -101,52 +148,88 @@ export default function FilesScreen() { ); } - if (error) { + if (error && files.length === 0) { return ( {error} - Retry + {t("common.retry")} ); } return ( - String(item.id)} - contentContainerStyle={styles.listContent} - renderItem={({ item }) => } - refreshControl={ - - } - onEndReached={handleLoadMore} - onEndReachedThreshold={0.4} - ListEmptyComponent={ - - 📂 - No documents yet. - - Upload a document from the Upload tab to get started. - - - } - ListFooterComponent={ - hasMore && files.length > 0 ? ( - - ) : null - } - /> + + {/* Search bar */} + + + + {searchQuery.length > 0 && ( + + + + )} + + + String(item.id)} + contentContainerStyle={styles.listContent} + renderItem={({ item }) => } + refreshControl={ + + } + onEndReached={handleLoadMore} + onEndReachedThreshold={0.4} + ListEmptyComponent={ + + + + {searchQuery ? t("files.search_empty") : t("files.empty_title")} + + + {searchQuery ? t("files.search_empty_hint") : t("files.empty_hint")} + + + } + ListFooterComponent={ + hasMore && files.length > 0 ? ( + + ) : null + } + /> + ); } -function FileRow({ file }: { file: FileRecord }) { +function FileRow({ file, onPress }: { file: FileRecord; onPress: (file: FileRecord) => void }) { const status = file.processing_status?.status ?? "pending"; + const icon = statusIcon(status); return ( - - {statusEmoji(status)} + onPress(file)} + accessibilityRole="button" + accessibilityLabel={`View details for ${file.original_filename}`} + > + {file.original_filename} @@ -155,14 +238,44 @@ function FileRow({ file }: { file: FileRecord }) { {formatDate(file.created_at)} · {formatBytes(file.file_size)} - {status} - + + {status} + + + ); } const styles = StyleSheet.create({ - list: { flex: 1, backgroundColor: "#f9fafb" }, - listContent: { padding: 16 }, + container: { flex: 1, backgroundColor: "#f9fafb" }, + list: { flex: 1 }, + listContent: { padding: 16, paddingTop: 0 }, + searchContainer: { + flexDirection: "row", + alignItems: "center", + backgroundColor: "#fff", + marginHorizontal: 16, + marginVertical: 12, + borderRadius: 10, + paddingHorizontal: 12, + borderWidth: 1, + borderColor: "#e5e7eb", + minHeight: 44, + }, + searchIcon: { marginRight: 8 }, + searchInput: { + flex: 1, + fontSize: 15, + color: "#111827", + paddingVertical: 10, + }, + clearButton: { + padding: 4, + minWidth: 44, + minHeight: 44, + alignItems: "center", + justifyContent: "center", + }, center: { flex: 1, alignItems: "center", @@ -179,7 +292,6 @@ const styles = StyleSheet.create({ }, retryText: { color: "#fff", fontWeight: "600" }, emptyState: { alignItems: "center", paddingTop: 60 }, - emptyEmoji: { fontSize: 48, marginBottom: 12 }, emptyText: { fontSize: 16, color: "#374151", marginBottom: 8 }, emptyHint: { fontSize: 13, @@ -203,7 +315,7 @@ const rowStyles = StyleSheet.create({ shadowRadius: 4, elevation: 2, }, - icon: { fontSize: 22, marginRight: 12 }, + icon: { marginRight: 12 }, info: { flex: 1 }, filename: { fontSize: 14, @@ -212,6 +324,11 @@ const rowStyles = StyleSheet.create({ marginBottom: 4, }, meta: { fontSize: 12, color: "#6b7280" }, + right: { + flexDirection: "row", + alignItems: "center", + gap: 6, + }, status: { fontSize: 11, color: "#6b7280", diff --git a/mobile/src/screens/LoginScreen.tsx b/mobile/src/screens/LoginScreen.tsx index f278b46e..e92a5596 100644 --- a/mobile/src/screens/LoginScreen.tsx +++ b/mobile/src/screens/LoginScreen.tsx @@ -1,13 +1,16 @@ /** - * LoginScreen – server URL entry and SSO sign-in. + * LoginScreen – server URL entry, SSO sign-in, and QR code login. * - * Renders a server URL input and a "Sign in with SSO" button that opens the - * DocuElevate web login page in the system browser. On success the - * AuthContext stores the API token and navigates to the main app. + * Renders a server URL input, a "Sign in with SSO" button that opens the + * DocuElevate web login page in the system browser, and a "Scan QR Code" + * button that opens the device camera to scan a QR code generated from the + * web interface. On success the AuthContext stores the API token and + * navigates to the main app. */ +import * as Linking from "expo-linking"; import { useRouter } from "expo-router"; -import React, { useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import { ActivityIndicator, Alert, @@ -21,21 +24,60 @@ import { View, } from "react-native"; import { useAuth } from "../context/AuthContext"; +import { useLocale, t } from "../i18n"; export default function LoginScreen() { - const { signIn } = useAuth(); + const { signIn, signInWithQR } = useAuth(); const router = useRouter(); - const [serverUrl, setServerUrl] = useState(""); + const [serverUrl, setServerUrl] = useState("https://app.docuelevate.org"); const [loading, setLoading] = useState(false); + const [qrLoading, setQrLoading] = useState(false); + // Subscribe to language changes so translated strings re-render. + useLocale(); + + // Handle incoming deep links for QR login (docuelevate://qr-login?token=...&server=...) + const handleDeepLink = useCallback( + async (event: { url: string }) => { + try { + const url = new URL(event.url); + if (url.hostname === "qr-login" || url.pathname === "/qr-login") { + const token = url.searchParams.get("token"); + const server = url.searchParams.get("server"); + if (token && server) { + setQrLoading(true); + await signInWithQR(server, token); + } + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : t("login.qr_login_failed"); + Alert.alert(t("login.qr_login_failed"), message); + } finally { + setQrLoading(false); + } + }, + [signInWithQR] + ); + + useEffect(() => { + // Listen for incoming deep links + const subscription = Linking.addEventListener("url", handleDeepLink); + + // Check if the app was opened via a deep link + Linking.getInitialURL().then((url) => { + if (url) handleDeepLink({ url }); + }); + + return () => subscription.remove(); + }, [handleDeepLink]); async function handleSignIn() { const url = serverUrl.trim(); if (!url) { - Alert.alert("Server URL required", "Please enter the URL of your DocuElevate server."); + Alert.alert(t("login.server_url_required"), t("login.server_url_required_msg")); return; } if (!url.startsWith("http://") && !url.startsWith("https://")) { - Alert.alert("Invalid URL", "The server URL must start with http:// or https://"); + Alert.alert(t("login.invalid_url"), t("login.invalid_url_msg")); return; } @@ -43,8 +85,8 @@ export default function LoginScreen() { try { await signIn(url); } catch (err: unknown) { - const message = err instanceof Error ? err.message : "Sign-in failed"; - Alert.alert("Sign-in failed", message); + const message = err instanceof Error ? err.message : t("login.sign_in_failed"); + Alert.alert(t("login.sign_in_failed"), message); } finally { setLoading(false); } @@ -65,12 +107,12 @@ export default function LoginScreen() { /> DocuElevate - Intelligent Document Processing + {t("welcome.tagline")} - Server URL + {t("login.server_url")} {loading ? ( ) : ( - Sign in with SSO + {t("login.sign_in_sso")} )} - - You will be redirected to your organisation's sign-in page. - + + + {t("login.or")} + + + + { + router.push("/(auth)/qr-scanner"); + }} + disabled={loading || qrLoading} + accessibilityRole="button" + accessibilityLabel={t("login.scan_qr")} + > + {qrLoading ? ( + + ) : ( + {t("login.scan_qr")} + )} + + + {t("login.hint")} router.back()} accessibilityRole="button" - accessibilityLabel="Back to welcome screen" + accessibilityLabel={t("login.back")} style={styles.backLink} > - ← Back + {t("login.back")} + + {/* Legal links – accessible pre-login for GDPR / Apple compliance */} + + { + const base = serverUrl.trim() || "https://app.docuelevate.org"; + Linking.openURL(`${base.replace(/\/$/, "")}/privacy`); + }} + accessibilityRole="link" + accessibilityLabel={t("legal.privacy_policy")} + style={styles.legalLinkButton} + > + {t("legal.privacy_policy")} + + · + { + const base = serverUrl.trim() || "https://app.docuelevate.org"; + Linking.openURL(`${base.replace(/\/$/, "")}/terms`); + }} + accessibilityRole="link" + accessibilityLabel={t("legal.terms")} + style={styles.legalLinkButton} + > + {t("legal.terms")} + + · + { + const base = serverUrl.trim() || "https://app.docuelevate.org"; + Linking.openURL(`${base.replace(/\/$/, "")}/imprint`); + }} + accessibilityRole="link" + accessibilityLabel={t("legal.imprint")} + style={styles.legalLinkButton} + > + {t("legal.imprint")} + + ); @@ -184,6 +285,36 @@ const styles = StyleSheet.create({ fontSize: 16, fontWeight: "600", }, + dividerRow: { + flexDirection: "row", + alignItems: "center", + marginVertical: 16, + }, + dividerLine: { + flex: 1, + height: 1, + backgroundColor: "#e5e7eb", + }, + dividerText: { + marginHorizontal: 12, + fontSize: 12, + color: "#9ca3af", + }, + qrButton: { + borderWidth: 1, + borderColor: "#1e40af", + borderRadius: 8, + paddingVertical: 14, + alignItems: "center", + justifyContent: "center", + minHeight: 48, + backgroundColor: "#eff6ff", + }, + qrButtonText: { + color: "#1e40af", + fontSize: 15, + fontWeight: "600", + }, hint: { marginTop: 16, fontSize: 12, @@ -200,4 +331,26 @@ const styles = StyleSheet.create({ fontSize: 13, color: "#6b7280", }, + legalLinks: { + flexDirection: "row", + justifyContent: "center", + alignItems: "center", + marginTop: 16, + flexWrap: "wrap", + }, + legalLinkButton: { + minHeight: 44, + justifyContent: "center", + paddingHorizontal: 4, + }, + legalLinkText: { + fontSize: 12, + color: "#9ca3af", + textDecorationLine: "underline", + }, + legalSeparator: { + fontSize: 12, + color: "#d1d5db", + marginHorizontal: 4, + }, }); diff --git a/mobile/src/screens/ProfileScreen.tsx b/mobile/src/screens/ProfileScreen.tsx index ecc39d72..3990e1df 100644 --- a/mobile/src/screens/ProfileScreen.tsx +++ b/mobile/src/screens/ProfileScreen.tsx @@ -2,6 +2,8 @@ * ProfileScreen – authenticated user profile and settings. */ +import Constants from "expo-constants"; +import * as Linking from "expo-linking"; import React from "react"; import { Alert, @@ -9,30 +11,84 @@ import { Pressable, ScrollView, StyleSheet, - Switch, Text, View, } from "react-native"; import { useAuth } from "../context/AuthContext"; +import { useLocale, getSupportedLanguages, t } from "../i18n"; +import api from "../services/api"; + +const DEFAULT_SERVER_URL = "https://app.docuelevate.org"; export default function ProfileScreen() { const { user, signOut, baseUrl } = useAuth(); + const { lang, setLang } = useLocale(); + + const effectiveBaseUrl = baseUrl || DEFAULT_SERVER_URL; + const appVersion = Constants.expoConfig?.version ?? "1.0.0"; + const languages = getSupportedLanguages(); + + async function handleLanguageSelect(code: string) { + await setLang(code); + // Fire-and-forget: sync the choice to the server so it persists across + // platforms (desktop web will reflect this preference too). + api.setServerLanguage(code).catch(() => { + // Network errors are non-critical – the local change is already applied. + }); + } function handleSignOut() { - Alert.alert("Sign out", "Are you sure you want to sign out?", [ - { text: "Cancel", style: "cancel" }, + Alert.alert(t("profile.sign_out_title"), t("profile.sign_out_msg"), [ + { text: t("common.cancel"), style: "cancel" }, { - text: "Sign out", + text: t("profile.sign_out"), style: "destructive", onPress: signOut, }, ]); } + function handleDeleteAccount() { + Alert.alert( + t("profile.delete_account_title"), + t("profile.delete_account_msg"), + [ + { text: t("common.cancel"), style: "cancel" }, + { + text: t("profile.delete_account"), + style: "destructive", + onPress: () => { + Linking.openURL(`${effectiveBaseUrl}/account/delete`).catch(() => { + Alert.alert(t("common.error"), t("profile.could_not_open", { page: t("profile.delete_account") })); + }); + }, + }, + ] + ); + } + + function openPrivacyPolicy() { + Linking.openURL(`${effectiveBaseUrl}/privacy`).catch(() => { + Alert.alert(t("common.error"), t("profile.could_not_open", { page: t("profile.privacy_policy") })); + }); + } + + function openTermsOfService() { + Linking.openURL(`${effectiveBaseUrl}/terms`).catch(() => { + Alert.alert(t("common.error"), t("profile.could_not_open", { page: t("profile.terms_of_service") })); + }); + } + + function openImprint() { + Linking.openURL(`${effectiveBaseUrl}/imprint`).catch(() => { + Alert.alert(t("common.error"), t("profile.could_not_open", { page: t("profile.imprint") })); + }); + } + if (!user) { return ( - Not signed in + {t("profile.not_signed_in")} ); } @@ -56,44 +112,121 @@ export default function ProfileScreen() { )} {user.display_name ?? user.owner_id} {user.email && {user.email}} - {user.is_admin && Admin} + {user.is_admin && {t("profile.admin")}} {/* Server info */} - Connection + {t("profile.connection")} - Server + {t("profile.server")} - {baseUrl || "–"} + {effectiveBaseUrl} - User ID + {t("profile.user_id")} {user.owner_id} - {/* Danger zone */} + {/* Settings */} + + {t("profile.settings")} + {t("profile.language")} + + {languages.map((l) => ( + handleLanguageSelect(l.code)} + accessibilityRole="button" + accessibilityLabel={`Set language to ${l.label}`} + accessibilityState={{ selected: lang === l.code }} + > + + {l.label} + + + ))} + + + + {/* Legal & Privacy */} + + {t("profile.legal")} + + {t("profile.privacy_policy")} + + + + {t("profile.terms_of_service")} + + + + {t("profile.imprint")} + + + + + {/* Sign out */} - Sign out + {t("profile.sign_out")} + + {/* Account deletion – Apple Guideline 5.1.1(v) */} + + + {t("profile.delete_account")} + + + + {/* App version */} + DocuElevate v{appVersion} ); } const styles = StyleSheet.create({ scroll: { flex: 1, backgroundColor: "#f9fafb" }, - content: { padding: 20 }, + content: { padding: 20, paddingBottom: 40 }, center: { flex: 1, alignItems: "center", @@ -180,6 +313,27 @@ const styles = StyleSheet.create({ maxWidth: "60%", textAlign: "right", }, + linkRow: { + flexDirection: "row", + justifyContent: "space-between", + alignItems: "center", + paddingVertical: 12, + borderBottomWidth: 1, + borderBottomColor: "#f3f4f6", + minHeight: 44, + }, + linkRowLast: { + borderBottomWidth: 0, + }, + linkText: { + fontSize: 15, + color: "#1e40af", + }, + linkChevron: { + fontSize: 18, + color: "#9ca3af", + fontWeight: "600", + }, signOutButton: { backgroundColor: "#fee2e2", borderRadius: 10, @@ -192,4 +346,58 @@ const styles = StyleSheet.create({ fontWeight: "700", fontSize: 15, }, + deleteAccountButton: { + backgroundColor: "#ffffff", + borderRadius: 10, + borderWidth: 1, + borderColor: "#dc2626", + paddingVertical: 14, + alignItems: "center", + minHeight: 48, + }, + deleteAccountText: { + color: "#dc2626", + fontWeight: "600", + fontSize: 14, + }, + versionText: { + fontSize: 12, + color: "#9ca3af", + textAlign: "center", + marginTop: 8, + }, + settingLabel: { + fontSize: 14, + color: "#374151", + fontWeight: "500", + marginBottom: 10, + }, + languageGrid: { + flexDirection: "row", + flexWrap: "wrap", + gap: 8, + }, + languageChip: { + paddingHorizontal: 14, + paddingVertical: 8, + borderRadius: 20, + backgroundColor: "#f3f4f6", + borderWidth: 1, + borderColor: "#e5e7eb", + minHeight: 36, + justifyContent: "center", + }, + languageChipActive: { + backgroundColor: "#dbeafe", + borderColor: "#1e40af", + }, + languageChipText: { + fontSize: 13, + color: "#6b7280", + fontWeight: "500", + }, + languageChipTextActive: { + color: "#1e40af", + fontWeight: "700", + }, }); diff --git a/mobile/src/screens/QRScannerScreen.tsx b/mobile/src/screens/QRScannerScreen.tsx new file mode 100644 index 00000000..696c8e2b --- /dev/null +++ b/mobile/src/screens/QRScannerScreen.tsx @@ -0,0 +1,295 @@ +/** + * QRScannerScreen – camera-based QR code scanner for mobile login. + * + * Opens the device camera and scans for QR codes containing a + * `docuelevate://qr-login?token=...&server=...` payload. On successful + * scan the token is claimed via the API and the user is signed in. + */ + +import { CameraView, useCameraPermissions } from "expo-camera"; +import { useRouter } from "expo-router"; +import React, { useCallback, useRef, useState } from "react"; +import { + ActivityIndicator, + Alert, + Pressable, + StyleSheet, + Text, + View, +} from "react-native"; +import { useAuth } from "../context/AuthContext"; + +export default function QRScannerScreen() { + const { signInWithQR } = useAuth(); + const router = useRouter(); + const [permission, requestPermission] = useCameraPermissions(); + const [scanned, setScanned] = useState(false); + const [processing, setProcessing] = useState(false); + const processingRef = useRef(false); + + const handleBarCodeScanned = useCallback( + async (result: { data: string }) => { + // Prevent duplicate scans while processing + if (processingRef.current) return; + + const { data } = result; + + // Only accept docuelevate:// QR codes + if (!data.startsWith("docuelevate://qr-login")) return; + + processingRef.current = true; + setScanned(true); + setProcessing(true); + + try { + const url = new URL(data); + const token = url.searchParams.get("token"); + const server = url.searchParams.get("server"); + + if (!token || !server) { + Alert.alert("Invalid QR Code", "This QR code does not contain valid login information."); + setScanned(false); + processingRef.current = false; + setProcessing(false); + return; + } + + await signInWithQR(server, token); + // signInWithQR updates AuthContext → AuthGuard redirects to main app + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "QR login failed"; + Alert.alert("QR Login Failed", message); + setScanned(false); + processingRef.current = false; + setProcessing(false); + } + }, + [signInWithQR] + ); + + // Permissions not yet determined + if (!permission) { + return ( + + + + ); + } + + // Permission denied + if (!permission.granted) { + return ( + + + Camera access is required to scan QR codes. + + + Grant Camera Access + + router.back()} + style={styles.backLink} + accessibilityRole="button" + accessibilityLabel="Go back" + > + ← Back + + + ); + } + + return ( + + + + {/* Overlay with scan area indicator */} + + + + + + + + + + + + + + {processing ? ( + + + Signing in… + + ) : ( + + Point your camera at the QR code{"\n"}shown on the DocuElevate web app + + )} + + router.back()} + style={styles.cancelButton} + accessibilityRole="button" + accessibilityLabel="Cancel QR scan" + > + Cancel + + + + + ); +} + +const SCAN_AREA_SIZE = 250; +const CORNER_SIZE = 24; +const CORNER_WIDTH = 3; + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: "#000", + }, + camera: { + flex: 1, + }, + centered: { + flex: 1, + justifyContent: "center", + alignItems: "center", + backgroundColor: "#f3f4f6", + padding: 24, + }, + permissionText: { + fontSize: 16, + color: "#374151", + textAlign: "center", + marginBottom: 20, + }, + permissionButton: { + backgroundColor: "#1e40af", + borderRadius: 8, + paddingVertical: 14, + paddingHorizontal: 24, + minHeight: 48, + alignItems: "center", + justifyContent: "center", + }, + permissionButtonText: { + color: "#fff", + fontSize: 16, + fontWeight: "600", + }, + backLink: { + marginTop: 20, + minHeight: 44, + alignItems: "center", + justifyContent: "center", + }, + backLinkText: { + fontSize: 14, + color: "#6b7280", + }, + overlay: { + ...StyleSheet.absoluteFillObject, + }, + overlayTop: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.5)", + }, + overlayMiddle: { + flexDirection: "row", + height: SCAN_AREA_SIZE, + }, + overlaySide: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.5)", + }, + scanArea: { + width: SCAN_AREA_SIZE, + height: SCAN_AREA_SIZE, + }, + corner: { + position: "absolute", + width: CORNER_SIZE, + height: CORNER_SIZE, + }, + cornerTopLeft: { + top: 0, + left: 0, + borderTopWidth: CORNER_WIDTH, + borderLeftWidth: CORNER_WIDTH, + borderColor: "#fff", + }, + cornerTopRight: { + top: 0, + right: 0, + borderTopWidth: CORNER_WIDTH, + borderRightWidth: CORNER_WIDTH, + borderColor: "#fff", + }, + cornerBottomLeft: { + bottom: 0, + left: 0, + borderBottomWidth: CORNER_WIDTH, + borderLeftWidth: CORNER_WIDTH, + borderColor: "#fff", + }, + cornerBottomRight: { + bottom: 0, + right: 0, + borderBottomWidth: CORNER_WIDTH, + borderRightWidth: CORNER_WIDTH, + borderColor: "#fff", + }, + overlayBottom: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.5)", + alignItems: "center", + paddingTop: 32, + }, + statusContainer: { + flexDirection: "row", + alignItems: "center", + }, + statusText: { + color: "#fff", + fontSize: 16, + fontWeight: "600", + marginLeft: 10, + }, + instructionText: { + color: "#fff", + fontSize: 15, + textAlign: "center", + lineHeight: 22, + }, + cancelButton: { + marginTop: 24, + paddingVertical: 12, + paddingHorizontal: 32, + borderRadius: 8, + borderWidth: 1, + borderColor: "rgba(255,255,255,0.5)", + minHeight: 44, + alignItems: "center", + justifyContent: "center", + }, + cancelButtonText: { + color: "#fff", + fontSize: 15, + fontWeight: "500", + }, +}); diff --git a/mobile/src/screens/UploadScreen.tsx b/mobile/src/screens/UploadScreen.tsx index 0a2788cf..daa00a01 100644 --- a/mobile/src/screens/UploadScreen.tsx +++ b/mobile/src/screens/UploadScreen.tsx @@ -12,7 +12,9 @@ * track the real-time processing status of each uploaded file. */ +import { Ionicons } from "@expo/vector-icons"; import * as DocumentPicker from "expo-document-picker"; +import * as FileSystem from "expo-file-system"; import * as ImagePicker from "expo-image-picker"; import React, { useCallback, useEffect, useRef, useState } from "react"; import { @@ -26,7 +28,9 @@ import { } from "react-native"; import { useAuth } from "../context/AuthContext"; import { useShare } from "../context/ShareContext"; +import { normalizeFileUri } from "../utils/normalizeUri"; import api from "../services/api"; +import { useLocale, t } from "../i18n"; /** Statuses that indicate processing has finished (no further polling needed). */ const TERMINAL_STATUSES = new Set(["completed", "failed", "duplicate"]); @@ -53,6 +57,8 @@ export default function UploadScreen() { const { isAuthenticated } = useAuth(); const { pendingFiles, clearPendingFiles } = useShare(); const [uploads, setUploads] = useState([]); + // Subscribe to language changes so translated strings re-render. + useLocale(); // Keep a ref in sync so the polling interval can read current state without // capturing a stale closure. @@ -61,30 +67,102 @@ export default function UploadScreen() { uploadsRef.current = uploads; }, [uploads]); + // Track URIs that have already been uploaded in this session so that + // duplicate share-sheet deliveries (iOS can fire both the Linking handler + // and +not-found.tsx for the same file) do not trigger repeated uploads. + const uploadedUrisRef = useRef>(new Set()); + // --------------------------------------------------------------------------- // Core helpers (declared before the effects that depend on them) // --------------------------------------------------------------------------- + /** + * Ensure a file URI is accessible for upload. + * + * Files received via the iOS Share Sheet / "Open In…" may reference paths + * outside the app's sandbox or use security-scoped URLs that React Native's + * fetch cannot read directly. This helper copies such files to the app's + * cache directory so the upload can proceed reliably. + * + * URIs from expo-image-picker and expo-document-picker are already in the + * app's cache and are returned unchanged. + */ + const ensureLocalUri = useCallback(async (uri: string, filename: string): Promise => { + // Android content:// URIs are handled natively by React Native's fetch. + if (!uri.startsWith("file://")) return uri; + + // Files already in the app's cache or documents directory are accessible. + const cacheDir = FileSystem.cacheDirectory; + const docDir = FileSystem.documentDirectory; + if (cacheDir && uri.startsWith(cacheDir)) return uri; + if (docDir && uri.startsWith(docDir)) return uri; + + // External file (e.g. from iOS Inbox or security-scoped URL) – copy to + // cache so the upload has guaranteed read access. + const safeName = filename.replace(/[^a-zA-Z0-9._-]/g, "_"); + const destUri = `${cacheDir}shared_${Date.now()}_${safeName}`; + try { + await FileSystem.copyAsync({ from: uri, to: destUri }); + return destUri; + } catch (copyErr) { + // Copy failed – fall back to the original URI (might work for some paths). + console.warn("[ensureLocalUri] copyAsync failed:", { from: uri, to: destUri, error: copyErr }); + return uri; + } + }, []); + const uploadFile = useCallback(async (uri: string, filename: string, mimeType?: string) => { - const id = `${Date.now()}-${filename}`; + // Deduplicate: skip if this exact URI was already uploaded in this session. + // This guards against duplicate share-sheet deliveries from iOS where the + // Linking handler and +not-found.tsx fire for the same file. + const normUri = normalizeFileUri(uri); + if (uploadedUrisRef.current.has(normUri)) { + console.debug("[uploadFile] skipping duplicate URI:", uri); + return; + } + uploadedUrisRef.current.add(normUri); + + const id = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}-${filename}`; setUploads((prev) => [{ id, filename, status: "uploading", uri, mimeType }, ...prev]); try { - const resp = await api.uploadFile(uri, filename, mimeType); - setUploads((prev) => - prev.map((item) => - item.id === id - ? { ...item, status: "done", taskId: resp.task_id, originalFilename: resp.original_filename } - : item - ) - ); + const localUri = await ensureLocalUri(uri, filename); + const resp = await api.uploadFile(localUri, filename, mimeType); + if (resp.status === "duplicate" && resp.duplicate_of) { + // Server rejected the file as a known duplicate — mark as done and + // set the server-side status to "duplicate" so it appears as a + // terminal status and is not polled further. + setUploads((prev) => + prev.map((item) => + item.id === id + ? { + ...item, + status: "done", + fileId: resp.duplicate_of!.original_file_id, + originalFilename: resp.original_filename, + serverStatus: "duplicate", + } + : item + ) + ); + } else { + setUploads((prev) => + prev.map((item) => + item.id === id + ? { ...item, status: "done", taskId: resp.task_id, originalFilename: resp.original_filename } + : item + ) + ); + } } catch (err: unknown) { + // Allow retrying this URI on failure. + uploadedUrisRef.current.delete(normUri); const msg = err instanceof Error ? err.message : "Upload failed"; setUploads((prev) => prev.map((item) => (item.id === id ? { ...item, status: "error", error: msg } : item)) ); } - }, []); + }, [ensureLocalUri]); const retryUpload = useCallback(async (item: UploadItem) => { if (!item.uri) return; @@ -99,21 +177,38 @@ export default function UploadScreen() { ); try { - const resp = await api.uploadFile(item.uri, item.filename, item.mimeType); - setUploads((prev) => - prev.map((u) => - u.id === item.id - ? { ...u, status: "done", taskId: resp.task_id, originalFilename: resp.original_filename } - : u - ) - ); + const localUri = await ensureLocalUri(item.uri, item.filename); + const resp = await api.uploadFile(localUri, item.filename, item.mimeType); + if (resp.status === "duplicate" && resp.duplicate_of) { + setUploads((prev) => + prev.map((u) => + u.id === item.id + ? { + ...u, + status: "done", + fileId: resp.duplicate_of!.original_file_id, + originalFilename: resp.original_filename, + serverStatus: "duplicate", + } + : u + ) + ); + } else { + setUploads((prev) => + prev.map((u) => + u.id === item.id + ? { ...u, status: "done", taskId: resp.task_id, originalFilename: resp.original_filename } + : u + ) + ); + } } catch (err: unknown) { const msg = err instanceof Error ? err.message : "Upload failed"; setUploads((prev) => prev.map((u) => (u.id === item.id ? { ...u, status: "error", error: msg } : u)) ); } - }, []); + }, [ensureLocalUri]); // --------------------------------------------------------------------------- // Polling – check server-side processing status every 5 seconds @@ -176,8 +271,8 @@ export default function UploadScreen() { const { status } = await ImagePicker.requestCameraPermissionsAsync(); if (status !== "granted") { Alert.alert( - "Camera access required", - "Please grant camera access in Settings to capture documents." + t("upload.camera_access_title"), + t("upload.camera_access_msg") ); return; } @@ -199,8 +294,8 @@ export default function UploadScreen() { const { status } = await ImagePicker.requestMediaLibraryPermissionsAsync(); if (status !== "granted") { Alert.alert( - "Photo library access required", - "Please grant photo library access in Settings to select images." + t("upload.photo_access_title"), + t("upload.photo_access_msg") ); return; } @@ -209,14 +304,17 @@ export default function UploadScreen() { mediaTypes: ["images"], quality: 0.9, allowsEditing: false, + allowsMultipleSelection: true, }); if (!result.canceled && result.assets.length > 0) { - const asset = result.assets[0]; - // Derive extension from MIME type so the filename matches the actual format - const ext = asset.mimeType?.split("/")[1]?.replace("jpeg", "jpg") ?? "jpg"; - const filename = asset.fileName ?? `photo_${Date.now()}.${ext}`; - await uploadFile(asset.uri, filename, asset.mimeType ?? "image/jpeg"); + for (let i = 0; i < result.assets.length; i++) { + const asset = result.assets[i]; + // Derive extension from MIME type so the filename matches the actual format + const ext = asset.mimeType?.split("/")[1]?.replace("jpeg", "jpg") ?? "jpg"; + const filename = asset.fileName ?? `photo_${Date.now()}_${i}.${ext}`; + await uploadFile(asset.uri, filename, asset.mimeType ?? "image/jpeg"); + } } } @@ -234,14 +332,14 @@ export default function UploadScreen() { } } } catch (err: unknown) { - Alert.alert("File picker error", err instanceof Error ? err.message : "Could not open file picker"); + Alert.alert(t("upload.file_picker_error"), err instanceof Error ? err.message : t("upload.file_picker_error_msg")); } } if (!isAuthenticated) { return ( - Please sign in to upload documents. + {t("upload.sign_in_required")} ); } @@ -254,30 +352,30 @@ export default function UploadScreen() { style={[styles.actionButton, styles.cameraButton]} onPress={handleCamera} accessibilityRole="button" - accessibilityLabel="Capture document with camera" + accessibilityLabel={t("upload.capture_label")} > - 📷 - Camera + + {t("upload.camera")} - 🖼️ - Photos + + {t("upload.photos")} - 📄 - Files + + {t("upload.files")} @@ -285,13 +383,9 @@ export default function UploadScreen() { {uploads.length === 0 ? ( - ☁️ - - Tap Camera, Photos, or Files to upload a document. - - - You can also share files from other apps directly to DocuElevate. - + + {t("upload.empty_title")} + {t("upload.empty_hint")} ) : ( uploads.map((item) => ( @@ -304,21 +398,24 @@ export default function UploadScreen() { } function UploadRow({ item, onRetry }: { item: UploadItem; onRetry: (item: UploadItem) => void }) { - const uploadIcons: Record = { - pending: "⏳", - uploading: "⬆️", - done: "✅", - error: "❌", + // Subscribe to language changes so status labels re-render. + useLocale(); + + const uploadIconProps: Record = { + pending: { name: "time-outline", color: "#6b7280" }, + uploading: { name: "arrow-up-circle-outline", color: "#1e40af" }, + done: { name: "checkmark-circle", color: "#059669" }, + error: { name: "close-circle", color: "#dc2626" }, }; /** Human-readable label for the server-side processing status. */ function serverStatusLabel(s: string): string { const labels: Record = { - pending: "Queued for processing…", - processing: "Processing…", - completed: "Processed ✓", - failed: "Processing failed", - duplicate: "Duplicate – already processed", + pending: t("upload.status_queued"), + processing: t("upload.status_processing"), + completed: t("upload.status_completed"), + failed: t("upload.status_failed"), + duplicate: t("upload.status_duplicate"), }; return labels[s] ?? s; } @@ -327,9 +424,9 @@ function UploadRow({ item, onRetry }: { item: UploadItem; onRetry: (item: Upload function handleLongPress() { if (!canRetry) return; - Alert.alert("Retry Upload", `Do you want to retry uploading "${item.filename}"?`, [ - { text: "Cancel", style: "cancel" }, - { text: "Retry", onPress: () => onRetry(item) }, + Alert.alert(t("upload.retry_title"), t("upload.retry_msg", { filename: item.filename }), [ + { text: t("common.cancel"), style: "cancel" }, + { text: t("common.retry"), onPress: () => onRetry(item) }, ]); } @@ -339,10 +436,10 @@ function UploadRow({ item, onRetry }: { item: UploadItem; onRetry: (item: Upload onPress={canRetry ? () => onRetry(item) : undefined} style={rowStyles.row} accessibilityRole={canRetry ? "button" : "none"} - accessibilityLabel={canRetry ? `Retry uploading ${item.filename}` : undefined} + accessibilityLabel={canRetry ? `${t("common.retry")} ${item.filename}` : undefined} accessibilityHint={canRetry ? "Tap or long-press to retry this upload" : undefined} > - {uploadIcons[item.status]} + {item.filename} @@ -351,7 +448,7 @@ function UploadRow({ item, onRetry }: { item: UploadItem; onRetry: (item: Upload )} {item.status === "done" && !item.serverStatus && ( - Queued for processing… + {t("upload.status_queued")} )} {item.status === "done" && item.serverStatus && ( {item.error} {canRetry && ( - Tap to retry + {t("upload.tap_retry")} )} )} @@ -397,7 +494,7 @@ const styles = StyleSheet.create({ cameraButton: { backgroundColor: "#1e40af" }, photoLibraryButton: { backgroundColor: "#7c3aed" }, fileButton: { backgroundColor: "#059669" }, - actionIcon: { fontSize: 28, marginBottom: 6 }, + actionIcon: { marginBottom: 6 }, actionLabel: { color: "#fff", fontSize: 14, @@ -409,7 +506,6 @@ const styles = StyleSheet.create({ alignItems: "center", paddingTop: 60, }, - emptyEmoji: { fontSize: 48, marginBottom: 12 }, emptyText: { fontSize: 16, color: "#374151", @@ -443,7 +539,7 @@ const rowStyles = StyleSheet.create({ shadowRadius: 4, elevation: 2, }, - icon: { fontSize: 22, marginRight: 12 }, + icon: { marginRight: 12 }, info: { flex: 1 }, filename: { fontSize: 14, diff --git a/mobile/src/screens/WelcomeScreen.tsx b/mobile/src/screens/WelcomeScreen.tsx index f779456c..8f3b8127 100644 --- a/mobile/src/screens/WelcomeScreen.tsx +++ b/mobile/src/screens/WelcomeScreen.tsx @@ -6,6 +6,7 @@ */ import { useRouter } from "expo-router"; +import * as Linking from "expo-linking"; import React from "react"; import { Image, @@ -16,27 +17,31 @@ import { View, } from "react-native"; import { SafeAreaView } from "react-native-safe-area-context"; - -const FEATURES: { icon: string; title: string; description: string }[] = [ - { - icon: "🔍", - title: "OCR & Text Extraction", - description: "Convert scanned PDFs and images into fully searchable text automatically.", - }, - { - icon: "🤖", - title: "AI Metadata Extraction", - description: "AI classifies documents and pulls out key fields like dates, amounts, and subjects.", - }, - { - icon: "☁️", - title: "Multi-Cloud Storage", - description: "Route processed files to Dropbox, Google Drive, OneDrive, S3, Nextcloud, and more.", - }, -]; +import { useLocale, t } from "../i18n"; export default function WelcomeScreen() { const router = useRouter(); + // Subscribe to language changes so translated strings re-render. + useLocale(); + + const features = [ + { + icon: "🔍", + title: t("welcome.feature_ocr_title"), + description: t("welcome.feature_ocr_desc"), + }, + { + icon: "🤖", + title: t("welcome.feature_ai_title"), + description: t("welcome.feature_ai_desc"), + }, + { + icon: "☁️", + title: t("welcome.feature_cloud_title"), + description: t("welcome.feature_cloud_desc"), + }, + ]; + return ( DocuElevate - Intelligent Document Processing - - Ingest documents, run OCR, extract metadata with AI, and route files - to your cloud storage — all in one seamless pipeline. - + {t("welcome.tagline")} + {t("welcome.description")} {/* Feature highlights */} - {FEATURES.map((feature) => ( + {features.map((feature) => ( {feature.icon} @@ -79,14 +81,42 @@ export default function WelcomeScreen() { style={({ pressed }) => [styles.button, pressed && styles.buttonPressed]} onPress={() => router.push("/(auth)/login")} accessibilityRole="button" - accessibilityLabel="Get started — connect to your DocuElevate server" + accessibilityLabel={t("welcome.get_started")} > - Get Started + {t("welcome.get_started")} - - Connect to your self-hosted or cloud DocuElevate server. - + {t("welcome.hint")} + + {/* Legal links – accessible pre-login for GDPR / Apple compliance */} + + Linking.openURL("https://app.docuelevate.org/privacy")} + accessibilityRole="link" + accessibilityLabel={t("legal.privacy_policy")} + style={styles.legalLinkButton} + > + {t("legal.privacy_policy")} + + · + Linking.openURL("https://app.docuelevate.org/terms")} + accessibilityRole="link" + accessibilityLabel={t("legal.terms")} + style={styles.legalLinkButton} + > + {t("legal.terms")} + + · + Linking.openURL("https://app.docuelevate.org/imprint")} + accessibilityRole="link" + accessibilityLabel={t("legal.imprint")} + style={styles.legalLinkButton} + > + {t("legal.imprint")} + + ); @@ -206,4 +236,26 @@ const styles = StyleSheet.create({ color: "rgba(255,255,255,0.55)", textAlign: "center", }, + legalLinks: { + flexDirection: "row", + justifyContent: "center", + alignItems: "center", + marginTop: 20, + flexWrap: "wrap", + }, + legalLinkButton: { + minHeight: 44, + justifyContent: "center", + paddingHorizontal: 4, + }, + legalLinkText: { + fontSize: 12, + color: "rgba(255,255,255,0.65)", + textDecorationLine: "underline", + }, + legalSeparator: { + fontSize: 12, + color: "rgba(255,255,255,0.45)", + marginHorizontal: 4, + }, }); diff --git a/mobile/src/services/api.ts b/mobile/src/services/api.ts index 1853422a..27861f85 100644 --- a/mobile/src/services/api.ts +++ b/mobile/src/services/api.ts @@ -26,6 +26,7 @@ export interface WhoAmIResponse { email: string | null; avatar_url: string | null; is_admin: boolean; + preferred_language: string | null; } export interface GenerateTokenResponse { @@ -35,6 +36,14 @@ export interface GenerateTokenResponse { created_at: string; } +export interface QRClaimResponse { + token: string; + token_id: number; + name: string; + owner_id: string; + created_at: string; +} + export interface DeviceRegistration { push_token: string; device_name?: string; @@ -58,10 +67,42 @@ export interface FileRecord { } export interface UploadResponse { - task_id: string; + task_id?: string; status: string; original_filename: string; stored_filename: string; + duplicate_of?: { + duplicate_type: string; + original_file_id: number; + original_filename: string; + message: string; + }; +} + +export interface ProcessingLog { + id: number; + task_id: string; + step_name: string; + status: string; + message: string; + timestamp: string; +} + +export interface FileDetail { + file: { + id: number; + filehash: string; + original_filename: string; + local_filename: string; + file_size: number; + mime_type: string; + created_at: string; + }; + processing_status: ProcessingStatus; + logs: ProcessingLog[]; + files_on_disk: { + original: boolean; + }; } // --------------------------------------------------------------------------- @@ -157,11 +198,23 @@ class DocuElevateAPI { }); } + /** Claim a QR login challenge and receive an API token. */ + async claimQRChallenge(challengeToken: string, deviceName: string): Promise { + return this.request("POST", "/api/qr-auth/claim", { + body: { challenge_token: challengeToken, device_name: deviceName }, + }); + } + /** Return profile information for the authenticated user. */ async whoAmI(): Promise { return this.request("GET", "/api/mobile/whoami"); } + /** Sync the user's preferred UI language to the server. */ + async setServerLanguage(lang: string): Promise { + await this.request("POST", "/api/i18n/language", { body: { language: lang } }); + } + // ------------------------------------------------------------------------- // Push notifications // ------------------------------------------------------------------------- @@ -208,6 +261,11 @@ class DocuElevateAPI { ); return data.processing_status; } + + /** Get full file details including processing logs. */ + async getFileDetail(fileId: number): Promise { + return this.request("GET", `/api/files/${fileId}`); + } } export const api = new DocuElevateAPI(); diff --git a/mobile/src/utils/mimeTypes.ts b/mobile/src/utils/mimeTypes.ts new file mode 100644 index 00000000..82f6de91 --- /dev/null +++ b/mobile/src/utils/mimeTypes.ts @@ -0,0 +1,44 @@ +/** + * Shared MIME type utilities for the DocuElevate mobile app. + * + * Used by the Linking handler in _layout.tsx, the catch-all +not-found.tsx, + * and any other code that needs to infer a MIME type from a file extension. + */ + +/** + * Common MIME type mappings for file extensions. + * Used to infer the MIME type of files shared via the Share Sheet / "Open In…" + * so the server receives a correct Content-Type instead of application/octet-stream. + */ +export const EXT_TO_MIME: Record = { + pdf: "application/pdf", + jpg: "image/jpeg", + jpeg: "image/jpeg", + png: "image/png", + gif: "image/gif", + bmp: "image/bmp", + tiff: "image/tiff", + tif: "image/tiff", + webp: "image/webp", + heic: "image/heic", + heif: "image/heif", + txt: "text/plain", + csv: "text/csv", + doc: "application/msword", + docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + xls: "application/vnd.ms-excel", + xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ppt: "application/vnd.ms-powerpoint", + pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation", + rtf: "application/rtf", + html: "text/html", + xml: "application/xml", + json: "application/json", + zip: "application/zip", +}; + +/** Infer MIME type from a filename's extension, or undefined if unknown. */ +export function mimeTypeFromFilename(filename: string): string | undefined { + const ext = filename.split(".").pop()?.toLowerCase(); + return ext ? EXT_TO_MIME[ext] : undefined; +} diff --git a/mobile/src/utils/normalizeUri.ts b/mobile/src/utils/normalizeUri.ts new file mode 100644 index 00000000..d32e7d7f --- /dev/null +++ b/mobile/src/utils/normalizeUri.ts @@ -0,0 +1,20 @@ +/** + * Normalise a file URI for deduplication. + * + * - Decode percent-encoding (`%20` → ` `) + * - Collapse consecutive slashes after the scheme (`file:////` → `file:///`) + * - Strip trailing slashes + */ +export function normalizeFileUri(uri: string): string { + let norm: string; + try { + norm = decodeURIComponent(uri); + } catch { + norm = uri; + } + // Collapse multiple slashes after the scheme (e.g. file://// → file:///) + norm = norm.replace(/^(file:\/\/)\/{2,}/, "$1/"); + // Strip trailing slash + norm = norm.replace(/\/+$/, ""); + return norm; +} diff --git a/requirements.txt b/requirements.txt index b4f35b22..a19105ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,3 +60,4 @@ sentry-sdk[fastapi,celery,sqlalchemy]>=2.20.0,<3.0.0 strawberry-graphql[fastapi]>=0.243.0,<1.0.0 aiofiles>=24.1.0 # Asynchronous file I/O support +segno>=1.6.0 # Pure-Python QR code generator (server-side rendering, no Pillow dependency) diff --git a/scripts/check_alembic_migrations.py b/scripts/check_alembic_migrations.py new file mode 100644 index 00000000..c399aef5 --- /dev/null +++ b/scripts/check_alembic_migrations.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Validate Alembic migration chain integrity. + +This script checks the migration files in ``migrations/versions/`` for +common problems that arise when multiple feature branches add migrations +in parallel and then get merged into *main*. + +Checks performed +~~~~~~~~~~~~~~~~ +1. **Multiple heads** – more than one migration without a child means the + chain has diverged and a merge migration is needed. +2. **Broken down-revision references** – a migration points to a + ``down_revision`` that does not exist. +3. **Duplicate revision IDs** – two files declare the same ``revision``. +4. **Revision / filename mismatch** – the ``revision`` variable inside a + file does not match the stem of the filename (minus the numeric + prefix). + +Exit codes +~~~~~~~~~~ +* **0** – all checks passed. +* **1** – one or more problems detected (details printed to *stderr*). +* **2** – unexpected runtime error. + +Usage:: + + python scripts/check_alembic_migrations.py # from repo root + python scripts/check_alembic_migrations.py --verbose # extra detail +""" + +from __future__ import annotations + +import argparse +import ast +import re +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_REVISION_RE = re.compile(r'^revision\s*(?::\s*str\s*)?=\s*["\'](.+?)["\']', re.MULTILINE) +_DOWN_REV_RE = re.compile( + r"^down_revision\s*(?::\s*Union\[str,\s*(?:None|tuple)\]\s*)?=\s*(.+)", + re.MULTILINE, +) + + +def _parse_down_revision(raw: str) -> list[str] | None: + """Parse a ``down_revision`` value into a list of parent revisions. + + Returns ``None`` for the root migration (``down_revision = None``). + Returns a list with one or more strings otherwise. Tuples are + returned for merge migrations (e.g. ``("017_a", "017_b")``). + """ + # Strip inline comments (e.g. ``None # type: ignore``) + raw = raw.strip() + if "#" in raw: + raw = raw[: raw.index("#")].strip() + try: + value = ast.literal_eval(raw) + except (ValueError, SyntaxError): + return [raw.strip("\"' ")] + + if value is None: + return None + if isinstance(value, str): + return [value] + if isinstance(value, (tuple, list)): + return [str(v) for v in value] + return [str(value)] + + +def _parse_migration(path: Path) -> dict | None: + """Extract ``revision`` and ``down_revision`` from a migration file.""" + text = path.read_text(encoding="utf-8") + + rev_match = _REVISION_RE.search(text) + down_match = _DOWN_REV_RE.search(text) + + if not rev_match: + return None # not a valid migration file + + revision = rev_match.group(1) + down_revision = _parse_down_revision(down_match.group(1)) if down_match else None + + return { + "path": path, + "revision": revision, + "down_revision": down_revision, + } + + +# --------------------------------------------------------------------------- +# Checks +# --------------------------------------------------------------------------- + + +def check_migrations(versions_dir: Path, *, verbose: bool = False) -> list[str]: + """Run all migration-chain checks and return a list of error messages.""" + errors: list[str] = [] + + # Collect all migrations ------------------------------------------------ + migrations: dict[str, dict] = {} + py_files = sorted(versions_dir.glob("*.py")) + if not py_files: + errors.append(f"No migration files found in {versions_dir}") + return errors + + for path in py_files: + if path.name == "__init__.py": + continue + info = _parse_migration(path) + if info is None: + if verbose: + print(f" SKIP {path.name} (no revision found)", file=sys.stderr) + continue + rev = info["revision"] + + # Check 1 – duplicate revision IDs + if rev in migrations: + errors.append(f"Duplicate revision '{rev}' in:\n - {migrations[rev]['path'].name}\n - {path.name}") + else: + migrations[rev] = info + + if verbose: + parents = info["down_revision"] or ["(root)"] + print(f" {rev} ← {', '.join(parents)}", file=sys.stderr) + + # Build child map ------------------------------------------------------- + all_revisions = set(migrations.keys()) + children: dict[str, list[str]] = {rev: [] for rev in all_revisions} + + for rev, info in migrations.items(): + parents = info["down_revision"] + if parents is None: + continue + for parent in parents: + # Check 2 – broken down_revision references + if parent not in all_revisions: + errors.append( + f"Broken chain: '{rev}' ({info['path'].name}) references " + f"down_revision '{parent}' which does not exist." + ) + else: + children[parent].append(rev) + + # Check 3 – multiple heads (revisions with no children) ----------------- + heads = [rev for rev, kids in children.items() if not kids] + if len(heads) > 1: + head_details = "\n".join(f" - {h} ({migrations[h]['path'].name})" for h in sorted(heads)) + errors.append( + f"Multiple migration heads detected ({len(heads)}). " + f"Create a merge migration to resolve:\n{head_details}\n\n" + f' Fix: alembic merge heads -m "merge_parallel_branches"' + ) + + # Check 4 – revision / filename consistency ----------------------------- + for rev, info in migrations.items(): + stem = info["path"].stem # e.g. "017_add_pipelines" + if rev != stem: + errors.append( + f"Filename mismatch: file '{info['path'].name}' declares " + f"revision='{rev}' but filename stem is '{stem}'." + ) + + return errors + + +# --------------------------------------------------------------------------- +# CLI entry-point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + """CLI entry-point. Returns 0 on success, 1 on failure, 2 on error.""" + parser = argparse.ArgumentParser(description="Check Alembic migration chain integrity.") + parser.add_argument( + "--versions-dir", + type=Path, + default=Path("migrations/versions"), + help="Path to Alembic versions directory (default: migrations/versions)", + ) + parser.add_argument("--verbose", "-v", action="store_true", help="Print extra diagnostic info") + args = parser.parse_args(argv) + + if not args.versions_dir.is_dir(): + print(f"ERROR: versions directory not found: {args.versions_dir}", file=sys.stderr) + return 2 + + if args.verbose: + print("Scanning migrations…", file=sys.stderr) + + errors = check_migrations(args.versions_dir, verbose=args.verbose) + + if errors: + print(f"\n{'=' * 60}", file=sys.stderr) + print(f" Migration chain problems found: {len(errors)}", file=sys.stderr) + print(f"{'=' * 60}\n", file=sys.stderr) + for i, err in enumerate(errors, 1): + print(f" [{i}] {err}\n", file=sys.stderr) + return 1 + + print("✓ Migration chain is valid.", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index f2655212..fd110d82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,6 +63,7 @@ from app.models import ( # noqa: F401, E402 ApiToken, AuditLog, AutomationHook, + ClassificationRuleModel, ComplianceTemplate, DocumentMetadata, FileRecord, @@ -116,6 +117,7 @@ def client(db_session) -> TestClient: # Import the canonical get_db function from app.database import get_db + from app.middleware.upload_rate_limit import require_upload_rate_limit # Override the get_db dependency to use our test database def override_get_db(): @@ -127,6 +129,14 @@ def client(db_session) -> TestClient: # Override the single canonical get_db dependency fastapi_app.dependency_overrides[get_db] = override_get_db + # Disable per-user upload rate limiting in tests so that upload-heavy + # test suites are not rejected with 429 Too Many Requests. + async def _no_rate_limit() -> None: + """No-op override: skip upload rate limiting during tests.""" + return None + + fastapi_app.dependency_overrides[require_upload_rate_limit] = _no_rate_limit + # Use base_url to satisfy TrustedHostMiddleware with TestClient(fastapi_app, base_url="http://localhost") as test_client: yield test_client diff --git a/tests/test_allowed_types.py b/tests/test_allowed_types.py index 23ad5f37..5802e5b2 100644 --- a/tests/test_allowed_types.py +++ b/tests/test_allowed_types.py @@ -83,6 +83,8 @@ class TestGotenbergCoverageDocuments: ".tif", ".webp", ".svg", + ".heic", + ".heif", } _html_extensions = {".html", ".htm"} _markdown_extensions = {".md", ".markdown"} diff --git a/tests/test_api_classification_rules.py b/tests/test_api_classification_rules.py new file mode 100644 index 00000000..a519c6a4 --- /dev/null +++ b/tests/test_api_classification_rules.py @@ -0,0 +1,290 @@ +"""Tests for the classification rules API endpoints. + +Covers CRUD operations, validation, and access control for +``/api/classification-rules``. +""" + +import pytest + +from app.models import ClassificationRuleModel + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_rule(db_session, owner_id="anonymous", **overrides): + """Insert a ClassificationRuleModel and return it.""" + defaults = { + "owner_id": owner_id, + "name": "test_rule", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": r"(?i)invoice", + "priority": 0, + "case_sensitive": False, + "enabled": True, + } + defaults.update(overrides) + rule = ClassificationRuleModel(**defaults) + db_session.add(rule) + db_session.commit() + db_session.refresh(rule) + return rule + + +# --------------------------------------------------------------------------- +# Categories & Rule Types endpoints +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestCategoriesEndpoint: + """Tests for GET /api/classification-rules/categories.""" + + def test_list_categories(self, client): + """Should return a dict of built-in categories.""" + r = client.get("/api/classification-rules/categories") + assert r.status_code == 200 + data = r.json() + assert isinstance(data, dict) + assert "invoice" in data + assert "contract" in data + assert "receipt" in data + assert "unknown" in data + + +@pytest.mark.unit +class TestRuleTypesEndpoint: + """Tests for GET /api/classification-rules/rule-types.""" + + def test_list_rule_types(self, client): + """Should return a list of valid rule types.""" + r = client.get("/api/classification-rules/rule-types") + assert r.status_code == 200 + data = r.json() + assert isinstance(data, list) + assert len(data) == 3 + type_values = {item["type"] for item in data} + assert "filename_pattern" in type_values + assert "content_keyword" in type_values + assert "metadata_match" in type_values + + +# --------------------------------------------------------------------------- +# CRUD Operations +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestClassificationRuleCRUD: + """Full CRUD test-suite for classification rules.""" + + def test_list_rules_empty(self, client): + """List returns an empty array when no rules exist.""" + r = client.get("/api/classification-rules/") + assert r.status_code == 200 + assert r.json() == [] + + def test_create_rule(self, client): + """POST should create a new classification rule.""" + r = client.post( + "/api/classification-rules/", + json={ + "name": "My Invoice Rule", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": r"(?i)rechnung", + "priority": 10, + }, + ) + assert r.status_code == 201 + data = r.json() + assert data["name"] == "My Invoice Rule" + assert data["category"] == "invoice" + assert data["rule_type"] == "filename_pattern" + assert data["priority"] == 10 + assert data["enabled"] is True + assert data["id"] is not None + + def test_create_rule_invalid_type_rejected(self, client): + """Creating a rule with an invalid rule_type should be rejected.""" + r = client.post( + "/api/classification-rules/", + json={ + "name": "Bad Rule", + "category": "test", + "rule_type": "invalid_type", + "pattern": "test", + }, + ) + assert r.status_code == 400 + + def test_create_duplicate_name_rejected(self, client): + """Creating two rules with the same name should be rejected.""" + payload = { + "name": "Dupe Rule", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "test", + } + r1 = client.post("/api/classification-rules/", json=payload) + assert r1.status_code == 201 + r2 = client.post("/api/classification-rules/", json=payload) + assert r2.status_code == 409 + + def test_get_rule(self, client): + """GET should return a specific rule by ID.""" + create_resp = client.post( + "/api/classification-rules/", + json={ + "name": "Get Test Rule", + "category": "contract", + "rule_type": "content_keyword", + "pattern": "agreement|terms", + }, + ) + rule_id = create_resp.json()["id"] + + r = client.get(f"/api/classification-rules/{rule_id}") + assert r.status_code == 200 + assert r.json()["name"] == "Get Test Rule" + assert r.json()["category"] == "contract" + + def test_get_nonexistent_rule(self, client): + """GET for a nonexistent rule should return 404.""" + r = client.get("/api/classification-rules/99999") + assert r.status_code == 404 + + def test_update_rule(self, client): + """PUT should update an existing rule.""" + create_resp = client.post( + "/api/classification-rules/", + json={ + "name": "Update Test", + "category": "receipt", + "rule_type": "filename_pattern", + "pattern": "receipt", + }, + ) + rule_id = create_resp.json()["id"] + + r = client.put( + f"/api/classification-rules/{rule_id}", + json={"category": "invoice", "priority": 50}, + ) + assert r.status_code == 200 + assert r.json()["category"] == "invoice" + assert r.json()["priority"] == 50 + # Name should be unchanged + assert r.json()["name"] == "Update Test" + + def test_update_nonexistent_rule(self, client): + """PUT for a nonexistent rule should return 404.""" + r = client.put("/api/classification-rules/99999", json={"category": "test"}) + assert r.status_code == 404 + + def test_update_invalid_rule_type_rejected(self, client): + """PUT with an invalid rule_type should be rejected.""" + create_resp = client.post( + "/api/classification-rules/", + json={ + "name": "Invalid Update", + "category": "test", + "rule_type": "filename_pattern", + "pattern": "test", + }, + ) + rule_id = create_resp.json()["id"] + + r = client.put( + f"/api/classification-rules/{rule_id}", + json={"rule_type": "bad_type"}, + ) + assert r.status_code == 400 + + def test_delete_rule(self, client): + """DELETE should remove the rule.""" + create_resp = client.post( + "/api/classification-rules/", + json={ + "name": "Delete Test", + "category": "test", + "rule_type": "content_keyword", + "pattern": "test", + }, + ) + rule_id = create_resp.json()["id"] + + r = client.delete(f"/api/classification-rules/{rule_id}") + assert r.status_code == 204 + + # Verify it's gone + r2 = client.get(f"/api/classification-rules/{rule_id}") + assert r2.status_code == 404 + + def test_delete_nonexistent_rule(self, client): + """DELETE for a nonexistent rule should return 404.""" + r = client.delete("/api/classification-rules/99999") + assert r.status_code == 404 + + def test_list_rules_after_create(self, client): + """List should return created rules.""" + client.post( + "/api/classification-rules/", + json={ + "name": "List Rule 1", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "test1", + }, + ) + client.post( + "/api/classification-rules/", + json={ + "name": "List Rule 2", + "category": "contract", + "rule_type": "content_keyword", + "pattern": "test2", + }, + ) + r = client.get("/api/classification-rules/") + assert r.status_code == 200 + assert len(r.json()) == 2 + + def test_create_rule_with_all_fields(self, client): + """Create a rule providing all optional fields.""" + r = client.post( + "/api/classification-rules/", + json={ + "name": "Full Rule", + "category": "tax_document", + "rule_type": "metadata_match", + "pattern": "department=finance", + "priority": 100, + "case_sensitive": True, + "enabled": False, + }, + ) + assert r.status_code == 201 + data = r.json() + assert data["case_sensitive"] is True + assert data["enabled"] is False + assert data["priority"] == 100 + + def test_create_rule_defaults(self, client): + """Create a rule with minimal fields to test defaults.""" + r = client.post( + "/api/classification-rules/", + json={ + "name": "Minimal Rule", + "category": "invoice", + "rule_type": "filename_pattern", + "pattern": "test", + }, + ) + assert r.status_code == 201 + data = r.json() + assert data["priority"] == 0 + assert data["case_sensitive"] is False + assert data["enabled"] is True diff --git a/tests/test_api_dropbox.py b/tests/test_api_dropbox.py index aacc57bb..71139d7b 100644 --- a/tests/test_api_dropbox.py +++ b/tests/test_api_dropbox.py @@ -416,3 +416,253 @@ class TestSaveDropboxSettings: # .env write is best-effort; endpoint should still succeed via DB write assert response.status_code == 200 assert response.json()["status"] == "success" + + +@pytest.mark.unit +class TestListDropboxFolders: + """Tests for list_dropbox_folders endpoint.""" + + @patch("app.api.dropbox.requests.post") + def test_list_folders_success(self, mock_post, client): + """Test successful folder listing at root.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "entries": [ + {".tag": "folder", "name": "Documents", "path_display": "/Documents", "id": "id:1"}, + {".tag": "folder", "name": "Photos", "path_display": "/Photos", "id": "id:2"}, + {".tag": "file", "name": "readme.txt", "path_display": "/readme.txt", "id": "id:3"}, + ], + "has_more": False, + } + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["folders"]) == 2 + assert data["folders"][0]["name"] == "Documents" + assert data["folders"][1]["name"] == "Photos" + assert data["path"] == "/" + assert data["has_more"] is False + + @patch("app.api.dropbox.requests.post") + def test_list_folders_subfolder(self, mock_post, client): + """Test listing folders in a subfolder.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "entries": [ + {".tag": "folder", "name": "Invoices", "path_display": "/Documents/Invoices", "id": "id:4"}, + ], + "has_more": False, + } + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": "/Documents"}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["folders"]) == 1 + assert data["folders"][0]["path"] == "/Documents/Invoices" + + @patch("app.api.dropbox.requests.post") + def test_list_folders_empty(self, mock_post, client): + """Test listing folders in an empty directory.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"entries": [], "has_more": False} + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": "/EmptyFolder"}, + ) + + assert response.status_code == 200 + assert len(response.json()["folders"]) == 0 + + @patch("app.api.dropbox.requests.post") + def test_list_folders_unauthorized(self, mock_post, client): + """Test listing folders with invalid token returns 401.""" + mock_response = Mock() + mock_response.status_code = 401 + mock_response.text = "Invalid access token" + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "bad-token", "path": ""}, + ) + + assert response.status_code == 401 + + @patch("app.api.dropbox.requests.post") + def test_list_folders_api_error(self, mock_post, client): + """Test listing folders when Dropbox API returns an error.""" + mock_response = Mock() + mock_response.status_code = 500 + mock_response.text = "Internal server error" + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 502 + + @patch("app.api.dropbox.requests.post") + def test_list_folders_root_path_normalization(self, mock_post, client): + """Test that '/' is normalized to empty string for Dropbox API.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"entries": [], "has_more": False} + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": "/"}, + ) + + assert response.status_code == 200 + # Check the actual API call used empty string for root + call_args = mock_post.call_args + assert call_args[1]["json"]["path"] == "" + + @patch("app.api.dropbox.requests.post") + def test_list_folders_sorted_alphabetically(self, mock_post, client): + """Test that folders are returned in alphabetical order.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "entries": [ + {".tag": "folder", "name": "Zebra", "path_display": "/Zebra", "id": "id:1"}, + {".tag": "folder", "name": "Alpha", "path_display": "/Alpha", "id": "id:2"}, + {".tag": "folder", "name": "middle", "path_display": "/middle", "id": "id:3"}, + ], + "has_more": False, + } + mock_post.return_value = mock_response + + response = client.post( + "/api/dropbox/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 200 + names = [f["name"] for f in response.json()["folders"]] + assert names == ["Alpha", "middle", "Zebra"] + + +class TestBuildDropboxRedirectUri: + """Tests for the _build_dropbox_redirect_uri helper.""" + + def test_uses_public_base_url_when_set(self): + """When PUBLIC_BASE_URL is configured, redirect URI should use it.""" + from unittest.mock import MagicMock + + with patch("app.api.dropbox.settings") as mock_settings: + mock_settings.public_base_url = "https://myapp.example.com" + from app.api.dropbox import _build_dropbox_redirect_uri + + mock_request = MagicMock() + result = _build_dropbox_redirect_uri(mock_request) + + assert result == "https://myapp.example.com/dropbox-callback" + + def test_uses_public_base_url_strips_trailing_slash(self): + """PUBLIC_BASE_URL with trailing slash should be handled correctly.""" + from unittest.mock import MagicMock + + with patch("app.api.dropbox.settings") as mock_settings: + mock_settings.public_base_url = "https://myapp.example.com/" + from app.api.dropbox import _build_dropbox_redirect_uri + + mock_request = MagicMock() + result = _build_dropbox_redirect_uri(mock_request) + + assert result == "https://myapp.example.com/dropbox-callback" + + def test_falls_back_to_request_when_public_base_url_not_set(self): + """When PUBLIC_BASE_URL is not set, use request scheme and netloc.""" + from unittest.mock import MagicMock + + with patch("app.api.dropbox.settings") as mock_settings: + mock_settings.public_base_url = None + from app.api.dropbox import _build_dropbox_redirect_uri + + mock_request = MagicMock() + mock_request.url.scheme = "https" + mock_request.url.netloc = "other.example.com" + result = _build_dropbox_redirect_uri(mock_request) + + assert result == "https://other.example.com/dropbox-callback" + + +@pytest.mark.unit +class TestGlobalAuthorizeUrl: + """Tests for GET /api/dropbox/global-authorize-url endpoint.""" + + @patch("app.api.dropbox.settings") + def test_returns_authorize_url(self, mock_settings, client): + """Test that a valid authorize URL is returned when global creds are configured.""" + mock_settings.dropbox_allow_global_credentials_for_integrations = True + mock_settings.dropbox_app_key = "test-app-key" + mock_settings.dropbox_app_secret = "test-app-secret" + mock_settings.public_base_url = "https://example.com" + + response = client.get("/api/dropbox/global-authorize-url") + + assert response.status_code == 200 + data = response.json() + assert "authorize_url" in data + assert "https://www.dropbox.com/oauth2/authorize" in data["authorize_url"] + assert "client_id=test-app-key" in data["authorize_url"] + # redirect_uri should be URL-encoded + assert "redirect_uri=" in data["authorize_url"] + assert "https%3A%2F%2Fexample.com%2Fdropbox-callback" in data["authorize_url"] + + @patch("app.api.dropbox.settings") + def test_returns_403_when_global_creds_disabled(self, mock_settings, client): + """Test 403 when global credentials for integrations are disabled.""" + mock_settings.dropbox_allow_global_credentials_for_integrations = False + mock_settings.dropbox_app_key = "test-app-key" + mock_settings.dropbox_app_secret = "test-app-secret" + + response = client.get("/api/dropbox/global-authorize-url") + + assert response.status_code == 403 + + @patch("app.api.dropbox.settings") + def test_returns_503_when_creds_not_configured(self, mock_settings, client): + """Test 503 when global Dropbox credentials are not configured.""" + mock_settings.dropbox_allow_global_credentials_for_integrations = True + mock_settings.dropbox_app_key = None + mock_settings.dropbox_app_secret = None + + response = client.get("/api/dropbox/global-authorize-url") + + assert response.status_code == 503 + + @patch("app.api.dropbox.settings") + def test_redirect_uri_uses_public_base_url(self, mock_settings, client): + """Redirect URI in authorize URL must use PUBLIC_BASE_URL when configured.""" + mock_settings.dropbox_allow_global_credentials_for_integrations = True + mock_settings.dropbox_app_key = "my-key" + mock_settings.dropbox_app_secret = "my-secret" + mock_settings.public_base_url = "https://prod.example.com" + + response = client.get("/api/dropbox/global-authorize-url") + + assert response.status_code == 200 + authorize_url = response.json()["authorize_url"] + # The redirect_uri must be URL-encoded and contain the public base URL + assert "https%3A%2F%2Fprod.example.com%2Fdropbox-callback" in authorize_url diff --git a/tests/test_api_integrations.py b/tests/test_api_integrations.py index 90d1c1f3..45336b2d 100644 --- a/tests/test_api_integrations.py +++ b/tests/test_api_integrations.py @@ -887,9 +887,9 @@ class TestConnectionTestEndpoint: def test_test_unsupported_type(self, int_client): """Unsupported integration types return a helpful non-error message.""" payload = { - "integration_type": "DROPBOX", + "integration_type": "FTP", "config": {}, - "credentials": {"token": "abc"}, + "credentials": {"username": "user", "password": "pass"}, } resp = int_client.post("/api/integrations/test", json=payload) assert resp.status_code == 200 @@ -897,6 +897,83 @@ class TestConnectionTestEndpoint: assert data["success"] is False assert "not yet supported" in data["message"] + def test_test_dropbox_missing_refresh_token(self, int_client): + """Dropbox test with missing refresh_token returns failure.""" + payload = { + "integration_type": "DROPBOX", + "config": {}, + "credentials": {"app_key": "key", "app_secret": "secret"}, + } + resp = int_client.post("/api/integrations/test", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is False + assert "refresh_token" in data["message"].lower() + + def test_test_dropbox_missing_app_key(self, int_client): + """Dropbox test with missing app_key/app_secret returns failure.""" + payload = { + "integration_type": "DROPBOX", + "config": {}, + "credentials": {"refresh_token": "rtoken"}, + } + resp = int_client.post("/api/integrations/test", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is False + assert "app_key" in data["message"].lower() + + def test_test_dropbox_invalid_credentials(self, int_client): + """Dropbox test with bad credentials returns an auth failure.""" + from unittest.mock import MagicMock, patch + + import dropbox.exceptions as dbx_exc + + with patch("app.api.integrations.dbx_lib") as mock_dbx: + mock_instance = MagicMock() + mock_dbx.Dropbox.return_value = mock_instance + mock_instance.users_get_current_account.side_effect = dbx_exc.AuthError("req_id", MagicMock()) + payload = { + "integration_type": "DROPBOX", + "config": {}, + "credentials": { + "app_key": "bad_key", + "app_secret": "bad_secret", + "refresh_token": "bad_token", + }, + } + resp = int_client.post("/api/integrations/test", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is False + assert "authentication failed" in data["message"].lower() + + def test_test_dropbox_success(self, int_client): + """Dropbox test with valid (mocked) credentials returns success.""" + from unittest.mock import MagicMock, patch + + with patch("app.api.integrations.dbx_lib") as mock_dbx: + mock_instance = MagicMock() + mock_dbx.Dropbox.return_value = mock_instance + mock_account = MagicMock() + mock_account.name.display_name = "Test User" + mock_instance.users_get_current_account.return_value = mock_account + + payload = { + "integration_type": "DROPBOX", + "config": {}, + "credentials": { + "app_key": "valid_key", + "app_secret": "valid_secret", + "refresh_token": "valid_token", + }, + } + resp = int_client.post("/api/integrations/test", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert "dropbox connection successful" in data["message"].lower() + def test_test_invalid_type_returns_400(self, int_client): """Invalid integration_type returns 400.""" payload = { diff --git a/tests/test_api_mobile.py b/tests/test_api_mobile.py index 54b08967..5fcd7229 100644 --- a/tests/test_api_mobile.py +++ b/tests/test_api_mobile.py @@ -329,7 +329,7 @@ class TestDeactivateDevice: """Tests for DELETE /api/mobile/devices/{device_id}.""" def test_deactivate_own_device(self, mob_engine, mob_session): - """Deactivating a device sets is_active to False.""" + """Deactivating an active device sets is_active to False (soft-delete, returns 200).""" from app.main import app device = MobileDevice( @@ -346,7 +346,8 @@ class TestDeactivateDevice: client = _make_client(mob_engine) try: resp = client.delete(f"/api/mobile/devices/{device_id}") - assert resp.status_code == 204 + assert resp.status_code == 200 + assert resp.json()["detail"] == "Device deactivated" mob_session.expire_all() updated = mob_session.get(MobileDevice, device_id) @@ -355,6 +356,33 @@ class TestDeactivateDevice: finally: _cleanup(app) + def test_delete_inactive_device(self, mob_engine, mob_session): + """Deleting an already-inactive device permanently removes it (hard-delete, returns 200).""" + from app.main import app + + device = MobileDevice( + owner_id=_OWNER, + push_token=_EXPO_TOKEN, + platform="ios", + is_active=False, + ) + mob_session.add(device) + mob_session.commit() + mob_session.refresh(device) + device_id = device.id + + client = _make_client(mob_engine) + try: + resp = client.delete(f"/api/mobile/devices/{device_id}") + assert resp.status_code == 200 + assert resp.json()["detail"] == "Device deleted" + + mob_session.expire_all() + deleted = mob_session.get(MobileDevice, device_id) + assert deleted is None + finally: + _cleanup(app) + def test_deactivate_other_users_device_returns_404(self, mob_engine, mob_session): """Attempting to deactivate another user's device returns 404.""" from app.main import app @@ -439,6 +467,42 @@ class TestWhoAmI: assert data["email"] == _OWNER assert data["avatar_url"] is not None # Gravatar URL assert data["is_admin"] is False + assert data["preferred_language"] is None # not set yet + finally: + _cleanup(app) + + def test_whoami_returns_preferred_language(self, mob_engine, mob_session): + """preferred_language from UserProfile is included in the whoami response.""" + from app.main import app + from app.models import UserProfile + + profile = UserProfile( + user_id=_OWNER, + display_name="Bob Test", + preferred_language="de", + ) + mob_session.add(profile) + mob_session.commit() + + client = _make_client(mob_engine) + try: + resp = client.get("/api/mobile/whoami") + assert resp.status_code == 200 + data = resp.json() + assert data["preferred_language"] == "de" + finally: + _cleanup(app) + + def test_whoami_no_profile_preferred_language_is_null(self, mob_engine): + """preferred_language is null when no UserProfile exists.""" + from app.main import app + + client = _make_client(mob_engine) + try: + resp = client.get("/api/mobile/whoami") + assert resp.status_code == 200 + data = resp.json() + assert data["preferred_language"] is None finally: _cleanup(app) diff --git a/tests/test_api_onedrive_comprehensive.py b/tests/test_api_onedrive_comprehensive.py index 1019031b..f46fb397 100644 --- a/tests/test_api_onedrive_comprehensive.py +++ b/tests/test_api_onedrive_comprehensive.py @@ -695,3 +695,182 @@ class TestOneDriveIntegration: # Verify env format is present (exact values may vary) assert "env_format" in config_data + + +@pytest.mark.unit +class TestListOneDriveFolders: + """Tests for list_onedrive_folders endpoint.""" + + @patch("app.api.onedrive.requests.get") + def test_list_folders_success(self, mock_get, client): + """Test successful folder listing at root.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "name": "Documents", + "id": "id:1", + "folder": {"childCount": 3}, + "parentReference": {"path": "/drive/root:"}, + }, + { + "name": "Pictures", + "id": "id:2", + "folder": {"childCount": 10}, + "parentReference": {"path": "/drive/root:"}, + }, + ], + } + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["folders"]) == 2 + assert data["folders"][0]["name"] == "Documents" + assert data["folders"][0]["path"] == "/Documents" + assert data["folders"][1]["name"] == "Pictures" + assert data["path"] == "/" + + @patch("app.api.onedrive.requests.get") + def test_list_folders_subfolder(self, mock_get, client): + """Test listing folders in a subfolder.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "name": "Invoices", + "id": "id:3", + "folder": {"childCount": 0}, + "parentReference": {"path": "/drive/root:/Documents"}, + }, + ], + } + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": "Documents"}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["folders"]) == 1 + assert data["folders"][0]["path"] == "/Documents/Invoices" + assert data["path"] == "/Documents" + + @patch("app.api.onedrive.requests.get") + def test_list_folders_empty(self, mock_get, client): + """Test listing folders in an empty directory.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"value": []} + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": "EmptyFolder"}, + ) + + assert response.status_code == 200 + assert len(response.json()["folders"]) == 0 + + @patch("app.api.onedrive.requests.get") + def test_list_folders_unauthorized(self, mock_get, client): + """Test listing folders with invalid token returns 401.""" + mock_response = Mock() + mock_response.status_code = 401 + mock_response.text = "Invalid access token" + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "bad-token", "path": ""}, + ) + + assert response.status_code == 401 + + @patch("app.api.onedrive.requests.get") + def test_list_folders_api_error(self, mock_get, client): + """Test listing folders when Graph API returns an error.""" + mock_response = Mock() + mock_response.status_code = 500 + mock_response.text = "Internal server error" + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 502 + + @patch("app.api.onedrive.requests.get") + def test_list_folders_sorted_alphabetically(self, mock_get, client): + """Test that folders are returned in alphabetical order.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "name": "Zebra", + "id": "id:1", + "folder": {"childCount": 0}, + "parentReference": {"path": "/drive/root:"}, + }, + { + "name": "Alpha", + "id": "id:2", + "folder": {"childCount": 0}, + "parentReference": {"path": "/drive/root:"}, + }, + { + "name": "middle", + "id": "id:3", + "folder": {"childCount": 0}, + "parentReference": {"path": "/drive/root:"}, + }, + ], + } + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 200 + names = [f["name"] for f in response.json()["folders"]] + assert names == ["Alpha", "middle", "Zebra"] + + @patch("app.api.onedrive.requests.get") + def test_list_folders_root_drive_parent(self, mock_get, client): + """Test folder path construction when parentReference.path is /drive/root.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "name": "TopLevel", + "id": "id:1", + "folder": {"childCount": 0}, + "parentReference": {"path": "/drive/root"}, + }, + ], + } + mock_get.return_value = mock_response + + response = client.post( + "/api/onedrive/list-folders", + data={"access_token": "test-token", "path": ""}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["folders"][0]["path"] == "/TopLevel" diff --git a/tests/test_api_tokens.py b/tests/test_api_tokens.py index dabf2dd8..2a2b124a 100644 --- a/tests/test_api_tokens.py +++ b/tests/test_api_tokens.py @@ -314,8 +314,8 @@ class TestTokenRevoke: _cleanup(app) @pytest.mark.unit - def test_revoke_already_revoked_token(self, tok_engine): - """Revoking an already-revoked token should return 400.""" + def test_delete_already_revoked_token(self, tok_engine): + """Deleting an already-revoked token should permanently remove it (hard-delete, 200).""" from app.main import app client = _make_client(tok_engine) @@ -324,9 +324,15 @@ class TestTokenRevoke: token_id = create_resp.json()["id"] client.delete(f"/api/api-tokens/{token_id}") + # Second DELETE should hard-delete the revoked token. resp = client.delete(f"/api/api-tokens/{token_id}") - assert resp.status_code == 400 - assert resp.json()["detail"] == "Token is already revoked" + assert resp.status_code == 200 + assert resp.json()["detail"] == "Token deleted" + + # Token must no longer appear in the list. + list_resp = client.get("/api/api-tokens/") + ids = [t["id"] for t in list_resp.json()] + assert token_id not in ids finally: _cleanup(app) @@ -677,3 +683,216 @@ class TestTokenUtils: token = "de_test_token_value" expected_hash = "9b89d9adf2f390c75bf2fd0ff2bb5622ef5a9dce438354cce6e39f2f5401129e" assert hash_token(token) == expected_hash + + +# --------------------------------------------------------------------------- +# Tests – Token reactivation +# --------------------------------------------------------------------------- + + +class TestTokenReactivate: + """Tests for POST /api/api-tokens/{id}/reactivate.""" + + @pytest.mark.unit + def test_reactivate_revoked_token(self, tok_engine): + """Reactivating a revoked token should set is_active=True and clear revoked_at.""" + from app.main import app + + client = _make_client(tok_engine) + try: + create_resp = client.post("/api/api-tokens/", json={"name": "Reactivate Me"}) + token_id = create_resp.json()["id"] + client.delete(f"/api/api-tokens/{token_id}") + + resp = client.post(f"/api/api-tokens/{token_id}/reactivate") + assert resp.status_code == 200 + data = resp.json() + assert data["is_active"] is True + assert data["revoked_at"] is None + finally: + _cleanup(app) + + @pytest.mark.unit + def test_reactivate_active_token_returns_400(self, tok_engine): + """Reactivating an already-active token should return 400.""" + from app.main import app + + client = _make_client(tok_engine) + try: + create_resp = client.post("/api/api-tokens/", json={"name": "Already Active"}) + token_id = create_resp.json()["id"] + + resp = client.post(f"/api/api-tokens/{token_id}/reactivate") + assert resp.status_code == 400 + assert resp.json()["detail"] == "Token is already active" + finally: + _cleanup(app) + + @pytest.mark.unit + def test_reactivate_nonexistent_token(self, tok_engine): + """Reactivating a non-existent token should return 404.""" + from app.main import app + + client = _make_client(tok_engine) + try: + resp = client.post("/api/api-tokens/99999/reactivate") + assert resp.status_code == 404 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_reactivate_other_users_token(self, tok_engine): + """A user cannot reactivate another user's token.""" + from app.main import app + + client_a = _make_client(tok_engine, _OWNER) + try: + create_resp = client_a.post("/api/api-tokens/", json={"name": "A Token"}) + token_id = create_resp.json()["id"] + client_a.delete(f"/api/api-tokens/{token_id}") + finally: + _cleanup(app) + + client_b = _make_client(tok_engine, _OTHER_OWNER) + try: + resp = client_b.post(f"/api/api-tokens/{token_id}/reactivate") + assert resp.status_code == 404 + finally: + _cleanup(app) + + +# --------------------------------------------------------------------------- +# Tests – Token lifetime (expires_at) +# --------------------------------------------------------------------------- + + +class TestTokenExpiry: + """Tests for token creation with optional lifetime and expiry enforcement.""" + + @pytest.mark.unit + def test_create_token_without_expiry(self, tok_engine): + """Creating a token without expires_in_days should leave expires_at as None.""" + from app.main import app + + client = _make_client(tok_engine) + try: + resp = client.post("/api/api-tokens/", json={"name": "No Expiry"}) + assert resp.status_code == 201 + data = resp.json() + assert data["expires_at"] is None + finally: + _cleanup(app) + + @pytest.mark.unit + def test_create_token_with_expiry(self, tok_engine, tok_session): + """Creating a token with expires_in_days should set expires_at in the future.""" + from datetime import datetime, timezone + + from app.main import app + + client = _make_client(tok_engine) + try: + resp = client.post("/api/api-tokens/", json={"name": "With Expiry", "expires_in_days": 30}) + assert resp.status_code == 201 + data = resp.json() + assert data["expires_at"] is not None + # Parse the returned datetime; handle both tz-aware and tz-naive serialisations + expires_str = data["expires_at"].replace("Z", "+00:00") + expires_at = datetime.fromisoformat(expires_str) + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + delta_days = (expires_at - now).days + assert 28 <= delta_days <= 30 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_expired_token_not_resolved(self, tok_engine, tok_session): + """A token past its expires_at should not authenticate.""" + from datetime import datetime, timedelta, timezone + from unittest.mock import MagicMock + + from app.api.api_tokens import generate_api_token, hash_token + from app.auth import _resolve_bearer_user + + plaintext = generate_api_token() + token_hash = hash_token(plaintext) + + db_token = ApiToken( + owner_id=_OWNER, + name="Expired Token", + token_hash=token_hash, + token_prefix=plaintext[:12], + is_active=True, + expires_at=datetime.now(timezone.utc) - timedelta(days=1), # expired yesterday + ) + tok_session.add(db_token) + tok_session.commit() + + mock_request = MagicMock() + mock_request.headers = {"authorization": f"Bearer {plaintext}"} + mock_request.client.host = "127.0.0.1" + + user = _resolve_bearer_user(mock_request, tok_session) + assert user is None + + @pytest.mark.unit + def test_non_expired_token_resolves(self, tok_engine, tok_session): + """A token before its expires_at should authenticate normally.""" + from datetime import datetime, timedelta, timezone + from unittest.mock import MagicMock + + from app.api.api_tokens import generate_api_token, hash_token + from app.auth import _resolve_bearer_user + + plaintext = generate_api_token() + token_hash = hash_token(plaintext) + + db_token = ApiToken( + owner_id=_OWNER, + name="Valid Token", + token_hash=token_hash, + token_prefix=plaintext[:12], + is_active=True, + expires_at=datetime.now(timezone.utc) + timedelta(days=30), # expires in 30 days + ) + tok_session.add(db_token) + tok_session.commit() + + mock_request = MagicMock() + mock_request.headers = {"authorization": f"Bearer {plaintext}"} + mock_request.client.host = "127.0.0.1" + + user = _resolve_bearer_user(mock_request, tok_session) + assert user is not None + assert user["preferred_username"] == _OWNER + + @pytest.mark.unit + def test_create_token_expires_in_days_zero_rejected(self, tok_engine): + """expires_in_days=0 should be rejected with 422 (ge=1).""" + from app.main import app + + client = _make_client(tok_engine) + try: + resp = client.post("/api/api-tokens/", json={"name": "Bad Expiry", "expires_in_days": 0}) + assert resp.status_code == 422 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_expires_at_included_in_list_response(self, tok_engine): + """List endpoint should include expires_at field.""" + from app.main import app + + client = _make_client(tok_engine) + try: + client.post("/api/api-tokens/", json={"name": "Listed", "expires_in_days": 7}) + resp = client.get("/api/api-tokens/") + assert resp.status_code == 200 + tokens = resp.json() + assert len(tokens) == 1 + assert "expires_at" in tokens[0] + assert tokens[0]["expires_at"] is not None + finally: + _cleanup(app) diff --git a/tests/test_check_alembic_migrations.py b/tests/test_check_alembic_migrations.py new file mode 100644 index 00000000..252d648b --- /dev/null +++ b/tests/test_check_alembic_migrations.py @@ -0,0 +1,199 @@ +"""Tests for scripts/check_alembic_migrations.py.""" + +# The script lives outside of the ``app`` package, so we import it by path. +import importlib.util +import textwrap +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "check_alembic_migrations.py" +_spec = importlib.util.spec_from_file_location("check_alembic_migrations", _SCRIPT) +assert _spec and _spec.loader +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) # type: ignore[union-attr] + +check_migrations = _mod.check_migrations +main = _mod.main + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _write_migration( + directory: Path, filename: str, revision: str, down_revision: str | tuple[str, ...] | None +) -> Path: + """Helper to create a minimal migration file.""" + if down_revision is None: + down_rev_str = "None" + elif isinstance(down_revision, tuple): + down_rev_str = repr(down_revision) + else: + down_rev_str = f'"{down_revision}"' + + content = textwrap.dedent(f'''\ + """Test migration.""" + from typing import Union + revision: str = "{revision}" + down_revision: Union[str, None] = {down_rev_str} + depends_on: Union[str, None] = None + def upgrade() -> None: + pass + def downgrade() -> None: + pass + ''') + path = directory / filename + path.write_text(content) + return path + + +@pytest.fixture +def versions_dir(tmp_path: Path) -> Path: + """Return a temporary versions directory.""" + d = tmp_path / "versions" + d.mkdir() + return d + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestCheckMigrations: + """Tests for the check_migrations function.""" + + def test_valid_linear_chain(self, versions_dir: Path) -> None: + """A simple linear chain should pass with no errors.""" + _write_migration(versions_dir, "001_initial.py", "001_initial", None) + _write_migration(versions_dir, "002_add_col.py", "002_add_col", "001_initial") + _write_migration(versions_dir, "003_add_table.py", "003_add_table", "002_add_col") + + errors = check_migrations(versions_dir) + assert errors == [] + + def test_valid_merge_migration(self, versions_dir: Path) -> None: + """A chain with a merge point should pass.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + _write_migration(versions_dir, "002_a.py", "002_a", "001_base") + _write_migration(versions_dir, "002_b.py", "002_b", "001_base") + + # Merge file with tuple down_revision + content = textwrap.dedent('''\ + """Merge.""" + from typing import Union + revision: str = "003_merge" + down_revision: Union[str, tuple] = ("002_a", "002_b") + depends_on: Union[str, None] = None + def upgrade() -> None: + pass + def downgrade() -> None: + pass + ''') + (versions_dir / "003_merge.py").write_text(content) + + errors = check_migrations(versions_dir) + assert errors == [] + + def test_multiple_heads_detected(self, versions_dir: Path) -> None: + """Two unmerged branches should report multiple heads.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + _write_migration(versions_dir, "002_a.py", "002_a", "001_base") + _write_migration(versions_dir, "002_b.py", "002_b", "001_base") + + errors = check_migrations(versions_dir) + assert len(errors) == 1 + assert "Multiple migration heads" in errors[0] + assert "002_a" in errors[0] + assert "002_b" in errors[0] + + def test_broken_down_revision(self, versions_dir: Path) -> None: + """A migration pointing to a non-existent parent should be flagged.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + _write_migration(versions_dir, "002_orphan.py", "002_orphan", "NONEXISTENT") + + errors = check_migrations(versions_dir) + assert any("Broken chain" in e for e in errors) + assert any("NONEXISTENT" in e for e in errors) + + def test_duplicate_revision(self, versions_dir: Path) -> None: + """Two files declaring the same revision should be flagged.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + _write_migration(versions_dir, "002_first.py", "002_dup", "001_base") + _write_migration(versions_dir, "002_second.py", "002_dup", "001_base") + + errors = check_migrations(versions_dir) + assert any("Duplicate revision" in e for e in errors) + + def test_filename_mismatch(self, versions_dir: Path) -> None: + """A file whose revision doesn't match its filename should be flagged.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + # filename stem is "002_wrong_name" but revision says "002_correct_name" + _write_migration(versions_dir, "002_wrong_name.py", "002_correct_name", "001_base") + + errors = check_migrations(versions_dir) + assert any("Filename mismatch" in e for e in errors) + + def test_empty_directory(self, versions_dir: Path) -> None: + """An empty versions directory should report an error.""" + errors = check_migrations(versions_dir) + assert len(errors) == 1 + assert "No migration files found" in errors[0] + + def test_init_py_is_skipped(self, versions_dir: Path) -> None: + """__init__.py files should be ignored.""" + (versions_dir / "__init__.py").write_text("") + _write_migration(versions_dir, "001_base.py", "001_base", None) + + errors = check_migrations(versions_dir) + assert errors == [] + + def test_non_migration_file_skipped(self, versions_dir: Path) -> None: + """A .py file without a revision variable should be silently skipped.""" + (versions_dir / "helper.py").write_text("# just a helper\nx = 1\n") + _write_migration(versions_dir, "001_base.py", "001_base", None) + + errors = check_migrations(versions_dir) + assert errors == [] + + +@pytest.mark.unit +class TestMainCLI: + """Tests for the CLI entry-point.""" + + def test_success_returns_zero(self, versions_dir: Path) -> None: + """Valid chain should exit 0.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + rc = main(["--versions-dir", str(versions_dir)]) + assert rc == 0 + + def test_failure_returns_one(self, versions_dir: Path) -> None: + """Invalid chain should exit 1.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + _write_migration(versions_dir, "002_a.py", "002_a", "001_base") + _write_migration(versions_dir, "002_b.py", "002_b", "001_base") + + rc = main(["--versions-dir", str(versions_dir)]) + assert rc == 1 + + def test_missing_directory_returns_two(self, tmp_path: Path) -> None: + """Non-existent versions directory should exit 2.""" + rc = main(["--versions-dir", str(tmp_path / "does_not_exist")]) + assert rc == 2 + + def test_verbose_flag(self, versions_dir: Path) -> None: + """The --verbose flag should not crash.""" + _write_migration(versions_dir, "001_base.py", "001_base", None) + rc = main(["--versions-dir", str(versions_dir), "--verbose"]) + assert rc == 0 + + def test_real_migrations(self) -> None: + """Smoke test against the actual project migrations.""" + real_dir = Path(__file__).resolve().parent.parent / "migrations" / "versions" + if not real_dir.is_dir(): + pytest.skip("migrations/versions directory not found in working tree") + rc = main(["--versions-dir", str(real_dir)]) + assert rc == 0 diff --git a/tests/test_classification_rules.py b/tests/test_classification_rules.py new file mode 100644 index 00000000..a4bacaec --- /dev/null +++ b/tests/test_classification_rules.py @@ -0,0 +1,422 @@ +"""Tests for the rule-based document classification engine. + +Covers the classification engine logic in ``app/utils/classification_rules.py``: +built-in rules, custom rules, confidence scoring, and edge cases. +""" + +import pytest + +from app.utils.classification_rules import ( + BUILTIN_CATEGORIES, + BUILTIN_RULES, + RULE_TYPE_CONTENT, + RULE_TYPE_FILENAME, + RULE_TYPE_METADATA, + ClassificationResult, + ClassificationRule, + MatchedRule, + classify_document, + db_rule_to_engine_rule, +) + +# --------------------------------------------------------------------------- +# Built-in categories & rules smoke tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestBuiltinCategories: + """Verify the pre-built categories and rules are sane.""" + + def test_builtin_categories_not_empty(self): + """There must be at least one built-in category.""" + assert len(BUILTIN_CATEGORIES) > 0 + + def test_unknown_category_exists(self): + """The 'unknown' fallback category must be present.""" + assert "unknown" in BUILTIN_CATEGORIES + + def test_core_categories_present(self): + """Invoice, contract, and receipt categories must exist.""" + for cat in ("invoice", "contract", "receipt"): + assert cat in BUILTIN_CATEGORIES, f"Missing built-in category: {cat}" + + def test_builtin_rules_not_empty(self): + """There must be at least one built-in rule.""" + assert len(BUILTIN_RULES) > 0 + + def test_all_builtin_rules_reference_valid_types(self): + """Every built-in rule must use a valid rule_type.""" + valid_types = {RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA} + for rule in BUILTIN_RULES: + assert rule.rule_type in valid_types, f"Rule {rule.name!r} has invalid type {rule.rule_type!r}" + + +# --------------------------------------------------------------------------- +# ClassificationRule dataclass validation +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestClassificationRuleValidation: + """Test ClassificationRule dataclass validation.""" + + def test_valid_rule_types(self): + """Valid rule types should not raise.""" + for rt in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA): + rule = ClassificationRule(name="test", category="test", rule_type=rt, pattern="test") + assert rule.rule_type == rt + + def test_invalid_rule_type_raises(self): + """An invalid rule_type should raise ValueError.""" + with pytest.raises(ValueError, match="Invalid rule_type"): + ClassificationRule(name="test", category="test", rule_type="invalid", pattern="test") + + +# --------------------------------------------------------------------------- +# Filename pattern matching +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestFilenamePatternMatching: + """Test classification via filename patterns.""" + + def test_invoice_filename(self): + """A filename containing 'invoice' should classify as invoice.""" + result = classify_document(filename="2024-03-01_Invoice_Acme.pdf") + assert result.category == "invoice" + assert result.confidence > 0 + + def test_german_invoice_filename(self): + """A filename containing 'Rechnung' should classify as invoice.""" + result = classify_document(filename="Rechnung_2024.pdf") + assert result.category == "invoice" + assert result.confidence > 0 + + def test_contract_filename(self): + """A filename containing 'contract' should classify as contract.""" + result = classify_document(filename="Service_Contract_2024.pdf") + assert result.category == "contract" + + def test_receipt_filename(self): + """A filename containing 'receipt' should classify as receipt.""" + result = classify_document(filename="Payment_Receipt.pdf") + assert result.category == "receipt" + + def test_unrecognized_filename(self): + """A generic filename with no keywords should return 'unknown'.""" + result = classify_document(filename="document_12345.pdf") + assert result.category == "unknown" + assert result.confidence == 0 + + def test_empty_filename(self): + """An empty filename should not match any rule.""" + result = classify_document(filename="") + assert result.category == "unknown" + + +# --------------------------------------------------------------------------- +# Content keyword matching +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestContentKeywordMatching: + """Test classification via content keywords.""" + + def test_invoice_content(self): + """Text containing 'invoice number' should classify as invoice.""" + result = classify_document(text="Please pay the invoice number 12345. Amount due: $500") + assert result.category == "invoice" + assert result.confidence > 0 + + def test_contract_content(self): + """Text containing 'terms and conditions' should classify as contract.""" + result = classify_document(text="The parties hereby agree to the following terms and conditions.") + assert result.category == "contract" + + def test_receipt_content(self): + """Text containing 'payment received' should classify as receipt.""" + result = classify_document(text="Thank you. Payment received for order #789.") + assert result.category == "receipt" + + def test_bank_statement_content(self): + """Text containing 'account statement' should classify as bank_statement.""" + result = classify_document(text="Monthly account statement. Opening balance: $1,000.") + assert result.category == "bank_statement" + + def test_empty_text(self): + """Empty text should not match any content rule.""" + result = classify_document(text="") + assert result.category == "unknown" + + def test_case_insensitive_matching(self): + """Content matching should be case-insensitive by default.""" + result = classify_document(text="INVOICE NUMBER 12345") + assert result.category == "invoice" + + +# --------------------------------------------------------------------------- +# Metadata matching +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestMetadataMatching: + """Test classification via metadata field matching.""" + + def test_document_type_invoice(self): + """metadata document_type=Invoice should classify as invoice.""" + result = classify_document(metadata={"document_type": "Invoice"}) + assert result.category == "invoice" + assert result.confidence >= 90 + + def test_document_type_contract(self): + """metadata document_type=Contract should classify as contract.""" + result = classify_document(metadata={"document_type": "Contract"}) + assert result.category == "contract" + + def test_kommunikationsart_rechnung(self): + """German classification metadata should classify as invoice.""" + result = classify_document(metadata={"kommunikationsart": "Rechnung"}) + assert result.category == "invoice" + + def test_no_metadata(self): + """None metadata should not match.""" + result = classify_document(metadata=None) + assert result.category == "unknown" + + def test_empty_metadata(self): + """Empty metadata dict should not match.""" + result = classify_document(metadata={}) + assert result.category == "unknown" + + def test_metadata_case_insensitive(self): + """Metadata matching should be case-insensitive by default.""" + result = classify_document(metadata={"document_type": "invoice"}) + assert result.category == "invoice" + + +# --------------------------------------------------------------------------- +# Combined matching / confidence boosting +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestCombinedMatching: + """Test that multiple matching rules boost confidence.""" + + def test_filename_and_content_boost(self): + """Filename + content matching should produce higher confidence than either alone.""" + filename_only = classify_document(filename="Invoice_2024.pdf") + combined = classify_document(filename="Invoice_2024.pdf", text="Invoice number: 12345. Amount due: $500.") + assert combined.confidence >= filename_only.confidence + assert len(combined.matched_rules) > len(filename_only.matched_rules) + + def test_all_three_signals(self): + """Filename + content + metadata should produce highest confidence.""" + result = classify_document( + filename="Invoice_Acme.pdf", + text="Invoice number: 12345. Amount due: $500.", + metadata={"document_type": "Invoice"}, + ) + assert result.category == "invoice" + assert result.confidence >= 90 + + def test_conflicting_signals_most_matches_wins(self): + """When filename says 'invoice' but content says 'contract', most matches wins.""" + result = classify_document( + filename="Invoice.pdf", + text="The parties hereby agree to the following terms and conditions. " + "This agreement between Company A and Company B is effective immediately.", + ) + # Content has more keyword matches for contract, but filename matches invoice. + # Either is acceptable as long as the result is deterministic. + assert result.category in ("invoice", "contract") + + +# --------------------------------------------------------------------------- +# Custom rules +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestCustomRules: + """Test user-defined custom classification rules.""" + + def test_custom_rule_matches(self): + """A custom filename rule should match when pattern hits.""" + custom = [ + ClassificationRule( + name="custom_hr_doc", + category="hr_document", + rule_type=RULE_TYPE_FILENAME, + pattern=r"(?i)employee|hiring|hr", + ) + ] + result = classify_document(filename="Employee_Handbook.pdf", custom_rules=custom) + assert result.category == "hr_document" + + def test_custom_content_rule(self): + """A custom content keyword rule should match.""" + custom = [ + ClassificationRule( + name="custom_medical", + category="medical", + rule_type=RULE_TYPE_CONTENT, + pattern="diagnosis|prescription|patient record", + ) + ] + result = classify_document(text="Patient record for Jane Doe. Diagnosis: common cold.", custom_rules=custom) + assert result.category == "medical" + + def test_custom_metadata_rule(self): + """A custom metadata rule should match.""" + custom = [ + ClassificationRule( + name="custom_legal", + category="legal", + rule_type=RULE_TYPE_METADATA, + pattern="department=legal", + ) + ] + result = classify_document(metadata={"department": "legal"}, custom_rules=custom) + assert result.category == "legal" + + def test_custom_rule_overrides_builtin(self): + """Custom rules with more matches should override built-in rules.""" + custom = [ + ClassificationRule( + name="custom_internal_invoice", + category="internal_invoice", + rule_type=RULE_TYPE_FILENAME, + pattern=r"(?i)invoice", + priority=100, + ), + ClassificationRule( + name="custom_internal_invoice_content", + category="internal_invoice", + rule_type=RULE_TYPE_CONTENT, + pattern="invoice number", + priority=100, + ), + ] + result = classify_document( + filename="Invoice_2024.pdf", + text="Invoice number: 12345", + custom_rules=custom, + ) + # Both builtin and custom rules for "invoice" patterns match, but custom + # has "internal_invoice" as category. The category with more total matches wins. + assert result.category in ("invoice", "internal_invoice") + assert result.confidence > 0 + + +# --------------------------------------------------------------------------- +# db_rule_to_engine_rule converter +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestDbRuleConversion: + """Test the database model to engine rule converter.""" + + def test_converts_basic_fields(self): + """All basic fields should be mapped correctly.""" + + class FakeDbRule: + name = "test_rule" + category = "invoice" + rule_type = RULE_TYPE_FILENAME + pattern = r"(?i)invoice" + priority = 10 + case_sensitive = True + + engine_rule = db_rule_to_engine_rule(FakeDbRule()) + assert engine_rule.name == "test_rule" + assert engine_rule.category == "invoice" + assert engine_rule.rule_type == RULE_TYPE_FILENAME + assert engine_rule.pattern == r"(?i)invoice" + assert engine_rule.priority == 10 + assert engine_rule.case_sensitive is True + + def test_defaults_case_sensitive_to_false(self): + """When case_sensitive is missing, default to False.""" + + class FakeDbRule: + name = "test" + category = "test" + rule_type = RULE_TYPE_CONTENT + pattern = "test" + priority = 0 + + engine_rule = db_rule_to_engine_rule(FakeDbRule()) + assert engine_rule.case_sensitive is False + + +# --------------------------------------------------------------------------- +# ClassificationResult +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestClassificationResult: + """Test the ClassificationResult dataclass.""" + + def test_default_matched_rules(self): + """matched_rules should default to an empty list.""" + result = ClassificationResult(category="test", confidence=50) + assert result.matched_rules == [] + + def test_with_matched_rules(self): + """matched_rules should be populated when provided.""" + match = MatchedRule(rule_name="test", rule_type=RULE_TYPE_FILENAME, category="invoice", confidence=60) + result = ClassificationResult(category="invoice", confidence=60, matched_rules=[match]) + assert len(result.matched_rules) == 1 + assert result.matched_rules[0].rule_name == "test" + + +# --------------------------------------------------------------------------- +# Edge cases +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestEdgeCases: + """Test edge cases in the classification engine.""" + + def test_no_inputs_at_all(self): + """No filename, text, or metadata should return 'unknown'.""" + result = classify_document() + assert result.category == "unknown" + assert result.confidence == 0 + assert result.matched_rules == [] + + def test_metadata_pattern_without_equals(self): + """A metadata pattern without '=' should not match.""" + custom = [ + ClassificationRule( + name="bad_pattern", + category="test", + rule_type=RULE_TYPE_METADATA, + pattern="no_equals_sign", + ) + ] + result = classify_document(metadata={"no_equals_sign": "value"}, custom_rules=custom) + assert result.category == "unknown" + + def test_confidence_capped_at_100(self): + """Confidence should never exceed 100.""" + # Create many rules that all match to test the cap + custom = [ + ClassificationRule( + name=f"flood_{i}", + category="flood", + rule_type=RULE_TYPE_CONTENT, + pattern="test keyword", + ) + for i in range(20) + ] + result = classify_document(text="test keyword is here", custom_rules=custom) + assert result.confidence <= 100 diff --git a/tests/test_classify_document.py b/tests/test_classify_document.py new file mode 100644 index 00000000..60fa2fe5 --- /dev/null +++ b/tests/test_classify_document.py @@ -0,0 +1,232 @@ +"""Tests for the classify_document Celery task. + +Covers the ``classify_document_task`` in ``app/tasks/classify_document.py``. +""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from app.models import ClassificationRuleModel, FileRecord +from app.tasks.classify_document import _load_custom_rules + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_file_record(db_session, **overrides): + """Insert a minimal FileRecord and return it.""" + defaults = { + "owner_id": "test-user", + "filehash": "abc123", + "original_filename": "Invoice_2024.pdf", + "local_filename": "/tmp/test.pdf", + "file_size": 1024, + "mime_type": "application/pdf", + "ocr_text": "Invoice number: 12345. Amount due: $500.", + "ai_metadata": None, + } + defaults.update(overrides) + fr = FileRecord(**defaults) + db_session.add(fr) + db_session.commit() + db_session.refresh(fr) + return fr + + +def _make_rule(db_session, **overrides): + """Insert a ClassificationRuleModel and return it.""" + defaults = { + "owner_id": None, + "name": "test_rule", + "category": "test_category", + "rule_type": "filename_pattern", + "pattern": r"(?i)test", + "priority": 0, + "case_sensitive": False, + "enabled": True, + } + defaults.update(overrides) + rule = ClassificationRuleModel(**defaults) + db_session.add(rule) + db_session.commit() + db_session.refresh(rule) + return rule + + +# --------------------------------------------------------------------------- +# _load_custom_rules +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestLoadCustomRules: + """Test the custom rule loading helper.""" + + @patch("app.tasks.classify_document.SessionLocal") + def test_loads_enabled_rules(self, mock_session_local): + """Should load enabled rules from the database.""" + mock_rule = MagicMock() + mock_rule.name = "rule1" + mock_rule.category = "invoice" + mock_rule.rule_type = "filename_pattern" + mock_rule.pattern = r"(?i)invoice" + mock_rule.priority = 10 + mock_rule.case_sensitive = False + + mock_db = MagicMock() + mock_query = MagicMock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [mock_rule] + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + rules = _load_custom_rules(owner_id="test-user") + assert len(rules) == 1 + assert rules[0].name == "rule1" + assert rules[0].category == "invoice" + + +# --------------------------------------------------------------------------- +# classify_document_task (integration-style with mocked DB and Celery) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestClassifyDocumentTask: + """Test the Celery classify_document_task.""" + + @patch("app.tasks.classify_document.log_task_progress") + @patch("app.tasks.classify_document._load_custom_rules", return_value=[]) + @patch("app.tasks.classify_document.SessionLocal") + def test_classify_invoice_file(self, mock_session_local, mock_load_rules, mock_log): + """Should classify a file with invoice filename and text as 'invoice'.""" + mock_file = MagicMock(spec=FileRecord) + mock_file.id = 1 + mock_file.original_filename = "Invoice_2024.pdf" + mock_file.ocr_text = "Invoice number: 12345. Amount due: $500." + mock_file.ai_metadata = None + mock_file.owner_id = "test-user" + + mock_db = MagicMock() + mock_db.query.return_value.filter.return_value.first.return_value = mock_file + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + from app.tasks.classify_document import classify_document_task + + # Call the underlying function directly via .run(), bypassing Celery + result = classify_document_task.run(1, owner_id="test-user") + + assert result["status"] == "success" + assert result["category"] == "invoice" + assert result["confidence"] > 0 + + # Verify ai_metadata was updated + assert mock_file.ai_metadata is not None + metadata = json.loads(mock_file.ai_metadata) + assert "classification" in metadata + assert metadata["classification"]["category"] == "invoice" + + @patch("app.tasks.classify_document.log_task_progress") + @patch("app.tasks.classify_document.SessionLocal") + def test_classify_file_not_found(self, mock_session_local, mock_log): + """Should return error when file record is not found.""" + mock_db = MagicMock() + mock_db.query.return_value.filter.return_value.first.return_value = None + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + from app.tasks.classify_document import classify_document_task + + result = classify_document_task.run(99999) + assert result["status"] == "error" + + @patch("app.tasks.classify_document.log_task_progress") + @patch("app.tasks.classify_document._load_custom_rules", return_value=[]) + @patch("app.tasks.classify_document.SessionLocal") + def test_classify_preserves_existing_metadata(self, mock_session_local, mock_load_rules, mock_log): + """Should preserve existing ai_metadata fields and add classification.""" + existing_meta = json.dumps({"document_type": "Invoice", "tags": ["finance"]}) + + mock_file = MagicMock(spec=FileRecord) + mock_file.id = 2 + mock_file.original_filename = "doc.pdf" + mock_file.ocr_text = "" + mock_file.ai_metadata = existing_meta + mock_file.owner_id = "test-user" + + mock_db = MagicMock() + mock_db.query.return_value.filter.return_value.first.return_value = mock_file + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + from app.tasks.classify_document import classify_document_task + + classify_document_task.run(2) + + # Check that existing fields are preserved + metadata = json.loads(mock_file.ai_metadata) + assert metadata["tags"] == ["finance"] + assert metadata["document_type"] == "Invoice" + assert "classification" in metadata + + @patch("app.tasks.classify_document.log_task_progress") + @patch("app.tasks.classify_document._load_custom_rules", return_value=[]) + @patch("app.tasks.classify_document.SessionLocal") + def test_classify_sets_document_type_when_missing(self, mock_session_local, mock_load_rules, mock_log): + """Should set document_type from classification when not already present.""" + mock_file = MagicMock(spec=FileRecord) + mock_file.id = 3 + mock_file.original_filename = "Invoice_2024.pdf" + mock_file.ocr_text = "Invoice number: 12345" + mock_file.ai_metadata = json.dumps({"tags": ["test"]}) + mock_file.owner_id = "test-user" + + mock_db = MagicMock() + mock_db.query.return_value.filter.return_value.first.return_value = mock_file + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + from app.tasks.classify_document import classify_document_task + + classify_document_task.run(3) + + metadata = json.loads(mock_file.ai_metadata) + assert metadata["document_type"] == "Invoice" + + @patch("app.tasks.classify_document.log_task_progress") + @patch("app.tasks.classify_document._load_custom_rules", return_value=[]) + @patch("app.tasks.classify_document.SessionLocal") + def test_classify_unknown_document(self, mock_session_local, mock_load_rules, mock_log): + """Should classify as 'unknown' when no rules match.""" + mock_file = MagicMock(spec=FileRecord) + mock_file.id = 4 + mock_file.original_filename = "random_file.pdf" + mock_file.ocr_text = "Lorem ipsum dolor sit amet." + mock_file.ai_metadata = None + mock_file.owner_id = "test-user" + + mock_db = MagicMock() + mock_db.query.return_value.filter.return_value.first.return_value = mock_file + mock_session_local.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_session_local.return_value.__exit__ = MagicMock(return_value=False) + + from app.tasks.classify_document import classify_document_task + + result = classify_document_task.run(4) + + assert result["category"] == "unknown" + assert result["confidence"] == 0 + + def test_classify_document_task_is_celery_task(self): + """Task should be registered as a Celery task.""" + from app.tasks.classify_document import classify_document_task + + assert hasattr(classify_document_task, "apply_async") + assert hasattr(classify_document_task, "delay") + assert callable(classify_document_task) diff --git a/tests/test_coverage_uploads_notification.py b/tests/test_coverage_uploads_notification.py index c59ada46..a255605c 100644 --- a/tests/test_coverage_uploads_notification.py +++ b/tests/test_coverage_uploads_notification.py @@ -665,6 +665,7 @@ def _all_should_upload_false(): "email", "onedrive", "s3", + "sharepoint", "icloud", ] return [patch(f"app.tasks.send_to_all._should_upload_to_{s}", return_value=False) for s in services] @@ -694,6 +695,7 @@ class TestSendToAllCoverage: patch("app.tasks.send_to_all._should_upload_to_email", return_value=False), patch("app.tasks.send_to_all._should_upload_to_onedrive", return_value=False), patch("app.tasks.send_to_all._should_upload_to_s3", return_value=False), + patch("app.tasks.send_to_all._should_upload_to_sharepoint", return_value=False), patch("app.tasks.send_to_all._should_upload_to_icloud", return_value=False), patch("app.tasks.send_to_all.SessionLocal") as mock_session_cls, ): @@ -806,6 +808,7 @@ class TestSendToAllCoverage: patch("app.tasks.send_to_all._should_upload_to_email", return_value=False), patch("app.tasks.send_to_all._should_upload_to_onedrive", return_value=False), patch("app.tasks.send_to_all._should_upload_to_s3", return_value=False), + patch("app.tasks.send_to_all._should_upload_to_sharepoint", return_value=False), patch("app.tasks.send_to_all._should_upload_to_icloud", return_value=False), patch("app.tasks.send_to_all.SessionLocal"), ): @@ -866,6 +869,7 @@ class TestSendToAllCoverage: patch("app.tasks.send_to_all._should_upload_to_email", return_value=False), patch("app.tasks.send_to_all._should_upload_to_onedrive", return_value=False), patch("app.tasks.send_to_all._should_upload_to_s3", return_value=False), + patch("app.tasks.send_to_all._should_upload_to_sharepoint", return_value=False), patch("app.tasks.send_to_all._should_upload_to_icloud", return_value=False), patch("app.tasks.send_to_all.SessionLocal"), ): diff --git a/tests/test_csrf.py b/tests/test_csrf.py index a6041038..a80e6e8a 100644 --- a/tests/test_csrf.py +++ b/tests/test_csrf.py @@ -337,6 +337,27 @@ class TestCSRFMiddlewareDispatch: call_next.assert_called_once_with(request) + @pytest.mark.asyncio + async def test_qr_auth_claim_is_exempt(self): + """QR auth claim path is exempt from CSRF validation. + + The mobile app calls this endpoint without a browser session and + therefore without a CSRF token. The cryptographically-random, + single-use challenge token provides equivalent protection. + """ + middleware = self._make_middleware() + request = self._make_request( + method="POST", + path="/api/qr-auth/claim", + session={}, + ) + call_next = AsyncMock(return_value=MagicMock()) + + with patch.object(CSRFMiddleware, "_get_submitted_token", new=AsyncMock(return_value=None)): + result = await middleware.dispatch(request, call_next) + + call_next.assert_called_once_with(request) + # --------------------------------------------------------------------------- # Integration tests – via TestClient @@ -362,6 +383,7 @@ class TestCSRFIntegration: assert "PATCH" in CSRF_PROTECTED_METHODS assert "GET" not in CSRF_PROTECTED_METHODS assert "/oauth-callback" in CSRF_EXEMPT_PATHS + assert "/api/qr-auth/claim" in CSRF_EXEMPT_PATHS def test_csrf_middleware_noop_when_auth_disabled(self): """When AUTH_ENABLED=False the middleware dispatch is a no-op (no validation).""" diff --git a/tests/test_database.py b/tests/test_database.py index 64e0e70e..3bd822f7 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -998,3 +998,49 @@ class TestAlembicUpgrade: # Verify head is reachable heads = script.get_heads() assert len(heads) == 1 # Should be a single linear chain + + +@pytest.mark.unit +class TestEnginePoolConfiguration: + """Tests for database engine pool configuration (pool class and options).""" + + def test_sqlite_engine_uses_null_pool(self): + """SQLite engines must use NullPool to prevent QueuePool exhaustion.""" + from sqlalchemy.pool import NullPool + + from app.database import engine + + # The test environment uses SQLite, so NullPool should be in effect. + assert isinstance(engine.pool, NullPool) + + def test_create_engine_sqlite_null_pool(self): + """Explicitly create a SQLite engine to confirm NullPool is applied.""" + from sqlalchemy import create_engine + from sqlalchemy.pool import NullPool + + test_engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=NullPool, + ) + assert isinstance(test_engine.pool, NullPool) + test_engine.dispose() + + def test_pool_settings_exist_in_config(self): + """Verify that pool tuning settings are exposed through config.""" + from app.config import settings + + assert hasattr(settings, "db_pool_size") + assert hasattr(settings, "db_max_overflow") + assert hasattr(settings, "db_pool_timeout") + assert hasattr(settings, "db_pool_recycle") + + def test_pool_settings_have_sensible_defaults(self): + """Default pool settings should be larger than SQLAlchemy's built-in defaults.""" + from app.config import settings + + # SQLAlchemy defaults: pool_size=5, max_overflow=10 + assert settings.db_pool_size >= 10 + assert settings.db_max_overflow >= 20 + assert settings.db_pool_timeout >= 30 + assert settings.db_pool_recycle >= 1800 diff --git a/tests/test_devices_page.py b/tests/test_devices_page.py new file mode 100644 index 00000000..106cba0c --- /dev/null +++ b/tests/test_devices_page.py @@ -0,0 +1,201 @@ +"""Tests for the Devices page and mobile token filtering (app/api/api_tokens.py mobile endpoint). + +These tests validate: +- ``GET /api/api-tokens/mobile`` returns only mobile tokens +- ``GET /api/api-tokens/`` excludes mobile tokens +- ``GET /devices`` renders the devices page +""" + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base, get_db +from app.models import ApiToken + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +_OWNER = "devices_user@example.com" +_OTHER_OWNER = "other_devices@example.com" + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def dev_engine(): + """In-memory SQLite engine.""" + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + yield engine + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def dev_session(dev_engine): + """DB session scoped to one test.""" + Session = sessionmaker(bind=dev_engine) + session = Session() + yield session + session.close() + + +def _make_client(dev_engine, owner_id: str = _OWNER) -> TestClient: + """Return a TestClient with *owner_id* injected as the authenticated user.""" + from app.api.api_tokens import _get_owner_id + from app.main import app + + Session = sessionmaker(bind=dev_engine) + + def _override_get_db(): + session = Session() + try: + yield session + finally: + session.close() + + def _override_owner(): + return owner_id + + app.dependency_overrides[get_db] = _override_get_db + app.dependency_overrides[_get_owner_id] = _override_owner + + client = TestClient(app, base_url="http://localhost", raise_server_exceptions=False) + return client + + +def _cleanup(app): + """Remove dependency overrides after test.""" + app.dependency_overrides.clear() + + +def _seed_tokens(session, owner_id: str = _OWNER): + """Create a mix of regular and mobile tokens for testing.""" + from app.api.api_tokens import generate_api_token, hash_token + + tokens = [] + # Regular API tokens + for name in ["CI Pipeline", "Webhook Upload"]: + pt = generate_api_token() + t = ApiToken(owner_id=owner_id, name=name, token_hash=hash_token(pt), token_prefix=pt[:12]) + session.add(t) + tokens.append(t) + + # Mobile tokens (various naming patterns) + for name in [ + "Mobile App – iPhone 15 Pro", + "Mobile App (QR) – Christian's iPad", + "Mobile App", + ]: + pt = generate_api_token() + t = ApiToken(owner_id=owner_id, name=name, token_hash=hash_token(pt), token_prefix=pt[:12]) + session.add(t) + tokens.append(t) + + session.commit() + return tokens + + +# --------------------------------------------------------------------------- +# Tests – Mobile Token Filtering +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestMobileTokenFiltering: + """Tests for GET /api/api-tokens/mobile and filtering from GET /api/api-tokens/.""" + + def test_list_mobile_tokens_returns_only_mobile(self, dev_engine, dev_session): + """GET /api/api-tokens/mobile should only return tokens starting with 'Mobile App'.""" + _seed_tokens(dev_session) + client = _make_client(dev_engine) + try: + res = client.get("/api/api-tokens/mobile") + assert res.status_code == 200 + data = res.json() + assert len(data) == 3 + for t in data: + assert t["name"].startswith("Mobile App") + finally: + _cleanup(client.app) + + def test_list_regular_tokens_excludes_mobile(self, dev_engine, dev_session): + """GET /api/api-tokens/ should NOT return tokens starting with 'Mobile App'.""" + _seed_tokens(dev_session) + client = _make_client(dev_engine) + try: + res = client.get("/api/api-tokens/") + assert res.status_code == 200 + data = res.json() + assert len(data) == 2 + for t in data: + assert not t["name"].startswith("Mobile App") + finally: + _cleanup(client.app) + + def test_list_mobile_tokens_empty(self, dev_engine): + """GET /api/api-tokens/mobile returns [] when no mobile tokens exist.""" + client = _make_client(dev_engine) + try: + res = client.get("/api/api-tokens/mobile") + assert res.status_code == 200 + assert res.json() == [] + finally: + _cleanup(client.app) + + def test_list_mobile_tokens_isolation(self, dev_engine, dev_session): + """Mobile tokens for other users should not appear.""" + _seed_tokens(dev_session, owner_id=_OTHER_OWNER) + client = _make_client(dev_engine, owner_id=_OWNER) + try: + res = client.get("/api/api-tokens/mobile") + assert res.status_code == 200 + assert res.json() == [] + finally: + _cleanup(client.app) + + def test_mobile_token_revoke_via_api_tokens_endpoint(self, dev_engine, dev_session): + """Mobile tokens can still be revoked via DELETE /api/api-tokens/{id}.""" + tokens = _seed_tokens(dev_session) + mobile_token = next(t for t in tokens if t.name.startswith("Mobile App")) + client = _make_client(dev_engine) + try: + res = client.delete(f"/api/api-tokens/{mobile_token.id}") + assert res.status_code == 200 + # Verify it's gone from mobile list + res2 = client.get("/api/api-tokens/mobile") + active_names = [t["name"] for t in res2.json() if t["is_active"]] + assert mobile_token.name not in active_names + finally: + _cleanup(client.app) + + +# --------------------------------------------------------------------------- +# Tests – Devices Page View +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestDevicesPageView: + """Tests for GET /devices page rendering.""" + + def test_devices_page_renders(self, dev_engine): + """GET /devices should return 200 with the devices template.""" + from app.views.devices import router as _ # noqa: F401 – ensures route is registered + + client = _make_client(dev_engine) + try: + res = client.get("/devices") + assert res.status_code == 200 + assert "devices.heading" in res.text or "Mobile Devices" in res.text + finally: + _cleanup(client.app) diff --git a/tests/test_diagnostic.py b/tests/test_diagnostic.py index fa47acc1..8d3f917a 100644 --- a/tests/test_diagnostic.py +++ b/tests/test_diagnostic.py @@ -5,6 +5,86 @@ from unittest.mock import MagicMock, patch import pytest +@pytest.mark.unit +class TestLivenessProbe: + """Tests for GET /api/diagnostic/healthz/live (unauthenticated).""" + + def test_liveness_returns_200(self, client): + """Liveness probe always returns 200 OK.""" + response = client.get("/api/diagnostic/healthz/live") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "ok" + + +@pytest.mark.unit +class TestReadinessProbe: + """Tests for GET /api/diagnostic/healthz/ready (unauthenticated).""" + + def test_readiness_returns_200_when_all_ok(self, client): + """Readiness probe returns 200 when database and Redis are reachable.""" + with ( + patch("app.api.diagnostic.engine") as mock_engine, + patch("app.api.diagnostic.redis_lib") as mock_redis, + ): + mock_conn = MagicMock() + mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn) + mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False) + mock_redis_inst = MagicMock() + mock_redis.from_url.return_value = mock_redis_inst + + response = client.get("/api/diagnostic/healthz/ready") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "ready" + assert data["checks"]["database"]["status"] == "ok" + + def test_readiness_returns_503_when_database_fails(self, client): + """Readiness probe returns 503 when database is unreachable.""" + with ( + patch("app.api.diagnostic.engine") as mock_engine, + patch("app.api.diagnostic.redis_lib") as mock_redis, + ): + mock_engine.connect.side_effect = Exception("DB unavailable") + mock_redis_inst = MagicMock() + mock_redis.from_url.return_value = mock_redis_inst + + response = client.get("/api/diagnostic/healthz/ready") + + assert response.status_code == 503 + data = response.json() + assert data["status"] == "not_ready" + assert data["checks"]["database"]["status"] == "error" + + def test_readiness_returns_200_when_redis_fails(self, client): + """Readiness remains 200 when only Redis is down (non-critical).""" + with ( + patch("app.api.diagnostic.engine") as mock_engine, + patch("app.api.diagnostic.redis_lib") as mock_redis, + ): + mock_conn = MagicMock() + mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn) + mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False) + mock_redis.from_url.return_value = MagicMock() + mock_redis.from_url.return_value.ping.side_effect = Exception("Connection refused") + + response = client.get("/api/diagnostic/healthz/ready") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "ready" + assert data["checks"]["redis"]["status"] == "error" + + def test_readiness_contains_checks_keys(self, client): + """Readiness response always contains database and redis checks.""" + response = client.get("/api/diagnostic/healthz/ready") + data = response.json() + assert "checks" in data + assert "database" in data["checks"] + assert "redis" in data["checks"] + + @pytest.mark.unit class TestHealthEndpoint: """Tests for GET /api/diagnostic/health endpoint.""" diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index 45486168..987b1599 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -3,11 +3,12 @@ Covers: - ``GET /api/duplicates`` — list all exact-duplicate groups - ``GET /api/files/{id}/duplicates`` — per-file exact + near-duplicate info -- ``POST /api/ui-upload`` — exact-duplicate warning in upload response +- ``POST /api/ui-upload`` — exact-duplicate rejection at upload time - ``GET /duplicates`` — duplicate management UI page """ import json +import os from unittest.mock import patch import pytest @@ -283,17 +284,25 @@ class TestGetFileDuplicates: # --------------------------------------------------------------------------- -# POST /api/ui-upload — exact-duplicate warning +# POST /api/ui-upload — exact-duplicate rejection # --------------------------------------------------------------------------- -class TestUploadDuplicateWarning: - """Tests for duplicate warning injected into the upload response.""" +class TestUploadDuplicateRejection: + """Tests for duplicate rejection at upload time. + + When ``ENABLE_DEDUPLICATION`` is ``True`` (the default) and the uploaded + file's SHA-256 hash matches an already-processed document, the upload + endpoint must: + - return ``status: "duplicate"`` instead of ``"queued"`` + - **not** enqueue a Celery task + - clean up the temporary file from disk + """ @pytest.mark.integration @patch("app.tasks.process_document.process_document.delay") def test_no_warning_for_unique_file(self, mock_delay, client: TestClient, tmp_path): - """Uploading a unique file should not produce a duplicate_warning.""" + """Uploading a unique file should not produce a duplicate response.""" mock_delay.return_value.id = "task-unique" pdf = tmp_path / "unique.pdf" pdf.write_bytes(b"%PDF-1.4\n%%EOF") @@ -306,14 +315,12 @@ class TestUploadDuplicateWarning: assert response.status_code == 200 data = response.json() - assert "duplicate_warning" not in data or data.get("duplicate_warning") is None + assert data["status"] == "queued" + assert "duplicate_of" not in data @pytest.mark.integration - @patch("app.tasks.process_document.process_document.delay") - def test_warning_for_exact_duplicate(self, mock_delay, client: TestClient, db_session, tmp_path): - """Uploading a file with the same hash as an existing record returns a warning.""" - mock_delay.return_value.id = "task-dup" - + def test_exact_duplicate_rejected(self, client: TestClient, db_session, tmp_path): + """Uploading a file with the same hash as an existing record is rejected.""" # Create a real PDF with known content pdf_bytes = b"%PDF-1.4\nsome unique content for test\n%%EOF" pdf = tmp_path / "existing.pdf" @@ -335,16 +342,14 @@ class TestUploadDuplicateWarning: assert response.status_code == 200 data = response.json() - assert "duplicate_warning" in data - assert data["duplicate_warning"]["duplicate_type"] == "exact" - assert data["duplicate_warning"]["original_file_id"] == existing.id + assert data["status"] == "duplicate" + assert "duplicate_of" in data + assert data["duplicate_of"]["duplicate_type"] == "exact" + assert data["duplicate_of"]["original_file_id"] == existing.id @pytest.mark.integration - @patch("app.tasks.process_document.process_document.delay") - def test_upload_still_queued_despite_warning(self, mock_delay, client: TestClient, db_session, tmp_path): - """Even when a duplicate is detected, the file should still be queued.""" - mock_delay.return_value.id = "task-still-queued" - + def test_duplicate_not_enqueued(self, client: TestClient, db_session, tmp_path): + """When a duplicate is detected, no Celery task should be created.""" pdf_bytes = b"%PDF-1.4\nqueue test content\n%%EOF" pdf = tmp_path / "queue_test.pdf" pdf.write_bytes(pdf_bytes) @@ -354,16 +359,47 @@ class TestUploadDuplicateWarning: filehash = hash_file(str(pdf)) _make_file(db_session, filehash=filehash, filename="queue_orig.pdf") - with open(pdf, "rb") as f: - response = client.post( - "/api/ui-upload", - files={"file": ("queue_test.pdf", f, "application/pdf")}, - ) + with patch("app.tasks.process_document.process_document.delay") as mock_delay: + with open(pdf, "rb") as f: + response = client.post( + "/api/ui-upload", + files={"file": ("queue_test.pdf", f, "application/pdf")}, + ) assert response.status_code == 200 data = response.json() - assert "task_id" in data - assert data["status"] == "queued" + assert data["status"] == "duplicate" + assert "task_id" not in data + mock_delay.assert_not_called() + + @pytest.mark.integration + def test_duplicate_temp_file_cleaned_up(self, client: TestClient, db_session, tmp_path): + """The temporary file saved to disk should be removed for a duplicate.""" + pdf_bytes = b"%PDF-1.4\ncleanup test content\n%%EOF" + pdf = tmp_path / "cleanup_test.pdf" + pdf.write_bytes(pdf_bytes) + + from app.utils.file_operations import hash_file + + filehash = hash_file(str(pdf)) + _make_file(db_session, filehash=filehash, filename="cleanup_orig.pdf") + + with patch("app.tasks.process_document.process_document.delay"): + with open(pdf, "rb") as f: + response = client.post( + "/api/ui-upload", + files={"file": ("cleanup_test.pdf", f, "application/pdf")}, + ) + + assert response.status_code == 200 + data = response.json() + # The stored_filename is returned so we can verify cleanup + stored = data.get("stored_filename") + assert stored is not None + + from app.config import settings + + assert not os.path.exists(os.path.join(settings.workdir, stored)) # --------------------------------------------------------------------------- diff --git a/tests/test_local_auth.py b/tests/test_local_auth.py index 8a53604c..babbbc30 100644 --- a/tests/test_local_auth.py +++ b/tests/test_local_auth.py @@ -322,6 +322,35 @@ def test_signup_duplicate_username(la_client, active_user): assert "Username" in resp.json()["detail"] +@pytest.mark.integration +def test_signup_invalid_username_with_dot(la_client): + """POST /api/auth/signup returns 422 with a list detail when username contains a dot. + + This is a regression test for the bug where ``data.detail`` was an array, + causing the frontend to display ``[object Object]`` instead of a message. + """ + with patch("app.api.local_auth.settings") as mock_settings: + mock_settings.allow_local_signup = True + mock_settings.multi_user_enabled = True + mock_settings.email_host = "smtp.example.com" + resp = la_client.post( + "/api/auth/signup", + json={ + "email": "a@example.com", + "username": "christian.louis", + "password": "password1", + "password_confirm": "password1", + }, + ) + assert resp.status_code == 422 + detail = resp.json()["detail"] + # FastAPI returns a list of validation errors for Pydantic constraint failures. + # Each entry must be a dict with a "msg" key so the frontend can extract a readable message. + assert isinstance(detail, list), "detail should be a list for Pydantic validation errors" + assert len(detail) > 0 + assert "msg" in detail[0] + + @pytest.mark.integration def test_signup_smtp_failure_cleans_up(la_client, la_session): """POST /api/auth/signup cleans up user records if email send fails.""" diff --git a/tests/test_send_to_all.py b/tests/test_send_to_all.py index 0bab975f..3cf0c42f 100644 --- a/tests/test_send_to_all.py +++ b/tests/test_send_to_all.py @@ -360,6 +360,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all.upload_to_dropbox") @@ -368,6 +369,7 @@ class TestSendToAllDestinations: mock_upload, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -397,6 +399,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False mock_upload.delay.return_value = MagicMock(id="task-123") @@ -410,6 +413,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all.settings") @patch("app.tasks.send_to_all._should_upload_to_dropbox") @patch("app.tasks.send_to_all._should_upload_to_icloud") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all._should_upload_to_nextcloud") @patch("app.tasks.send_to_all._should_upload_to_paperless") @@ -434,6 +438,7 @@ class TestSendToAllDestinations: mock_paperless, mock_nextcloud, mock_should_s3, + mock_sharepoint, mock_icloud, mock_should_dropbox, mock_settings, @@ -456,6 +461,7 @@ class TestSendToAllDestinations: mock_sftp.return_value = False mock_email.return_value = False mock_onedrive.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False mock_dropbox_upload.delay.return_value = MagicMock(id="dropbox-task") mock_s3_upload.delay.return_value = MagicMock(id="s3-task") @@ -478,12 +484,14 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") def test_skips_unconfigured_services( self, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -513,6 +521,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False result = send_to_all_destinations.apply(args=[str(test_file), False, 1]) @@ -534,6 +543,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all.upload_to_dropbox") @@ -542,6 +552,7 @@ class TestSendToAllDestinations: mock_upload, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -571,6 +582,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False mock_upload.delay.return_value = MagicMock(id="task-123") @@ -593,6 +605,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all.get_configured_services_from_validator") @@ -603,6 +616,7 @@ class TestSendToAllDestinations: mock_validator, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -633,6 +647,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False mock_upload.delay.return_value = MagicMock(id="task-123") @@ -653,6 +668,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all.get_configured_services_from_validator") @@ -661,6 +677,7 @@ class TestSendToAllDestinations: mock_validator, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -691,6 +708,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False # Should not raise, should fall back to individual checks @@ -710,6 +728,7 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all.upload_to_dropbox") @@ -718,6 +737,7 @@ class TestSendToAllDestinations: mock_upload, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -747,6 +767,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False mock_upload.delay.side_effect = Exception("Queue error") @@ -758,6 +779,7 @@ class TestSendToAllDestinations: assert "dropbox_error" in result.result["tasks"] @patch("app.tasks.send_to_all._should_upload_to_icloud") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_s3") @patch("app.tasks.send_to_all._should_upload_to_onedrive") @patch("app.tasks.send_to_all._should_upload_to_email") @@ -786,6 +808,7 @@ class TestSendToAllDestinations: mock_email, mock_onedrive, mock_s3, + mock_sharepoint, mock_icloud, tmp_path, ): @@ -808,6 +831,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False # Mock database session @@ -836,12 +860,14 @@ class TestSendToAllDestinations: @patch("app.tasks.send_to_all._should_upload_to_sftp") @patch("app.tasks.send_to_all._should_upload_to_email") @patch("app.tasks.send_to_all._should_upload_to_onedrive") + @patch("app.tasks.send_to_all._should_upload_to_sharepoint") @patch("app.tasks.send_to_all._should_upload_to_icloud") @patch("app.tasks.send_to_all._should_upload_to_s3") def test_should_upload_check_exception_handling( self, mock_s3, mock_icloud, + mock_sharepoint, mock_onedrive, mock_email, mock_sftp, @@ -871,6 +897,7 @@ class TestSendToAllDestinations: mock_email.return_value = False mock_onedrive.return_value = False mock_s3.return_value = False + mock_sharepoint.return_value = False mock_icloud.return_value = False # Should not raise, should treat as not configured diff --git a/tests/test_session_management.py b/tests/test_session_management.py new file mode 100644 index 00000000..e11c93ca --- /dev/null +++ b/tests/test_session_management.py @@ -0,0 +1,699 @@ +"""Tests for server-side session management and QR code login. + +Covers: +* Session creation, validation, revocation, and cleanup +* "Log off everywhere" (revoke all sessions) +* QR login challenge creation, validation, claiming, and status polling +* Session management API endpoints (list, revoke, revoke-all) +* QR auth API endpoints (challenge, status, claim) +* Device info parsing from User-Agent strings +""" + +from __future__ import annotations + +import secrets +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from app.database import Base +from app.models import ApiToken, QRLoginChallenge, UserSession + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def db_session(): + """Provide an in-memory SQLite session with all tables created.""" + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + TestSession = sessionmaker(bind=engine) + session = TestSession() + yield session + session.close() + Base.metadata.drop_all(engine) + + +@pytest.fixture() +def sample_user_id(): + return "user@example.com" + + +# --------------------------------------------------------------------------- +# Model Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestUserSessionModel: + """Tests for the UserSession ORM model.""" + + def test_create_user_session(self, db_session: Session, sample_user_id: str): + """Test creating a UserSession record.""" + now = datetime.now(timezone.utc) + session = UserSession( + session_token=secrets.token_urlsafe(64), + user_id=sample_user_id, + ip_address="192.168.1.1", + user_agent="Mozilla/5.0", + device_info="Chrome on macOS", + expires_at=now + timedelta(days=30), + ) + db_session.add(session) + db_session.commit() + + assert session.id is not None + assert session.user_id == sample_user_id + assert session.is_revoked is False + assert session.device_info == "Chrome on macOS" + + def test_session_default_values(self, db_session: Session, sample_user_id: str): + """Test that default values are set correctly.""" + session = UserSession( + session_token="test_token_123", + user_id=sample_user_id, + expires_at=datetime.now(timezone.utc) + timedelta(days=30), + ) + db_session.add(session) + db_session.commit() + + assert session.is_revoked is False + assert session.revoked_at is None + + +@pytest.mark.unit +class TestQRLoginChallengeModel: + """Tests for the QRLoginChallenge ORM model.""" + + def test_create_challenge(self, db_session: Session, sample_user_id: str): + """Test creating a QRLoginChallenge record.""" + challenge = QRLoginChallenge( + challenge_token=secrets.token_urlsafe(64), + user_id=sample_user_id, + created_by_ip="10.0.0.1", + expires_at=datetime.now(timezone.utc) + timedelta(seconds=120), + ) + db_session.add(challenge) + db_session.commit() + + assert challenge.id is not None + assert challenge.is_claimed is False + assert challenge.is_cancelled is False + + def test_challenge_default_values(self, db_session: Session, sample_user_id: str): + """Test that QRLoginChallenge defaults are correct.""" + challenge = QRLoginChallenge( + challenge_token="challenge_test_123", + user_id=sample_user_id, + expires_at=datetime.now(timezone.utc) + timedelta(seconds=120), + ) + db_session.add(challenge) + db_session.commit() + + assert challenge.is_claimed is False + assert challenge.is_cancelled is False + assert challenge.claimed_at is None + assert challenge.device_name is None + + +# --------------------------------------------------------------------------- +# Session Manager Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestSessionManager: + """Tests for app/utils/session_manager.py functions.""" + + @patch("app.utils.session_manager.settings") + def test_get_session_lifetime_days_default(self, mock_settings): + """Test default session lifetime.""" + from app.utils.session_manager import get_session_lifetime_days + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + assert get_session_lifetime_days() == 30 + + @patch("app.utils.session_manager.settings") + def test_get_session_lifetime_days_custom(self, mock_settings): + """Test custom session lifetime overrides default.""" + from app.utils.session_manager import get_session_lifetime_days + + mock_settings.session_lifetime_custom_days = 90 + mock_settings.session_lifetime_days = 30 + assert get_session_lifetime_days() == 90 + + @patch("app.utils.session_manager.settings") + def test_get_session_lifetime_days_minimum(self, mock_settings): + """Test session lifetime has a minimum of 1 day.""" + from app.utils.session_manager import get_session_lifetime_days + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 0 + assert get_session_lifetime_days() == 1 + + @patch("app.utils.session_manager.settings") + def test_get_session_max_age_seconds(self, mock_settings): + """Test session max age in seconds.""" + from app.utils.session_manager import get_session_max_age_seconds + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + assert get_session_max_age_seconds() == 30 * 86400 + + @patch("app.utils.session_manager.settings") + def test_create_session(self, mock_settings, db_session: Session, sample_user_id: str): + """Test creating a server-side session.""" + from app.utils.session_manager import create_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + mock_settings.qr_login_challenge_ttl_seconds = 120 + + user_session = create_session( + db_session, + user_id=sample_user_id, + ip_address="10.0.0.1", + user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0", + ) + + assert user_session.id is not None + assert user_session.user_id == sample_user_id + assert user_session.ip_address == "10.0.0.1" + assert user_session.session_token is not None + assert len(user_session.session_token) > 32 + assert user_session.is_revoked is False + assert user_session.device_info is not None + + @patch("app.utils.session_manager.settings") + def test_validate_session_valid(self, mock_settings, db_session: Session, sample_user_id: str): + """Test validating a valid session.""" + from app.utils.session_manager import create_session, validate_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + user_session = create_session(db_session, user_id=sample_user_id) + result = validate_session(db_session, user_session.session_token) + assert result is not None + assert result.id == user_session.id + + @patch("app.utils.session_manager.settings") + def test_validate_session_revoked(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that revoked sessions are rejected.""" + from app.utils.session_manager import create_session, validate_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + user_session = create_session(db_session, user_id=sample_user_id) + user_session.is_revoked = True + db_session.commit() + + result = validate_session(db_session, user_session.session_token) + assert result is None + + @patch("app.utils.session_manager.settings") + def test_validate_session_expired(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that expired sessions are rejected.""" + from app.utils.session_manager import create_session, validate_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + user_session = create_session(db_session, user_id=sample_user_id) + user_session.expires_at = datetime.now(timezone.utc) - timedelta(hours=1) + db_session.commit() + + result = validate_session(db_session, user_session.session_token) + assert result is None + + def test_validate_session_empty_token(self, db_session: Session): + """Test that empty token returns None.""" + from app.utils.session_manager import validate_session + + assert validate_session(db_session, "") is None + assert validate_session(db_session, None) is None + + def test_validate_session_nonexistent_token(self, db_session: Session): + """Test that nonexistent token returns None.""" + from app.utils.session_manager import validate_session + + assert validate_session(db_session, "nonexistent_token_xyz") is None + + @patch("app.utils.session_manager.settings") + def test_revoke_session(self, mock_settings, db_session: Session, sample_user_id: str): + """Test revoking a single session.""" + from app.utils.session_manager import create_session, revoke_session, validate_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + user_session = create_session(db_session, user_id=sample_user_id) + assert revoke_session(db_session, user_session.id, sample_user_id) is True + + # Session should now be invalid + assert validate_session(db_session, user_session.session_token) is None + + @patch("app.utils.session_manager.settings") + def test_revoke_session_wrong_user(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that a user cannot revoke another user's session.""" + from app.utils.session_manager import create_session, revoke_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + user_session = create_session(db_session, user_id=sample_user_id) + assert revoke_session(db_session, user_session.id, "other_user@example.com") is False + + @patch("app.utils.session_manager.settings") + def test_revoke_all_sessions(self, mock_settings, db_session: Session, sample_user_id: str): + """Test revoking all sessions for a user.""" + from app.utils.session_manager import create_session, list_user_sessions, revoke_all_sessions + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + s1 = create_session(db_session, user_id=sample_user_id) + s2 = create_session(db_session, user_id=sample_user_id) + s3 = create_session(db_session, user_id=sample_user_id) + + count = revoke_all_sessions(db_session, sample_user_id, revoke_api_tokens=False) + assert count == 3 + + # All sessions should be revoked + active = list_user_sessions(db_session, sample_user_id) + assert len(active) == 0 + + @patch("app.utils.session_manager.settings") + def test_revoke_all_except_current(self, mock_settings, db_session: Session, sample_user_id: str): + """Test revoking all sessions except the current one.""" + from app.utils.session_manager import create_session, list_user_sessions, revoke_all_sessions + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + s1 = create_session(db_session, user_id=sample_user_id) + s2 = create_session(db_session, user_id=sample_user_id) + s3 = create_session(db_session, user_id=sample_user_id) + + count = revoke_all_sessions( + db_session, + sample_user_id, + except_session_id=s1.id, + revoke_api_tokens=False, + ) + assert count == 2 + + active = list_user_sessions(db_session, sample_user_id) + assert len(active) == 1 + assert active[0].id == s1.id + + @patch("app.utils.session_manager.settings") + def test_revoke_all_includes_api_tokens(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that revoke-all also revokes API tokens.""" + from app.utils.session_manager import create_session, revoke_all_sessions + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + create_session(db_session, user_id=sample_user_id) + + # Create an API token + token = ApiToken( + owner_id=sample_user_id, + name="Test Token", + token_hash="abc123hash", + token_prefix="de_abc12345", + ) + db_session.add(token) + db_session.commit() + + revoke_all_sessions(db_session, sample_user_id, revoke_api_tokens=True) + + db_session.refresh(token) + assert token.is_active is False + + @patch("app.utils.session_manager.settings") + def test_list_user_sessions(self, mock_settings, db_session: Session, sample_user_id: str): + """Test listing active sessions for a user.""" + from app.utils.session_manager import create_session, list_user_sessions + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + create_session(db_session, user_id=sample_user_id) + create_session(db_session, user_id=sample_user_id) + create_session(db_session, user_id="other@example.com") + + sessions = list_user_sessions(db_session, sample_user_id) + assert len(sessions) == 2 + + @patch("app.utils.session_manager.settings") + def test_cleanup_expired_sessions(self, mock_settings, db_session: Session, sample_user_id: str): + """Test cleaning up expired sessions.""" + from app.utils.session_manager import cleanup_expired_sessions, create_session + + mock_settings.session_lifetime_custom_days = None + mock_settings.session_lifetime_days = 30 + + # Create a session that expired 10 days ago + session = create_session(db_session, user_id=sample_user_id) + session.expires_at = datetime.now(timezone.utc) - timedelta(days=10) + db_session.commit() + + count = cleanup_expired_sessions(db_session) + assert count == 1 + + +# --------------------------------------------------------------------------- +# QR Login Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestQRLogin: + """Tests for QR login challenge/claim flow.""" + + @patch("app.utils.session_manager.settings") + def test_create_qr_challenge(self, mock_settings, db_session: Session, sample_user_id: str): + """Test creating a QR login challenge.""" + from app.utils.session_manager import create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id, ip_address="10.0.0.1") + + assert challenge.id is not None + assert challenge.user_id == sample_user_id + assert challenge.challenge_token is not None + assert len(challenge.challenge_token) > 32 + assert challenge.is_claimed is False + assert challenge.created_by_ip == "10.0.0.1" + # SQLite returns naive datetimes; normalise before comparison + expires = challenge.expires_at + if expires.tzinfo is None: + expires = expires.replace(tzinfo=timezone.utc) + assert expires > datetime.now(timezone.utc) + + @patch("app.utils.session_manager.settings") + def test_create_qr_challenge_ttl_seconds(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that ttl_seconds can be derived from created_at and expires_at. + + The API endpoint computes ttl_seconds = (expires_at - created_at) to + allow the client to run a countdown timer without comparing absolute + timestamps (avoiding clock-skew issues). + """ + from app.utils.session_manager import create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + + ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds())) + assert ttl_seconds == 120 + + @patch("app.utils.session_manager.settings") + def test_create_qr_challenge_custom_ttl(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that a custom TTL is correctly reflected in the challenge timestamps.""" + from app.utils.session_manager import create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 300 + + challenge = create_qr_challenge(db_session, sample_user_id) + + ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds())) + assert ttl_seconds == 300 + + @patch("app.utils.session_manager.settings") + def test_validate_qr_challenge_valid(self, mock_settings, db_session: Session, sample_user_id: str): + """Test validating a valid QR challenge.""" + from app.utils.session_manager import create_qr_challenge, validate_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + result = validate_qr_challenge(db_session, challenge.challenge_token) + assert result is not None + assert result.id == challenge.id + + @patch("app.utils.session_manager.settings") + def test_validate_qr_challenge_expired(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that expired challenges are rejected.""" + from app.utils.session_manager import create_qr_challenge, validate_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + challenge.expires_at = datetime.now(timezone.utc) - timedelta(minutes=1) + db_session.commit() + + result = validate_qr_challenge(db_session, challenge.challenge_token) + assert result is None + + @patch("app.utils.session_manager.settings") + def test_validate_qr_challenge_claimed(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that claimed challenges are rejected (replay protection).""" + from app.utils.session_manager import create_qr_challenge, validate_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + challenge.is_claimed = True + db_session.commit() + + result = validate_qr_challenge(db_session, challenge.challenge_token) + assert result is None + + @patch("app.utils.session_manager.settings") + def test_validate_qr_challenge_cancelled(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that cancelled challenges are rejected.""" + from app.utils.session_manager import create_qr_challenge, validate_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + challenge.is_cancelled = True + db_session.commit() + + result = validate_qr_challenge(db_session, challenge.challenge_token) + assert result is None + + def test_validate_qr_challenge_empty(self, db_session: Session): + """Test that empty challenge token returns None.""" + from app.utils.session_manager import validate_qr_challenge + + assert validate_qr_challenge(db_session, "") is None + assert validate_qr_challenge(db_session, None) is None + + @patch("app.utils.session_manager.settings") + def test_claim_qr_challenge_success(self, mock_settings, db_session: Session, sample_user_id: str): + """Test successfully claiming a QR challenge.""" + from app.utils.session_manager import claim_qr_challenge, create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + result = claim_qr_challenge( + db_session, + challenge.challenge_token, + device_name="Christian's iPhone 15 Pro", + ip_address="192.168.1.100", + ) + + assert result is not None + assert result["token"].startswith("de_") + assert result["token_id"] is not None + assert result["owner_id"] == sample_user_id + assert "QR" in result["name"] + + # Challenge should now be claimed + db_session.refresh(challenge) + assert challenge.is_claimed is True + assert challenge.claimed_by_ip == "192.168.1.100" + assert challenge.device_name == "Christian's iPhone 15 Pro" + + @patch("app.utils.session_manager.settings") + def test_claim_qr_challenge_replay_protection(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that a claimed challenge cannot be claimed again.""" + from app.utils.session_manager import claim_qr_challenge, create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + + # First claim succeeds + result1 = claim_qr_challenge(db_session, challenge.challenge_token) + assert result1 is not None + + # Second claim fails (replay protection) + result2 = claim_qr_challenge(db_session, challenge.challenge_token) + assert result2 is None + + @patch("app.utils.session_manager.settings") + def test_claim_qr_challenge_expired(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that expired challenges cannot be claimed.""" + from app.utils.session_manager import claim_qr_challenge, create_qr_challenge + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + challenge.expires_at = datetime.now(timezone.utc) - timedelta(minutes=1) + db_session.commit() + + result = claim_qr_challenge(db_session, challenge.challenge_token) + assert result is None + + def test_claim_qr_challenge_invalid_token(self, db_session: Session): + """Test claiming with an invalid token.""" + from app.utils.session_manager import claim_qr_challenge + + result = claim_qr_challenge(db_session, "nonexistent_token_xyz") + assert result is None + + @patch("app.utils.session_manager.settings") + def test_get_challenge_status_pending(self, mock_settings, db_session: Session, sample_user_id: str): + """Test getting status of a pending challenge.""" + from app.utils.session_manager import create_qr_challenge, get_challenge_status + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + status = get_challenge_status(db_session, challenge.id, sample_user_id) + + assert status is not None + assert status["status"] == "pending" + + @patch("app.utils.session_manager.settings") + def test_get_challenge_status_claimed(self, mock_settings, db_session: Session, sample_user_id: str): + """Test getting status of a claimed challenge.""" + from app.utils.session_manager import claim_qr_challenge, create_qr_challenge, get_challenge_status + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + claim_qr_challenge(db_session, challenge.challenge_token, device_name="Test Device") + + status = get_challenge_status(db_session, challenge.id, sample_user_id) + assert status is not None + assert status["status"] == "claimed" + assert status["device_name"] == "Test Device" + + @patch("app.utils.session_manager.settings") + def test_get_challenge_status_expired(self, mock_settings, db_session: Session, sample_user_id: str): + """Test getting status of an expired challenge.""" + from app.utils.session_manager import create_qr_challenge, get_challenge_status + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + challenge.expires_at = datetime.now(timezone.utc) - timedelta(minutes=1) + db_session.commit() + + status = get_challenge_status(db_session, challenge.id, sample_user_id) + assert status["status"] == "expired" + + @patch("app.utils.session_manager.settings") + def test_get_challenge_status_wrong_user(self, mock_settings, db_session: Session, sample_user_id: str): + """Test that a user cannot see another user's challenge status.""" + from app.utils.session_manager import create_qr_challenge, get_challenge_status + + mock_settings.qr_login_challenge_ttl_seconds = 120 + + challenge = create_qr_challenge(db_session, sample_user_id) + status = get_challenge_status(db_session, challenge.id, "other@example.com") + assert status is None + + +# --------------------------------------------------------------------------- +# Device Info Parsing Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestDeviceInfoParsing: + """Tests for User-Agent parsing.""" + + def test_chrome_macos(self): + from app.utils.session_manager import _parse_device_info + + ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + result = _parse_device_info(ua) + assert "Chrome" in result + assert "macOS" in result + + def test_safari_iphone(self): + from app.utils.session_manager import _parse_device_info + + ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1" + result = _parse_device_info(ua) + assert "Safari" in result + assert "iPhone" in result + + def test_firefox_windows(self): + from app.utils.session_manager import _parse_device_info + + ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0" + result = _parse_device_info(ua) + assert "Firefox" in result + assert "Windows" in result + + def test_edge_windows(self): + from app.utils.session_manager import _parse_device_info + + ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0" + result = _parse_device_info(ua) + assert "Edge" in result + assert "Windows" in result + + def test_android_chrome(self): + from app.utils.session_manager import _parse_device_info + + ua = "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.210 Mobile Safari/537.36" + result = _parse_device_info(ua) + assert "Chrome" in result + assert "Android" in result + + def test_none_user_agent(self): + from app.utils.session_manager import _parse_device_info + + assert _parse_device_info(None) is None + + def test_empty_user_agent(self): + from app.utils.session_manager import _parse_device_info + + assert _parse_device_info("") is None + + +# --------------------------------------------------------------------------- +# Config Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestSessionConfig: + """Tests for session-related configuration fields.""" + + def test_session_lifetime_days_field_exists(self): + """Verify session_lifetime_days field is defined in Settings.""" + from app.config import Settings + + # Check the field exists in the model + assert "session_lifetime_days" in Settings.model_fields + + def test_session_lifetime_custom_days_field_exists(self): + """Verify session_lifetime_custom_days field is defined in Settings.""" + from app.config import Settings + + assert "session_lifetime_custom_days" in Settings.model_fields + + def test_qr_login_challenge_ttl_field_exists(self): + """Verify qr_login_challenge_ttl_seconds field is defined in Settings.""" + from app.config import Settings + + assert "qr_login_challenge_ttl_seconds" in Settings.model_fields diff --git a/tests/test_system_reset.py b/tests/test_system_reset.py new file mode 100644 index 00000000..570ebd19 --- /dev/null +++ b/tests/test_system_reset.py @@ -0,0 +1,393 @@ +"""Tests for the system reset feature (app/api/system_reset.py, app/utils/system_reset.py, app/views/system_reset.py).""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base +from app.models import ( + DocumentMetadata, + FileProcessingStep, + FileRecord, + ProcessingLog, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def reset_workdir(): + """Create a temporary workdir populated with sample user data.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create data subdirectories with dummy files + for subdir in ("original", "processed", "tmp", "pdfa", "backups"): + d = Path(tmpdir) / subdir + d.mkdir() + (d / "sample.pdf").write_bytes(b"%PDF-1.4 fake") + + # Create cache files + for cache in ("watch_folder_processed.json", "ftp_ingest_processed.json"): + (Path(tmpdir) / cache).write_text("{}") + + # Create a per-user watch folder cache + (Path(tmpdir) / "user_wf_42.json").write_text("{}") + + # Create a loose PDF in workdir root + (Path(tmpdir) / "abc123.pdf").write_bytes(b"%PDF-1.4 loose") + + yield tmpdir + + +@pytest.fixture +def reset_db_session(): + """Fresh in-memory database with sample user data rows.""" + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + session = Session() + + # Seed with sample data + fr = FileRecord( + filehash="abc123", + original_filename="test.pdf", + local_filename="uuid.pdf", + file_size=1024, + mime_type="application/pdf", + ) + session.add(fr) + session.flush() + + session.add(ProcessingLog(file_id=fr.id, task_id="t1", step_name="hash_file", status="success")) + session.add(FileProcessingStep(file_id=fr.id, step_name="hash_file", status="success")) + session.add(DocumentMetadata(filename="test.pdf", sender="Alice", recipient="Bob")) + session.commit() + + yield session + + session.close() + Base.metadata.drop_all(bind=engine) + + +# --------------------------------------------------------------------------- +# Unit tests for app/utils/system_reset.py +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestWipeWorkdirData: + """Tests for _wipe_workdir_data().""" + + def test_removes_data_subdirs(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + result = _wipe_workdir_data(reset_workdir) + + # All data subdirectories should be gone + for subdir in ("original", "processed", "tmp", "pdfa", "backups"): + assert not (Path(reset_workdir) / subdir).exists() + + assert result["deleted_dirs"] == 5 + + def test_removes_cache_files(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + result = _wipe_workdir_data(reset_workdir) + + assert not (Path(reset_workdir) / "watch_folder_processed.json").exists() + assert not (Path(reset_workdir) / "ftp_ingest_processed.json").exists() + assert not (Path(reset_workdir) / "user_wf_42.json").exists() + assert result["deleted_files"] >= 3 + + def test_removes_loose_document_files(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + _wipe_workdir_data(reset_workdir) + assert not (Path(reset_workdir) / "abc123.pdf").exists() + + def test_preserves_workdir_directory(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + _wipe_workdir_data(reset_workdir) + assert Path(reset_workdir).is_dir() + + def test_handles_empty_workdir(self): + """No errors when workdir has no data dirs or caches.""" + from app.utils.system_reset import _wipe_workdir_data + + with tempfile.TemporaryDirectory() as empty_dir: + result = _wipe_workdir_data(empty_dir) + assert result["deleted_dirs"] == 0 + assert result["deleted_files"] == 0 + + +@pytest.mark.unit +class TestWipeDatabase: + """Tests for _wipe_database().""" + + def test_deletes_all_user_data(self, reset_db_session): + from app.utils.system_reset import _wipe_database + + result = _wipe_database(reset_db_session) + + assert result.get("files", 0) >= 1 + assert result.get("processing_logs", 0) >= 1 + assert result.get("file_processing_steps", 0) >= 1 + assert result.get("document_metadata", 0) >= 1 + + def test_tables_are_empty_after_wipe(self, reset_db_session): + from app.utils.system_reset import _wipe_database + + _wipe_database(reset_db_session) + + assert reset_db_session.query(FileRecord).count() == 0 + assert reset_db_session.query(ProcessingLog).count() == 0 + assert reset_db_session.query(FileProcessingStep).count() == 0 + assert reset_db_session.query(DocumentMetadata).count() == 0 + + +@pytest.mark.unit +class TestPerformFullReset: + """Tests for perform_full_reset().""" + + def test_wipes_db_and_filesystem(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_full_reset + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + result = perform_full_reset(reset_db_session) + + assert "database" in result + assert "filesystem" in result + assert reset_db_session.query(FileRecord).count() == 0 + assert not (Path(reset_workdir) / "original").exists() + + +@pytest.mark.unit +class TestPerformResetAndReimport: + """Tests for perform_reset_and_reimport().""" + + def test_copies_originals_to_reimport_then_wipes(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_reset_and_reimport + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + mock_settings.watch_folders = "" + mock_settings.watch_folder_delete_after_process = False + result = perform_reset_and_reimport(reset_db_session) + + reimport_dir = Path(reset_workdir) / "reimport" + assert reimport_dir.is_dir() + assert result["reimport"]["files_moved"] >= 1 + + # DB should be wiped + assert reset_db_session.query(FileRecord).count() == 0 + + # Reimport folder should contain the original file + reimport_files = list(reimport_dir.iterdir()) + assert len(reimport_files) >= 1 + + def test_configures_watch_folder(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_reset_and_reimport + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + mock_settings.watch_folders = "/some/other/folder" + mock_settings.watch_folder_delete_after_process = False + perform_reset_and_reimport(reset_db_session) + + reimport_path = str(Path(reset_workdir) / "reimport") + # watch_folders should now include the reimport path + assert reimport_path in mock_settings.watch_folders + + +@pytest.mark.unit +class TestStartupReset: + """Tests for perform_startup_reset().""" + + def test_startup_reset_calls_full_reset(self): + from app.utils.system_reset import perform_startup_reset + + with patch("app.utils.system_reset.perform_full_reset") as mock_reset: + with patch("app.database.SessionLocal") as mock_sl: + mock_db = mock_sl.return_value + perform_startup_reset() + + mock_reset.assert_called_once_with(mock_db) + mock_db.close.assert_called_once() + + def test_startup_reset_handles_errors(self): + from app.utils.system_reset import perform_startup_reset + + with patch("app.utils.system_reset.perform_full_reset", side_effect=RuntimeError("boom")): + with patch("app.database.SessionLocal") as mock_sl: + mock_db = mock_sl.return_value + # Should not raise + perform_startup_reset() + mock_db.rollback.assert_called_once() + mock_db.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# Integration tests for API endpoints +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestSystemResetApi: + """Tests for the /api/admin/system-reset/ endpoints.""" + + def test_full_reset_requires_admin(self, client): + """Non-admin users get 403.""" + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + assert response.status_code == 403 + + def test_full_reset_requires_feature_flag(self, client): + """Returns 404 when ENABLE_FACTORY_RESET is false.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = False + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + assert response.status_code == 404 + + def test_full_reset_requires_confirmation(self, client): + """Wrong confirmation string gets 400.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "WRONG"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 400 + + def test_reimport_requires_confirmation(self, client): + """Wrong confirmation string gets 400.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + response = client.post( + "/api/admin/system-reset/reimport", + json={"confirmation": "WRONG"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 400 + + def test_status_endpoint(self, client): + """The status endpoint returns feature-flag state.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + response = client.get("/api/admin/system-reset/status") + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + data = response.json() + assert "enabled" in data + assert "factory_reset_on_startup" in data + + def test_full_reset_success(self, client): + """Full reset succeeds with correct confirmation and feature flag.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + with patch( + "app.utils.system_reset.perform_full_reset", return_value={"database": {}, "filesystem": {}} + ): + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + def test_reimport_success(self, client): + """Reimport succeeds with correct confirmation.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + with patch( + "app.utils.system_reset.perform_reset_and_reimport", + return_value={"database": {}, "filesystem": {}, "reimport": {"files_moved": 3}}, + ): + response = client.post( + "/api/admin/system-reset/reimport", + json={"confirmation": "REIMPORT"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +# --------------------------------------------------------------------------- +# Integration tests for the view +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestSystemResetView: + """Tests for the /admin/system-reset view.""" + + def test_view_redirects_when_disabled(self, client): + """When ENABLE_FACTORY_RESET=False, accessing the page redirects away.""" + with client: + client.cookies.set("session", "test") + with patch("app.views.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = False + response = client.get("/admin/system-reset", follow_redirects=False) + # Redirect to /settings (302) when disabled, or to login (302/307) when unauthenticated + assert response.status_code in (302, 307) + + def test_view_requires_auth(self, client): + """Unauthenticated users are redirected away from the page.""" + with patch("app.views.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + mock_s.factory_reset_on_startup = False + response = client.get("/admin/system-reset", follow_redirects=False) + # Should redirect to login since there's no active session + assert response.status_code in (302, 307) diff --git a/tests/test_upload_rate_limit.py b/tests/test_upload_rate_limit.py new file mode 100644 index 00000000..4e23e6dc --- /dev/null +++ b/tests/test_upload_rate_limit.py @@ -0,0 +1,265 @@ +"""Tests for per-user health-aware upload rate limiting (app/middleware/upload_rate_limit.py).""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from app.middleware.upload_rate_limit import compute_effective_limit + +# --------------------------------------------------------------------------- +# Tests for compute_effective_limit (pure function, no Redis needed) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestComputeEffectiveLimit: + """Tests for the health-aware effective-limit calculation.""" + + def test_normal_conditions_return_base_limit(self): + """Under normal conditions the full base limit should be returned.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=0, cpu_load_ratio=0.0) + assert effective == 20 + assert factor == 1.0 + assert reason == "normal" + + def test_moderate_queue_halves_limit(self): + """Queue depth > 50 should halve the base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=60, cpu_load_ratio=0.0) + assert effective == 10 + assert factor == 0.5 + assert "moderate_queue" in reason + + def test_high_queue_quarters_limit(self): + """Queue depth > 100 should quarter the base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=120, cpu_load_ratio=0.0) + assert effective == 5 + assert factor == 0.25 + assert "high_queue" in reason + + def test_critical_queue_drops_to_ten_percent(self): + """Queue depth > 200 should drop to 10% of base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=250, cpu_load_ratio=0.0) + assert effective == 2 + assert factor == 0.10 + assert "critical_queue" in reason + + def test_moderate_cpu_halves_limit(self): + """CPU load ratio > 1.5 should halve the base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=0, cpu_load_ratio=1.8) + assert effective == 10 + assert factor == 0.5 + assert "moderate_cpu" in reason + + def test_high_cpu_quarters_limit(self): + """CPU load ratio > 2.0 should quarter the base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=0, cpu_load_ratio=2.5) + assert effective == 5 + assert factor == 0.25 + assert "high_cpu" in reason + + def test_critical_cpu_drops_to_ten_percent(self): + """CPU load ratio > 3.0 should drop to 10% of base limit.""" + effective, factor, reason = compute_effective_limit(20, queue_depth=0, cpu_load_ratio=4.0) + assert effective == 2 + assert factor == 0.10 + assert "critical_cpu" in reason + + def test_worst_metric_wins(self): + """The lowest factor from queue and CPU should be applied.""" + # Queue says 0.5, CPU says 0.25 → 0.25 wins + effective, factor, reason = compute_effective_limit(20, queue_depth=60, cpu_load_ratio=2.5) + assert effective == 5 + assert factor == 0.25 + + def test_minimum_effective_limit_is_one(self): + """Even under extreme load the effective limit must be ≥ 1.""" + effective, _factor, _reason = compute_effective_limit(1, queue_depth=999, cpu_load_ratio=10.0) + assert effective >= 1 + + def test_zero_base_limit_returns_zero(self): + """A base limit of 0 (disabled) should clamp to at least 1.""" + effective, _factor, _reason = compute_effective_limit(0, queue_depth=0, cpu_load_ratio=0.0) + # max(1, int(0 * 1.0)) = max(1, 0) = 1 + # A base_limit of 0 means "disabled" and is handled upstream + # (the dependency skips the check entirely), but the pure function + # still clamps to 1 as a safety net. + assert effective == 1 + + +# --------------------------------------------------------------------------- +# Tests for the FastAPI dependency (mocked Redis) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestRequireUploadRateLimit: + """Tests for the require_upload_rate_limit FastAPI dependency.""" + + @pytest.mark.asyncio + async def test_allows_request_when_redis_unavailable(self): + """When Redis is down the dependency should fail open (allow the request).""" + from app.middleware.upload_rate_limit import require_upload_rate_limit + + mock_request = MagicMock() + mock_request.session = {} + mock_request.client = MagicMock() + mock_request.client.host = "127.0.0.1" + + with patch("app.middleware.upload_rate_limit._get_redis", return_value=None): + # Should NOT raise + result = await require_upload_rate_limit(mock_request) + assert result is None + + @pytest.mark.asyncio + async def test_allows_request_under_limit(self): + """A user below the rate limit should be allowed through.""" + from app.middleware.upload_rate_limit import require_upload_rate_limit + + mock_request = MagicMock() + mock_request.session = {"user": {"username": "testuser"}} + mock_request.client = MagicMock() + mock_request.client.host = "10.0.0.1" + + mock_redis = MagicMock() + mock_pipe = MagicMock() + mock_pipe.execute.return_value = [ + 0, # zremrangebyscore result + 5, # zcard — current count (under limit of 20) + [], # zrange oldest + ] + mock_redis.pipeline.return_value = mock_pipe + mock_redis.llen.return_value = 0 # empty queues + + mock_pipe2 = MagicMock() + mock_pipe2.execute.return_value = [True, True] + # The second pipeline call (record upload) + mock_redis.pipeline.side_effect = [mock_pipe, mock_pipe2] + + with ( + patch("app.middleware.upload_rate_limit._get_redis", return_value=mock_redis), + patch("app.middleware.upload_rate_limit.get_current_owner_id", return_value="testuser"), + patch("app.middleware.upload_rate_limit._get_cpu_load_ratio", return_value=0.1), + ): + result = await require_upload_rate_limit(mock_request) + assert result is None + + @pytest.mark.asyncio + async def test_rejects_request_over_limit(self): + """A user at or over the rate limit should receive a 429.""" + from fastapi import HTTPException + + from app.middleware.upload_rate_limit import require_upload_rate_limit + + mock_request = MagicMock() + mock_request.session = {"user": {"username": "spammer"}} + mock_request.client = MagicMock() + mock_request.client.host = "10.0.0.2" + + mock_redis = MagicMock() + mock_pipe = MagicMock() + mock_pipe.execute.return_value = [ + 0, # zremrangebyscore + 20, # zcard — at limit + [("oldest_entry", 1000000.0)], # oldest entry for retry_after + ] + mock_redis.pipeline.return_value = mock_pipe + mock_redis.llen.return_value = 0 + + with ( + patch("app.middleware.upload_rate_limit._get_redis", return_value=mock_redis), + patch("app.middleware.upload_rate_limit.get_current_owner_id", return_value="spammer"), + patch("app.middleware.upload_rate_limit._get_cpu_load_ratio", return_value=0.0), + ): + with pytest.raises(HTTPException) as exc_info: + await require_upload_rate_limit(mock_request) + assert exc_info.value.status_code == 429 + assert "Retry-After" in exc_info.value.headers + + @pytest.mark.asyncio + async def test_health_reduces_effective_limit(self): + """When queues are deep, the effective limit should drop, causing a 429 sooner.""" + from fastapi import HTTPException + + from app.middleware.upload_rate_limit import require_upload_rate_limit + + mock_request = MagicMock() + mock_request.session = {"user": {"username": "normaluser"}} + mock_request.client = MagicMock() + mock_request.client.host = "10.0.0.3" + + mock_redis = MagicMock() + mock_pipe = MagicMock() + # 12 uploads already — under normal limit of 20 but over health-reduced limit + mock_pipe.execute.return_value = [ + 0, # zremrangebyscore + 12, # zcard — 12 uploads in window + [("oldest", 1000000.0)], + ] + mock_redis.pipeline.return_value = mock_pipe + # Simulate deep queue (>100) → effective limit = 25% of 20 = 5 + mock_redis.llen.return_value = 40 # 40 per queue * 3 = 120 total + + with ( + patch("app.middleware.upload_rate_limit._get_redis", return_value=mock_redis), + patch("app.middleware.upload_rate_limit.get_current_owner_id", return_value="normaluser"), + patch("app.middleware.upload_rate_limit._get_cpu_load_ratio", return_value=0.0), + ): + with pytest.raises(HTTPException) as exc_info: + await require_upload_rate_limit(mock_request) + assert exc_info.value.status_code == 429 + + @pytest.mark.asyncio + async def test_falls_back_to_ip_when_no_user(self): + """Unauthenticated requests should use IP-based rate limiting.""" + from app.middleware.upload_rate_limit import require_upload_rate_limit + + mock_request = MagicMock() + mock_request.session = {} + mock_request.client = MagicMock() + mock_request.client.host = "192.168.1.100" + + mock_redis = MagicMock() + mock_pipe = MagicMock() + mock_pipe.execute.return_value = [0, 0, []] + mock_redis.pipeline.return_value = mock_pipe + mock_redis.llen.return_value = 0 + + mock_pipe2 = MagicMock() + mock_pipe2.execute.return_value = [True, True] + mock_redis.pipeline.side_effect = [mock_pipe, mock_pipe2] + + with ( + patch("app.middleware.upload_rate_limit._get_redis", return_value=mock_redis), + patch("app.middleware.upload_rate_limit.get_current_owner_id", return_value=None), + patch("app.middleware.upload_rate_limit._get_cpu_load_ratio", return_value=0.0), + ): + result = await require_upload_rate_limit(mock_request) + assert result is None + + +# --------------------------------------------------------------------------- +# Tests for configuration +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestUploadRateLimitConfig: + """Tests for upload rate limit configuration settings.""" + + def test_settings_exist(self): + """Verify per-user upload rate limit settings are exposed in config.""" + from app.config import settings + + assert hasattr(settings, "upload_rate_limit_per_user") + assert hasattr(settings, "upload_rate_limit_window") + + def test_sensible_defaults(self): + """Default values should be reasonable for a multi-user system.""" + from app.config import settings + + assert settings.upload_rate_limit_per_user >= 10 + assert settings.upload_rate_limit_per_user <= 100 + assert settings.upload_rate_limit_window >= 30 + assert settings.upload_rate_limit_window <= 300 diff --git a/tests/test_upload_to_sharepoint.py b/tests/test_upload_to_sharepoint.py new file mode 100644 index 00000000..1acfa40b --- /dev/null +++ b/tests/test_upload_to_sharepoint.py @@ -0,0 +1,554 @@ +""" +Tests for app/tasks/upload_to_sharepoint.py module. + +Covers get_sharepoint_token, resolve_sharepoint_drive, +create_sharepoint_upload_session, upload_large_file_sharepoint, +and upload_to_sharepoint Celery task. +""" + +from unittest.mock import Mock, patch + +import pytest + + +@pytest.mark.unit +class TestGetSharepointToken: + """Tests for get_sharepoint_token function.""" + + @patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_refresh_token_flow(self, mock_settings, mock_msal): + """Test token acquisition using refresh token.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "client-secret" + mock_settings.sharepoint_refresh_token = "refresh-token" + mock_settings.sharepoint_tenant_id = "common" + + mock_app = Mock() + mock_app.acquire_token_by_refresh_token.return_value = { + "access_token": "new-access-token", + } + mock_msal.return_value = mock_app + + token = get_sharepoint_token() + assert token == "new-access-token" + + @patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_refresh_token_updates_new_token(self, mock_settings, mock_msal): + """Test that a new refresh token updates settings.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "client-secret" + mock_settings.sharepoint_refresh_token = "old-refresh-token" + mock_settings.sharepoint_tenant_id = "common" + + mock_app = Mock() + mock_app.acquire_token_by_refresh_token.return_value = { + "access_token": "access-token", + "refresh_token": "new-refresh-token", + } + mock_msal.return_value = mock_app + + get_sharepoint_token() + assert mock_settings.sharepoint_refresh_token == "new-refresh-token" + + @patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_refresh_token_failure(self, mock_settings, mock_msal): + """Test error handling when refresh token fails.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "client-secret" + mock_settings.sharepoint_refresh_token = "expired-token" + mock_settings.sharepoint_tenant_id = "common" + + mock_app = Mock() + mock_app.acquire_token_by_refresh_token.return_value = { + "error": "invalid_grant", + "error_description": "Token expired", + } + mock_msal.return_value = mock_app + + with pytest.raises(ValueError, match="Failed to get SharePoint access token"): + get_sharepoint_token() + + @patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_client_credentials_flow(self, mock_settings, mock_msal): + """Test token acquisition using client credentials (org accounts).""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "client-secret" + mock_settings.sharepoint_refresh_token = "" + mock_settings.sharepoint_tenant_id = "org-tenant-id" + + mock_app = Mock() + mock_app.acquire_token_for_client.return_value = { + "access_token": "client-cred-token", + } + mock_msal.return_value = mock_app + + token = get_sharepoint_token() + assert token == "client-cred-token" + + @patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_client_credentials_failure(self, mock_settings, mock_msal): + """Test error handling when client credentials flow fails.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "client-secret" + mock_settings.sharepoint_refresh_token = "" + mock_settings.sharepoint_tenant_id = "org-tenant-id" + + mock_app = Mock() + mock_app.acquire_token_for_client.return_value = { + "error": "unauthorized_client", + "error_description": "Not authorized", + } + mock_msal.return_value = mock_app + + with pytest.raises(ValueError, match="Failed to get SharePoint access token"): + get_sharepoint_token() + + @patch("app.tasks.upload_to_sharepoint.settings") + def test_missing_client_id(self, mock_settings): + """Test error when client ID is missing.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "" + mock_settings.sharepoint_client_secret = "secret" + + with pytest.raises(ValueError, match="client ID and client secret"): + get_sharepoint_token() + + @patch("app.tasks.upload_to_sharepoint.settings") + def test_no_refresh_token_common_tenant(self, mock_settings): + """Test error for common tenant without refresh token.""" + from app.tasks.upload_to_sharepoint import get_sharepoint_token + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "secret" + mock_settings.sharepoint_refresh_token = "" + mock_settings.sharepoint_tenant_id = "common" + + with pytest.raises(ValueError, match="either a refresh token or a non-'common' tenant ID"): + get_sharepoint_token() + + +@pytest.mark.unit +class TestResolveSharepointDrive: + """Tests for resolve_sharepoint_drive function.""" + + @patch("app.tasks.upload_to_sharepoint.requests.get") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_successful_resolution(self, mock_settings, mock_get): + """Test successful site and drive resolution.""" + from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive + + mock_settings.http_request_timeout = 30 + + site_resp = Mock() + site_resp.status_code = 200 + site_resp.json.return_value = {"id": "site-id-123"} + + drives_resp = Mock() + drives_resp.status_code = 200 + drives_resp.json.return_value = { + "value": [ + {"id": "drive-1", "name": "Documents"}, + {"id": "drive-2", "name": "Site Assets"}, + ] + } + + mock_get.side_effect = [site_resp, drives_resp] + + site_id, drive_id = resolve_sharepoint_drive( + "access-token", "https://tenant.sharepoint.com/sites/mysite", "Documents" + ) + + assert site_id == "site-id-123" + assert drive_id == "drive-1" + + @patch("app.tasks.upload_to_sharepoint.requests.get") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_library_not_found(self, mock_settings, mock_get): + """Test error when document library is not found.""" + from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive + + mock_settings.http_request_timeout = 30 + + site_resp = Mock() + site_resp.status_code = 200 + site_resp.json.return_value = {"id": "site-id-123"} + + drives_resp = Mock() + drives_resp.status_code = 200 + drives_resp.json.return_value = { + "value": [ + {"id": "drive-1", "name": "Documents"}, + ] + } + + mock_get.side_effect = [site_resp, drives_resp] + + with pytest.raises(RuntimeError, match="not found on site"): + resolve_sharepoint_drive("access-token", "https://tenant.sharepoint.com/sites/mysite", "NonExistentLibrary") + + @patch("app.tasks.upload_to_sharepoint.requests.get") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_site_resolution_failure(self, mock_settings, mock_get): + """Test error when site resolution fails.""" + from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive + + mock_settings.http_request_timeout = 30 + + site_resp = Mock() + site_resp.status_code = 404 + site_resp.text = "Site not found" + + mock_get.return_value = site_resp + + with pytest.raises(RuntimeError, match="Failed to resolve SharePoint site"): + resolve_sharepoint_drive("access-token", "https://tenant.sharepoint.com/sites/nonexistent", "Documents") + + def test_invalid_site_url(self): + """Test error with invalid site URL.""" + from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive + + with pytest.raises(ValueError, match="Invalid SharePoint site URL"): + resolve_sharepoint_drive("access-token", "not-a-url", "Documents") + + @patch("app.tasks.upload_to_sharepoint.requests.get") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_case_insensitive_library_match(self, mock_settings, mock_get): + """Test that library name matching is case-insensitive.""" + from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive + + mock_settings.http_request_timeout = 30 + + site_resp = Mock() + site_resp.status_code = 200 + site_resp.json.return_value = {"id": "site-id"} + + drives_resp = Mock() + drives_resp.status_code = 200 + drives_resp.json.return_value = { + "value": [ + {"id": "drive-1", "name": "Shared Documents"}, + ] + } + + mock_get.side_effect = [site_resp, drives_resp] + + site_id, drive_id = resolve_sharepoint_drive( + "access-token", "https://tenant.sharepoint.com/sites/mysite", "shared documents" + ) + + assert drive_id == "drive-1" + + +@pytest.mark.unit +class TestCreateSharepointUploadSession: + """Tests for create_sharepoint_upload_session function.""" + + @patch("app.tasks.upload_to_sharepoint.requests.post") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_successful_session_creation(self, mock_settings, mock_post): + """Test successful upload session creation.""" + from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session + + mock_settings.http_request_timeout = 30 + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"uploadUrl": "https://upload.url/session123"} + mock_post.return_value = mock_response + + url = create_sharepoint_upload_session("test.pdf", "Uploads", "drive-id", "site-id", "access-token") + + assert url == "https://upload.url/session123" + + @patch("app.tasks.upload_to_sharepoint.requests.post") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_session_without_folder(self, mock_settings, mock_post): + """Test upload session creation without folder path.""" + from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session + + mock_settings.http_request_timeout = 30 + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"uploadUrl": "https://upload.url/session456"} + mock_post.return_value = mock_response + + url = create_sharepoint_upload_session("test.pdf", None, "drive-id", "site-id", "access-token") + + assert url == "https://upload.url/session456" + + @patch("app.tasks.upload_to_sharepoint.requests.post") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_session_creation_failure(self, mock_settings, mock_post): + """Test error handling when session creation fails.""" + from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session + + mock_settings.http_request_timeout = 30 + + mock_response = Mock() + mock_response.status_code = 403 + mock_response.text = "Access denied" + mock_post.return_value = mock_response + + with pytest.raises(RuntimeError, match="Failed to create SharePoint upload session"): + create_sharepoint_upload_session("test.pdf", "Uploads", "drive-id", "site-id", "access-token") + + @patch("app.tasks.upload_to_sharepoint.requests.post") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_url_encoding_special_characters(self, mock_settings, mock_post): + """Test that special characters in folder path are URL-encoded.""" + from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session + + mock_settings.http_request_timeout = 30 + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"uploadUrl": "https://upload.url/session"} + mock_post.return_value = mock_response + + create_sharepoint_upload_session("file with spaces.pdf", "My Documents/Uploads", "drive-id", "site-id", "token") + + call_url = mock_post.call_args[0][0] + assert "My%20Documents" in call_url + assert "file%20with%20spaces.pdf" in call_url + + +@pytest.mark.unit +class TestUploadLargeFileSharepoint: + """Tests for upload_large_file_sharepoint function.""" + + @patch("app.tasks.upload_to_sharepoint.requests.put") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_small_single_chunk_upload(self, mock_settings, mock_put, tmp_path): + """Test uploading a file that fits in a single chunk.""" + from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint + + mock_settings.http_request_timeout = 30 + + test_file = tmp_path / "small.pdf" + test_file.write_bytes(b"small content") + + mock_response = Mock() + mock_response.status_code = 201 + mock_response.json.return_value = {"id": "file123", "name": "small.pdf"} + mock_put.return_value = mock_response + + result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session") + + assert result["id"] == "file123" + + @patch("app.tasks.upload_to_sharepoint.time.sleep") + @patch("app.tasks.upload_to_sharepoint.requests.put") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_chunk_upload_retry_on_failure(self, mock_settings, mock_put, mock_sleep, tmp_path): + """Test retry logic when a chunk upload fails.""" + from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint + + mock_settings.http_request_timeout = 30 + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_fail = Mock() + mock_fail.status_code = 500 + + mock_success = Mock() + mock_success.status_code = 201 + mock_success.json.return_value = {"id": "file123"} + + mock_put.side_effect = [mock_fail, mock_success] + + result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session") + + assert result["id"] == "file123" + + @patch("app.tasks.upload_to_sharepoint.time.sleep") + @patch("app.tasks.upload_to_sharepoint.requests.put") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_chunk_upload_retry_on_exception(self, mock_settings, mock_put, mock_sleep, tmp_path): + """Test retry logic when an exception occurs during upload.""" + from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint + + mock_settings.http_request_timeout = 30 + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_success = Mock() + mock_success.status_code = 201 + mock_success.json.return_value = {"id": "file123"} + + mock_put.side_effect = [Exception("Network error"), mock_success] + + result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session") + + assert result["id"] == "file123" + + @patch("app.tasks.upload_to_sharepoint.time.sleep") + @patch("app.tasks.upload_to_sharepoint.requests.put") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_all_retries_exhausted(self, mock_settings, mock_put, mock_sleep, tmp_path): + """Test that exhausting all retries raises an exception.""" + from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint + + mock_settings.http_request_timeout = 30 + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_fail = Mock() + mock_fail.status_code = 500 + mock_fail.text = "Server Error" + mock_put.return_value = mock_fail + + with pytest.raises(RuntimeError, match="Failed to upload chunk"): + upload_large_file_sharepoint(str(test_file), "https://upload.url/session") + + +@pytest.mark.unit +class TestUploadToSharepoint: + """Tests for upload_to_sharepoint Celery task.""" + + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + def test_file_not_found(self, mock_log): + """Test that missing file raises FileNotFoundError.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + with pytest.raises(FileNotFoundError): + upload_to_sharepoint.__wrapped__("/nonexistent/file.pdf", file_id=1) + + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_missing_client_id(self, mock_settings, mock_log, tmp_path): + """Test error when SharePoint client ID is not configured.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + mock_settings.sharepoint_client_id = "" + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + with pytest.raises(ValueError, match="client ID is not configured"): + upload_to_sharepoint.__wrapped__(str(test_file), file_id=1) + + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_missing_site_url(self, mock_settings, mock_log, tmp_path): + """Test error when SharePoint site URL is not configured.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_site_url = "" + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + with pytest.raises(ValueError, match="site URL is not configured"): + upload_to_sharepoint.__wrapped__(str(test_file), file_id=1) + + @patch("app.tasks.upload_to_sharepoint.upload_large_file_sharepoint") + @patch("app.tasks.upload_to_sharepoint.create_sharepoint_upload_session") + @patch("app.tasks.upload_to_sharepoint.resolve_sharepoint_drive") + @patch("app.tasks.upload_to_sharepoint.get_sharepoint_token") + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_successful_upload( + self, mock_settings, mock_log, mock_token, mock_resolve, mock_session, mock_upload, tmp_path + ): + """Test successful SharePoint upload.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "secret" + mock_settings.sharepoint_refresh_token = "token" + mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite" + mock_settings.sharepoint_document_library = "Documents" + mock_settings.sharepoint_folder_path = "Uploads" + mock_settings.sharepoint_tenant_id = "common" + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_token.return_value = "access-token" + mock_resolve.return_value = ("site-id", "drive-id") + mock_session.return_value = "https://upload.url/session" + mock_upload.return_value = {"webUrl": "https://tenant.sharepoint.com/sites/mysite/test.pdf"} + + result = upload_to_sharepoint.apply(args=[str(test_file)], kwargs={"file_id": 1}).get() + + assert result["status"] == "Completed" + assert "Uploads" in result["sharepoint_path"] + assert result["web_url"] == "https://tenant.sharepoint.com/sites/mysite/test.pdf" + + @patch("app.tasks.upload_to_sharepoint.get_sharepoint_token") + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_upload_exception_handling(self, mock_settings, mock_log, mock_token, tmp_path): + """Test that upload errors are properly handled.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "secret" + mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite" + mock_settings.sharepoint_folder_path = "Uploads" + mock_settings.sharepoint_document_library = "Documents" + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_token.side_effect = ValueError("Token error") + + with pytest.raises(RuntimeError, match="Failed to upload"): + upload_to_sharepoint.apply(args=[str(test_file)], kwargs={"file_id": 1}).get() + + @patch("app.tasks.upload_to_sharepoint.upload_large_file_sharepoint") + @patch("app.tasks.upload_to_sharepoint.create_sharepoint_upload_session") + @patch("app.tasks.upload_to_sharepoint.resolve_sharepoint_drive") + @patch("app.tasks.upload_to_sharepoint.get_sharepoint_token") + @patch("app.tasks.upload_to_sharepoint.log_task_progress") + @patch("app.tasks.upload_to_sharepoint.settings") + def test_folder_override( + self, mock_settings, mock_log, mock_token, mock_resolve, mock_session, mock_upload, tmp_path + ): + """Test that folder_override is used instead of settings.""" + from app.tasks.upload_to_sharepoint import upload_to_sharepoint + + mock_settings.sharepoint_client_id = "client-id" + mock_settings.sharepoint_client_secret = "secret" + mock_settings.sharepoint_refresh_token = "token" + mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite" + mock_settings.sharepoint_document_library = "Documents" + mock_settings.sharepoint_folder_path = "DefaultFolder" + mock_settings.sharepoint_tenant_id = "common" + + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"test content") + + mock_token.return_value = "access-token" + mock_resolve.return_value = ("site-id", "drive-id") + mock_session.return_value = "https://upload.url/session" + mock_upload.return_value = {"webUrl": "https://example.com/test.pdf"} + + result = upload_to_sharepoint.apply( + args=[str(test_file)], kwargs={"file_id": 1, "folder_override": "CustomFolder"} + ).get() + + # Verify the session was created with the override folder + mock_session.assert_called_once_with("test.pdf", "CustomFolder", "drive-id", "site-id", "access-token") + assert result["status"] == "Completed" diff --git a/tests/test_views_dropbox.py b/tests/test_views_dropbox.py index d42b9304..79dad26c 100644 --- a/tests/test_views_dropbox.py +++ b/tests/test_views_dropbox.py @@ -144,3 +144,37 @@ class TestDropboxViews: assert response.status_code == 200 assert b"/Documents/Uploads" in response.content assert b"Back to Integrations" in response.content + + +@pytest.mark.integration +class TestDropboxCallbackUrl: + """Tests that the callback_url is correctly passed to templates.""" + + def test_setup_page_includes_callback_url(self, client): + """Setup page should include the callback_url variable in its response.""" + response = client.get("/dropbox-setup") + assert response.status_code == 200 + # callback_url is embedded in the JS as the dropboxCallbackUrl constant + assert b"dropboxCallbackUrl" in response.content + + def test_callback_page_includes_callback_url(self, client): + """Callback page should embed the server-side callback URL.""" + response = client.get("/dropbox-callback?code=testcode") + assert response.status_code == 200 + # callback_url is used as the redirectUri + assert b"redirectUri" in response.content + + def test_setup_page_uses_public_base_url_when_set(self, client): + """When PUBLIC_BASE_URL is configured, it should appear in the redirect URI hint.""" + with patch("app.views.dropbox.settings") as mock_settings: + mock_settings.public_base_url = "https://configured.example.com" + mock_settings.dropbox_app_key = "" + mock_settings.dropbox_app_secret = "" + mock_settings.dropbox_refresh_token = "" + mock_settings.dropbox_folder = "" + mock_settings.dropbox_allow_global_credentials_for_integrations = False + response = client.get("/dropbox-setup") + assert response.status_code == 200 + # The configured public_base_url hostname must appear in the page (redirect URI display) + page_text = response.text + assert "configured.example.com/dropbox-callback" in page_text