diff --git a/.env.demo b/.env.demo index f92b6d8e..6920b998 100644 --- a/.env.demo +++ b/.env.demo @@ -3,10 +3,21 @@ WORKDIR=/workdir DATABASE_URL=sqlite:///./app/database.db REDIS_URL=redis://redis:6379/0 EXTERNAL_HOSTNAME=docuelevate.example.com +# PUBLIC_BASE_URL=https://docuelevate.example.com # Full URL with scheme; required when X-Forwarded-Proto is not forwarded by your proxy GOTENBERG_URL=http://gotenberg:3000 ALLOW_FILE_DELETE=true # Allow deletion of file records COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2) +# **Database Connection Pool** (PostgreSQL / MySQL only; ignored for SQLite) +# DB_POOL_SIZE=10 # Persistent connections per worker (default: 10) +# DB_MAX_OVERFLOW=20 # Extra connections under burst (default: 20) +# DB_POOL_TIMEOUT=30 # Seconds to wait for a pool connection (default: 30) +# DB_POOL_RECYCLE=1800 # Recycle connections after N seconds (default: 1800) + +# **Per-User Upload Rate Limiting** (health-aware, Redis-backed) +# UPLOAD_RATE_LIMIT_PER_USER=20 # Max uploads per user per window (default: 20) +# UPLOAD_RATE_LIMIT_WINDOW=60 # Sliding window in seconds (default: 60) + # **System Reset / Factory Reset** # FACTORY_RESET_ON_STARTUP=false # Wipe all user data on every startup (demo/testing only) # ENABLE_FACTORY_RESET=false # Show the System Reset page in admin UI diff --git a/BUILD_DATE b/BUILD_DATE index 13dc73cd..6f198e0a 100644 --- a/BUILD_DATE +++ b/BUILD_DATE @@ -1 +1 @@ -2026-03-17T13:12:02Z +2026-03-20T12:55:52Z diff --git a/CHANGELOG.md b/CHANGELOG.md index 53af22b1..b85c7f44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,223 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## v0.161.0 (2026-03-20) + +### Documentation + +- Update scaling, health probe, and beat scheduler documentation + ([`4d019d5`](https://github.com/christianlouis/DocuElevate/commit/4d019d53d9ad161a68639d6ba2109ba3ac77df34)) + +### Features + +- **scaling**: Enable horizontal scaling for API and worker pods + ([`f75b125`](https://github.com/christianlouis/DocuElevate/commit/f75b12599291050f97aa452d980f17547ddd7bd6)) + + +## v0.160.3 (2026-03-20) + +### Bug Fixes + +- **mobile**: Wire i18n reactivity, translate all screens, sync language with server + ([`3b5ca04`](https://github.com/christianlouis/DocuElevate/commit/3b5ca04ebc8b6dd20f877bc4797e364e0997d840)) + +### Chores + +- **mobile**: Upgrade ESLint to v9 with flat config and fix expo-localization version + ([`0e6a4c5`](https://github.com/christianlouis/DocuElevate/commit/0e6a4c5084b3704653965f0091c36b9c78b8ad60)) + + +## v0.160.2 (2026-03-20) + +### Bug Fixes + +- **dropbox**: Fix Invalid redirect_uri error by adding PUBLIC_BASE_URL config and URL-encoding + ([`5e3e2b1`](https://github.com/christianlouis/DocuElevate/commit/5e3e2b19997d3af6570fbaa1e75a49cbfe6cf78d)) + + +## v0.160.1 (2026-03-20) + +### Bug Fixes + +- **mobile**: Update expo-localization version from ~16.0.6 to ~16.1.0 + ([`78c3717`](https://github.com/christianlouis/DocuElevate/commit/78c3717661923b43f1762fa7728c75e803938fb7)) + + +## v0.160.0 (2026-03-20) + +### Bug Fixes + +- **mobile**: Address code review feedback - error handling, filename collision, hash display + ([`6541529`](https://github.com/christianlouis/DocuElevate/commit/65415292507aa37408428400aef7e87e006abfd0)) + +### Features + +- **mobile**: Add pre-login legal pages, multi-image selection, file detail view, search, i18n, HEIC + support + ([`67c17e7`](https://github.com/christianlouis/DocuElevate/commit/67c17e7baa8edf76be394d0aff42c2adeae351e1)) + + +## v0.159.0 (2026-03-19) + +### Code Style + +- Apply ruff auto-fix + ([`910fb29`](https://github.com/christianlouis/DocuElevate/commit/910fb297ba1122b751250b323a330e15e276daf6)) + +### Features + +- **integrations**: Add Dropbox connection test and global-credential sharing + ([`d1f9819`](https://github.com/christianlouis/DocuElevate/commit/d1f9819f4e12320bb901b76366fa8d4a8cd23a66)) + + +## v0.158.4 (2026-03-19) + +### Bug Fixes + +- **ui**: Show proper error when signup username has invalid characters + ([`689c616`](https://github.com/christianlouis/DocuElevate/commit/689c616e448bf3edfcb91b347ae9f6fac43b1983)) + + +## v0.158.3 (2026-03-19) + +### Bug Fixes + +- **upload**: Reject exact duplicates at upload time and prevent duplicate mobile uploads + ([`d5c18cc`](https://github.com/christianlouis/DocuElevate/commit/d5c18ccf07efa28532724a78e63cc6bd7eab9515)) + +### Refactoring + +- **mobile**: Extract normalizeFileUri to shared utility module + ([`ec88221`](https://github.com/christianlouis/DocuElevate/commit/ec882214e29d22da9d8025ea857ba754fbddae37)) + + +## v0.158.2 (2026-03-19) + +### Bug Fixes + +- **mobile**: Add user feedback when server URL is unavailable + ([`1366317`](https://github.com/christianlouis/DocuElevate/commit/136631762bc81399aecd5766663da7d923cba985)) + +- **mobile**: Apple App Store compliance fixes + ([`5c15a23`](https://github.com/christianlouis/DocuElevate/commit/5c15a2395a0dcfa5fe5cf1a34a98ae743d29eabc)) + +- **mobile**: Fix file sharing deep-link conflicts and add MIME type inference + ([`1559686`](https://github.com/christianlouis/DocuElevate/commit/1559686f903808e2ab4547f21575460aaeca839e)) + +- **mobile**: Fix shared file upload hanging by copying to cache + ([`f549505`](https://github.com/christianlouis/DocuElevate/commit/f549505bfd625100e77a170cfcae372913680835)) + +### Documentation + +- Add Apple App Store Compliance audit report + ([`1572f32`](https://github.com/christianlouis/DocuElevate/commit/1572f322d72583c45c88e55ea51b2456e548de4c)) + +### Refactoring + +- **mobile**: Extract shared MIME type utility and improve error handling + ([`cfe83d7`](https://github.com/christianlouis/DocuElevate/commit/cfe83d7efa7f93ca51bfcac417cfafc650bbc36b)) + + +## v0.158.1 (2026-03-19) + +### Bug Fixes + +- **mobile**: Add shared file to ShareContext directly in +not-found.tsx + ([`71a7a57`](https://github.com/christianlouis/DocuElevate/commit/71a7a57adc1bb6de88c060dba563afa433283e38)) + + +## v0.158.0 (2026-03-19) + +### Bug Fixes + +- Address code review feedback (assertion, exc_info logging) + ([`faa68ad`](https://github.com/christianlouis/DocuElevate/commit/faa68adaa143774a8757fda4fd0248cc3aef551e)) + +- **config**: Add SETTING_METADATA for db pool and upload rate limit settings + ([`1d7286c`](https://github.com/christianlouis/DocuElevate/commit/1d7286c4c68de903951d9819bcb58ae1300dab7b)) + +- **config**: Remove duplicate dictionary keys and class fields from merge + ([`d34b8bc`](https://github.com/christianlouis/DocuElevate/commit/d34b8bceb9f96719bf5c922156d5487a4c6426b4)) + +- **db**: Use NullPool for SQLite and expose pool tuning settings + ([`56bf665`](https://github.com/christianlouis/DocuElevate/commit/56bf66539757cf1d98fe251ad64fb75b481462ef)) + +- **tests**: Add docstring to rate limiter no-op override + ([`c9f9001`](https://github.com/christianlouis/DocuElevate/commit/c9f900124407baa230b38f6e1d0f5ee78e0bedcc)) + +- **tests**: Disable upload rate limiter in test client fixture + ([`6bf121f`](https://github.com/christianlouis/DocuElevate/commit/6bf121f02f2c32f7b73435f42d30b9681710089e)) + +### Features + +- **api**: Add per-user health-aware upload rate limiting + ([`571cc81`](https://github.com/christianlouis/DocuElevate/commit/571cc817893a1f4cbd7cfbee7f1c46ca1a4f14d8)) + + +## v0.157.2 (2026-03-19) + +### Bug Fixes + +- **ui**: Improve devices page table layout to prevent horizontal scrolling + ([`b12e891`](https://github.com/christianlouis/DocuElevate/commit/b12e8916824273eefb85f0bb498d4991e8c66d5e)) + + +## v0.157.1 (2026-03-19) + +### Bug Fixes + +- **mobile**: Resolve iOS "unmatched route docuelevate://" error in Open In share flow + ([`f2b7db8`](https://github.com/christianlouis/DocuElevate/commit/f2b7db88ba86e894a9bf2e4e7d6a54cd7f423194)) + + +## v0.157.0 (2026-03-19) + +### Features + +- **api**: Allow disabled tokens/devices to be deleted & reactivated; add token lifetime + ([`e4749b4`](https://github.com/christianlouis/DocuElevate/commit/e4749b4e7cdd78ab95627736d3fcd84fecc55e53)) + + +## v0.156.3 (2026-03-18) + +### Bug Fixes + +- **auth**: Exempt /api/qr-auth/claim from CSRF to fix mobile QR login + ([`a4aaebf`](https://github.com/christianlouis/DocuElevate/commit/a4aaebfe6649ef051790b6af24e0739bdd9ecfed)) + +### Documentation + +- **changelog**: Update changelog [skip ci] + ([`d8d2016`](https://github.com/christianlouis/DocuElevate/commit/d8d2016f8597f775cdf3848e7baeaae98cb998b0)) + + +## Unreleased + + +## v0.156.2 (2026-03-18) + +### Bug Fixes + +- **qr-login**: Render QR code server-side using segno instead of CDN JS library + ([`a8eb650`](https://github.com/christianlouis/DocuElevate/commit/a8eb6504ac100c14b10f57511fde8f1202deeffe)) + +### Chores + +- Initial plan for server-side QR code rendering + ([`6727253`](https://github.com/christianlouis/DocuElevate/commit/6727253958a6ae8436fc1184736ec43eef2ab820)) + +- Remove accidentally committed =1.6.0 file + ([`ba8c88b`](https://github.com/christianlouis/DocuElevate/commit/ba8c88bc17d3245cba6079ac5ba3be5fddf36366)) + + +## v0.156.1 (2026-03-18) + +### Bug Fixes + +- Add missing SETTING_METADATA entries for db pool and upload rate limit settings + ([`dc0a19b`](https://github.com/christianlouis/DocuElevate/commit/dc0a19bd118d3503ad50608f55fb0bc4ce104948)) + + ## v0.156.0 (2026-03-17) ### Bug Fixes diff --git a/GIT_SHA b/GIT_SHA index 0872f7b3..072810ee 100644 --- a/GIT_SHA +++ b/GIT_SHA @@ -1 +1 @@ -a3c657b +91eecd9 diff --git a/RUNTIME_INFO b/RUNTIME_INFO index a46c1eb1..2c5982a3 100644 --- a/RUNTIME_INFO +++ b/RUNTIME_INFO @@ -1,10 +1,10 @@ DocuElevate Build Information ============================== -Version: 0.156.0 -Build Date: 2026-03-17T13:12:02Z -Git Commit: a3c657b947d774255ede33bcf6143ab60c69d1ac -Git Short SHA: a3c657b +Version: 0.161.0 +Build Date: 2026-03-20T12:55:52Z +Git Commit: 91eecd93963e31b06c0dcf76aaf3fb1cfd242d89 +Git Short SHA: 91eecd9 Git Branch: main -Commit Date: 2026-03-17T14:11:41+01:00 -Build Timestamp: 2026-03-17T13:12:02Z +Commit Date: 2026-03-20T13:55:31+01:00 +Build Timestamp: 2026-03-20T12:55:52Z ============================== diff --git a/VERSION b/VERSION index b97a9dda..94a94ed3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.156.0 +0.161.0 diff --git a/app/api/api_tokens.py b/app/api/api_tokens.py index 1beef61b..62074b8a 100644 --- a/app/api/api_tokens.py +++ b/app/api/api_tokens.py @@ -13,7 +13,7 @@ plaintext is returned exactly once at creation time. import hashlib import logging import secrets -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Annotated, Any from fastapi import APIRouter, Depends, HTTPException, Request, status @@ -105,6 +105,7 @@ def _token_to_dict(t: ApiToken) -> dict[str, Any]: "last_used_ip": t.last_used_ip, "created_at": t.created_at, "revoked_at": t.revoked_at, + "expires_at": t.expires_at, } @@ -117,6 +118,12 @@ class TokenCreate(BaseModel): """Schema for creating a new API token.""" name: str = Field(..., min_length=1, max_length=255, description="Human-readable label for the token") + expires_in_days: int | None = Field( + default=None, + ge=1, + le=3650, # Maximum 10 years; keeps tokens from being effectively permanent while allowing long-lived CI/CD tokens. + description="Optional lifetime in days. If omitted the token never expires.", + ) class TokenResponse(BaseModel): @@ -130,6 +137,7 @@ class TokenResponse(BaseModel): last_used_ip: str | None created_at: datetime | None revoked_at: datetime | None + expires_at: datetime | None model_config = {"from_attributes": True} @@ -160,11 +168,16 @@ async def create_token( token_hash_value = hash_token(plaintext) prefix = plaintext[:12] # "de_" prefix + 9 random chars = 12 chars total + expires_at = None + if body.expires_in_days is not None: + expires_at = datetime.now(timezone.utc) + timedelta(days=body.expires_in_days) + db_token = ApiToken( owner_id=owner_id, name=body.name, token_hash=token_hash_value, token_prefix=prefix, + expires_at=expires_at, ) try: db.add(db_token) @@ -185,6 +198,7 @@ async def create_token( "last_used_ip": db_token.last_used_ip, "created_at": db_token.created_at, "revoked_at": db_token.revoked_at, + "expires_at": db_token.expires_at, "token": plaintext, } @@ -235,30 +249,73 @@ async def list_mobile_tokens( @router.delete("/{token_id}", status_code=status.HTTP_200_OK) -async def revoke_token( +async def revoke_or_delete_token( token_id: int, owner_id: CurrentOwner, db: DbSession, ) -> dict[str, str]: - """Revoke (soft-delete) an API token. + """Revoke or permanently delete an API token. - The token row is kept for audit purposes but marked inactive with a - ``revoked_at`` timestamp. + * **Active token** – soft-revoked: the row is kept for audit purposes + but marked inactive with a ``revoked_at`` timestamp. + * **Already-revoked token** – hard-deleted: the row is permanently + removed from the database. """ db_token = db.query(ApiToken).filter(ApiToken.id == token_id, ApiToken.owner_id == owner_id).first() if not db_token: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found") - if not db_token.is_active: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Token is already revoked") + if db_token.is_active: + # Soft-revoke the active token. + try: + db_token.is_active = False + db_token.revoked_at = datetime.now(timezone.utc) + db.commit() + except Exception: + db.rollback() + raise + logger.info("API token revoked: id=%s owner=%s", token_id, owner_id) + return {"detail": "Token revoked"} + # Hard-delete an already-revoked token. try: - db_token.is_active = False - db_token.revoked_at = datetime.now(timezone.utc) + db.delete(db_token) db.commit() except Exception: db.rollback() raise + logger.info("API token permanently deleted: id=%s owner=%s", token_id, owner_id) + return {"detail": "Token deleted"} - logger.info("API token revoked: id=%s owner=%s", token_id, owner_id) - return {"detail": "Token revoked"} + +@router.post("/{token_id}/reactivate", status_code=status.HTTP_200_OK, response_model=TokenResponse) +async def reactivate_token( + token_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Reactivate a previously revoked API token. + + Clears the ``revoked_at`` timestamp and sets ``is_active`` back to + ``True``. The token can be used for authentication again immediately. + If the token had an ``expires_at`` in the past the caller should + consider re-creating a new token instead. + """ + db_token = db.query(ApiToken).filter(ApiToken.id == token_id, ApiToken.owner_id == owner_id).first() + if not db_token: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found") + + if db_token.is_active: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Token is already active") + + try: + db_token.is_active = True + db_token.revoked_at = None + db.commit() + db.refresh(db_token) + except Exception: + db.rollback() + raise + + logger.info("API token reactivated: id=%s owner=%s", token_id, owner_id) + return _token_to_dict(db_token) diff --git a/app/api/diagnostic.py b/app/api/diagnostic.py index a329da46..d21a3d39 100644 --- a/app/api/diagnostic.py +++ b/app/api/diagnostic.py @@ -21,6 +21,67 @@ _DEFAULT_REDIS_URL = "redis://localhost:6379/0" router = APIRouter() +# --------------------------------------------------------------------------- +# Unauthenticated probe endpoints for Kubernetes liveness / readiness checks. +# These intentionally skip authentication so that kubelet can reach them +# without credentials. They live under /diagnostic/healthz/* so that the +# existing authenticated /diagnostic/health endpoint is unaffected. +# --------------------------------------------------------------------------- + + +@router.get("/diagnostic/healthz/live") +async def liveness_probe() -> JSONResponse: + """Lightweight liveness probe for Kubernetes. + + Returns **200 OK** as long as the process is running. Kubernetes uses + this to decide whether to *restart* the container — it should therefore + be as cheap as possible and **never** check external dependencies. + + **Authentication:** None (designed for kubelet probes). + """ + return JSONResponse(content={"status": "ok"}, status_code=200) + + +@router.get("/diagnostic/healthz/ready") +async def readiness_probe() -> JSONResponse: + """Readiness probe for Kubernetes. + + Verifies that the application can serve traffic by checking the database + and Redis. Kubernetes uses this to decide whether to *route traffic* to + the pod. + + Returns **200 OK** when all critical subsystems are reachable, or + **503 Service Unavailable** when the database is down. + + **Authentication:** None (designed for kubelet probes). + """ + checks: dict[str, dict[str, str]] = {} + db_ok = False + + # ── Database check ───────────────────────────────────────────────── + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + checks["database"] = {"status": "ok"} + db_ok = True + except Exception as exc: + logger.warning("Readiness probe: database check failed: %s", exc) + checks["database"] = {"status": "error", "detail": str(exc)} + + # ── Redis check ──────────────────────────────────────────────────── + try: + redis_url = settings.redis_url or _DEFAULT_REDIS_URL + r = redis_lib.from_url(redis_url, socket_connect_timeout=2, socket_timeout=2) + r.ping() + checks["redis"] = {"status": "ok"} + except Exception as exc: + logger.warning("Readiness probe: Redis check failed: %s", exc) + checks["redis"] = {"status": "error", "detail": str(exc)} + + http_status = 503 if not db_ok else 200 + overall = "ready" if db_ok else "not_ready" + return JSONResponse(content={"status": overall, "checks": checks}, status_code=http_status) + @router.get("/diagnostic/health") @require_login diff --git a/app/api/dropbox.py b/app/api/dropbox.py index da52c758..73cddec4 100644 --- a/app/api/dropbox.py +++ b/app/api/dropbox.py @@ -5,6 +5,7 @@ Dropbox API endpoints import logging import os from typing import Annotated, Optional +from urllib.parse import quote import httpx from fastapi import APIRouter, Depends, Form, HTTPException, Request, status @@ -23,6 +24,93 @@ logger = logging.getLogger(__name__) router = APIRouter() +def _build_dropbox_redirect_uri(request: Request) -> str: + """Build the Dropbox OAuth callback redirect URI. + + Uses ``PUBLIC_BASE_URL`` when configured (recommended for deployments behind + a reverse proxy that doesn't forward ``X-Forwarded-Proto``). Falls back to + deriving the URI from the incoming request's scheme and host headers. + """ + if settings.public_base_url: + return settings.public_base_url.rstrip("/") + "/dropbox-callback" + return f"{request.url.scheme}://{request.url.netloc}/dropbox-callback" + + +@router.get("/dropbox/global-authorize-url") +@require_login +async def dropbox_global_authorize_url(request: Request): + """Return the Dropbox OAuth authorization URL using the global app credentials. + + This endpoint is used when ``DROPBOX_ALLOW_GLOBAL_CREDENTIALS_FOR_INTEGRATIONS`` + is enabled so that users can authorize their personal Dropbox integration without + needing to supply their own app key/secret. Only the public ``app_key`` is + embedded in the URL; the ``app_secret`` is never sent to the browser. + """ + if not settings.dropbox_allow_global_credentials_for_integrations: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Global credentials for integrations are not enabled", + ) + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Global Dropbox credentials are not configured", + ) + redirect_uri = _build_dropbox_redirect_uri(request) + authorize_url = ( + "https://www.dropbox.com/oauth2/authorize" + f"?client_id={settings.dropbox_app_key}" + "&response_type=code" + "&token_access_type=offline" + f"&redirect_uri={quote(redirect_uri, safe='')}" + ) + return {"authorize_url": authorize_url} + + +@router.post("/dropbox/exchange-token-global") +@require_login +async def exchange_dropbox_token_global( + request: Request, + code: Annotated[str, Form(...)], + redirect_uri: Annotated[str, Form(...)], +): + """Exchange an authorization code using the global Dropbox app credentials. + + Used when ``DROPBOX_ALLOW_GLOBAL_CREDENTIALS_FOR_INTEGRATIONS`` is enabled so + that the ``app_secret`` is never exposed to the browser. Only the OAuth code + and redirect URI need to be supplied by the client. + """ + if not settings.dropbox_allow_global_credentials_for_integrations: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Global credentials for integrations are not enabled", + ) + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Global Dropbox credentials are not configured", + ) + + token_url = "https://api.dropboxapi.com/oauth2/token" + payload = { + "client_id": settings.dropbox_app_key, + "client_secret": settings.dropbox_app_secret, + "code": code, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + } + + token_data = exchange_oauth_token(provider_name="Dropbox", token_url=token_url, payload=payload) + + return { + "refresh_token": token_data["refresh_token"], + "access_token": token_data["access_token"], + "expires_in": token_data.get("expires_in", 14400), + # Return the public app_key so the callback can store it in the integration + "app_key": settings.dropbox_app_key, + } + + @router.post("/dropbox/exchange-token") @require_login async def exchange_dropbox_token( diff --git a/app/api/files.py b/app/api/files.py index 64aa62b0..8eae6e55 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -20,6 +20,7 @@ from sqlalchemy.orm import Session from app.auth import require_login from app.config import settings from app.database import get_db +from app.middleware.upload_rate_limit import require_upload_rate_limit from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.process_document import process_document @@ -346,7 +347,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession): try: # Find all file records - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -385,7 +388,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): """ try: # Find all file records - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -457,7 +462,9 @@ def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: Db Useful for re-running OCR on files with poor text quality or missing OCR text. """ try: - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -537,7 +544,9 @@ def bulk_download_files(request: Request, file_ids: List[int], db: DbSession): Files not found on disk are silently skipped. """ try: - file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)) + query = apply_owner_filter(query, request) + file_records = query.all() if not file_records: raise HTTPException(status_code=404, detail="No files found with the provided IDs") @@ -619,7 +628,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession): """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -675,7 +686,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession): """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -938,7 +951,9 @@ def retry_subtask( """ try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1080,7 +1095,9 @@ def get_file_preview( try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1160,7 +1177,9 @@ def download_file( try: # Find the file record - file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + query = db.query(FileRecord).filter(FileRecord.id == file_id) + query = apply_owner_filter(query, request) + file_record = query.first() if not file_record: raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") @@ -1249,7 +1268,12 @@ async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None: - """Check for an exact duplicate of the uploaded file and return a warning if found.""" + """Check for an exact duplicate of the uploaded file. + + Returns a dict with duplicate info when the file's SHA-256 hash matches an + already-processed document, or ``None`` when no duplicate is found (or + deduplication is disabled). + """ if not settings.enable_deduplication: return None @@ -1268,8 +1292,8 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s "original_file_id": existing.id, "original_filename": existing.original_filename, "message": ( - "This file appears to be an exact duplicate of an already-processed document. " - "It will still be queued but will be flagged as a duplicate." + "This file is an exact duplicate of an already-processed document. " + "It has not been queued for processing again." ), } except Exception as e: @@ -1280,7 +1304,12 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s @router.post("/ui-upload") @require_login -async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)): +async def ui_upload( + request: Request, + db: DbSession, + file: UploadFile = File(...), + _rate_ok: None = Depends(require_upload_rate_limit), +): """Endpoint to accept a user-uploaded file and enqueue it for processing.""" workdir = settings.workdir @@ -1366,6 +1395,25 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") file_size = written_size + # ── Early duplicate rejection ────────────────────────────────────────── + # Check for exact duplicates (same SHA-256 hash) BEFORE enqueuing a + # processing task. When deduplication is enabled and the file already + # exists, we skip processing entirely, clean up the temp file, and + # return the existing file's information to the caller. + exact_duplicate = _check_for_exact_duplicate(db, target_path, safe_filename) + if exact_duplicate: + # Remove the just-saved temp file — it's a duplicate. + try: + os.remove(target_path) + except OSError: + pass + return { + "status": "duplicate", + "original_filename": safe_filename, + "stored_filename": target_filename, + "duplicate_of": exact_duplicate, + } + # Determine if the file is a PDF or needs conversion mime_type, _ = mimetypes.guess_type(target_path) file_ext = os.path.splitext(target_path)[1].lower() @@ -1429,6 +1477,8 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... ".tif", ".webp", ".svg", + ".heic", + ".heif", }: # If it's an image, convert to PDF first task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id) @@ -1442,20 +1492,12 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(... logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id) - # Check for exact duplicates (same SHA-256 hash) before returning. - # This gives the caller an immediate warning without waiting for the pipeline. - # Only performed when deduplication is enabled in settings. - exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename) - - response: dict = { + return { "task_id": task.id, "status": "queued", "original_filename": safe_filename, "stored_filename": target_filename, } - if exact_duplicate_warning: - response["duplicate_warning"] = exact_duplicate_warning - return response # --------------------------------------------------------------------------- diff --git a/app/api/integrations.py b/app/api/integrations.py index 8d2d9209..c0d58893 100644 --- a/app/api/integrations.py +++ b/app/api/integrations.py @@ -32,6 +32,21 @@ from app.utils.encryption import decrypt_value, encrypt_value from app.utils.subscription import get_tier, get_user_tier_id from app.utils.user_scope import get_current_owner_id +# Optional Dropbox SDK — imported at module level so tests can patch it cleanly. +try: + import dropbox as dbx_lib + from dropbox.exceptions import AuthError as _DropboxAuthError + from dropbox.exceptions import BadInputError as _DropboxBadInputError +except ImportError: # pragma: no cover + dbx_lib = None # type: ignore[assignment] + + class _DropboxAuthError(Exception): # type: ignore[no-redef] + """Stub — only used when the dropbox package is missing.""" + + class _DropboxBadInputError(Exception): # type: ignore[no-redef] + """Stub — only used when the dropbox package is missing.""" + + logger = logging.getLogger(__name__) router = APIRouter(prefix="/integrations", tags=["integrations"]) @@ -550,6 +565,47 @@ def _test_s3_connection(config: dict[str, Any] | None, credentials: dict[str, An return {"success": False, "message": "S3 connection failed"} +def _test_dropbox_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]: + """Test a Dropbox connection by verifying OAuth credentials via the Dropbox API.""" + if dbx_lib is None: + return {"success": False, "message": "dropbox package is not installed"} # pragma: no cover + + creds = credentials or {} + app_key = creds.get("app_key", "") + app_secret = creds.get("app_secret", "") + refresh_token = creds.get("refresh_token", "") + + if not refresh_token: + return {"success": False, "message": "Missing required credential: refresh_token"} + if not app_key or not app_secret: + return {"success": False, "message": "Missing required credentials: app_key and app_secret"} + + try: + dbx = dbx_lib.Dropbox( + app_key=app_key, + app_secret=app_secret, + oauth2_refresh_token=refresh_token, + ) + account = dbx.users_get_current_account() + display_name = getattr(account, "name", None) + name_str = "" + if display_name: + name_str = f" ({getattr(display_name, 'display_name', '') or ''})" + return {"success": True, "message": f"Dropbox connection successful{name_str}"} + except _DropboxAuthError as exc: + logger.warning("Dropbox auth error: %s", exc) + return { + "success": False, + "message": "Dropbox authentication failed — check app_key, app_secret, and refresh_token", + } + except _DropboxBadInputError as exc: + logger.warning("Dropbox bad input error: %s", exc) + return {"success": False, "message": "Dropbox connection failed — invalid credentials format"} + except Exception as exc: # noqa: BLE001 + logger.warning("Dropbox connection error: %s", exc) + return {"success": False, "message": "Dropbox connection failed — check credentials and network connectivity"} + + def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str, Any] | None) -> dict[str, Any]: """Test a WebDAV/Nextcloud connection by issuing an HTTP PROPFIND.""" import urllib.request @@ -596,6 +652,7 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str _CONNECTION_TESTERS: dict[str, Any] = { + IntegrationType.DROPBOX: _test_dropbox_connection, IntegrationType.IMAP: _test_imap_connection, IntegrationType.S3: _test_s3_connection, IntegrationType.WEBDAV: _test_webdav_connection, diff --git a/app/api/mobile.py b/app/api/mobile.py index 465872ab..5305c5d2 100644 --- a/app/api/mobile.py +++ b/app/api/mobile.py @@ -120,6 +120,7 @@ class WhoAmIResponse(BaseModel): email: str | None avatar_url: str | None is_admin: bool + preferred_language: str | None # --------------------------------------------------------------------------- @@ -273,31 +274,44 @@ async def list_devices( return [_device_to_response(d) for d in devices] -@router.delete("/devices/{device_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/devices/{device_id}", status_code=status.HTTP_200_OK) @require_login async def deactivate_device( request: Request, device_id: int, owner_id: CurrentOwner, db: DbSession, -) -> None: - """Deactivate a push-notification device registration. +) -> dict[str, str]: + """Deactivate or permanently delete a push-notification device registration. - The device record is kept for audit purposes but will no longer receive - push notifications. + * **Active device** – soft-deactivated: the record is kept for audit + purposes but will no longer receive push notifications. + * **Already-inactive device** – hard-deleted: the record is permanently + removed from the database. """ device = db.get(MobileDevice, device_id) if not device or device.owner_id != owner_id: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Device not found") - device.is_active = False + if device.is_active: + device.is_active = False + try: + db.commit() + except Exception: + db.rollback() + raise + logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id) + return {"detail": "Device deactivated"} + + # Hard-delete an already-inactive device. try: + db.delete(device) db.commit() except Exception: db.rollback() raise - - logger.info("Mobile device deactivated: id=%s owner=%s", device_id, owner_id) + logger.info("Mobile device permanently deleted: id=%s owner=%s", device_id, owner_id) + return {"detail": "Device deleted"} @router.get("/whoami", response_model=WhoAmIResponse) @@ -344,4 +358,5 @@ async def whoami( "email": email, "avatar_url": avatar_url, "is_admin": is_admin, + "preferred_language": profile.preferred_language if profile else None, } diff --git a/app/api/qr_auth.py b/app/api/qr_auth.py index b9dbaa5d..8f891e4e 100644 --- a/app/api/qr_auth.py +++ b/app/api/qr_auth.py @@ -19,10 +19,13 @@ Security properties: from __future__ import annotations +import base64 +import io import logging from datetime import datetime from typing import Annotated, Any +import segno from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -72,6 +75,7 @@ class CreateChallengeResponse(BaseModel): expires_at: datetime ttl_seconds: int = Field(description="Seconds until the challenge expires (use for client-side countdown).") qr_payload: str = Field(description="The string to encode in the QR code.") + qr_code_svg: str = Field(description="Base64-encoded SVG data URI of the QR code, ready for use in an src.") class ChallengeStatusResponse(BaseModel): @@ -106,6 +110,29 @@ class ClaimChallengeResponse(BaseModel): created_at: datetime +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# QR code rendering parameters +_QR_ERROR_LEVEL = "M" # Medium error correction (~15% recovery); sufficient for on-screen display +_QR_SCALE = 4 # Each QR module is rendered as 4×4 SVG pixels + + +def _generate_qr_svg(payload: str) -> str: + """Generate a QR code for *payload* and return it as a base64 SVG data URI. + + Using ``segno`` (pure-Python, no Pillow dependency) and SVG output so the + QR code scales crisply at any resolution without requiring a canvas or any + client-side JavaScript library. + """ + qr = segno.make(payload, error=_QR_ERROR_LEVEL) + buf = io.BytesIO() + qr.save(buf, kind="svg", scale=_QR_SCALE, xmldecl=False, svgclass=None, lineclass=None, omitsize=True) + svg_bytes = buf.getvalue() + return "data:image/svg+xml;base64," + base64.b64encode(svg_bytes).decode("ascii") + + # --------------------------------------------------------------------------- # Endpoints # --------------------------------------------------------------------------- @@ -143,6 +170,7 @@ async def create_challenge( "expires_at": challenge.expires_at, "ttl_seconds": ttl_seconds, "qr_payload": qr_payload, + "qr_code_svg": _generate_qr_svg(qr_payload), } diff --git a/app/api/url_upload.py b/app/api/url_upload.py index ae286ad3..e93eaea3 100644 --- a/app/api/url_upload.py +++ b/app/api/url_upload.py @@ -11,11 +11,12 @@ from typing import Optional import aiofiles import httpx -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel, HttpUrl, field_validator from app.auth import require_login from app.config import settings +from app.middleware.upload_rate_limit import require_upload_rate_limit from app.tasks.process_document import process_document from app.utils.allowed_types import ALLOWED_MIME_TYPES from app.utils.filename_utils import sanitize_filename @@ -107,7 +108,11 @@ def validate_file_type(content_type: str, filename: str) -> bool: @router.post("/process-url") @require_login -async def process_url(request: Request, url_request: URLUploadRequest): +async def process_url( + request: Request, + url_request: URLUploadRequest, + _rate_ok: None = Depends(require_upload_rate_limit), +): """ Download a file from a URL and enqueue it for processing. diff --git a/app/auth.py b/app/auth.py index c5883dac..5b92b156 100644 --- a/app/auth.py +++ b/app/auth.py @@ -103,11 +103,18 @@ if AUTH_ENABLED and settings.social_auth_apple_enabled: logger.warning("SOCIAL_AUTH_APPLE_ENABLED=true but client ID/team ID not configured") if AUTH_ENABLED and settings.social_auth_dropbox_enabled: - if settings.social_auth_dropbox_client_id and settings.social_auth_dropbox_client_secret: + # Determine which credentials to use for Dropbox social login + _dropbox_client_id = settings.social_auth_dropbox_client_id + _dropbox_client_secret = settings.social_auth_dropbox_client_secret + if settings.social_auth_dropbox_use_global_credentials and not _dropbox_client_id: + _dropbox_client_id = settings.dropbox_app_key + _dropbox_client_secret = settings.dropbox_app_secret + + if _dropbox_client_id and _dropbox_client_secret: oauth.register( name="dropbox", - client_id=settings.social_auth_dropbox_client_id, - client_secret=settings.social_auth_dropbox_client_secret, + client_id=_dropbox_client_id, + client_secret=_dropbox_client_secret, authorize_url="https://www.dropbox.com/oauth2/authorize", access_token_url="https://api.dropboxapi.com/oauth2/token", userinfo_endpoint="https://api.dropboxapi.com/2/users/get_current_account", @@ -186,6 +193,16 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None: logger.debug("[AUTH] _resolve_bearer_user: no active API token matched the provided hash") return None + # Reject tokens that have passed their optional expiry. + if db_token.expires_at is not None: + now_utc = datetime.now(timezone.utc) + expires_aware = db_token.expires_at + if expires_aware.tzinfo is None: + expires_aware = expires_aware.replace(tzinfo=timezone.utc) + if now_utc > expires_aware: + logger.debug("[AUTH] _resolve_bearer_user: API token id=%s has expired", db_token.id) + return None + logger.debug( "[AUTH] _resolve_bearer_user: matched API token id=%s owner=%s", db_token.id, diff --git a/app/config.py b/app/config.py index cff42826..70ba5c36 100644 --- a/app/config.py +++ b/app/config.py @@ -13,6 +13,24 @@ class Settings(BaseSettings): database_url: str redis_url: str + + # Database connection-pool tuning (ignored for SQLite, which uses NullPool). + db_pool_size: int = Field( + default=10, + description="Number of persistent connections kept in the pool per worker process.", + ) + db_max_overflow: int = Field( + default=20, + description="Additional connections allowed beyond db_pool_size under burst load.", + ) + db_pool_timeout: int = Field( + default=30, + description="Seconds to wait for a connection from the pool before raising a TimeoutError.", + ) + db_pool_recycle: int = Field( + default=1800, + description="Recycle (close and reopen) connections after this many seconds to avoid stale connections.", + ) openai_api_key: str openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint openai_model: str = "gpt-4o-mini" # Default model @@ -102,6 +120,16 @@ class Settings(BaseSettings): dropbox_app_secret: Optional[str] = None dropbox_folder: Optional[str] = None dropbox_refresh_token: Optional[str] = None + dropbox_allow_global_credentials_for_integrations: bool = Field( + default=False, + description=( + "When True, users may authorize their personal Dropbox integrations using the global " + "DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without " + "needing to create their own Dropbox app. The Dropbox OAuth flow is initiated " + "server-side so the app secret is never exposed to the browser. " + "Default: False (each user must supply their own app credentials)." + ), + ) # Making Nextcloud optional nextcloud_enabled: bool = Field( @@ -165,6 +193,16 @@ class Settings(BaseSettings): google_docai_processor_id: Optional[str] = None google_docai_location: str = "us" # Processor location, e.g. "us" or "eu" external_hostname: str = "localhost" # Default to localhost + public_base_url: Optional[str] = Field( + default=None, + description=( + "The full public base URL of the application, including scheme " + "(e.g., 'https://docuelevate.example.com'). " + "When set, this overrides the auto-detected URL for OAuth redirect URIs. " + "This is required when the application is behind a reverse proxy that does " + "not forward X-Forwarded-Proto headers correctly." + ), + ) # --------------------------------------------------------------------------- # Document Translation Settings @@ -299,6 +337,16 @@ class Settings(BaseSettings): social_auth_dropbox_enabled: bool = False social_auth_dropbox_client_id: Optional[str] = None social_auth_dropbox_client_secret: Optional[str] = None + social_auth_dropbox_use_global_credentials: bool = Field( + default=False, + description=( + "When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET " + "credentials (the storage integration credentials) instead of requiring separate " + "SOCIAL_AUTH_DROPBOX_CLIENT_ID / SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. " + "Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and the global Dropbox app credentials to be set. " + "Default: False." + ), + ) # Local user signup allow_local_signup: bool = Field( @@ -1115,6 +1163,20 @@ class Settings(BaseSettings): ), ) + # Per-user upload rate limiting (health-aware, Redis-backed sliding window) + upload_rate_limit_per_user: int = Field( + default=20, + description=( + "Maximum number of file uploads allowed per user within the sliding window. " + "The effective limit may be reduced dynamically when the system is under heavy load " + "(high queue depth or CPU usage). Set to 0 to disable per-user upload rate limiting." + ), + ) + upload_rate_limit_window: int = Field( + default=60, + description="Sliding window size in seconds for per-user upload rate limiting (default: 60).", + ) + # Rate Limiting Configuration (see SECURITY_AUDIT.md and docs/API.md) # Protects against DoS attacks and API abuse rate_limiting_enabled: bool = Field( diff --git a/app/database.py b/app/database.py index f563931a..701e4673 100644 --- a/app/database.py +++ b/app/database.py @@ -10,6 +10,7 @@ from typing import Any from sqlalchemy import create_engine, exc from sqlalchemy.engine.url import make_url from sqlalchemy.orm import Session, declarative_base, sessionmaker +from sqlalchemy.pool import NullPool, QueuePool from app.config import settings @@ -17,9 +18,37 @@ logger = logging.getLogger(__name__) Base = declarative_base() -# Parse the DATABASE_URL +# --------------------------------------------------------------------------- +# Engine construction +# --------------------------------------------------------------------------- DB_URL = settings.database_url -engine = create_engine(DB_URL, connect_args={"check_same_thread": False}) +_parsed_url = make_url(DB_URL) + +_connect_args: dict[str, Any] = {} +_engine_kwargs: dict[str, Any] = { + "pool_pre_ping": True, # detect stale / dropped connections before use +} + +if _parsed_url.get_backend_name() == "sqlite": + # SQLite does not benefit from connection pooling and is prone to + # QueuePool exhaustion under concurrent access. NullPool opens a fresh + # connection for each request and closes it immediately afterwards, + # completely avoiding the "QueuePool limit reached" TimeoutError. + _connect_args["check_same_thread"] = False + _engine_kwargs["poolclass"] = NullPool +else: + # PostgreSQL / MySQL — use a bounded QueuePool with configurable limits. + _engine_kwargs["poolclass"] = QueuePool + _engine_kwargs.update( + { + "pool_size": settings.db_pool_size, + "max_overflow": settings.db_max_overflow, + "pool_timeout": settings.db_pool_timeout, + "pool_recycle": settings.db_pool_recycle, + } + ) + +engine = create_engine(DB_URL, connect_args=_connect_args, **_engine_kwargs) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/app/middleware/csrf.py b/app/middleware/csrf.py index 842958af..f21088ad 100644 --- a/app/middleware/csrf.py +++ b/app/middleware/csrf.py @@ -20,6 +20,9 @@ How it works: Exempt paths (CSRF is not checked even for state-changing methods): - ``/oauth-callback`` – OAuth 2.0 callback; protected by the ``state`` parameter. +- ``/api/qr-auth/claim`` – Called by the unauthenticated mobile app; the + cryptographically-random, single-use challenge token provides equivalent + protection. """ import logging @@ -39,6 +42,10 @@ CSRF_PROTECTED_METHODS = {"POST", "PUT", "DELETE", "PATCH"} # their own replay-protection mechanism). CSRF_EXEMPT_PATHS = { "/oauth-callback", + # The mobile app calls this endpoint without a browser session/CSRF token. + # The cryptographically-random, single-use challenge token already provides + # equivalent protection against cross-site request forgery. + "/api/qr-auth/claim", } diff --git a/app/middleware/upload_rate_limit.py b/app/middleware/upload_rate_limit.py new file mode 100644 index 00000000..16b7d907 --- /dev/null +++ b/app/middleware/upload_rate_limit.py @@ -0,0 +1,290 @@ +"""Per-user, health-aware upload rate limiter for DocuElevate. + +This module provides a FastAPI dependency that enforces per-user upload rate +limits using a Redis-backed sliding window counter. The effective limit is +dynamically reduced when the system is under heavy load (high Celery queue +depth or elevated CPU load average), ensuring the server remains responsive +to all users even during bulk-upload scenarios. + +Usage in an endpoint:: + + from app.middleware.upload_rate_limit import require_upload_rate_limit + + @router.post("/ui-upload") + @require_login + async def ui_upload( + request: Request, + _rate_ok: None = Depends(require_upload_rate_limit), + ... + ): + ... + +See ``docs/ConfigurationGuide.md`` for the configuration options +(``UPLOAD_RATE_LIMIT_PER_USER``, ``UPLOAD_RATE_LIMIT_WINDOW``). +""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +import redis +from fastapi import HTTPException, Request, status + +from app.config import settings +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Redis key prefix +# --------------------------------------------------------------------------- +_KEY_PREFIX = "docuelevate:upload_rate" + +# --------------------------------------------------------------------------- +# Health-check queue names (Celery defaults used by DocuElevate) +# --------------------------------------------------------------------------- +_CELERY_QUEUES = ("document_processor", "default", "celery") + +# --------------------------------------------------------------------------- +# Singleton Redis client (lazy-initialised; fail-open when unavailable) +# --------------------------------------------------------------------------- +_redis_client: redis.Redis | None = None + + +def _get_redis() -> redis.Redis | None: + """Return a shared Redis client, or *None* when Redis is unavailable.""" + global _redis_client + if _redis_client is not None: + return _redis_client + try: + _redis_client = redis.Redis.from_url( + settings.redis_url, + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + ) + # Quick connectivity check – raises on failure. + _redis_client.ping() + return _redis_client + except Exception: # noqa: BLE001 + logger.debug("Redis unavailable for upload rate limiter – falling back to allow-all", exc_info=True) + _redis_client = None + return None + + +# --------------------------------------------------------------------------- +# Health metrics helpers +# --------------------------------------------------------------------------- + + +def _get_queue_depth(r: redis.Redis) -> int: + """Return the total number of pending tasks across all Celery queues.""" + total = 0 + for queue_name in _CELERY_QUEUES: + try: + total += r.llen(queue_name) + except Exception: # noqa: BLE001, S110 + logger.debug("Could not read queue length for %r", queue_name, exc_info=True) + return total + + +def _get_cpu_load_ratio() -> float: + """Return the 1-minute load average divided by the number of CPU cores. + + Returns ``0.0`` on platforms that do not support :func:`os.getloadavg` + (e.g. Windows) so that the limiter never penalises on those systems. + """ + try: + load_1m = os.getloadavg()[0] + cpu_count = os.cpu_count() or 1 + return load_1m / cpu_count + except (OSError, AttributeError): + return 0.0 + + +def compute_effective_limit( + base_limit: int, + queue_depth: int = 0, + cpu_load_ratio: float = 0.0, +) -> tuple[int, float, str]: + """Compute the effective upload rate limit based on system health. + + The function applies a *reduction factor* (``0.0 < factor ≤ 1.0``) to the + configured base limit. Both queue depth and CPU load contribute + independently; the lowest factor wins. + + Args: + base_limit: The configured maximum uploads per window. + queue_depth: Total pending tasks in Celery queues. + cpu_load_ratio: 1-minute load average divided by CPU count. + + Returns: + A 3-tuple of ``(effective_limit, factor, reason)`` where *reason* + is a human-readable tag for logging. + """ + factor = 1.0 + reason = "normal" + + # --- Queue-depth thresholds --- + if queue_depth > 200: + factor, reason = min(factor, 0.10), f"critical_queue({queue_depth})" + elif queue_depth > 100: + factor, reason = min(factor, 0.25), f"high_queue({queue_depth})" + elif queue_depth > 50: + factor, reason = min(factor, 0.50), f"moderate_queue({queue_depth})" + + # --- CPU-load thresholds --- + if cpu_load_ratio > 3.0: + new_factor = 0.10 + if new_factor < factor: + factor, reason = new_factor, f"critical_cpu({cpu_load_ratio:.1f})" + elif cpu_load_ratio > 2.0: + new_factor = 0.25 + if new_factor < factor: + factor, reason = new_factor, f"high_cpu({cpu_load_ratio:.1f})" + elif cpu_load_ratio > 1.5: + new_factor = 0.50 + if new_factor < factor: + factor, reason = new_factor, f"moderate_cpu({cpu_load_ratio:.1f})" + + effective = max(1, int(base_limit * factor)) + return effective, factor, reason + + +# --------------------------------------------------------------------------- +# Core sliding-window check (Redis sorted set) +# --------------------------------------------------------------------------- + + +def _check_and_record( + r: redis.Redis, + user_id: str, + window: int, + effective_limit: int, +) -> dict[str, Any] | None: + """Atomically check the user's upload count and record the new upload. + + Uses a Redis sorted set where each member is a unique timestamp-based ID + and the score is the Unix timestamp. Entries older than *window* seconds + are pruned on every call so the set never grows unbounded. + + Returns: + ``None`` if the request is allowed, or a ``dict`` with ``count``, + ``limit``, and ``retry_after`` if the limit is exceeded. + """ + key = f"{_KEY_PREFIX}:{user_id}" + now = time.time() + window_start = now - window + + pipe = r.pipeline(transaction=True) + # 1. Remove entries outside the window + pipe.zremrangebyscore(key, "-inf", window_start) + # 2. Count current entries + pipe.zcard(key) + # 3. Retrieve the oldest entry's score (to compute retry_after) + pipe.zrange(key, 0, 0, withscores=True) + results = pipe.execute() + + current_count: int = results[1] + oldest_entries: list = results[2] + + if current_count >= effective_limit: + # Compute how long until the oldest entry expires from the window. + if oldest_entries: + oldest_score = oldest_entries[0][1] + retry_after = max(1, int((oldest_score + window) - now)) + else: + retry_after = max(1, window // 2) + return { + "count": current_count, + "limit": effective_limit, + "retry_after": retry_after, + } + + # 4. Record this upload (unique member = timestamp with random suffix) + member = f"{now}:{os.urandom(4).hex()}" + pipe2 = r.pipeline(transaction=True) + pipe2.zadd(key, {member: now}) + pipe2.expire(key, window + 60) # TTL slightly longer than window + pipe2.execute() + + return None + + +# --------------------------------------------------------------------------- +# FastAPI dependency +# --------------------------------------------------------------------------- + + +async def require_upload_rate_limit(request: Request) -> None: + """FastAPI dependency that enforces per-user upload rate limits. + + The dependency is designed to **fail open**: if Redis is unavailable the + request is allowed through so that uploads are never blocked by a + monitoring outage. + + Raises: + HTTPException: 429 Too Many Requests when the per-user upload limit + is exceeded. The ``Retry-After`` header indicates how many + seconds the client should wait before retrying. + """ + r = _get_redis() + if r is None: + # Redis unavailable – fail open. + return + + # Identify the user (owner_id for multi-user, IP fallback). + user_id = get_current_owner_id(request) + if not user_id: + user_id = f"ip:{request.client.host}" if request.client else "ip:unknown" + + base_limit: int = settings.upload_rate_limit_per_user + window: int = settings.upload_rate_limit_window + + # Gather health metrics and compute effective limit. + try: + queue_depth = _get_queue_depth(r) + except Exception: # noqa: BLE001 + queue_depth = 0 + + cpu_load_ratio = _get_cpu_load_ratio() + effective_limit, factor, health_reason = compute_effective_limit(base_limit, queue_depth, cpu_load_ratio) + + # Sliding-window check. + try: + rejection = _check_and_record(r, user_id, window, effective_limit) + except Exception as exc: # noqa: BLE001 + logger.warning("Upload rate-limit check failed (allowing request): %s", exc) + return + + if rejection is not None: + retry_after = rejection["retry_after"] + logger.warning( + "Upload rate limit exceeded: user=%s count=%d/%d window=%ds health=%s retry_after=%ds", + user_id, + rejection["count"], + rejection["limit"], + window, + health_reason, + retry_after, + ) + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail=( + f"Upload rate limit exceeded ({rejection['count']}/{rejection['limit']} " + f"in {window}s). Retry after {retry_after}s." + ), + headers={"Retry-After": str(retry_after)}, + ) + + if factor < 1.0: + logger.info( + "Upload allowed with reduced limit: user=%s effective=%d/%d health=%s", + user_id, + effective_limit, + base_limit, + health_reason, + ) diff --git a/app/models.py b/app/models.py index 10e964f4..60323f02 100644 --- a/app/models.py +++ b/app/models.py @@ -786,6 +786,9 @@ class ApiToken(Base): created_at = Column(DateTime(timezone=True), server_default=func.now()) revoked_at = Column(DateTime(timezone=True), nullable=True) + # Optional expiry: if set, the token is rejected after this timestamp. + expires_at = Column(DateTime(timezone=True), nullable=True) + class SharedLink(Base): """Shareable, time-limited or view-limited document link. diff --git a/app/tasks/convert_to_pdf.py b/app/tasks/convert_to_pdf.py index 2a0ff4ac..08320f55 100644 --- a/app/tasks/convert_to_pdf.py +++ b/app/tasks/convert_to_pdf.py @@ -205,7 +205,7 @@ def convert_to_pdf( ".pdf", # PDF (already in PDF format but can be processed) } - IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg"} + IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg", ".heic", ".heif"} HTML_EXTENSIONS = {".html", ".htm"} diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py index eaa37e6e..7212c0f8 100644 --- a/app/utils/allowed_types.py +++ b/app/utils/allowed_types.py @@ -68,6 +68,8 @@ IMAGE_MIME_TYPES: set[str] = { "image/tiff", "image/webp", "image/svg+xml", + "image/heic", + "image/heif", } # --------------------------------------------------------------------------- @@ -124,6 +126,8 @@ ALLOWED_EXTENSIONS: set[str] = { ".tif", ".webp", ".svg", + ".heic", + ".heif", # Web ".html", ".htm", @@ -234,7 +238,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { }, "images": { "label": "Images", - "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)", + "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)", "mime_types": frozenset( { "image/jpeg", @@ -245,6 +249,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { "image/tiff", "image/webp", "image/svg+xml", + "image/heic", + "image/heif", } ), "extensions": frozenset( @@ -258,6 +264,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = { ".tif", ".webp", ".svg", + ".heic", + ".heif", } ), }, diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 41e870ec..644a62eb 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -39,6 +39,50 @@ SETTING_METADATA = { "required": True, "restart_required": True, }, + "db_pool_size": { + "category": "Core", + "description": ( + "Number of persistent database connections kept in the pool per worker process. " + "Ignored for SQLite (which uses NullPool). Default: 10." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_max_overflow": { + "category": "Core", + "description": ( + "Additional database connections allowed beyond db_pool_size under burst load. " + "Ignored for SQLite. Default: 20." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_pool_timeout": { + "category": "Core", + "description": ( + "Seconds to wait for a database connection from the pool before raising a TimeoutError. " + "Ignored for SQLite. Default: 30." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "db_pool_recycle": { + "category": "Core", + "description": ( + "Recycle (close and reopen) database connections after this many seconds " + "to avoid stale connections. Ignored for SQLite. Default: 1800." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": True, + }, "workdir": { "category": "Core", "description": "Working directory for file storage and processing", @@ -55,6 +99,18 @@ SETTING_METADATA = { "required": True, # Required for OAuth redirects and external URLs "restart_required": True, }, + "public_base_url": { + "category": "Core", + "description": ( + "Full public base URL including scheme (e.g., https://docuelevate.example.com). " + "When set, overrides auto-detected URLs for OAuth redirect URIs. " + "Required when behind a reverse proxy that does not forward X-Forwarded-Proto." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, "debug": { "category": "Core", "description": "Enable debug mode for verbose logging", @@ -326,6 +382,19 @@ SETTING_METADATA = { "required": False, "restart_required": True, }, + "social_auth_dropbox_use_global_credentials": { + "category": "Social Login", + "description": ( + "When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET " + "credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / " + "SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. " + "Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, "social_auth_dropbox_enabled": { "category": "Social Login", "description": ( @@ -719,6 +788,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "dropbox_allow_global_credentials_for_integrations": { + "category": "Storage Providers", + "description": ( + "When True, users may authorize their personal Dropbox integrations using the global " + "DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without " + "needing to create their own Dropbox app." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Storage Providers - Nextcloud "nextcloud_enabled": { "category": "Storage Providers", @@ -2557,6 +2638,27 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Per-user upload rate limiting + "upload_rate_limit_per_user": { + "category": "Security", + "description": ( + "Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. " + "The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. " + "Default: 20." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "upload_rate_limit_window": { + "category": "Security", + "description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Rate Limiting "rate_limiting_enabled": { "category": "Security", diff --git a/app/views/dropbox.py b/app/views/dropbox.py index f623a7c6..5723650a 100644 --- a/app/views/dropbox.py +++ b/app/views/dropbox.py @@ -14,6 +14,19 @@ from app.views.base import APIRouter, Depends, get_db, require_login, settings, router = APIRouter() +def _get_dropbox_callback_url(request: Request) -> str: + """Return the Dropbox OAuth callback URL. + + Uses ``PUBLIC_BASE_URL`` when configured so that the redirect URI displayed + to the user (and registered in the Dropbox developer console) matches the + one used in the OAuth authorization request. Falls back to deriving the URL + from the incoming request when ``PUBLIC_BASE_URL`` is not set. + """ + if settings.public_base_url: + return settings.public_base_url.rstrip("/") + "/dropbox-callback" + return f"{request.url.scheme}://{request.url.netloc}/dropbox-callback" + + @router.get("/dropbox-setup") @require_login async def dropbox_setup_page( @@ -30,6 +43,8 @@ async def dropbox_setup_page( path from the integration's existing config is pre-populated; global admin credentials are never exposed in this mode. """ + callback_url = _get_dropbox_callback_url(request) + if integration_id is not None: owner_id = get_current_owner_id(request) integration = ( @@ -46,6 +61,12 @@ async def dropbox_setup_page( cfg = {} # Support both "folder" (DROPBOX destination) and "folder_path" (WATCH_FOLDER source) folder_path = cfg.get("folder", cfg.get("folder_path", "")) + # Determine if global credentials are available for users to reuse + global_creds_available = bool( + settings.dropbox_allow_global_credentials_for_integrations + and settings.dropbox_app_key + and settings.dropbox_app_secret + ) return templates.TemplateResponse( "dropbox.html", { @@ -56,9 +77,12 @@ async def dropbox_setup_page( "integration_name": integration.name, "integration_type": integration.integration_type, "folder_path": folder_path, - "app_key_value": "", + # Only expose the public app key (not the secret) when global creds are allowed + "app_key_value": settings.dropbox_app_key if global_creds_available else "", "app_secret_value": "", "refresh_token_value": "", + "global_creds_available": global_creds_available, + "callback_url": callback_url, }, ) @@ -78,6 +102,7 @@ async def dropbox_setup_page( "integration_id": integration_id, "integration_name": None, "integration_type": None, + "callback_url": callback_url, }, ) @@ -108,5 +133,6 @@ async def dropbox_callback(request: Request, code: str = None, error: str = None "app_key_value": "", # The callback will prioritize sessionStorage values "app_secret_value": "", # The callback will prioritize sessionStorage values "folder_path": "", # The callback will prioritize sessionStorage values + "callback_url": _get_dropbox_callback_url(request), }, ) diff --git a/docker-compose.yaml b/docker-compose.yaml index 05e1584c..10c370f6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile - container_name: document_api + # No container_name — allows `docker compose up --scale api=N` restart: always # We'll keep the code in /app, but set working_dir to the shared data directory @@ -24,7 +24,7 @@ services: depends_on: - redis - - worker + - beat # Mount the shared working directory for data volumes: @@ -34,13 +34,14 @@ services: build: context: . dockerfile: Dockerfile - container_name: document_worker + # No container_name — allows `docker compose up --scale worker=N` restart: always # same shared working directory working_dir: /workdir - command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"] + # Workers process tasks only — no -B flag (Beat runs in the dedicated beat service) + command: ["celery", "-A", "app.celery_worker", "worker", "--loglevel=info", "-Q", "document_processor,default,celery"] env_file: - .env environment: @@ -54,6 +55,26 @@ services: volumes: - /var/docparse/workdir:/workdir + # Dedicated Celery Beat scheduler — exactly one instance must run at all times. + # Beat publishes periodic tasks to the Redis broker; workers pick them up. + # Do NOT scale this service (replicas must stay at 1). + beat: + build: + context: . + dockerfile: Dockerfile + container_name: document_beat + restart: always + working_dir: /workdir + command: ["celery", "-A", "app.celery_worker", "beat", "--loglevel=info"] + env_file: + - .env + environment: + - PYTHONPATH=/app + depends_on: + - redis + volumes: + - /var/docparse/workdir:/workdir + gotenberg: image: gotenberg/gotenberg:latest container_name: gotenberg diff --git a/docs/API.md b/docs/API.md index de9fdf62..775bd593 100644 --- a/docs/API.md +++ b/docs/API.md @@ -27,9 +27,11 @@ DocuElevate implements rate limiting to protect against abuse and DoS attacks. R ### Default Limits - **Default endpoints**: 100 requests per minute -- **File upload**: 600 requests per minute +- **File upload**: 600 requests per minute (global) + 20 per user per 60 s (per-user, health-aware) - **Authentication**: 10 requests per minute +**Per-user upload rate limiting**: Upload endpoints (`/api/ui-upload`, `/api/process-url`) enforce a per-user sliding-window limit that adapts to system load. Under heavy queue depth or high CPU usage, the effective limit is reduced automatically. See the [Configuration Guide](ConfigurationGuide.md#per-user-upload-rate-limiting) for details. + **Note**: Document processing endpoints (OCR, metadata extraction) use built-in queue throttling to control processing rates and prevent upstream API overloads. No additional API-level rate limit is applied to processing endpoints. ### Rate Limit Headers @@ -53,6 +55,10 @@ RATE_LIMITING_ENABLED=true RATE_LIMIT_DEFAULT=100/minute RATE_LIMIT_UPLOAD=600/minute RATE_LIMIT_AUTH=10/minute + +# Per-user upload rate limiting (health-aware) +UPLOAD_RATE_LIMIT_PER_USER=20 # Max uploads per user per window +UPLOAD_RATE_LIMIT_WINDOW=60 # Sliding window in seconds ``` See [Configuration Guide](ConfigurationGuide.md) for more details. @@ -115,7 +121,8 @@ curl -X GET "http:///api/files" \ |--------|----------|-------------| | `POST` | `/api/api-tokens/` | Create a new token | | `GET` | `/api/api-tokens/` | List all your tokens | -| `DELETE` | `/api/api-tokens/{id}` | Revoke a token | +| `DELETE` | `/api/api-tokens/{id}` | Revoke (active) or permanently delete (revoked) a token | +| `POST` | `/api/api-tokens/{id}/reactivate` | Reactivate a revoked token | ### Session Authentication @@ -235,17 +242,33 @@ The DocuElevate browser extension uses this endpoint to send files directly from **POST** `/api/ui-upload` -Upload one or more files from your computer for processing. +Upload a file from your computer for processing. **Request**: -- Multipart form data with file(s) +- Multipart form data with a single `file` field -**Response**: +**Response** (new file): ```json { - "success": true, - "file_ids": [123, 124], - "message": "Files uploaded and queued for processing" + "task_id": "abc-123", + "status": "queued", + "original_filename": "invoice.pdf", + "stored_filename": "a1b2c3d4.pdf" +} +``` + +**Response** (exact duplicate, when `ENABLE_DEDUPLICATION=True`): +```json +{ + "status": "duplicate", + "original_filename": "invoice.pdf", + "stored_filename": "e5f6a7b8.pdf", + "duplicate_of": { + "duplicate_type": "exact", + "original_file_id": 42, + "original_filename": "invoice.pdf", + "message": "This file is an exact duplicate of an already-processed document. It has not been queued for processing again." + } } ``` @@ -1356,7 +1379,7 @@ Test an integration connection without saving. Useful for "Test connection" UI b {"success": true, "message": "IMAP connection successful"} ``` -Supported connection tests: `IMAP`, `S3`, `WEBDAV`, `NEXTCLOUD`. Other types return a message that testing is not yet supported. +Supported connection tests: `DROPBOX`, `IMAP`, `S3`, `WEBDAV`, `NEXTCLOUD`. Other types return a message that testing is not yet supported. ### GET /api/integrations/quota/ @@ -1571,6 +1594,47 @@ Lightweight endpoint returning the total number of queued + in-progress items. D ## Diagnostic +### GET /api/diagnostic/healthz/live + +Lightweight liveness probe for Kubernetes. Returns **200 OK** as long as the process is running. This endpoint does **not** check external dependencies and is intentionally cheap. + +**Authentication:** None (designed for kubelet probes) + +**Response (200 OK):** +```json +{ + "status": "ok" +} +``` + +### GET /api/diagnostic/healthz/ready + +Readiness probe for Kubernetes. Verifies that the application can serve traffic by checking database and Redis connectivity. + +**Authentication:** None (designed for kubelet probes) + +**Response (200 OK) – ready to serve traffic:** +```json +{ + "status": "ready", + "checks": { + "database": {"status": "ok"}, + "redis": {"status": "ok"} + } +} +``` + +**Response (503 Service Unavailable) – database unreachable:** +```json +{ + "status": "not_ready", + "checks": { + "database": {"status": "error", "detail": "..."}, + "redis": {"status": "ok"} + } +} +``` + ### GET /api/diagnostic/health System health endpoint designed for monitoring tools such as Grafana, Uptime Kuma, Prometheus blackbox exporter, or any HTTP-based health checker. @@ -2127,12 +2191,14 @@ Usage tracking records when each token was last used and from which IP address. ### POST /api/api-tokens/ -Create a new API token. +Create a new API token. Optionally specify a lifetime in days via +`expires_in_days` (1–3650). If omitted the token never expires. **Request:** ```json { - "name": "CI Pipeline" + "name": "CI Pipeline", + "expires_in_days": 90 } ``` @@ -2147,7 +2213,8 @@ Create a new API token. "last_used_at": null, "last_used_ip": null, "created_at": "2026-03-08T12:00:00Z", - "revoked_at": null + "revoked_at": null, + "expires_at": "2026-06-06T12:00:00Z" } ``` @@ -2169,15 +2236,20 @@ List all tokens for the authenticated user. The full token value is never includ "last_used_at": "2026-03-08T15:30:00Z", "last_used_ip": "203.0.113.42", "created_at": "2026-03-08T12:00:00Z", - "revoked_at": null + "revoked_at": null, + "expires_at": "2026-06-06T12:00:00Z" } ] ``` ### DELETE /api/api-tokens/{token_id} -Revoke a token. The token is soft-deleted (kept for audit purposes) and can no -longer be used for authentication. +Revoke or permanently delete a token: + +* **Active token** – soft-revoked (kept for audit purposes, marked inactive). + Response: `{"detail": "Token revoked"}` +* **Already-revoked token** – permanently deleted from the database. + Response: `{"detail": "Token deleted"}` **Response (200):** ```json @@ -2186,6 +2258,13 @@ longer be used for authentication. } ``` +### POST /api/api-tokens/{token_id}/reactivate + +Reactivate a previously revoked token. Clears `revoked_at` and sets +`is_active` back to `true`. + +**Response (200):** The updated `TokenResponse` object. + ### Using API Tokens Include the token in the `Authorization` header of any API request: @@ -2425,9 +2504,14 @@ List all registered push-notification devices for the current user. ### DELETE /api/mobile/devices/{device_id} -Deactivate a push-notification device. The device will no longer receive push notifications. +Deactivate or permanently delete a push-notification device: -**Response (204 No Content)** +* **Active device** – soft-deactivated (record kept, will no longer receive push notifications). + Response: `{"detail": "Device deactivated"}` +* **Already-inactive device** – permanently deleted from the database. + Response: `{"detail": "Device deleted"}` + +**Response (200)** ### GET /api/mobile/whoami diff --git a/docs/AppleAppStoreCompliance.md b/docs/AppleAppStoreCompliance.md new file mode 100644 index 00000000..2df05a2a --- /dev/null +++ b/docs/AppleAppStoreCompliance.md @@ -0,0 +1,285 @@ +# Apple App Store Compliance Audit Report + +This document details the findings from a comprehensive audit of the DocuElevate mobile app against Apple's App Store Review Guidelines, Human Interface Guidelines (HIG), and privacy requirements. It covers all areas of compliance, risks for rejection, and recommendations. + +> **Last Audited:** March 2026 +> **App Version:** 1.0.0 +> **Expo SDK:** 54.0.0 +> **Bundle ID:** `org.docuelevate.mobile` + +--- + +## Executive Summary + +The DocuElevate mobile app is broadly compliant with Apple's App Store requirements. The following issues were identified and resolved as part of this audit: + +| Issue | Severity | Status | +|-------|----------|--------| +| Unused `fetch` background mode declared | High | ✅ Fixed | +| Missing privacy manifest for required reason APIs | High | ✅ Fixed | +| No account deletion option (Guideline 5.1.1(v)) | Critical | ✅ Fixed | +| No Privacy Policy / Terms of Service links in-app | High | ✅ Fixed | +| Emoji used as UI icons instead of platform-native icons | Medium | ✅ Fixed | +| Missing app version display | Low | ✅ Fixed | +| Unused `Switch` import in ProfileScreen | Low | ✅ Fixed | + +--- + +## 1. Human Interface Guidelines (HIG) + +### 1.1 Navigation & Tab Bar ✅ + +- The app uses a standard bottom tab bar with three tabs: Upload, Files, and Profile. +- Tab icons use **Ionicons** (an icon set that closely maps to Apple's SF Symbols). +- Active/inactive tab colors follow iOS conventions (`#1e40af` active, `#9ca3af` inactive). +- Header styling uses a solid color background with white text, consistent with iOS navigation bar patterns. + +### 1.2 Icons & Visual Assets ✅ + +- **App icon:** Custom `icon.png` provided at root level; Expo handles generating all required sizes. +- **Splash screen:** Uses branded splash with `contain` resize mode and matching background color. +- **Adaptive icon (Android):** Properly configured with foreground image and background color. +- **Action buttons:** Previously used emoji characters (📷, 🖼️, 📄) which render inconsistently across iOS versions. **Fixed:** Now using Ionicons (`camera-outline`, `images-outline`, `document-outline`). +- **Status indicators:** Previously used emoji (✅, ❌, ⏳, ⚙️). **Fixed:** Now using Ionicons with semantic colors. + +### 1.3 Typography & Colors ✅ + +- Uses system fonts (default React Native text rendering uses San Francisco on iOS). +- Color palette (`#1e40af` primary blue, semantic reds/greens/grays) provides sufficient contrast ratios. +- Text sizes follow iOS recommended minimums (body text ≥ 13pt). + +### 1.4 Touch Targets ✅ + +- All interactive elements have `minHeight: 44` or `minHeight: 48` (meets Apple's 44×44pt minimum). +- Back links, cancel buttons, and retry buttons all meet minimum touch target requirements. + +### 1.5 Safe Areas ✅ + +- The app uses `react-native-safe-area-context` (`SafeAreaProvider`) to respect device notches, Dynamic Island, and home indicator. + +### 1.6 Dark Mode ✅ + +- `userInterfaceStyle: "automatic"` is set in `app.json`, enabling automatic dark mode support. + +--- + +## 2. Privacy & Data Usage + +### 2.1 Permission Descriptions ✅ + +All iOS permission strings (Info.plist keys) are present and provide clear, specific descriptions of why each permission is needed: + +| Permission | Key | Description | +|-----------|-----|-------------| +| Camera | `NSCameraUsageDescription` | "DocuElevate uses the camera to scan QR codes for login and to capture documents for upload." | +| Photo Library (Read) | `NSPhotoLibraryUsageDescription` | "DocuElevate accesses your photo library to select documents for upload." | +| Photo Library (Write) | `NSPhotoLibraryAddUsageDescription` | "DocuElevate saves scanned documents to your photo library." | + +**Assessment:** All descriptions clearly explain the purpose, which is a requirement for App Review approval. + +### 2.2 Push Notifications ✅ + +- Push notification permission is requested at runtime (not at launch) when the user enters the authenticated area. +- The app works gracefully without push notifications if permission is denied. +- Device tokens are registered via a dedicated backend endpoint. + +### 2.3 Background Modes ✅ (Fixed) + +- **Previous state:** `UIBackgroundModes` included `["fetch", "remote-notification"]`. +- **Issue:** The app does not implement background fetch (`application:performFetchWithCompletionHandler:`). Apple may reject apps that declare background modes they don't actively use (Guideline 2.5.4). +- **Fix:** Removed `fetch` from `UIBackgroundModes`. Only `remote-notification` remains, which is required for push notification delivery. + +### 2.4 Privacy Manifest ✅ (Fixed) + +Starting in Spring 2024, Apple requires a privacy manifest (`PrivacyInfo.xcprivacy`) for apps using specific APIs. The following required reason APIs are used by the app's dependencies: + +| API Category | Reason Code | Justification | +|-------------|-------------|---------------| +| `NSPrivacyAccessedAPICategoryUserDefaults` | `CA92.1` | Used by `@react-native-async-storage/async-storage` for user preferences | +| `NSPrivacyAccessedAPICategoryFileTimestamp` | `C617.1` | Used by `expo-file-system` to read file metadata | +| `NSPrivacyAccessedAPICategoryDiskSpace` | `E174.1` | Used by Expo runtime for storage space checks | +| `NSPrivacyAccessedAPICategorySystemBootTime` | `35F9.1` | Used by React Native's timing APIs | + +The privacy manifest is configured via `expo-build-properties` plugin in `app.json`, which ensures it is included in the generated Xcode project during EAS Build. + +### 2.5 Tracking & Analytics ✅ + +- `NSPrivacyTracking: false` — the app does **not** track users. +- `NSPrivacyCollectedDataTypes: []` — no data types are collected for tracking. +- No analytics SDKs (Firebase Analytics, Amplitude, Mixpanel, etc.) are included. +- No App Tracking Transparency (ATT) prompt is needed. + +### 2.6 Encryption Declaration ✅ + +- `ITSAppUsesNonExemptEncryption: false` — the app uses only standard HTTPS/TLS for network communication, which is exempt from export compliance requirements. + +### 2.7 Data Storage Security ✅ + +- API tokens are stored in the device keychain via `expo-secure-store` (uses iOS Keychain Services). +- No sensitive data is stored in `AsyncStorage` or `UserDefaults`. +- Server URL is stored in secure storage, not in plain text files. + +--- + +## 3. App Store Review Guidelines Compliance + +### 3.1 Functionality (Guideline 2.x) ✅ + +- **2.1 App Completeness:** The app provides a complete, functional experience. All advertised features (camera capture, file upload, document list, push notifications) work as described. +- **2.3 Accurate Metadata:** App name ("DocuElevate"), description, and screenshots should accurately reflect the app's functionality. +- **2.5.4 Background Modes:** Only `remote-notification` is declared, which is actively used. ✅ Fixed. + +### 3.2 Content & Intellectual Property (Guideline 3.x) ✅ + +- No third-party trademarked content is used. +- The app does not display user-generated content publicly (documents are private to each user). +- No copyrighted content is bundled with the app. + +### 3.3 Business (Guideline 3.1.x) ✅ + +- The app does not include in-app purchases, subscriptions, or payment processing. +- No physical goods or services are sold through the app. +- Authentication is handled via self-hosted or enterprise SSO — no Apple Sign-In requirement applies (Apple Sign-In is required only when third-party social login options like Google/Facebook are offered as the primary login method; enterprise SSO to a self-hosted server is exempt). + +### 3.4 Safety & Privacy (Guideline 5.x) ✅ + +- **5.1.1 Data Collection and Storage:** The app collects only what is necessary for its functionality (server URL, auth token, push token). +- **5.1.1(v) Account Deletion:** ✅ Fixed. Users can now initiate account deletion from the Profile screen, which opens the server's account deletion page in the browser. +- **5.1.2 Data Use and Sharing:** No data is shared with third parties or used for advertising. + +### 3.5 Privacy Policy ✅ (Fixed) + +- **Requirement:** Apple requires all apps to have an accessible privacy policy. +- **Fix:** Privacy Policy and Terms of Service links are now accessible from the Profile screen, opening the server's hosted policy pages. +- **App Store Connect:** The privacy policy URL must also be provided in App Store Connect during submission. + +### 3.6 Login & Authentication ✅ + +- Two login methods are available: SSO (browser-based OAuth) and QR code scanning. +- Both methods provide clear error messages on failure. +- The app correctly handles authentication cancellation. +- Session restoration on app launch is implemented. +- **Demo Account:** For App Review, a demo account may need to be provided in App Store Connect's review notes. Ensure the review team can access a test server. + +--- + +## 4. Technical Compliance + +### 4.1 API Usage ✅ + +- No private APIs are used (all functionality comes from Expo SDK and React Native public APIs). +- No deprecated APIs are used that would trigger rejection. + +### 4.2 Network Security ✅ + +- The app validates server URLs require `http://` or `https://` scheme. +- All API calls use Bearer token authentication over HTTPS. +- App Transport Security (ATS) is not explicitly disabled — default iOS ATS rules apply. + +### 4.3 Deep Linking ✅ + +- Custom URL scheme `docuelevate://` is properly registered. +- Deep link handling for QR login (`docuelevate://qr-login`) and file sharing is implemented correctly. +- `WebBrowser.openAuthSessionAsync` is used for OAuth, which properly handles the authentication session lifecycle. + +### 4.4 Document Handling ✅ + +- `CFBundleDocumentTypes` properly declares supported file types. +- `LSSupportsOpeningDocumentsInPlace: false` ensures iOS copies shared files to the app's accessible Inbox directory, avoiding security-scoped URL issues. +- The `+not-found.tsx` handler correctly intercepts iOS "Open In…" file paths. +- `UploadScreen` uses `expo-file-system` to copy external files to cache before uploading for reliable file access. + +### 4.5 Crash Resistance ✅ + +- All network calls are wrapped in try/catch blocks. +- Error states are displayed to users with actionable recovery options (retry buttons). +- Permission denials are handled gracefully with explanatory messages. + +--- + +## 5. Onboarding & First-Run Experience + +### 5.1 Welcome Screen ✅ + +- Clean, informative welcome screen with app branding and feature highlights. +- Clear "Get Started" call-to-action leading to the login screen. +- No misleading claims or functionality promises. + +### 5.2 Login Flow ✅ + +- Server URL entry with input validation. +- Two clear authentication options (SSO and QR code). +- Error handling with user-friendly alert dialogs. +- Back navigation available from all auth screens. + +### 5.3 First-Run Permissions ✅ + +- Camera permission is requested at the point of use (when tapping Camera button), not at launch. +- Photo library permission is requested at the point of use. +- Push notification permission is requested after authentication, not before. +- All permission requests include clear usage descriptions. + +--- + +## 6. Remaining Recommendations + +### 6.1 App Store Connect Preparation + +Before submission, ensure the following are configured in App Store Connect: + +- [ ] **Privacy Policy URL** — must point to the server's `/privacy` endpoint +- [ ] **App Store description** — accurate description of features +- [ ] **Screenshots** — for iPhone and iPad (since `supportsTablet: true`) +- [ ] **App category** — "Business" or "Productivity" +- [ ] **Age rating** — complete the questionnaire (likely 4+) +- [ ] **Review notes** — provide demo server URL and test credentials for the Apple review team +- [ ] **Privacy Nutrition Labels** — declare data types collected (device ID for push notifications, authentication tokens) + +### 6.2 Accessibility Enhancements (Recommended) + +While the app includes `accessibilityRole` and `accessibilityLabel` on interactive elements, consider: + +- Adding `accessibilityHint` to buttons where the action isn't immediately obvious. +- Testing with VoiceOver to ensure all screens are fully navigable. +- Ensuring all status changes are announced to screen readers. + +### 6.3 iPad Support + +The app declares `supportsTablet: true`. Ensure: + +- UI scales appropriately on iPad screen sizes. +- Split View and Slide Over multitasking work correctly. +- Touch targets remain accessible on larger screens. + +### 6.4 Localization (Future Enhancement) + +- The app currently uses English-only strings. +- For broader App Store reach, consider localizing the app name, description, and in-app strings. + +--- + +## 7. Compliance Checklist Summary + +| Area | Status | Notes | +|------|--------|-------| +| Human Interface Guidelines | ✅ Pass | Ionicons used for platform-consistent iconography | +| App Icons & Visual Assets | ✅ Pass | All required assets provided | +| Device Data Usage | ✅ Pass | Camera, photos, notifications properly handled | +| Privacy Disclosures | ✅ Pass | Info.plist keys and privacy manifest configured | +| Background Modes | ✅ Pass | Only `remote-notification` declared | +| Restricted APIs | ✅ Pass | No private or deprecated APIs used | +| Content Standards | ✅ Pass | No misleading or inappropriate content | +| Functionality | ✅ Pass | Complete, functional app experience | +| Business Model | ✅ Pass | No IAP conflicts | +| Safety & Privacy | ✅ Pass | Account deletion available, privacy policy linked | +| Onboarding | ✅ Pass | Clear, permission-respectful first-run experience | +| Privacy Manifest | ✅ Pass | Required reason APIs declared | + +--- + +## References + +- [Apple App Store Review Guidelines](https://developer.apple.com/app-store/review/guidelines/) +- [Apple Human Interface Guidelines](https://developer.apple.com/design/human-interface-guidelines/) +- [Apple Privacy Manifest Requirements](https://developer.apple.com/documentation/bundleresources/privacy_manifest_files) +- [App Store Connect Help](https://developer.apple.com/help/app-store-connect/) diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 4e68e91a..a84ce393 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -11,10 +11,15 @@ Configuration is primarily done through environment variables specified in a `.e | **Variable** | **Description** | **Example** | |------------------------|----------------------------------------------------------|--------------------------------| | `DATABASE_URL` | Path/URL to the SQLite database (or other SQL backend). Use the [Database Wizard](/database-wizard) for guided setup. See [Database Configuration](DatabaseConfiguration.md). | `sqlite:///./app/database.db` | +| `DB_POOL_SIZE` | Number of persistent connections in the pool per worker (PostgreSQL/MySQL only; ignored for SQLite). | `10` | +| `DB_MAX_OVERFLOW` | Additional connections beyond `DB_POOL_SIZE` under burst load (PostgreSQL/MySQL only). | `20` | +| `DB_POOL_TIMEOUT` | Seconds to wait for a pool connection before raising `TimeoutError` (PostgreSQL/MySQL only). | `30` | +| `DB_POOL_RECYCLE` | Recycle connections after this many seconds to avoid stale connections (PostgreSQL/MySQL only). | `1800` | | `REDIS_URL` | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0` | | `WORKDIR` | Working directory for the application. | `/workdir` | | `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` | | `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` | +| `PUBLIC_BASE_URL` | Full public base URL including scheme (e.g., `https://docuelevate.example.com`). When set, overrides auto-detected URLs used for OAuth redirect URIs. **Required when your reverse proxy does not forward `X-Forwarded-Proto` headers.** | *(not set)* | | `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` | | `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` | | `FACTORY_RESET_ON_STARTUP` | Wipe all user data on every startup (demo/testing). | `false` | @@ -81,6 +86,28 @@ Control how the web UI queues and paces file uploads to avoid overwhelming the b **Example**: With `UPLOAD_CONCURRENCY=3` and `UPLOAD_QUEUE_DELAY_MS=500`, a directory of 5,000 files is uploaded ≈ 3 at a time with 500 ms pacing – the backend processes files at its own rate while the queue drains in the background without triggering API rate limits. +### Per-User Upload Rate Limiting + +Server-side rate limiting that prevents any single user from overwhelming the system with bulk uploads. The limiter uses a Redis-backed sliding window and dynamically adjusts limits based on system health. + +| **Variable** | **Description** | **Default** | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------|-------------| +| `UPLOAD_RATE_LIMIT_PER_USER` | Maximum uploads allowed per user within the sliding window. Effective limit may be reduced under load. | `20` | +| `UPLOAD_RATE_LIMIT_WINDOW` | Sliding window size in seconds. | `60` | + +**Health-aware dynamic limiting**: The effective per-user limit is automatically reduced when the system is under heavy load: + +| **System condition** | **Effective limit** | **Trigger** | +|--------------------------------|---------------------|--------------------------------| +| Normal | 100 % of base | Queue < 50, CPU load normal | +| Moderate load | 50 % of base | Queue 50–100 or CPU > 1.5× | +| High load | 25 % of base | Queue 100–200 or CPU > 2× | +| Critical load | 10 % of base | Queue > 200 or CPU > 3× | + +When a user exceeds the limit, the server returns **HTTP 429 Too Many Requests** with a `Retry-After` header. The browser client (see *Client-Side Upload Throttling* above) automatically pauses and retries. + +> **Note**: The limiter fails open — if Redis is unavailable, all uploads are allowed through so that a monitoring outage never blocks document processing. + ### File Upload Size Limits **Security Feature**: Control file upload sizes to prevent resource exhaustion attacks. See [SECURITY_AUDIT.md](../SECURITY_AUDIT.md#5-file-upload-size-limits) for security details. @@ -1555,14 +1582,30 @@ DocuElevate detects and flags documents that share the same content, even if the ### Exact Duplicate Detection (SHA-256) -When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the new document is stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=`) and no further processing is performed. +When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the upload is rejected immediately — no processing task is created, and the temporary file is removed from disk. The `/api/ui-upload` response returns `"status": "duplicate"` together with a `duplicate_of` object that identifies the original file. + +If the same file somehow reaches the Celery worker (e.g. via a watch-folder ingest) it is still caught there and stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=`) with no further processing. | Variable | Description | Default | |---|---|---| | `ENABLE_DEDUPLICATION` | Hash-based exact duplicate detection on ingest. | `True` | | `SHOW_DEDUPLICATION_STEP` | Show the "Check for Duplicates" step in the processing timeline UI. | `True` | -An immediate duplicate warning is also included in the `/api/ui-upload` JSON response so the frontend can alert the user before the pipeline completes. +When the upload is an exact duplicate the `/api/ui-upload` response looks like: + +```json +{ + "status": "duplicate", + "original_filename": "invoice.pdf", + "stored_filename": "abc-123.pdf", + "duplicate_of": { + "duplicate_type": "exact", + "original_file_id": 42, + "original_filename": "invoice.pdf", + "message": "This file is an exact duplicate of an already-processed document. It has not been queued for processing again." + } +} +``` ### Near-Duplicate Detection (Content Similarity) diff --git a/docs/DatabaseConfiguration.md b/docs/DatabaseConfiguration.md index 4930820f..06fd2916 100644 --- a/docs/DatabaseConfiguration.md +++ b/docs/DatabaseConfiguration.md @@ -337,18 +337,26 @@ The Helm chart includes a pre-install and pre-upgrade Job hook that runs `alembi ## Connection Pooling -SQLAlchemy manages a connection pool automatically. The defaults are suitable for most deployments. For high-concurrency or Kubernetes deployments you may want to tune: +SQLAlchemy manages a connection pool automatically. DocuElevate selects the pool +strategy based on the database backend: + +- **SQLite** — uses `NullPool` (a fresh connection per request, closed immediately). + This avoids the `QueuePool limit reached` `TimeoutError` that can occur under + concurrent load because SQLite does not benefit from persistent connection pooling. +- **PostgreSQL / MySQL** — uses a bounded `QueuePool` whose size is configurable + via environment variables. ```bash -# Optional — these are set via environment variables if you extend app/database.py -# Typical production values: -DB_POOL_SIZE=10 # Number of persistent connections per worker -DB_MAX_OVERFLOW=20 # Additional connections allowed beyond pool_size -DB_POOL_TIMEOUT=30 # Seconds to wait for a connection from the pool -DB_POOL_RECYCLE=1800 # Recycle connections after 30 minutes (avoids stale connections) +# Tune these for PostgreSQL / MySQL (ignored when using SQLite): +DB_POOL_SIZE=10 # Number of persistent connections per worker (default: 10) +DB_MAX_OVERFLOW=20 # Additional connections allowed beyond pool_size (default: 20) +DB_POOL_TIMEOUT=30 # Seconds to wait for a connection from the pool (default: 30) +DB_POOL_RECYCLE=1800 # Recycle connections after 30 minutes (default: 1800) ``` -> **Note:** These environment variables are not exposed in the default `app/config.py`. If you need to tune them, extend the database engine creation in `app/database.py`. +All backends also enable `pool_pre_ping`, which sends a lightweight health-check +before each connection is handed out. This detects stale or dropped connections +and transparently reconnects. For **PgBouncer** (external connection pooling), point `DATABASE_URL` at your PgBouncer instance and use transaction-mode pooling: @@ -512,4 +520,13 @@ Then retry `alembic upgrade head`. Either increase `max_connections` in `postgresql.conf` or add PgBouncer in front of PostgreSQL. The default PostgreSQL `max_connections` is `100`; reduce `DB_POOL_SIZE` per worker to stay within this limit. +### "QueuePool limit reached" TimeoutError (SQLite) + +If you see `TimeoutError: QueuePool limit of size 5 overflow 10 reached`, your +deployment is still running an older version of DocuElevate that used a bounded +connection pool for SQLite. Upgrade to the latest release — SQLite now uses +`NullPool`, which eliminates this error entirely. If you are already on the +latest version and are still seeing pool exhaustion, ensure you are not +overriding the engine creation manually. + For more help, see the [Troubleshooting Guide](Troubleshooting.md). diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index 94dc634f..43345465 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -349,16 +349,18 @@ workdir: ## Scaling +DocuElevate is designed for horizontal scaling. Both API and worker pods are stateless and can be scaled independently. + ### Docker Compose -Add more worker containers: +Scale workers (task processing) and API pods (request handling) independently: -```yaml -worker: - deploy: - replicas: 3 +```bash +docker compose up -d --scale worker=3 --scale api=2 ``` +> **Note:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. Do not scale it. It publishes periodic tasks to the Redis broker; workers pick them up. + ### Kubernetes / Helm Enable HPA: @@ -377,13 +379,15 @@ worker: maxReplicas: 10 ``` +The Helm chart deploys a separate **beat** pod (always 1 replica, `Recreate` strategy) so that scheduled tasks are never duplicated when workers scale. + --- ## Monitoring - **Docker Compose**: `docker-compose logs -f`, `docker stats` - **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f` -- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed. +- **Prometheus / Grafana**: Scrape the `/api/diagnostic/healthz/ready` endpoint for readiness; add custom metrics as needed. - **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring. --- diff --git a/docs/DropboxSetup.md b/docs/DropboxSetup.md index 8923a595..b0e44852 100644 --- a/docs/DropboxSetup.md +++ b/docs/DropboxSetup.md @@ -128,7 +128,31 @@ If you encounter issues with Dropbox integration: 1. **Authentication Errors**: Make sure your App Key and App Secret are correct 2. **Token Expired**: Click "Refresh Token" button on the setup page to obtain a new token 3. **Folder Permissions**: Ensure your app has the correct permissions enabled for file operations -4. **Invalid Redirect URI**: Verify that the redirect URI in your app settings matches the one used in the authentication flow +4. **Invalid Redirect URI**: See section below for the most common cause and fix. 5. **Rate Limiting**: Dropbox API has rate limits; if exceeded, wait and try again +### Fixing "Invalid redirect_uri" Error + +This error appears on the Dropbox authorization page when the redirect URI in the OAuth request does not match any URI registered in your Dropbox app console. + +**Most common cause**: The application is deployed behind a reverse proxy (Traefik, Nginx, Caddy) that does **not** forward the `X-Forwarded-Proto: https` header to DocuElevate. Without this header, the server cannot determine that it is being accessed over HTTPS and may construct an `http://` redirect URI, while the registered URI in Dropbox is `https://`. + +**Fix**: + +Option 1 – Configure your proxy to forward `X-Forwarded-Proto`: + +```nginx +proxy_set_header X-Forwarded-Proto $scheme; +``` + +Option 2 – Set `PUBLIC_BASE_URL` in your environment (recommended for most deployments): + +```bash +PUBLIC_BASE_URL=https://docuelevate.example.com +``` + +When `PUBLIC_BASE_URL` is set, DocuElevate uses it directly for all OAuth redirect URIs instead of trying to infer the scheme from request headers. This is the most reliable option. + +After setting `PUBLIC_BASE_URL`, ensure the Dropbox app console redirect URI matches exactly (e.g., `https://docuelevate.example.com/dropbox-callback`). The setup wizard at `/dropbox-setup` will show you the exact URI to register. + For more general configuration issues, see the [Configuration Troubleshooting Guide](ConfigurationTroubleshooting.md). diff --git a/docs/KubernetesDeployment.md b/docs/KubernetesDeployment.md index 94247873..c23ddc81 100644 --- a/docs/KubernetesDeployment.md +++ b/docs/KubernetesDeployment.md @@ -373,6 +373,8 @@ worker: replicaCount: 4 ``` +> **Beat scheduler:** The Helm chart deploys a dedicated `beat` pod (always exactly 1 replica with `Recreate` strategy) that publishes periodic tasks to the Redis broker. Workers consume these tasks — scaling workers does **not** duplicate scheduled jobs. + ### Horizontal Pod Autoscaler ```yaml @@ -433,24 +435,30 @@ externalRedis: ### Kubernetes Probes -The Helm chart configures liveness and readiness probes on the API pods via `/api/health`. Default settings: +The Helm chart configures **unauthenticated** liveness and readiness probes on the API pods so kubelet can reach them without credentials. Default settings: ```yaml api: livenessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/live port: 8000 initialDelaySeconds: 30 - periodSeconds: 30 + periodSeconds: 20 readinessProbe: httpGet: - path: /api/health + path: /api/diagnostic/healthz/ready port: 8000 - initialDelaySeconds: 10 + initialDelaySeconds: 15 periodSeconds: 10 ``` +| Endpoint | Auth | Purpose | +|----------|------|---------| +| `/api/diagnostic/healthz/live` | None | Lightweight liveness check — returns 200 if the process is running | +| `/api/diagnostic/healthz/ready` | None | Readiness check — verifies database and Redis connectivity (503 when DB is down) | +| `/api/diagnostic/health` | Required | Full health status for monitoring dashboards (Grafana, Uptime Kuma) | + ### Prometheus Scraping Add annotations to expose metrics (if using a Prometheus-compatible exporter): diff --git a/docs/MobileApp.md b/docs/MobileApp.md index 8b8e6f79..07ff077b 100644 --- a/docs/MobileApp.md +++ b/docs/MobileApp.md @@ -12,9 +12,14 @@ DocuElevate includes a native mobile application for iOS and Android built with | Auto-generated API token | ✅ | ✅ | | Camera capture → upload | ✅ | ✅ | | File picker upload | ✅ | ✅ | +| Multi-image selection from library | ✅ | ✅ | | Share Sheet / Share Intent | ✅ | ✅ | | Push notifications | ✅ | ✅ | -| Document list | ✅ | ✅ | +| Document list with search | ✅ | ✅ | +| File detail view with processing logs | ✅ | ✅ | +| Pre-login legal pages (GDPR) | ✅ | ✅ | +| Localization (EN, DE, ES, FR, IT) | ✅ | ✅ | +| Language selection | ✅ | ✅ | | Dark mode | ✅ | ✅ | ## Getting Started (Development) @@ -171,8 +176,8 @@ curl -X DELETE -H "Authorization: Bearer " https://your-server/api/mobile 1. Open the **Upload** tab. 2. Tap **Photos**. -3. Select an existing photo from the device's photo library. -4. The image is uploaded and queued for processing. +3. Select one or more photos from the device's photo library (multi-selection is supported). +4. All selected images are uploaded and queued for processing. ### File Picker @@ -198,6 +203,32 @@ The app registers itself as a share target so any file can be sent directly to D The URL may arrive as a standard `file://` path **or** under the app's custom `docuelevate://` scheme (e.g. `docuelevate://private/var/mobile/Library/…/file.pdf`). The root layout detects the custom-scheme form and rewrites it to a `file://` URL before forwarding it to the Upload screen through `ShareContext`. +##### Handling "unmatched route" errors from "Open In…" + +iOS sometimes delivers the file path under the `docuelevate://` scheme, e.g.: + +``` +docuelevate://private/var/mobile/Library/Mobile Documents/…/Invoice.pdf +``` + +expo-router strips the scheme and tries to match `/private/var/mobile/…` as an in-app route. Because no such route exists, it previously threw an **"unmatched route docuelevate://"** error and the upload never completed. + +The fix is a catch-all `+not-found.tsx` route (see `mobile/app/+not-found.tsx`). When expo-router cannot match the path, it renders this screen instead. The screen detects that the path is a filesystem path rather than a real in-app route, adds the file directly to `ShareContext`, and redirects to the Upload tab. `UploadScreen` picks up the pending file and begins uploading automatically. The `Linking` listener in the root layout may also fire for the same URL; `ShareContext.addPendingFile` deduplicates by URI so the file is only uploaded once. + +##### File accessibility and local caching + +Shared files may reference paths outside the app's sandbox or use security-scoped URLs that React Native's `fetch` cannot read directly. To guarantee reliable uploads: + +- **`LSSupportsOpeningDocumentsInPlace`** is set to `false` in `app.json`, which tells iOS to copy shared files into the app's `Documents/Inbox` directory before handing them to the app. +- **`UploadScreen`** uses `expo-file-system` (`FileSystem.copyAsync`) to copy any `file://` URI that is outside the app's cache/documents directory to a local cache path before uploading. This ensures the file is readable regardless of its origin. +- **MIME type inference**: Both `+not-found.tsx` and the `Linking` handler in `_layout.tsx` infer the MIME type from the file extension (e.g. `.pdf` → `application/pdf`) so the server receives a correct `Content-Type` instead of `application/octet-stream`. + +##### iOS Action / Share Extension (future enhancement) + +Apps like DeepL ("Translate in DeepL") and Microsoft Word ("Convert to Word") appear as **Action Extensions** in the iOS share sheet — a system-level feature that requires a separate Xcode target built with Swift or Objective-C. A proper Action Extension runs in its own process and must share authentication credentials with the main app via an iOS **App Group** (shared keychain / shared container). + +This level of iOS-native integration is a planned future enhancement. Until it is available, the recommended workflow is the current one: tap **Share → DocuElevate** (the app appears in the "Open With" row of the share sheet via `CFBundleDocumentTypes`). + #### Android implementation `app.json` declares `ACTION_SEND` and `ACTION_SEND_MULTIPLE` intent filters for `mimeType: "*/*"` in the `android.intentFilters` section. Incoming content URIs are received the same way as on iOS. @@ -215,6 +246,75 @@ If a file upload fails (e.g. due to network issues or a server error), the faile The retry re-uses the original file URI so no re-selection is needed. +## Document Search + +The **Files** tab includes a search bar at the top that lets users search through their processed documents by filename. Searches are debounced (400ms) to avoid excessive API calls. Clear the search with the ✕ button to return to the full list. + +## File Detail View + +Tapping any document in the **Files** tab opens a detail view showing: + +- **File metadata**: filename, file size, MIME type, upload date, and file hash +- **Processing status**: current status with a colour-coded icon +- **Processing log**: chronological list of processing steps with individual status indicators and timestamps + +Pull-to-refresh updates the detail view. This replicates the web interface at `/files/{id}` and `/files/{id}/detail` in a mobile-friendly layout. + +## Legal & Compliance + +### GDPR & Apple App Store Compliance + +Privacy Policy, Terms of Service, and Imprint links are accessible **before login** from both the **Welcome Screen** and the **Login Screen**. This ensures compliance with: + +- **GDPR** (General Data Protection Regulation) – users must be able to review the privacy policy before providing personal data +- **Apple App Store Review Guidelines** – apps must provide accessible privacy information before account creation + +Post-login, the same links are available in the **Profile** tab under the "Legal" section. + +## Localization (i18n) + +The mobile app supports five languages with automatic device-locale detection: + +| Language | Code | Status | +|----------|------|--------| +| English | `en` | ✅ Complete | +| German (Deutsch) | `de` | ✅ Complete | +| Spanish (Español) | `es` | ✅ Complete | +| French (Français) | `fr` | ✅ Complete | +| Italian (Italiano) | `it` | ✅ Complete | + +### How it works + +Language priority (highest to lowest): + +1. **Server preference** — `preferred_language` returned by `GET /api/mobile/whoami` on login or app resume. Allows a language set on the desktop web interface to propagate to mobile automatically. +2. **AsyncStorage** — the last language explicitly selected on the device, used as an offline fallback when the server is unreachable. +3. **Device locale** — detected via `expo-localization` on first launch. +4. **English** — final fallback when none of the above match a supported locale. + +When a user selects a language on mobile the choice is: +- Applied immediately to all screens (via `LocaleContext`) +- Persisted locally to AsyncStorage +- Synced to the server via `POST /api/i18n/language` (fire-and-forget), so the next desktop login reflects the same preference. + +> **Note**: If the server's preferred language is not supported by the mobile app (e.g. a locale added to the web frontend but not yet translated for mobile), the mobile app falls back to the next priority in the list above. + +### Adding a new language + +1. Create a new translation file in `mobile/src/i18n/` (e.g. `pt.json` for Portuguese) +2. Copy the structure from `en.json` and translate all values +3. Import the new file in `mobile/src/i18n/index.ts` +4. Add it to the `translations` object and `getSupportedLanguages()` array + +## User Settings + +The **Profile** tab includes a **Settings** section where users can: + +- **Change language**: Select from the supported languages (English, German, Spanish, French, Italian) +- View server connection details +- Access legal documents (Privacy Policy, Terms of Service, Imprint) +- Sign out or delete their account + ## Mobile API Endpoints The backend exposes a dedicated `/api/mobile/` namespace: @@ -225,7 +325,8 @@ The backend exposes a dedicated `/api/mobile/` namespace: | `POST` | `/api/mobile/register-device` | Bearer | Register Expo push token | | `GET` | `/api/mobile/devices` | Bearer | List registered devices | | `DELETE` | `/api/mobile/devices/{id}` | Bearer | Deactivate a device | -| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile | +| `GET` | `/api/mobile/whoami` | Bearer | Get current user profile (includes `preferred_language`) | +| `POST` | `/api/i18n/language` | Bearer | Sync language preference to server | All other API endpoints (file upload, file listing, etc.) work with Bearer token authentication. @@ -269,7 +370,7 @@ Re-registering the same token is safe (idempotent). ### GET /api/mobile/whoami -Returns the current user's profile. +Returns the current user's profile, including the server-stored language preference. **Response (200):** ```json @@ -278,10 +379,15 @@ Returns the current user's profile. "display_name": "John Doe", "email": "john@example.com", "avatar_url": "https://www.gravatar.com/avatar/...", - "is_admin": false + "is_admin": false, + "preferred_language": "de" } ``` +`preferred_language` is `null` when no preference has been saved. The mobile +app applies this value on login / app resume, falling back to AsyncStorage and +then the device locale when it is `null` or unsupported. + ## Configuration No server-side configuration is required to enable the mobile app. The Expo push notification routing does not need FCM or APNs credentials on the server. @@ -320,8 +426,20 @@ mobile/ │ ├── LoginScreen.tsx # Server URL + SSO button + QR code scanner │ ├── QRScannerScreen.tsx # Camera-based QR code scanner for login │ ├── UploadScreen.tsx # Camera capture + photo library + file picker - │ ├── FilesScreen.tsx # Processed document list - │ └── ProfileScreen.tsx # User profile + sign out + │ ├── FilesScreen.tsx # Processed document list with search + │ ├── FileDetailScreen.tsx # File detail view with processing logs + │ ├── ProfileScreen.tsx # User profile + settings + sign out + │ └── WelcomeScreen.tsx # Pre-login welcome with legal links + ├── i18n/ # Localization (i18n) + │ ├── index.ts # i18n module (locale detection, t() function) + │ ├── en.json # English translations + │ ├── de.json # German translations + │ ├── es.json # Spanish translations + │ ├── fr.json # French translations + │ └── it.json # Italian translations + ├── utils/ + │ ├── mimeTypes.ts # MIME type mapping for file extensions + │ └── normalizeUri.ts # URI normalization for deduplication └── services/ └── api.ts # DocuElevate REST API client ``` @@ -408,3 +526,4 @@ eas build --platform ios - [API Documentation](./API.md) - [Configuration Guide](./ConfigurationGuide.md) - [Deployment Guide](./DeploymentGuide.md) +- [Apple App Store Compliance Audit](./AppleAppStoreCompliance.md) diff --git a/docs/ProductionReadiness.md b/docs/ProductionReadiness.md index 2f93c7c5..a938e39e 100644 --- a/docs/ProductionReadiness.md +++ b/docs/ProductionReadiness.md @@ -34,7 +34,7 @@ Use this checklist to track readiness before going live. - [ ] **Redis** — Running and accessible only from internal network - [ ] **Meilisearch** — Running and accessible only from internal network - [ ] **Worker replicas** — At least 2 workers configured for redundancy -- [ ] **Monitoring** — `/api/health` polled by uptime checker +- [ ] **Monitoring** — `/api/diagnostic/health` polled by uptime checker - [ ] **Backups** — Automated backup of database, workdir, and Meilisearch data - [ ] **Log retention** — Logs shipped to a persistent store or aggregator - [ ] **Secrets management** — API keys not committed to source control @@ -285,22 +285,24 @@ For SSO/OIDC (Authentik, Keycloak, Auth0, etc.) see the [Authentication Setup Gu ### Docker Compose -Use the `deploy.replicas` setting (requires Docker Swarm mode) or simply run multiple workers: - -```yaml -worker: - deploy: - replicas: 3 -``` - -Or scale after deployment: +Scale workers independently: ```bash -docker-compose up -d --scale worker=3 +docker compose up -d --scale worker=3 ``` Each worker processes tasks from the Celery queue independently. Ensure the shared `workdir` volume is accessible from all worker containers. +> **Important:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. It is defined as a dedicated service in `docker-compose.yaml` with a fixed `container_name`. Do not scale it. + +### Scaling the API + +API pods are fully stateless (sessions use encrypted cookies, not server-side state) and can be scaled behind a load balancer: + +```bash +docker compose up -d --scale api=3 +``` + ### Kubernetes (Helm) ```yaml @@ -339,11 +341,32 @@ celery -A app.celery_worker worker -Q default,celery --concurrency=2 ### Health Check Endpoint -DocuElevate exposes `/api/health` for readiness probing. Configure your uptime monitor to poll this endpoint: +DocuElevate exposes three health-related endpoints: + +| Endpoint | Auth | Purpose | +|----------|------|---------| +| `GET /api/diagnostic/healthz/live` | None | Lightweight liveness probe — returns 200 if the process is running | +| `GET /api/diagnostic/healthz/ready` | None | Readiness probe — checks database and Redis (503 when DB is down) | +| `GET /api/diagnostic/health` | Required | Full status for monitoring dashboards (Grafana, Uptime Kuma) | + +For **Kubernetes probes**, use the unauthenticated endpoints: + +```yaml +livenessProbe: + httpGet: + path: /api/diagnostic/healthz/live + port: 8000 +readinessProbe: + httpGet: + path: /api/diagnostic/healthz/ready + port: 8000 +``` + +For **uptime monitors** (Uptime Kuma, Grafana, etc.), use the authenticated endpoint: ```bash -curl http://docuelevate.example.com/api/health -# Expected: {"status": "ok", ...} +curl http://docuelevate.example.com/api/diagnostic/health +# Expected: {"status": "healthy", ...} ``` Set `UPTIME_KUMA_URL` to your Uptime Kuma push URL for heartbeat monitoring: @@ -502,4 +525,4 @@ For a dedicated Kubernetes deployment guide, including architecture diagrams, PV - **Image Pull Policy**: Use `IfNotPresent` in production with pinned image tags (not `latest`) for reproducible deployments. -- **Liveness & Readiness Probes**: Already configured in the Helm chart via `/api/health`. Verify they are tuned to your startup time. +- **Liveness & Readiness Probes**: Already configured in the Helm chart via unauthenticated endpoints (`/api/diagnostic/healthz/live` and `/api/diagnostic/healthz/ready`). Verify they are tuned to your startup time. diff --git a/docs/UserGuide.md b/docs/UserGuide.md index c216e07a..8f791cfb 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -87,7 +87,7 @@ DocuElevate provides multiple convenient ways to upload documents to the system. #### Supported File Types - **Documents**: PDF, Word (.doc, .docx), Excel (.xls, .xlsx), PowerPoint (.ppt, .pptx) -- **Images**: JPEG, PNG, GIF, BMP, TIFF, WebP, SVG +- **Images**: JPEG, PNG, GIF, BMP, TIFF, WebP, SVG, HEIC, HEIF - **Text**: Plain text (.txt), CSV, RTF, HTML, XML, Markdown - **Maximum file size**: 500MB per file diff --git a/frontend/static/js/upload.js b/frontend/static/js/upload.js index 0e25ce7b..b8ecd62a 100644 --- a/frontend/static/js/upload.js +++ b/frontend/static/js/upload.js @@ -433,9 +433,17 @@ function _uploadSingleFile(file, progressBar, statusEl, onTerminal) { if (xhr.status === 200) { const result = JSON.parse(xhr.responseText); progressBar.style.width = '100%'; - progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full'; - statusEl.textContent = `Success: Task ID: ${result.task_id}`; - statusEl.className = 'text-xs text-green-600 mt-1'; + + if (result.status === 'duplicate' && result.duplicate_of) { + // Exact duplicate – no processing task was created + progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full'; + statusEl.textContent = `Duplicate – already processed (file #${result.duplicate_of.original_file_id})`; + statusEl.className = 'text-xs text-yellow-600 mt-1'; + } else { + progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full'; + statusEl.textContent = `Success: Task ID: ${result.task_id}`; + statusEl.className = 'text-xs text-green-600 mt-1'; + } _onUploadSuccess(); onTerminal(); resolve({ rateLimited: false, retryAfterSeconds: 0 }); diff --git a/frontend/templates/api_tokens.html b/frontend/templates/api_tokens.html index 5e4633cf..b4facee8 100644 --- a/frontend/templates/api_tokens.html +++ b/frontend/templates/api_tokens.html @@ -33,6 +33,20 @@ aria-required="true" /> +
+ + +
+ + + + @@ -209,9 +258,10 @@ function apiTokens() { tokens: [], loading: true, creating: false, - revoking: null, + acting: null, error: null, newTokenName: '', + newTokenExpiresDays: null, newlyCreatedToken: null, copied: false, baseUrl: window.location.origin, @@ -238,13 +288,15 @@ function apiTokens() { this.error = null; this.newlyCreatedToken = null; try { + const body = { name: this.newTokenName.trim() }; + if (this.newTokenExpiresDays) body.expires_in_days = parseInt(this.newTokenExpiresDays); const res = await fetch('/api/api-tokens/', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken, }, - body: JSON.stringify({ name: this.newTokenName.trim() }), + body: JSON.stringify(body), }); if (!res.ok) { const data = await res.json().catch(() => ({})); @@ -253,6 +305,7 @@ function apiTokens() { const data = await res.json(); this.newlyCreatedToken = data.token; this.newTokenName = ''; + this.newTokenExpiresDays = null; await this.loadTokens(); } catch (e) { this.error = e.message; @@ -263,7 +316,7 @@ function apiTokens() { async revokeToken(token) { if (!confirm(`Revoke token "${token.name}"? This cannot be undone.`)) return; - this.revoking = token.id; + this.acting = token.id; this.error = null; try { const res = await fetch(`/api/api-tokens/${token.id}`, { @@ -278,10 +331,66 @@ function apiTokens() { } catch (e) { this.error = e.message; } finally { - this.revoking = null; + this.acting = null; } }, + async reactivateToken(token) { + if (!confirm({{ _("api_tokens.reactivate_confirm") | tojson }})) return; + this.acting = token.id; + this.error = null; + try { + const res = await fetch(`/api/api-tokens/${token.id}/reactivate`, { + method: 'POST', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to reactivate token'); + } + await this.loadTokens(); + } catch (e) { + this.error = e.message; + } finally { + this.acting = null; + } + }, + + async deleteToken(token) { + if (!confirm({{ _("api_tokens.delete_confirm") | tojson }})) return; + this.acting = token.id; + this.error = null; + try { + const res = await fetch(`/api/api-tokens/${token.id}`, { + method: 'DELETE', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to delete token'); + } + await this.loadTokens(); + } catch (e) { + this.error = e.message; + } finally { + this.acting = null; + } + }, + + tokenStatusClass(token) { + if (!token.is_active) return 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400'; + if (token.expires_at && new Date(token.expires_at) < new Date()) + return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-400'; + return 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400'; + }, + + tokenStatusLabel(token) { + if (!token.is_active) return '{{ _("api_tokens.status_revoked") }}'; + if (token.expires_at && new Date(token.expires_at) < new Date()) + return '{{ _("api_tokens.status_expired") }}'; + return '{{ _("api_tokens.status_active") }}'; + }, + copyToken() { if (this.newlyCreatedToken) { navigator.clipboard.writeText(this.newlyCreatedToken); diff --git a/frontend/templates/devices.html b/frontend/templates/devices.html index 40e0a255..b640d950 100644 --- a/frontend/templates/devices.html +++ b/frontend/templates/devices.html @@ -3,7 +3,7 @@ {% block title %}{{ _("devices.page_title") }}{% endblock %} {% block content %} -
+
@@ -48,55 +48,85 @@ - - - - - - + + + + + + @@ -182,16 +212,31 @@ x-show="device.is_active" type="button" @click="deactivateDevice(device)" - :disabled="deactivatingDevice === device.id" + :disabled="actingDevice === device.id" class="flex-shrink-0 inline-flex items-center px-3 py-1.5 text-sm font-medium text-red-600 hover:text-red-800 dark:text-red-400 dark:hover:text-red-300 hover:bg-red-50 dark:hover:bg-red-900/20 rounded-md focus:outline-none focus:ring-2 focus:ring-red-500 disabled:opacity-50 transition-colors" style="min-height:36px; min-width:44px;" :aria-label="'{{ _('devices.deactivate_device') }} ' + (device.device_name || 'device')" > - + {{ _("devices.deactivate_device") }} + + @@ -242,8 +287,8 @@ function devicesPage() { devices: [], loadingTokens: true, loadingDevices: true, - revokingToken: null, - deactivatingDevice: null, + actingToken: null, + actingDevice: null, tokenError: null, deviceError: null, banner: { visible: false, error: false, message: '' }, @@ -286,7 +331,7 @@ function devicesPage() { async revokeToken(token) { if (!confirm({{ _("devices.confirm_revoke_token") | tojson }})) return; - this.revokingToken = token.id; + this.actingToken = token.id; try { const res = await fetch(`/api/api-tokens/${token.id}`, { method: 'DELETE', @@ -301,19 +346,61 @@ function devicesPage() { } catch (e) { this._showBanner(e.message, true); } finally { - this.revokingToken = null; + this.actingToken = null; + } + }, + + async reactivateMobileToken(token) { + if (!confirm({{ _("devices.reactivate_token_confirm") | tojson }})) return; + this.actingToken = token.id; + try { + const res = await fetch(`/api/api-tokens/${token.id}/reactivate`, { + method: 'POST', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to reactivate token'); + } + await this.loadMobileTokens(); + this._showBanner({{ _("devices.token_reactivated_success") | tojson }}, false); + } catch (e) { + this._showBanner(e.message, true); + } finally { + this.actingToken = null; + } + }, + + async deleteMobileToken(token) { + if (!confirm({{ _("devices.delete_token_confirm") | tojson }})) return; + this.actingToken = token.id; + try { + const res = await fetch(`/api/api-tokens/${token.id}`, { + method: 'DELETE', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to delete token'); + } + await this.loadMobileTokens(); + this._showBanner({{ _("devices.token_deleted_success") | tojson }}, false); + } catch (e) { + this._showBanner(e.message, true); + } finally { + this.actingToken = null; } }, async deactivateDevice(device) { if (!confirm({{ _("devices.confirm_deactivate_device") | tojson }})) return; - this.deactivatingDevice = device.id; + this.actingDevice = device.id; try { const res = await fetch(`/api/mobile/devices/${device.id}`, { method: 'DELETE', headers: { 'X-CSRF-Token': csrfToken }, }); - if (!res.ok && res.status !== 204) { + if (!res.ok) { const data = await res.json().catch(() => ({})); throw new Error(data.detail || 'Failed to remove device'); } @@ -322,7 +409,28 @@ function devicesPage() { } catch (e) { this._showBanner(e.message, true); } finally { - this.deactivatingDevice = null; + this.actingDevice = null; + } + }, + + async deleteDevice(device) { + if (!confirm({{ _("devices.delete_device_confirm") | tojson }})) return; + this.actingDevice = device.id; + try { + const res = await fetch(`/api/mobile/devices/${device.id}`, { + method: 'DELETE', + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.detail || 'Failed to delete device'); + } + await this.loadDevices(); + this._showBanner({{ _("devices.device_deleted_success") | tojson }}, false); + } catch (e) { + this._showBanner(e.message, true); + } finally { + this.actingDevice = null; } }, diff --git a/frontend/templates/dropbox.html b/frontend/templates/dropbox.html index a94a5b96..6c41bd96 100644 --- a/frontend/templates/dropbox.html +++ b/frontend/templates/dropbox.html @@ -104,7 +104,7 @@

Step 3: Set OAuth 2 Redirect URI

  1. In your app's settings page, go to the "OAuth 2" section
    2. -
  2. Add a redirect URI: {{ request.url.scheme }}://{{ request.url.netloc }}/dropbox-callback
    3. +
  3. Add a redirect URI: {{ callback_url }}
  4. Click "Add" to save the redirect URI
@@ -116,6 +116,34 @@
+ {% if user_mode and global_creds_available %} + +
+
+ +
+

Using shared application credentials

+

Your administrator has enabled shared Dropbox app credentials. You can authorize your account without supplying your own App Key and Secret.

+
+
+
+ + {% if folder_path %} +
+

Target folder (from integration settings)

+

{{ folder_path }}

+
+ {% endif %} + +
+ +
+ {% else %}
@@ -146,6 +174,7 @@ Start Authentication Flow
+ {% endif %} {% if not user_mode %} @@ -268,6 +297,10 @@ DROPBOX_FOLDER={{ folder_path|default('/Documents/Uploads', true) }} document.addEventListener('DOMContentLoaded', function() { const userMode = {{ 'true' if user_mode else 'false' }}; + const globalCredsAvailable = {{ 'true' if global_creds_available else 'false' }}; + // Redirect URI for OAuth: prefer server-provided value (respects PUBLIC_BASE_URL), + // fall back to window.location.origin for resilience. + const dropboxCallbackUrl = {{ callback_url | tojson }} || (window.location.origin + "/dropbox-callback"); // Store integration_id if provided (for per-user OAuth flow) const integrationId = "{{ integration_id or '' }}"; @@ -277,6 +310,7 @@ document.addEventListener('DOMContentLoaded', function() { // Elements const startAuthFlowBtn = document.getElementById('start-auth-flow'); + const startAuthFlowGlobalBtn = document.getElementById('start-auth-flow-global'); const testTokenBtn = document.getElementById('test-token'); const refreshTokenBtn = document.getElementById('refresh-token-btn'); const tokenStatus = document.getElementById('token-status'); @@ -328,11 +362,38 @@ document.addEventListener('DOMContentLoaded', function() { } }); + // Global-credentials "Authorize with Dropbox" button (user mode, admin-provided creds) + if (startAuthFlowGlobalBtn) { + startAuthFlowGlobalBtn.addEventListener('click', async function() { + startAuthFlowGlobalBtn.disabled = true; + startAuthFlowGlobalBtn.innerHTML = '⟳ Redirecting…'; + try { + const resp = await fetch('/api/dropbox/global-authorize-url'); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + showModal('error', 'Error', err.detail || 'Could not retrieve authorization URL.'); + startAuthFlowGlobalBtn.disabled = false; + startAuthFlowGlobalBtn.innerHTML = 'Authorize with Dropbox'; + return; + } + const data = await resp.json(); + // Signal to the callback that global credentials should be used for the exchange + sessionStorage.setItem('dropbox_use_global_creds', 'true'); + window.location.href = data.authorize_url; + } catch (err) { + showModal('error', 'Network Error', err.message || 'Unknown error'); + startAuthFlowGlobalBtn.disabled = false; + startAuthFlowGlobalBtn.innerHTML = 'Authorize with Dropbox'; + } + }); + } + // Start Authentication Flow button click - startAuthFlowBtn.addEventListener('click', function() { + if (startAuthFlowBtn) { + startAuthFlowBtn.addEventListener('click', function() { const appKey = document.getElementById('app-key').value.trim(); const appSecret = appSecretInput.value.trim(); - const redirectUri = window.location.origin + "/dropbox-callback"; + const redirectUri = dropboxCallbackUrl; if (!appKey) { showModal('error', 'Validation Error', 'Please enter your App Key'); @@ -362,6 +423,7 @@ document.addEventListener('DOMContentLoaded', function() { // Redirect the user to the Dropbox login page window.location.href = authUrl; }); + } // end if (startAuthFlowBtn) // Test Token button click (admin mode only) if (testTokenBtn) { diff --git a/frontend/templates/dropbox_callback.html b/frontend/templates/dropbox_callback.html index fd7b0f31..5b2efd3f 100644 --- a/frontend/templates/dropbox_callback.html +++ b/frontend/templates/dropbox_callback.html @@ -94,6 +94,7 @@ document.addEventListener('DOMContentLoaded', function() { const code = "{{ code }}"; // Get credentials from session storage (these take precedence over server-provided values) + const useGlobalCreds = sessionStorage.getItem('dropbox_use_global_creds') === 'true'; const appKey = sessionStorage.getItem('dropbox_app_key') || "{{ app_key_value }}"; const appSecret = sessionStorage.getItem('dropbox_app_secret') || "{{ app_secret_value }}"; const folderPath = sessionStorage.getItem('dropbox_folder_path') || "{{ folder_path }}" || '/Documents/Uploads'; @@ -107,20 +108,100 @@ document.addEventListener('DOMContentLoaded', function() { } } - const redirectUri = window.location.origin + "/dropbox-callback"; + // Use server-provided callback URL (respects PUBLIC_BASE_URL) with fallback to window.location.origin + const redirectUri = {{ callback_url | tojson }} || (window.location.origin + "/dropbox-callback"); // Automatically exchange the code for a refresh token if (code) { - if (!appKey || !appSecret) { - showError("Missing App Key or App Secret. Please go back to the setup page and try again."); - return; + if (useGlobalCreds) { + // Global credentials: the server handles the exchange using the admin app secret + exchangeCodeGlobal(code, redirectUri); + } else { + if (!appKey || !appSecret) { + showError("Missing App Key or App Secret. Please go back to the setup page and try again."); + return; + } + exchangeCode(code, appKey, appSecret, redirectUri); } - - exchangeCode(code, appKey, appSecret, redirectUri); } else { showError("No authorization code was found in the URL"); } + function exchangeCodeGlobal(code, redirectUri) { + const formData = new FormData(); + formData.append('code', code); + formData.append('redirect_uri', redirectUri); + + document.getElementById('processing-message').innerHTML = + '

Exchanging authorization code using shared credentials…

'; + + fetch('/api/dropbox/exchange-token-global', { + method: 'POST', + body: formData + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to exchange token'); + }); + } + return response.json(); + }) + .then(data => { + if (data.refresh_token) { + const resolvedAppKey = data.app_key || ''; + if (integrationId) { + const creds = { + refresh_token: data.refresh_token, + // Store public app_key with the integration (no secret stored browser-side) + app_key: resolvedAppKey, + // Flag so the backend knows to use global app_secret for future operations + use_global_app_secret: true, + }; + + const body = { credentials: creds }; + + document.getElementById('processing-message').innerHTML = + '

Saving credentials to your integration…

'; + + return fetch(`/api/integrations/${integrationId}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }).then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error('Failed to save credentials: ' + (err.detail || 'Unknown error')); + }); + } + return response.json(); + }).then(() => { + // Clean up + sessionStorage.removeItem('dropbox_use_global_creds'); + sessionStorage.removeItem('oauth_integration_id'); + + document.getElementById('processing-message').innerHTML = + '

✓ Dropbox authorized successfully!

' + + '

Redirecting to Integrations…

'; + document.querySelector('.animate-spin').parentNode.classList.add('hidden'); + setTimeout(() => { window.location.href = '/integrations'; }, 2000); + }); + } + // Global admin flow — the exchange-token-global endpoint is intended for + // per-user integrations, so reaching here without an integrationId is unexpected. + console.warn('dropbox_callback: global creds flow reached without integration_id'); + sessionStorage.removeItem('dropbox_use_global_creds'); + showSuccess(data.refresh_token, resolvedAppKey, '', folderPath); + setTimeout(() => { window.location.href = '/status'; }, 10000); + } else { + throw new Error('No refresh token was received from the server'); + } + }) + .catch(error => { + showError(error.message); + }); + } + function exchangeCode(code, appKey, appSecret, redirectUri) { const formData = new FormData(); formData.append('client_id', appKey); diff --git a/frontend/templates/integrations_dashboard.html b/frontend/templates/integrations_dashboard.html index 335b6745..5de07de2 100644 --- a/frontend/templates/integrations_dashboard.html +++ b/frontend/templates/integrations_dashboard.html @@ -1392,7 +1392,7 @@ function integrationsDashboard() { body: JSON.stringify({ integration_type: intg.integration_type, config: intg.config, - credentials: creds, + credentials: creds.credentials, }), }); const data = await resp.json(); diff --git a/frontend/templates/qr_login.html b/frontend/templates/qr_login.html index 7b29f594..a2ecac26 100644 --- a/frontend/templates/qr_login.html +++ b/frontend/templates/qr_login.html @@ -32,7 +32,13 @@ id="qr-container" aria-label="{{ _('qr_login.description') }}" > - + {{ _('qr_login.description') }}

{{ _("qr_login.description") }} @@ -110,8 +116,7 @@ - - +

{{ _("devices.col_device") }}{{ _("devices.col_token_prefix") }}{{ _("devices.col_created") }}{{ _("devices.col_last_used") }}{{ _("devices.col_status") }}{{ _("common.actions") }}{{ _("devices.col_device") }}{{ _("devices.col_token_prefix") }}{{ _("devices.col_created") }}{{ _("devices.col_last_used") }}{{ _("devices.col_status") }}{{ _("common.actions") }}