Merge remote-tracking branch 'origin/main' into copilot/fix-watch-folder-settings

# Conflicts:
#	app/views/dropbox.py
#	frontend/templates/dropbox.html
#	frontend/templates/integrations_dashboard.html
#	tests/test_api_dropbox.py
#	tests/test_views_dropbox.py

app/views/__init__.py

@@ -6,8 +6,11 @@ from fastapi import APIRouter
 
 from app.views.admin_users import router as admin_users_router
 from app.views.api_tokens import router as api_tokens_router
+from app.views.audit_logs import router as audit_logs_router
 from app.views.backup import router as backup_router
+from app.views.compliance import router as compliance_router
 from app.views.db_wizard import router as db_wizard_router
+from app.views.devices import router as devices_router  # Mobile devices dashboard
 from app.views.dropbox import router as dropbox_router
 from app.views.filemanager import router as filemanager_router
 
@@ -23,6 +26,8 @@ from app.views.onboarding import router as onboarding_router
 from app.views.onedrive import router as onedrive_router
 from app.views.pipelines import router as pipelines_router  # Processing pipelines
 from app.views.plans import router as plans_router  # Admin Plan Designer
+from app.views.profile import router as profile_router  # User self-service profile
+from app.views.qr_login import router as qr_login_router  # QR code mobile login
 from app.views.queue import router as queue_router
 from app.views.scheduled_jobs import router as scheduled_jobs_router  # Scheduled batch jobs
 from app.views.search import router as search_router
@@ -31,6 +36,7 @@ from app.views.share import router as share_router
 from app.views.shared_links import router as shared_links_router
 from app.views.status import router as status_router
 from app.views.subscriptions import router as subscriptions_router  # Pricing + subscription pages
+from app.views.system_reset import router as system_reset_router  # System reset / factory reset
 from app.views.wizard import router as wizard_router
 
 # Create a main router that includes all the view routers
@@ -56,8 +62,14 @@ router.include_router(subscriptions_router)  # Pricing + subscription pages
 router.include_router(plans_router)  # Admin Plan Designer
 router.include_router(onboarding_router)  # User onboarding wizard
 router.include_router(pipelines_router)  # Processing pipelines
+router.include_router(profile_router)  # User self-service profile settings
+router.include_router(qr_login_router)  # QR code mobile login page
 router.include_router(imap_accounts_router)  # Per-user IMAP ingestion accounts
 router.include_router(integrations_router)  # Unified integrations dashboard
 router.include_router(notifications_router)  # User notification dashboard
 router.include_router(scheduled_jobs_router)  # Admin scheduled batch jobs
+router.include_router(audit_logs_router)  # Comprehensive audit log viewer
 router.include_router(help_router)  # Built-in help / How-To docs
+router.include_router(compliance_router)  # Compliance templates dashboard
+router.include_router(devices_router)  # Mobile devices dashboard
+router.include_router(system_reset_router)  # System reset / factory reset

app/views/audit_logs.py

@@ -0,0 +1,46 @@
+"""
+Audit log viewer UI — admin-only page with filtering and SIEM status.
+"""
+
+import logging
+
+from fastapi import Depends, HTTPException, Request, status
+from sqlalchemy.orm import Session
+
+from app.views.base import APIRouter, get_db, require_login, settings, templates
+from app.views.settings import require_admin_access
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/admin/audit-logs")
+@require_login
+@require_admin_access
+async def audit_logs_page(request: Request, db: Session = Depends(get_db)):
+    """Comprehensive audit log viewer with filtering controls.
+
+    Displays a chronological log of all significant actions: logins,
+    document operations, settings changes, and admin actions.  The
+    actual data is fetched client-side via the ``/api/audit-logs`` JSON
+    endpoint so that filters, pagination, and live refresh work without
+    full-page reloads.
+    """
+    try:
+        siem_enabled = settings.audit_siem_enabled
+        siem_transport = settings.audit_siem_transport if siem_enabled else None
+        return templates.TemplateResponse(
+            "audit_logs.html",
+            {
+                "request": request,
+                "app_version": settings.version,
+                "siem_enabled": siem_enabled,
+                "siem_transport": siem_transport,
+            },
+        )
+    except Exception as e:
+        logger.error("Error loading audit logs page: %s", e)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to load audit logs page",
+        )

app/views/base.py

@@ -11,7 +11,17 @@ from sqlalchemy.orm import Session  # noqa: F401
 
 from app.auth import require_login  # noqa: F401
 from app.config import settings
-from app.database import get_db  # noqa: F401
+from app.database import SessionLocal, get_db  # noqa: F401
+from app.models import UserProfile
+from app.utils.i18n import (
+    SUPPORTED_LANGUAGES,
+    detect_language,
+    format_date,
+    format_datetime,
+    format_number,
+    get_suggested_languages,
+    translate,
+)
 
 # Set up Jinja2 templates
 templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
@@ -21,10 +31,58 @@ templates = Jinja2Templates(directory=str(templates_dir))
 templates.env.globals["min"] = min
 templates.env.globals["max"] = max
 
+# ---------------------------------------------------------------------------
+# i18n Jinja2 integration
+# ---------------------------------------------------------------------------
+# The _() function is available in every template to translate UI strings.
+# Usage: {{ _("nav.dashboard") }} or {{ _("upload.max_size", size="10 MB") }}
+# The locale is automatically resolved from the request context.
+# A default English implementation is registered as a global so error handlers
+# that don't go through _inject_global_context still have the function available.
+# ---------------------------------------------------------------------------
+
+templates.env.globals["supported_languages"] = SUPPORTED_LANGUAGES
+templates.env.globals["_"] = lambda key, **kwargs: translate(key, "en", **kwargs)
+
 # Customize Jinja2Templates to include app_version in all templates
 original_template_response = templates.TemplateResponse
 
 
+def _hydrate_language_from_db(request: Request, session_user: object) -> None:
+    """Load the user's preferred language from the DB into the session.
+
+    Called once per session when ``preferred_language`` is not yet in the
+    session.  A lightweight DB query fetches the stored preference so that
+    :func:`detect_language` picks it up from the session on all subsequent
+    requests without further DB access.
+    """
+    from app.utils.i18n import SUPPORTED_LANGUAGE_CODES
+
+    user_id: str | None = None
+    if isinstance(session_user, dict):
+        user_id = (
+            session_user.get("sub")
+            or session_user.get("preferred_username")
+            or session_user.get("email")
+            or session_user.get("id")
+        )
+    elif isinstance(session_user, str):
+        user_id = session_user
+
+    if not user_id:
+        return
+
+    db = SessionLocal()
+    try:
+        profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
+        if profile and profile.preferred_language and profile.preferred_language in SUPPORTED_LANGUAGE_CODES:
+            request.session["preferred_language"] = profile.preferred_language
+    except Exception:  # noqa: BLE001 — intentionally broad; DB may be temporarily unavailable
+        logger.debug("Could not hydrate language preference for user_id=%s", user_id)
+    finally:
+        db.close()
+
+
 def _inject_global_context(ctx: dict) -> None:
     """Inject shared global variables into every template context dict."""
     ctx.setdefault("version", settings.version)
@@ -36,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None:
         "allow_signup",
         getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False),
     )
+    ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False))
 
     req = ctx.get("request")
     if req is not None:
@@ -48,8 +107,43 @@ def _inject_global_context(ctx: dict) -> None:
             session_user = req.session.get("user")
         # When auth is disabled every visitor is effectively "logged in"
         ctx.setdefault("is_logged_in", not getattr(settings, "auth_enabled", True) or session_user is not None)
+
+        # --- Hydrate session language from DB (once per session) ---
+        # If the session doesn't have a preferred_language yet but the user
+        # is logged in, load the stored preference from the database so that
+        # detect_language() picks it up from the session on this and all
+        # subsequent requests.
+        if hasattr(req, "session") and "preferred_language" not in req.session and session_user is not None:
+            _hydrate_language_from_db(req, session_user)
+
+        # --- i18n: detect language and register template helpers ---
+        current_locale = detect_language(req)
+        ctx.setdefault("current_locale", current_locale)
+
+        # Smart language suggestions for the compact nav-bar dropdown (5-7 languages)
+        accept_header = req.headers.get("accept-language", "") if hasattr(req, "headers") else ""
+        ctx.setdefault("suggested_languages", get_suggested_languages(current_locale, accept_header))
+
+        def _translate(key: str, **kwargs: object) -> str:
+            return translate(key, current_locale, **kwargs)
+
+        def _format_date(value: object, short: bool = False) -> str:
+            return format_date(value, current_locale, short=short)  # type: ignore[arg-type]
+
+        def _format_datetime(value: object) -> str:
+            return format_datetime(value, current_locale)  # type: ignore[arg-type]
+
+        def _format_number(value: object) -> str:
+            return format_number(value, current_locale)  # type: ignore[arg-type]
+
+        ctx.setdefault("_", _translate)
+        ctx.setdefault("format_date_l10n", _format_date)
+        ctx.setdefault("format_datetime_l10n", _format_datetime)
+        ctx.setdefault("format_number_l10n", _format_number)
     else:
         ctx.setdefault("is_logged_in", not getattr(settings, "auth_enabled", True))
+        ctx.setdefault("current_locale", "en")
+        ctx.setdefault("_", lambda key, **kw: translate(key, "en", **kw))
 
 
 def template_response_with_version(*args, **kwargs):

app/views/compliance.py

@@ -0,0 +1,48 @@
+"""Admin view: compliance templates dashboard page."""
+
+import logging
+
+from fastapi import HTTPException, Request, status
+from fastapi.responses import RedirectResponse
+
+from app.views.base import APIRouter, require_login, settings, templates
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _require_admin(request: Request):
+    """Return the session user if they are an admin, else None."""
+    user = request.session.get("user")
+    if not user or not user.get("is_admin"):
+        logger.warning("Non-admin user attempted to access /admin/compliance")
+        return None
+    return user
+
+
+@router.get("/admin/compliance")
+@require_login
+async def compliance_page(request: Request):
+    """Admin compliance templates dashboard page.
+
+    Displays GDPR, HIPAA, and SOC2 compliance templates with their current
+    status and one-click apply functionality.
+    """
+    user = _require_admin(request)
+    if user is None:
+        return RedirectResponse(url="/", status_code=status.HTTP_302_FOUND)
+
+    try:
+        return templates.TemplateResponse(
+            "compliance.html",
+            {
+                "request": request,
+                "app_version": settings.version,
+            },
+        )
+    except Exception as e:
+        logger.error(f"Error loading compliance page: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to load compliance page",
+        )

app/views/devices.py

@@ -0,0 +1,25 @@
+"""View route for the Devices management page.
+
+Renders the ``devices.html`` template where users can see their registered
+mobile devices, mobile API tokens (created via the mobile SSO flow or QR
+code login), and revoke access per-device.
+"""
+
+import logging
+
+from fastapi import APIRouter, Request
+
+from app.views.base import require_login, templates
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/devices", include_in_schema=False)
+@require_login
+async def devices_page(request: Request):
+    """Render the Devices management page."""
+    return templates.TemplateResponse(
+        "devices.html",
+        {"request": request, "page_title": "Devices"},
+    )

app/views/dropbox.py

@@ -14,6 +14,19 @@ from app.views.base import APIRouter, Depends, get_db, require_login, settings,
 router = APIRouter()
 
 
+def _get_dropbox_callback_url(request: Request) -> str:
+    """Return the Dropbox OAuth callback URL.
+
+    Uses ``PUBLIC_BASE_URL`` when configured so that the redirect URI displayed
+    to the user (and registered in the Dropbox developer console) matches the
+    one used in the OAuth authorization request.  Falls back to deriving the URL
+    from the incoming request when ``PUBLIC_BASE_URL`` is not set.
+    """
+    if settings.public_base_url:
+        return settings.public_base_url.rstrip("/") + "/dropbox-callback"
+    return f"{request.url.scheme}://{request.url.netloc}/dropbox-callback"
+
+
 @router.get("/dropbox-setup")
 @require_login
 async def dropbox_setup_page(
@@ -30,6 +43,8 @@ async def dropbox_setup_page(
     path from the integration's existing config is pre-populated; global
     admin credentials are never exposed in this mode.
     """
+    callback_url = _get_dropbox_callback_url(request)
+
     if integration_id is not None:
         owner_id = get_current_owner_id(request)
         integration = (
@@ -46,9 +61,12 @@ async def dropbox_setup_page(
                     cfg = {}
             # Support both "folder" (DROPBOX destination) and "folder_path" (WATCH_FOLDER source)
             folder_path = cfg.get("folder", cfg.get("folder_path", ""))
-            # Provide system-wide app credentials when available so users can
-            # authorize without creating their own Dropbox app.
-            has_system_credentials = bool(settings.dropbox_app_key and settings.dropbox_app_secret)
+            # Determine if global credentials are available for users to reuse
+            global_creds_available = bool(
+                settings.dropbox_allow_global_credentials_for_integrations
+                and settings.dropbox_app_key
+                and settings.dropbox_app_secret
+            )
             return templates.TemplateResponse(
                 "dropbox.html",
                 {
@@ -59,10 +77,12 @@ async def dropbox_setup_page(
                     "integration_name": integration.name,
                     "integration_type": integration.integration_type,
                     "folder_path": folder_path,
-                    "has_system_credentials": has_system_credentials,
-                    "app_key_value": settings.dropbox_app_key or "" if has_system_credentials else "",
-                    "app_secret_value": settings.dropbox_app_secret or "" if has_system_credentials else "",
+                    # Only expose the public app key (not the secret) when global creds are allowed
+                    "app_key_value": settings.dropbox_app_key if global_creds_available else "",
+                    "app_secret_value": "",
                     "refresh_token_value": "",
+                    "global_creds_available": global_creds_available,
+                    "callback_url": callback_url,
                 },
             )
 
@@ -75,7 +95,6 @@ async def dropbox_setup_page(
             "request": request,
             "user_mode": False,
             "is_configured": is_configured,
-            "has_system_credentials": bool(settings.dropbox_app_key and settings.dropbox_app_secret),
             "app_key_value": settings.dropbox_app_key or "",
             "app_secret_value": settings.dropbox_app_secret if settings.dropbox_app_secret else "",
             "refresh_token_value": settings.dropbox_refresh_token if settings.dropbox_refresh_token else "",
@@ -83,6 +102,7 @@ async def dropbox_setup_page(
             "integration_id": integration_id,
             "integration_name": None,
             "integration_type": None,
+            "callback_url": callback_url,
         },
     )
 
@@ -113,5 +133,6 @@ async def dropbox_callback(request: Request, code: str = None, error: str = None
             "app_key_value": "",  # The callback will prioritize sessionStorage values
             "app_secret_value": "",  # The callback will prioritize sessionStorage values
             "folder_path": "",  # The callback will prioritize sessionStorage values
+            "callback_url": _get_dropbox_callback_url(request),
         },
     )

app/views/files.py

@@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
         )
 
 
+@router.get("/files/{file_id}/text/default-language")
+@require_login
+def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)):
+    """Return the persisted default-language translation for the file view."""
+    from fastapi import status
+    from fastapi.responses import JSONResponse
+
+    from app.models import FileRecord
+
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+    if not file_record:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND)
+
+    if not file_record.default_language_text:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="No default-language translation available",
+        )
+
+    return JSONResponse(
+        content={
+            "text": file_record.default_language_text,
+            "language_code": file_record.default_language_code,
+            "detected_language": file_record.detected_language,
+        }
+    )
+
+
 @router.get("/duplicates")
 @require_login
 def duplicates_page(

app/views/imap_accounts.py

@@ -1,11 +1,13 @@
 """User-facing view for the per-user IMAP ingestion dashboard."""
 
+import json
 import logging
 
 from fastapi import Request
 from sqlalchemy.orm import Session
 
-from app.models import UserImapAccount
+from app.models import ImapIngestionProfile, UserImapAccount
+from app.utils.allowed_types import DEFAULT_CATEGORIES, FILE_TYPE_CATEGORIES
 from app.utils.subscription import get_tier, get_user_tier_id
 from app.utils.user_scope import get_current_owner_id
 from app.views.base import APIRouter, Depends, get_db, require_login, templates
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
     return max_mb
 
 
+def _serialize_profile(profile: ImapIngestionProfile) -> dict:
+    """Serialize a profile for JSON embedding in the template."""
+    try:
+        categories = json.loads(profile.allowed_categories)
+    except (ValueError, TypeError):
+        categories = []
+    return {
+        "id": profile.id,
+        "name": profile.name,
+        "description": profile.description,
+        "owner_id": profile.owner_id,
+        "allowed_categories": categories,
+        "is_builtin": profile.is_builtin,
+    }
+
+
 @router.get("/imap-accounts")
 @require_login
 async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
@@ -49,11 +67,35 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
         max_mailboxes = _get_max_mailboxes(tier)
         can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
 
+    # Load ingestion profiles: system-global + user's own
+    profiles = (
+        db.query(ImapIngestionProfile)
+        .filter(
+            # SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
+            (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id)  # noqa: E711
+        )
+        .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
+        .all()
+    )
+
+    # Category definitions for the UI checkbox builder
+    categories = [
+        {
+            "key": key,
+            "label": info["label"],
+            "description": info["description"],
+        }
+        for key, info in FILE_TYPE_CATEGORIES.items()
+    ]
+
     return templates.TemplateResponse(
         "imap_accounts.html",
         {
             "request": request,
             "accounts": accounts,
+            "profiles": [_serialize_profile(p) for p in profiles],
+            "categories": categories,
+            "default_categories": DEFAULT_CATEGORIES,
             "current_count": current_count,
             "max_mailboxes": max_mailboxes,
             "can_add": can_add,

app/views/onboarding.py

@@ -27,6 +27,7 @@ _DESTINATION_META: list[dict] = [
     {"id": "webdav", "name": "WebDAV", "icon": "fas fa-server"},
     {"id": "sftp", "name": "SFTP", "icon": "fas fa-terminal"},
     {"id": "ftp", "name": "FTP", "icon": "fas fa-server"},
+    {"id": "icloud", "name": "iCloud Drive", "icon": "fab fa-apple"},
 ]
 
 
@@ -51,6 +52,7 @@ def _get_configured_destinations(cfg: Settings) -> list[dict]:
         "webdav": bool(cfg.webdav_url and cfg.webdav_username),
         "sftp": bool(cfg.sftp_host and cfg.sftp_username),
         "ftp": bool(cfg.ftp_host and cfg.ftp_username),
+        "icloud": bool(cfg.icloud_username and cfg.icloud_password),
     }
     return [meta for meta in _DESTINATION_META if checks.get(meta["id"], False)]
 

app/views/plans.py

@@ -3,12 +3,11 @@
 from fastapi import Request
 from fastapi.responses import HTMLResponse
 from fastapi.routing import APIRouter
-from fastapi.templating import Jinja2Templates
 
 from app.auth import require_login
+from app.views.base import templates
 
 router = APIRouter()
-templates = Jinja2Templates(directory="frontend/templates")
 
 
 @router.get("/admin/plans", response_class=HTMLResponse)

app/views/profile.py

@@ -0,0 +1,40 @@
+"""View route for the user self-service profile settings page.
+
+Route:
+  GET /profile  — renders the profile settings HTML page (requires login)
+"""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import Depends, Request
+from sqlalchemy.orm import Session
+
+from app.models import UserProfile
+from app.utils.i18n import SUPPORTED_LANGUAGES
+from app.views.base import APIRouter, get_db, require_login, templates
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/profile", include_in_schema=False)
+@require_login
+async def profile_page(request: Request, db: Session = Depends(get_db)):
+    """Serve the user profile settings page."""
+    user = request.session.get("user") or {}
+    user_id = user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
+
+    profile = None
+    if user_id:
+        profile = db.query(UserProfile).filter(UserProfile.user_id == user_id).first()
+
+    return templates.TemplateResponse(
+        "profile.html",
+        {
+            "request": request,
+            "profile": profile,
+            "supported_languages": SUPPORTED_LANGUAGES,
+        },
+    )

app/views/qr_login.py

@@ -0,0 +1,26 @@
+"""View route for the QR code mobile login page.
+
+Route:
+  GET /qr-login  — renders the QR login page (requires login)
+"""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import Request
+
+from app.views.base import APIRouter, require_login, templates
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/qr-login", include_in_schema=False)
+@require_login
+async def qr_login_page(request: Request):
+    """Serve the QR code login page for mobile app authentication."""
+    return templates.TemplateResponse(
+        "qr_login.html",
+        {"request": request},
+    )

app/views/system_reset.py

@@ -0,0 +1,40 @@
+"""
+System reset view — admin-only UI page.
+
+Renders a confirmation-heavy page that allows administrators to:
+1. **Full Reset** — wipe all user data (DB + disk) for a fresh start.
+2. **Reset & Re-import** — move originals to a reimport folder, wipe,
+   and let the watch-folder mechanism re-ingest them.
+
+Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag.
+"""
+
+import logging
+
+from fastapi import Depends, Request
+from fastapi.responses import RedirectResponse, Response
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.views.base import APIRouter, get_db, require_login, templates
+from app.views.settings import require_admin_access
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/admin/system-reset")
+@require_login
+@require_admin_access
+async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response:
+    """Render the system reset administration page."""
+    if not settings.enable_factory_reset:
+        return RedirectResponse(url="/settings", status_code=302)
+
+    return templates.TemplateResponse(
+        "system_reset.html",
+        {
+            "request": request,
+            "factory_reset_on_startup": settings.factory_reset_on_startup,
+        },
+    )