gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into copilot/add-conditional-routing

Browse Source 
Resolve conflicts in app/api/__init__.py and app/models.py.
Renumber migration 027_add_routing_rules → 035_add_routing_rules.
Fix migration chain: down_revision → 034_add_user_profile_settings.
Add PipelineRoutingRule to migrations/env.py.
This commit is contained in:
copilot-swe-agent[bot]
2026-03-12 22:07:54 +00:00
parent 1e5e35a26a 77f392b29b
commit f70fa65969
163 changed files with 45250 additions and 955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/allowed_types.py
+154
View File
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
".md",
".markdown",
}
# ---------------------------------------------------------------------------
# Fine-grained file-type categories used by IMAP ingestion profiles.
# Each category groups related MIME types and extensions so that users can
# enable/disable a logical collection of formats (e.g. "images") rather than
# having to manage individual MIME strings.
# ---------------------------------------------------------------------------
FILE_TYPE_CATEGORIES: dict[str, dict] = {
"pdf": {
"label": "PDF",
"description": "PDF documents (.pdf)",
"mime_types": frozenset({"application/pdf"}),
"extensions": frozenset({".pdf"}),
},
"office": {
"label": "Microsoft Office",
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
"mime_types": frozenset(
{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"application/vnd.ms-word.document.macroEnabled.12",
"application/vnd.ms-word.template.macroEnabled.12",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"application/vnd.ms-excel.sheet.macroEnabled.12",
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.presentationml.template",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
}
),
"extensions": frozenset(
{
".doc",
".docx",
".docm",
".dot",
".dotx",
".dotm",
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw",
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx",
}
),
},
"opendocument": {
"label": "OpenDocument (LibreOffice)",
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
"mime_types": frozenset(
{
"application/vnd.oasis.opendocument.text",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.graphics",
"application/vnd.oasis.opendocument.formula",
}
),
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
},
"text": {
"label": "Text & Data",
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
"mime_types": frozenset(
{
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
),
"extensions": frozenset({".txt", ".csv", ".rtf"}),
},
"web": {
"label": "Web & Markup",
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
"mime_types": frozenset(
{
"text/html",
"text/markdown",
"text/x-markdown",
}
),
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"mime_types": frozenset(
{
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
),
"extensions": frozenset(
{
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
),
},
}
# Default categories for the "documents only" built-in profile (no images)
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
# All categories including images
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
def get_allowed_types_for_categories(
categories: list[str],
) -> tuple[frozenset[str], frozenset[str]]:
"""Return ``(mime_types, extensions)`` for the given category list.
Unknown category names are silently ignored so that future categories
don't break existing profiles.
"""
mime_types: set[str] = set()
extensions: set[str] = set()
for cat in categories:
info = FILE_TYPE_CATEGORIES.get(cat)
if info:
mime_types |= info["mime_types"]
extensions |= info["extensions"]
return frozenset(mime_types), frozenset(extensions)
app/utils/audit_service.py
+331
View File
@@ -0,0 +1,331 @@
"""
Comprehensive audit-event service for DocuElevate.
Provides helpers to **record** audit events (append-only database writes)
and to optionally **forward** them to external SIEM systems.
Supported SIEM transports:
* **Syslog** – RFC 5424 structured-data messages over UDP or TCP.
* **HTTP** – JSON POST payloads compatible with Splunk HEC, Logstash
HTTP input, Grafana Loki push API, and any generic webhook endpoint.
"""
import json
import logging
import re
import socket
import threading
from datetime import datetime, timezone
from typing import Any
import httpx
from fastapi import Request
from sqlalchemy.orm import Session
from app.config import settings
from app.middleware.audit_log import get_client_ip, get_username
from app.models import AuditLog
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Public helpers
# ---------------------------------------------------------------------------
def record_event(
db: Session,
*,
action: str,
user: str = "system",
resource_type: str | None = None,
resource_id: str | None = None,
ip_address: str | None = None,
details: dict[str, Any] | None = None,
severity: str = "info",
) -> AuditLog:
"""Persist an audit event and optionally forward it to SIEM.
Args:
db: Active SQLAlchemy session.
action: Short action identifier (e.g. ``"login"``, ``"document.create"``).
user: Username performing the action.
resource_type: Category of the affected resource (``"document"``, ``"user"`` …).
resource_id: Identifier of the affected resource.
ip_address: Client IP address (``None`` when not applicable).
details: Arbitrary key/value context serialised as JSON.
severity: One of ``info``, ``warning``, ``error``, ``critical``.
Returns:
The newly created :class:`AuditLog` row.
"""
details_json = json.dumps(details, default=str) if details else None
entry = AuditLog(
user=user,
action=action,
resource_type=resource_type,
resource_id=str(resource_id) if resource_id is not None else None,
ip_address=ip_address,
details=details_json,
severity=severity,
)
db.add(entry)
db.commit()
db.refresh(entry)
# Fire-and-forget SIEM forwarding in a background thread so we never
# block the request path.
if settings.audit_siem_enabled:
payload = _build_siem_payload(entry)
thread = threading.Thread(target=_forward_to_siem, args=(payload,), daemon=True)
thread.start()
return entry
def record_event_from_request(
db: Session,
request: Request,
*,
action: str,
resource_type: str | None = None,
resource_id: str | None = None,
details: dict[str, Any] | None = None,
severity: str = "info",
) -> AuditLog:
"""Convenience wrapper that extracts user and IP from a :class:`Request`.
Args:
db: Active SQLAlchemy session.
request: The current HTTP request.
action: Short action identifier.
resource_type: Category of the affected resource.
resource_id: Identifier of the affected resource.
details: Arbitrary key/value context serialised as JSON.
severity: One of ``info``, ``warning``, ``error``, ``critical``.
Returns:
The newly created :class:`AuditLog` row.
"""
return record_event(
db,
action=action,
user=get_username(request),
resource_type=resource_type,
resource_id=resource_id,
ip_address=get_client_ip(request),
details=details,
severity=severity,
)
def query_events(
db: Session,
*,
action: str | None = None,
user: str | None = None,
resource_type: str | None = None,
severity: str | None = None,
since: datetime | None = None,
until: datetime | None = None,
limit: int = 200,
offset: int = 0,
) -> list[AuditLog]:
"""Query audit log entries with optional filtering.
Args:
db: Active SQLAlchemy session.
action: Filter by action string (exact match).
user: Filter by username (exact match).
resource_type: Filter by resource type (exact match).
severity: Filter by severity level (exact match).
since: Only events at or after this timestamp.
until: Only events at or before this timestamp.
limit: Maximum number of rows to return.
offset: Number of rows to skip (for pagination).
Returns:
List of :class:`AuditLog` rows ordered by *timestamp descending*.
"""
q = db.query(AuditLog)
if action:
q = q.filter(AuditLog.action == action)
if user:
q = q.filter(AuditLog.user == user)
if resource_type:
q = q.filter(AuditLog.resource_type == resource_type)
if severity:
q = q.filter(AuditLog.severity == severity)
if since:
q = q.filter(AuditLog.timestamp >= since)
if until:
q = q.filter(AuditLog.timestamp <= until)
return q.order_by(AuditLog.timestamp.desc()).offset(offset).limit(limit).all()
def count_events(
db: Session,
*,
action: str | None = None,
user: str | None = None,
resource_type: str | None = None,
severity: str | None = None,
since: datetime | None = None,
until: datetime | None = None,
) -> int:
"""Return the total count of events matching the given filters.
Args:
db: Active SQLAlchemy session.
action: Filter by action string.
user: Filter by username.
resource_type: Filter by resource type.
severity: Filter by severity level.
since: Only events at or after this timestamp.
until: Only events at or before this timestamp.
Returns:
Integer count.
"""
q = db.query(AuditLog)
if action:
q = q.filter(AuditLog.action == action)
if user:
q = q.filter(AuditLog.user == user)
if resource_type:
q = q.filter(AuditLog.resource_type == resource_type)
if severity:
q = q.filter(AuditLog.severity == severity)
if since:
q = q.filter(AuditLog.timestamp >= since)
if until:
q = q.filter(AuditLog.timestamp <= until)
return q.count()
# ---------------------------------------------------------------------------
# SIEM forwarding internals
# ---------------------------------------------------------------------------
_SYSLOG_FACILITY_LOCAL0 = 16
_SYSLOG_SEVERITY_MAP = {
"info": 6,
"warning": 4,
"error": 3,
"critical": 2,
}
def _build_siem_payload(entry: AuditLog) -> dict[str, Any]:
"""Convert an :class:`AuditLog` row into a plain dict for SIEM delivery."""
ts = entry.timestamp if entry.timestamp else datetime.now(timezone.utc)
return {
"id": entry.id,
"timestamp": ts.isoformat(),
"user": entry.user,
"action": entry.action,
"resource_type": entry.resource_type,
"resource_id": entry.resource_id,
"ip_address": entry.ip_address,
"details": entry.details,
"severity": entry.severity,
"source": "docuelevate",
}
def _forward_to_siem(payload: dict[str, Any]) -> None:
"""Route a SIEM payload to the configured transport."""
transport = settings.audit_siem_transport.lower()
try:
if transport == "syslog":
_send_syslog(payload)
elif transport == "http":
_send_http(payload)
else:
logger.warning("Unknown SIEM transport %r; skipping forwarding", transport)
except Exception:
logger.exception("Failed to forward audit event to SIEM (%s)", transport)
def _send_syslog(payload: dict[str, Any]) -> None:
"""Send a RFC 5424 syslog message to the configured receiver."""
severity_num = _SYSLOG_SEVERITY_MAP.get(payload.get("severity", "info"), 6)
priority = _SYSLOG_FACILITY_LOCAL0 * 8 + severity_num
ts = payload.get("timestamp", datetime.now(timezone.utc).isoformat())
hostname = socket.gethostname()
app_name = "docuelevate"
msg_id = payload.get("action", "-")
# Structured data (SD) element with key event fields.
sd = (
f'[docuelevate@0 user="{payload.get("user", "-")}" '
f'action="{payload.get("action", "-")}" '
f'resource_type="{payload.get("resource_type", "-")}" '
f'resource_id="{payload.get("resource_id", "-")}" '
f'ip="{payload.get("ip_address", "-")}"]'
)
message = json.dumps(payload, default=str)
syslog_msg = f"<{priority}>1 {ts} {hostname} {app_name} - {msg_id} {sd} {message}"
proto = settings.audit_siem_syslog_protocol.lower()
host = settings.audit_siem_syslog_host
port = settings.audit_siem_syslog_port
if proto == "tcp":
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(5)
sock.connect((host, port))
sock.sendall(syslog_msg.encode("utf-8"))
else:
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
sock.settimeout(5)
sock.sendto(syslog_msg.encode("utf-8"), (host, port))
logger.debug("Syslog audit event sent to %s:%s (%s)", host, port, proto)
def _send_http(payload: dict[str, Any]) -> None:
"""POST a JSON audit event to the configured HTTP endpoint."""
url = settings.audit_siem_http_url
if not url:
logger.warning("SIEM HTTP URL not configured; skipping HTTP forwarding")
return
headers: dict[str, str] = {"Content-Type": "application/json"}
token = settings.audit_siem_http_token
if token:
headers["Authorization"] = f"Bearer {token}"
# Parse custom headers (comma-separated "Key:Value" pairs).
# Reject headers that could override security-critical ones already set,
# and validate that header names contain only RFC 7230 token characters.
_PROTECTED_HEADERS = {"authorization", "content-type", "host"}
_VALID_HEADER_NAME = re.compile(r"^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$")
raw_custom = settings.audit_siem_http_custom_headers
if raw_custom:
for raw_pair in raw_custom.split(","):
pair = raw_pair.strip()
if ":" in pair:
k, _, v = pair.partition(":")
name = k.strip()
if not name or not _VALID_HEADER_NAME.match(name):
logger.warning("Skipping invalid SIEM custom header name: %r", name)
continue
if name.lower() in _PROTECTED_HEADERS:
logger.warning("Skipping protected SIEM custom header: %r", name)
continue
headers[name] = v.strip()
# Wrap in Splunk HEC-style envelope when URL contains ``/services/collector``.
body: dict[str, Any]
if "/services/collector" in url:
body = {"event": payload, "sourcetype": "docuelevate:audit", "source": "docuelevate"}
else:
body = payload
with httpx.Client(timeout=10) as client:
resp = client.post(url, json=body, headers=headers)
resp.raise_for_status()
logger.debug("HTTP audit event forwarded to %s (status %s)", url, resp.status_code)
app/utils/compliance_service.py
+433
View File
@@ -0,0 +1,433 @@
"""Compliance service for managing GDPR, HIPAA, and SOC2 compliance templates.
Provides pre-built compliance configurations that can be applied with one click
to ensure the DocuElevate instance meets regulatory requirements.
"""
import json
import logging
from datetime import datetime, timezone
from typing import Any
from sqlalchemy.orm import Session
from app.models import ComplianceTemplate
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Pre-built compliance template definitions
# ---------------------------------------------------------------------------
COMPLIANCE_TEMPLATES: dict[str, dict[str, Any]] = {
"gdpr": {
"display_name": "GDPR (General Data Protection Regulation)",
"description": (
"European Union regulation for data protection and privacy. "
"Enforces data minimisation, encryption at rest, audit logging, "
"and limits PII exposure in telemetry."
),
"settings": {
"auth_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "User authentication must be enabled to control access to personal data.",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Personally identifiable information must not be sent to external monitoring services.",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "HTTP security headers protect against common web vulnerabilities.",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "HTTP Strict Transport Security ensures encrypted connections.",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "CSP headers prevent cross-site scripting and data injection attacks.",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "X-Frame-Options header prevents clickjacking attacks.",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Data minimisation: avoid storing duplicate documents.",
},
],
},
"hipaa": {
"display_name": "HIPAA (Health Insurance Portability and Accountability Act)",
"description": (
"United States regulation for protecting health information. "
"Requires strong access controls, audit trails, encryption, "
"and strict session management."
),
"settings": {
"auth_enabled": "True",
"multi_user_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "Access controls are required to protect electronic Protected Health Information (ePHI).",
},
{
"key": "multi_user_enabled",
"expected": "True",
"label": "Multi-user mode enabled",
"description": "Individual user accounts required for access accountability.",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Protected Health Information must not be sent to external services.",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "Security headers protect ePHI during transmission.",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "Encrypted transport required for all ePHI transmissions.",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "CSP prevents injection attacks that could expose ePHI.",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "Prevents embedding the application in unauthorized frames.",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Minimise data footprint for ePHI.",
},
],
},
"soc2": {
"display_name": "SOC 2 (Service Organization Control 2)",
"description": (
"Trust Service Criteria framework for service organisations. "
"Focuses on security, availability, processing integrity, "
"confidentiality, and privacy."
),
"settings": {
"auth_enabled": "True",
"multi_user_enabled": "True",
"sentry_send_default_pii": "False",
"security_headers_enabled": "True",
"security_header_hsts_enabled": "True",
"security_header_csp_enabled": "True",
"security_header_x_frame_options_enabled": "True",
"enable_deduplication": "True",
},
"checks": [
{
"key": "auth_enabled",
"expected": "True",
"label": "Authentication enabled",
"description": "Logical access controls required (CC6.1).",
},
{
"key": "multi_user_enabled",
"expected": "True",
"label": "Multi-user mode enabled",
"description": "Individual user accounts for access management (CC6.2).",
},
{
"key": "sentry_send_default_pii",
"expected": "False",
"label": "PII excluded from telemetry",
"description": "Confidential information must not leak to external services (CC6.7).",
},
{
"key": "security_headers_enabled",
"expected": "True",
"label": "Security headers enabled",
"description": "Protection against common web threats (CC6.6).",
},
{
"key": "security_header_hsts_enabled",
"expected": "True",
"label": "HSTS enabled",
"description": "Encrypted transport in transit (CC6.7).",
},
{
"key": "security_header_csp_enabled",
"expected": "True",
"label": "Content Security Policy enabled",
"description": "Application-level security controls (CC6.6).",
},
{
"key": "security_header_x_frame_options_enabled",
"expected": "True",
"label": "Clickjacking protection enabled",
"description": "UI redress attack prevention (CC6.6).",
},
{
"key": "enable_deduplication",
"expected": "True",
"label": "Deduplication enabled",
"description": "Data integrity through deduplication (PI1.1).",
},
],
},
}
def seed_compliance_templates(db: Session) -> None:
"""Create or update the built-in compliance template rows.
Called once at application startup to ensure the ``compliance_templates``
table always contains the latest definitions.
"""
for name, defn in COMPLIANCE_TEMPLATES.items():
existing = db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first()
if existing is None:
template = ComplianceTemplate(
name=name,
display_name=defn["display_name"],
description=defn["description"],
settings_json=json.dumps(defn["settings"]),
enabled=False,
status="not_applied",
)
db.add(template)
logger.info(f"Seeded compliance template: {name}")
else:
# Update display_name and description if changed, but preserve user state
existing.display_name = defn["display_name"]
existing.description = defn["description"]
try:
db.commit()
except Exception:
db.rollback()
logger.exception("Failed to seed compliance templates")
def get_all_templates(db: Session) -> list[dict[str, Any]]:
"""Return all compliance templates with their current status."""
templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all()
result = []
for t in templates:
defn = COMPLIANCE_TEMPLATES.get(t.name, {})
checks = defn.get("checks", [])
result.append(
{
"id": t.id,
"name": t.name,
"display_name": t.display_name,
"description": t.description,
"enabled": t.enabled,
"status": t.status,
"applied_at": t.applied_at.isoformat() if t.applied_at else None,
"applied_by": t.applied_by,
"settings": json.loads(t.settings_json) if t.settings_json else {},
"checks": checks,
"check_count": len(checks),
}
)
return result
def get_template_by_name(db: Session, name: str) -> ComplianceTemplate | None:
"""Retrieve a single compliance template by name."""
return db.query(ComplianceTemplate).filter(ComplianceTemplate.name == name).first()
def evaluate_template_status(db: Session, name: str) -> dict[str, Any]:
"""Evaluate the compliance status of a template against live settings.
Returns a dict with ``status``, ``total``, ``passed``, ``failed``, and
a list of individual ``check_results``.
"""
from app.config import settings as app_settings
from app.utils.settings_service import get_all_settings_from_db
defn = COMPLIANCE_TEMPLATES.get(name)
if defn is None:
return {"status": "unknown", "total": 0, "passed": 0, "failed": 0, "check_results": []}
db_settings = get_all_settings_from_db(db)
checks = defn.get("checks", [])
results: list[dict[str, Any]] = []
passed = 0
for check in checks:
key = check["key"]
expected = check["expected"]
# Resolve effective value: DB > config object
if key in db_settings and db_settings[key] is not None:
actual = str(db_settings[key])
else:
actual = str(getattr(app_settings, key, ""))
is_passing = actual.lower() == expected.lower()
if is_passing:
passed += 1
results.append(
{
"key": key,
"label": check["label"],
"description": check["description"],
"expected": expected,
"actual": actual,
"passing": is_passing,
}
)
total = len(checks)
if passed == total:
status = "compliant"
elif passed > 0:
status = "partial"
else:
status = "non_compliant"
return {
"status": status,
"total": total,
"passed": passed,
"failed": total - passed,
"check_results": results,
}
def apply_template(db: Session, name: str, applied_by: str = "admin") -> dict[str, Any]:
"""Apply a compliance template by writing its settings to the database.
Returns a summary of what was applied.
"""
from app.utils.settings_service import save_setting_to_db
defn = COMPLIANCE_TEMPLATES.get(name)
if defn is None:
return {"success": False, "error": f"Unknown template: {name}"}
template = get_template_by_name(db, name)
if template is None:
return {"success": False, "error": f"Template not found in database: {name}"}
applied_settings: dict[str, str] = {}
errors: list[str] = []
for key, value in defn["settings"].items():
try:
save_setting_to_db(db, key, value, changed_by=f"compliance:{name}")
applied_settings[key] = value
except Exception as e:
errors.append(f"{key}: {e}")
logger.error(f"Failed to apply compliance setting {key}={value}: {e}")
# Update the template record
now = datetime.now(timezone.utc)
template.enabled = True
template.settings_json = json.dumps(applied_settings)
template.applied_at = now
template.applied_by = applied_by
# Evaluate and store status
eval_result = evaluate_template_status(db, name)
template.status = eval_result["status"]
try:
db.commit()
except Exception:
db.rollback()
logger.exception(f"Failed to update compliance template record: {name}")
return {"success": False, "error": "Database commit failed"}
logger.info(f"Applied compliance template '{name}' by {applied_by}: {len(applied_settings)} settings written")
return {
"success": len(errors) == 0,
"template": name,
"applied_settings": applied_settings,
"errors": errors,
"status": eval_result,
}
def get_compliance_summary(db: Session) -> dict[str, Any]:
"""Return a high-level compliance dashboard summary across all templates."""
templates = db.query(ComplianceTemplate).order_by(ComplianceTemplate.name).all()
summary: list[dict[str, Any]] = []
total_checks = 0
total_passed = 0
for t in templates:
eval_result = evaluate_template_status(db, t.name)
total_checks += eval_result["total"]
total_passed += eval_result["passed"]
summary.append(
{
"name": t.name,
"display_name": t.display_name,
"enabled": t.enabled,
"status": eval_result["status"],
"total": eval_result["total"],
"passed": eval_result["passed"],
"failed": eval_result["failed"],
"applied_at": t.applied_at.isoformat() if t.applied_at else None,
"applied_by": t.applied_by,
}
)
overall = "compliant" if total_checks > 0 and total_passed == total_checks else "non_compliant"
if 0 < total_passed < total_checks:
overall = "partial"
return {
"overall_status": overall,
"total_checks": total_checks,
"total_passed": total_passed,
"total_failed": total_checks - total_passed,
"templates": summary,
}
app/utils/config_validator/providers.py
+25 -10
View File
@@ -144,7 +144,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
),
"enabled": True,
"enabled": getattr(settings, "dropbox_enabled", True),
"description": "Upload files to Dropbox cloud storage",
"details": {
"folder": getattr(settings, "dropbox_folder", "Not set"),
@@ -161,7 +161,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"configured": bool(
getattr(settings, "dest_email_host", None) and getattr(settings, "dest_email_default_recipient", None)
),
"enabled": True,
"enabled": getattr(settings, "dest_email_enabled", True),
"description": "Send documents via email",
"details": {
"host": getattr(settings, "dest_email_host", "Not set"),
@@ -183,7 +183,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "ftp_username", None)
and getattr(settings, "ftp_password", None)
),
"enabled": True,
"enabled": getattr(settings, "ftp_enabled", True),
"description": "Upload files to FTP server",
"details": {
"host": getattr(settings, "ftp_host", "Not set"),
@@ -214,7 +214,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"name": "Google Drive",
"icon": "fa-brands fa-google-drive",
"configured": is_configured and bool(getattr(settings, "google_drive_folder_id", None)),
"enabled": True,
"enabled": getattr(settings, "google_drive_enabled", True),
"description": "Store documents in Google Drive",
"details": {
"auth_type": "OAuth" if use_oauth else "Service Account",
@@ -250,7 +250,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
),
"enabled": True,
"enabled": getattr(settings, "nextcloud_enabled", True),
"description": "Store documents in NextCloud",
"details": {
"url": getattr(settings, "nextcloud_upload_url", "Not set"),
@@ -270,7 +270,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
),
"enabled": True,
"enabled": getattr(settings, "onedrive_enabled", True),
"description": "Store documents in Microsoft OneDrive",
"details": {
"client_id": getattr(settings, "onedrive_client_id", "Not set"),
@@ -288,7 +288,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
"configured": bool(
getattr(settings, "paperless_host", None) and getattr(settings, "paperless_ngx_api_token", None)
),
"enabled": True,
"enabled": getattr(settings, "paperless_enabled", True),
"description": "Document management system for digital archives",
"details": {
"host": getattr(settings, "paperless_host", "Not set"),
@@ -305,7 +305,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "aws_access_key_id", None)
and getattr(settings, "aws_secret_access_key", None)
),
"enabled": True,
"enabled": getattr(settings, "s3_enabled", True),
"description": "Store documents in S3-compatible object storage",
"details": {
"bucket": getattr(settings, "s3_bucket_name", "Not set"),
@@ -327,7 +327,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "sftp_username", None)
and (getattr(settings, "sftp_password", None) or getattr(settings, "sftp_private_key", None))
),
"enabled": True,
"enabled": getattr(settings, "sftp_enabled", True),
"description": "Upload files to SFTP server",
"details": {
"host": getattr(settings, "sftp_host", "Not set"),
@@ -362,7 +362,7 @@ def get_provider_status() -> dict[str, dict[str, object]]:
and getattr(settings, "webdav_username", None)
and getattr(settings, "webdav_password", None)
),
"enabled": True,
"enabled": getattr(settings, "webdav_enabled", True),
"description": "Store documents on WebDAV servers",
"details": {
"url": getattr(settings, "webdav_url", "Not set"),
@@ -373,4 +373,19 @@ def get_provider_status() -> dict[str, dict[str, object]]:
},
}
# Check iCloud Drive configuration
providers["iCloud Drive"] = {
"name": "iCloud Drive",
"icon": "fa-brands fa-apple",
"configured": bool(getattr(settings, "icloud_username", None) and getattr(settings, "icloud_password", None)),
"enabled": getattr(settings, "icloud_enabled", True),
"description": "Store documents in Apple iCloud Drive",
"details": {
"username": getattr(settings, "icloud_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "icloud_password", None)),
"folder": getattr(settings, "icloud_folder", "Not set"),
"cookie_directory": getattr(settings, "icloud_cookie_directory", "Not set"),
},
}
return providers
app/utils/config_validator/validators.py
+32 -2
View File
@@ -59,13 +59,43 @@ def validate_auth_config() -> list[str]:
and getattr(settings, "authentik_config_url", None)
)
if not using_simple_auth and not using_oidc:
issues.append("Neither simple authentication nor OIDC are properly configured")
# Check if any social login provider is enabled
using_social_login = any(
getattr(settings, f"social_auth_{p}_enabled", False) for p in ("google", "microsoft", "apple", "dropbox")
)
if not using_simple_auth and not using_oidc and not using_social_login:
issues.append("Neither simple authentication, OIDC, nor social login are properly configured")
# If using OIDC, check for provider name
if using_oidc and not getattr(settings, "oauth_provider_name", None):
issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled")
# Validate individual social login provider configs
if getattr(settings, "social_auth_google_enabled", False):
if not getattr(settings, "social_auth_google_client_id", None):
issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_ID is required when Google login is enabled")
if not getattr(settings, "social_auth_google_client_secret", None):
issues.append("SOCIAL_AUTH_GOOGLE_CLIENT_SECRET is required when Google login is enabled")
if getattr(settings, "social_auth_microsoft_enabled", False):
if not getattr(settings, "social_auth_microsoft_client_id", None):
issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_ID is required when Microsoft login is enabled")
if not getattr(settings, "social_auth_microsoft_client_secret", None):
issues.append("SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET is required when Microsoft login is enabled")
if getattr(settings, "social_auth_apple_enabled", False):
if not getattr(settings, "social_auth_apple_client_id", None):
issues.append("SOCIAL_AUTH_APPLE_CLIENT_ID is required when Apple login is enabled")
if not getattr(settings, "social_auth_apple_team_id", None):
issues.append("SOCIAL_AUTH_APPLE_TEAM_ID is required when Apple login is enabled")
if getattr(settings, "social_auth_dropbox_enabled", False):
if not getattr(settings, "social_auth_dropbox_client_id", None):
issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_ID is required when Dropbox login is enabled")
if not getattr(settings, "social_auth_dropbox_client_secret", None):
issues.append("SOCIAL_AUTH_DROPBOX_CLIENT_SECRET is required when Dropbox login is enabled")
return issues
app/utils/db_migrate.py
+2
View File
@@ -32,8 +32,10 @@ _TABLE_ORDER = [
"processing_logs",
"application_settings",
"settings_audit_log",
"audit_logs",
"saved_searches",
"webhook_configs",
"shared_links",
]
app/utils/i18n.py
+542
View File
@@ -0,0 +1,542 @@
"""Internationalization (i18n) and localization (l10n) utilities.
Provides a JSON-based translation system for the DocuElevate UI with:
* **31 supported languages** covering all major European languages plus ZH
* Browser ``Accept-Language`` detection with cookie & user-profile persistence
* AI-powered fallback translation via the configured LLM provider
* Locale-aware date, number, and file-size formatting helpers
* Jinja2 integration via a ``_()`` global function
Language resolution order:
1. User profile ``preferred_language`` (persisted in DB)
2. ``docuelevate_lang`` cookie
3. ``Accept-Language`` HTTP header
4. Default (``en``)
"""
from __future__ import annotations
import json
import logging
from datetime import date, datetime
from functools import lru_cache
from pathlib import Path
from typing import Any
from starlette.requests import Request
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Supported languages (ordered by priority)
# ---------------------------------------------------------------------------
SUPPORTED_LANGUAGES: list[dict[str, str]] = [
# --- Tier 1: Primary European languages ---
{"code": "en", "name": "English", "native": "English", "flag": "🇬🇧"},
{"code": "de", "name": "German", "native": "Deutsch", "flag": "🇩🇪"},
{"code": "fr", "name": "French", "native": "Français", "flag": "🇫🇷"},
{"code": "es", "name": "Spanish", "native": "Español", "flag": "🇪🇸"},
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "🇮🇹"},
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "🇵🇹"},
# --- Tier 2: Western & Northern European ---
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "🇳🇱"},
{"code": "nb", "name": "Norwegian", "native": "Norsk", "flag": "🇳🇴"},
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "🇩🇰"},
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "🇸🇪"},
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "🇫🇮"},
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "🇮🇸"},
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "🇮🇪"},
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "🇱🇺"},
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "🏴"},
# --- Tier 3: Central & Eastern European ---
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "🇵🇱"},
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "🇨🇿"},
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "🇸🇰"},
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "🇭🇺"},
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "🇸🇮"},
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "🇭🇷"},
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "🇷🇴"},
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "🇧🇬"},
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "🇬🇷"},
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "🇪🇪"},
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "🇱🇻"},
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "🇱🇹"},
# --- Tier 4: Non-EU European & Other ---
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "🇹🇷"},
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "🇺🇦"},
{"code": "ru", "name": "Russian", "native": "Русский", "flag": "🇷🇺"},
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "🇨🇳"},
]
SUPPORTED_LANGUAGE_CODES: set[str] = {lang["code"] for lang in SUPPORTED_LANGUAGES}
DEFAULT_LANGUAGE = "en"
# ---------------------------------------------------------------------------
# Translation file loading
# ---------------------------------------------------------------------------
_TRANSLATIONS_DIR = Path(__file__).resolve().parent.parent.parent / "frontend" / "translations"
_translation_cache: dict[str, dict[str, str]] = {}
def _load_translations(locale: str) -> dict[str, str]:
"""Load the translation JSON file for *locale*, with caching."""
if locale in _translation_cache:
return _translation_cache[locale]
filepath = _TRANSLATIONS_DIR / f"{locale}.json"
if not filepath.is_file():
logger.warning("Translation file not found for locale '%s'", locale)
_translation_cache[locale] = {}
return {}
try:
data: dict[str, str] = json.loads(filepath.read_text(encoding="utf-8"))
_translation_cache[locale] = data
return data
except (json.JSONDecodeError, OSError):
logger.exception("Failed to load translations for '%s'", locale)
_translation_cache[locale] = {}
return {}
def reload_translations() -> None:
"""Clear the translation cache so files are re-read on next access."""
_translation_cache.clear()
# ---------------------------------------------------------------------------
# Core translation function
# ---------------------------------------------------------------------------
def translate(key: str, locale: str | None = None, **kwargs: Any) -> str:
"""Return the translated string for *key* in *locale*.
Falls back through:
1. Requested *locale*
2. English (``en``)
3. The raw key itself (to keep the UI functional)
Positional placeholders ``{0}``, ``{1}`` or named placeholders
``{name}`` in the translated string are interpolated via *kwargs*.
"""
locale = locale if locale and locale in SUPPORTED_LANGUAGE_CODES else DEFAULT_LANGUAGE
translations = _load_translations(locale)
value = translations.get(key)
# Fallback to English
if value is None and locale != DEFAULT_LANGUAGE:
en_translations = _load_translations(DEFAULT_LANGUAGE)
value = en_translations.get(key)
# Fallback to key itself
if value is None:
value = key
if kwargs:
try:
value = value.format(**kwargs)
except (KeyError, IndexError):
pass # Return unformatted string rather than crash
return value
# ---------------------------------------------------------------------------
# AI fallback translation (best-effort, non-blocking)
# ---------------------------------------------------------------------------
_ai_translation_cache: dict[tuple[str, str], str] = {}
def translate_with_ai_fallback(text: str, target_locale: str) -> str:
"""Translate *text* using the configured AI provider as a fallback.
Returns the original *text* unchanged when:
* The target locale is English (source language)
* The AI provider is unavailable or returns an error
* The translation has already been cached
Results are cached in-memory for the lifetime of the process.
"""
if target_locale == DEFAULT_LANGUAGE or target_locale not in SUPPORTED_LANGUAGE_CODES:
return text
cache_key = (text, target_locale)
if cache_key in _ai_translation_cache:
return _ai_translation_cache[cache_key]
target_name = next(
(lang["name"] for lang in SUPPORTED_LANGUAGES if lang["code"] == target_locale),
target_locale,
)
try:
from litellm import completion # type: ignore[import-untyped]
from app.config import settings
model = getattr(settings, "ai_model", None) or getattr(settings, "openai_model", "gpt-4o-mini")
response = completion(
model=model,
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following UI text "
f"from English to {target_name}. Return ONLY the translated text, "
f"nothing else. Keep any HTML tags, placeholders like {{name}}, "
f"and special characters intact."
),
},
{"role": "user", "content": text},
],
max_tokens=256,
temperature=0.1,
)
translated = response.choices[0].message.content.strip()
_ai_translation_cache[cache_key] = translated
return translated
except Exception:
logger.debug("AI fallback translation failed for '%s' → %s", text[:50], target_locale)
return text
# ---------------------------------------------------------------------------
# Language detection
# ---------------------------------------------------------------------------
def detect_language(request: Request) -> str:
"""Determine the preferred UI language from the request context.
Resolution order:
1. ``preferred_language`` stored in the user session
2. ``docuelevate_lang`` cookie
3. ``Accept-Language`` HTTP header (best match)
4. Default → ``en``
"""
# 1. User session preference
if hasattr(request, "session"):
session_lang = request.session.get("preferred_language")
if isinstance(session_lang, str) and session_lang in SUPPORTED_LANGUAGE_CODES:
return session_lang
# 2. Cookie
if hasattr(request, "cookies"):
cookie_lang = request.cookies.get("docuelevate_lang")
if isinstance(cookie_lang, str) and cookie_lang in SUPPORTED_LANGUAGE_CODES:
return cookie_lang
# 3. Accept-Language header
accept = ""
if hasattr(request, "headers"):
accept = request.headers.get("accept-language", "")
lang = _parse_accept_language(accept)
if lang:
return lang
return DEFAULT_LANGUAGE
def _parse_accept_language(header: str) -> str | None:
"""Extract the best matching language from an ``Accept-Language`` header.
Parses quality values and returns the highest-priority match among
:data:`SUPPORTED_LANGUAGE_CODES`, or ``None`` if nothing matches.
"""
if not header:
return None
entries: list[tuple[float, str]] = []
for raw_part in header.split(","):
part = raw_part.strip()
if not part:
continue
if ";q=" in part:
lang_tag, _, q_str = part.partition(";q=")
try:
quality = float(q_str.strip())
except ValueError:
quality = 0.0
else:
lang_tag = part
quality = 1.0
entries.append((quality, lang_tag.strip().lower()))
# Sort by quality descending
entries.sort(key=lambda e: e[0], reverse=True)
for _quality, tag in entries:
# Try exact match first (e.g., "de", "zh")
code = tag.split("-")[0]
if code in SUPPORTED_LANGUAGE_CODES:
return code
return None
# ---------------------------------------------------------------------------
# Localization helpers (l10n)
# ---------------------------------------------------------------------------
# Locale-specific formatting rules for date/number display
_LOCALE_FORMATS: dict[str, dict[str, Any]] = {
"en": {
"date": "%B %d, %Y",
"date_short": "%m/%d/%Y",
"datetime": "%B %d, %Y %I:%M %p",
"thousands_sep": ",",
"decimal_sep": ".",
},
"de": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"fr": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u202f",
"decimal_sep": ",",
},
"es": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"it": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"pt": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"nl": {
"date": "%d %B %Y",
"date_short": "%d-%m-%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"nb": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"da": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"sv": {
"date": "%d %B %Y",
"date_short": "%Y-%m-%d",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"fi": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"is": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ga": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"lb": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ca": {
"date": "%d de %B de %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d de %B de %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"pl": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"cs": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"sk": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"hu": {
"date": "%Y. %B %d.",
"date_short": "%Y.%m.%d.",
"datetime": "%Y. %B %d. %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"sl": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"hr": {
"date": "%d. %B %Y.",
"date_short": "%d.%m.%Y.",
"datetime": "%d. %B %Y. %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"ro": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"bg": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"el": {
"date": "%d %B %Y",
"date_short": "%d/%m/%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"et": {
"date": "%d. %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d. %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"lv": {
"date": "%Y. gada %d. %B",
"date_short": "%d.%m.%Y.",
"datetime": "%Y. gada %d. %B %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"lt": {
"date": "%Y m. %B %d d.",
"date_short": "%Y-%m-%d",
"datetime": "%Y m. %B %d d. %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"tr": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": ".",
"decimal_sep": ",",
},
"uk": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
"zh": {
"date": "%Y年%m月%d日",
"date_short": "%Y/%m/%d",
"datetime": "%Y年%m月%d日 %H:%M",
"thousands_sep": ",",
"decimal_sep": ".",
},
"ru": {
"date": "%d %B %Y",
"date_short": "%d.%m.%Y",
"datetime": "%d %B %Y %H:%M",
"thousands_sep": "\u00a0",
"decimal_sep": ",",
},
}
def format_date(value: date | datetime | None, locale: str = DEFAULT_LANGUAGE, short: bool = False) -> str:
"""Format a date/datetime value according to the locale conventions."""
if value is None:
return ""
fmt_key = "date_short" if short else "date"
fmt = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])[fmt_key]
return value.strftime(fmt)
def format_datetime(value: datetime | None, locale: str = DEFAULT_LANGUAGE) -> str:
"""Format a datetime value according to the locale conventions."""
if value is None:
return ""
fmt = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])["datetime"]
return value.strftime(fmt)
def format_number(value: int | float, locale: str = DEFAULT_LANGUAGE) -> str:
"""Format a number with locale-appropriate thousand separators."""
lf = _LOCALE_FORMATS.get(locale, _LOCALE_FORMATS[DEFAULT_LANGUAGE])
if isinstance(value, float):
int_part, _, dec_part = f"{value:,.2f}".partition(".")
formatted_int = int_part.replace(",", lf["thousands_sep"])
return f"{formatted_int}{lf['decimal_sep']}{dec_part}"
return f"{value:,}".replace(",", lf["thousands_sep"])
@lru_cache(maxsize=32)
def get_language_info(code: str) -> dict[str, str] | None:
"""Return the metadata dict for a supported language code, or ``None``."""
for lang in SUPPORTED_LANGUAGES:
if lang["code"] == code:
return lang
return None
app/utils/push_notification.py
+130
View File
@@ -0,0 +1,130 @@
"""Push notification sender for the DocuElevate mobile app.
Uses the **Expo Push Notification** service to deliver notifications to both
iOS (via APNs) and Android (via FCM) without requiring server-side APNs keys
or FCM credentials. The mobile app obtains an ``ExponentPushToken[…]`` at
startup and registers it with the backend via the mobile API.
Reference: https://docs.expo.dev/push-notifications/sending-notifications/
"""
import logging
from typing import Any
import httpx
from app.database import SessionLocal
from app.models import MobileDevice
logger = logging.getLogger(__name__)
EXPO_PUSH_URL = "https://exp.host/--/api/v2/push/send"
# Maximum tokens per batch request (Expo limit).
_EXPO_BATCH_LIMIT = 100
def send_expo_push_notification(
tokens: list[str],
title: str,
body: str,
data: dict[str, Any] | None = None,
sound: str = "default",
badge: int | None = None,
) -> list[dict[str, Any]]:
"""Send a push notification to one or more Expo push tokens.
Args:
tokens: List of Expo push tokens (``ExponentPushToken[…]``).
title: Notification title shown in the system tray.
body: Notification body text.
data: Optional JSON-serialisable dict attached to the notification
(available in the app via ``notification.request.content.data``).
sound: Notification sound. Use ``"default"`` or ``None`` for silent.
badge: iOS badge count. Pass ``0`` to clear.
Returns:
List of Expo push receipt dicts (one per token).
"""
if not tokens:
return []
results: list[dict[str, Any]] = []
# Send in batches to stay within Expo's per-request limit.
for i in range(0, len(tokens), _EXPO_BATCH_LIMIT):
batch = tokens[i : i + _EXPO_BATCH_LIMIT]
messages = []
for token in batch:
msg: dict[str, Any] = {
"to": token,
"title": title,
"body": body,
"sound": sound,
}
if data:
msg["data"] = data
if badge is not None:
msg["badge"] = badge
messages.append(msg)
try:
resp = httpx.post(
EXPO_PUSH_URL,
json=messages,
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate",
"Content-Type": "application/json",
},
timeout=15,
)
resp.raise_for_status()
payload = resp.json()
batch_results = payload.get("data", [])
results.extend(batch_results)
logger.debug("Expo push batch sent: %d tokens, %d results", len(batch), len(batch_results))
except httpx.HTTPStatusError as exc:
logger.error("Expo push HTTP error: %s – %s", exc.response.status_code, exc.response.text)
except Exception:
logger.exception("Expo push notification failed for batch starting at index %d", i)
return results
def send_push_to_owner(
owner_id: str,
title: str,
body: str,
data: dict[str, Any] | None = None,
) -> None:
"""Look up all active push tokens for *owner_id* and send them a notification.
This function is safe to call from Celery task workers. Database errors
and push failures are logged but never raised so that the caller task is
not retried due to a notification failure.
"""
db = SessionLocal()
try:
devices = (
db.query(MobileDevice)
.filter(
MobileDevice.owner_id == owner_id,
MobileDevice.is_active.is_(True),
MobileDevice.push_token.isnot(None),
)
.all()
)
tokens = [d.push_token for d in devices if d.push_token]
except Exception:
logger.exception("Failed to query mobile devices for owner_id=%s", owner_id)
return
finally:
db.close()
if not tokens:
logger.debug("No active push tokens for owner_id=%s", owner_id)
return
logger.info("Sending push notification to %d device(s) for owner_id=%s", len(tokens), owner_id)
send_expo_push_notification(tokens=tokens, title=title, body=body, data=data)
app/utils/settings_service.py
+364
View File
@@ -182,6 +182,153 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
# Social Login Providers
"social_auth_google_enabled": {
"category": "Social Login",
"description": (
"Enable Google Sign-In. Requires SOCIAL_AUTH_GOOGLE_CLIENT_ID and "
"SOCIAL_AUTH_GOOGLE_CLIENT_SECRET from the Google Cloud Console."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://console.cloud.google.com/apis/credentials",
"help_link_label": "Google Cloud Console",
},
"social_auth_google_client_id": {
"category": "Social Login",
"description": "Google OAuth2 client ID from the Google Cloud Console.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_google_client_secret": {
"category": "Social Login",
"description": "Google OAuth2 client secret from the Google Cloud Console.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_enabled": {
"category": "Social Login",
"description": (
"Enable Microsoft Sign-In (Azure AD / Microsoft Entra ID). Requires "
"SOCIAL_AUTH_MICROSOFT_CLIENT_ID and SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET "
"from Azure App Registrations."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade",
"help_link_label": "Azure Portal",
},
"social_auth_microsoft_client_id": {
"category": "Social Login",
"description": "Microsoft OAuth2 application (client) ID from Azure App Registrations.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_client_secret": {
"category": "Social Login",
"description": "Microsoft OAuth2 client secret from Azure App Registrations.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_tenant": {
"category": "Social Login",
"description": (
"Azure AD tenant ID or one of 'common', 'organizations', 'consumers'. "
"Use 'common' to allow any Microsoft account. Use a specific GUID to "
"restrict to a single organization."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_enabled": {
"category": "Social Login",
"description": (
"Enable Sign in with Apple. Requires an Apple Developer account with "
"a Services ID configured for Sign in with Apple."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://developer.apple.com/account/resources/identifiers/list/serviceId",
"help_link_label": "Apple Developer Portal",
},
"social_auth_apple_client_id": {
"category": "Social Login",
"description": "Apple Services ID (e.g. com.example.docuelevate).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_team_id": {
"category": "Social Login",
"description": "Apple Developer Team ID (10-character alphanumeric string).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_key_id": {
"category": "Social Login",
"description": "Apple Sign-In private key ID from the Apple Developer Portal.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_private_key": {
"category": "Social Login",
"description": (
"Apple Sign-In private key (PEM format). Generate this in the Apple Developer Portal. "
"Paste the entire key content including BEGIN/END headers."
),
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_enabled": {
"category": "Social Login",
"description": (
"Enable Dropbox Sign-In. Uses the same Dropbox App you may already have "
"configured for storage, or a separate one."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_client_id": {
"category": "Social Login",
"description": "Dropbox OAuth2 App Key from the Dropbox App Console.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_client_secret": {
"category": "Social Login",
"description": "Dropbox OAuth2 App Secret from the Dropbox App Console.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
# AI Services
"openai_api_key": {
"category": "AI Services",
@@ -495,6 +642,14 @@ SETTING_METADATA = {
"options": ["us", "eu"],
},
# Storage Providers - Dropbox
"dropbox_enabled": {
"category": "Storage Providers",
"description": "Enable Dropbox as an upload destination. When disabled, no documents will be sent to Dropbox even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"dropbox_app_key": {
"category": "Storage Providers",
"description": "Dropbox app key for OAuth authentication",
@@ -528,6 +683,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_enabled": {
"category": "Storage Providers",
"description": "Enable Nextcloud as an upload destination. When disabled, no documents will be sent to Nextcloud even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"nextcloud_upload_url": {
"category": "Storage Providers",
"description": "Nextcloud WebDAV upload URL",
@@ -561,6 +724,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Paperless-ngx
"paperless_enabled": {
"category": "Storage Providers",
"description": "Enable Paperless-ngx as an upload destination. When disabled, no documents will be sent to Paperless-ngx even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"paperless_ngx_api_token": {
"category": "Storage Providers",
"description": "Paperless-ngx API authentication token",
@@ -578,6 +749,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - Google Drive
"google_drive_enabled": {
"category": "Storage Providers",
"description": "Enable Google Drive as an upload destination. When disabled, no documents will be sent to Google Drive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"google_drive_credentials_json": {
"category": "Storage Providers",
"description": "Google Drive service account credentials JSON",
@@ -635,6 +814,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - OneDrive
"onedrive_enabled": {
"category": "Storage Providers",
"description": "Enable OneDrive as an upload destination. When disabled, no documents will be sent to OneDrive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"onedrive_client_id": {
"category": "Storage Providers",
"description": "OneDrive OAuth client ID",
@@ -676,6 +863,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_enabled": {
"category": "Storage Providers",
"description": "Enable WebDAV as an upload destination. When disabled, no documents will be sent to WebDAV even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"webdav_url": {
"category": "Storage Providers",
"description": "WebDAV server URL",
@@ -717,6 +912,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - FTP
"ftp_enabled": {
"category": "Storage Providers",
"description": "Enable FTP as an upload destination. When disabled, no documents will be sent to FTP even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"ftp_host": {
"category": "Storage Providers",
"description": "FTP server hostname or IP address",
@@ -774,6 +977,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Storage Providers - SFTP
"sftp_enabled": {
"category": "Storage Providers",
"description": "Enable SFTP as an upload destination. When disabled, no documents will be sent to SFTP even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sftp_host": {
"category": "Storage Providers",
"description": "SFTP server hostname or IP address",
@@ -838,7 +1049,56 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - iCloud Drive
"icloud_enabled": {
"category": "Storage Providers",
"description": "Enable iCloud Drive as an upload destination. When disabled, no documents will be sent to iCloud Drive even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_username": {
"category": "Storage Providers",
"description": "Apple ID email address for iCloud Drive authentication",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_password": {
"category": "Storage Providers",
"description": "App-specific password for iCloud Drive (generate at https://appleid.apple.com)",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"icloud_folder": {
"category": "Storage Providers",
"description": "Target folder path in iCloud Drive (e.g. Documents/Uploads)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"icloud_cookie_directory": {
"category": "Storage Providers",
"description": "Directory for persisting iCloud session cookies (default: ~/.pyicloud)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - AWS S3
"s3_enabled": {
"category": "Storage Providers",
"description": "Enable Amazon S3 as an upload destination. When disabled, no documents will be sent to S3 even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"aws_access_key_id": {
"category": "Storage Providers",
"description": "AWS access key ID for S3",
@@ -972,6 +1232,14 @@ SETTING_METADATA = {
"restart_required": False,
},
# Email Destination Settings (dedicated SMTP for document delivery)
"dest_email_enabled": {
"category": "Email Destination",
"description": "Enable Email as an upload destination. When disabled, no documents will be delivered via email even if credentials are configured.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"dest_email_host": {
"category": "Email Destination",
"description": "SMTP server hostname for document delivery (separate from shared email settings)",
@@ -1392,6 +1660,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"imap_attachment_filter": {
"category": "IMAP",
"description": (
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
"Per-user IMAP accounts can override this global default."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
@@ -1595,6 +1875,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"compliance_enabled": {
"category": "Feature Flags",
"description": (
"Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). "
"When enabled, admins can view compliance status and apply "
"pre-built regulatory configurations. Default: True."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Backup / Restore
"backup_enabled": {
"category": "Backup",
@@ -2065,6 +2357,78 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"audit_siem_enabled": {
"category": "Security",
"description": "Enable forwarding of audit events to an external SIEM system (Syslog, Splunk, Logstash, etc.).",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_transport": {
"category": "Security",
"description": (
"Transport used to forward audit events. 'syslog' sends RFC 5424 messages over UDP/TCP. "
"'http' sends JSON POST payloads to a webhook URL (Splunk HEC, Logstash, Grafana Loki, etc.)."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["syslog", "http"],
},
"audit_siem_syslog_host": {
"category": "Security",
"description": "Hostname or IP of the syslog receiver.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_syslog_port": {
"category": "Security",
"description": "Port of the syslog receiver. Default: 514.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_syslog_protocol": {
"category": "Security",
"description": "Protocol for syslog transport: 'udp' or 'tcp'. Default: udp.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["udp", "tcp"],
},
"audit_siem_http_url": {
"category": "Security",
"description": (
"HTTP endpoint URL for SIEM webhook delivery. Supports Splunk HEC, "
"Logstash HTTP input, Grafana Loki push API, or any JSON-accepting endpoint."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"audit_siem_http_token": {
"category": "Security",
"description": "Bearer / HEC token included in the Authorization header of SIEM HTTP requests.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"audit_siem_http_custom_headers": {
"category": "Security",
"description": "Comma-separated 'Key:Value' pairs of extra headers for SIEM HTTP requests.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Rate Limiting
"rate_limiting_enabled": {
"category": "Security",
app/utils/user_notification.py
+13
View File
@@ -210,6 +210,19 @@ def dispatch_user_notification(
finally:
db.close()
# 3. Send push notifications to registered mobile devices
try:
from app.utils.push_notification import send_push_to_owner
send_push_to_owner(
owner_id=owner_id,
title=title,
body=message,
data={"event_type": event_type, "file_id": file_id},
)
except Exception:
logger.exception("Error sending push notification for owner_id=%s event=%s", owner_id, event_type)
def notify_user_document_processed(owner_id: str, filename: str, file_id: int | None = None) -> None:
"""Notify a user that their document was successfully processed."""