feat(audit): add comprehensive audit logging with SIEM integration

- Add AuditLog model with append-only design (timestamp, user, action, resource, IP, details, severity)
- Add audit_service.py with record/query helpers and SIEM forwarding (Syslog RFC 5424, HTTP/webhook)
- Add /api/audit-logs REST endpoints with filtering and pagination
- Add /admin/audit-logs viewer UI with real-time filters
- Add SIEM config settings (syslog, HTTP for Splunk HEC/Logstash/Grafana Loki)
- Add Alembic migration 027_add_audit_logs
- Add navigation link in admin menu
- Add comprehensive tests (20 tests covering model, service, SIEM, API, view)

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/__init__.py

@@ -8,6 +8,7 @@ from fastapi import APIRouter
 
 from app.api.admin_users import router as admin_users_router
 from app.api.api_tokens import router as api_tokens_router
+from app.api.audit_logs import router as audit_logs_router
 from app.api.azure import router as azure_router
 from app.api.backup import router as backup_router
 from app.api.billing import router as billing_router
@@ -82,3 +83,4 @@ router.include_router(imap_accounts_router)
 router.include_router(integrations_router)
 router.include_router(notifications_router)
 router.include_router(scheduled_jobs_router)
+router.include_router(audit_logs_router)

app/api/audit_logs.py

@@ -0,0 +1,115 @@
+"""
+Audit log REST API endpoints.
+
+Provides read-only access to the comprehensive audit log for admin users.
+Events are append-only — there are no update or delete endpoints.
+"""
+
+import logging
+from datetime import datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.orm import Session
+
+from app.auth import require_login
+from app.database import get_db
+from app.utils.audit_service import count_events, query_events
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/audit-logs")
+@require_login
+async def list_audit_logs(
+    request: Request,
+    db: Session = Depends(get_db),
+    action: str | None = Query(None, description="Filter by action (exact match)"),
+    user: str | None = Query(None, description="Filter by username"),
+    resource_type: str | None = Query(None, description="Filter by resource type"),
+    severity: str | None = Query(None, description="Filter by severity level"),
+    since: datetime | None = Query(None, description="Only events at or after this ISO-8601 timestamp"),
+    until: datetime | None = Query(None, description="Only events at or before this ISO-8601 timestamp"),
+    limit: int = Query(50, ge=1, le=500, description="Max rows to return"),
+    offset: int = Query(0, ge=0, description="Rows to skip for pagination"),
+) -> dict[str, Any]:
+    """Return audit log entries with optional filtering and pagination.
+
+    Requires authentication. Returns events in reverse chronological order.
+    """
+    entries = query_events(
+        db,
+        action=action,
+        user=user,
+        resource_type=resource_type,
+        severity=severity,
+        since=since,
+        until=until,
+        limit=limit,
+        offset=offset,
+    )
+    total = count_events(
+        db,
+        action=action,
+        user=user,
+        resource_type=resource_type,
+        severity=severity,
+        since=since,
+        until=until,
+    )
+    return {
+        "items": [_serialize(e) for e in entries],
+        "total": total,
+        "limit": limit,
+        "offset": offset,
+    }
+
+
+@router.get("/audit-logs/actions")
+@require_login
+async def list_distinct_actions(
+    request: Request,
+    db: Session = Depends(get_db),
+) -> list[str]:
+    """Return the distinct action values present in the audit log."""
+    from app.models import AuditLog
+
+    rows = db.query(AuditLog.action).distinct().order_by(AuditLog.action).all()
+    return [r[0] for r in rows]
+
+
+@router.get("/audit-logs/users")
+@require_login
+async def list_distinct_users(
+    request: Request,
+    db: Session = Depends(get_db),
+) -> list[str]:
+    """Return the distinct user values present in the audit log."""
+    from app.models import AuditLog
+
+    rows = db.query(AuditLog.user).distinct().order_by(AuditLog.user).all()
+    return [r[0] for r in rows]
+
+
+# ------------------------------------------------------------------
+# Helpers
+# ------------------------------------------------------------------
+
+
+def _serialize(entry) -> dict[str, Any]:
+    """Convert an AuditLog row to a JSON-safe dict."""
+    import json as _json
+
+    return {
+        "id": entry.id,
+        "timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
+        "user": entry.user,
+        "action": entry.action,
+        "resource_type": entry.resource_type,
+        "resource_id": entry.resource_id,
+        "ip_address": entry.ip_address,
+        "details": _json.loads(entry.details) if entry.details else None,
+        "severity": entry.severity,
+    }

app/config.py

@@ -849,6 +849,49 @@ class Settings(BaseSettings):
         ),
     )
 
+    # SIEM / External Audit Log Forwarding
+    # Forward audit events to external SIEM systems for centralised monitoring.
+    audit_siem_enabled: bool = Field(
+        default=False,
+        description="Enable forwarding of audit events to an external SIEM system.",
+    )
+    audit_siem_transport: str = Field(
+        default="syslog",
+        description=(
+            "Transport used to forward audit events. "
+            "Options: 'syslog' (RFC 5424 over UDP/TCP), 'http' (JSON POST to a webhook URL, "
+            "compatible with Splunk HEC, Logstash HTTP input, Grafana Loki, etc.)."
+        ),
+    )
+    audit_siem_syslog_host: str = Field(
+        default="localhost",
+        description="Hostname or IP of the syslog receiver.",
+    )
+    audit_siem_syslog_port: int = Field(
+        default=514,
+        description="Port of the syslog receiver.",
+    )
+    audit_siem_syslog_protocol: str = Field(
+        default="udp",
+        description="Protocol for syslog transport: 'udp' or 'tcp'.",
+    )
+    audit_siem_http_url: str = Field(
+        default="",
+        description=(
+            "HTTP endpoint URL for SIEM webhook delivery. "
+            "Supports Splunk HEC (https://splunk:8088/services/collector/event), "
+            "Logstash HTTP input, Grafana Loki push API, or any JSON-accepting endpoint."
+        ),
+    )
+    audit_siem_http_token: str = Field(
+        default="",
+        description="Bearer / HEC token included in the Authorization header of SIEM HTTP requests.",
+    )
+    audit_siem_http_custom_headers: str = Field(
+        default="",
+        description="Comma-separated 'Key:Value' pairs of extra headers for SIEM HTTP requests.",
+    )
+
     # UI / Appearance
     ui_default_color_scheme: str = Field(
         default="system",

app/models.py

@@ -146,6 +146,27 @@ class SettingsAuditLog(Base):
     action = Column(String, nullable=False)  # "update" or "delete"
 
 
+class AuditLog(Base):
+    """Comprehensive audit log for compliance tracking.
+
+    Records all significant actions: login/logout, document CRUD, settings
+    changes, and administrative operations.  Rows are append-only; the API
+    and service layer never update or delete entries.
+    """
+
+    __tablename__ = "audit_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+    timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
+    user = Column(String, nullable=False, index=True)  # Username or "anonymous" / "system"
+    action = Column(String, nullable=False, index=True)  # e.g. "login", "document.create", "settings.update"
+    resource_type = Column(String, nullable=True, index=True)  # e.g. "document", "user", "settings"
+    resource_id = Column(String, nullable=True)  # ID of the affected resource
+    ip_address = Column(String, nullable=True)  # Client IP address
+    details = Column(Text, nullable=True)  # JSON-encoded extra context
+    severity = Column(String(16), nullable=False, server_default="info")  # info / warning / error / critical
+
+
 class SavedSearch(Base):
     """User-defined saved search filters for quick access to frequently used filter combinations."""
 

app/utils/audit_service.py

@@ -0,0 +1,319 @@
+"""
+Comprehensive audit-event service for DocuElevate.
+
+Provides helpers to **record** audit events (append-only database writes)
+and to optionally **forward** them to external SIEM systems.
+
+Supported SIEM transports:
+* **Syslog** – RFC 5424 structured-data messages over UDP or TCP.
+* **HTTP** – JSON POST payloads compatible with Splunk HEC, Logstash
+  HTTP input, Grafana Loki push API, and any generic webhook endpoint.
+"""
+
+import json
+import logging
+import socket
+import threading
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+from fastapi import Request
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.middleware.audit_log import get_client_ip, get_username
+from app.models import AuditLog
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Public helpers
+# ---------------------------------------------------------------------------
+
+
+def record_event(
+    db: Session,
+    *,
+    action: str,
+    user: str = "system",
+    resource_type: str | None = None,
+    resource_id: str | None = None,
+    ip_address: str | None = None,
+    details: dict[str, Any] | None = None,
+    severity: str = "info",
+) -> AuditLog:
+    """Persist an audit event and optionally forward it to SIEM.
+
+    Args:
+        db: Active SQLAlchemy session.
+        action: Short action identifier (e.g. ``"login"``, ``"document.create"``).
+        user: Username performing the action.
+        resource_type: Category of the affected resource (``"document"``, ``"user"`` …).
+        resource_id: Identifier of the affected resource.
+        ip_address: Client IP address (``None`` when not applicable).
+        details: Arbitrary key/value context serialised as JSON.
+        severity: One of ``info``, ``warning``, ``error``, ``critical``.
+
+    Returns:
+        The newly created :class:`AuditLog` row.
+    """
+    details_json = json.dumps(details, default=str) if details else None
+
+    entry = AuditLog(
+        user=user,
+        action=action,
+        resource_type=resource_type,
+        resource_id=str(resource_id) if resource_id is not None else None,
+        ip_address=ip_address,
+        details=details_json,
+        severity=severity,
+    )
+    db.add(entry)
+    db.commit()
+    db.refresh(entry)
+
+    # Fire-and-forget SIEM forwarding in a background thread so we never
+    # block the request path.
+    if settings.audit_siem_enabled:
+        payload = _build_siem_payload(entry)
+        thread = threading.Thread(target=_forward_to_siem, args=(payload,), daemon=True)
+        thread.start()
+
+    return entry
+
+
+def record_event_from_request(
+    db: Session,
+    request: Request,
+    *,
+    action: str,
+    resource_type: str | None = None,
+    resource_id: str | None = None,
+    details: dict[str, Any] | None = None,
+    severity: str = "info",
+) -> AuditLog:
+    """Convenience wrapper that extracts user and IP from a :class:`Request`.
+
+    Args:
+        db: Active SQLAlchemy session.
+        request: The current HTTP request.
+        action: Short action identifier.
+        resource_type: Category of the affected resource.
+        resource_id: Identifier of the affected resource.
+        details: Arbitrary key/value context serialised as JSON.
+        severity: One of ``info``, ``warning``, ``error``, ``critical``.
+
+    Returns:
+        The newly created :class:`AuditLog` row.
+    """
+    return record_event(
+        db,
+        action=action,
+        user=get_username(request),
+        resource_type=resource_type,
+        resource_id=resource_id,
+        ip_address=get_client_ip(request),
+        details=details,
+        severity=severity,
+    )
+
+
+def query_events(
+    db: Session,
+    *,
+    action: str | None = None,
+    user: str | None = None,
+    resource_type: str | None = None,
+    severity: str | None = None,
+    since: datetime | None = None,
+    until: datetime | None = None,
+    limit: int = 200,
+    offset: int = 0,
+) -> list[AuditLog]:
+    """Query audit log entries with optional filtering.
+
+    Args:
+        db: Active SQLAlchemy session.
+        action: Filter by action string (exact match).
+        user: Filter by username (exact match).
+        resource_type: Filter by resource type (exact match).
+        severity: Filter by severity level (exact match).
+        since: Only events at or after this timestamp.
+        until: Only events at or before this timestamp.
+        limit: Maximum number of rows to return.
+        offset: Number of rows to skip (for pagination).
+
+    Returns:
+        List of :class:`AuditLog` rows ordered by *timestamp descending*.
+    """
+    q = db.query(AuditLog)
+    if action:
+        q = q.filter(AuditLog.action == action)
+    if user:
+        q = q.filter(AuditLog.user == user)
+    if resource_type:
+        q = q.filter(AuditLog.resource_type == resource_type)
+    if severity:
+        q = q.filter(AuditLog.severity == severity)
+    if since:
+        q = q.filter(AuditLog.timestamp >= since)
+    if until:
+        q = q.filter(AuditLog.timestamp <= until)
+    return q.order_by(AuditLog.timestamp.desc()).offset(offset).limit(limit).all()
+
+
+def count_events(
+    db: Session,
+    *,
+    action: str | None = None,
+    user: str | None = None,
+    resource_type: str | None = None,
+    severity: str | None = None,
+    since: datetime | None = None,
+    until: datetime | None = None,
+) -> int:
+    """Return the total count of events matching the given filters.
+
+    Args:
+        db: Active SQLAlchemy session.
+        action: Filter by action string.
+        user: Filter by username.
+        resource_type: Filter by resource type.
+        severity: Filter by severity level.
+        since: Only events at or after this timestamp.
+        until: Only events at or before this timestamp.
+
+    Returns:
+        Integer count.
+    """
+    q = db.query(AuditLog)
+    if action:
+        q = q.filter(AuditLog.action == action)
+    if user:
+        q = q.filter(AuditLog.user == user)
+    if resource_type:
+        q = q.filter(AuditLog.resource_type == resource_type)
+    if severity:
+        q = q.filter(AuditLog.severity == severity)
+    if since:
+        q = q.filter(AuditLog.timestamp >= since)
+    if until:
+        q = q.filter(AuditLog.timestamp <= until)
+    return q.count()
+
+
+# ---------------------------------------------------------------------------
+# SIEM forwarding internals
+# ---------------------------------------------------------------------------
+
+_SYSLOG_FACILITY_LOCAL0 = 16
+_SYSLOG_SEVERITY_MAP = {
+    "info": 6,
+    "warning": 4,
+    "error": 3,
+    "critical": 2,
+}
+
+
+def _build_siem_payload(entry: AuditLog) -> dict[str, Any]:
+    """Convert an :class:`AuditLog` row into a plain dict for SIEM delivery."""
+    ts = entry.timestamp if entry.timestamp else datetime.now(timezone.utc)
+    return {
+        "id": entry.id,
+        "timestamp": ts.isoformat(),
+        "user": entry.user,
+        "action": entry.action,
+        "resource_type": entry.resource_type,
+        "resource_id": entry.resource_id,
+        "ip_address": entry.ip_address,
+        "details": entry.details,
+        "severity": entry.severity,
+        "source": "docuelevate",
+    }
+
+
+def _forward_to_siem(payload: dict[str, Any]) -> None:
+    """Route a SIEM payload to the configured transport."""
+    transport = settings.audit_siem_transport.lower()
+    try:
+        if transport == "syslog":
+            _send_syslog(payload)
+        elif transport == "http":
+            _send_http(payload)
+        else:
+            logger.warning("Unknown SIEM transport %r; skipping forwarding", transport)
+    except Exception:
+        logger.exception("Failed to forward audit event to SIEM (%s)", transport)
+
+
+def _send_syslog(payload: dict[str, Any]) -> None:
+    """Send a RFC 5424 syslog message to the configured receiver."""
+    severity_num = _SYSLOG_SEVERITY_MAP.get(payload.get("severity", "info"), 6)
+    priority = _SYSLOG_FACILITY_LOCAL0 * 8 + severity_num
+    ts = payload.get("timestamp", datetime.now(timezone.utc).isoformat())
+    hostname = socket.gethostname()
+    app_name = "docuelevate"
+    msg_id = payload.get("action", "-")
+
+    # Structured data (SD) element with key event fields.
+    sd = (
+        f'[docuelevate@0 user="{payload.get("user", "-")}" '
+        f'action="{payload.get("action", "-")}" '
+        f'resource_type="{payload.get("resource_type", "-")}" '
+        f'resource_id="{payload.get("resource_id", "-")}" '
+        f'ip="{payload.get("ip_address", "-")}"]'
+    )
+    message = json.dumps(payload, default=str)
+    syslog_msg = f"<{priority}>1 {ts} {hostname} {app_name} - {msg_id} {sd} {message}"
+
+    proto = settings.audit_siem_syslog_protocol.lower()
+    host = settings.audit_siem_syslog_host
+    port = settings.audit_siem_syslog_port
+
+    if proto == "tcp":
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.settimeout(5)
+            sock.connect((host, port))
+            sock.sendall(syslog_msg.encode("utf-8"))
+    else:
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+            sock.settimeout(5)
+            sock.sendto(syslog_msg.encode("utf-8"), (host, port))
+
+    logger.debug("Syslog audit event sent to %s:%s (%s)", host, port, proto)
+
+
+def _send_http(payload: dict[str, Any]) -> None:
+    """POST a JSON audit event to the configured HTTP endpoint."""
+    url = settings.audit_siem_http_url
+    if not url:
+        logger.warning("SIEM HTTP URL not configured; skipping HTTP forwarding")
+        return
+
+    headers: dict[str, str] = {"Content-Type": "application/json"}
+    token = settings.audit_siem_http_token
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    # Parse custom headers (comma-separated "Key:Value" pairs).
+    raw_custom = settings.audit_siem_http_custom_headers
+    if raw_custom:
+        for raw_pair in raw_custom.split(","):
+            pair = raw_pair.strip()
+            if ":" in pair:
+                k, _, v = pair.partition(":")
+                headers[k.strip()] = v.strip()
+
+    # Wrap in Splunk HEC-style envelope when URL contains ``/services/collector``.
+    body: dict[str, Any]
+    if "/services/collector" in url:
+        body = {"event": payload, "sourcetype": "docuelevate:audit", "source": "docuelevate"}
+    else:
+        body = payload
+
+    with httpx.Client(timeout=10) as client:
+        resp = client.post(url, json=body, headers=headers)
+        resp.raise_for_status()
+
+    logger.debug("HTTP audit event forwarded to %s (status %s)", url, resp.status_code)

app/utils/db_migrate.py

@@ -32,6 +32,7 @@ _TABLE_ORDER = [
     "processing_logs",
     "application_settings",
     "settings_audit_log",
+    "audit_logs",
     "saved_searches",
     "webhook_configs",
 ]

app/views/__init__.py

@@ -6,6 +6,7 @@ from fastapi import APIRouter
 
 from app.views.admin_users import router as admin_users_router
 from app.views.api_tokens import router as api_tokens_router
+from app.views.audit_logs import router as audit_logs_router
 from app.views.backup import router as backup_router
 from app.views.db_wizard import router as db_wizard_router
 from app.views.dropbox import router as dropbox_router
@@ -60,4 +61,5 @@ router.include_router(imap_accounts_router)  # Per-user IMAP ingestion accounts
 router.include_router(integrations_router)  # Unified integrations dashboard
 router.include_router(notifications_router)  # User notification dashboard
 router.include_router(scheduled_jobs_router)  # Admin scheduled batch jobs
+router.include_router(audit_logs_router)  # Comprehensive audit log viewer
 router.include_router(help_router)  # Built-in help / How-To docs

app/views/audit_logs.py

@@ -0,0 +1,46 @@
+"""
+Audit log viewer UI — admin-only page with filtering and SIEM status.
+"""
+
+import logging
+
+from fastapi import Depends, HTTPException, Request, status
+from sqlalchemy.orm import Session
+
+from app.views.base import APIRouter, get_db, require_login, settings, templates
+from app.views.settings import require_admin_access
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.get("/admin/audit-logs")
+@require_login
+@require_admin_access
+async def audit_logs_page(request: Request, db: Session = Depends(get_db)):
+    """Comprehensive audit log viewer with filtering controls.
+
+    Displays a chronological log of all significant actions: logins,
+    document operations, settings changes, and admin actions.  The
+    actual data is fetched client-side via the ``/api/audit-logs`` JSON
+    endpoint so that filters, pagination, and live refresh work without
+    full-page reloads.
+    """
+    try:
+        siem_enabled = settings.audit_siem_enabled
+        siem_transport = settings.audit_siem_transport if siem_enabled else None
+        return templates.TemplateResponse(
+            "audit_logs.html",
+            {
+                "request": request,
+                "app_version": settings.version,
+                "siem_enabled": siem_enabled,
+                "siem_transport": siem_transport,
+            },
+        )
+    except Exception as e:
+        logger.error("Error loading audit logs page: %s", e)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to load audit logs page",
+        )