feat(config): add JSON structured logging and syslog forwarding for application logs

- Add LOG_FORMAT setting (text/json) for structured JSON log output
- Add LOG_SYSLOG_* settings for direct syslog forwarding of app logs
- JSON format compatible with Grafana Loki, Splunk, ELK, Datadog
- Syslog forwarding uses Python's SysLogHandler (UDP/TCP)
- Update .env.demo and ConfigurationGuide.md with all new settings
- Add tests for JSON formatter and syslog config fields

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

.env.demo

@@ -14,6 +14,17 @@ COMPLIANCE_ENABLED=true  # Enable compliance templates dashboard (GDPR, HIPAA, S
 # LOG_LEVEL=INFO
 # DEBUG=false
 
+# Log output format: "text" (human-readable, default) or "json" (structured JSON lines).
+# Use "json" when shipping logs to Grafana Loki, Splunk, ELK, Datadog, or any SIEM.
+# LOG_FORMAT=text
+
+# Forward application logs to a syslog receiver (in addition to stdout).
+# Useful for traditional (non-container) deployments and centralised SIEM ingestion.
+# LOG_SYSLOG_ENABLED=false
+# LOG_SYSLOG_HOST=localhost
+# LOG_SYSLOG_PORT=514
+# LOG_SYSLOG_PROTOCOL=udp          # udp | tcp
+
 # **UI / Appearance**
 # Default colour scheme: system (follow OS), light, or dark
 # Individual users can always override with the navbar dark-mode toggle.

app/config.py

@@ -62,6 +62,37 @@ class Settings(BaseSettings):
         ),
     )
 
+    # Log output format.  ``text`` is the human-readable default.
+    # ``json`` emits one JSON object per line, ideal for log collectors
+    # (Promtail, Fluentd, Filebeat, Datadog agent) and SIEM ingestion.
+    log_format: str = Field(
+        default="text",
+        description=(
+            "Log output format: 'text' (human-readable, default) or "
+            "'json' (structured JSON lines for SIEM / log aggregation)."
+        ),
+    )
+
+    # Optional syslog forwarding for application logs (not just audit events).
+    # When enabled, a Python SysLogHandler is added to the root logger so that
+    # every log message is also sent to the configured syslog receiver.
+    log_syslog_enabled: bool = Field(
+        default=False,
+        description="Forward application logs to a syslog receiver in addition to stdout.",
+    )
+    log_syslog_host: str = Field(
+        default="localhost",
+        description="Hostname or IP of the syslog receiver for application logs.",
+    )
+    log_syslog_port: int = Field(
+        default=514,
+        description="Port of the syslog receiver for application logs.",
+    )
+    log_syslog_protocol: str = Field(
+        default="udp",
+        description="Protocol for syslog transport: 'udp' or 'tcp'.",
+    )
+
     # Making Dropbox optional
     dropbox_enabled: bool = Field(
         default=True,

app/main.py

@@ -47,6 +47,13 @@ from app.views.files import router as files_router
 #     level is automatically lowered to ``DEBUG``.
 #   • Default (neither flag set): ``INFO``.
 #
+# ``LOG_FORMAT=json`` enables structured JSON lines on stdout, suitable for
+# Promtail, Fluentd, Filebeat, Datadog, Splunk UF, or any log collector.
+#
+# ``LOG_SYSLOG_ENABLED=true`` adds a Python SysLogHandler so that every log
+# message is also forwarded to the configured syslog receiver — useful for
+# traditional (non-container) deployments and centralised SIEM ingestion.
+#
 # Noisy third-party loggers (httpx, httpcore, authlib, etc.) are pinned to
 # WARNING when the app-level is DEBUG to keep output useful.
 # ---------------------------------------------------------------------------
@@ -58,12 +65,66 @@ else:
 
 _effective_level_int = getattr(logging, _effective_level, logging.INFO)
 
-logging.basicConfig(
+
-    level=_effective_level_int,
+class _JsonFormatter(logging.Formatter):
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    """Emit one JSON object per log line for machine consumption.
-    datefmt="%Y-%m-%d %H:%M:%S",
+
-    force=True,
+    Fields emitted: ``timestamp``, ``level``, ``logger``, ``message``,
-)
+    ``module``, ``funcName``, ``lineno``, and — when present — ``exc_info``.
+    Compatible with Grafana Loki, Splunk, ELK, Datadog, and most SIEM tools.
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        import json as _json
+        from datetime import datetime as _dt
+        from datetime import timezone as _tz
+
+        log_entry: dict = {
+            "timestamp": _dt.fromtimestamp(record.created, tz=_tz.utc).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "module": record.module,
+            "funcName": record.funcName,
+            "lineno": record.lineno,
+        }
+        if record.exc_info and record.exc_info[1] is not None:
+            log_entry["exc_info"] = self.formatException(record.exc_info)
+        return _json.dumps(log_entry, default=str)
+
+
+# Choose formatter based on LOG_FORMAT setting
+if settings.log_format.lower() == "json":
+    _handler = logging.StreamHandler()
+    _handler.setFormatter(_JsonFormatter())
+    logging.root.handlers = [_handler]
+    logging.root.setLevel(_effective_level_int)
+else:
+    logging.basicConfig(
+        level=_effective_level_int,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        force=True,
+    )
+
+# Optional: forward application logs to a syslog receiver
+if settings.log_syslog_enabled:
+    import logging.handlers as _lh
+    import socket as _socket
+
+    _proto = settings.log_syslog_protocol.lower()
+    _socktype = _socket.SOCK_STREAM if _proto == "tcp" else _socket.SOCK_DGRAM
+    _syslog_handler = _lh.SysLogHandler(
+        address=(settings.log_syslog_host, settings.log_syslog_port),
+        socktype=_socktype,
+    )
+    _syslog_handler.setLevel(_effective_level_int)
+    # Use the same formatter as stdout (text or JSON)
+    if settings.log_format.lower() == "json":
+        _syslog_handler.setFormatter(_JsonFormatter())
+    else:
+        _syslog_handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s"))
+    logging.root.addHandler(_syslog_handler)
 
 # Keep noisy third-party loggers quiet at DEBUG level
 if _effective_level_int <= logging.DEBUG:
@@ -78,7 +139,14 @@ if _effective_level_int <= logging.DEBUG:
     ):
         logging.getLogger(_noisy).setLevel(logging.WARNING)
 
-logging.getLogger(__name__).info("Root logging level set to %s (debug=%s)", _effective_level, settings.debug)
+_startup_logger = logging.getLogger(__name__)
+_startup_logger.info(
+    "Root logging level set to %s (debug=%s, format=%s, syslog=%s)",
+    _effective_level,
+    settings.debug,
+    settings.log_format,
+    settings.log_syslog_enabled,
+)
 
 # Load configuration from .env for the session key
 config = Config(".env")

docs/ConfigurationGuide.md

@@ -485,6 +485,59 @@ LOG_LEVEL=WARNING
 
 > **Tip:** At `DEBUG` level, noisy third-party libraries (httpx, authlib, urllib3, etc.) are automatically pinned to `WARNING` so that application debug output remains readable.
 
+#### Structured JSON Logging
+
+Set `LOG_FORMAT=json` to emit structured JSON lines on stdout — one JSON object per log message. This is the standard format for log collectors and SIEM tools:
+
+| **Variable** | **Description** | **Default** |
+|-------------|----------------|-------------|
+| `LOG_FORMAT` | Log output format: `text` (human-readable) or `json` (structured JSON lines). | `text` |
+
+Each JSON log line contains: `timestamp` (ISO 8601), `level`, `logger`, `message`, `module`, `funcName`, `lineno`, and `exc_info` (when an exception is logged).
+
+```bash
+# Enable JSON logging for SIEM / log aggregation
+LOG_FORMAT=json
+```
+
+**Example JSON output:**
+```json
+{"timestamp": "2025-03-16T09:18:05.192000+00:00", "level": "INFO", "logger": "app.auth", "message": "[SECURITY] OAUTH_LOGIN_SUCCESS user=alice@example.com admin=False", "module": "auth", "funcName": "oauth_callback", "lineno": 654}
+```
+
+**Compatible with:**
+- **Grafana Loki** — Promtail scrapes JSON from Docker stdout
+- **Splunk** — Universal Forwarder or HEC with JSON sourcetype
+- **ELK / OpenSearch** — Filebeat with JSON codec
+- **Datadog** — Agent auto-parses JSON logs
+- **Fluentd / Vector** — JSON input plugin
+- **Docker log drivers** — `--log-driver=json-file` (default) preserves structure
+
+#### Syslog Forwarding (Application Logs)
+
+For traditional (non-container) deployments, application logs can be forwarded directly to a syslog receiver. This is **separate** from audit-log SIEM forwarding (see below) — it sends _every_ Python log message, not just audit events.
+
+| **Variable** | **Description** | **Default** |
+|-------------|----------------|-------------|
+| `LOG_SYSLOG_ENABLED` | Forward application logs to a syslog receiver in addition to stdout. | `false` |
+| `LOG_SYSLOG_HOST` | Hostname or IP of the syslog receiver. | `localhost` |
+| `LOG_SYSLOG_PORT` | Port of the syslog receiver. | `514` |
+| `LOG_SYSLOG_PROTOCOL` | Protocol: `udp` or `tcp`. | `udp` |
+
+```bash
+# Forward all application logs to syslog
+LOG_SYSLOG_ENABLED=true
+LOG_SYSLOG_HOST=syslog.internal.example.com
+LOG_SYSLOG_PORT=514
+LOG_SYSLOG_PROTOCOL=udp
+
+# Combine with JSON format for structured syslog messages
+LOG_FORMAT=json
+LOG_SYSLOG_ENABLED=true
+```
+
+> **Note:** When `LOG_FORMAT=json`, syslog messages are also sent as JSON. When `LOG_FORMAT=text`, syslog messages use the standard `name - level - message` format.
+
 ### Audit Logging
 
 DocuElevate provides comprehensive audit logging that records significant actions (logins, document CRUD, settings changes) to an append-only database table. Every entry captures the timestamp, user, action, resource, client IP, and optional JSON details.

tests/test_logging_config.py

@@ -75,8 +75,6 @@ class TestEffectiveLogLevel:
             env = os.environ.copy()
             env.pop("LOG_LEVEL", None)
             with patch.dict(os.environ, env, clear=True):
-                from app.config import Settings as S
-
                 s = Settings(
                     database_url="sqlite:///test.db",
                     redis_url="redis://localhost:6379",
@@ -100,8 +98,6 @@ class TestEffectiveLogLevel:
     def test_explicit_log_level_overrides_debug(self):
         """When LOG_LEVEL is explicitly set, it takes precedence over DEBUG=True."""
         with patch.dict(os.environ, {"LOG_LEVEL": "WARNING", "DEBUG": "true"}, clear=False):
-            from app.config import Settings as S
-
             s = Settings(
                 database_url="sqlite:///test.db",
                 redis_url="redis://localhost:6379",
@@ -171,3 +167,119 @@ class TestLoggingConfiguredAtStartup:
         root = logging.getLogger()
         # The test env doesn't set DEBUG=True, so the level should be INFO (20)
         assert root.level <= logging.INFO
+
+
+@pytest.mark.unit
+class TestJsonFormatter:
+    """Tests for the _JsonFormatter used when LOG_FORMAT=json."""
+
+    def _make_formatter(self):
+        """Lazily import the JSON formatter from main module."""
+        from app.main import _JsonFormatter
+
+        return _JsonFormatter()
+
+    def test_output_is_valid_json(self):
+        """JSON formatter output should be parseable JSON."""
+        import json
+
+        fmt = self._make_formatter()
+        record = logging.LogRecord(
+            name="test.logger",
+            level=logging.INFO,
+            pathname="test.py",
+            lineno=42,
+            msg="Hello %s",
+            args=("world",),
+            exc_info=None,
+        )
+        result = fmt.format(record)
+        parsed = json.loads(result)
+        assert parsed["level"] == "INFO"
+        assert parsed["logger"] == "test.logger"
+        assert parsed["message"] == "Hello world"
+        assert parsed["lineno"] == 42
+
+    def test_includes_timestamp_iso8601(self):
+        """JSON output should contain an ISO 8601 timestamp."""
+        import json
+
+        fmt = self._make_formatter()
+        record = logging.LogRecord(
+            name="x",
+            level=logging.DEBUG,
+            pathname="x.py",
+            lineno=1,
+            msg="test",
+            args=(),
+            exc_info=None,
+        )
+        parsed = json.loads(fmt.format(record))
+        assert "timestamp" in parsed
+        # ISO 8601 timestamps contain "T" and "+00:00" (UTC)
+        assert "T" in parsed["timestamp"]
+
+    def test_includes_exc_info_when_present(self):
+        """JSON output should include exc_info when an exception is logged."""
+        import json
+
+        fmt = self._make_formatter()
+        try:
+            raise ValueError("boom")  # noqa: TRY301
+        except ValueError:
+            import sys
+
+            record = logging.LogRecord(
+                name="x",
+                level=logging.ERROR,
+                pathname="x.py",
+                lineno=1,
+                msg="error",
+                args=(),
+                exc_info=sys.exc_info(),
+            )
+        parsed = json.loads(fmt.format(record))
+        assert "exc_info" in parsed
+        assert "ValueError" in parsed["exc_info"]
+
+
+@pytest.mark.unit
+class TestLogFormatSetting:
+    """Tests for the log_format and log_syslog_* config fields."""
+
+    _BASE_KWARGS = {
+        "database_url": "sqlite:///test.db",
+        "redis_url": "redis://localhost:6379",
+        "openai_api_key": "test",
+        "azure_ai_key": "test",
+        "azure_region": "test",
+        "azure_endpoint": "https://test.example.com",
+        "gotenberg_url": "http://localhost:3000",
+        "workdir": "/tmp",
+        "auth_enabled": False,
+        "session_secret": None,
+    }
+
+    def test_log_format_default_is_text(self):
+        """Test that log_format defaults to 'text'."""
+        config = Settings(**self._BASE_KWARGS)
+        assert config.log_format == "text"
+
+    def test_log_format_accepts_json(self):
+        """Test that log_format accepts 'json'."""
+        config = Settings(**self._BASE_KWARGS, log_format="json")
+        assert config.log_format == "json"
+
+    def test_log_syslog_defaults(self):
+        """Test syslog forwarding defaults."""
+        config = Settings(**self._BASE_KWARGS)
+        assert config.log_syslog_enabled is False
+        assert config.log_syslog_host == "localhost"
+        assert config.log_syslog_port == 514
+        assert config.log_syslog_protocol == "udp"
+
+    def test_log_syslog_can_be_enabled(self):
+        """Test that syslog forwarding can be enabled."""
+        config = Settings(**self._BASE_KWARGS, log_syslog_enabled=True, log_syslog_host="syslog.example.com")
+        assert config.log_syslog_enabled is True
+        assert config.log_syslog_host == "syslog.example.com"