diff --git a/.env.demo b/.env.demo index ba0cff69..9412e754 100644 --- a/.env.demo +++ b/.env.demo @@ -14,6 +14,17 @@ COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, S # LOG_LEVEL=INFO # DEBUG=false +# Log output format: "text" (human-readable, default) or "json" (structured JSON lines). +# Use "json" when shipping logs to Grafana Loki, Splunk, ELK, Datadog, or any SIEM. +# LOG_FORMAT=text + +# Forward application logs to a syslog receiver (in addition to stdout). +# Useful for traditional (non-container) deployments and centralised SIEM ingestion. +# LOG_SYSLOG_ENABLED=false +# LOG_SYSLOG_HOST=localhost +# LOG_SYSLOG_PORT=514 +# LOG_SYSLOG_PROTOCOL=udp # udp | tcp + # **UI / Appearance** # Default colour scheme: system (follow OS), light, or dark # Individual users can always override with the navbar dark-mode toggle. diff --git a/app/config.py b/app/config.py index ebcd078e..30b434d1 100644 --- a/app/config.py +++ b/app/config.py @@ -62,6 +62,37 @@ class Settings(BaseSettings): ), ) + # Log output format. ``text`` is the human-readable default. + # ``json`` emits one JSON object per line, ideal for log collectors + # (Promtail, Fluentd, Filebeat, Datadog agent) and SIEM ingestion. + log_format: str = Field( + default="text", + description=( + "Log output format: 'text' (human-readable, default) or " + "'json' (structured JSON lines for SIEM / log aggregation)." + ), + ) + + # Optional syslog forwarding for application logs (not just audit events). + # When enabled, a Python SysLogHandler is added to the root logger so that + # every log message is also sent to the configured syslog receiver. + log_syslog_enabled: bool = Field( + default=False, + description="Forward application logs to a syslog receiver in addition to stdout.", + ) + log_syslog_host: str = Field( + default="localhost", + description="Hostname or IP of the syslog receiver for application logs.", + ) + log_syslog_port: int = Field( + default=514, + description="Port of the syslog receiver for application logs.", + ) + log_syslog_protocol: str = Field( + default="udp", + description="Protocol for syslog transport: 'udp' or 'tcp'.", + ) + # Making Dropbox optional dropbox_enabled: bool = Field( default=True, diff --git a/app/main.py b/app/main.py index 63af224e..61b1a890 100644 --- a/app/main.py +++ b/app/main.py @@ -47,6 +47,13 @@ from app.views.files import router as files_router # level is automatically lowered to ``DEBUG``. # • Default (neither flag set): ``INFO``. # +# ``LOG_FORMAT=json`` enables structured JSON lines on stdout, suitable for +# Promtail, Fluentd, Filebeat, Datadog, Splunk UF, or any log collector. +# +# ``LOG_SYSLOG_ENABLED=true`` adds a Python SysLogHandler so that every log +# message is also forwarded to the configured syslog receiver — useful for +# traditional (non-container) deployments and centralised SIEM ingestion. +# # Noisy third-party loggers (httpx, httpcore, authlib, etc.) are pinned to # WARNING when the app-level is DEBUG to keep output useful. # --------------------------------------------------------------------------- @@ -58,12 +65,66 @@ else: _effective_level_int = getattr(logging, _effective_level, logging.INFO) -logging.basicConfig( - level=_effective_level_int, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - force=True, -) + +class _JsonFormatter(logging.Formatter): + """Emit one JSON object per log line for machine consumption. + + Fields emitted: ``timestamp``, ``level``, ``logger``, ``message``, + ``module``, ``funcName``, ``lineno``, and — when present — ``exc_info``. + Compatible with Grafana Loki, Splunk, ELK, Datadog, and most SIEM tools. + """ + + def format(self, record: logging.LogRecord) -> str: + import json as _json + from datetime import datetime as _dt + from datetime import timezone as _tz + + log_entry: dict = { + "timestamp": _dt.fromtimestamp(record.created, tz=_tz.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "module": record.module, + "funcName": record.funcName, + "lineno": record.lineno, + } + if record.exc_info and record.exc_info[1] is not None: + log_entry["exc_info"] = self.formatException(record.exc_info) + return _json.dumps(log_entry, default=str) + + +# Choose formatter based on LOG_FORMAT setting +if settings.log_format.lower() == "json": + _handler = logging.StreamHandler() + _handler.setFormatter(_JsonFormatter()) + logging.root.handlers = [_handler] + logging.root.setLevel(_effective_level_int) +else: + logging.basicConfig( + level=_effective_level_int, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + force=True, + ) + +# Optional: forward application logs to a syslog receiver +if settings.log_syslog_enabled: + import logging.handlers as _lh + import socket as _socket + + _proto = settings.log_syslog_protocol.lower() + _socktype = _socket.SOCK_STREAM if _proto == "tcp" else _socket.SOCK_DGRAM + _syslog_handler = _lh.SysLogHandler( + address=(settings.log_syslog_host, settings.log_syslog_port), + socktype=_socktype, + ) + _syslog_handler.setLevel(_effective_level_int) + # Use the same formatter as stdout (text or JSON) + if settings.log_format.lower() == "json": + _syslog_handler.setFormatter(_JsonFormatter()) + else: + _syslog_handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s")) + logging.root.addHandler(_syslog_handler) # Keep noisy third-party loggers quiet at DEBUG level if _effective_level_int <= logging.DEBUG: @@ -78,7 +139,14 @@ if _effective_level_int <= logging.DEBUG: ): logging.getLogger(_noisy).setLevel(logging.WARNING) -logging.getLogger(__name__).info("Root logging level set to %s (debug=%s)", _effective_level, settings.debug) +_startup_logger = logging.getLogger(__name__) +_startup_logger.info( + "Root logging level set to %s (debug=%s, format=%s, syslog=%s)", + _effective_level, + settings.debug, + settings.log_format, + settings.log_syslog_enabled, +) # Load configuration from .env for the session key config = Config(".env") diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 9b9bdc71..37acf9c2 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -485,6 +485,59 @@ LOG_LEVEL=WARNING > **Tip:** At `DEBUG` level, noisy third-party libraries (httpx, authlib, urllib3, etc.) are automatically pinned to `WARNING` so that application debug output remains readable. +#### Structured JSON Logging + +Set `LOG_FORMAT=json` to emit structured JSON lines on stdout — one JSON object per log message. This is the standard format for log collectors and SIEM tools: + +| **Variable** | **Description** | **Default** | +|-------------|----------------|-------------| +| `LOG_FORMAT` | Log output format: `text` (human-readable) or `json` (structured JSON lines). | `text` | + +Each JSON log line contains: `timestamp` (ISO 8601), `level`, `logger`, `message`, `module`, `funcName`, `lineno`, and `exc_info` (when an exception is logged). + +```bash +# Enable JSON logging for SIEM / log aggregation +LOG_FORMAT=json +``` + +**Example JSON output:** +```json +{"timestamp": "2025-03-16T09:18:05.192000+00:00", "level": "INFO", "logger": "app.auth", "message": "[SECURITY] OAUTH_LOGIN_SUCCESS user=alice@example.com admin=False", "module": "auth", "funcName": "oauth_callback", "lineno": 654} +``` + +**Compatible with:** +- **Grafana Loki** — Promtail scrapes JSON from Docker stdout +- **Splunk** — Universal Forwarder or HEC with JSON sourcetype +- **ELK / OpenSearch** — Filebeat with JSON codec +- **Datadog** — Agent auto-parses JSON logs +- **Fluentd / Vector** — JSON input plugin +- **Docker log drivers** — `--log-driver=json-file` (default) preserves structure + +#### Syslog Forwarding (Application Logs) + +For traditional (non-container) deployments, application logs can be forwarded directly to a syslog receiver. This is **separate** from audit-log SIEM forwarding (see below) — it sends _every_ Python log message, not just audit events. + +| **Variable** | **Description** | **Default** | +|-------------|----------------|-------------| +| `LOG_SYSLOG_ENABLED` | Forward application logs to a syslog receiver in addition to stdout. | `false` | +| `LOG_SYSLOG_HOST` | Hostname or IP of the syslog receiver. | `localhost` | +| `LOG_SYSLOG_PORT` | Port of the syslog receiver. | `514` | +| `LOG_SYSLOG_PROTOCOL` | Protocol: `udp` or `tcp`. | `udp` | + +```bash +# Forward all application logs to syslog +LOG_SYSLOG_ENABLED=true +LOG_SYSLOG_HOST=syslog.internal.example.com +LOG_SYSLOG_PORT=514 +LOG_SYSLOG_PROTOCOL=udp + +# Combine with JSON format for structured syslog messages +LOG_FORMAT=json +LOG_SYSLOG_ENABLED=true +``` + +> **Note:** When `LOG_FORMAT=json`, syslog messages are also sent as JSON. When `LOG_FORMAT=text`, syslog messages use the standard `name - level - message` format. + ### Audit Logging DocuElevate provides comprehensive audit logging that records significant actions (logins, document CRUD, settings changes) to an append-only database table. Every entry captures the timestamp, user, action, resource, client IP, and optional JSON details. diff --git a/tests/test_logging_config.py b/tests/test_logging_config.py index bcec61ea..9c24c13c 100644 --- a/tests/test_logging_config.py +++ b/tests/test_logging_config.py @@ -75,8 +75,6 @@ class TestEffectiveLogLevel: env = os.environ.copy() env.pop("LOG_LEVEL", None) with patch.dict(os.environ, env, clear=True): - from app.config import Settings as S - s = Settings( database_url="sqlite:///test.db", redis_url="redis://localhost:6379", @@ -100,8 +98,6 @@ class TestEffectiveLogLevel: def test_explicit_log_level_overrides_debug(self): """When LOG_LEVEL is explicitly set, it takes precedence over DEBUG=True.""" with patch.dict(os.environ, {"LOG_LEVEL": "WARNING", "DEBUG": "true"}, clear=False): - from app.config import Settings as S - s = Settings( database_url="sqlite:///test.db", redis_url="redis://localhost:6379", @@ -171,3 +167,119 @@ class TestLoggingConfiguredAtStartup: root = logging.getLogger() # The test env doesn't set DEBUG=True, so the level should be INFO (20) assert root.level <= logging.INFO + + +@pytest.mark.unit +class TestJsonFormatter: + """Tests for the _JsonFormatter used when LOG_FORMAT=json.""" + + def _make_formatter(self): + """Lazily import the JSON formatter from main module.""" + from app.main import _JsonFormatter + + return _JsonFormatter() + + def test_output_is_valid_json(self): + """JSON formatter output should be parseable JSON.""" + import json + + fmt = self._make_formatter() + record = logging.LogRecord( + name="test.logger", + level=logging.INFO, + pathname="test.py", + lineno=42, + msg="Hello %s", + args=("world",), + exc_info=None, + ) + result = fmt.format(record) + parsed = json.loads(result) + assert parsed["level"] == "INFO" + assert parsed["logger"] == "test.logger" + assert parsed["message"] == "Hello world" + assert parsed["lineno"] == 42 + + def test_includes_timestamp_iso8601(self): + """JSON output should contain an ISO 8601 timestamp.""" + import json + + fmt = self._make_formatter() + record = logging.LogRecord( + name="x", + level=logging.DEBUG, + pathname="x.py", + lineno=1, + msg="test", + args=(), + exc_info=None, + ) + parsed = json.loads(fmt.format(record)) + assert "timestamp" in parsed + # ISO 8601 timestamps contain "T" and "+00:00" (UTC) + assert "T" in parsed["timestamp"] + + def test_includes_exc_info_when_present(self): + """JSON output should include exc_info when an exception is logged.""" + import json + + fmt = self._make_formatter() + try: + raise ValueError("boom") # noqa: TRY301 + except ValueError: + import sys + + record = logging.LogRecord( + name="x", + level=logging.ERROR, + pathname="x.py", + lineno=1, + msg="error", + args=(), + exc_info=sys.exc_info(), + ) + parsed = json.loads(fmt.format(record)) + assert "exc_info" in parsed + assert "ValueError" in parsed["exc_info"] + + +@pytest.mark.unit +class TestLogFormatSetting: + """Tests for the log_format and log_syslog_* config fields.""" + + _BASE_KWARGS = { + "database_url": "sqlite:///test.db", + "redis_url": "redis://localhost:6379", + "openai_api_key": "test", + "azure_ai_key": "test", + "azure_region": "test", + "azure_endpoint": "https://test.example.com", + "gotenberg_url": "http://localhost:3000", + "workdir": "/tmp", + "auth_enabled": False, + "session_secret": None, + } + + def test_log_format_default_is_text(self): + """Test that log_format defaults to 'text'.""" + config = Settings(**self._BASE_KWARGS) + assert config.log_format == "text" + + def test_log_format_accepts_json(self): + """Test that log_format accepts 'json'.""" + config = Settings(**self._BASE_KWARGS, log_format="json") + assert config.log_format == "json" + + def test_log_syslog_defaults(self): + """Test syslog forwarding defaults.""" + config = Settings(**self._BASE_KWARGS) + assert config.log_syslog_enabled is False + assert config.log_syslog_host == "localhost" + assert config.log_syslog_port == 514 + assert config.log_syslog_protocol == "udp" + + def test_log_syslog_can_be_enabled(self): + """Test that syslog forwarding can be enabled.""" + config = Settings(**self._BASE_KWARGS, log_syslog_enabled=True, log_syslog_host="syslog.example.com") + assert config.log_syslog_enabled is True + assert config.log_syslog_host == "syslog.example.com"