gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): add JSON structured logging and syslog forwarding for application logs

Browse Source 
- Add LOG_FORMAT setting (text/json) for structured JSON log output
- Add LOG_SYSLOG_* settings for direct syslog forwarding of app logs
- JSON format compatible with Grafana Loki, Splunk, ELK, Datadog
- Syslog forwarding uses Python's SysLogHandler (UDP/TCP)
- Update .env.demo and ConfigurationGuide.md with all new settings
- Add tests for JSON formatter and syslog config fields

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-16 09:51:16 +00:00
parent 18c49c6b2d
commit 6da1e6fd81
5 changed files with 286 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.demo
+11
View File
@@ -14,6 +14,17 @@ COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, S
# LOG_LEVEL=INFO
# DEBUG=false
# Log output format: "text" (human-readable, default) or "json" (structured JSON lines).
# Use "json" when shipping logs to Grafana Loki, Splunk, ELK, Datadog, or any SIEM.
# LOG_FORMAT=text
# Forward application logs to a syslog receiver (in addition to stdout).
# Useful for traditional (non-container) deployments and centralised SIEM ingestion.
# LOG_SYSLOG_ENABLED=false
# LOG_SYSLOG_HOST=localhost
# LOG_SYSLOG_PORT=514
# LOG_SYSLOG_PROTOCOL=udp # udp | tcp
# **UI / Appearance**
# Default colour scheme: system (follow OS), light, or dark
# Individual users can always override with the navbar dark-mode toggle.
app/config.py
+31
View File
@@ -62,6 +62,37 @@ class Settings(BaseSettings):
),
)
# Log output format. ``text`` is the human-readable default.
# ``json`` emits one JSON object per line, ideal for log collectors
# (Promtail, Fluentd, Filebeat, Datadog agent) and SIEM ingestion.
log_format: str = Field(
default="text",
description=(
"Log output format: 'text' (human-readable, default) or "
"'json' (structured JSON lines for SIEM / log aggregation)."
),
)
# Optional syslog forwarding for application logs (not just audit events).
# When enabled, a Python SysLogHandler is added to the root logger so that
# every log message is also sent to the configured syslog receiver.
log_syslog_enabled: bool = Field(
default=False,
description="Forward application logs to a syslog receiver in addition to stdout.",
)
log_syslog_host: str = Field(
default="localhost",
description="Hostname or IP of the syslog receiver for application logs.",
)
log_syslog_port: int = Field(
default=514,
description="Port of the syslog receiver for application logs.",
)
log_syslog_protocol: str = Field(
default="udp",
description="Protocol for syslog transport: 'udp' or 'tcp'.",
)
# Making Dropbox optional
dropbox_enabled: bool = Field(
default=True,
app/main.py
+75 -7
View File
@@ -47,6 +47,13 @@ from app.views.files import router as files_router
# level is automatically lowered to ``DEBUG``.
# • Default (neither flag set): ``INFO``.
#
# ``LOG_FORMAT=json`` enables structured JSON lines on stdout, suitable for
# Promtail, Fluentd, Filebeat, Datadog, Splunk UF, or any log collector.
#
# ``LOG_SYSLOG_ENABLED=true`` adds a Python SysLogHandler so that every log
# message is also forwarded to the configured syslog receiver — useful for
# traditional (non-container) deployments and centralised SIEM ingestion.
#
# Noisy third-party loggers (httpx, httpcore, authlib, etc.) are pinned to
# WARNING when the app-level is DEBUG to keep output useful.
# ---------------------------------------------------------------------------
@@ -58,12 +65,66 @@ else:
_effective_level_int = getattr(logging, _effective_level, logging.INFO)
logging.basicConfig(
level=_effective_level_int,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
class _JsonFormatter(logging.Formatter):
"""Emit one JSON object per log line for machine consumption.
Fields emitted: ``timestamp``, ``level``, ``logger``, ``message``,
``module``, ``funcName``, ``lineno``, and — when present — ``exc_info``.
Compatible with Grafana Loki, Splunk, ELK, Datadog, and most SIEM tools.
"""
def format(self, record: logging.LogRecord) -> str:
import json as _json
from datetime import datetime as _dt
from datetime import timezone as _tz
log_entry: dict = {
"timestamp": _dt.fromtimestamp(record.created, tz=_tz.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
"module": record.module,
"funcName": record.funcName,
"lineno": record.lineno,
}
if record.exc_info and record.exc_info[1] is not None:
log_entry["exc_info"] = self.formatException(record.exc_info)
return _json.dumps(log_entry, default=str)
# Choose formatter based on LOG_FORMAT setting
if settings.log_format.lower() == "json":
_handler = logging.StreamHandler()
_handler.setFormatter(_JsonFormatter())
logging.root.handlers = [_handler]
logging.root.setLevel(_effective_level_int)
else:
logging.basicConfig(
level=_effective_level_int,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
# Optional: forward application logs to a syslog receiver
if settings.log_syslog_enabled:
import logging.handlers as _lh
import socket as _socket
_proto = settings.log_syslog_protocol.lower()
_socktype = _socket.SOCK_STREAM if _proto == "tcp" else _socket.SOCK_DGRAM
_syslog_handler = _lh.SysLogHandler(
address=(settings.log_syslog_host, settings.log_syslog_port),
socktype=_socktype,
)
_syslog_handler.setLevel(_effective_level_int)
# Use the same formatter as stdout (text or JSON)
if settings.log_format.lower() == "json":
_syslog_handler.setFormatter(_JsonFormatter())
else:
_syslog_handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s"))
logging.root.addHandler(_syslog_handler)
# Keep noisy third-party loggers quiet at DEBUG level
if _effective_level_int <= logging.DEBUG:
@@ -78,7 +139,14 @@ if _effective_level_int <= logging.DEBUG:
):
logging.getLogger(_noisy).setLevel(logging.WARNING)
logging.getLogger(__name__).info("Root logging level set to %s (debug=%s)", _effective_level, settings.debug)
_startup_logger = logging.getLogger(__name__)
_startup_logger.info(
"Root logging level set to %s (debug=%s, format=%s, syslog=%s)",
_effective_level,
settings.debug,
settings.log_format,
settings.log_syslog_enabled,
)
# Load configuration from .env for the session key
config = Config(".env")
docs/ConfigurationGuide.md
+53
View File
@@ -485,6 +485,59 @@ LOG_LEVEL=WARNING
> **Tip:** At `DEBUG` level, noisy third-party libraries (httpx, authlib, urllib3, etc.) are automatically pinned to `WARNING` so that application debug output remains readable.
#### Structured JSON Logging
Set `LOG_FORMAT=json` to emit structured JSON lines on stdout — one JSON object per log message. This is the standard format for log collectors and SIEM tools:
| **Variable** | **Description** | **Default** |
|-------------|----------------|-------------|
| `LOG_FORMAT` | Log output format: `text` (human-readable) or `json` (structured JSON lines). | `text` |
Each JSON log line contains: `timestamp` (ISO 8601), `level`, `logger`, `message`, `module`, `funcName`, `lineno`, and `exc_info` (when an exception is logged).
```bash
# Enable JSON logging for SIEM / log aggregation
LOG_FORMAT=json
```
**Example JSON output:**
```json
{"timestamp": "2025-03-16T09:18:05.192000+00:00", "level": "INFO", "logger": "app.auth", "message": "[SECURITY] OAUTH_LOGIN_SUCCESS user=alice@example.com admin=False", "module": "auth", "funcName": "oauth_callback", "lineno": 654}
```
**Compatible with:**
- **Grafana Loki** — Promtail scrapes JSON from Docker stdout
- **Splunk** — Universal Forwarder or HEC with JSON sourcetype
- **ELK / OpenSearch** — Filebeat with JSON codec
- **Datadog** — Agent auto-parses JSON logs
- **Fluentd / Vector** — JSON input plugin
- **Docker log drivers** — `--log-driver=json-file` (default) preserves structure
#### Syslog Forwarding (Application Logs)
For traditional (non-container) deployments, application logs can be forwarded directly to a syslog receiver. This is **separate** from audit-log SIEM forwarding (see below) — it sends _every_ Python log message, not just audit events.
| **Variable** | **Description** | **Default** |
|-------------|----------------|-------------|
| `LOG_SYSLOG_ENABLED` | Forward application logs to a syslog receiver in addition to stdout. | `false` |
| `LOG_SYSLOG_HOST` | Hostname or IP of the syslog receiver. | `localhost` |
| `LOG_SYSLOG_PORT` | Port of the syslog receiver. | `514` |
| `LOG_SYSLOG_PROTOCOL` | Protocol: `udp` or `tcp`. | `udp` |
```bash
# Forward all application logs to syslog
LOG_SYSLOG_ENABLED=true
LOG_SYSLOG_HOST=syslog.internal.example.com
LOG_SYSLOG_PORT=514
LOG_SYSLOG_PROTOCOL=udp
# Combine with JSON format for structured syslog messages
LOG_FORMAT=json
LOG_SYSLOG_ENABLED=true
```
> **Note:** When `LOG_FORMAT=json`, syslog messages are also sent as JSON. When `LOG_FORMAT=text`, syslog messages use the standard `name - level - message` format.
### Audit Logging
DocuElevate provides comprehensive audit logging that records significant actions (logins, document CRUD, settings changes) to an append-only database table. Every entry captures the timestamp, user, action, resource, client IP, and optional JSON details.
tests/test_logging_config.py
+116 -4
View File
@@ -75,8 +75,6 @@ class TestEffectiveLogLevel:
env = os.environ.copy()
env.pop("LOG_LEVEL", None)
with patch.dict(os.environ, env, clear=True):
from app.config import Settings as S
s = Settings(
database_url="sqlite:///test.db",
redis_url="redis://localhost:6379",
@@ -100,8 +98,6 @@ class TestEffectiveLogLevel:
def test_explicit_log_level_overrides_debug(self):
"""When LOG_LEVEL is explicitly set, it takes precedence over DEBUG=True."""
with patch.dict(os.environ, {"LOG_LEVEL": "WARNING", "DEBUG": "true"}, clear=False):
from app.config import Settings as S
s = Settings(
database_url="sqlite:///test.db",
redis_url="redis://localhost:6379",
@@ -171,3 +167,119 @@ class TestLoggingConfiguredAtStartup:
root = logging.getLogger()
# The test env doesn't set DEBUG=True, so the level should be INFO (20)
assert root.level <= logging.INFO
@pytest.mark.unit
class TestJsonFormatter:
"""Tests for the _JsonFormatter used when LOG_FORMAT=json."""
def _make_formatter(self):
"""Lazily import the JSON formatter from main module."""
from app.main import _JsonFormatter
return _JsonFormatter()
def test_output_is_valid_json(self):
"""JSON formatter output should be parseable JSON."""
import json
fmt = self._make_formatter()
record = logging.LogRecord(
name="test.logger",
level=logging.INFO,
pathname="test.py",
lineno=42,
msg="Hello %s",
args=("world",),
exc_info=None,
)
result = fmt.format(record)
parsed = json.loads(result)
assert parsed["level"] == "INFO"
assert parsed["logger"] == "test.logger"
assert parsed["message"] == "Hello world"
assert parsed["lineno"] == 42
def test_includes_timestamp_iso8601(self):
"""JSON output should contain an ISO 8601 timestamp."""
import json
fmt = self._make_formatter()
record = logging.LogRecord(
name="x",
level=logging.DEBUG,
pathname="x.py",
lineno=1,
msg="test",
args=(),
exc_info=None,
)
parsed = json.loads(fmt.format(record))
assert "timestamp" in parsed
# ISO 8601 timestamps contain "T" and "+00:00" (UTC)
assert "T" in parsed["timestamp"]
def test_includes_exc_info_when_present(self):
"""JSON output should include exc_info when an exception is logged."""
import json
fmt = self._make_formatter()
try:
raise ValueError("boom") # noqa: TRY301
except ValueError:
import sys
record = logging.LogRecord(
name="x",
level=logging.ERROR,
pathname="x.py",
lineno=1,
msg="error",
args=(),
exc_info=sys.exc_info(),
)
parsed = json.loads(fmt.format(record))
assert "exc_info" in parsed
assert "ValueError" in parsed["exc_info"]
@pytest.mark.unit
class TestLogFormatSetting:
"""Tests for the log_format and log_syslog_* config fields."""
_BASE_KWARGS = {
"database_url": "sqlite:///test.db",
"redis_url": "redis://localhost:6379",
"openai_api_key": "test",
"azure_ai_key": "test",
"azure_region": "test",
"azure_endpoint": "https://test.example.com",
"gotenberg_url": "http://localhost:3000",
"workdir": "/tmp",
"auth_enabled": False,
"session_secret": None,
}
def test_log_format_default_is_text(self):
"""Test that log_format defaults to 'text'."""
config = Settings(**self._BASE_KWARGS)
assert config.log_format == "text"
def test_log_format_accepts_json(self):
"""Test that log_format accepts 'json'."""
config = Settings(**self._BASE_KWARGS, log_format="json")
assert config.log_format == "json"
def test_log_syslog_defaults(self):
"""Test syslog forwarding defaults."""
config = Settings(**self._BASE_KWARGS)
assert config.log_syslog_enabled is False
assert config.log_syslog_host == "localhost"
assert config.log_syslog_port == 514
assert config.log_syslog_protocol == "udp"
def test_log_syslog_can_be_enabled(self):
"""Test that syslog forwarding can be enabled."""
config = Settings(**self._BASE_KWARGS, log_syslog_enabled=True, log_syslog_host="syslog.example.com")
assert config.log_syslog_enabled is True
assert config.log_syslog_host == "syslog.example.com"