gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add SIEM integration templates

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-05-23 18:26:50 +02:00
parent 946925d370
commit c1b53cf81f
9 changed files with 707 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/api_v1/api.py
+2
View File
@@ -7,6 +7,7 @@ from app.api.api_v1.endpoints import (
forensics,
health,
imap,
integrations,
mail_sources,
public,
reports,
@@ -30,6 +31,7 @@ api_router.include_router(reports.router, prefix="/reports", tags=["reports"])
api_router.include_router(forensics.router, prefix="/forensics", tags=["forensics"])
api_router.include_router(setup.router, prefix="/setup", tags=["setup"])
api_router.include_router(imap.router, prefix="/imap", tags=["imap"])
api_router.include_router(integrations.router, prefix="/integrations", tags=["integrations"])
api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["mail-sources"])
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
backend/app/api/api_v1/endpoints/integrations.py
+14
View File
@@ -0,0 +1,14 @@
"""Integration template endpoints."""
from fastapi import APIRouter, Depends
from app.core.security import require_admin_auth
from app.services.siem_templates import get_siem_templates
router = APIRouter()
@router.get("/siem/templates")
async def siem_templates(_auth: dict = Depends(require_admin_auth)):
"""Return versioned schemas and examples for SIEM ingestion."""
return get_siem_templates()
backend/app/services/siem_templates.py
+405
View File
@@ -0,0 +1,405 @@
"""Versioned SIEM integration templates and validation helpers."""
from __future__ import annotations
import copy
from typing import Any, Dict, List
from app.services.webhook_events import (
EVENT_ALERT_CREATED,
EVENT_COMPLIANCE_DROP,
EVENT_REPORT_IMPORTED,
EVENT_REPORTS_MISSING,
EVENT_SENDER_NEW,
)
SIEM_SCHEMA_VERSION = "dmarq.siem.event.v1"
SIEM_TEMPLATE_VERSION = "2026-05-23"
SIEM_EVENT_TYPES = [
EVENT_REPORT_IMPORTED,
EVENT_SENDER_NEW,
EVENT_COMPLIANCE_DROP,
EVENT_REPORTS_MISSING,
EVENT_ALERT_CREATED,
]
SIEM_SEVERITIES = ["info", "low", "medium", "high", "critical"]
SIEM_EVENT_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://dmarq.app/schemas/siem/dmarq-siem-event-v1.schema.json",
"title": "DMARQ SIEM Event",
"description": "Stable SIEM envelope for DMARQ posture, report, and alert events.",
"type": "object",
"additionalProperties": False,
"required": [
"schema_version",
"event_type",
"event_id",
"event_time",
"severity",
"source",
"entity",
"metrics",
"redaction",
],
"properties": {
"schema_version": {"const": SIEM_SCHEMA_VERSION},
"event_type": {"type": "string", "enum": SIEM_EVENT_TYPES},
"event_id": {
"type": "string",
"description": "Stable event or delivery identifier for deduplication.",
"minLength": 8,
},
"event_time": {"type": "string", "format": "date-time"},
"severity": {"type": "string", "enum": SIEM_SEVERITIES},
"source": {
"type": "object",
"additionalProperties": False,
"required": ["application", "instance", "environment"],
"properties": {
"application": {"const": "dmarq"},
"instance": {"type": "string"},
"environment": {"type": "string"},
},
},
"entity": {
"type": "object",
"additionalProperties": False,
"required": ["domain"],
"properties": {
"domain": {"type": "string"},
"sender_ip": {"type": ["string", "null"]},
"sender_org": {"type": ["string", "null"]},
"reporter": {"type": ["string", "null"]},
"alert_rule": {"type": ["string", "null"]},
},
},
"metrics": {
"type": "object",
"additionalProperties": False,
"properties": {
"message_count": {"type": ["integer", "null"], "minimum": 0},
"aligned_count": {"type": ["integer", "null"], "minimum": 0},
"failed_count": {"type": ["integer", "null"], "minimum": 0},
"compliance_rate": {"type": ["number", "null"], "minimum": 0, "maximum": 100},
"previous_compliance_rate": {
"type": ["number", "null"],
"minimum": 0,
"maximum": 100,
},
"drop_points": {"type": ["number", "null"], "minimum": 0, "maximum": 100},
"missing_days": {"type": ["integer", "null"], "minimum": 0},
},
},
"alert": {
"type": ["object", "null"],
"additionalProperties": False,
"properties": {
"title": {"type": "string"},
"detail": {"type": "string"},
"status": {"type": "string", "enum": ["active", "resolved", "informational"]},
},
},
"redaction": {
"type": "object",
"additionalProperties": False,
"required": ["pii_redacted", "secret_fields_removed", "raw_report_included"],
"properties": {
"pii_redacted": {"const": True},
"secret_fields_removed": {"const": True},
"raw_report_included": {"const": False},
"notes": {"type": "string"},
},
},
"links": {
"type": "object",
"additionalProperties": False,
"properties": {
"domain_url": {"type": ["string", "null"], "format": "uri"},
"report_url": {"type": ["string", "null"], "format": "uri"},
},
},
"tags": {"type": "array", "items": {"type": "string"}},
"extensions": {
"type": "object",
"description": "Optional namespaced integration fields.",
"additionalProperties": True,
},
},
}
BASE_REDACTION = {
"pii_redacted": True,
"secret_fields_removed": True,
"raw_report_included": False,
"notes": "Contains aggregate counts and posture metadata only.",
}
SIEM_EVENT_EXAMPLES: Dict[str, Dict[str, Any]] = {
"sender_new": {
"schema_version": SIEM_SCHEMA_VERSION,
"event_type": EVENT_SENDER_NEW,
"event_id": "dmarq-sender-new-20260523-example-com-20301135",
"event_time": "2026-05-23T16:30:00Z",
"severity": "medium",
"source": {
"application": "dmarq",
"instance": "dmarq-preprod",
"environment": "preprod",
},
"entity": {
"domain": "example.com",
"sender_ip": "203.0.113.5",
"sender_org": "Example SaaS Mail",
"reporter": "google.com",
"alert_rule": "new_sender_source",
},
"metrics": {
"message_count": 42,
"aligned_count": 40,
"failed_count": 2,
"compliance_rate": 95.24,
"previous_compliance_rate": None,
"drop_points": None,
"missing_days": None,
},
"alert": {
"title": "New sending source for example.com",
"detail": "203.0.113.5 was first observed sending authenticated mail.",
"status": "active",
},
"redaction": BASE_REDACTION,
"links": {
"domain_url": "https://dmarq.example/domains/example.com",
"report_url": None,
},
"tags": ["email-security", "dmarc", "new-sender"],
"extensions": {},
},
"compliance_drop": {
"schema_version": SIEM_SCHEMA_VERSION,
"event_type": EVENT_COMPLIANCE_DROP,
"event_id": "dmarq-compliance-drop-20260523-example-com",
"event_time": "2026-05-23T16:35:00Z",
"severity": "high",
"source": {
"application": "dmarq",
"instance": "dmarq-preprod",
"environment": "preprod",
},
"entity": {
"domain": "example.com",
"sender_ip": None,
"sender_org": None,
"reporter": None,
"alert_rule": "compliance_drop",
},
"metrics": {
"message_count": 1280,
"aligned_count": 914,
"failed_count": 366,
"compliance_rate": 71.41,
"previous_compliance_rate": 94.8,
"drop_points": 23.39,
"missing_days": None,
},
"alert": {
"title": "DMARC compliance dropped for example.com",
"detail": "Compliance fell by 23.39 percentage points in the current window.",
"status": "active",
},
"redaction": BASE_REDACTION,
"links": {
"domain_url": "https://dmarq.example/domains/example.com",
"report_url": "https://dmarq.example/domains/example.com/reports",
},
"tags": ["email-security", "dmarc", "compliance-drop"],
"extensions": {},
},
"alert_created": {
"schema_version": SIEM_SCHEMA_VERSION,
"event_type": EVENT_ALERT_CREATED,
"event_id": "dmarq-alert-created-20260523-example-com",
"event_time": "2026-05-23T16:40:00Z",
"severity": "medium",
"source": {
"application": "dmarq",
"instance": "dmarq-preprod",
"environment": "preprod",
},
"entity": {
"domain": "example.com",
"sender_ip": None,
"sender_org": None,
"reporter": None,
"alert_rule": "missing_reports",
},
"metrics": {
"message_count": None,
"aligned_count": None,
"failed_count": None,
"compliance_rate": None,
"previous_compliance_rate": None,
"drop_points": None,
"missing_days": 3,
},
"alert": {
"title": "Missing DMARC reports for example.com",
"detail": "No aggregate reports have been imported for 3 days.",
"status": "active",
},
"redaction": BASE_REDACTION,
"links": {
"domain_url": "https://dmarq.example/domains/example.com",
"report_url": None,
},
"tags": ["email-security", "dmarc", "missing-reports"],
"extensions": {},
},
}
SIEM_INGESTION_EXAMPLES: Dict[str, Dict[str, Any]] = {
"splunk_hec": {
"time": 1779554100,
"host": "dmarq-preprod",
"source": "dmarq:webhook",
"sourcetype": "_json",
"index": "email_security",
"event": SIEM_EVENT_EXAMPLES["compliance_drop"],
},
"elastic_ecs": {
"@timestamp": "2026-05-23T16:35:00Z",
"ecs.version": "8.11.0",
"event.kind": "alert",
"event.category": ["email"],
"event.type": ["info"],
"event.dataset": "dmarq.siem",
"observer.vendor": "DMARQ",
"observer.product": "DMARQ",
"rule.name": "compliance_drop",
"host.name": "dmarq-preprod",
"dmarq": SIEM_EVENT_EXAMPLES["compliance_drop"],
},
"microsoft_sentinel_custom_log": [
{
"TimeGenerated": "2026-05-23T16:35:00Z",
"EventType": EVENT_COMPLIANCE_DROP,
"Domain": "example.com",
"Severity": "high",
"ComplianceRate": 71.41,
"PreviousComplianceRate": 94.8,
"DropPoints": 23.39,
"DmarqEvent": SIEM_EVENT_EXAMPLES["compliance_drop"],
}
],
}
SIEM_CONFIG_TEMPLATES: Dict[str, Dict[str, Any]] = {
"splunk_hec": {
"webhook_url_pattern": "https://splunk.example:8088/services/collector/event",
"headers": {
"Authorization": "Splunk ${SPLUNK_HEC_TOKEN}",
"Content-Type": "application/json",
},
"recommended_index": "email_security",
"recommended_sourcetype": "_json",
"notes": [
"Store the HEC token in the receiving proxy or SIEM secret store.",
"Deduplicate with event.event_id or X-DMARQ-Idempotency-Key.",
],
},
"elastic_logstash_http": {
"webhook_url_pattern": "https://logstash.example:5044/dmarq",
"pipeline_hint": "Parse the JSON body, move event_time to @timestamp, and keep the full payload under dmarq.",
"recommended_index": "logs-dmarq.email_security-default",
"notes": [
"Use a Logstash secret store entry for downstream Elasticsearch credentials.",
"Map severity to event.risk_score or event.severity in the pipeline.",
],
},
"microsoft_sentinel": {
"webhook_url_pattern": "https://ingest.monitor.azure.com/dataCollectionRules/${DCR_IMMUTABLE_ID}/streams/Custom-Dmarq_CL",
"table_name": "Dmarq_CL",
"notes": [
"Use an Azure Function, Logic App, or protected relay to add OAuth credentials.",
"Keep TimeGenerated mapped from event_time for query accuracy.",
],
},
}
REDACTION_GUIDANCE = [
"Forward aggregate counts, domains, sender IPs, reporter names, and alert metadata.",
"Do not forward raw report XML, raw RFC 822 messages, mailbox credentials, API tokens, or webhook signing secrets.",
"Keep recipient addresses, local-parts, authentication headers, and forensic message content redacted unless a separate privacy review approves them.",
"Use event_id or X-DMARQ-Idempotency-Key for deduplication instead of hashing raw payloads that may include sensitive fields.",
]
def _append_required_field_errors(event: Dict[str, Any], errors: List[str]) -> None:
for field in SIEM_EVENT_SCHEMA["required"]:
if field not in event:
errors.append(f"missing required field: {field}")
def _append_enum_errors(event: Dict[str, Any], errors: List[str]) -> None:
checks = [
(
event.get("schema_version") == SIEM_SCHEMA_VERSION,
"schema_version must be dmarq.siem.event.v1",
),
(event.get("event_type") in SIEM_EVENT_TYPES, "event_type is not a supported SIEM event"),
(event.get("severity") in SIEM_SEVERITIES, "severity is not supported"),
]
errors.extend(message for passed, message in checks if not passed)
def _append_redaction_errors(event: Dict[str, Any], errors: List[str]) -> None:
redaction = event.get("redaction") or {}
checks = [
(redaction.get("pii_redacted") is True, "redaction.pii_redacted must be true"),
(
redaction.get("secret_fields_removed") is True,
"redaction.secret_fields_removed must be true",
),
(
redaction.get("raw_report_included") is False,
"redaction.raw_report_included must be false",
),
]
errors.extend(message for passed, message in checks if not passed)
def get_siem_templates() -> Dict[str, Any]:
"""Return a copy of the versioned SIEM template bundle."""
return copy.deepcopy(
{
"template_version": SIEM_TEMPLATE_VERSION,
"schema_version": SIEM_SCHEMA_VERSION,
"event_types": SIEM_EVENT_TYPES,
"event_schema": SIEM_EVENT_SCHEMA,
"event_examples": SIEM_EVENT_EXAMPLES,
"ingestion_examples": SIEM_INGESTION_EXAMPLES,
"config_templates": SIEM_CONFIG_TEMPLATES,
"redaction_guidance": REDACTION_GUIDANCE,
}
)
def validate_siem_event(event: Dict[str, Any]) -> List[str]:
"""Run lightweight validation for bundled examples without extra dependencies."""
errors: List[str] = []
_append_required_field_errors(event, errors)
_append_enum_errors(event, errors)
source = event.get("source") or {}
if source.get("application") != "dmarq":
errors.append("source.application must be dmarq")
entity = event.get("entity") or {}
if not entity.get("domain"):
errors.append("entity.domain is required")
_append_redaction_errors(event, errors)
return errors
backend/app/tests/test_siem_templates.py
+77
View File
@@ -0,0 +1,77 @@
import json
from fastapi.testclient import TestClient
from app.services.siem_templates import (
SIEM_EVENT_TYPES,
SIEM_SCHEMA_VERSION,
get_siem_templates,
validate_siem_event,
)
def _string_values(value):
if isinstance(value, dict):
for item in value.values():
yield from _string_values(item)
elif isinstance(value, list):
for item in value:
yield from _string_values(item)
elif isinstance(value, str):
yield value
def test_siem_template_bundle_is_versioned_and_examples_match_schema():
"""Bundled SIEM examples stay aligned with the stable event envelope."""
templates = get_siem_templates()
assert templates["schema_version"] == SIEM_SCHEMA_VERSION
assert templates["event_schema"]["properties"]["schema_version"]["const"] == SIEM_SCHEMA_VERSION
assert templates["event_schema"]["properties"]["event_type"]["enum"] == SIEM_EVENT_TYPES
examples = templates["event_examples"]
assert {"sender_new", "compliance_drop", "alert_created"}.issubset(examples)
for name, example in examples.items():
assert validate_siem_event(example) == [], name
encoded = json.dumps(list(_string_values(example))).lower()
assert "secret" not in encoded
assert "token" not in encoded
assert "raw_report_xml" not in encoded
def test_siem_ingestion_examples_wrap_valid_dmarq_events():
"""SIEM-specific examples keep the normalized event intact."""
templates = get_siem_templates()
ingestion_examples = templates["ingestion_examples"]
splunk_event = ingestion_examples["splunk_hec"]["event"]
elastic_event = ingestion_examples["elastic_ecs"]["dmarq"]
sentinel_event = ingestion_examples["microsoft_sentinel_custom_log"][0]["DmarqEvent"]
for wrapped_event in [splunk_event, elastic_event, sentinel_event]:
assert validate_siem_event(wrapped_event) == []
assert wrapped_event["schema_version"] == SIEM_SCHEMA_VERSION
def test_siem_templates_endpoint_returns_common_configs(authed_client: TestClient):
"""Administrators can fetch schemas, examples, and SIEM config hints."""
response = authed_client.get("/api/v1/integrations/siem/templates")
assert response.status_code == 200
body = response.json()
assert body["schema_version"] == SIEM_SCHEMA_VERSION
assert set(body["config_templates"]) == {
"splunk_hec",
"elastic_logstash_http",
"microsoft_sentinel",
}
assert body["event_examples"]["compliance_drop"]["redaction"]["pii_redacted"] is True
assert "raw report" in " ".join(body["redaction_guidance"]).lower()
def test_siem_templates_endpoint_requires_admin_auth(client: TestClient):
"""Template endpoint follows the same admin boundary as other integrations."""
response = client.get("/api/v1/integrations/siem/templates")
assert response.status_code == 401