Merge pull request #191 from christianlouis/codex/m14-siem-templates
feat: add SIEM integration templates
This commit is contained in:
@@ -7,6 +7,7 @@ from app.api.api_v1.endpoints import (
|
||||
forensics,
|
||||
health,
|
||||
imap,
|
||||
integrations,
|
||||
mail_sources,
|
||||
public,
|
||||
reports,
|
||||
@@ -30,6 +31,7 @@ api_router.include_router(reports.router, prefix="/reports", tags=["reports"])
|
||||
api_router.include_router(forensics.router, prefix="/forensics", tags=["forensics"])
|
||||
api_router.include_router(setup.router, prefix="/setup", tags=["setup"])
|
||||
api_router.include_router(imap.router, prefix="/imap", tags=["imap"])
|
||||
api_router.include_router(integrations.router, prefix="/integrations", tags=["integrations"])
|
||||
api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
|
||||
api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["mail-sources"])
|
||||
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Integration template endpoints."""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.core.security import require_admin_auth
|
||||
from app.services.siem_templates import get_siem_templates
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/siem/templates")
|
||||
async def siem_templates(_auth: dict = Depends(require_admin_auth)):
|
||||
"""Return versioned schemas and examples for SIEM ingestion."""
|
||||
return get_siem_templates()
|
||||
@@ -0,0 +1,405 @@
|
||||
"""Versioned SIEM integration templates and validation helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.services.webhook_events import (
|
||||
EVENT_ALERT_CREATED,
|
||||
EVENT_COMPLIANCE_DROP,
|
||||
EVENT_REPORT_IMPORTED,
|
||||
EVENT_REPORTS_MISSING,
|
||||
EVENT_SENDER_NEW,
|
||||
)
|
||||
|
||||
SIEM_SCHEMA_VERSION = "dmarq.siem.event.v1"
|
||||
SIEM_TEMPLATE_VERSION = "2026-05-23"
|
||||
|
||||
SIEM_EVENT_TYPES = [
|
||||
EVENT_REPORT_IMPORTED,
|
||||
EVENT_SENDER_NEW,
|
||||
EVENT_COMPLIANCE_DROP,
|
||||
EVENT_REPORTS_MISSING,
|
||||
EVENT_ALERT_CREATED,
|
||||
]
|
||||
|
||||
SIEM_SEVERITIES = ["info", "low", "medium", "high", "critical"]
|
||||
|
||||
SIEM_EVENT_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://dmarq.app/schemas/siem/dmarq-siem-event-v1.schema.json",
|
||||
"title": "DMARQ SIEM Event",
|
||||
"description": "Stable SIEM envelope for DMARQ posture, report, and alert events.",
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": [
|
||||
"schema_version",
|
||||
"event_type",
|
||||
"event_id",
|
||||
"event_time",
|
||||
"severity",
|
||||
"source",
|
||||
"entity",
|
||||
"metrics",
|
||||
"redaction",
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {"const": SIEM_SCHEMA_VERSION},
|
||||
"event_type": {"type": "string", "enum": SIEM_EVENT_TYPES},
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"description": "Stable event or delivery identifier for deduplication.",
|
||||
"minLength": 8,
|
||||
},
|
||||
"event_time": {"type": "string", "format": "date-time"},
|
||||
"severity": {"type": "string", "enum": SIEM_SEVERITIES},
|
||||
"source": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["application", "instance", "environment"],
|
||||
"properties": {
|
||||
"application": {"const": "dmarq"},
|
||||
"instance": {"type": "string"},
|
||||
"environment": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"entity": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["domain"],
|
||||
"properties": {
|
||||
"domain": {"type": "string"},
|
||||
"sender_ip": {"type": ["string", "null"]},
|
||||
"sender_org": {"type": ["string", "null"]},
|
||||
"reporter": {"type": ["string", "null"]},
|
||||
"alert_rule": {"type": ["string", "null"]},
|
||||
},
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"message_count": {"type": ["integer", "null"], "minimum": 0},
|
||||
"aligned_count": {"type": ["integer", "null"], "minimum": 0},
|
||||
"failed_count": {"type": ["integer", "null"], "minimum": 0},
|
||||
"compliance_rate": {"type": ["number", "null"], "minimum": 0, "maximum": 100},
|
||||
"previous_compliance_rate": {
|
||||
"type": ["number", "null"],
|
||||
"minimum": 0,
|
||||
"maximum": 100,
|
||||
},
|
||||
"drop_points": {"type": ["number", "null"], "minimum": 0, "maximum": 100},
|
||||
"missing_days": {"type": ["integer", "null"], "minimum": 0},
|
||||
},
|
||||
},
|
||||
"alert": {
|
||||
"type": ["object", "null"],
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"detail": {"type": "string"},
|
||||
"status": {"type": "string", "enum": ["active", "resolved", "informational"]},
|
||||
},
|
||||
},
|
||||
"redaction": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["pii_redacted", "secret_fields_removed", "raw_report_included"],
|
||||
"properties": {
|
||||
"pii_redacted": {"const": True},
|
||||
"secret_fields_removed": {"const": True},
|
||||
"raw_report_included": {"const": False},
|
||||
"notes": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"links": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"domain_url": {"type": ["string", "null"], "format": "uri"},
|
||||
"report_url": {"type": ["string", "null"], "format": "uri"},
|
||||
},
|
||||
},
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
"extensions": {
|
||||
"type": "object",
|
||||
"description": "Optional namespaced integration fields.",
|
||||
"additionalProperties": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
BASE_REDACTION = {
|
||||
"pii_redacted": True,
|
||||
"secret_fields_removed": True,
|
||||
"raw_report_included": False,
|
||||
"notes": "Contains aggregate counts and posture metadata only.",
|
||||
}
|
||||
|
||||
SIEM_EVENT_EXAMPLES: Dict[str, Dict[str, Any]] = {
|
||||
"sender_new": {
|
||||
"schema_version": SIEM_SCHEMA_VERSION,
|
||||
"event_type": EVENT_SENDER_NEW,
|
||||
"event_id": "dmarq-sender-new-20260523-example-com-20301135",
|
||||
"event_time": "2026-05-23T16:30:00Z",
|
||||
"severity": "medium",
|
||||
"source": {
|
||||
"application": "dmarq",
|
||||
"instance": "dmarq-preprod",
|
||||
"environment": "preprod",
|
||||
},
|
||||
"entity": {
|
||||
"domain": "example.com",
|
||||
"sender_ip": "203.0.113.5",
|
||||
"sender_org": "Example SaaS Mail",
|
||||
"reporter": "google.com",
|
||||
"alert_rule": "new_sender_source",
|
||||
},
|
||||
"metrics": {
|
||||
"message_count": 42,
|
||||
"aligned_count": 40,
|
||||
"failed_count": 2,
|
||||
"compliance_rate": 95.24,
|
||||
"previous_compliance_rate": None,
|
||||
"drop_points": None,
|
||||
"missing_days": None,
|
||||
},
|
||||
"alert": {
|
||||
"title": "New sending source for example.com",
|
||||
"detail": "203.0.113.5 was first observed sending authenticated mail.",
|
||||
"status": "active",
|
||||
},
|
||||
"redaction": BASE_REDACTION,
|
||||
"links": {
|
||||
"domain_url": "https://dmarq.example/domains/example.com",
|
||||
"report_url": None,
|
||||
},
|
||||
"tags": ["email-security", "dmarc", "new-sender"],
|
||||
"extensions": {},
|
||||
},
|
||||
"compliance_drop": {
|
||||
"schema_version": SIEM_SCHEMA_VERSION,
|
||||
"event_type": EVENT_COMPLIANCE_DROP,
|
||||
"event_id": "dmarq-compliance-drop-20260523-example-com",
|
||||
"event_time": "2026-05-23T16:35:00Z",
|
||||
"severity": "high",
|
||||
"source": {
|
||||
"application": "dmarq",
|
||||
"instance": "dmarq-preprod",
|
||||
"environment": "preprod",
|
||||
},
|
||||
"entity": {
|
||||
"domain": "example.com",
|
||||
"sender_ip": None,
|
||||
"sender_org": None,
|
||||
"reporter": None,
|
||||
"alert_rule": "compliance_drop",
|
||||
},
|
||||
"metrics": {
|
||||
"message_count": 1280,
|
||||
"aligned_count": 914,
|
||||
"failed_count": 366,
|
||||
"compliance_rate": 71.41,
|
||||
"previous_compliance_rate": 94.8,
|
||||
"drop_points": 23.39,
|
||||
"missing_days": None,
|
||||
},
|
||||
"alert": {
|
||||
"title": "DMARC compliance dropped for example.com",
|
||||
"detail": "Compliance fell by 23.39 percentage points in the current window.",
|
||||
"status": "active",
|
||||
},
|
||||
"redaction": BASE_REDACTION,
|
||||
"links": {
|
||||
"domain_url": "https://dmarq.example/domains/example.com",
|
||||
"report_url": "https://dmarq.example/domains/example.com/reports",
|
||||
},
|
||||
"tags": ["email-security", "dmarc", "compliance-drop"],
|
||||
"extensions": {},
|
||||
},
|
||||
"alert_created": {
|
||||
"schema_version": SIEM_SCHEMA_VERSION,
|
||||
"event_type": EVENT_ALERT_CREATED,
|
||||
"event_id": "dmarq-alert-created-20260523-example-com",
|
||||
"event_time": "2026-05-23T16:40:00Z",
|
||||
"severity": "medium",
|
||||
"source": {
|
||||
"application": "dmarq",
|
||||
"instance": "dmarq-preprod",
|
||||
"environment": "preprod",
|
||||
},
|
||||
"entity": {
|
||||
"domain": "example.com",
|
||||
"sender_ip": None,
|
||||
"sender_org": None,
|
||||
"reporter": None,
|
||||
"alert_rule": "missing_reports",
|
||||
},
|
||||
"metrics": {
|
||||
"message_count": None,
|
||||
"aligned_count": None,
|
||||
"failed_count": None,
|
||||
"compliance_rate": None,
|
||||
"previous_compliance_rate": None,
|
||||
"drop_points": None,
|
||||
"missing_days": 3,
|
||||
},
|
||||
"alert": {
|
||||
"title": "Missing DMARC reports for example.com",
|
||||
"detail": "No aggregate reports have been imported for 3 days.",
|
||||
"status": "active",
|
||||
},
|
||||
"redaction": BASE_REDACTION,
|
||||
"links": {
|
||||
"domain_url": "https://dmarq.example/domains/example.com",
|
||||
"report_url": None,
|
||||
},
|
||||
"tags": ["email-security", "dmarc", "missing-reports"],
|
||||
"extensions": {},
|
||||
},
|
||||
}
|
||||
|
||||
SIEM_INGESTION_EXAMPLES: Dict[str, Dict[str, Any]] = {
|
||||
"splunk_hec": {
|
||||
"time": 1779554100,
|
||||
"host": "dmarq-preprod",
|
||||
"source": "dmarq:webhook",
|
||||
"sourcetype": "_json",
|
||||
"index": "email_security",
|
||||
"event": SIEM_EVENT_EXAMPLES["compliance_drop"],
|
||||
},
|
||||
"elastic_ecs": {
|
||||
"@timestamp": "2026-05-23T16:35:00Z",
|
||||
"ecs.version": "8.11.0",
|
||||
"event.kind": "alert",
|
||||
"event.category": ["email"],
|
||||
"event.type": ["info"],
|
||||
"event.dataset": "dmarq.siem",
|
||||
"observer.vendor": "DMARQ",
|
||||
"observer.product": "DMARQ",
|
||||
"rule.name": "compliance_drop",
|
||||
"host.name": "dmarq-preprod",
|
||||
"dmarq": SIEM_EVENT_EXAMPLES["compliance_drop"],
|
||||
},
|
||||
"microsoft_sentinel_custom_log": [
|
||||
{
|
||||
"TimeGenerated": "2026-05-23T16:35:00Z",
|
||||
"EventType": EVENT_COMPLIANCE_DROP,
|
||||
"Domain": "example.com",
|
||||
"Severity": "high",
|
||||
"ComplianceRate": 71.41,
|
||||
"PreviousComplianceRate": 94.8,
|
||||
"DropPoints": 23.39,
|
||||
"DmarqEvent": SIEM_EVENT_EXAMPLES["compliance_drop"],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
SIEM_CONFIG_TEMPLATES: Dict[str, Dict[str, Any]] = {
|
||||
"splunk_hec": {
|
||||
"webhook_url_pattern": "https://splunk.example:8088/services/collector/event",
|
||||
"headers": {
|
||||
"Authorization": "Splunk ${SPLUNK_HEC_TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"recommended_index": "email_security",
|
||||
"recommended_sourcetype": "_json",
|
||||
"notes": [
|
||||
"Store the HEC token in the receiving proxy or SIEM secret store.",
|
||||
"Deduplicate with event.event_id or X-DMARQ-Idempotency-Key.",
|
||||
],
|
||||
},
|
||||
"elastic_logstash_http": {
|
||||
"webhook_url_pattern": "https://logstash.example:5044/dmarq",
|
||||
"pipeline_hint": "Parse the JSON body, move event_time to @timestamp, and keep the full payload under dmarq.",
|
||||
"recommended_index": "logs-dmarq.email_security-default",
|
||||
"notes": [
|
||||
"Use a Logstash secret store entry for downstream Elasticsearch credentials.",
|
||||
"Map severity to event.risk_score or event.severity in the pipeline.",
|
||||
],
|
||||
},
|
||||
"microsoft_sentinel": {
|
||||
"webhook_url_pattern": "https://ingest.monitor.azure.com/dataCollectionRules/${DCR_IMMUTABLE_ID}/streams/Custom-Dmarq_CL",
|
||||
"table_name": "Dmarq_CL",
|
||||
"notes": [
|
||||
"Use an Azure Function, Logic App, or protected relay to add OAuth credentials.",
|
||||
"Keep TimeGenerated mapped from event_time for query accuracy.",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
REDACTION_GUIDANCE = [
|
||||
"Forward aggregate counts, domains, sender IPs, reporter names, and alert metadata.",
|
||||
"Do not forward raw report XML, raw RFC 822 messages, mailbox credentials, API tokens, or webhook signing secrets.",
|
||||
"Keep recipient addresses, local-parts, authentication headers, and forensic message content redacted unless a separate privacy review approves them.",
|
||||
"Use event_id or X-DMARQ-Idempotency-Key for deduplication instead of hashing raw payloads that may include sensitive fields.",
|
||||
]
|
||||
|
||||
|
||||
def _append_required_field_errors(event: Dict[str, Any], errors: List[str]) -> None:
|
||||
for field in SIEM_EVENT_SCHEMA["required"]:
|
||||
if field not in event:
|
||||
errors.append(f"missing required field: {field}")
|
||||
|
||||
|
||||
def _append_enum_errors(event: Dict[str, Any], errors: List[str]) -> None:
|
||||
checks = [
|
||||
(
|
||||
event.get("schema_version") == SIEM_SCHEMA_VERSION,
|
||||
"schema_version must be dmarq.siem.event.v1",
|
||||
),
|
||||
(event.get("event_type") in SIEM_EVENT_TYPES, "event_type is not a supported SIEM event"),
|
||||
(event.get("severity") in SIEM_SEVERITIES, "severity is not supported"),
|
||||
]
|
||||
errors.extend(message for passed, message in checks if not passed)
|
||||
|
||||
|
||||
def _append_redaction_errors(event: Dict[str, Any], errors: List[str]) -> None:
|
||||
redaction = event.get("redaction") or {}
|
||||
checks = [
|
||||
(redaction.get("pii_redacted") is True, "redaction.pii_redacted must be true"),
|
||||
(
|
||||
redaction.get("secret_fields_removed") is True,
|
||||
"redaction.secret_fields_removed must be true",
|
||||
),
|
||||
(
|
||||
redaction.get("raw_report_included") is False,
|
||||
"redaction.raw_report_included must be false",
|
||||
),
|
||||
]
|
||||
errors.extend(message for passed, message in checks if not passed)
|
||||
|
||||
|
||||
def get_siem_templates() -> Dict[str, Any]:
|
||||
"""Return a copy of the versioned SIEM template bundle."""
|
||||
return copy.deepcopy(
|
||||
{
|
||||
"template_version": SIEM_TEMPLATE_VERSION,
|
||||
"schema_version": SIEM_SCHEMA_VERSION,
|
||||
"event_types": SIEM_EVENT_TYPES,
|
||||
"event_schema": SIEM_EVENT_SCHEMA,
|
||||
"event_examples": SIEM_EVENT_EXAMPLES,
|
||||
"ingestion_examples": SIEM_INGESTION_EXAMPLES,
|
||||
"config_templates": SIEM_CONFIG_TEMPLATES,
|
||||
"redaction_guidance": REDACTION_GUIDANCE,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def validate_siem_event(event: Dict[str, Any]) -> List[str]:
|
||||
"""Run lightweight validation for bundled examples without extra dependencies."""
|
||||
errors: List[str] = []
|
||||
_append_required_field_errors(event, errors)
|
||||
_append_enum_errors(event, errors)
|
||||
|
||||
source = event.get("source") or {}
|
||||
if source.get("application") != "dmarq":
|
||||
errors.append("source.application must be dmarq")
|
||||
|
||||
entity = event.get("entity") or {}
|
||||
if not entity.get("domain"):
|
||||
errors.append("entity.domain is required")
|
||||
|
||||
_append_redaction_errors(event, errors)
|
||||
return errors
|
||||
@@ -0,0 +1,77 @@
|
||||
import json
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.services.siem_templates import (
|
||||
SIEM_EVENT_TYPES,
|
||||
SIEM_SCHEMA_VERSION,
|
||||
get_siem_templates,
|
||||
validate_siem_event,
|
||||
)
|
||||
|
||||
|
||||
def _string_values(value):
|
||||
if isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from _string_values(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from _string_values(item)
|
||||
elif isinstance(value, str):
|
||||
yield value
|
||||
|
||||
|
||||
def test_siem_template_bundle_is_versioned_and_examples_match_schema():
|
||||
"""Bundled SIEM examples stay aligned with the stable event envelope."""
|
||||
templates = get_siem_templates()
|
||||
|
||||
assert templates["schema_version"] == SIEM_SCHEMA_VERSION
|
||||
assert templates["event_schema"]["properties"]["schema_version"]["const"] == SIEM_SCHEMA_VERSION
|
||||
assert templates["event_schema"]["properties"]["event_type"]["enum"] == SIEM_EVENT_TYPES
|
||||
|
||||
examples = templates["event_examples"]
|
||||
assert {"sender_new", "compliance_drop", "alert_created"}.issubset(examples)
|
||||
|
||||
for name, example in examples.items():
|
||||
assert validate_siem_event(example) == [], name
|
||||
encoded = json.dumps(list(_string_values(example))).lower()
|
||||
assert "secret" not in encoded
|
||||
assert "token" not in encoded
|
||||
assert "raw_report_xml" not in encoded
|
||||
|
||||
|
||||
def test_siem_ingestion_examples_wrap_valid_dmarq_events():
|
||||
"""SIEM-specific examples keep the normalized event intact."""
|
||||
templates = get_siem_templates()
|
||||
ingestion_examples = templates["ingestion_examples"]
|
||||
|
||||
splunk_event = ingestion_examples["splunk_hec"]["event"]
|
||||
elastic_event = ingestion_examples["elastic_ecs"]["dmarq"]
|
||||
sentinel_event = ingestion_examples["microsoft_sentinel_custom_log"][0]["DmarqEvent"]
|
||||
|
||||
for wrapped_event in [splunk_event, elastic_event, sentinel_event]:
|
||||
assert validate_siem_event(wrapped_event) == []
|
||||
assert wrapped_event["schema_version"] == SIEM_SCHEMA_VERSION
|
||||
|
||||
|
||||
def test_siem_templates_endpoint_returns_common_configs(authed_client: TestClient):
|
||||
"""Administrators can fetch schemas, examples, and SIEM config hints."""
|
||||
response = authed_client.get("/api/v1/integrations/siem/templates")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["schema_version"] == SIEM_SCHEMA_VERSION
|
||||
assert set(body["config_templates"]) == {
|
||||
"splunk_hec",
|
||||
"elastic_logstash_http",
|
||||
"microsoft_sentinel",
|
||||
}
|
||||
assert body["event_examples"]["compliance_drop"]["redaction"]["pii_redacted"] is True
|
||||
assert "raw report" in " ".join(body["redaction_guidance"]).lower()
|
||||
|
||||
|
||||
def test_siem_templates_endpoint_requires_admin_auth(client: TestClient):
|
||||
"""Template endpoint follows the same admin boundary as other integrations."""
|
||||
response = client.get("/api/v1/integrations/siem/templates")
|
||||
|
||||
assert response.status_code == 401
|
||||
+2
-1
@@ -237,7 +237,8 @@ Goal: let DMARQ integrate cleanly into existing security and operations workflow
|
||||
Planned:
|
||||
- A stable, documented read-only API surface for posture and reporting queries. Delivered with scoped `reports:read`, `posture:read`, and `tls-reports:read` API tokens, public read-only endpoints, and per-token usage audit fields.
|
||||
- Webhook event delivery for key events (new sender source, compliance drop, missing reports, alert lifecycle). Delivered with encrypted webhook endpoints, signed delivery headers, idempotency keys, retry/backoff state, test sends, and delivery inspection.
|
||||
- Integration templates for SIEM and ticketing workflows (export formats, payload schemas, examples).
|
||||
- SIEM integration templates delivered with a stable `dmarq.siem.event.v1` schema, source/compliance/alert examples, Splunk HEC, Elastic ECS, and Microsoft Sentinel ingestion shapes, and sensitive-field redaction guidance.
|
||||
- Ticketing/chatops integration templates (Jira, GitHub, Slack, Teams).
|
||||
- Token/scoping model for API access that matches governance needs (service accounts, least privilege).
|
||||
|
||||
Exit criteria:
|
||||
|
||||
@@ -546,3 +546,16 @@ Receivers should verify these headers:
|
||||
Non-2xx responses are retried with exponential backoff until the endpoint's
|
||||
maximum attempt count is reached. Operators can inspect the delivery status,
|
||||
last response code, error text, and response excerpt without reading logs.
|
||||
|
||||
## Integration Templates
|
||||
|
||||
DMARQ ships operator-ready templates for normalized SIEM ingestion:
|
||||
|
||||
| Endpoint | Purpose |
|
||||
| --- | --- |
|
||||
| `GET /api/v1/integrations/siem/templates` | Return versioned SIEM schemas, examples, config hints, and redaction guidance |
|
||||
|
||||
The SIEM template bundle includes the stable `dmarq.siem.event.v1` envelope,
|
||||
examples for sender, compliance-drop, and alert events, and ingestion shapes for
|
||||
Splunk HEC, Elastic ECS, and Microsoft Sentinel custom logs. See
|
||||
[SIEM Integration Templates](siem-integrations.md) for the full operator guide.
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# SIEM Integration Templates
|
||||
|
||||
DMARQ publishes a versioned SIEM event envelope for security analytics
|
||||
pipelines. Use it when forwarding webhook events or API-derived posture data to
|
||||
Splunk, Elastic, Microsoft Sentinel, or another JSON-capable SIEM.
|
||||
|
||||
## Template Endpoint
|
||||
|
||||
Administrators can fetch the current template bundle from:
|
||||
|
||||
```http
|
||||
GET /api/v1/integrations/siem/templates
|
||||
```
|
||||
|
||||
The response contains:
|
||||
|
||||
- `template_version`: release date for the template bundle.
|
||||
- `schema_version`: stable event envelope identifier, currently `dmarq.siem.event.v1`.
|
||||
- `event_schema`: JSON Schema for normalized SIEM events.
|
||||
- `event_examples`: source, compliance-drop, and alert examples.
|
||||
- `ingestion_examples`: Splunk HEC, Elastic ECS, and Microsoft Sentinel shapes.
|
||||
- `config_templates`: endpoint and mapping hints for common SIEM ingestion paths.
|
||||
- `redaction_guidance`: sensitive-field handling rules.
|
||||
|
||||
## Stable Event Envelope
|
||||
|
||||
Every normalized SIEM event uses the same envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "dmarq.siem.event.v1",
|
||||
"event_type": "dmarq.compliance.drop",
|
||||
"event_id": "dmarq-compliance-drop-20260523-example-com",
|
||||
"event_time": "2026-05-23T16:35:00Z",
|
||||
"severity": "high",
|
||||
"source": {
|
||||
"application": "dmarq",
|
||||
"instance": "dmarq-preprod",
|
||||
"environment": "preprod"
|
||||
},
|
||||
"entity": {
|
||||
"domain": "example.com",
|
||||
"sender_ip": null,
|
||||
"sender_org": null,
|
||||
"reporter": null,
|
||||
"alert_rule": "compliance_drop"
|
||||
},
|
||||
"metrics": {
|
||||
"message_count": 1280,
|
||||
"aligned_count": 914,
|
||||
"failed_count": 366,
|
||||
"compliance_rate": 71.41,
|
||||
"previous_compliance_rate": 94.8,
|
||||
"drop_points": 23.39,
|
||||
"missing_days": null
|
||||
},
|
||||
"alert": {
|
||||
"title": "DMARC compliance dropped for example.com",
|
||||
"detail": "Compliance fell by 23.39 percentage points in the current window.",
|
||||
"status": "active"
|
||||
},
|
||||
"redaction": {
|
||||
"pii_redacted": true,
|
||||
"secret_fields_removed": true,
|
||||
"raw_report_included": false,
|
||||
"notes": "Contains aggregate counts and posture metadata only."
|
||||
},
|
||||
"links": {
|
||||
"domain_url": "https://dmarq.example/domains/example.com",
|
||||
"report_url": "https://dmarq.example/domains/example.com/reports"
|
||||
},
|
||||
"tags": ["email-security", "dmarc", "compliance-drop"],
|
||||
"extensions": {}
|
||||
}
|
||||
```
|
||||
|
||||
Supported normalized event types are:
|
||||
|
||||
- `dmarq.report.imported`
|
||||
- `dmarq.sender.new`
|
||||
- `dmarq.compliance.drop`
|
||||
- `dmarq.reports.missing`
|
||||
- `dmarq.alert.created`
|
||||
|
||||
## Splunk HEC
|
||||
|
||||
Use the webhook URL for a relay or directly for Splunk HEC when your network
|
||||
allows it:
|
||||
|
||||
```text
|
||||
https://splunk.example:8088/services/collector/event
|
||||
```
|
||||
|
||||
Send the normalized event as the `event` field:
|
||||
|
||||
```json
|
||||
{
|
||||
"time": 1779554100,
|
||||
"host": "dmarq-preprod",
|
||||
"source": "dmarq:webhook",
|
||||
"sourcetype": "_json",
|
||||
"index": "email_security",
|
||||
"event": {
|
||||
"schema_version": "dmarq.siem.event.v1",
|
||||
"event_type": "dmarq.compliance.drop",
|
||||
"event_id": "dmarq-compliance-drop-20260523-example-com"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Keep the HEC token in Splunk, a receiving proxy, or a secret manager. Do not
|
||||
store Splunk credentials in the DMARQ webhook URL.
|
||||
|
||||
## Elastic ECS
|
||||
|
||||
For Elastic or Logstash, keep the full DMARQ event under `dmarq` and map common
|
||||
fields to ECS:
|
||||
|
||||
```json
|
||||
{
|
||||
"@timestamp": "2026-05-23T16:35:00Z",
|
||||
"ecs.version": "8.11.0",
|
||||
"event.kind": "alert",
|
||||
"event.category": ["email"],
|
||||
"event.type": ["info"],
|
||||
"event.dataset": "dmarq.siem",
|
||||
"observer.vendor": "DMARQ",
|
||||
"observer.product": "DMARQ",
|
||||
"rule.name": "compliance_drop",
|
||||
"host.name": "dmarq-preprod",
|
||||
"dmarq": {
|
||||
"schema_version": "dmarq.siem.event.v1",
|
||||
"event_type": "dmarq.compliance.drop",
|
||||
"event_id": "dmarq-compliance-drop-20260523-example-com"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Index into a dedicated dataset such as
|
||||
`logs-dmarq.email_security-default` so retention, dashboards, and alerts can be
|
||||
managed independently from application logs.
|
||||
|
||||
## Microsoft Sentinel
|
||||
|
||||
For Sentinel custom logs, use a protected relay such as Azure Function or Logic
|
||||
App to add Azure credentials and forward the JSON body to a Data Collection
|
||||
Rule stream:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"TimeGenerated": "2026-05-23T16:35:00Z",
|
||||
"EventType": "dmarq.compliance.drop",
|
||||
"Domain": "example.com",
|
||||
"Severity": "high",
|
||||
"ComplianceRate": 71.41,
|
||||
"PreviousComplianceRate": 94.8,
|
||||
"DropPoints": 23.39,
|
||||
"DmarqEvent": {
|
||||
"schema_version": "dmarq.siem.event.v1",
|
||||
"event_type": "dmarq.compliance.drop",
|
||||
"event_id": "dmarq-compliance-drop-20260523-example-com"
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Map `TimeGenerated` from `event_time` so KQL queries and scheduled analytics
|
||||
rules use DMARQ's event time instead of relay receipt time.
|
||||
|
||||
## Redaction Guidance
|
||||
|
||||
Forward:
|
||||
|
||||
- Aggregate counts and compliance rates.
|
||||
- Monitored domains, sender IPs, reporter names, alert rule names, and event IDs.
|
||||
- Links back to DMARQ pages when the receiving SIEM user is authorized to open them.
|
||||
|
||||
Do not forward:
|
||||
|
||||
- Raw aggregate report XML.
|
||||
- Raw RFC 822 mail content or forensic message bodies.
|
||||
- Mailbox credentials, API tokens, webhook signing secrets, authorization headers, or SIEM ingest tokens.
|
||||
- Recipient local-parts, authentication headers, and forensic identifiers unless a separate privacy review approves them.
|
||||
|
||||
Use `event_id` or `X-DMARQ-Idempotency-Key` for deduplication. Avoid deriving
|
||||
deduplication hashes from raw payloads that may accidentally include sensitive
|
||||
fields.
|
||||
@@ -101,6 +101,11 @@ DMARQ signs each delivery with `X-DMARQ-Signature` and includes
|
||||
`X-DMARQ-Idempotency-Key` so receivers can reject replays and deduplicate
|
||||
retries. Endpoint URLs and signing secrets are encrypted at rest.
|
||||
|
||||
For SIEM pipelines, DMARQ also exposes a versioned template bundle at
|
||||
`/api/v1/integrations/siem/templates`. It includes the stable
|
||||
`dmarq.siem.event.v1` event schema, Splunk HEC, Elastic ECS, and Microsoft
|
||||
Sentinel examples, plus redaction guidance for sensitive fields.
|
||||
|
||||
## API Access
|
||||
|
||||
DMARQ provides an API for integration with other systems:
|
||||
|
||||
@@ -40,6 +40,7 @@ nav:
|
||||
- Configuration: deployment/configuration.md
|
||||
- Technical Reference:
|
||||
- API Reference: reference/api.md
|
||||
- SIEM Integrations: reference/siem-integrations.md
|
||||
- Architecture: reference/architecture.md
|
||||
- Database Schema: reference/database.md
|
||||
- Development:
|
||||
|
||||
Reference in New Issue
Block a user