gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #179 from christianlouis/codex/m11-persist-aggregate-fields

Browse Source 
[codex] Persist DMARC aggregate metadata
This commit is contained in:
Christian Krakau-Louis
2026-05-23 14:20:23 +02:00
committed by GitHub
parent d9e68e721a 9d0b12efea
commit 6818d52a8b
7 changed files with 345 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/versions/e2f3a4b5c6d7_add_dmarcbis_aggregate_fields.py
+54
View File
@@ -0,0 +1,54 @@
"""add dmarcbis aggregate fields
Revision ID: e2f3a4b5c6d7
Revises: d1e2f3a4b5c6
Create Date: 2026-05-23 00:00:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "e2f3a4b5c6d7"
down_revision: Union[str, Sequence[str], None] = "d1e2f3a4b5c6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Persist optional RFC 9990 / DMARCbis aggregate metadata."""
op.add_column("dmarc_reports", sa.Column("extra_contact_info", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("generator", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("report_errors", sa.Text(), nullable=True))
op.add_column("dmarc_reports", sa.Column("non_subdomain_policy", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("failure_options", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("testing", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("discovery_method", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("schema_version", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("report_variant", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("xml_namespace", sa.String(), nullable=True))
op.add_column("dmarc_reports", sa.Column("report_extensions", sa.Text(), nullable=True))
op.add_column("report_records", sa.Column("envelope_to", sa.String(), nullable=True))
op.add_column("report_records", sa.Column("policy_override_reasons", sa.Text(), nullable=True))
op.add_column("report_records", sa.Column("record_extensions", sa.Text(), nullable=True))
def downgrade() -> None:
"""Remove optional RFC 9990 / DMARCbis aggregate metadata."""
op.drop_column("report_records", "record_extensions")
op.drop_column("report_records", "policy_override_reasons")
op.drop_column("report_records", "envelope_to")
op.drop_column("dmarc_reports", "report_extensions")
op.drop_column("dmarc_reports", "xml_namespace")
op.drop_column("dmarc_reports", "report_variant")
op.drop_column("dmarc_reports", "schema_version")
op.drop_column("dmarc_reports", "discovery_method")
op.drop_column("dmarc_reports", "testing")
op.drop_column("dmarc_reports", "failure_options")
op.drop_column("dmarc_reports", "non_subdomain_policy")
op.drop_column("dmarc_reports", "report_errors")
op.drop_column("dmarc_reports", "generator")
op.drop_column("dmarc_reports", "extra_contact_info")
backend/app/api/api_v1/endpoints/domains.py
+32 -8
View File
@@ -357,11 +357,15 @@ def _domain_exists(db: Session, store: ReportStore, domain_name: str) -> bool:
)
def _record_evidence(label: str, value: Optional[str], href: str = "#dns-records") -> DNSHealthEvidence:
def _record_evidence(
label: str, value: Optional[str], href: str = "#dns-records"
) -> DNSHealthEvidence:
return DNSHealthEvidence(label=label, value=value or "Not found", href=href)
def _summary_evidence(label: str, value: object, href: str = "#compliance-chart") -> DNSHealthEvidence:
def _summary_evidence(
label: str, value: object, href: str = "#compliance-chart"
) -> DNSHealthEvidence:
return DNSHealthEvidence(label=label, value=str(value), href=href)
@@ -549,8 +553,7 @@ async def read_domains(db: Session = Depends(get_db)):
domains = _domain_names_for_summary(db, store)
summaries = store.get_all_domain_summaries()
stored = {
domain.name: domain
for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
domain.name: domain for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
}
result = []
@@ -582,9 +585,7 @@ async def create_domain(
):
"""Create a monitored domain before any DMARC reports have arrived."""
name = _normalize_domain_name(payload.name)
validation = validate_domain_config(
{"name": name, "description": payload.description or ""}
)
validation = validate_domain_config({"name": name, "description": payload.description or ""})
if not validation["valid"]:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -818,7 +819,9 @@ async def get_domain_dns_health(
recommendations.append(_enforcement_recommendation(policy, summary))
failed_checks = sum(1 for check in checks if check.status == "fail")
health_status = "healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical"
health_status = (
"healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical"
)
return DNSHealthResponse(
status=health_status,
policy=policy,
@@ -999,6 +1002,16 @@ async def export_domain_reports(
"failed",
"pass_rate",
"policy",
"subdomain_policy",
"non_subdomain_policy",
"adkim",
"aspf",
"failure_options",
"testing",
"discovery_method",
"schema_version",
"report_variant",
"generator",
]
)
@@ -1008,6 +1021,7 @@ async def export_domain_reports(
passed = int(summary.get("passed_count", report.get("passed_count", 0)) or 0)
failed = int(summary.get("failed_count", report.get("failed_count", 0)) or 0)
policy = report.get("policy", "none")
policy_parts = policy if isinstance(policy, dict) else {}
if isinstance(policy, dict):
policy = policy.get("p", "none")
writer.writerow(
@@ -1022,6 +1036,16 @@ async def export_domain_reports(
failed,
report.get("pass_rate", 0.0),
policy,
policy_parts.get("sp", ""),
policy_parts.get("np", ""),
policy_parts.get("adkim", ""),
policy_parts.get("aspf", ""),
policy_parts.get("fo", ""),
policy_parts.get("testing", ""),
policy_parts.get("discovery_method", ""),
report.get("schema_version", ""),
report.get("variant", ""),
report.get("generator", ""),
]
)
backend/app/models/report.py
+14
View File
@@ -20,15 +20,26 @@ class DMARCReport(Base):
begin_date = Column(Integer, nullable=False, index=True) # Unix timestamp
end_date = Column(Integer, nullable=False, index=True) # Unix timestamp
source_email = Column(String, nullable=True)
extra_contact_info = Column(String, nullable=True)
generator = Column(String, nullable=True)
report_errors = Column(Text, nullable=True)
# Policy information
policy = Column(String, nullable=True) # none, quarantine, reject (indexed via __table_args__)
subdomain_policy = Column(String, nullable=True)
non_subdomain_policy = Column(String, nullable=True)
adkim = Column(String(1), nullable=True) # r (relaxed) or s (strict)
aspf = Column(String(1), nullable=True) # r (relaxed) or s (strict)
percentage = Column(Integer, nullable=True)
failure_options = Column(String, nullable=True)
testing = Column(String, nullable=True)
discovery_method = Column(String, nullable=True)
# Processing metadata
schema_version = Column(String, nullable=True)
report_variant = Column(String, nullable=True)
xml_namespace = Column(String, nullable=True)
report_extensions = Column(Text, nullable=True)
processed_at = Column(DateTime, default=datetime.utcnow)
raw_data = Column(Text, nullable=True) # Original XML content (optional)
@@ -72,10 +83,13 @@ class ReportRecord(Base):
# Identifiers
header_from = Column(String, nullable=True, index=True)
envelope_from = Column(String, nullable=True)
envelope_to = Column(String, nullable=True)
# Authentication details (optional JSON fields)
dkim_auth_details = Column(Text, nullable=True) # JSON array of DKIM results
spf_auth_details = Column(Text, nullable=True) # JSON array of SPF results
policy_override_reasons = Column(Text, nullable=True) # JSON array of policy reasons
record_extensions = Column(Text, nullable=True) # JSON object of extension values
# Relationships
report = relationship("DMARCReport", back_populates="records")
backend/app/services/report_persistence.py
+51
View File
@@ -41,6 +41,22 @@ def _loads_json_list(value: Optional[str]) -> Optional[List[Dict[str, Any]]]:
return decoded if isinstance(decoded, list) else None
def _loads_json_dict(value: Optional[str]) -> Optional[Dict[str, Any]]:
if not value:
return None
try:
decoded = json.loads(value)
except (json.JSONDecodeError, TypeError):
return None
return decoded if isinstance(decoded, dict) else None
def _json_or_none(value: Any) -> Optional[str]:
if value in (None, "", [], {}):
return None
return json.dumps(value, sort_keys=True)
def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]:
policy = report.get("policy") or {}
if isinstance(policy, str):
@@ -51,8 +67,12 @@ def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]:
"p": policy.get("p", "none"),
"sp": policy.get("sp", ""),
"pct": str(policy.get("pct", "100")),
"np": policy.get("np"),
"fo": policy.get("fo"),
"adkim": policy.get("adkim") or report.get("adkim"),
"aspf": policy.get("aspf") or report.get("aspf"),
"testing": policy.get("testing"),
"discovery_method": policy.get("discovery_method"),
}
@@ -106,11 +126,22 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport
begin_date=begin_ts,
end_date=end_ts,
source_email=report.get("email") or report.get("source_email"),
extra_contact_info=report.get("extra_contact_info") or None,
generator=report.get("generator") or None,
report_errors=_json_or_none(report.get("errors")),
policy=policy["p"],
subdomain_policy=policy.get("sp") or None,
non_subdomain_policy=policy.get("np") or None,
adkim=policy.get("adkim") or None,
aspf=policy.get("aspf") or None,
percentage=pct,
failure_options=policy.get("fo") or None,
testing=policy.get("testing") or None,
discovery_method=policy.get("discovery_method") or None,
schema_version=report.get("schema_version") or None,
report_variant=report.get("variant") or None,
xml_namespace=report.get("xml_namespace") or None,
report_extensions=_json_or_none(report.get("extensions")),
)
db.add(db_report)
db.flush()
@@ -126,12 +157,15 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport
spf=record.get("spf_result") or record.get("spf") or "unknown",
header_from=record.get("header_from"),
envelope_from=record.get("envelope_from"),
envelope_to=record.get("envelope_to"),
dkim_auth_details=(
json.dumps(record.get("dkim")) if isinstance(record.get("dkim"), list) else None
),
spf_auth_details=(
json.dumps(record.get("spf")) if isinstance(record.get("spf"), list) else None
),
policy_override_reasons=_json_or_none(record.get("policy_override_reasons")),
record_extensions=_json_or_none(record.get("extensions")),
)
)
@@ -160,8 +194,12 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
"dkim_result": dkim_result,
"spf_result": spf_result,
"header_from": record.header_from or "",
"envelope_from": record.envelope_from or "",
"envelope_to": record.envelope_to or "",
"dkim": _loads_json_list(record.dkim_auth_details) or [],
"spf": _loads_json_list(record.spf_auth_details) or [],
"policy_override_reasons": (_loads_json_list(record.policy_override_reasons) or []),
"extensions": _loads_json_dict(record.record_extensions) or {},
}
)
@@ -172,6 +210,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
"report_id": report.report_id,
"org_name": report.org_name,
"email": report.source_email or "",
"extra_contact_info": report.extra_contact_info or "",
"generator": report.generator or "",
"errors": _loads_json_list(report.report_errors) or [],
"variant": report.report_variant or "",
"schema_version": report.schema_version or "",
"xml_namespace": report.xml_namespace or "",
"extensions": _loads_json_dict(report.report_extensions) or {},
"begin_date": _iso_from_timestamp(report.begin_date),
"end_date": _iso_from_timestamp(report.end_date),
"begin_timestamp": report.begin_date,
@@ -179,7 +224,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
"policy": {
"p": report.policy or "none",
"sp": report.subdomain_policy or "",
"np": report.non_subdomain_policy or "",
"pct": str(report.percentage or 100),
"fo": report.failure_options or "",
"adkim": report.adkim or "",
"aspf": report.aspf or "",
"testing": report.testing or "",
"discovery_method": report.discovery_method or "",
},
"records": records,
"summary": {
backend/app/tests/test_domain_detail_endpoints.py
+30 -4
View File
@@ -33,7 +33,20 @@ REPORT_DICT_POLICY = {
"end_date": "2020-08-15T23:59:59",
"begin_timestamp": 1597449600,
"end_timestamp": 1597535999,
"policy": {"p": "reject", "sp": "reject", "pct": "100"},
"policy": {
"p": "reject",
"sp": "reject",
"pct": "100",
"np": "none",
"adkim": "s",
"aspf": "r",
"fo": "1",
"testing": "y",
"discovery_method": "treewalk",
},
"schema_version": "1.0",
"variant": "rfc9990",
"generator": "ExampleRUA 2.0",
"records": [
{
"source_ip": "209.85.220.1",
@@ -139,6 +152,15 @@ def test_export_domain_reports_returns_csv(seeded_client: TestClient):
assert rows[0]["passed"] == "10"
assert rows[0]["failed"] == "0"
assert rows[0]["policy"] == "reject"
assert rows[0]["non_subdomain_policy"] == "none"
assert rows[0]["adkim"] == "s"
assert rows[0]["aspf"] == "r"
assert rows[0]["failure_options"] == "1"
assert rows[0]["testing"] == "y"
assert rows[0]["discovery_method"] == "treewalk"
assert rows[0]["schema_version"] == "1.0"
assert rows[0]["report_variant"] == "rfc9990"
assert rows[0]["generator"] == "ExampleRUA 2.0"
def test_export_domain_reports_filters_by_date_range(client: TestClient):
@@ -243,7 +265,9 @@ def test_get_domain_sources_returns_rollup_counts(client: TestClient):
assert source["disposition_counts"] == {"none": 4, "quarantine": 6}
def test_get_domain_sources_returns_recommendations(client: TestClient, monkeypatch: pytest.MonkeyPatch):
def test_get_domain_sources_returns_recommendations(
client: TestClient, monkeypatch: pytest.MonkeyPatch
):
"""Endpoint includes actionable guidance for common failure patterns."""
async def fake_ptr_lookup(_provider, _ip, timeout=3.0): # pylint: disable=unused-argument
@@ -339,8 +363,10 @@ def test_source_recommendations_cover_common_cases():
]
for source, hostname, spf_fix_hint, expected_types in cases:
recommendations = domains_endpoint._source_recommendations( # pylint: disable=protected-access
source["source_ip"], source, hostname, spf_fix_hint
recommendations = (
domains_endpoint._source_recommendations( # pylint: disable=protected-access
source["source_ip"], source, hostname, spf_fix_hint
)
)
assert {item.type for item in recommendations} == expected_types
backend/app/tests/test_reports_api.py
+162
View File
@@ -3,11 +3,81 @@ import zipfile
from fastapi.testclient import TestClient
from app.models.domain import Domain
from app.models.report import DMARCReport, ReportRecord
from app.services.report_persistence import persisted_report_to_dict
from app.services.report_store import ReportStore
from app.tests.test_data import SAMPLE_XML
SAMPLE_RFC9990_XML = """
<feedback xmlns="urn:ietf:params:xml:ns:dmarc-2.0" xmlns:vendor="https://reports.example.test/dmarc">
<version>1.0</version>
<report_metadata>
<org_name>Example Receiver</org_name>
<email>dmarc@example.test</email>
<extra_contact_info>https://example.test/dmarc</extra_contact_info>
<report_id>2026-05-23-example.org</report_id>
<date_range>
<begin>1779494400</begin>
<end>1779580799</end>
</date_range>
<error>Multiple records ignored.</error>
<generator>ExampleRUA 2.0</generator>
</report_metadata>
<policy_published>
<domain>example.org</domain>
<discovery_method>treewalk</discovery_method>
<p>quarantine</p>
<sp>reject</sp>
<np>none</np>
<fo>1</fo>
<adkim>s</adkim>
<aspf>r</aspf>
<testing>y</testing>
</policy_published>
<extension>
<vendor:receiver>mx1.example.test</vendor:receiver>
</extension>
<record>
<row>
<source_ip>2001:db8::1</source_ip>
<count>5</count>
<policy_evaluated>
<disposition>quarantine</disposition>
<dkim>fail</dkim>
<spf>pass</spf>
<reason>
<type>local_policy</type>
<comment>trusted relay</comment>
</reason>
</policy_evaluated>
</row>
<identifiers>
<header_from>news.example.org</header_from>
<envelope_from>bounce.example.org</envelope_from>
<envelope_to>customer.example.net</envelope_to>
</identifiers>
<auth_results>
<dkim>
<domain>example.net</domain>
<selector>selector1</selector>
<result>fail</result>
<human_result>body hash did not verify</human_result>
</dkim>
<spf>
<domain>bounce.example.org</domain>
<scope>mfrom</scope>
<result>pass</result>
<human_result>sender authorized</human_result>
</spf>
</auth_results>
<vendor:source>mail-platform</vendor:source>
</record>
</feedback>
"""
def _make_zip(xml_content: str) -> bytes:
"""Create a ZIP file containing the given XML content."""
buf = io.BytesIO()
@@ -44,6 +114,98 @@ def test_upload_persists_report_rows(client: TestClient, db_session):
assert db_session.query(ReportRecord).filter_by(report_id=report.id).count() == 1
def test_upload_persists_rfc9990_optional_fields(client: TestClient, db_session):
"""RFC 9990 / DMARCbis metadata is kept for exports and future UI use."""
zip_bytes = _make_zip(SAMPLE_RFC9990_XML)
response = client.post(
"/api/v1/reports/upload",
files={"file": ("report.zip", zip_bytes, "application/zip")},
)
assert response.status_code == 200
report = db_session.query(DMARCReport).filter_by(report_id="2026-05-23-example.org").one()
assert report.domain.name == "example.org"
assert report.extra_contact_info == "https://example.test/dmarc"
assert report.generator == "ExampleRUA 2.0"
assert report.report_variant == "rfc9990"
assert report.schema_version == "1.0"
assert report.xml_namespace == "urn:ietf:params:xml:ns:dmarc-2.0"
assert report.non_subdomain_policy == "none"
assert report.failure_options == "1"
assert report.testing == "y"
assert report.discovery_method == "treewalk"
assert "Multiple records ignored." in report.report_errors
assert "mx1.example.test" in report.report_extensions
record = db_session.query(ReportRecord).filter_by(report_id=report.id).one()
assert record.envelope_from == "bounce.example.org"
assert record.envelope_to == "customer.example.net"
assert "local_policy" in record.policy_override_reasons
assert "mail-platform" in record.record_extensions
assert "human_result" in record.dkim_auth_details
assert "scope" in record.spf_auth_details
def test_persisted_report_to_dict_hydrates_optional_json_metadata(db_session):
"""Persisted RFC 9990 extension metadata is restored safely for readers."""
domain = Domain(name="metadata.example", dmarc_policy="reject")
report = DMARCReport(
domain=domain,
report_id="metadata-report",
org_name="Example Receiver",
begin_date=1779494400,
end_date=1779580799,
source_email="dmarc@example.test",
report_errors='["Multiple records ignored."]',
policy="reject",
subdomain_policy="quarantine",
non_subdomain_policy="none",
adkim="s",
aspf="r",
percentage=100,
failure_options="1",
testing="y",
discovery_method="treewalk",
schema_version="1.0",
report_variant="rfc9990",
xml_namespace="urn:ietf:params:xml:ns:dmarc-2.0",
report_extensions='{"vendor:receiver": "mx1.example.test"}',
)
report.records.append(
ReportRecord(
source_ip="2001:db8::1",
count=5,
disposition="quarantine",
dkim="fail",
spf="pass",
header_from="news.metadata.example",
envelope_from="bounce.metadata.example",
envelope_to="customer.example.net",
policy_override_reasons='[{"type": "local_policy"}]',
record_extensions='{"vendor:source": "mail-platform"}',
)
)
db_session.add(report)
db_session.flush()
hydrated = persisted_report_to_dict(report)
assert hydrated["errors"] == ["Multiple records ignored."]
assert hydrated["extensions"] == {"vendor:receiver": "mx1.example.test"}
assert hydrated["policy"]["np"] == "none"
assert hydrated["policy"]["fo"] == "1"
assert hydrated["records"][0]["envelope_to"] == "customer.example.net"
assert hydrated["records"][0]["extensions"] == {"vendor:source": "mail-platform"}
report.report_extensions = "{not-json"
report.records[0].record_extensions = "{not-json"
hydrated_with_bad_json = persisted_report_to_dict(report)
assert hydrated_with_bad_json["extensions"] == {}
assert hydrated_with_bad_json["records"][0]["extensions"] == {}
def test_report_reads_hydrate_from_persisted_rows(client: TestClient):
"""Report read APIs rebuild the in-memory projection from the database."""
zip_bytes = _make_zip(SAMPLE_XML)
docs/milestones.md
+2 -2
View File
@@ -189,10 +189,10 @@ Goal: keep DMARQ compatible with evolving DMARC report formats and nomenclature
Delivered:
- Add parser compatibility for RFC 9990-style aggregate report namespaces, version detection, policy metadata, identifiers, override reasons, auth-result details, and namespaced extensions.
- Keep legacy RFC 7489-style reports backward compatible through fixture coverage.
- Persist and CSV-export newly introduced aggregate metadata with nullable, backward-safe database fields.
Planned:
- Store newly introduced fields with safe defaults.
- Update CSV export and domain/source reporting to include new metadata where it improves operator actionability.
- Update domain/source reporting to include new metadata where it improves operator actionability.
- Add fixture-driven tests for representative real-world DMARCbis-style reports.
- Update documentation to clarify supported formats and terminology.