feat: persist DMARC aggregate metadata

backend/alembic/versions/e2f3a4b5c6d7_add_dmarcbis_aggregate_fields.py

@@ -0,0 +1,54 @@
+"""add dmarcbis aggregate fields
+
+Revision ID: e2f3a4b5c6d7
+Revises: d1e2f3a4b5c6
+Create Date: 2026-05-23 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "e2f3a4b5c6d7"
+down_revision: Union[str, Sequence[str], None] = "d1e2f3a4b5c6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Persist optional RFC 9990 / DMARCbis aggregate metadata."""
+    op.add_column("dmarc_reports", sa.Column("extra_contact_info", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("generator", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("report_errors", sa.Text(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("non_subdomain_policy", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("failure_options", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("testing", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("discovery_method", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("schema_version", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("report_variant", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("xml_namespace", sa.String(), nullable=True))
+    op.add_column("dmarc_reports", sa.Column("report_extensions", sa.Text(), nullable=True))
+    op.add_column("report_records", sa.Column("envelope_to", sa.String(), nullable=True))
+    op.add_column("report_records", sa.Column("policy_override_reasons", sa.Text(), nullable=True))
+    op.add_column("report_records", sa.Column("record_extensions", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    """Remove optional RFC 9990 / DMARCbis aggregate metadata."""
+    op.drop_column("report_records", "record_extensions")
+    op.drop_column("report_records", "policy_override_reasons")
+    op.drop_column("report_records", "envelope_to")
+    op.drop_column("dmarc_reports", "report_extensions")
+    op.drop_column("dmarc_reports", "xml_namespace")
+    op.drop_column("dmarc_reports", "report_variant")
+    op.drop_column("dmarc_reports", "schema_version")
+    op.drop_column("dmarc_reports", "discovery_method")
+    op.drop_column("dmarc_reports", "testing")
+    op.drop_column("dmarc_reports", "failure_options")
+    op.drop_column("dmarc_reports", "non_subdomain_policy")
+    op.drop_column("dmarc_reports", "report_errors")
+    op.drop_column("dmarc_reports", "generator")
+    op.drop_column("dmarc_reports", "extra_contact_info")

backend/app/api/api_v1/endpoints/domains.py

@@ -357,11 +357,15 @@ def _domain_exists(db: Session, store: ReportStore, domain_name: str) -> bool:
     )
 
 
-def _record_evidence(label: str, value: Optional[str], href: str = "#dns-records") -> DNSHealthEvidence:
+def _record_evidence(
+    label: str, value: Optional[str], href: str = "#dns-records"
+) -> DNSHealthEvidence:
     return DNSHealthEvidence(label=label, value=value or "Not found", href=href)
 
 
-def _summary_evidence(label: str, value: object, href: str = "#compliance-chart") -> DNSHealthEvidence:
+def _summary_evidence(
+    label: str, value: object, href: str = "#compliance-chart"
+) -> DNSHealthEvidence:
     return DNSHealthEvidence(label=label, value=str(value), href=href)
 
 
@@ -549,8 +553,7 @@ async def read_domains(db: Session = Depends(get_db)):
     domains = _domain_names_for_summary(db, store)
     summaries = store.get_all_domain_summaries()
     stored = {
-        domain.name: domain
+        domain.name: domain for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
-        for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
     }
 
     result = []
@@ -582,9 +585,7 @@ async def create_domain(
 ):
     """Create a monitored domain before any DMARC reports have arrived."""
     name = _normalize_domain_name(payload.name)
-    validation = validate_domain_config(
+    validation = validate_domain_config({"name": name, "description": payload.description or ""})
-        {"name": name, "description": payload.description or ""}
-    )
     if not validation["valid"]:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -818,7 +819,9 @@ async def get_domain_dns_health(
     recommendations.append(_enforcement_recommendation(policy, summary))
 
     failed_checks = sum(1 for check in checks if check.status == "fail")
-    health_status = "healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical"
+    health_status = (
+        "healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical"
+    )
     return DNSHealthResponse(
         status=health_status,
         policy=policy,
@@ -999,6 +1002,16 @@ async def export_domain_reports(
             "failed",
             "pass_rate",
             "policy",
+            "subdomain_policy",
+            "non_subdomain_policy",
+            "adkim",
+            "aspf",
+            "failure_options",
+            "testing",
+            "discovery_method",
+            "schema_version",
+            "report_variant",
+            "generator",
         ]
     )
 
@@ -1008,6 +1021,7 @@ async def export_domain_reports(
         passed = int(summary.get("passed_count", report.get("passed_count", 0)) or 0)
         failed = int(summary.get("failed_count", report.get("failed_count", 0)) or 0)
         policy = report.get("policy", "none")
+        policy_parts = policy if isinstance(policy, dict) else {}
         if isinstance(policy, dict):
             policy = policy.get("p", "none")
         writer.writerow(
@@ -1022,6 +1036,16 @@ async def export_domain_reports(
                 failed,
                 report.get("pass_rate", 0.0),
                 policy,
+                policy_parts.get("sp", ""),
+                policy_parts.get("np", ""),
+                policy_parts.get("adkim", ""),
+                policy_parts.get("aspf", ""),
+                policy_parts.get("fo", ""),
+                policy_parts.get("testing", ""),
+                policy_parts.get("discovery_method", ""),
+                report.get("schema_version", ""),
+                report.get("variant", ""),
+                report.get("generator", ""),
             ]
         )
 

backend/app/models/report.py

@@ -20,15 +20,26 @@ class DMARCReport(Base):
     begin_date = Column(Integer, nullable=False, index=True)  # Unix timestamp
     end_date = Column(Integer, nullable=False, index=True)  # Unix timestamp
     source_email = Column(String, nullable=True)
+    extra_contact_info = Column(String, nullable=True)
+    generator = Column(String, nullable=True)
+    report_errors = Column(Text, nullable=True)
 
     # Policy information
     policy = Column(String, nullable=True)  # none, quarantine, reject (indexed via __table_args__)
     subdomain_policy = Column(String, nullable=True)
+    non_subdomain_policy = Column(String, nullable=True)
     adkim = Column(String(1), nullable=True)  # r (relaxed) or s (strict)
     aspf = Column(String(1), nullable=True)  # r (relaxed) or s (strict)
     percentage = Column(Integer, nullable=True)
+    failure_options = Column(String, nullable=True)
+    testing = Column(String, nullable=True)
+    discovery_method = Column(String, nullable=True)
 
     # Processing metadata
+    schema_version = Column(String, nullable=True)
+    report_variant = Column(String, nullable=True)
+    xml_namespace = Column(String, nullable=True)
+    report_extensions = Column(Text, nullable=True)
     processed_at = Column(DateTime, default=datetime.utcnow)
     raw_data = Column(Text, nullable=True)  # Original XML content (optional)
 
@@ -72,10 +83,13 @@ class ReportRecord(Base):
     # Identifiers
     header_from = Column(String, nullable=True, index=True)
     envelope_from = Column(String, nullable=True)
+    envelope_to = Column(String, nullable=True)
 
     # Authentication details (optional JSON fields)
     dkim_auth_details = Column(Text, nullable=True)  # JSON array of DKIM results
     spf_auth_details = Column(Text, nullable=True)  # JSON array of SPF results
+    policy_override_reasons = Column(Text, nullable=True)  # JSON array of policy reasons
+    record_extensions = Column(Text, nullable=True)  # JSON object of extension values
 
     # Relationships
     report = relationship("DMARCReport", back_populates="records")

backend/app/services/report_persistence.py

@@ -41,6 +41,22 @@ def _loads_json_list(value: Optional[str]) -> Optional[List[Dict[str, Any]]]:
     return decoded if isinstance(decoded, list) else None
 
 
+def _loads_json_dict(value: Optional[str]) -> Optional[Dict[str, Any]]:
+    if not value:
+        return None
+    try:
+        decoded = json.loads(value)
+    except (json.JSONDecodeError, TypeError):
+        return None
+    return decoded if isinstance(decoded, dict) else None
+
+
+def _json_or_none(value: Any) -> Optional[str]:
+    if value in (None, "", [], {}):
+        return None
+    return json.dumps(value, sort_keys=True)
+
+
 def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]:
     policy = report.get("policy") or {}
     if isinstance(policy, str):
@@ -51,8 +67,12 @@ def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]:
         "p": policy.get("p", "none"),
         "sp": policy.get("sp", ""),
         "pct": str(policy.get("pct", "100")),
+        "np": policy.get("np"),
+        "fo": policy.get("fo"),
         "adkim": policy.get("adkim") or report.get("adkim"),
         "aspf": policy.get("aspf") or report.get("aspf"),
+        "testing": policy.get("testing"),
+        "discovery_method": policy.get("discovery_method"),
     }
 
 
@@ -106,11 +126,22 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport
         begin_date=begin_ts,
         end_date=end_ts,
         source_email=report.get("email") or report.get("source_email"),
+        extra_contact_info=report.get("extra_contact_info") or None,
+        generator=report.get("generator") or None,
+        report_errors=_json_or_none(report.get("errors")),
         policy=policy["p"],
         subdomain_policy=policy.get("sp") or None,
+        non_subdomain_policy=policy.get("np") or None,
         adkim=policy.get("adkim") or None,
         aspf=policy.get("aspf") or None,
         percentage=pct,
+        failure_options=policy.get("fo") or None,
+        testing=policy.get("testing") or None,
+        discovery_method=policy.get("discovery_method") or None,
+        schema_version=report.get("schema_version") or None,
+        report_variant=report.get("variant") or None,
+        xml_namespace=report.get("xml_namespace") or None,
+        report_extensions=_json_or_none(report.get("extensions")),
     )
     db.add(db_report)
     db.flush()
@@ -126,12 +157,15 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport
                 spf=record.get("spf_result") or record.get("spf") or "unknown",
                 header_from=record.get("header_from"),
                 envelope_from=record.get("envelope_from"),
+                envelope_to=record.get("envelope_to"),
                 dkim_auth_details=(
                     json.dumps(record.get("dkim")) if isinstance(record.get("dkim"), list) else None
                 ),
                 spf_auth_details=(
                     json.dumps(record.get("spf")) if isinstance(record.get("spf"), list) else None
                 ),
+                policy_override_reasons=_json_or_none(record.get("policy_override_reasons")),
+                record_extensions=_json_or_none(record.get("extensions")),
             )
         )
 
@@ -160,8 +194,12 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
                 "dkim_result": dkim_result,
                 "spf_result": spf_result,
                 "header_from": record.header_from or "",
+                "envelope_from": record.envelope_from or "",
+                "envelope_to": record.envelope_to or "",
                 "dkim": _loads_json_list(record.dkim_auth_details) or [],
                 "spf": _loads_json_list(record.spf_auth_details) or [],
+                "policy_override_reasons": (_loads_json_list(record.policy_override_reasons) or []),
+                "extensions": _loads_json_dict(record.record_extensions) or {},
             }
         )
 
@@ -172,6 +210,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
         "report_id": report.report_id,
         "org_name": report.org_name,
         "email": report.source_email or "",
+        "extra_contact_info": report.extra_contact_info or "",
+        "generator": report.generator or "",
+        "errors": _loads_json_list(report.report_errors) or [],
+        "variant": report.report_variant or "",
+        "schema_version": report.schema_version or "",
+        "xml_namespace": report.xml_namespace or "",
+        "extensions": _loads_json_dict(report.report_extensions) or {},
         "begin_date": _iso_from_timestamp(report.begin_date),
         "end_date": _iso_from_timestamp(report.end_date),
         "begin_timestamp": report.begin_date,
@@ -179,7 +224,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
         "policy": {
             "p": report.policy or "none",
             "sp": report.subdomain_policy or "",
+            "np": report.non_subdomain_policy or "",
             "pct": str(report.percentage or 100),
+            "fo": report.failure_options or "",
+            "adkim": report.adkim or "",
+            "aspf": report.aspf or "",
+            "testing": report.testing or "",
+            "discovery_method": report.discovery_method or "",
         },
         "records": records,
         "summary": {

backend/app/tests/test_domain_detail_endpoints.py

@@ -33,7 +33,20 @@ REPORT_DICT_POLICY = {
     "end_date": "2020-08-15T23:59:59",
     "begin_timestamp": 1597449600,
     "end_timestamp": 1597535999,
-    "policy": {"p": "reject", "sp": "reject", "pct": "100"},
+    "policy": {
+        "p": "reject",
+        "sp": "reject",
+        "pct": "100",
+        "np": "none",
+        "adkim": "s",
+        "aspf": "r",
+        "fo": "1",
+        "testing": "y",
+        "discovery_method": "treewalk",
+    },
+    "schema_version": "1.0",
+    "variant": "rfc9990",
+    "generator": "ExampleRUA 2.0",
     "records": [
         {
             "source_ip": "209.85.220.1",
@@ -139,6 +152,15 @@ def test_export_domain_reports_returns_csv(seeded_client: TestClient):
     assert rows[0]["passed"] == "10"
     assert rows[0]["failed"] == "0"
     assert rows[0]["policy"] == "reject"
+    assert rows[0]["non_subdomain_policy"] == "none"
+    assert rows[0]["adkim"] == "s"
+    assert rows[0]["aspf"] == "r"
+    assert rows[0]["failure_options"] == "1"
+    assert rows[0]["testing"] == "y"
+    assert rows[0]["discovery_method"] == "treewalk"
+    assert rows[0]["schema_version"] == "1.0"
+    assert rows[0]["report_variant"] == "rfc9990"
+    assert rows[0]["generator"] == "ExampleRUA 2.0"
 
 
 def test_export_domain_reports_filters_by_date_range(client: TestClient):
@@ -243,7 +265,9 @@ def test_get_domain_sources_returns_rollup_counts(client: TestClient):
     assert source["disposition_counts"] == {"none": 4, "quarantine": 6}
 
 
-def test_get_domain_sources_returns_recommendations(client: TestClient, monkeypatch: pytest.MonkeyPatch):
+def test_get_domain_sources_returns_recommendations(
+    client: TestClient, monkeypatch: pytest.MonkeyPatch
+):
     """Endpoint includes actionable guidance for common failure patterns."""
 
     async def fake_ptr_lookup(_provider, _ip, timeout=3.0):  # pylint: disable=unused-argument
@@ -339,8 +363,10 @@ def test_source_recommendations_cover_common_cases():
     ]
 
     for source, hostname, spf_fix_hint, expected_types in cases:
-        recommendations = domains_endpoint._source_recommendations(  # pylint: disable=protected-access
+        recommendations = (
-            source["source_ip"], source, hostname, spf_fix_hint
+            domains_endpoint._source_recommendations(  # pylint: disable=protected-access
+                source["source_ip"], source, hostname, spf_fix_hint
+            )
         )
         assert {item.type for item in recommendations} == expected_types
 

backend/app/tests/test_reports_api.py

@@ -3,11 +3,81 @@ import zipfile
 
 from fastapi.testclient import TestClient
 
+from app.models.domain import Domain
 from app.models.report import DMARCReport, ReportRecord
+from app.services.report_persistence import persisted_report_to_dict
 from app.services.report_store import ReportStore
 from app.tests.test_data import SAMPLE_XML
 
 
+SAMPLE_RFC9990_XML = """
+<feedback xmlns="urn:ietf:params:xml:ns:dmarc-2.0" xmlns:vendor="https://reports.example.test/dmarc">
+    <version>1.0</version>
+    <report_metadata>
+        <org_name>Example Receiver</org_name>
+        <email>dmarc@example.test</email>
+        <extra_contact_info>https://example.test/dmarc</extra_contact_info>
+        <report_id>2026-05-23-example.org</report_id>
+        <date_range>
+            <begin>1779494400</begin>
+            <end>1779580799</end>
+        </date_range>
+        <error>Multiple records ignored.</error>
+        <generator>ExampleRUA 2.0</generator>
+    </report_metadata>
+    <policy_published>
+        <domain>example.org</domain>
+        <discovery_method>treewalk</discovery_method>
+        <p>quarantine</p>
+        <sp>reject</sp>
+        <np>none</np>
+        <fo>1</fo>
+        <adkim>s</adkim>
+        <aspf>r</aspf>
+        <testing>y</testing>
+    </policy_published>
+    <extension>
+        <vendor:receiver>mx1.example.test</vendor:receiver>
+    </extension>
+    <record>
+        <row>
+            <source_ip>2001:db8::1</source_ip>
+            <count>5</count>
+            <policy_evaluated>
+                <disposition>quarantine</disposition>
+                <dkim>fail</dkim>
+                <spf>pass</spf>
+                <reason>
+                    <type>local_policy</type>
+                    <comment>trusted relay</comment>
+                </reason>
+            </policy_evaluated>
+        </row>
+        <identifiers>
+            <header_from>news.example.org</header_from>
+            <envelope_from>bounce.example.org</envelope_from>
+            <envelope_to>customer.example.net</envelope_to>
+        </identifiers>
+        <auth_results>
+            <dkim>
+                <domain>example.net</domain>
+                <selector>selector1</selector>
+                <result>fail</result>
+                <human_result>body hash did not verify</human_result>
+            </dkim>
+            <spf>
+                <domain>bounce.example.org</domain>
+                <scope>mfrom</scope>
+                <result>pass</result>
+                <human_result>sender authorized</human_result>
+            </spf>
+        </auth_results>
+        <vendor:source>mail-platform</vendor:source>
+    </record>
+</feedback>
+"""
+
+
 def _make_zip(xml_content: str) -> bytes:
     """Create a ZIP file containing the given XML content."""
     buf = io.BytesIO()
@@ -44,6 +114,98 @@ def test_upload_persists_report_rows(client: TestClient, db_session):
     assert db_session.query(ReportRecord).filter_by(report_id=report.id).count() == 1
 
 
+def test_upload_persists_rfc9990_optional_fields(client: TestClient, db_session):
+    """RFC 9990 / DMARCbis metadata is kept for exports and future UI use."""
+    zip_bytes = _make_zip(SAMPLE_RFC9990_XML)
+    response = client.post(
+        "/api/v1/reports/upload",
+        files={"file": ("report.zip", zip_bytes, "application/zip")},
+    )
+    assert response.status_code == 200
+
+    report = db_session.query(DMARCReport).filter_by(report_id="2026-05-23-example.org").one()
+    assert report.domain.name == "example.org"
+    assert report.extra_contact_info == "https://example.test/dmarc"
+    assert report.generator == "ExampleRUA 2.0"
+    assert report.report_variant == "rfc9990"
+    assert report.schema_version == "1.0"
+    assert report.xml_namespace == "urn:ietf:params:xml:ns:dmarc-2.0"
+    assert report.non_subdomain_policy == "none"
+    assert report.failure_options == "1"
+    assert report.testing == "y"
+    assert report.discovery_method == "treewalk"
+    assert "Multiple records ignored." in report.report_errors
+    assert "mx1.example.test" in report.report_extensions
+
+    record = db_session.query(ReportRecord).filter_by(report_id=report.id).one()
+    assert record.envelope_from == "bounce.example.org"
+    assert record.envelope_to == "customer.example.net"
+    assert "local_policy" in record.policy_override_reasons
+    assert "mail-platform" in record.record_extensions
+    assert "human_result" in record.dkim_auth_details
+    assert "scope" in record.spf_auth_details
+
+
+def test_persisted_report_to_dict_hydrates_optional_json_metadata(db_session):
+    """Persisted RFC 9990 extension metadata is restored safely for readers."""
+    domain = Domain(name="metadata.example", dmarc_policy="reject")
+    report = DMARCReport(
+        domain=domain,
+        report_id="metadata-report",
+        org_name="Example Receiver",
+        begin_date=1779494400,
+        end_date=1779580799,
+        source_email="dmarc@example.test",
+        report_errors='["Multiple records ignored."]',
+        policy="reject",
+        subdomain_policy="quarantine",
+        non_subdomain_policy="none",
+        adkim="s",
+        aspf="r",
+        percentage=100,
+        failure_options="1",
+        testing="y",
+        discovery_method="treewalk",
+        schema_version="1.0",
+        report_variant="rfc9990",
+        xml_namespace="urn:ietf:params:xml:ns:dmarc-2.0",
+        report_extensions='{"vendor:receiver": "mx1.example.test"}',
+    )
+    report.records.append(
+        ReportRecord(
+            source_ip="2001:db8::1",
+            count=5,
+            disposition="quarantine",
+            dkim="fail",
+            spf="pass",
+            header_from="news.metadata.example",
+            envelope_from="bounce.metadata.example",
+            envelope_to="customer.example.net",
+            policy_override_reasons='[{"type": "local_policy"}]',
+            record_extensions='{"vendor:source": "mail-platform"}',
+        )
+    )
+    db_session.add(report)
+    db_session.flush()
+
+    hydrated = persisted_report_to_dict(report)
+
+    assert hydrated["errors"] == ["Multiple records ignored."]
+    assert hydrated["extensions"] == {"vendor:receiver": "mx1.example.test"}
+    assert hydrated["policy"]["np"] == "none"
+    assert hydrated["policy"]["fo"] == "1"
+    assert hydrated["records"][0]["envelope_to"] == "customer.example.net"
+    assert hydrated["records"][0]["extensions"] == {"vendor:source": "mail-platform"}
+
+    report.report_extensions = "{not-json"
+    report.records[0].record_extensions = "{not-json"
+
+    hydrated_with_bad_json = persisted_report_to_dict(report)
+
+    assert hydrated_with_bad_json["extensions"] == {}
+    assert hydrated_with_bad_json["records"][0]["extensions"] == {}
+
+
 def test_report_reads_hydrate_from_persisted_rows(client: TestClient):
     """Report read APIs rebuild the in-memory projection from the database."""
     zip_bytes = _make_zip(SAMPLE_XML)

docs/milestones.md

@@ -189,10 +189,10 @@ Goal: keep DMARQ compatible with evolving DMARC report formats and nomenclature
 Delivered:
 - Add parser compatibility for RFC 9990-style aggregate report namespaces, version detection, policy metadata, identifiers, override reasons, auth-result details, and namespaced extensions.
 - Keep legacy RFC 7489-style reports backward compatible through fixture coverage.
+- Persist and CSV-export newly introduced aggregate metadata with nullable, backward-safe database fields.
 
 Planned:
-- Store newly introduced fields with safe defaults.
+- Update domain/source reporting to include new metadata where it improves operator actionability.
-- Update CSV export and domain/source reporting to include new metadata where it improves operator actionability.
 - Add fixture-driven tests for representative real-world DMARCbis-style reports.
 - Update documentation to clarify supported formats and terminology.