diff --git a/backend/alembic/versions/e2f3a4b5c6d7_add_dmarcbis_aggregate_fields.py b/backend/alembic/versions/e2f3a4b5c6d7_add_dmarcbis_aggregate_fields.py new file mode 100644 index 0000000..ece79dc --- /dev/null +++ b/backend/alembic/versions/e2f3a4b5c6d7_add_dmarcbis_aggregate_fields.py @@ -0,0 +1,54 @@ +"""add dmarcbis aggregate fields + +Revision ID: e2f3a4b5c6d7 +Revises: d1e2f3a4b5c6 +Create Date: 2026-05-23 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e2f3a4b5c6d7" +down_revision: Union[str, Sequence[str], None] = "d1e2f3a4b5c6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Persist optional RFC 9990 / DMARCbis aggregate metadata.""" + op.add_column("dmarc_reports", sa.Column("extra_contact_info", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("generator", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("report_errors", sa.Text(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("non_subdomain_policy", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("failure_options", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("testing", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("discovery_method", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("schema_version", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("report_variant", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("xml_namespace", sa.String(), nullable=True)) + op.add_column("dmarc_reports", sa.Column("report_extensions", sa.Text(), nullable=True)) + op.add_column("report_records", sa.Column("envelope_to", sa.String(), nullable=True)) + op.add_column("report_records", sa.Column("policy_override_reasons", sa.Text(), nullable=True)) + op.add_column("report_records", sa.Column("record_extensions", sa.Text(), nullable=True)) + + +def downgrade() -> None: + """Remove optional RFC 9990 / DMARCbis aggregate metadata.""" + op.drop_column("report_records", "record_extensions") + op.drop_column("report_records", "policy_override_reasons") + op.drop_column("report_records", "envelope_to") + op.drop_column("dmarc_reports", "report_extensions") + op.drop_column("dmarc_reports", "xml_namespace") + op.drop_column("dmarc_reports", "report_variant") + op.drop_column("dmarc_reports", "schema_version") + op.drop_column("dmarc_reports", "discovery_method") + op.drop_column("dmarc_reports", "testing") + op.drop_column("dmarc_reports", "failure_options") + op.drop_column("dmarc_reports", "non_subdomain_policy") + op.drop_column("dmarc_reports", "report_errors") + op.drop_column("dmarc_reports", "generator") + op.drop_column("dmarc_reports", "extra_contact_info") diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py index 5c075f6..f4bf63b 100644 --- a/backend/app/api/api_v1/endpoints/domains.py +++ b/backend/app/api/api_v1/endpoints/domains.py @@ -357,11 +357,15 @@ def _domain_exists(db: Session, store: ReportStore, domain_name: str) -> bool: ) -def _record_evidence(label: str, value: Optional[str], href: str = "#dns-records") -> DNSHealthEvidence: +def _record_evidence( + label: str, value: Optional[str], href: str = "#dns-records" +) -> DNSHealthEvidence: return DNSHealthEvidence(label=label, value=value or "Not found", href=href) -def _summary_evidence(label: str, value: object, href: str = "#compliance-chart") -> DNSHealthEvidence: +def _summary_evidence( + label: str, value: object, href: str = "#compliance-chart" +) -> DNSHealthEvidence: return DNSHealthEvidence(label=label, value=str(value), href=href) @@ -549,8 +553,7 @@ async def read_domains(db: Session = Depends(get_db)): domains = _domain_names_for_summary(db, store) summaries = store.get_all_domain_summaries() stored = { - domain.name: domain - for domain in db.query(Domain).filter(Domain.name.in_(domains)).all() + domain.name: domain for domain in db.query(Domain).filter(Domain.name.in_(domains)).all() } result = [] @@ -582,9 +585,7 @@ async def create_domain( ): """Create a monitored domain before any DMARC reports have arrived.""" name = _normalize_domain_name(payload.name) - validation = validate_domain_config( - {"name": name, "description": payload.description or ""} - ) + validation = validate_domain_config({"name": name, "description": payload.description or ""}) if not validation["valid"]: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -818,7 +819,9 @@ async def get_domain_dns_health( recommendations.append(_enforcement_recommendation(policy, summary)) failed_checks = sum(1 for check in checks if check.status == "fail") - health_status = "healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical" + health_status = ( + "healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical" + ) return DNSHealthResponse( status=health_status, policy=policy, @@ -999,6 +1002,16 @@ async def export_domain_reports( "failed", "pass_rate", "policy", + "subdomain_policy", + "non_subdomain_policy", + "adkim", + "aspf", + "failure_options", + "testing", + "discovery_method", + "schema_version", + "report_variant", + "generator", ] ) @@ -1008,6 +1021,7 @@ async def export_domain_reports( passed = int(summary.get("passed_count", report.get("passed_count", 0)) or 0) failed = int(summary.get("failed_count", report.get("failed_count", 0)) or 0) policy = report.get("policy", "none") + policy_parts = policy if isinstance(policy, dict) else {} if isinstance(policy, dict): policy = policy.get("p", "none") writer.writerow( @@ -1022,6 +1036,16 @@ async def export_domain_reports( failed, report.get("pass_rate", 0.0), policy, + policy_parts.get("sp", ""), + policy_parts.get("np", ""), + policy_parts.get("adkim", ""), + policy_parts.get("aspf", ""), + policy_parts.get("fo", ""), + policy_parts.get("testing", ""), + policy_parts.get("discovery_method", ""), + report.get("schema_version", ""), + report.get("variant", ""), + report.get("generator", ""), ] ) diff --git a/backend/app/models/report.py b/backend/app/models/report.py index addab6f..b1db09c 100644 --- a/backend/app/models/report.py +++ b/backend/app/models/report.py @@ -20,15 +20,26 @@ class DMARCReport(Base): begin_date = Column(Integer, nullable=False, index=True) # Unix timestamp end_date = Column(Integer, nullable=False, index=True) # Unix timestamp source_email = Column(String, nullable=True) + extra_contact_info = Column(String, nullable=True) + generator = Column(String, nullable=True) + report_errors = Column(Text, nullable=True) # Policy information policy = Column(String, nullable=True) # none, quarantine, reject (indexed via __table_args__) subdomain_policy = Column(String, nullable=True) + non_subdomain_policy = Column(String, nullable=True) adkim = Column(String(1), nullable=True) # r (relaxed) or s (strict) aspf = Column(String(1), nullable=True) # r (relaxed) or s (strict) percentage = Column(Integer, nullable=True) + failure_options = Column(String, nullable=True) + testing = Column(String, nullable=True) + discovery_method = Column(String, nullable=True) # Processing metadata + schema_version = Column(String, nullable=True) + report_variant = Column(String, nullable=True) + xml_namespace = Column(String, nullable=True) + report_extensions = Column(Text, nullable=True) processed_at = Column(DateTime, default=datetime.utcnow) raw_data = Column(Text, nullable=True) # Original XML content (optional) @@ -72,10 +83,13 @@ class ReportRecord(Base): # Identifiers header_from = Column(String, nullable=True, index=True) envelope_from = Column(String, nullable=True) + envelope_to = Column(String, nullable=True) # Authentication details (optional JSON fields) dkim_auth_details = Column(Text, nullable=True) # JSON array of DKIM results spf_auth_details = Column(Text, nullable=True) # JSON array of SPF results + policy_override_reasons = Column(Text, nullable=True) # JSON array of policy reasons + record_extensions = Column(Text, nullable=True) # JSON object of extension values # Relationships report = relationship("DMARCReport", back_populates="records") diff --git a/backend/app/services/report_persistence.py b/backend/app/services/report_persistence.py index d386c83..5bca2f2 100644 --- a/backend/app/services/report_persistence.py +++ b/backend/app/services/report_persistence.py @@ -41,6 +41,22 @@ def _loads_json_list(value: Optional[str]) -> Optional[List[Dict[str, Any]]]: return decoded if isinstance(decoded, list) else None +def _loads_json_dict(value: Optional[str]) -> Optional[Dict[str, Any]]: + if not value: + return None + try: + decoded = json.loads(value) + except (json.JSONDecodeError, TypeError): + return None + return decoded if isinstance(decoded, dict) else None + + +def _json_or_none(value: Any) -> Optional[str]: + if value in (None, "", [], {}): + return None + return json.dumps(value, sort_keys=True) + + def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]: policy = report.get("policy") or {} if isinstance(policy, str): @@ -51,8 +67,12 @@ def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]: "p": policy.get("p", "none"), "sp": policy.get("sp", ""), "pct": str(policy.get("pct", "100")), + "np": policy.get("np"), + "fo": policy.get("fo"), "adkim": policy.get("adkim") or report.get("adkim"), "aspf": policy.get("aspf") or report.get("aspf"), + "testing": policy.get("testing"), + "discovery_method": policy.get("discovery_method"), } @@ -106,11 +126,22 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport begin_date=begin_ts, end_date=end_ts, source_email=report.get("email") or report.get("source_email"), + extra_contact_info=report.get("extra_contact_info") or None, + generator=report.get("generator") or None, + report_errors=_json_or_none(report.get("errors")), policy=policy["p"], subdomain_policy=policy.get("sp") or None, + non_subdomain_policy=policy.get("np") or None, adkim=policy.get("adkim") or None, aspf=policy.get("aspf") or None, percentage=pct, + failure_options=policy.get("fo") or None, + testing=policy.get("testing") or None, + discovery_method=policy.get("discovery_method") or None, + schema_version=report.get("schema_version") or None, + report_variant=report.get("variant") or None, + xml_namespace=report.get("xml_namespace") or None, + report_extensions=_json_or_none(report.get("extensions")), ) db.add(db_report) db.flush() @@ -126,12 +157,15 @@ def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport spf=record.get("spf_result") or record.get("spf") or "unknown", header_from=record.get("header_from"), envelope_from=record.get("envelope_from"), + envelope_to=record.get("envelope_to"), dkim_auth_details=( json.dumps(record.get("dkim")) if isinstance(record.get("dkim"), list) else None ), spf_auth_details=( json.dumps(record.get("spf")) if isinstance(record.get("spf"), list) else None ), + policy_override_reasons=_json_or_none(record.get("policy_override_reasons")), + record_extensions=_json_or_none(record.get("extensions")), ) ) @@ -160,8 +194,12 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]: "dkim_result": dkim_result, "spf_result": spf_result, "header_from": record.header_from or "", + "envelope_from": record.envelope_from or "", + "envelope_to": record.envelope_to or "", "dkim": _loads_json_list(record.dkim_auth_details) or [], "spf": _loads_json_list(record.spf_auth_details) or [], + "policy_override_reasons": (_loads_json_list(record.policy_override_reasons) or []), + "extensions": _loads_json_dict(record.record_extensions) or {}, } ) @@ -172,6 +210,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]: "report_id": report.report_id, "org_name": report.org_name, "email": report.source_email or "", + "extra_contact_info": report.extra_contact_info or "", + "generator": report.generator or "", + "errors": _loads_json_list(report.report_errors) or [], + "variant": report.report_variant or "", + "schema_version": report.schema_version or "", + "xml_namespace": report.xml_namespace or "", + "extensions": _loads_json_dict(report.report_extensions) or {}, "begin_date": _iso_from_timestamp(report.begin_date), "end_date": _iso_from_timestamp(report.end_date), "begin_timestamp": report.begin_date, @@ -179,7 +224,13 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]: "policy": { "p": report.policy or "none", "sp": report.subdomain_policy or "", + "np": report.non_subdomain_policy or "", "pct": str(report.percentage or 100), + "fo": report.failure_options or "", + "adkim": report.adkim or "", + "aspf": report.aspf or "", + "testing": report.testing or "", + "discovery_method": report.discovery_method or "", }, "records": records, "summary": { diff --git a/backend/app/tests/test_domain_detail_endpoints.py b/backend/app/tests/test_domain_detail_endpoints.py index 516da64..a97585a 100644 --- a/backend/app/tests/test_domain_detail_endpoints.py +++ b/backend/app/tests/test_domain_detail_endpoints.py @@ -33,7 +33,20 @@ REPORT_DICT_POLICY = { "end_date": "2020-08-15T23:59:59", "begin_timestamp": 1597449600, "end_timestamp": 1597535999, - "policy": {"p": "reject", "sp": "reject", "pct": "100"}, + "policy": { + "p": "reject", + "sp": "reject", + "pct": "100", + "np": "none", + "adkim": "s", + "aspf": "r", + "fo": "1", + "testing": "y", + "discovery_method": "treewalk", + }, + "schema_version": "1.0", + "variant": "rfc9990", + "generator": "ExampleRUA 2.0", "records": [ { "source_ip": "209.85.220.1", @@ -139,6 +152,15 @@ def test_export_domain_reports_returns_csv(seeded_client: TestClient): assert rows[0]["passed"] == "10" assert rows[0]["failed"] == "0" assert rows[0]["policy"] == "reject" + assert rows[0]["non_subdomain_policy"] == "none" + assert rows[0]["adkim"] == "s" + assert rows[0]["aspf"] == "r" + assert rows[0]["failure_options"] == "1" + assert rows[0]["testing"] == "y" + assert rows[0]["discovery_method"] == "treewalk" + assert rows[0]["schema_version"] == "1.0" + assert rows[0]["report_variant"] == "rfc9990" + assert rows[0]["generator"] == "ExampleRUA 2.0" def test_export_domain_reports_filters_by_date_range(client: TestClient): @@ -243,7 +265,9 @@ def test_get_domain_sources_returns_rollup_counts(client: TestClient): assert source["disposition_counts"] == {"none": 4, "quarantine": 6} -def test_get_domain_sources_returns_recommendations(client: TestClient, monkeypatch: pytest.MonkeyPatch): +def test_get_domain_sources_returns_recommendations( + client: TestClient, monkeypatch: pytest.MonkeyPatch +): """Endpoint includes actionable guidance for common failure patterns.""" async def fake_ptr_lookup(_provider, _ip, timeout=3.0): # pylint: disable=unused-argument @@ -339,8 +363,10 @@ def test_source_recommendations_cover_common_cases(): ] for source, hostname, spf_fix_hint, expected_types in cases: - recommendations = domains_endpoint._source_recommendations( # pylint: disable=protected-access - source["source_ip"], source, hostname, spf_fix_hint + recommendations = ( + domains_endpoint._source_recommendations( # pylint: disable=protected-access + source["source_ip"], source, hostname, spf_fix_hint + ) ) assert {item.type for item in recommendations} == expected_types diff --git a/backend/app/tests/test_reports_api.py b/backend/app/tests/test_reports_api.py index 10adeb5..1e9889e 100644 --- a/backend/app/tests/test_reports_api.py +++ b/backend/app/tests/test_reports_api.py @@ -3,11 +3,81 @@ import zipfile from fastapi.testclient import TestClient +from app.models.domain import Domain from app.models.report import DMARCReport, ReportRecord +from app.services.report_persistence import persisted_report_to_dict from app.services.report_store import ReportStore from app.tests.test_data import SAMPLE_XML +SAMPLE_RFC9990_XML = """ + + 1.0 + + Example Receiver + dmarc@example.test + https://example.test/dmarc + 2026-05-23-example.org + + 1779494400 + 1779580799 + + Multiple records ignored. + ExampleRUA 2.0 + + + example.org + treewalk +

quarantine

+ reject + none + 1 + s + r + y +
+ + mx1.example.test + + + + 2001:db8::1 + 5 + + quarantine + fail + pass + + local_policy + trusted relay + + + + + news.example.org + bounce.example.org + customer.example.net + + + + example.net + selector1 + fail + body hash did not verify + + + bounce.example.org + mfrom + pass + sender authorized + + + mail-platform + +
+""" + + def _make_zip(xml_content: str) -> bytes: """Create a ZIP file containing the given XML content.""" buf = io.BytesIO() @@ -44,6 +114,98 @@ def test_upload_persists_report_rows(client: TestClient, db_session): assert db_session.query(ReportRecord).filter_by(report_id=report.id).count() == 1 +def test_upload_persists_rfc9990_optional_fields(client: TestClient, db_session): + """RFC 9990 / DMARCbis metadata is kept for exports and future UI use.""" + zip_bytes = _make_zip(SAMPLE_RFC9990_XML) + response = client.post( + "/api/v1/reports/upload", + files={"file": ("report.zip", zip_bytes, "application/zip")}, + ) + assert response.status_code == 200 + + report = db_session.query(DMARCReport).filter_by(report_id="2026-05-23-example.org").one() + assert report.domain.name == "example.org" + assert report.extra_contact_info == "https://example.test/dmarc" + assert report.generator == "ExampleRUA 2.0" + assert report.report_variant == "rfc9990" + assert report.schema_version == "1.0" + assert report.xml_namespace == "urn:ietf:params:xml:ns:dmarc-2.0" + assert report.non_subdomain_policy == "none" + assert report.failure_options == "1" + assert report.testing == "y" + assert report.discovery_method == "treewalk" + assert "Multiple records ignored." in report.report_errors + assert "mx1.example.test" in report.report_extensions + + record = db_session.query(ReportRecord).filter_by(report_id=report.id).one() + assert record.envelope_from == "bounce.example.org" + assert record.envelope_to == "customer.example.net" + assert "local_policy" in record.policy_override_reasons + assert "mail-platform" in record.record_extensions + assert "human_result" in record.dkim_auth_details + assert "scope" in record.spf_auth_details + + +def test_persisted_report_to_dict_hydrates_optional_json_metadata(db_session): + """Persisted RFC 9990 extension metadata is restored safely for readers.""" + domain = Domain(name="metadata.example", dmarc_policy="reject") + report = DMARCReport( + domain=domain, + report_id="metadata-report", + org_name="Example Receiver", + begin_date=1779494400, + end_date=1779580799, + source_email="dmarc@example.test", + report_errors='["Multiple records ignored."]', + policy="reject", + subdomain_policy="quarantine", + non_subdomain_policy="none", + adkim="s", + aspf="r", + percentage=100, + failure_options="1", + testing="y", + discovery_method="treewalk", + schema_version="1.0", + report_variant="rfc9990", + xml_namespace="urn:ietf:params:xml:ns:dmarc-2.0", + report_extensions='{"vendor:receiver": "mx1.example.test"}', + ) + report.records.append( + ReportRecord( + source_ip="2001:db8::1", + count=5, + disposition="quarantine", + dkim="fail", + spf="pass", + header_from="news.metadata.example", + envelope_from="bounce.metadata.example", + envelope_to="customer.example.net", + policy_override_reasons='[{"type": "local_policy"}]', + record_extensions='{"vendor:source": "mail-platform"}', + ) + ) + db_session.add(report) + db_session.flush() + + hydrated = persisted_report_to_dict(report) + + assert hydrated["errors"] == ["Multiple records ignored."] + assert hydrated["extensions"] == {"vendor:receiver": "mx1.example.test"} + assert hydrated["policy"]["np"] == "none" + assert hydrated["policy"]["fo"] == "1" + assert hydrated["records"][0]["envelope_to"] == "customer.example.net" + assert hydrated["records"][0]["extensions"] == {"vendor:source": "mail-platform"} + + report.report_extensions = "{not-json" + report.records[0].record_extensions = "{not-json" + + hydrated_with_bad_json = persisted_report_to_dict(report) + + assert hydrated_with_bad_json["extensions"] == {} + assert hydrated_with_bad_json["records"][0]["extensions"] == {} + + def test_report_reads_hydrate_from_persisted_rows(client: TestClient): """Report read APIs rebuild the in-memory projection from the database.""" zip_bytes = _make_zip(SAMPLE_XML) diff --git a/docs/milestones.md b/docs/milestones.md index a717df0..860a709 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -189,10 +189,10 @@ Goal: keep DMARQ compatible with evolving DMARC report formats and nomenclature Delivered: - Add parser compatibility for RFC 9990-style aggregate report namespaces, version detection, policy metadata, identifiers, override reasons, auth-result details, and namespaced extensions. - Keep legacy RFC 7489-style reports backward compatible through fixture coverage. +- Persist and CSV-export newly introduced aggregate metadata with nullable, backward-safe database fields. Planned: -- Store newly introduced fields with safe defaults. -- Update CSV export and domain/source reporting to include new metadata where it improves operator actionability. +- Update domain/source reporting to include new metadata where it improves operator actionability. - Add fixture-driven tests for representative real-world DMARCbis-style reports. - Update documentation to clarify supported formats and terminology.