gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: persist DMARC aggregate metadata

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-05-23 14:11:36 +02:00
parent 0200a13069
commit 9d0b12efea
7 changed files with 345 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/tests/test_domain_detail_endpoints.py
+30 -4
View File
@@ -33,7 +33,20 @@ REPORT_DICT_POLICY = {
"end_date": "2020-08-15T23:59:59",
"begin_timestamp": 1597449600,
"end_timestamp": 1597535999,
"policy": {"p": "reject", "sp": "reject", "pct": "100"},
"policy": {
"p": "reject",
"sp": "reject",
"pct": "100",
"np": "none",
"adkim": "s",
"aspf": "r",
"fo": "1",
"testing": "y",
"discovery_method": "treewalk",
},
"schema_version": "1.0",
"variant": "rfc9990",
"generator": "ExampleRUA 2.0",
"records": [
{
"source_ip": "209.85.220.1",
@@ -139,6 +152,15 @@ def test_export_domain_reports_returns_csv(seeded_client: TestClient):
assert rows[0]["passed"] == "10"
assert rows[0]["failed"] == "0"
assert rows[0]["policy"] == "reject"
assert rows[0]["non_subdomain_policy"] == "none"
assert rows[0]["adkim"] == "s"
assert rows[0]["aspf"] == "r"
assert rows[0]["failure_options"] == "1"
assert rows[0]["testing"] == "y"
assert rows[0]["discovery_method"] == "treewalk"
assert rows[0]["schema_version"] == "1.0"
assert rows[0]["report_variant"] == "rfc9990"
assert rows[0]["generator"] == "ExampleRUA 2.0"
def test_export_domain_reports_filters_by_date_range(client: TestClient):
@@ -243,7 +265,9 @@ def test_get_domain_sources_returns_rollup_counts(client: TestClient):
assert source["disposition_counts"] == {"none": 4, "quarantine": 6}
def test_get_domain_sources_returns_recommendations(client: TestClient, monkeypatch: pytest.MonkeyPatch):
def test_get_domain_sources_returns_recommendations(
client: TestClient, monkeypatch: pytest.MonkeyPatch
):
"""Endpoint includes actionable guidance for common failure patterns."""
async def fake_ptr_lookup(_provider, _ip, timeout=3.0): # pylint: disable=unused-argument
@@ -339,8 +363,10 @@ def test_source_recommendations_cover_common_cases():
]
for source, hostname, spf_fix_hint, expected_types in cases:
recommendations = domains_endpoint._source_recommendations( # pylint: disable=protected-access
source["source_ip"], source, hostname, spf_fix_hint
recommendations = (
domains_endpoint._source_recommendations( # pylint: disable=protected-access
source["source_ip"], source, hostname, spf_fix_hint
)
)
assert {item.type for item in recommendations} == expected_types
backend/app/tests/test_reports_api.py
+162
View File
@@ -3,11 +3,81 @@ import zipfile
from fastapi.testclient import TestClient
from app.models.domain import Domain
from app.models.report import DMARCReport, ReportRecord
from app.services.report_persistence import persisted_report_to_dict
from app.services.report_store import ReportStore
from app.tests.test_data import SAMPLE_XML
SAMPLE_RFC9990_XML = """
<feedback xmlns="urn:ietf:params:xml:ns:dmarc-2.0" xmlns:vendor="https://reports.example.test/dmarc">
<version>1.0</version>
<report_metadata>
<org_name>Example Receiver</org_name>
<email>dmarc@example.test</email>
<extra_contact_info>https://example.test/dmarc</extra_contact_info>
<report_id>2026-05-23-example.org</report_id>
<date_range>
<begin>1779494400</begin>
<end>1779580799</end>
</date_range>
<error>Multiple records ignored.</error>
<generator>ExampleRUA 2.0</generator>
</report_metadata>
<policy_published>
<domain>example.org</domain>
<discovery_method>treewalk</discovery_method>
<p>quarantine</p>
<sp>reject</sp>
<np>none</np>
<fo>1</fo>
<adkim>s</adkim>
<aspf>r</aspf>
<testing>y</testing>
</policy_published>
<extension>
<vendor:receiver>mx1.example.test</vendor:receiver>
</extension>
<record>
<row>
<source_ip>2001:db8::1</source_ip>
<count>5</count>
<policy_evaluated>
<disposition>quarantine</disposition>
<dkim>fail</dkim>
<spf>pass</spf>
<reason>
<type>local_policy</type>
<comment>trusted relay</comment>
</reason>
</policy_evaluated>
</row>
<identifiers>
<header_from>news.example.org</header_from>
<envelope_from>bounce.example.org</envelope_from>
<envelope_to>customer.example.net</envelope_to>
</identifiers>
<auth_results>
<dkim>
<domain>example.net</domain>
<selector>selector1</selector>
<result>fail</result>
<human_result>body hash did not verify</human_result>
</dkim>
<spf>
<domain>bounce.example.org</domain>
<scope>mfrom</scope>
<result>pass</result>
<human_result>sender authorized</human_result>
</spf>
</auth_results>
<vendor:source>mail-platform</vendor:source>
</record>
</feedback>
"""
def _make_zip(xml_content: str) -> bytes:
"""Create a ZIP file containing the given XML content."""
buf = io.BytesIO()
@@ -44,6 +114,98 @@ def test_upload_persists_report_rows(client: TestClient, db_session):
assert db_session.query(ReportRecord).filter_by(report_id=report.id).count() == 1
def test_upload_persists_rfc9990_optional_fields(client: TestClient, db_session):
"""RFC 9990 / DMARCbis metadata is kept for exports and future UI use."""
zip_bytes = _make_zip(SAMPLE_RFC9990_XML)
response = client.post(
"/api/v1/reports/upload",
files={"file": ("report.zip", zip_bytes, "application/zip")},
)
assert response.status_code == 200
report = db_session.query(DMARCReport).filter_by(report_id="2026-05-23-example.org").one()
assert report.domain.name == "example.org"
assert report.extra_contact_info == "https://example.test/dmarc"
assert report.generator == "ExampleRUA 2.0"
assert report.report_variant == "rfc9990"
assert report.schema_version == "1.0"
assert report.xml_namespace == "urn:ietf:params:xml:ns:dmarc-2.0"
assert report.non_subdomain_policy == "none"
assert report.failure_options == "1"
assert report.testing == "y"
assert report.discovery_method == "treewalk"
assert "Multiple records ignored." in report.report_errors
assert "mx1.example.test" in report.report_extensions
record = db_session.query(ReportRecord).filter_by(report_id=report.id).one()
assert record.envelope_from == "bounce.example.org"
assert record.envelope_to == "customer.example.net"
assert "local_policy" in record.policy_override_reasons
assert "mail-platform" in record.record_extensions
assert "human_result" in record.dkim_auth_details
assert "scope" in record.spf_auth_details
def test_persisted_report_to_dict_hydrates_optional_json_metadata(db_session):
"""Persisted RFC 9990 extension metadata is restored safely for readers."""
domain = Domain(name="metadata.example", dmarc_policy="reject")
report = DMARCReport(
domain=domain,
report_id="metadata-report",
org_name="Example Receiver",
begin_date=1779494400,
end_date=1779580799,
source_email="dmarc@example.test",
report_errors='["Multiple records ignored."]',
policy="reject",
subdomain_policy="quarantine",
non_subdomain_policy="none",
adkim="s",
aspf="r",
percentage=100,
failure_options="1",
testing="y",
discovery_method="treewalk",
schema_version="1.0",
report_variant="rfc9990",
xml_namespace="urn:ietf:params:xml:ns:dmarc-2.0",
report_extensions='{"vendor:receiver": "mx1.example.test"}',
)
report.records.append(
ReportRecord(
source_ip="2001:db8::1",
count=5,
disposition="quarantine",
dkim="fail",
spf="pass",
header_from="news.metadata.example",
envelope_from="bounce.metadata.example",
envelope_to="customer.example.net",
policy_override_reasons='[{"type": "local_policy"}]',
record_extensions='{"vendor:source": "mail-platform"}',
)
)
db_session.add(report)
db_session.flush()
hydrated = persisted_report_to_dict(report)
assert hydrated["errors"] == ["Multiple records ignored."]
assert hydrated["extensions"] == {"vendor:receiver": "mx1.example.test"}
assert hydrated["policy"]["np"] == "none"
assert hydrated["policy"]["fo"] == "1"
assert hydrated["records"][0]["envelope_to"] == "customer.example.net"
assert hydrated["records"][0]["extensions"] == {"vendor:source": "mail-platform"}
report.report_extensions = "{not-json"
report.records[0].record_extensions = "{not-json"
hydrated_with_bad_json = persisted_report_to_dict(report)
assert hydrated_with_bad_json["extensions"] == {}
assert hydrated_with_bad_json["records"][0]["extensions"] == {}
def test_report_reads_hydrate_from_persisted_rows(client: TestClient):
"""Report read APIs rebuild the in-memory projection from the database."""
zip_bytes = _make_zip(SAMPLE_XML)