From 14f9f6defdd4d0d80746b6057d0714b659e75625 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 22 May 2026 20:35:56 +0200 Subject: [PATCH] feat: add source pass fail rollups --- backend/app/api/api_v1/endpoints/domains.py | 28 +++- backend/app/services/report_persistence.py | 4 +- backend/app/services/report_store.py | 83 ++++++++++- backend/app/templates/domain_details.html | 36 ++++- backend/app/tests/test_dns_endpoints.py | 32 +++++ .../app/tests/test_domain_detail_endpoints.py | 45 ++++++ backend/app/tests/test_report_store.py | 77 +++++++++- backend/app/tests/test_stats_summarizer.py | 83 ++++++++++- backend/app/utils/stats_summarizer.py | 133 +++++++++++++++--- docs/development/roadmap.md | 31 ++-- docs/milestones.md | 14 +- docs/todo.md | 2 +- 12 files changed, 503 insertions(+), 65 deletions(-) diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py index eb1488a..5fa0453 100644 --- a/backend/app/api/api_v1/endpoints/domains.py +++ b/backend/app/api/api_v1/endpoints/domains.py @@ -91,6 +91,13 @@ class SourceEntry(BaseModel): dkim: str dmarc: str disposition: str + spf_pass_count: int = 0 + spf_fail_count: int = 0 + dkim_pass_count: int = 0 + dkim_fail_count: int = 0 + dmarc_pass_count: int = 0 + dmarc_fail_count: int = 0 + disposition_counts: Dict[str, int] = Field(default_factory=dict) hostname: Optional[str] = None spf_fix_hint: Optional[str] = None @@ -135,7 +142,10 @@ def _get_selectors_from_reports(store: "ReportStore", domain: str) -> List[str]: selectors: List[str] = [] for report in store.get_domain_reports(domain): for record in report.get("records", []): - for dkim_entry in record.get("dkim", []): + dkim_entries = record.get("dkim") or [] + for dkim_entry in dkim_entries: + if not isinstance(dkim_entry, dict): + continue sel = dkim_entry.get("selector", "").strip() if sel and sel not in selectors: selectors.append(sel) @@ -470,12 +480,12 @@ def _build_compliance_timeline(store: ReportStore, domain: str) -> List[Timeline return timeline -def _spf_fix_hint(ip: str, spf_result: str) -> Optional[str]: +def _spf_fix_hint(ip: str, spf_result: str, failed_count: int = 0) -> Optional[str]: """Return a copy-paste SPF mechanism (e.g. ``ip4:1.2.3.4``) for a failing IP. Returns ``None`` when SPF did not fail or when *ip* is not a valid address. """ - if spf_result != "fail": + if spf_result != "fail" and failed_count <= 0: return None try: addr = ipaddress.ip_address(ip) @@ -534,10 +544,18 @@ async def get_domain_sources( count=source.get("count", 0), spf=spf_result, dkim=dkim_result, - dmarc=("pass" if spf_result == "pass" or dkim_result == "pass" else "fail"), + dmarc=source.get("dmarc_result") + or ("pass" if spf_result == "pass" or dkim_result == "pass" else "fail"), disposition=source.get("disposition", "none"), + spf_pass_count=source.get("spf_pass_count", 0), + spf_fail_count=source.get("spf_fail_count", 0), + dkim_pass_count=source.get("dkim_pass_count", 0), + dkim_fail_count=source.get("dkim_fail_count", 0), + dmarc_pass_count=source.get("dmarc_pass_count", 0), + dmarc_fail_count=source.get("dmarc_fail_count", 0), + disposition_counts=source.get("disposition_counts", {}), hostname=hostname, - spf_fix_hint=_spf_fix_hint(ip, spf_result), + spf_fix_hint=_spf_fix_hint(ip, spf_result, source.get("spf_fail_count", 0)), ) ) diff --git a/backend/app/services/report_persistence.py b/backend/app/services/report_persistence.py index 6a6099e..2b5769a 100644 --- a/backend/app/services/report_persistence.py +++ b/backend/app/services/report_persistence.py @@ -160,8 +160,8 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]: "dkim_result": dkim_result, "spf_result": spf_result, "header_from": record.header_from or "", - "dkim": _loads_json_list(record.dkim_auth_details), - "spf": _loads_json_list(record.spf_auth_details), + "dkim": _loads_json_list(record.dkim_auth_details) or [], + "spf": _loads_json_list(record.spf_auth_details) or [], } ) diff --git a/backend/app/services/report_store.py b/backend/app/services/report_store.py index d6a13e0..06704fe 100644 --- a/backend/app/services/report_store.py +++ b/backend/app/services/report_store.py @@ -2,6 +2,26 @@ import threading from typing import Any, Dict, List, Optional +def _auth_status_from_counts(pass_count: int, fail_count: int, unknown_count: int = 0) -> str: + """Return a compact status label for aggregated authentication results.""" + if pass_count > 0 and fail_count > 0: + return "mixed" + if pass_count > 0: + return "pass" + if fail_count > 0: + return "fail" + if unknown_count > 0: + return "unknown" + return "none" + + +def _dominant_result(counts: Dict[str, int], default: str = "none") -> str: + """Return the highest-volume result from a result/count mapping.""" + if not counts: + return default + return max(counts.items(), key=lambda item: item[1])[0] + + class ReportStore: """ In-memory store for DMARC reports @@ -77,14 +97,65 @@ class ReportStore: if source_ip not in sources: sources[source_ip] = { "count": 0, - "spf_result": "unknown", - "dkim_result": "unknown", + "spf_pass_count": 0, + "spf_fail_count": 0, + "spf_unknown_count": 0, + "dkim_pass_count": 0, + "dkim_fail_count": 0, + "dkim_unknown_count": 0, + "dmarc_pass_count": 0, + "dmarc_fail_count": 0, + "disposition_counts": {}, + "spf_result": "none", + "dkim_result": "none", + "dmarc_result": "none", "disposition": "none", } - sources[source_ip]["count"] += record.get("count", 0) - sources[source_ip]["spf_result"] = record.get("spf_result", "unknown") - sources[source_ip]["dkim_result"] = record.get("dkim_result", "unknown") - sources[source_ip]["disposition"] = record.get("disposition", "none") + count = int(record.get("count") or 0) + spf_result = record.get("spf_result", "unknown") or "unknown" + dkim_result = record.get("dkim_result", "unknown") or "unknown" + disposition = record.get("disposition", "none") or "none" + source = sources[source_ip] + + source["count"] += count + + if spf_result == "pass": + source["spf_pass_count"] += count + elif spf_result == "fail": + source["spf_fail_count"] += count + else: + source["spf_unknown_count"] += count + + if dkim_result == "pass": + source["dkim_pass_count"] += count + elif dkim_result == "fail": + source["dkim_fail_count"] += count + else: + source["dkim_unknown_count"] += count + + if spf_result == "pass" or dkim_result == "pass": + source["dmarc_pass_count"] += count + else: + source["dmarc_fail_count"] += count + + disposition_counts = source["disposition_counts"] + disposition_counts[disposition] = disposition_counts.get(disposition, 0) + count + + source["spf_result"] = _auth_status_from_counts( + source["spf_pass_count"], + source["spf_fail_count"], + source["spf_unknown_count"], + ) + source["dkim_result"] = _auth_status_from_counts( + source["dkim_pass_count"], + source["dkim_fail_count"], + source["dkim_unknown_count"], + ) + source["dmarc_result"] = _auth_status_from_counts( + source["dmarc_pass_count"], + source["dmarc_fail_count"], + ) + source["disposition"] = _dominant_result(disposition_counts) total = summary["total_count"] summary["compliance_rate"] = ( diff --git a/backend/app/templates/domain_details.html b/backend/app/templates/domain_details.html index 41aef8e..5eb73c9 100644 --- a/backend/app/templates/domain_details.html +++ b/backend/app/templates/domain_details.html @@ -287,11 +287,21 @@ Fail -