diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py
index eb1488a..5fa0453 100644
--- a/backend/app/api/api_v1/endpoints/domains.py
+++ b/backend/app/api/api_v1/endpoints/domains.py
@@ -91,6 +91,13 @@ class SourceEntry(BaseModel):
dkim: str
dmarc: str
disposition: str
+ spf_pass_count: int = 0
+ spf_fail_count: int = 0
+ dkim_pass_count: int = 0
+ dkim_fail_count: int = 0
+ dmarc_pass_count: int = 0
+ dmarc_fail_count: int = 0
+ disposition_counts: Dict[str, int] = Field(default_factory=dict)
hostname: Optional[str] = None
spf_fix_hint: Optional[str] = None
@@ -135,7 +142,10 @@ def _get_selectors_from_reports(store: "ReportStore", domain: str) -> List[str]:
selectors: List[str] = []
for report in store.get_domain_reports(domain):
for record in report.get("records", []):
- for dkim_entry in record.get("dkim", []):
+ dkim_entries = record.get("dkim") or []
+ for dkim_entry in dkim_entries:
+ if not isinstance(dkim_entry, dict):
+ continue
sel = dkim_entry.get("selector", "").strip()
if sel and sel not in selectors:
selectors.append(sel)
@@ -470,12 +480,12 @@ def _build_compliance_timeline(store: ReportStore, domain: str) -> List[Timeline
return timeline
-def _spf_fix_hint(ip: str, spf_result: str) -> Optional[str]:
+def _spf_fix_hint(ip: str, spf_result: str, failed_count: int = 0) -> Optional[str]:
"""Return a copy-paste SPF mechanism (e.g. ``ip4:1.2.3.4``) for a failing IP.
Returns ``None`` when SPF did not fail or when *ip* is not a valid address.
"""
- if spf_result != "fail":
+ if spf_result != "fail" and failed_count <= 0:
return None
try:
addr = ipaddress.ip_address(ip)
@@ -534,10 +544,18 @@ async def get_domain_sources(
count=source.get("count", 0),
spf=spf_result,
dkim=dkim_result,
- dmarc=("pass" if spf_result == "pass" or dkim_result == "pass" else "fail"),
+ dmarc=source.get("dmarc_result")
+ or ("pass" if spf_result == "pass" or dkim_result == "pass" else "fail"),
disposition=source.get("disposition", "none"),
+ spf_pass_count=source.get("spf_pass_count", 0),
+ spf_fail_count=source.get("spf_fail_count", 0),
+ dkim_pass_count=source.get("dkim_pass_count", 0),
+ dkim_fail_count=source.get("dkim_fail_count", 0),
+ dmarc_pass_count=source.get("dmarc_pass_count", 0),
+ dmarc_fail_count=source.get("dmarc_fail_count", 0),
+ disposition_counts=source.get("disposition_counts", {}),
hostname=hostname,
- spf_fix_hint=_spf_fix_hint(ip, spf_result),
+ spf_fix_hint=_spf_fix_hint(ip, spf_result, source.get("spf_fail_count", 0)),
)
)
diff --git a/backend/app/services/report_persistence.py b/backend/app/services/report_persistence.py
index 6a6099e..2b5769a 100644
--- a/backend/app/services/report_persistence.py
+++ b/backend/app/services/report_persistence.py
@@ -160,8 +160,8 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
"dkim_result": dkim_result,
"spf_result": spf_result,
"header_from": record.header_from or "",
- "dkim": _loads_json_list(record.dkim_auth_details),
- "spf": _loads_json_list(record.spf_auth_details),
+ "dkim": _loads_json_list(record.dkim_auth_details) or [],
+ "spf": _loads_json_list(record.spf_auth_details) or [],
}
)
diff --git a/backend/app/services/report_store.py b/backend/app/services/report_store.py
index d6a13e0..06704fe 100644
--- a/backend/app/services/report_store.py
+++ b/backend/app/services/report_store.py
@@ -2,6 +2,26 @@ import threading
from typing import Any, Dict, List, Optional
+def _auth_status_from_counts(pass_count: int, fail_count: int, unknown_count: int = 0) -> str:
+ """Return a compact status label for aggregated authentication results."""
+ if pass_count > 0 and fail_count > 0:
+ return "mixed"
+ if pass_count > 0:
+ return "pass"
+ if fail_count > 0:
+ return "fail"
+ if unknown_count > 0:
+ return "unknown"
+ return "none"
+
+
+def _dominant_result(counts: Dict[str, int], default: str = "none") -> str:
+ """Return the highest-volume result from a result/count mapping."""
+ if not counts:
+ return default
+ return max(counts.items(), key=lambda item: item[1])[0]
+
+
class ReportStore:
"""
In-memory store for DMARC reports
@@ -77,14 +97,65 @@ class ReportStore:
if source_ip not in sources:
sources[source_ip] = {
"count": 0,
- "spf_result": "unknown",
- "dkim_result": "unknown",
+ "spf_pass_count": 0,
+ "spf_fail_count": 0,
+ "spf_unknown_count": 0,
+ "dkim_pass_count": 0,
+ "dkim_fail_count": 0,
+ "dkim_unknown_count": 0,
+ "dmarc_pass_count": 0,
+ "dmarc_fail_count": 0,
+ "disposition_counts": {},
+ "spf_result": "none",
+ "dkim_result": "none",
+ "dmarc_result": "none",
"disposition": "none",
}
- sources[source_ip]["count"] += record.get("count", 0)
- sources[source_ip]["spf_result"] = record.get("spf_result", "unknown")
- sources[source_ip]["dkim_result"] = record.get("dkim_result", "unknown")
- sources[source_ip]["disposition"] = record.get("disposition", "none")
+ count = int(record.get("count") or 0)
+ spf_result = record.get("spf_result", "unknown") or "unknown"
+ dkim_result = record.get("dkim_result", "unknown") or "unknown"
+ disposition = record.get("disposition", "none") or "none"
+ source = sources[source_ip]
+
+ source["count"] += count
+
+ if spf_result == "pass":
+ source["spf_pass_count"] += count
+ elif spf_result == "fail":
+ source["spf_fail_count"] += count
+ else:
+ source["spf_unknown_count"] += count
+
+ if dkim_result == "pass":
+ source["dkim_pass_count"] += count
+ elif dkim_result == "fail":
+ source["dkim_fail_count"] += count
+ else:
+ source["dkim_unknown_count"] += count
+
+ if spf_result == "pass" or dkim_result == "pass":
+ source["dmarc_pass_count"] += count
+ else:
+ source["dmarc_fail_count"] += count
+
+ disposition_counts = source["disposition_counts"]
+ disposition_counts[disposition] = disposition_counts.get(disposition, 0) + count
+
+ source["spf_result"] = _auth_status_from_counts(
+ source["spf_pass_count"],
+ source["spf_fail_count"],
+ source["spf_unknown_count"],
+ )
+ source["dkim_result"] = _auth_status_from_counts(
+ source["dkim_pass_count"],
+ source["dkim_fail_count"],
+ source["dkim_unknown_count"],
+ )
+ source["dmarc_result"] = _auth_status_from_counts(
+ source["dmarc_pass_count"],
+ source["dmarc_fail_count"],
+ )
+ source["disposition"] = _dominant_result(disposition_counts)
total = summary["total_count"]
summary["compliance_rate"] = (
diff --git a/backend/app/templates/domain_details.html b/backend/app/templates/domain_details.html
index 41aef8e..5eb73c9 100644
--- a/backend/app/templates/domain_details.html
+++ b/backend/app/templates/domain_details.html
@@ -287,11 +287,21 @@
Fail
-
+
+
+ Mixed
+
+
+
+
+
+ /
+
+
{% endcall %}
{% call td() %}
@@ -304,11 +314,21 @@
Fail
-
+
+
+ Mixed
+
+
+
+
+
+ /
+
+
{% endcall %}
{% call td() %}
@@ -321,6 +341,16 @@
Fail
+
+
+ Mixed
+
+
+
+
+ /
+
+
{% endcall %}
{% call td() %}
@@ -742,4 +772,4 @@ function domainDetailsApp(domainId) {
};
}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/backend/app/tests/test_dns_endpoints.py b/backend/app/tests/test_dns_endpoints.py
index aa887d2..e78a5c6 100644
--- a/backend/app/tests/test_dns_endpoints.py
+++ b/backend/app/tests/test_dns_endpoints.py
@@ -14,6 +14,7 @@ import pytest
from fastapi.testclient import TestClient
from app.api.api_v1.endpoints import domains as domains_endpoint
+from app.api.api_v1.endpoints.domains import _spf_fix_hint
from app.models.domain import Domain
from app.services.dns_resolver import DomainDNSResult
from app.services.report_store import ReportStore
@@ -189,6 +190,32 @@ def test_get_selectors_includes_report_selectors(client: TestClient):
assert "google" in data["report_selectors"]
+def test_get_selectors_ignores_missing_dkim_detail_lists(client: TestClient):
+ """Missing or malformed DKIM auth-detail arrays should not break selectors."""
+ ReportStore.get_instance().add_report(
+ {
+ **MINIMAL_REPORT,
+ "report_id": "missing-dkim-details",
+ "records": [
+ {
+ "source_ip": "203.0.113.10",
+ "count": 1,
+ "disposition": "none",
+ "dkim_result": "pass",
+ "spf_result": "pass",
+ "dkim": ["not-a-dict", {"selector": "mail"}],
+ "spf": None,
+ }
+ ],
+ }
+ )
+
+ response = client.get(f"/api/v1/domains/{DOMAIN}/selectors")
+
+ assert response.status_code == 200
+ assert "mail" in response.json()["report_selectors"]
+
+
def test_get_selectors_report_selector_moves_to_manual_when_added(client: TestClient):
"""A selector discovered from reports should appear only in 'selectors' once added manually."""
# Confirm it's in report_selectors before adding
@@ -425,3 +452,8 @@ def test_sources_endpoint_no_fix_hint_when_spf_passes(client: TestClient):
passing = next((s for s in sources if s["ip"] == "1.2.3.4"), None)
if passing is not None:
assert passing["spf_fix_hint"] is None
+
+
+def test_spf_fix_hint_returns_none_for_invalid_ip_with_failures():
+ """Invalid source IP values should not generate SPF snippets."""
+ assert _spf_fix_hint("not-an-ip", "mixed", failed_count=3) is None
diff --git a/backend/app/tests/test_domain_detail_endpoints.py b/backend/app/tests/test_domain_detail_endpoints.py
index 554f856..10529dc 100644
--- a/backend/app/tests/test_domain_detail_endpoints.py
+++ b/backend/app/tests/test_domain_detail_endpoints.py
@@ -132,6 +132,51 @@ def test_get_domain_sources_returns_200(seeded_client: TestClient):
assert source["dmarc"] == "pass"
+def test_get_domain_sources_returns_rollup_counts(client: TestClient):
+ """Endpoint reports pass/fail totals instead of only the latest IP result."""
+ report = {
+ **REPORT_DICT_POLICY,
+ "report_id": "rpt-mixed-source",
+ "records": [
+ {
+ "source_ip": "209.85.220.9",
+ "count": 4,
+ "disposition": "none",
+ "dkim_result": "pass",
+ "spf_result": "fail",
+ "header_from": DOMAIN,
+ },
+ {
+ "source_ip": "209.85.220.9",
+ "count": 6,
+ "disposition": "quarantine",
+ "dkim_result": "fail",
+ "spf_result": "fail",
+ "header_from": DOMAIN,
+ },
+ ],
+ "summary": {"total_count": 10, "passed_count": 4, "failed_count": 6},
+ }
+ ReportStore.get_instance().add_report(report)
+
+ response = client.get(f"/api/v1/domains/{DOMAIN}/sources")
+
+ assert response.status_code == 200
+ source = response.json()["sources"][0]
+ assert source["ip"] == "209.85.220.9"
+ assert source["count"] == 10
+ assert source["spf"] == "fail"
+ assert source["dkim"] == "mixed"
+ assert source["dmarc"] == "mixed"
+ assert source["spf_pass_count"] == 0
+ assert source["spf_fail_count"] == 10
+ assert source["dkim_pass_count"] == 4
+ assert source["dkim_fail_count"] == 6
+ assert source["dmarc_pass_count"] == 4
+ assert source["dmarc_fail_count"] == 6
+ assert source["disposition_counts"] == {"none": 4, "quarantine": 6}
+
+
def test_get_domain_sources_days_param_accepted(seeded_client: TestClient):
"""The 'days' query parameter is accepted without raising a TypeError."""
response = seeded_client.get(f"/api/v1/domains/{DOMAIN}/sources?days=7")
diff --git a/backend/app/tests/test_report_store.py b/backend/app/tests/test_report_store.py
index fb4de17..ac5dfe6 100644
--- a/backend/app/tests/test_report_store.py
+++ b/backend/app/tests/test_report_store.py
@@ -1,4 +1,4 @@
-from app.services.report_store import ReportStore
+from app.services.report_store import ReportStore, _auth_status_from_counts, _dominant_result
def _sample_report(domain: str = "example.com") -> dict:
@@ -91,6 +91,81 @@ class TestReportStore:
assert [source["source_ip"] for source in sources] == ["203.0.113.2", "203.0.113.1"]
assert [source["count"] for source in sources] == [12, 5]
+ def test_get_domain_sources_rolls_up_pass_fail_counts_per_ip(self):
+ store = ReportStore.get_instance()
+ report = _sample_report("test.com")
+ report["records"] = [
+ {
+ "source_ip": "203.0.113.9",
+ "count": 7,
+ "disposition": "none",
+ "dkim_result": "pass",
+ "spf_result": "fail",
+ "header_from": "test.com",
+ },
+ {
+ "source_ip": "203.0.113.9",
+ "count": 3,
+ "disposition": "quarantine",
+ "dkim_result": "fail",
+ "spf_result": "pass",
+ "header_from": "test.com",
+ },
+ {
+ "source_ip": "203.0.113.9",
+ "count": 2,
+ "disposition": "reject",
+ "dkim_result": "fail",
+ "spf_result": "fail",
+ "header_from": "test.com",
+ },
+ ]
+
+ store.add_report(report)
+
+ source = store.get_domain_sources("test.com")[0]
+ assert source["source_ip"] == "203.0.113.9"
+ assert source["count"] == 12
+ assert source["spf_result"] == "mixed"
+ assert source["dkim_result"] == "mixed"
+ assert source["dmarc_result"] == "mixed"
+ assert source["spf_pass_count"] == 3
+ assert source["spf_fail_count"] == 9
+ assert source["dkim_pass_count"] == 7
+ assert source["dkim_fail_count"] == 5
+ assert source["dmarc_pass_count"] == 10
+ assert source["dmarc_fail_count"] == 2
+ assert source["disposition_counts"] == {"none": 7, "quarantine": 3, "reject": 2}
+
+ def test_get_domain_sources_rolls_up_unknown_auth_results(self):
+ store = ReportStore.get_instance()
+ report = _sample_report("test.com")
+ report["records"] = [
+ {
+ "source_ip": "203.0.113.10",
+ "count": 4,
+ "disposition": "none",
+ "dkim_result": "temperror",
+ "spf_result": "neutral",
+ "header_from": "test.com",
+ }
+ ]
+
+ store.add_report(report)
+
+ source = store.get_domain_sources("test.com")[0]
+ assert source["spf_result"] == "unknown"
+ assert source["dkim_result"] == "unknown"
+ assert source["spf_unknown_count"] == 4
+ assert source["dkim_unknown_count"] == 4
+ assert source["dmarc_result"] == "fail"
+
+ def test_dominant_result_returns_default_for_empty_counts(self):
+ assert _dominant_result({}) == "none"
+
+ def test_auth_status_returns_none_without_counts(self):
+ assert _auth_status_from_counts(0, 0) == "none"
+
def test_clear(self):
store = ReportStore.get_instance()
store.add_report(_sample_report("test.com"))
diff --git a/backend/app/tests/test_stats_summarizer.py b/backend/app/tests/test_stats_summarizer.py
index 0bc3fa2..ce866d9 100644
--- a/backend/app/tests/test_stats_summarizer.py
+++ b/backend/app/tests/test_stats_summarizer.py
@@ -13,7 +13,7 @@ import app.models.user # noqa: F401
from app.core.database import Base
from app.models.domain import Domain
from app.models.report import DMARCReport, ReportRecord
-from app.utils.stats_summarizer import StatsSummarizer
+from app.utils.stats_summarizer import StatsSummarizer, _auth_status_from_counts
@pytest.fixture()
@@ -81,6 +81,51 @@ def _seed_domain_and_reports(db, domain_name="example.com"):
return domain
+def _seed_mixed_source_records(db, domain_name="example.com"):
+ """Insert multiple auth outcomes for one source IP."""
+ domain = Domain(name=domain_name)
+ db.add(domain)
+ db.flush()
+
+ report = DMARCReport(
+ domain_id=domain.id,
+ report_id="rpt-mixed",
+ org_name="google.com",
+ begin_date=1597449600,
+ end_date=1597535999,
+ policy="none",
+ )
+ db.add(report)
+ db.flush()
+
+ db.add_all(
+ [
+ ReportRecord(
+ report_id=report.id,
+ source_ip="203.0.113.55",
+ count=8,
+ disposition="none",
+ dkim="pass",
+ spf="fail",
+ ),
+ ReportRecord(
+ report_id=report.id,
+ source_ip="203.0.113.55",
+ count=2,
+ disposition="reject",
+ dkim="fail",
+ spf="fail",
+ ),
+ ]
+ )
+ db.flush()
+ return domain
+
+
+def test_auth_status_from_counts_returns_none_without_results():
+ assert _auth_status_from_counts(0, 0) == "none"
+
+
class TestStatsSummarizerGlobal:
"""Tests for global statistics."""
@@ -114,6 +159,24 @@ class TestStatsSummarizerGlobal:
assert stats["top_sources"][0]["ip"] == "203.0.113.1"
assert stats["top_sources"][0]["count"] == 5
+ def test_global_top_sources_include_pass_fail_rollups(self, db_session, summarizer):
+ _seed_mixed_source_records(db_session)
+ db_session.commit()
+
+ stats = summarizer.calculate_summary_statistics(db_session)
+ source = stats["top_sources"][0]
+ assert source["ip"] == "203.0.113.55"
+ assert source["count"] == 10
+ assert source["spf_pass_count"] == 0
+ assert source["spf_fail_count"] == 10
+ assert source["dkim_pass_count"] == 8
+ assert source["dkim_fail_count"] == 2
+ assert source["dmarc_pass_count"] == 8
+ assert source["dmarc_fail_count"] == 2
+ assert source["spf"] == "fail"
+ assert source["dkim"] == "mixed"
+ assert source["dmarc"] == "mixed"
+
def test_multiple_domains(self, db_session, summarizer):
_seed_domain_and_reports(db_session, "example.com")
_seed_domain_and_reports(db_session, "test.org")
@@ -155,6 +218,24 @@ class TestStatsSummarizerDomain:
assert stats["sources"][0]["ip"] == "203.0.113.1"
assert stats["sources"][0]["count"] == 5
+ def test_domain_sources_group_by_ip_with_pass_fail_rollups(self, db_session, summarizer):
+ _seed_mixed_source_records(db_session, "example.com")
+ db_session.commit()
+
+ stats = summarizer.calculate_summary_statistics(db_session, domain_id="example.com")
+ assert len(stats["sources"]) == 1
+ source = stats["sources"][0]
+ assert source["ip"] == "203.0.113.55"
+ assert source["count"] == 10
+ assert source["spf_fail_count"] == 10
+ assert source["dkim_pass_count"] == 8
+ assert source["dkim_fail_count"] == 2
+ assert source["dmarc_pass_count"] == 8
+ assert source["dmarc_fail_count"] == 2
+ assert source["spf"] == "fail"
+ assert source["dkim"] == "mixed"
+ assert source["dmarc"] == "mixed"
+
def test_domain_isolation(self, db_session, summarizer):
"""Stats for one domain should not include data from another."""
_seed_domain_and_reports(db_session, "example.com")
diff --git a/backend/app/utils/stats_summarizer.py b/backend/app/utils/stats_summarizer.py
index 5cbbbb9..8f2f423 100644
--- a/backend/app/utils/stats_summarizer.py
+++ b/backend/app/utils/stats_summarizer.py
@@ -14,6 +14,17 @@ from app.models.report import DMARCReport, ReportRecord
logger = logging.getLogger(__name__)
+def _auth_status_from_counts(pass_count: int, fail_count: int) -> str:
+ """Return pass, fail, mixed, or none from aggregate pass/fail counts."""
+ if pass_count > 0 and fail_count > 0:
+ return "mixed"
+ if pass_count > 0:
+ return "pass"
+ if fail_count > 0:
+ return "fail"
+ return "none"
+
+
class StatsSummarizer:
"""
Utility class for summarizing and caching dashboard statistics
@@ -273,6 +284,27 @@ class StatsSummarizer:
db.query(
ReportRecord.source_ip,
func.sum(ReportRecord.count).label("total_count"),
+ func.sum(case((ReportRecord.spf == "pass", ReportRecord.count), else_=0)).label(
+ "spf_pass_count"
+ ),
+ func.sum(case((ReportRecord.spf == "fail", ReportRecord.count), else_=0)).label(
+ "spf_fail_count"
+ ),
+ func.sum(case((ReportRecord.dkim == "pass", ReportRecord.count), else_=0)).label(
+ "dkim_pass_count"
+ ),
+ func.sum(case((ReportRecord.dkim == "fail", ReportRecord.count), else_=0)).label(
+ "dkim_fail_count"
+ ),
+ func.sum(
+ case(
+ (
+ (ReportRecord.dkim == "pass") | (ReportRecord.spf == "pass"),
+ ReportRecord.count,
+ ),
+ else_=0,
+ )
+ ).label("dmarc_pass_count"),
)
.group_by(ReportRecord.source_ip)
.order_by(func.sum(ReportRecord.count).desc())
@@ -280,33 +312,88 @@ class StatsSummarizer:
.all()
)
- return [{"ip": row.source_ip, "count": int(row.total_count)} for row in results]
-
- def _get_domain_sources(
- self, db: Session, domain_db_id: int, limit: int = 10
- ) -> List[Dict[str, Any]]:
- """Get top sending sources for a specific domain."""
- results = (
- db.query(
- ReportRecord.source_ip,
- func.sum(ReportRecord.count).label("total_count"),
- ReportRecord.spf,
- ReportRecord.dkim,
- )
- .join(DMARCReport, ReportRecord.report_id == DMARCReport.id)
- .filter(DMARCReport.domain_id == domain_db_id)
- .group_by(ReportRecord.source_ip, ReportRecord.spf, ReportRecord.dkim)
- .order_by(func.sum(ReportRecord.count).desc())
- .limit(limit)
- .all()
- )
-
return [
{
"ip": row.source_ip,
"count": int(row.total_count),
- "spf": row.spf or "unknown",
- "dkim": row.dkim or "unknown",
+ "spf_pass_count": int(row.spf_pass_count or 0),
+ "spf_fail_count": int(row.spf_fail_count or 0),
+ "dkim_pass_count": int(row.dkim_pass_count or 0),
+ "dkim_fail_count": int(row.dkim_fail_count or 0),
+ "dmarc_pass_count": int(row.dmarc_pass_count or 0),
+ "dmarc_fail_count": int(row.total_count) - int(row.dmarc_pass_count or 0),
+ "spf": _auth_status_from_counts(
+ int(row.spf_pass_count or 0), int(row.spf_fail_count or 0)
+ ),
+ "dkim": _auth_status_from_counts(
+ int(row.dkim_pass_count or 0), int(row.dkim_fail_count or 0)
+ ),
+ "dmarc": _auth_status_from_counts(
+ int(row.dmarc_pass_count or 0),
+ int(row.total_count) - int(row.dmarc_pass_count or 0),
+ ),
+ }
+ for row in results
+ ]
+
+ def _get_domain_sources(
+ self, db: Session, domain_db_id: int, limit: int = 10
+ ) -> List[Dict[str, Any]]:
+ """Get top sending sources for a specific domain."""
+ results = (
+ db.query(
+ ReportRecord.source_ip,
+ func.sum(ReportRecord.count).label("total_count"),
+ func.sum(case((ReportRecord.spf == "pass", ReportRecord.count), else_=0)).label(
+ "spf_pass_count"
+ ),
+ func.sum(case((ReportRecord.spf == "fail", ReportRecord.count), else_=0)).label(
+ "spf_fail_count"
+ ),
+ func.sum(case((ReportRecord.dkim == "pass", ReportRecord.count), else_=0)).label(
+ "dkim_pass_count"
+ ),
+ func.sum(case((ReportRecord.dkim == "fail", ReportRecord.count), else_=0)).label(
+ "dkim_fail_count"
+ ),
+ func.sum(
+ case(
+ (
+ (ReportRecord.dkim == "pass") | (ReportRecord.spf == "pass"),
+ ReportRecord.count,
+ ),
+ else_=0,
+ )
+ ).label("dmarc_pass_count"),
+ )
+ .join(DMARCReport, ReportRecord.report_id == DMARCReport.id)
+ .filter(DMARCReport.domain_id == domain_db_id)
+ .group_by(ReportRecord.source_ip)
+ .order_by(func.sum(ReportRecord.count).desc())
+ .limit(limit)
+ .all()
+ )
+
+ return [
+ {
+ "ip": row.source_ip,
+ "count": int(row.total_count),
+ "spf_pass_count": int(row.spf_pass_count or 0),
+ "spf_fail_count": int(row.spf_fail_count or 0),
+ "dkim_pass_count": int(row.dkim_pass_count or 0),
+ "dkim_fail_count": int(row.dkim_fail_count or 0),
+ "dmarc_pass_count": int(row.dmarc_pass_count or 0),
+ "dmarc_fail_count": int(row.total_count) - int(row.dmarc_pass_count or 0),
+ "spf": _auth_status_from_counts(
+ int(row.spf_pass_count or 0), int(row.spf_fail_count or 0)
+ ),
+ "dkim": _auth_status_from_counts(
+ int(row.dkim_pass_count or 0), int(row.dkim_fail_count or 0)
+ ),
+ "dmarc": _auth_status_from_counts(
+ int(row.dmarc_pass_count or 0),
+ int(row.total_count) - int(row.dmarc_pass_count or 0),
+ ),
}
for row in results
]
diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md
index 2b402de..faf1271 100644
--- a/docs/development/roadmap.md
+++ b/docs/development/roadmap.md
@@ -31,37 +31,38 @@ Recently improved:
- Import-history rows include sanitized per-attachment outcomes and imported report IDs.
- Mail source backfills can be launched from the UI with configurable search windows.
- The current Alpine-based UI is allowed by CSP and renders dynamic tables in real browsers.
+- Sending-source summaries now retain SPF, DKIM, DMARC, and disposition pass/fail totals per IP instead of showing only the latest result.
Implementation note:
- The legacy `ReportStore` remains as a projection layer for existing report/dashboard code, but durable report data now lives in the database.
-## Active Milestone: Reporting Quality and Import Confidence
-
-Objective: make mailbox imports auditable and make report totals trustworthy.
-
-Priority tasks:
-- Report duplicate skips separately from parse failures.
-- Improve source rollups so a source IP tracks pass/fail counts over time.
-
-Quality bar:
-- Importing the same mailbox twice must not change aggregate totals.
-- Parse failures must be visible and actionable.
-- The user should be able to tell whether a mail source is healthy without reading logs.
-
-## Next Milestone: Meaningful Reports
+## Active Milestone: Meaningful Reports
Objective: turn parsed DMARC data into administrator-friendly reports.
Priority tasks:
- Add time-series charts for volume and compliance.
- Add per-domain daily rollups.
-- Add sender/source breakdowns with SPF, DKIM, and disposition counts.
- Add "what changed" summaries for newly observed senders and sudden compliance drops.
- Add exportable reports for a domain and date range.
+- Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns.
Quality bar:
- A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.
+## Completed Milestone: Reporting Quality and Import Confidence
+
+Objective: make mailbox imports auditable and make report totals trustworthy.
+
+Delivered:
+- Duplicate skips are reported separately from parse failures.
+- Import history exposes per-attachment results and sanitized errors.
+- Individual sources can be manually imported and backfilled from the UI.
+- Source rollups track pass/fail counts over time by sender IP.
+
+Quality bar:
+- Importing the same mailbox twice does not change aggregate totals, parse failures are visible, and source totals are not overwritten by the latest result.
+
## Production Hardening
Objective: make self-hosted deployments safer.
diff --git a/docs/milestones.md b/docs/milestones.md
index c395481..7799ca0 100644
--- a/docs/milestones.md
+++ b/docs/milestones.md
@@ -49,13 +49,13 @@ Delivered:
Implementation note:
- The existing `ReportStore` remains as a compatibility projection for dashboard/report code, but persisted database rows are now the durable source for uploads and mailbox imports.
-## Milestone 4: Reporting Quality and Import Confidence - In Progress
+## Milestone 4: Reporting Quality and Import Confidence - Complete
-Status: In progress
+Status: Complete
Goal: make reports trustworthy for day-to-day administration and make import failures obvious.
-Recently delivered:
+Delivered:
- Gmail import now handles real inbox metadata patterns and Google-style ZIP filenames.
- Gmail/IMAP imports skip duplicate report IDs to avoid inflated totals.
- Tests now cover a real Google-style ZIP attachment path rather than only mocked parser behavior.
@@ -66,16 +66,14 @@ Recently delivered:
- Import history now includes per-attachment details for imported reports, duplicates, parse errors, unsupported attachments, and imported report IDs.
- Mail sources can be backfilled from the UI with 7-day, 30-day, 90-day, or custom search windows.
- The current Alpine-based UI can run under the configured CSP, so dynamic tables render in real browsers.
-
-Next tasks:
-- Improve source aggregation so each sender IP keeps pass/fail totals instead of only the latest result.
+- Source aggregation now keeps per-sender-IP SPF, DKIM, DMARC, and disposition totals instead of overwriting each IP with only the latest result.
Exit criteria:
- A user can connect a mailbox, run a backfill, see exactly what was imported or skipped, and trust that totals are not double-counted.
-## Milestone 5: Dashboard and Meaningful Reports - Next
+## Milestone 5: Dashboard and Meaningful Reports - In Progress
-Status: Planned
+Status: In progress
Goal: convert raw DMARC data into useful operational reporting.
diff --git a/docs/todo.md b/docs/todo.md
index 34326e8..bf765a3 100644
--- a/docs/todo.md
+++ b/docs/todo.md
@@ -149,7 +149,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA
### Meaningful Reports
- [ ] Add per-domain daily rollups
-- [ ] Add sender/source pass/fail totals
+- [x] Add sender/source pass/fail totals
- [ ] Add newly observed source detection
- [ ] Add exportable domain reports
- [ ] Add actionable recommendations for common DMARC failure patterns