From 67e5fedce5df54d92e9715cefa103c34e75f992e Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 22 May 2026 21:40:59 +0200 Subject: [PATCH] feat: add dashboard change summaries --- backend/app/templates/index.html | 75 ++++++++++ backend/app/tests/test_stats_summarizer.py | 163 +++++++++++++++++++++ backend/app/utils/stats_summarizer.py | 118 ++++++++++++++- docs/development/roadmap.md | 6 +- docs/milestones.md | 5 +- docs/todo.md | 2 +- 6 files changed, 361 insertions(+), 8 deletions(-) diff --git a/backend/app/templates/index.html b/backend/app/templates/index.html index a7304e1..407949f 100644 --- a/backend/app/templates/index.html +++ b/backend/app/templates/index.html @@ -116,6 +116,23 @@ {% endcall %} + +
+ {% call card() %} + {% call card_header() %} + {% call card_title() %}What Changed{% endcall %} + {% call card_description() %} + New sending sources and sharp compliance changes from the last 30 days + {% endcall %} + {% endcall %} + {% call card_content() %} +
+ +
+ {% endcall %} + {% endcall %} +
+
{% call card() %} @@ -309,12 +326,14 @@ function dashboardApp() { } else { this.hasDomainData = false; this.clearDashboardCharts(); + this.populateChangeSummary([]); this.populateTopSources([]); } } catch (error) { console.error('Error fetching domain summary:', error); this.hasDomainData = false; this.clearDashboardCharts(); + this.populateChangeSummary([]); this.populateTopSources([]); } }, @@ -329,6 +348,7 @@ function dashboardApp() { const data = await response.json(); this.renderDashboardCharts(data.compliance_trend || []); + this.populateChangeSummary(data.change_summary || []); this.populateTopSources(data.top_sources || []); } catch (error) { console.error('Error fetching dashboard stats:', error); @@ -507,6 +527,61 @@ function dashboardApp() { return new Intl.NumberFormat(undefined, { notation: 'compact' }).format(value); }, + populateChangeSummary(changes) { + const list = document.getElementById('change-summary-list'); + if (!list) return; + + list.textContent = ''; + + if (!changes || !changes.length) { + const empty = document.createElement('p'); + empty.className = 'text-sm text-muted-foreground'; + empty.textContent = 'No major sender or compliance changes detected.'; + list.appendChild(empty); + return; + } + + changes.forEach(change => { + const item = document.createElement('div'); + item.className = 'rounded-md border border-base-300 bg-base-100 p-3'; + + const header = document.createElement('div'); + header.className = 'flex items-start gap-2'; + + const dot = document.createElement('span'); + dot.className = `mt-1 inline-flex h-2.5 w-2.5 flex-none rounded-full ${this.changeSeverityClass(change.severity)}`; + header.appendChild(dot); + + const content = document.createElement('div'); + content.className = 'min-w-0'; + + const title = document.createElement('p'); + title.className = 'text-sm font-semibold'; + title.textContent = change.title || 'Change detected'; + content.appendChild(title); + + const detail = document.createElement('p'); + detail.className = 'mt-1 text-sm text-muted-foreground'; + detail.textContent = change.detail || ''; + content.appendChild(detail); + + const action = document.createElement('p'); + action.className = 'mt-1 text-sm'; + action.textContent = change.action || ''; + content.appendChild(action); + + header.appendChild(content); + item.appendChild(header); + list.appendChild(item); + }); + }, + + changeSeverityClass(severity) { + if (severity === 'error') return 'bg-red-500'; + if (severity === 'warning') return 'bg-yellow-500'; + return 'bg-blue-500'; + }, + populateTopSources(sources) { const tableBody = document.getElementById('top-sources-table-body'); if (!tableBody) return; diff --git a/backend/app/tests/test_stats_summarizer.py b/backend/app/tests/test_stats_summarizer.py index e1cc302..7567d1c 100644 --- a/backend/app/tests/test_stats_summarizer.py +++ b/backend/app/tests/test_stats_summarizer.py @@ -200,6 +200,120 @@ def _seed_recent_trend_records(db, domain_name="example.com"): return domain +def _seed_new_source_records(db, domain_name="example.com"): + """Insert an old source and a current first-seen source.""" + domain = Domain(name=domain_name) + db.add(domain) + db.flush() + + old_report = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-old-source", + org_name="google.com", + begin_date=_timestamp_days_ago(10), + end_date=_timestamp_days_ago(10) + 3600, + policy="none", + ) + current_report = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-new-source", + org_name="google.com", + begin_date=_timestamp_days_ago(1), + end_date=_timestamp_days_ago(1) + 3600, + policy="none", + ) + db.add_all([old_report, current_report]) + db.flush() + + db.add_all( + [ + ReportRecord( + report_id=old_report.id, + source_ip="203.0.113.20", + count=12, + disposition="none", + dkim="pass", + spf="pass", + ), + ReportRecord( + report_id=current_report.id, + source_ip="203.0.113.21", + count=7, + disposition="none", + dkim="pass", + spf="fail", + ), + ] + ) + db.flush() + return domain + + +def _seed_compliance_drop_records(db, domain_name="example.com"): + """Insert a recent compliance drop with a source that is not new.""" + domain = Domain(name=domain_name) + db.add(domain) + db.flush() + + old_report = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-known-source", + org_name="google.com", + begin_date=_timestamp_days_ago(10), + end_date=_timestamp_days_ago(10) + 3600, + policy="none", + ) + previous_report = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-passing-day", + org_name="google.com", + begin_date=_timestamp_days_ago(2), + end_date=_timestamp_days_ago(2) + 3600, + policy="none", + ) + current_report = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-failing-day", + org_name="google.com", + begin_date=_timestamp_days_ago(0), + end_date=_timestamp_days_ago(0) + 3600, + policy="none", + ) + db.add_all([old_report, previous_report, current_report]) + db.flush() + + db.add_all( + [ + ReportRecord( + report_id=old_report.id, + source_ip="203.0.113.30", + count=3, + disposition="none", + dkim="pass", + spf="pass", + ), + ReportRecord( + report_id=previous_report.id, + source_ip="203.0.113.30", + count=10, + disposition="none", + dkim="pass", + spf="pass", + ), + ReportRecord( + report_id=current_report.id, + source_ip="203.0.113.30", + count=10, + disposition="none", + dkim="fail", + spf="fail", + ), + ] + ) + db.flush() + return domain + + def test_auth_status_from_counts_returns_none_without_results(): assert _auth_status_from_counts(0, 0) == "none" @@ -289,6 +403,31 @@ class TestStatsSummarizerGlobal: assert len(stats["compliance_trend"]) == 1 assert stats["compliance_trend"][0]["total"] == 5 + def test_global_change_summary_detects_new_source(self, db_session, summarizer): + _seed_new_source_records(db_session) + db_session.commit() + + stats = summarizer.calculate_summary_statistics(db_session, period_days=7) + new_sources = [item for item in stats["change_summary"] if item["type"] == "new_source"] + + assert len(new_sources) == 1 + assert new_sources[0]["domain"] == "example.com" + assert new_sources[0]["source_ip"] == "203.0.113.21" + assert new_sources[0]["message_count"] == 7 + + def test_global_change_summary_detects_compliance_drop(self, db_session, summarizer): + _seed_compliance_drop_records(db_session) + db_session.commit() + + stats = summarizer.calculate_summary_statistics(db_session, period_days=7) + drops = [item for item in stats["change_summary"] if item["type"] == "compliance_drop"] + + assert len(drops) == 1 + assert drops[0]["previous_rate"] == 100.0 + assert drops[0]["current_rate"] == 0.0 + assert drops[0]["drop"] == 100.0 + assert drops[0]["failed"] == 10 + class TestStatsSummarizerDomain: """Tests for domain-specific statistics.""" @@ -357,6 +496,20 @@ class TestStatsSummarizerDomain: ) assert [point["total"] for point in stats["compliance_trend"]] == [10, 5] + def test_domain_change_summary_isolated_to_domain(self, db_session, summarizer): + _seed_new_source_records(db_session, "example.com") + _seed_new_source_records(db_session, "other.org") + db_session.commit() + + stats = summarizer.calculate_summary_statistics( + db_session, domain_id="example.com", period_days=7 + ) + new_sources = [item for item in stats["change_summary"] if item["type"] == "new_source"] + + assert len(new_sources) == 1 + assert new_sources[0]["domain"] == "example.com" + assert new_sources[0]["source_ip"] == "203.0.113.21" + class TestStatsSummarizerCaching: """Tests for the caching layer.""" @@ -388,3 +541,13 @@ class TestStatsSummarizerCaching: assert len(stats_7_days["compliance_trend"]) == 2 assert len(stats_1_day["compliance_trend"]) == 1 + + def test_old_cache_without_change_summary_is_refreshed(self, db_session, summarizer): + _seed_new_source_records(db_session) + db_session.commit() + summarizer.save_summary({"total_domains": 99}, period_days=7) + + stats = summarizer.calculate_summary_statistics(db_session, period_days=7) + + assert stats["total_domains"] == 1 + assert "change_summary" in stats diff --git a/backend/app/utils/stats_summarizer.py b/backend/app/utils/stats_summarizer.py index 64dc756..a3a7551 100644 --- a/backend/app/utils/stats_summarizer.py +++ b/backend/app/utils/stats_summarizer.py @@ -177,7 +177,7 @@ class StatsSummarizer: # First check if we have cached stats cached_stats = self.get_cached_summary(domain_id, period_days=period_days) - if cached_stats: + if cached_stats and "change_summary" in cached_stats: return cached_stats if domain_id is None: @@ -223,6 +223,9 @@ class StatsSummarizer: # Compliance trend over recent days compliance_trend = self._get_compliance_trend(db, days=period_days) + # Recently changed source and compliance signals + change_summary = self._get_change_summary(db, days=period_days, trend=compliance_trend) + return { "total_domains": total_domains, "total_emails": total_emails, @@ -231,6 +234,7 @@ class StatsSummarizer: "reports_processed": reports_processed, "top_sources": top_sources, "compliance_trend": compliance_trend, + "change_summary": change_summary, } def _calculate_domain_statistics( @@ -248,6 +252,7 @@ class StatsSummarizer: "reports_processed": 0, "sources": [], "compliance_trend": [], + "change_summary": [], } # Aggregate email counts for this domain @@ -285,6 +290,14 @@ class StatsSummarizer: # Compliance trend for this domain compliance_trend = self._get_compliance_trend(db, domain.id, days=period_days) + # Recently changed source and compliance signals + change_summary = self._get_change_summary( + db, + domain.id, + days=period_days, + trend=compliance_trend, + ) + return { "domain": domain_id, "total_emails": total_emails, @@ -293,6 +306,7 @@ class StatsSummarizer: "reports_processed": reports_processed, "sources": sources, "compliance_trend": compliance_trend, + "change_summary": change_summary, } def _get_top_sources(self, db: Session, limit: int = 10) -> List[Dict[str, Any]]: @@ -481,3 +495,105 @@ class StatsSummarizer: ) return trend + + def _get_change_summary( + self, + db: Session, + domain_db_id: Optional[int] = None, + days: int = 30, + trend: Optional[List[Dict[str, Any]]] = None, + limit: int = 5, + ) -> List[Dict[str, Any]]: + """Return notable source and compliance changes for the reporting window.""" + days = max(1, int(days or 30)) + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + cutoff_ts = int(cutoff.timestamp()) + changes: List[Dict[str, Any]] = [] + + current_query = ( + db.query( + Domain.name.label("domain"), + ReportRecord.source_ip.label("source_ip"), + func.sum(ReportRecord.count).label("message_count"), + ) + .join(DMARCReport, ReportRecord.report_id == DMARCReport.id) + .join(Domain, DMARCReport.domain_id == Domain.id) + .filter(DMARCReport.begin_date >= cutoff_ts) + ) + previous_query = ( + db.query(Domain.name.label("domain"), ReportRecord.source_ip.label("source_ip")) + .join(DMARCReport, ReportRecord.report_id == DMARCReport.id) + .join(Domain, DMARCReport.domain_id == Domain.id) + .filter(DMARCReport.begin_date < cutoff_ts) + ) + + if domain_db_id is not None: + current_query = current_query.filter(DMARCReport.domain_id == domain_db_id) + previous_query = previous_query.filter(DMARCReport.domain_id == domain_db_id) + + previous_sources = {(row.domain, row.source_ip) for row in previous_query.distinct().all()} + current_sources = ( + current_query.group_by(Domain.name, ReportRecord.source_ip) + .order_by(func.sum(ReportRecord.count).desc()) + .all() + ) + + for row in current_sources: + source_key = (row.domain, row.source_ip) + if source_key in previous_sources: + continue + changes.append( + { + "type": "new_source", + "severity": "warning", + "title": "New sending source", + "domain": row.domain, + "source_ip": row.source_ip, + "message_count": int(row.message_count or 0), + "detail": ( + f"{row.source_ip} first appeared for {row.domain} in the last " + f"{days} days with {int(row.message_count or 0)} messages." + ), + "action": "Review whether this source is legitimate before changing SPF or DKIM.", + } + ) + if len(changes) >= limit: + break + + compliance_drop = self._build_compliance_drop_change(trend or []) + if compliance_drop: + changes.append(compliance_drop) + + return changes + + @staticmethod + def _build_compliance_drop_change(trend: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Return a change item when the latest compliance point drops sharply.""" + if len(trend) < 2: + return None + + previous = trend[-2] + current = trend[-1] + previous_rate = float(previous.get("compliance_rate", previous.get("rate", 0)) or 0) + current_rate = float(current.get("compliance_rate", current.get("rate", 0)) or 0) + drop = round(previous_rate - current_rate, 1) + failed = int(current.get("failed", 0) or 0) + + if drop < 10 or failed <= 0: + return None + + return { + "type": "compliance_drop", + "severity": "error" if drop >= 25 else "warning", + "title": "Compliance dropped", + "date": current.get("date"), + "previous_rate": previous_rate, + "current_rate": current_rate, + "drop": drop, + "failed": failed, + "detail": ( + f"Compliance fell from {previous_rate}% to {current_rate}% " + f"on {current.get('date')}." + ), + "action": "Review sources from that date and prioritize any new or failing senders.", + } diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index 43ec12a..f4c554e 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -36,19 +36,17 @@ Recently improved: Implementation note: - The legacy `ReportStore` remains as a projection layer for existing report/dashboard code, but durable report data now lives in the database. -## Active Milestone: Meaningful Reports +## Completed Milestone: Meaningful Reports Objective: turn parsed DMARC data into administrator-friendly reports. -Priority tasks: -- Add "what changed" summaries for newly observed senders and sudden compliance drops. - Delivered: - Dashboard time-series charts show daily mail volume, compliance rate, and failure rate. - Top sending sources show DMARC, SPF, and DKIM pass/fail breakdowns on the dashboard. - Per-domain timelines include daily volume, pass, fail, compliance-rate, and failure-rate rollups. - Domain reports can be exported to CSV for a selected date range. - Source reports include actionable recommendations for unknown sources, SPF-only passes, DKIM-only passes, full failures, and unenforced policies. +- What changed summaries identify newly observed senders and sudden compliance drops. Quality bar: - A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next. diff --git a/docs/milestones.md b/docs/milestones.md index d60b6fb..f402c2e 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -71,9 +71,9 @@ Delivered: Exit criteria: - A user can connect a mailbox, run a backfill, see exactly what was imported or skipped, and trust that totals are not double-counted. -## Milestone 5: Dashboard and Meaningful Reports - In Progress +## Milestone 5: Dashboard and Meaningful Reports - Complete -Status: In progress +Status: Complete Goal: convert raw DMARC data into useful operational reporting. @@ -83,6 +83,7 @@ Delivered: - Per-domain report timeline and daily rollups. - Exportable reports for a selected domain and date range. - Clear recommendations for common cases: unknown source, SPF-only pass, DKIM-only pass, full fail, and policy not enforced. +- What changed summaries for newly observed sources and sudden compliance drops. Exit criteria: - A domain owner can answer: who is sending as my domain, what is failing, what changed recently, and what should I fix next? diff --git a/docs/todo.md b/docs/todo.md index 31516ea..ceb7535 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -150,7 +150,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA ### Meaningful Reports - [x] Add per-domain daily rollups - [x] Add sender/source pass/fail totals -- [ ] Add newly observed source detection +- [x] Add newly observed source detection - [x] Add exportable domain reports - [x] Add actionable recommendations for common DMARC failure patterns