Merge pull request #110 from christianlouis/codex/change-summaries

feat: add dashboard change summaries

backend/app/templates/index.html

@@ -116,6 +116,23 @@
         {% endcall %}
     </div>
 
+    <!-- What Changed -->
+    <div x-show="hasDomainData" x-cloak>
+        {% call card() %}
+            {% call card_header() %}
+                {% call card_title() %}What Changed{% endcall %}
+                {% call card_description() %}
+                    New sending sources and sharp compliance changes from the last 30 days
+                {% endcall %}
+            {% endcall %}
+            {% call card_content() %}
+                <div id="change-summary-list" class="space-y-3">
+                    <!-- Change summaries will be populated here via JavaScript -->
+                </div>
+            {% endcall %}
+        {% endcall %}
+    </div>
+
     <!-- Top Sending Sources -->
     <div x-show="hasDomainData" x-cloak>
         {% call card() %}
@@ -309,12 +326,14 @@ function dashboardApp() {
                 } else {
                     this.hasDomainData = false;
                     this.clearDashboardCharts();
+                    this.populateChangeSummary([]);
                     this.populateTopSources([]);
                 }
             } catch (error) {
                 console.error('Error fetching domain summary:', error);
                 this.hasDomainData = false;
                 this.clearDashboardCharts();
+                this.populateChangeSummary([]);
                 this.populateTopSources([]);
             }
         },
@@ -329,6 +348,7 @@ function dashboardApp() {
 
                 const data = await response.json();
                 this.renderDashboardCharts(data.compliance_trend || []);
+                this.populateChangeSummary(data.change_summary || []);
                 this.populateTopSources(data.top_sources || []);
             } catch (error) {
                 console.error('Error fetching dashboard stats:', error);
@@ -507,6 +527,61 @@ function dashboardApp() {
             return new Intl.NumberFormat(undefined, { notation: 'compact' }).format(value);
         },
 
+        populateChangeSummary(changes) {
+            const list = document.getElementById('change-summary-list');
+            if (!list) return;
+
+            list.textContent = '';
+
+            if (!changes || !changes.length) {
+                const empty = document.createElement('p');
+                empty.className = 'text-sm text-muted-foreground';
+                empty.textContent = 'No major sender or compliance changes detected.';
+                list.appendChild(empty);
+                return;
+            }
+
+            changes.forEach(change => {
+                const item = document.createElement('div');
+                item.className = 'rounded-md border border-base-300 bg-base-100 p-3';
+
+                const header = document.createElement('div');
+                header.className = 'flex items-start gap-2';
+
+                const dot = document.createElement('span');
+                dot.className = `mt-1 inline-flex h-2.5 w-2.5 flex-none rounded-full ${this.changeSeverityClass(change.severity)}`;
+                header.appendChild(dot);
+
+                const content = document.createElement('div');
+                content.className = 'min-w-0';
+
+                const title = document.createElement('p');
+                title.className = 'text-sm font-semibold';
+                title.textContent = change.title || 'Change detected';
+                content.appendChild(title);
+
+                const detail = document.createElement('p');
+                detail.className = 'mt-1 text-sm text-muted-foreground';
+                detail.textContent = change.detail || '';
+                content.appendChild(detail);
+
+                const action = document.createElement('p');
+                action.className = 'mt-1 text-sm';
+                action.textContent = change.action || '';
+                content.appendChild(action);
+
+                header.appendChild(content);
+                item.appendChild(header);
+                list.appendChild(item);
+            });
+        },
+
+        changeSeverityClass(severity) {
+            if (severity === 'error') return 'bg-red-500';
+            if (severity === 'warning') return 'bg-yellow-500';
+            return 'bg-blue-500';
+        },
+
         populateTopSources(sources) {
             const tableBody = document.getElementById('top-sources-table-body');
             if (!tableBody) return;

backend/app/tests/test_stats_summarizer.py

@@ -200,6 +200,120 @@ def _seed_recent_trend_records(db, domain_name="example.com"):
     return domain
 
 
+def _seed_new_source_records(db, domain_name="example.com"):
+    """Insert an old source and a current first-seen source."""
+    domain = Domain(name=domain_name)
+    db.add(domain)
+    db.flush()
+
+    old_report = DMARCReport(
+        domain_id=domain.id,
+        report_id=f"{domain_name}-old-source",
+        org_name="google.com",
+        begin_date=_timestamp_days_ago(10),
+        end_date=_timestamp_days_ago(10) + 3600,
+        policy="none",
+    )
+    current_report = DMARCReport(
+        domain_id=domain.id,
+        report_id=f"{domain_name}-new-source",
+        org_name="google.com",
+        begin_date=_timestamp_days_ago(1),
+        end_date=_timestamp_days_ago(1) + 3600,
+        policy="none",
+    )
+    db.add_all([old_report, current_report])
+    db.flush()
+
+    db.add_all(
+        [
+            ReportRecord(
+                report_id=old_report.id,
+                source_ip="203.0.113.20",
+                count=12,
+                disposition="none",
+                dkim="pass",
+                spf="pass",
+            ),
+            ReportRecord(
+                report_id=current_report.id,
+                source_ip="203.0.113.21",
+                count=7,
+                disposition="none",
+                dkim="pass",
+                spf="fail",
+            ),
+        ]
+    )
+    db.flush()
+    return domain
+
+
+def _seed_compliance_drop_records(db, domain_name="example.com"):
+    """Insert a recent compliance drop with a source that is not new."""
+    domain = Domain(name=domain_name)
+    db.add(domain)
+    db.flush()
+
+    old_report = DMARCReport(
+        domain_id=domain.id,
+        report_id=f"{domain_name}-known-source",
+        org_name="google.com",
+        begin_date=_timestamp_days_ago(10),
+        end_date=_timestamp_days_ago(10) + 3600,
+        policy="none",
+    )
+    previous_report = DMARCReport(
+        domain_id=domain.id,
+        report_id=f"{domain_name}-passing-day",
+        org_name="google.com",
+        begin_date=_timestamp_days_ago(2),
+        end_date=_timestamp_days_ago(2) + 3600,
+        policy="none",
+    )
+    current_report = DMARCReport(
+        domain_id=domain.id,
+        report_id=f"{domain_name}-failing-day",
+        org_name="google.com",
+        begin_date=_timestamp_days_ago(0),
+        end_date=_timestamp_days_ago(0) + 3600,
+        policy="none",
+    )
+    db.add_all([old_report, previous_report, current_report])
+    db.flush()
+
+    db.add_all(
+        [
+            ReportRecord(
+                report_id=old_report.id,
+                source_ip="203.0.113.30",
+                count=3,
+                disposition="none",
+                dkim="pass",
+                spf="pass",
+            ),
+            ReportRecord(
+                report_id=previous_report.id,
+                source_ip="203.0.113.30",
+                count=10,
+                disposition="none",
+                dkim="pass",
+                spf="pass",
+            ),
+            ReportRecord(
+                report_id=current_report.id,
+                source_ip="203.0.113.30",
+                count=10,
+                disposition="none",
+                dkim="fail",
+                spf="fail",
+            ),
+        ]
+    )
+    db.flush()
+    return domain
+
+
 def test_auth_status_from_counts_returns_none_without_results():
     assert _auth_status_from_counts(0, 0) == "none"
 
@@ -289,6 +403,31 @@ class TestStatsSummarizerGlobal:
         assert len(stats["compliance_trend"]) == 1
         assert stats["compliance_trend"][0]["total"] == 5
 
+    def test_global_change_summary_detects_new_source(self, db_session, summarizer):
+        _seed_new_source_records(db_session)
+        db_session.commit()
+
+        stats = summarizer.calculate_summary_statistics(db_session, period_days=7)
+        new_sources = [item for item in stats["change_summary"] if item["type"] == "new_source"]
+
+        assert len(new_sources) == 1
+        assert new_sources[0]["domain"] == "example.com"
+        assert new_sources[0]["source_ip"] == "203.0.113.21"
+        assert new_sources[0]["message_count"] == 7
+
+    def test_global_change_summary_detects_compliance_drop(self, db_session, summarizer):
+        _seed_compliance_drop_records(db_session)
+        db_session.commit()
+
+        stats = summarizer.calculate_summary_statistics(db_session, period_days=7)
+        drops = [item for item in stats["change_summary"] if item["type"] == "compliance_drop"]
+
+        assert len(drops) == 1
+        assert drops[0]["previous_rate"] == 100.0
+        assert drops[0]["current_rate"] == 0.0
+        assert drops[0]["drop"] == 100.0
+        assert drops[0]["failed"] == 10
+
 
 class TestStatsSummarizerDomain:
     """Tests for domain-specific statistics."""
@@ -357,6 +496,20 @@ class TestStatsSummarizerDomain:
         )
         assert [point["total"] for point in stats["compliance_trend"]] == [10, 5]
 
+    def test_domain_change_summary_isolated_to_domain(self, db_session, summarizer):
+        _seed_new_source_records(db_session, "example.com")
+        _seed_new_source_records(db_session, "other.org")
+        db_session.commit()
+
+        stats = summarizer.calculate_summary_statistics(
+            db_session, domain_id="example.com", period_days=7
+        )
+        new_sources = [item for item in stats["change_summary"] if item["type"] == "new_source"]
+
+        assert len(new_sources) == 1
+        assert new_sources[0]["domain"] == "example.com"
+        assert new_sources[0]["source_ip"] == "203.0.113.21"
+
 
 class TestStatsSummarizerCaching:
     """Tests for the caching layer."""
@@ -388,3 +541,13 @@ class TestStatsSummarizerCaching:
 
         assert len(stats_7_days["compliance_trend"]) == 2
         assert len(stats_1_day["compliance_trend"]) == 1
+
+    def test_old_cache_without_change_summary_is_refreshed(self, db_session, summarizer):
+        _seed_new_source_records(db_session)
+        db_session.commit()
+        summarizer.save_summary({"total_domains": 99}, period_days=7)
+
+        stats = summarizer.calculate_summary_statistics(db_session, period_days=7)
+
+        assert stats["total_domains"] == 1
+        assert "change_summary" in stats

backend/app/utils/stats_summarizer.py

@@ -177,7 +177,7 @@ class StatsSummarizer:
 
         # First check if we have cached stats
         cached_stats = self.get_cached_summary(domain_id, period_days=period_days)
-        if cached_stats:
+        if cached_stats and "change_summary" in cached_stats:
             return cached_stats
 
         if domain_id is None:
@@ -223,6 +223,9 @@ class StatsSummarizer:
         # Compliance trend over recent days
         compliance_trend = self._get_compliance_trend(db, days=period_days)
 
+        # Recently changed source and compliance signals
+        change_summary = self._get_change_summary(db, days=period_days, trend=compliance_trend)
+
         return {
             "total_domains": total_domains,
             "total_emails": total_emails,
@@ -231,6 +234,7 @@ class StatsSummarizer:
             "reports_processed": reports_processed,
             "top_sources": top_sources,
             "compliance_trend": compliance_trend,
+            "change_summary": change_summary,
         }
 
     def _calculate_domain_statistics(
@@ -248,6 +252,7 @@ class StatsSummarizer:
                 "reports_processed": 0,
                 "sources": [],
                 "compliance_trend": [],
+                "change_summary": [],
             }
 
         # Aggregate email counts for this domain
@@ -285,6 +290,14 @@ class StatsSummarizer:
         # Compliance trend for this domain
         compliance_trend = self._get_compliance_trend(db, domain.id, days=period_days)
 
+        # Recently changed source and compliance signals
+        change_summary = self._get_change_summary(
+            db,
+            domain.id,
+            days=period_days,
+            trend=compliance_trend,
+        )
+
         return {
             "domain": domain_id,
             "total_emails": total_emails,
@@ -293,6 +306,7 @@ class StatsSummarizer:
             "reports_processed": reports_processed,
             "sources": sources,
             "compliance_trend": compliance_trend,
+            "change_summary": change_summary,
         }
 
     def _get_top_sources(self, db: Session, limit: int = 10) -> List[Dict[str, Any]]:
@@ -481,3 +495,105 @@ class StatsSummarizer:
             )
 
         return trend
+
+    def _get_change_summary(
+        self,
+        db: Session,
+        domain_db_id: Optional[int] = None,
+        days: int = 30,
+        trend: Optional[List[Dict[str, Any]]] = None,
+        limit: int = 5,
+    ) -> List[Dict[str, Any]]:
+        """Return notable source and compliance changes for the reporting window."""
+        days = max(1, int(days or 30))
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+        cutoff_ts = int(cutoff.timestamp())
+        changes: List[Dict[str, Any]] = []
+
+        current_query = (
+            db.query(
+                Domain.name.label("domain"),
+                ReportRecord.source_ip.label("source_ip"),
+                func.sum(ReportRecord.count).label("message_count"),
+            )
+            .join(DMARCReport, ReportRecord.report_id == DMARCReport.id)
+            .join(Domain, DMARCReport.domain_id == Domain.id)
+            .filter(DMARCReport.begin_date >= cutoff_ts)
+        )
+        previous_query = (
+            db.query(Domain.name.label("domain"), ReportRecord.source_ip.label("source_ip"))
+            .join(DMARCReport, ReportRecord.report_id == DMARCReport.id)
+            .join(Domain, DMARCReport.domain_id == Domain.id)
+            .filter(DMARCReport.begin_date < cutoff_ts)
+        )
+
+        if domain_db_id is not None:
+            current_query = current_query.filter(DMARCReport.domain_id == domain_db_id)
+            previous_query = previous_query.filter(DMARCReport.domain_id == domain_db_id)
+
+        previous_sources = {(row.domain, row.source_ip) for row in previous_query.distinct().all()}
+        current_sources = (
+            current_query.group_by(Domain.name, ReportRecord.source_ip)
+            .order_by(func.sum(ReportRecord.count).desc())
+            .all()
+        )
+
+        for row in current_sources:
+            source_key = (row.domain, row.source_ip)
+            if source_key in previous_sources:
+                continue
+            changes.append(
+                {
+                    "type": "new_source",
+                    "severity": "warning",
+                    "title": "New sending source",
+                    "domain": row.domain,
+                    "source_ip": row.source_ip,
+                    "message_count": int(row.message_count or 0),
+                    "detail": (
+                        f"{row.source_ip} first appeared for {row.domain} in the last "
+                        f"{days} days with {int(row.message_count or 0)} messages."
+                    ),
+                    "action": "Review whether this source is legitimate before changing SPF or DKIM.",
+                }
+            )
+            if len(changes) >= limit:
+                break
+
+        compliance_drop = self._build_compliance_drop_change(trend or [])
+        if compliance_drop:
+            changes.append(compliance_drop)
+
+        return changes
+
+    @staticmethod
+    def _build_compliance_drop_change(trend: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        """Return a change item when the latest compliance point drops sharply."""
+        if len(trend) < 2:
+            return None
+
+        previous = trend[-2]
+        current = trend[-1]
+        previous_rate = float(previous.get("compliance_rate", previous.get("rate", 0)) or 0)
+        current_rate = float(current.get("compliance_rate", current.get("rate", 0)) or 0)
+        drop = round(previous_rate - current_rate, 1)
+        failed = int(current.get("failed", 0) or 0)
+
+        if drop < 10 or failed <= 0:
+            return None
+
+        return {
+            "type": "compliance_drop",
+            "severity": "error" if drop >= 25 else "warning",
+            "title": "Compliance dropped",
+            "date": current.get("date"),
+            "previous_rate": previous_rate,
+            "current_rate": current_rate,
+            "drop": drop,
+            "failed": failed,
+            "detail": (
+                f"Compliance fell from {previous_rate}% to {current_rate}% "
+                f"on {current.get('date')}."
+            ),
+            "action": "Review sources from that date and prioritize any new or failing senders.",
+        }

docs/development/roadmap.md

@@ -36,19 +36,17 @@ Recently improved:
 Implementation note:
 - The legacy `ReportStore` remains as a projection layer for existing report/dashboard code, but durable report data now lives in the database.
 
-## Active Milestone: Meaningful Reports
+## Completed Milestone: Meaningful Reports
 
 Objective: turn parsed DMARC data into administrator-friendly reports.
 
-Priority tasks:
-- Add "what changed" summaries for newly observed senders and sudden compliance drops.
-
 Delivered:
 - Dashboard time-series charts show daily mail volume, compliance rate, and failure rate.
 - Top sending sources show DMARC, SPF, and DKIM pass/fail breakdowns on the dashboard.
 - Per-domain timelines include daily volume, pass, fail, compliance-rate, and failure-rate rollups.
 - Domain reports can be exported to CSV for a selected date range.
 - Source reports include actionable recommendations for unknown sources, SPF-only passes, DKIM-only passes, full failures, and unenforced policies.
+- What changed summaries identify newly observed senders and sudden compliance drops.
 
 Quality bar:
 - A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.

docs/milestones.md

@@ -71,9 +71,9 @@ Delivered:
 Exit criteria:
 - A user can connect a mailbox, run a backfill, see exactly what was imported or skipped, and trust that totals are not double-counted.
 
-## Milestone 5: Dashboard and Meaningful Reports - In Progress
+## Milestone 5: Dashboard and Meaningful Reports - Complete
 
-Status: In progress
+Status: Complete
 
 Goal: convert raw DMARC data into useful operational reporting.
 
@@ -83,6 +83,7 @@ Delivered:
 - Per-domain report timeline and daily rollups.
 - Exportable reports for a selected domain and date range.
 - Clear recommendations for common cases: unknown source, SPF-only pass, DKIM-only pass, full fail, and policy not enforced.
+- What changed summaries for newly observed sources and sudden compliance drops.
 
 Exit criteria:
 - A domain owner can answer: who is sending as my domain, what is failing, what changed recently, and what should I fix next?

docs/todo.md

@@ -150,7 +150,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA
 ### Meaningful Reports
 - [x] Add per-domain daily rollups
 - [x] Add sender/source pass/fail totals
-- [ ] Add newly observed source detection
+- [x] Add newly observed source detection
 - [x] Add exportable domain reports
 - [x] Add actionable recommendations for common DMARC failure patterns