From 84855144451a9340db9a6693398736820351e759 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 22 May 2026 20:52:47 +0200 Subject: [PATCH] feat: add dashboard trend charts --- backend/app/api/api_v1/endpoints/stats.py | 8 +- backend/app/templates/index.html | 212 ++++++++++++++++++++- backend/app/tests/test_stats_endpoints.py | 21 ++ backend/app/tests/test_stats_summarizer.py | 122 ++++++++++++ backend/app/utils/stats_summarizer.py | 84 +++++--- docs/development/roadmap.md | 4 +- docs/milestones.md | 4 +- docs/todo.md | 8 +- 8 files changed, 426 insertions(+), 37 deletions(-) diff --git a/backend/app/api/api_v1/endpoints/stats.py b/backend/app/api/api_v1/endpoints/stats.py index 7863b1a..a6b5403 100644 --- a/backend/app/api/api_v1/endpoints/stats.py +++ b/backend/app/api/api_v1/endpoints/stats.py @@ -13,7 +13,7 @@ router = APIRouter() async def get_dashboard_statistics( db: Session = Depends(get_db), force_refresh: bool = Query(False, title="Force refresh of statistics"), - period_days: int = Query(30, title="Period in days for time-based statistics"), + period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"), ) -> Dict[str, Any]: """ Get optimized statistics for the dashboard using cached data when possible. @@ -34,7 +34,7 @@ async def get_dashboard_statistics( stats_summarizer.invalidate_cache() # Get statistics (from cache or calculate if needed) - stats = stats_summarizer.calculate_summary_statistics(db) + stats = stats_summarizer.calculate_summary_statistics(db, period_days=period_days) # Add version and timestamp stats["api_version"] = "1.0" @@ -48,7 +48,7 @@ async def get_domain_statistics( domain_id: str = Path(..., title="The domain ID or name"), db: Session = Depends(get_db), force_refresh: bool = Query(False, title="Force refresh of statistics"), - period_days: int = Query(30, title="Period in days for time-based statistics"), + period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"), ) -> Dict[str, Any]: """ Get optimized statistics for a specific domain using cached data when possible. @@ -69,7 +69,7 @@ async def get_domain_statistics( stats_summarizer.invalidate_cache(domain_id) # Get domain statistics (from cache or calculate if needed) - stats = stats_summarizer.calculate_summary_statistics(db, domain_id) + stats = stats_summarizer.calculate_summary_statistics(db, domain_id, period_days=period_days) # Add version and timestamp stats["api_version"] = "1.0" diff --git a/backend/app/templates/index.html b/backend/app/templates/index.html index badb426..3c12e34 100644 --- a/backend/app/templates/index.html +++ b/backend/app/templates/index.html @@ -84,6 +84,37 @@ {% endcall %} {% endcall %} + + +
+ {% call card() %} + {% call card_header() %} + {% call card_title() %}Mail Volume Trend{% endcall %} + {% call card_description() %} + Daily DMARC mail volume over the last 30 days + {% endcall %} + {% endcall %} + {% call card_content() %} +
+ +
+ {% endcall %} + {% endcall %} + + {% call card() %} + {% call card_header() %} + {% call card_title() %}Authentication Trend{% endcall %} + {% call card_description() %} + Daily compliance and failure rates over the last 30 days + {% endcall %} + {% endcall %} + {% call card_content() %} +
+ +
+ {% endcall %} + {% endcall %} +
@@ -184,10 +215,13 @@ {% endblock %} {% block scripts %} + -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/backend/app/tests/test_stats_endpoints.py b/backend/app/tests/test_stats_endpoints.py index 6a970dd..dc330a4 100644 --- a/backend/app/tests/test_stats_endpoints.py +++ b/backend/app/tests/test_stats_endpoints.py @@ -32,6 +32,16 @@ class TestDashboardStatistics: data = response.json() assert data["period_days"] == 7 + def test_dashboard_passes_period_days_to_summarizer(self, client: TestClient): + with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer: + mock_instance = MagicMock() + mock_instance.calculate_summary_statistics.return_value = {"total": 0} + MockSummarizer.return_value = mock_instance + + response = client.get("/api/v1/stats/dashboard?period_days=7") + assert response.status_code == 200 + assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 7 + def test_dashboard_force_refresh(self, client: TestClient): """force_refresh=true should trigger cache invalidation without error.""" response = client.get("/api/v1/stats/dashboard?force_refresh=true") @@ -85,6 +95,17 @@ class TestDomainStatistics: data = response.json() assert data["period_days"] == 14 + def test_domain_stats_passes_period_days_to_summarizer(self, client: TestClient): + with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer: + mock_instance = MagicMock() + mock_instance.calculate_summary_statistics.return_value = {"total": 0} + MockSummarizer.return_value = mock_instance + + response = client.get("/api/v1/stats/domain/example.com?period_days=14") + assert response.status_code == 200 + assert mock_instance.calculate_summary_statistics.call_args.args[1] == "example.com" + assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 14 + def test_domain_stats_force_refresh(self, client: TestClient): response = client.get("/api/v1/stats/domain/example.com?force_refresh=true") assert response.status_code == 200 diff --git a/backend/app/tests/test_stats_summarizer.py b/backend/app/tests/test_stats_summarizer.py index ce866d9..e1cc302 100644 --- a/backend/app/tests/test_stats_summarizer.py +++ b/backend/app/tests/test_stats_summarizer.py @@ -2,6 +2,7 @@ import shutil import tempfile +from datetime import datetime, timedelta, timezone import pytest from sqlalchemy import create_engine @@ -122,6 +123,83 @@ def _seed_mixed_source_records(db, domain_name="example.com"): return domain +def _timestamp_days_ago(days): + return int((datetime.now(timezone.utc) - timedelta(days=days)).timestamp()) + + +def _seed_recent_trend_records(db, domain_name="example.com"): + """Insert recent reports across multiple days for trend calculations.""" + domain = Domain(name=domain_name) + db.add(domain) + db.flush() + + report1 = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-recent-1", + org_name="google.com", + begin_date=_timestamp_days_ago(2), + end_date=_timestamp_days_ago(2) + 3600, + policy="none", + ) + report2 = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-recent-2", + org_name="google.com", + begin_date=_timestamp_days_ago(0), + end_date=_timestamp_days_ago(0) + 3600, + policy="none", + ) + report3 = DMARCReport( + domain_id=domain.id, + report_id=f"{domain_name}-old", + org_name="google.com", + begin_date=_timestamp_days_ago(20), + end_date=_timestamp_days_ago(20) + 3600, + policy="none", + ) + db.add_all([report1, report2, report3]) + db.flush() + + db.add_all( + [ + ReportRecord( + report_id=report1.id, + source_ip="203.0.113.10", + count=6, + disposition="none", + dkim="pass", + spf="fail", + ), + ReportRecord( + report_id=report1.id, + source_ip="203.0.113.11", + count=4, + disposition="reject", + dkim="fail", + spf="fail", + ), + ReportRecord( + report_id=report2.id, + source_ip="203.0.113.12", + count=5, + disposition="none", + dkim="pass", + spf="pass", + ), + ReportRecord( + report_id=report3.id, + source_ip="203.0.113.13", + count=99, + disposition="none", + dkim="pass", + spf="pass", + ), + ] + ) + db.flush() + return domain + + def test_auth_status_from_counts_returns_none_without_results(): assert _auth_status_from_counts(0, 0) == "none" @@ -187,6 +265,30 @@ class TestStatsSummarizerGlobal: assert stats["total_emails"] == 16 # 8 * 2 assert stats["reports_processed"] == 2 + def test_global_trend_includes_volume_and_failure_rate(self, db_session, summarizer): + _seed_recent_trend_records(db_session) + db_session.commit() + + stats = summarizer.calculate_summary_statistics(db_session, period_days=7) + assert len(stats["compliance_trend"]) == 2 + + first_day = stats["compliance_trend"][0] + assert first_day["total"] == 10 + assert first_day["volume"] == 10 + assert first_day["passed"] == 6 + assert first_day["failed"] == 4 + assert first_day["rate"] == 60.0 + assert first_day["compliance_rate"] == 60.0 + assert first_day["failure_rate"] == 40.0 + + def test_global_trend_respects_period_days(self, db_session, summarizer): + _seed_recent_trend_records(db_session) + db_session.commit() + + stats = summarizer.calculate_summary_statistics(db_session, period_days=1) + assert len(stats["compliance_trend"]) == 1 + assert stats["compliance_trend"][0]["total"] == 5 + class TestStatsSummarizerDomain: """Tests for domain-specific statistics.""" @@ -245,6 +347,16 @@ class TestStatsSummarizerDomain: stats = summarizer.calculate_summary_statistics(db_session, domain_id="example.com") assert stats["total_emails"] == 8 # Only example.com's data + def test_domain_trend_isolation(self, db_session, summarizer): + _seed_recent_trend_records(db_session, "example.com") + _seed_recent_trend_records(db_session, "other.org") + db_session.commit() + + stats = summarizer.calculate_summary_statistics( + db_session, domain_id="example.com", period_days=7 + ) + assert [point["total"] for point in stats["compliance_trend"]] == [10, 5] + class TestStatsSummarizerCaching: """Tests for the caching layer.""" @@ -266,3 +378,13 @@ class TestStatsSummarizerCaching: # Should recalculate after invalidation stats = summarizer.calculate_summary_statistics(db_session) assert stats["total_domains"] == 1 + + def test_period_days_uses_separate_cache_files(self, db_session, summarizer): + _seed_recent_trend_records(db_session) + db_session.commit() + + stats_7_days = summarizer.calculate_summary_statistics(db_session, period_days=7) + stats_1_day = summarizer.calculate_summary_statistics(db_session, period_days=1) + + assert len(stats_7_days["compliance_trend"]) == 2 + assert len(stats_1_day["compliance_trend"]) == 1 diff --git a/backend/app/utils/stats_summarizer.py b/backend/app/utils/stats_summarizer.py index 8f2f423..64dc756 100644 --- a/backend/app/utils/stats_summarizer.py +++ b/backend/app/utils/stats_summarizer.py @@ -52,7 +52,10 @@ class StatsSummarizer: os.makedirs(self.cache_dir, exist_ok=True) def get_cached_summary( - self, domain_id: Optional[str] = None, max_age_minutes: int = 60 + self, + domain_id: Optional[str] = None, + max_age_minutes: int = 60, + period_days: int = 30, ) -> Optional[Dict[str, Any]]: """ Get cached summary statistics if available and not too old @@ -61,11 +64,12 @@ class StatsSummarizer: domain_id: Optional domain ID to get domain-specific stats If None, gets global summary max_age_minutes: Maximum age of cache in minutes + period_days: Number of days used for time-based trend data Returns: Cached statistics or None if not available or too old """ - cache_file = self._get_cache_filename(domain_id) + cache_file = self._get_cache_filename(domain_id, period_days) try: if not os.path.exists(cache_file): @@ -86,18 +90,21 @@ class StatsSummarizer: logger.warning("Error reading cache file %s: %s", cache_file, str(e)) return None - def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool: + def save_summary( + self, stats: Dict[str, Any], domain_id: Optional[str] = None, period_days: int = 30 + ) -> bool: """ Save summary statistics to cache Args: stats: Dictionary of statistics to cache domain_id: Optional domain ID for domain-specific stats + period_days: Number of days used for time-based trend data Returns: True if save was successful, False otherwise """ - cache_file = self._get_cache_filename(domain_id) + cache_file = self._get_cache_filename(domain_id, period_days) try: # Add timestamp @@ -121,34 +128,39 @@ class StatsSummarizer: If None, invalidates global summary cache """ if domain_id is None: - # Invalidate all caches - cache_file = self._get_cache_filename() - if os.path.exists(cache_file): - os.remove(cache_file) + self._remove_cache_files("global_summary") else: - # Invalidate specific domain cache - cache_file = self._get_cache_filename(domain_id) - if os.path.exists(cache_file): - os.remove(cache_file) + safe_domain = domain_id.replace(".", "_").replace("/", "_") + self._remove_cache_files(f"domain_{safe_domain}") - def _get_cache_filename(self, domain_id: Optional[str] = None) -> str: + def _remove_cache_files(self, prefix: str) -> None: + """Remove cached summary files that begin with the provided prefix.""" + for filename in os.listdir(self.cache_dir): + if filename.startswith(prefix) and filename.endswith(".json"): + os.remove(os.path.join(self.cache_dir, filename)) + + def _get_cache_filename( + self, domain_id: Optional[str] = None, period_days: int = 30 + ) -> str: """ Get the filename for a cache file Args: domain_id: Optional domain ID for domain-specific cache + period_days: Number of days used for time-based trend data Returns: Path to the cache file """ + period_days = max(1, int(period_days or 30)) if domain_id is None: - return os.path.join(self.cache_dir, "global_summary.json") + return os.path.join(self.cache_dir, f"global_summary_{period_days}d.json") # Sanitize domain_id to use as filename safe_domain = domain_id.replace(".", "_").replace("/", "_") - return os.path.join(self.cache_dir, f"domain_{safe_domain}.json") + return os.path.join(self.cache_dir, f"domain_{safe_domain}_{period_days}d.json") def calculate_summary_statistics( - self, db: Session, domain_id: Optional[str] = None + self, db: Session, domain_id: Optional[str] = None, period_days: int = 30 ) -> Dict[str, Any]: """ Calculate summary statistics from the database @@ -156,26 +168,29 @@ class StatsSummarizer: Args: db: Database session domain_id: Optional domain ID to calculate domain-specific stats + period_days: Number of days used for time-based trend data Returns: Dictionary with summary statistics """ + period_days = max(1, int(period_days or 30)) + # First check if we have cached stats - cached_stats = self.get_cached_summary(domain_id) + cached_stats = self.get_cached_summary(domain_id, period_days=period_days) if cached_stats: return cached_stats if domain_id is None: - stats = self._calculate_global_statistics(db) + stats = self._calculate_global_statistics(db, period_days) else: - stats = self._calculate_domain_statistics(db, domain_id) + stats = self._calculate_domain_statistics(db, domain_id, period_days) # Cache the statistics - self.save_summary(stats, domain_id) + self.save_summary(stats, domain_id, period_days) return stats - def _calculate_global_statistics(self, db: Session) -> Dict[str, Any]: + def _calculate_global_statistics(self, db: Session, period_days: int = 30) -> Dict[str, Any]: """Calculate global statistics across all domains from the database.""" # Count total domains total_domains = db.query(func.count(Domain.id)).scalar() or 0 @@ -206,7 +221,7 @@ class StatsSummarizer: top_sources = self._get_top_sources(db) # Compliance trend over recent days - compliance_trend = self._get_compliance_trend(db) + compliance_trend = self._get_compliance_trend(db, days=period_days) return { "total_domains": total_domains, @@ -218,7 +233,9 @@ class StatsSummarizer: "compliance_trend": compliance_trend, } - def _calculate_domain_statistics(self, db: Session, domain_id: str) -> Dict[str, Any]: + def _calculate_domain_statistics( + self, db: Session, domain_id: str, period_days: int = 30 + ) -> Dict[str, Any]: """Calculate statistics for a specific domain from the database.""" # Look up the domain by name domain = db.query(Domain).filter(Domain.name == domain_id).first() @@ -266,7 +283,7 @@ class StatsSummarizer: sources = self._get_domain_sources(db, domain.id) # Compliance trend for this domain - compliance_trend = self._get_compliance_trend(db, domain.id) + compliance_trend = self._get_compliance_trend(db, domain.id, days=period_days) return { "domain": domain_id, @@ -445,7 +462,22 @@ class StatsSummarizer: trend = [] for date_str in sorted(daily.keys()): data = daily[date_str] - rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0 - trend.append({"date": date_str, "rate": rate}) + total = data["total"] + passed = data["passed"] + failed = max(0, total - passed) + compliance_rate = round((passed / total) * 100, 1) if total > 0 else 0.0 + failure_rate = round((failed / total) * 100, 1) if total > 0 else 0.0 + trend.append( + { + "date": date_str, + "total": total, + "volume": total, + "passed": passed, + "failed": failed, + "rate": compliance_rate, + "compliance_rate": compliance_rate, + "failure_rate": failure_rate, + } + ) return trend diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index faf1271..8aacb8d 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -41,12 +41,14 @@ Implementation note: Objective: turn parsed DMARC data into administrator-friendly reports. Priority tasks: -- Add time-series charts for volume and compliance. - Add per-domain daily rollups. - Add "what changed" summaries for newly observed senders and sudden compliance drops. - Add exportable reports for a domain and date range. - Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns. +Delivered: +- Dashboard time-series charts show daily mail volume, compliance rate, and failure rate. + Quality bar: - A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next. diff --git a/docs/milestones.md b/docs/milestones.md index 7799ca0..94b671b 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -77,8 +77,10 @@ Status: In progress Goal: convert raw DMARC data into useful operational reporting. -Planned: +Delivered: - Dashboard trend charts for volume, compliance rate, and failure rate. + +Planned: - Top sender/source reports with pass/fail breakdowns. - Per-domain report timeline and daily rollups. - Exportable reports for a selected domain and date range. diff --git a/docs/todo.md b/docs/todo.md index bf765a3..cc691c1 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -128,16 +128,16 @@ This file tracks the specific implementation tasks for each milestone of the DMA ## Milestone 4: Dashboard Enhancements ### Data Visualization -- [ ] Integrate Chart.js library -- [ ] Create time-series charts for DMARC compliance -- [ ] Add volume charts for email traffic +- [x] Integrate Chart.js library +- [x] Create time-series charts for DMARC compliance +- [x] Add volume charts for email traffic - [ ] Implement sender breakdown visualizations - [ ] Create policy distribution charts ### Dashboard Widgets - [ ] Create compliance rate summary widget - [ ] Add enforcement rate widget -- [ ] Implement email volume trends widget +- [x] Implement email volume trends widget - [ ] Create top sender sources widget - [ ] Add alert status summary (for later integration)