diff --git a/backend/app/api/api_v1/endpoints/stats.py b/backend/app/api/api_v1/endpoints/stats.py
index 7863b1a..a6b5403 100644
--- a/backend/app/api/api_v1/endpoints/stats.py
+++ b/backend/app/api/api_v1/endpoints/stats.py
@@ -13,7 +13,7 @@ router = APIRouter()
async def get_dashboard_statistics(
db: Session = Depends(get_db),
force_refresh: bool = Query(False, title="Force refresh of statistics"),
- period_days: int = Query(30, title="Period in days for time-based statistics"),
+ period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"),
) -> Dict[str, Any]:
"""
Get optimized statistics for the dashboard using cached data when possible.
@@ -34,7 +34,7 @@ async def get_dashboard_statistics(
stats_summarizer.invalidate_cache()
# Get statistics (from cache or calculate if needed)
- stats = stats_summarizer.calculate_summary_statistics(db)
+ stats = stats_summarizer.calculate_summary_statistics(db, period_days=period_days)
# Add version and timestamp
stats["api_version"] = "1.0"
@@ -48,7 +48,7 @@ async def get_domain_statistics(
domain_id: str = Path(..., title="The domain ID or name"),
db: Session = Depends(get_db),
force_refresh: bool = Query(False, title="Force refresh of statistics"),
- period_days: int = Query(30, title="Period in days for time-based statistics"),
+ period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"),
) -> Dict[str, Any]:
"""
Get optimized statistics for a specific domain using cached data when possible.
@@ -69,7 +69,7 @@ async def get_domain_statistics(
stats_summarizer.invalidate_cache(domain_id)
# Get domain statistics (from cache or calculate if needed)
- stats = stats_summarizer.calculate_summary_statistics(db, domain_id)
+ stats = stats_summarizer.calculate_summary_statistics(db, domain_id, period_days=period_days)
# Add version and timestamp
stats["api_version"] = "1.0"
diff --git a/backend/app/templates/index.html b/backend/app/templates/index.html
index badb426..3c12e34 100644
--- a/backend/app/templates/index.html
+++ b/backend/app/templates/index.html
@@ -84,6 +84,37 @@
{% endcall %}
{% endcall %}
+
+
+
+ {% call card() %}
+ {% call card_header() %}
+ {% call card_title() %}Mail Volume Trend{% endcall %}
+ {% call card_description() %}
+ Daily DMARC mail volume over the last 30 days
+ {% endcall %}
+ {% endcall %}
+ {% call card_content() %}
+
+
+
+ {% endcall %}
+ {% endcall %}
+
+ {% call card() %}
+ {% call card_header() %}
+ {% call card_title() %}Authentication Trend{% endcall %}
+ {% call card_description() %}
+ Daily compliance and failure rates over the last 30 days
+ {% endcall %}
+ {% endcall %}
+ {% call card_content() %}
+
+
+
+ {% endcall %}
+ {% endcall %}
+
@@ -184,10 +215,13 @@
{% endblock %}
{% block scripts %}
+
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/backend/app/tests/test_stats_endpoints.py b/backend/app/tests/test_stats_endpoints.py
index 6a970dd..dc330a4 100644
--- a/backend/app/tests/test_stats_endpoints.py
+++ b/backend/app/tests/test_stats_endpoints.py
@@ -32,6 +32,16 @@ class TestDashboardStatistics:
data = response.json()
assert data["period_days"] == 7
+ def test_dashboard_passes_period_days_to_summarizer(self, client: TestClient):
+ with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer:
+ mock_instance = MagicMock()
+ mock_instance.calculate_summary_statistics.return_value = {"total": 0}
+ MockSummarizer.return_value = mock_instance
+
+ response = client.get("/api/v1/stats/dashboard?period_days=7")
+ assert response.status_code == 200
+ assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 7
+
def test_dashboard_force_refresh(self, client: TestClient):
"""force_refresh=true should trigger cache invalidation without error."""
response = client.get("/api/v1/stats/dashboard?force_refresh=true")
@@ -85,6 +95,17 @@ class TestDomainStatistics:
data = response.json()
assert data["period_days"] == 14
+ def test_domain_stats_passes_period_days_to_summarizer(self, client: TestClient):
+ with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer:
+ mock_instance = MagicMock()
+ mock_instance.calculate_summary_statistics.return_value = {"total": 0}
+ MockSummarizer.return_value = mock_instance
+
+ response = client.get("/api/v1/stats/domain/example.com?period_days=14")
+ assert response.status_code == 200
+ assert mock_instance.calculate_summary_statistics.call_args.args[1] == "example.com"
+ assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 14
+
def test_domain_stats_force_refresh(self, client: TestClient):
response = client.get("/api/v1/stats/domain/example.com?force_refresh=true")
assert response.status_code == 200
diff --git a/backend/app/tests/test_stats_summarizer.py b/backend/app/tests/test_stats_summarizer.py
index ce866d9..e1cc302 100644
--- a/backend/app/tests/test_stats_summarizer.py
+++ b/backend/app/tests/test_stats_summarizer.py
@@ -2,6 +2,7 @@
import shutil
import tempfile
+from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import create_engine
@@ -122,6 +123,83 @@ def _seed_mixed_source_records(db, domain_name="example.com"):
return domain
+def _timestamp_days_ago(days):
+ return int((datetime.now(timezone.utc) - timedelta(days=days)).timestamp())
+
+
+def _seed_recent_trend_records(db, domain_name="example.com"):
+ """Insert recent reports across multiple days for trend calculations."""
+ domain = Domain(name=domain_name)
+ db.add(domain)
+ db.flush()
+
+ report1 = DMARCReport(
+ domain_id=domain.id,
+ report_id=f"{domain_name}-recent-1",
+ org_name="google.com",
+ begin_date=_timestamp_days_ago(2),
+ end_date=_timestamp_days_ago(2) + 3600,
+ policy="none",
+ )
+ report2 = DMARCReport(
+ domain_id=domain.id,
+ report_id=f"{domain_name}-recent-2",
+ org_name="google.com",
+ begin_date=_timestamp_days_ago(0),
+ end_date=_timestamp_days_ago(0) + 3600,
+ policy="none",
+ )
+ report3 = DMARCReport(
+ domain_id=domain.id,
+ report_id=f"{domain_name}-old",
+ org_name="google.com",
+ begin_date=_timestamp_days_ago(20),
+ end_date=_timestamp_days_ago(20) + 3600,
+ policy="none",
+ )
+ db.add_all([report1, report2, report3])
+ db.flush()
+
+ db.add_all(
+ [
+ ReportRecord(
+ report_id=report1.id,
+ source_ip="203.0.113.10",
+ count=6,
+ disposition="none",
+ dkim="pass",
+ spf="fail",
+ ),
+ ReportRecord(
+ report_id=report1.id,
+ source_ip="203.0.113.11",
+ count=4,
+ disposition="reject",
+ dkim="fail",
+ spf="fail",
+ ),
+ ReportRecord(
+ report_id=report2.id,
+ source_ip="203.0.113.12",
+ count=5,
+ disposition="none",
+ dkim="pass",
+ spf="pass",
+ ),
+ ReportRecord(
+ report_id=report3.id,
+ source_ip="203.0.113.13",
+ count=99,
+ disposition="none",
+ dkim="pass",
+ spf="pass",
+ ),
+ ]
+ )
+ db.flush()
+ return domain
+
+
def test_auth_status_from_counts_returns_none_without_results():
assert _auth_status_from_counts(0, 0) == "none"
@@ -187,6 +265,30 @@ class TestStatsSummarizerGlobal:
assert stats["total_emails"] == 16 # 8 * 2
assert stats["reports_processed"] == 2
+ def test_global_trend_includes_volume_and_failure_rate(self, db_session, summarizer):
+ _seed_recent_trend_records(db_session)
+ db_session.commit()
+
+ stats = summarizer.calculate_summary_statistics(db_session, period_days=7)
+ assert len(stats["compliance_trend"]) == 2
+
+ first_day = stats["compliance_trend"][0]
+ assert first_day["total"] == 10
+ assert first_day["volume"] == 10
+ assert first_day["passed"] == 6
+ assert first_day["failed"] == 4
+ assert first_day["rate"] == 60.0
+ assert first_day["compliance_rate"] == 60.0
+ assert first_day["failure_rate"] == 40.0
+
+ def test_global_trend_respects_period_days(self, db_session, summarizer):
+ _seed_recent_trend_records(db_session)
+ db_session.commit()
+
+ stats = summarizer.calculate_summary_statistics(db_session, period_days=1)
+ assert len(stats["compliance_trend"]) == 1
+ assert stats["compliance_trend"][0]["total"] == 5
+
class TestStatsSummarizerDomain:
"""Tests for domain-specific statistics."""
@@ -245,6 +347,16 @@ class TestStatsSummarizerDomain:
stats = summarizer.calculate_summary_statistics(db_session, domain_id="example.com")
assert stats["total_emails"] == 8 # Only example.com's data
+ def test_domain_trend_isolation(self, db_session, summarizer):
+ _seed_recent_trend_records(db_session, "example.com")
+ _seed_recent_trend_records(db_session, "other.org")
+ db_session.commit()
+
+ stats = summarizer.calculate_summary_statistics(
+ db_session, domain_id="example.com", period_days=7
+ )
+ assert [point["total"] for point in stats["compliance_trend"]] == [10, 5]
+
class TestStatsSummarizerCaching:
"""Tests for the caching layer."""
@@ -266,3 +378,13 @@ class TestStatsSummarizerCaching:
# Should recalculate after invalidation
stats = summarizer.calculate_summary_statistics(db_session)
assert stats["total_domains"] == 1
+
+ def test_period_days_uses_separate_cache_files(self, db_session, summarizer):
+ _seed_recent_trend_records(db_session)
+ db_session.commit()
+
+ stats_7_days = summarizer.calculate_summary_statistics(db_session, period_days=7)
+ stats_1_day = summarizer.calculate_summary_statistics(db_session, period_days=1)
+
+ assert len(stats_7_days["compliance_trend"]) == 2
+ assert len(stats_1_day["compliance_trend"]) == 1
diff --git a/backend/app/utils/stats_summarizer.py b/backend/app/utils/stats_summarizer.py
index 8f2f423..64dc756 100644
--- a/backend/app/utils/stats_summarizer.py
+++ b/backend/app/utils/stats_summarizer.py
@@ -52,7 +52,10 @@ class StatsSummarizer:
os.makedirs(self.cache_dir, exist_ok=True)
def get_cached_summary(
- self, domain_id: Optional[str] = None, max_age_minutes: int = 60
+ self,
+ domain_id: Optional[str] = None,
+ max_age_minutes: int = 60,
+ period_days: int = 30,
) -> Optional[Dict[str, Any]]:
"""
Get cached summary statistics if available and not too old
@@ -61,11 +64,12 @@ class StatsSummarizer:
domain_id: Optional domain ID to get domain-specific stats
If None, gets global summary
max_age_minutes: Maximum age of cache in minutes
+ period_days: Number of days used for time-based trend data
Returns:
Cached statistics or None if not available or too old
"""
- cache_file = self._get_cache_filename(domain_id)
+ cache_file = self._get_cache_filename(domain_id, period_days)
try:
if not os.path.exists(cache_file):
@@ -86,18 +90,21 @@ class StatsSummarizer:
logger.warning("Error reading cache file %s: %s", cache_file, str(e))
return None
- def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool:
+ def save_summary(
+ self, stats: Dict[str, Any], domain_id: Optional[str] = None, period_days: int = 30
+ ) -> bool:
"""
Save summary statistics to cache
Args:
stats: Dictionary of statistics to cache
domain_id: Optional domain ID for domain-specific stats
+ period_days: Number of days used for time-based trend data
Returns:
True if save was successful, False otherwise
"""
- cache_file = self._get_cache_filename(domain_id)
+ cache_file = self._get_cache_filename(domain_id, period_days)
try:
# Add timestamp
@@ -121,34 +128,39 @@ class StatsSummarizer:
If None, invalidates global summary cache
"""
if domain_id is None:
- # Invalidate all caches
- cache_file = self._get_cache_filename()
- if os.path.exists(cache_file):
- os.remove(cache_file)
+ self._remove_cache_files("global_summary")
else:
- # Invalidate specific domain cache
- cache_file = self._get_cache_filename(domain_id)
- if os.path.exists(cache_file):
- os.remove(cache_file)
+ safe_domain = domain_id.replace(".", "_").replace("/", "_")
+ self._remove_cache_files(f"domain_{safe_domain}")
- def _get_cache_filename(self, domain_id: Optional[str] = None) -> str:
+ def _remove_cache_files(self, prefix: str) -> None:
+ """Remove cached summary files that begin with the provided prefix."""
+ for filename in os.listdir(self.cache_dir):
+ if filename.startswith(prefix) and filename.endswith(".json"):
+ os.remove(os.path.join(self.cache_dir, filename))
+
+ def _get_cache_filename(
+ self, domain_id: Optional[str] = None, period_days: int = 30
+ ) -> str:
"""
Get the filename for a cache file
Args:
domain_id: Optional domain ID for domain-specific cache
+ period_days: Number of days used for time-based trend data
Returns:
Path to the cache file
"""
+ period_days = max(1, int(period_days or 30))
if domain_id is None:
- return os.path.join(self.cache_dir, "global_summary.json")
+ return os.path.join(self.cache_dir, f"global_summary_{period_days}d.json")
# Sanitize domain_id to use as filename
safe_domain = domain_id.replace(".", "_").replace("/", "_")
- return os.path.join(self.cache_dir, f"domain_{safe_domain}.json")
+ return os.path.join(self.cache_dir, f"domain_{safe_domain}_{period_days}d.json")
def calculate_summary_statistics(
- self, db: Session, domain_id: Optional[str] = None
+ self, db: Session, domain_id: Optional[str] = None, period_days: int = 30
) -> Dict[str, Any]:
"""
Calculate summary statistics from the database
@@ -156,26 +168,29 @@ class StatsSummarizer:
Args:
db: Database session
domain_id: Optional domain ID to calculate domain-specific stats
+ period_days: Number of days used for time-based trend data
Returns:
Dictionary with summary statistics
"""
+ period_days = max(1, int(period_days or 30))
+
# First check if we have cached stats
- cached_stats = self.get_cached_summary(domain_id)
+ cached_stats = self.get_cached_summary(domain_id, period_days=period_days)
if cached_stats:
return cached_stats
if domain_id is None:
- stats = self._calculate_global_statistics(db)
+ stats = self._calculate_global_statistics(db, period_days)
else:
- stats = self._calculate_domain_statistics(db, domain_id)
+ stats = self._calculate_domain_statistics(db, domain_id, period_days)
# Cache the statistics
- self.save_summary(stats, domain_id)
+ self.save_summary(stats, domain_id, period_days)
return stats
- def _calculate_global_statistics(self, db: Session) -> Dict[str, Any]:
+ def _calculate_global_statistics(self, db: Session, period_days: int = 30) -> Dict[str, Any]:
"""Calculate global statistics across all domains from the database."""
# Count total domains
total_domains = db.query(func.count(Domain.id)).scalar() or 0
@@ -206,7 +221,7 @@ class StatsSummarizer:
top_sources = self._get_top_sources(db)
# Compliance trend over recent days
- compliance_trend = self._get_compliance_trend(db)
+ compliance_trend = self._get_compliance_trend(db, days=period_days)
return {
"total_domains": total_domains,
@@ -218,7 +233,9 @@ class StatsSummarizer:
"compliance_trend": compliance_trend,
}
- def _calculate_domain_statistics(self, db: Session, domain_id: str) -> Dict[str, Any]:
+ def _calculate_domain_statistics(
+ self, db: Session, domain_id: str, period_days: int = 30
+ ) -> Dict[str, Any]:
"""Calculate statistics for a specific domain from the database."""
# Look up the domain by name
domain = db.query(Domain).filter(Domain.name == domain_id).first()
@@ -266,7 +283,7 @@ class StatsSummarizer:
sources = self._get_domain_sources(db, domain.id)
# Compliance trend for this domain
- compliance_trend = self._get_compliance_trend(db, domain.id)
+ compliance_trend = self._get_compliance_trend(db, domain.id, days=period_days)
return {
"domain": domain_id,
@@ -445,7 +462,22 @@ class StatsSummarizer:
trend = []
for date_str in sorted(daily.keys()):
data = daily[date_str]
- rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0
- trend.append({"date": date_str, "rate": rate})
+ total = data["total"]
+ passed = data["passed"]
+ failed = max(0, total - passed)
+ compliance_rate = round((passed / total) * 100, 1) if total > 0 else 0.0
+ failure_rate = round((failed / total) * 100, 1) if total > 0 else 0.0
+ trend.append(
+ {
+ "date": date_str,
+ "total": total,
+ "volume": total,
+ "passed": passed,
+ "failed": failed,
+ "rate": compliance_rate,
+ "compliance_rate": compliance_rate,
+ "failure_rate": failure_rate,
+ }
+ )
return trend
diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md
index faf1271..8aacb8d 100644
--- a/docs/development/roadmap.md
+++ b/docs/development/roadmap.md
@@ -41,12 +41,14 @@ Implementation note:
Objective: turn parsed DMARC data into administrator-friendly reports.
Priority tasks:
-- Add time-series charts for volume and compliance.
- Add per-domain daily rollups.
- Add "what changed" summaries for newly observed senders and sudden compliance drops.
- Add exportable reports for a domain and date range.
- Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns.
+Delivered:
+- Dashboard time-series charts show daily mail volume, compliance rate, and failure rate.
+
Quality bar:
- A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.
diff --git a/docs/milestones.md b/docs/milestones.md
index 7799ca0..94b671b 100644
--- a/docs/milestones.md
+++ b/docs/milestones.md
@@ -77,8 +77,10 @@ Status: In progress
Goal: convert raw DMARC data into useful operational reporting.
-Planned:
+Delivered:
- Dashboard trend charts for volume, compliance rate, and failure rate.
+
+Planned:
- Top sender/source reports with pass/fail breakdowns.
- Per-domain report timeline and daily rollups.
- Exportable reports for a selected domain and date range.
diff --git a/docs/todo.md b/docs/todo.md
index bf765a3..cc691c1 100644
--- a/docs/todo.md
+++ b/docs/todo.md
@@ -128,16 +128,16 @@ This file tracks the specific implementation tasks for each milestone of the DMA
## Milestone 4: Dashboard Enhancements
### Data Visualization
-- [ ] Integrate Chart.js library
-- [ ] Create time-series charts for DMARC compliance
-- [ ] Add volume charts for email traffic
+- [x] Integrate Chart.js library
+- [x] Create time-series charts for DMARC compliance
+- [x] Add volume charts for email traffic
- [ ] Implement sender breakdown visualizations
- [ ] Create policy distribution charts
### Dashboard Widgets
- [ ] Create compliance rate summary widget
- [ ] Add enforcement rate widget
-- [ ] Implement email volume trends widget
+- [x] Implement email volume trends widget
- [ ] Create top sender sources widget
- [ ] Add alert status summary (for later integration)