gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #105 from christianlouis/codex/dashboard-trend-charts

Browse Source 
feat: add dashboard trend charts
This commit is contained in:
Christian Krakau-Louis
2026-05-22 20:55:45 +02:00
committed by GitHub
parent 1c8eaa5733 8485514445
commit 884d83ef83
8 changed files with 426 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/api_v1/endpoints/stats.py
+4 -4
View File
@@ -13,7 +13,7 @@ router = APIRouter()
async def get_dashboard_statistics(
db: Session = Depends(get_db),
force_refresh: bool = Query(False, title="Force refresh of statistics"),
period_days: int = Query(30, title="Period in days for time-based statistics"),
period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"),
) -> Dict[str, Any]:
"""
Get optimized statistics for the dashboard using cached data when possible.
@@ -34,7 +34,7 @@ async def get_dashboard_statistics(
stats_summarizer.invalidate_cache()
# Get statistics (from cache or calculate if needed)
stats = stats_summarizer.calculate_summary_statistics(db)
stats = stats_summarizer.calculate_summary_statistics(db, period_days=period_days)
# Add version and timestamp
stats["api_version"] = "1.0"
@@ -48,7 +48,7 @@ async def get_domain_statistics(
domain_id: str = Path(..., title="The domain ID or name"),
db: Session = Depends(get_db),
force_refresh: bool = Query(False, title="Force refresh of statistics"),
period_days: int = Query(30, title="Period in days for time-based statistics"),
period_days: int = Query(30, ge=1, le=365, title="Period in days for time-based statistics"),
) -> Dict[str, Any]:
"""
Get optimized statistics for a specific domain using cached data when possible.
@@ -69,7 +69,7 @@ async def get_domain_statistics(
stats_summarizer.invalidate_cache(domain_id)
# Get domain statistics (from cache or calculate if needed)
stats = stats_summarizer.calculate_summary_statistics(db, domain_id)
stats = stats_summarizer.calculate_summary_statistics(db, domain_id, period_days=period_days)
# Add version and timestamp
stats["api_version"] = "1.0"
backend/app/templates/index.html
+211 -1
View File
@@ -84,6 +84,37 @@
{% endcall %}
{% endcall %}
</div>
<!-- Dashboard Trends -->
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4" x-show="hasDomainData" x-cloak>
{% call card() %}
{% call card_header() %}
{% call card_title() %}Mail Volume Trend{% endcall %}
{% call card_description() %}
Daily DMARC mail volume over the last 30 days
{% endcall %}
{% endcall %}
{% call card_content() %}
<div class="h-64">
<canvas id="volume-trend-chart" aria-label="Daily DMARC mail volume trend"></canvas>
</div>
{% endcall %}
{% endcall %}
{% call card() %}
{% call card_header() %}
{% call card_title() %}Authentication Trend{% endcall %}
{% call card_description() %}
Daily compliance and failure rates over the last 30 days
{% endcall %}
{% endcall %}
{% call card_content() %}
<div class="h-64">
<canvas id="compliance-trend-chart" aria-label="Daily DMARC compliance and failure trend"></canvas>
</div>
{% endcall %}
{% endcall %}
</div>
<!-- Domain Compliance Table -->
<div x-show="hasDomainData" x-cloak>
@@ -184,10 +215,13 @@
{% endblock %}
{% block scripts %}
<script src="/static/js/chart.umd.min.js"></script>
<script>
function dashboardApp() {
return {
hasDomainData: false,
volumeTrendChart: null,
complianceTrendChart: null,
init() {
// Fetch domain summary on page load
@@ -239,12 +273,30 @@ function dashboardApp() {
this.hasDomainData = true;
this.updateDashboardStats(data);
this.populateDomainsTable(data.domains);
this.$nextTick(() => this.fetchDashboardTrends());
} else {
this.hasDomainData = false;
this.clearDashboardCharts();
}
} catch (error) {
console.error('Error fetching domain summary:', error);
this.hasDomainData = false;
this.clearDashboardCharts();
}
},
async fetchDashboardTrends() {
try {
const response = await fetch('/api/v1/stats/dashboard?period_days=30');
if (!response.ok) {
console.error('Error fetching dashboard trends:', response.status);
return;
}
const data = await response.json();
this.renderDashboardCharts(data.compliance_trend || []);
} catch (error) {
console.error('Error fetching dashboard trends:', error);
}
},
@@ -261,6 +313,164 @@ function dashboardApp() {
if (passRate) passRate.textContent = `${data.overall_pass_rate || 0}%`;
if (reportsProcessed) reportsProcessed.textContent = data.reports_processed || 0;
},
renderDashboardCharts(trendData) {
if (!window.Chart || !trendData.length) {
this.clearDashboardCharts();
return;
}
this.renderVolumeTrend(trendData);
this.renderComplianceTrend(trendData);
},
renderVolumeTrend(trendData) {
const canvas = document.getElementById('volume-trend-chart');
if (!canvas) return;
const labels = trendData.map(item => item.date);
const volumes = trendData.map(item => item.volume || item.total || 0);
if (this.volumeTrendChart) {
this.volumeTrendChart.destroy();
}
this.volumeTrendChart = new Chart(canvas.getContext('2d'), {
type: 'bar',
data: {
labels,
datasets: [{
label: 'Messages',
data: volumes,
backgroundColor: 'rgba(37, 99, 235, 0.72)',
borderColor: 'rgb(37, 99, 235)',
borderWidth: 1,
borderRadius: 4,
maxBarThickness: 28
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
y: {
beginAtZero: true,
ticks: {
precision: 0,
callback: value => this.formatLargeNumber(value)
},
title: {
display: true,
text: 'Messages'
}
},
x: {
grid: {
display: false
}
}
},
plugins: {
legend: {
display: false
},
tooltip: {
callbacks: {
label: context => `${this.formatLargeNumber(context.parsed.y)} messages`
}
}
}
}
});
},
renderComplianceTrend(trendData) {
const canvas = document.getElementById('compliance-trend-chart');
if (!canvas) return;
const labels = trendData.map(item => item.date);
const complianceRates = trendData.map(item => item.compliance_rate ?? item.rate ?? 0);
const failureRates = trendData.map(item => item.failure_rate ?? 0);
if (this.complianceTrendChart) {
this.complianceTrendChart.destroy();
}
this.complianceTrendChart = new Chart(canvas.getContext('2d'), {
type: 'line',
data: {
labels,
datasets: [
{
label: 'Compliance Rate',
data: complianceRates,
borderColor: 'rgb(22, 163, 74)',
backgroundColor: 'rgba(22, 163, 74, 0.12)',
pointBackgroundColor: 'rgb(22, 163, 74)',
pointRadius: 3,
pointHoverRadius: 5,
tension: 0.35,
fill: true
},
{
label: 'Failure Rate',
data: failureRates,
borderColor: 'rgb(220, 38, 38)',
backgroundColor: 'rgba(220, 38, 38, 0.08)',
pointBackgroundColor: 'rgb(220, 38, 38)',
pointRadius: 3,
pointHoverRadius: 5,
tension: 0.35,
fill: false
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
y: {
beginAtZero: true,
max: 100,
ticks: {
callback: value => `${value}%`
},
title: {
display: true,
text: 'Rate'
}
},
x: {
grid: {
display: false
}
}
},
plugins: {
tooltip: {
callbacks: {
label: context => `${context.dataset.label}: ${context.parsed.y}%`
}
}
}
}
});
},
clearDashboardCharts() {
if (this.volumeTrendChart) {
this.volumeTrendChart.destroy();
this.volumeTrendChart = null;
}
if (this.complianceTrendChart) {
this.complianceTrendChart.destroy();
this.complianceTrendChart = null;
}
},
formatLargeNumber(value) {
return new Intl.NumberFormat(undefined, { notation: 'compact' }).format(value);
},
populateDomainsTable(domains) {
if (!domains || !domains.length) return;
@@ -300,4 +510,4 @@ function dashboardApp() {
}
}
</script>
{% endblock %}
{% endblock %}
backend/app/tests/test_stats_endpoints.py
+21
View File
@@ -32,6 +32,16 @@ class TestDashboardStatistics:
data = response.json()
assert data["period_days"] == 7
def test_dashboard_passes_period_days_to_summarizer(self, client: TestClient):
with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer:
mock_instance = MagicMock()
mock_instance.calculate_summary_statistics.return_value = {"total": 0}
MockSummarizer.return_value = mock_instance
response = client.get("/api/v1/stats/dashboard?period_days=7")
assert response.status_code == 200
assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 7
def test_dashboard_force_refresh(self, client: TestClient):
"""force_refresh=true should trigger cache invalidation without error."""
response = client.get("/api/v1/stats/dashboard?force_refresh=true")
@@ -85,6 +95,17 @@ class TestDomainStatistics:
data = response.json()
assert data["period_days"] == 14
def test_domain_stats_passes_period_days_to_summarizer(self, client: TestClient):
with patch("app.api.api_v1.endpoints.stats.StatsSummarizer") as MockSummarizer:
mock_instance = MagicMock()
mock_instance.calculate_summary_statistics.return_value = {"total": 0}
MockSummarizer.return_value = mock_instance
response = client.get("/api/v1/stats/domain/example.com?period_days=14")
assert response.status_code == 200
assert mock_instance.calculate_summary_statistics.call_args.args[1] == "example.com"
assert mock_instance.calculate_summary_statistics.call_args.kwargs["period_days"] == 14
def test_domain_stats_force_refresh(self, client: TestClient):
response = client.get("/api/v1/stats/domain/example.com?force_refresh=true")
assert response.status_code == 200
backend/app/tests/test_stats_summarizer.py
+122
View File
@@ -2,6 +2,7 @@
import shutil
import tempfile
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import create_engine
@@ -122,6 +123,83 @@ def _seed_mixed_source_records(db, domain_name="example.com"):
return domain
def _timestamp_days_ago(days):
return int((datetime.now(timezone.utc) - timedelta(days=days)).timestamp())
def _seed_recent_trend_records(db, domain_name="example.com"):
"""Insert recent reports across multiple days for trend calculations."""
domain = Domain(name=domain_name)
db.add(domain)
db.flush()
report1 = DMARCReport(
domain_id=domain.id,
report_id=f"{domain_name}-recent-1",
org_name="google.com",
begin_date=_timestamp_days_ago(2),
end_date=_timestamp_days_ago(2) + 3600,
policy="none",
)
report2 = DMARCReport(
domain_id=domain.id,
report_id=f"{domain_name}-recent-2",
org_name="google.com",
begin_date=_timestamp_days_ago(0),
end_date=_timestamp_days_ago(0) + 3600,
policy="none",
)
report3 = DMARCReport(
domain_id=domain.id,
report_id=f"{domain_name}-old",
org_name="google.com",
begin_date=_timestamp_days_ago(20),
end_date=_timestamp_days_ago(20) + 3600,
policy="none",
)
db.add_all([report1, report2, report3])
db.flush()
db.add_all(
[
ReportRecord(
report_id=report1.id,
source_ip="203.0.113.10",
count=6,
disposition="none",
dkim="pass",
spf="fail",
),
ReportRecord(
report_id=report1.id,
source_ip="203.0.113.11",
count=4,
disposition="reject",
dkim="fail",
spf="fail",
),
ReportRecord(
report_id=report2.id,
source_ip="203.0.113.12",
count=5,
disposition="none",
dkim="pass",
spf="pass",
),
ReportRecord(
report_id=report3.id,
source_ip="203.0.113.13",
count=99,
disposition="none",
dkim="pass",
spf="pass",
),
]
)
db.flush()
return domain
def test_auth_status_from_counts_returns_none_without_results():
assert _auth_status_from_counts(0, 0) == "none"
@@ -187,6 +265,30 @@ class TestStatsSummarizerGlobal:
assert stats["total_emails"] == 16 # 8 * 2
assert stats["reports_processed"] == 2
def test_global_trend_includes_volume_and_failure_rate(self, db_session, summarizer):
_seed_recent_trend_records(db_session)
db_session.commit()
stats = summarizer.calculate_summary_statistics(db_session, period_days=7)
assert len(stats["compliance_trend"]) == 2
first_day = stats["compliance_trend"][0]
assert first_day["total"] == 10
assert first_day["volume"] == 10
assert first_day["passed"] == 6
assert first_day["failed"] == 4
assert first_day["rate"] == 60.0
assert first_day["compliance_rate"] == 60.0
assert first_day["failure_rate"] == 40.0
def test_global_trend_respects_period_days(self, db_session, summarizer):
_seed_recent_trend_records(db_session)
db_session.commit()
stats = summarizer.calculate_summary_statistics(db_session, period_days=1)
assert len(stats["compliance_trend"]) == 1
assert stats["compliance_trend"][0]["total"] == 5
class TestStatsSummarizerDomain:
"""Tests for domain-specific statistics."""
@@ -245,6 +347,16 @@ class TestStatsSummarizerDomain:
stats = summarizer.calculate_summary_statistics(db_session, domain_id="example.com")
assert stats["total_emails"] == 8 # Only example.com's data
def test_domain_trend_isolation(self, db_session, summarizer):
_seed_recent_trend_records(db_session, "example.com")
_seed_recent_trend_records(db_session, "other.org")
db_session.commit()
stats = summarizer.calculate_summary_statistics(
db_session, domain_id="example.com", period_days=7
)
assert [point["total"] for point in stats["compliance_trend"]] == [10, 5]
class TestStatsSummarizerCaching:
"""Tests for the caching layer."""
@@ -266,3 +378,13 @@ class TestStatsSummarizerCaching:
# Should recalculate after invalidation
stats = summarizer.calculate_summary_statistics(db_session)
assert stats["total_domains"] == 1
def test_period_days_uses_separate_cache_files(self, db_session, summarizer):
_seed_recent_trend_records(db_session)
db_session.commit()
stats_7_days = summarizer.calculate_summary_statistics(db_session, period_days=7)
stats_1_day = summarizer.calculate_summary_statistics(db_session, period_days=1)
assert len(stats_7_days["compliance_trend"]) == 2
assert len(stats_1_day["compliance_trend"]) == 1
backend/app/utils/stats_summarizer.py
+58 -26
View File
@@ -52,7 +52,10 @@ class StatsSummarizer:
os.makedirs(self.cache_dir, exist_ok=True)
def get_cached_summary(
self, domain_id: Optional[str] = None, max_age_minutes: int = 60
self,
domain_id: Optional[str] = None,
max_age_minutes: int = 60,
period_days: int = 30,
) -> Optional[Dict[str, Any]]:
"""
Get cached summary statistics if available and not too old
@@ -61,11 +64,12 @@ class StatsSummarizer:
domain_id: Optional domain ID to get domain-specific stats
If None, gets global summary
max_age_minutes: Maximum age of cache in minutes
period_days: Number of days used for time-based trend data
Returns:
Cached statistics or None if not available or too old
"""
cache_file = self._get_cache_filename(domain_id)
cache_file = self._get_cache_filename(domain_id, period_days)
try:
if not os.path.exists(cache_file):
@@ -86,18 +90,21 @@ class StatsSummarizer:
logger.warning("Error reading cache file %s: %s", cache_file, str(e))
return None
def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool:
def save_summary(
self, stats: Dict[str, Any], domain_id: Optional[str] = None, period_days: int = 30
) -> bool:
"""
Save summary statistics to cache
Args:
stats: Dictionary of statistics to cache
domain_id: Optional domain ID for domain-specific stats
period_days: Number of days used for time-based trend data
Returns:
True if save was successful, False otherwise
"""
cache_file = self._get_cache_filename(domain_id)
cache_file = self._get_cache_filename(domain_id, period_days)
try:
# Add timestamp
@@ -121,34 +128,39 @@ class StatsSummarizer:
If None, invalidates global summary cache
"""
if domain_id is None:
# Invalidate all caches
cache_file = self._get_cache_filename()
if os.path.exists(cache_file):
os.remove(cache_file)
self._remove_cache_files("global_summary")
else:
# Invalidate specific domain cache
cache_file = self._get_cache_filename(domain_id)
if os.path.exists(cache_file):
os.remove(cache_file)
safe_domain = domain_id.replace(".", "_").replace("/", "_")
self._remove_cache_files(f"domain_{safe_domain}")
def _get_cache_filename(self, domain_id: Optional[str] = None) -> str:
def _remove_cache_files(self, prefix: str) -> None:
"""Remove cached summary files that begin with the provided prefix."""
for filename in os.listdir(self.cache_dir):
if filename.startswith(prefix) and filename.endswith(".json"):
os.remove(os.path.join(self.cache_dir, filename))
def _get_cache_filename(
self, domain_id: Optional[str] = None, period_days: int = 30
) -> str:
"""
Get the filename for a cache file
Args:
domain_id: Optional domain ID for domain-specific cache
period_days: Number of days used for time-based trend data
Returns:
Path to the cache file
"""
period_days = max(1, int(period_days or 30))
if domain_id is None:
return os.path.join(self.cache_dir, "global_summary.json")
return os.path.join(self.cache_dir, f"global_summary_{period_days}d.json")
# Sanitize domain_id to use as filename
safe_domain = domain_id.replace(".", "_").replace("/", "_")
return os.path.join(self.cache_dir, f"domain_{safe_domain}.json")
return os.path.join(self.cache_dir, f"domain_{safe_domain}_{period_days}d.json")
def calculate_summary_statistics(
self, db: Session, domain_id: Optional[str] = None
self, db: Session, domain_id: Optional[str] = None, period_days: int = 30
) -> Dict[str, Any]:
"""
Calculate summary statistics from the database
@@ -156,26 +168,29 @@ class StatsSummarizer:
Args:
db: Database session
domain_id: Optional domain ID to calculate domain-specific stats
period_days: Number of days used for time-based trend data
Returns:
Dictionary with summary statistics
"""
period_days = max(1, int(period_days or 30))
# First check if we have cached stats
cached_stats = self.get_cached_summary(domain_id)
cached_stats = self.get_cached_summary(domain_id, period_days=period_days)
if cached_stats:
return cached_stats
if domain_id is None:
stats = self._calculate_global_statistics(db)
stats = self._calculate_global_statistics(db, period_days)
else:
stats = self._calculate_domain_statistics(db, domain_id)
stats = self._calculate_domain_statistics(db, domain_id, period_days)
# Cache the statistics
self.save_summary(stats, domain_id)
self.save_summary(stats, domain_id, period_days)
return stats
def _calculate_global_statistics(self, db: Session) -> Dict[str, Any]:
def _calculate_global_statistics(self, db: Session, period_days: int = 30) -> Dict[str, Any]:
"""Calculate global statistics across all domains from the database."""
# Count total domains
total_domains = db.query(func.count(Domain.id)).scalar() or 0
@@ -206,7 +221,7 @@ class StatsSummarizer:
top_sources = self._get_top_sources(db)
# Compliance trend over recent days
compliance_trend = self._get_compliance_trend(db)
compliance_trend = self._get_compliance_trend(db, days=period_days)
return {
"total_domains": total_domains,
@@ -218,7 +233,9 @@ class StatsSummarizer:
"compliance_trend": compliance_trend,
}
def _calculate_domain_statistics(self, db: Session, domain_id: str) -> Dict[str, Any]:
def _calculate_domain_statistics(
self, db: Session, domain_id: str, period_days: int = 30
) -> Dict[str, Any]:
"""Calculate statistics for a specific domain from the database."""
# Look up the domain by name
domain = db.query(Domain).filter(Domain.name == domain_id).first()
@@ -266,7 +283,7 @@ class StatsSummarizer:
sources = self._get_domain_sources(db, domain.id)
# Compliance trend for this domain
compliance_trend = self._get_compliance_trend(db, domain.id)
compliance_trend = self._get_compliance_trend(db, domain.id, days=period_days)
return {
"domain": domain_id,
@@ -445,7 +462,22 @@ class StatsSummarizer:
trend = []
for date_str in sorted(daily.keys()):
data = daily[date_str]
rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0
trend.append({"date": date_str, "rate": rate})
total = data["total"]
passed = data["passed"]
failed = max(0, total - passed)
compliance_rate = round((passed / total) * 100, 1) if total > 0 else 0.0
failure_rate = round((failed / total) * 100, 1) if total > 0 else 0.0
trend.append(
{
"date": date_str,
"total": total,
"volume": total,
"passed": passed,
"failed": failed,
"rate": compliance_rate,
"compliance_rate": compliance_rate,
"failure_rate": failure_rate,
}
)
return trend
docs/development/roadmap.md
+3 -1
View File
@@ -41,12 +41,14 @@ Implementation note:
Objective: turn parsed DMARC data into administrator-friendly reports.
Priority tasks:
- Add time-series charts for volume and compliance.
- Add per-domain daily rollups.
- Add "what changed" summaries for newly observed senders and sudden compliance drops.
- Add exportable reports for a domain and date range.
- Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns.
Delivered:
- Dashboard time-series charts show daily mail volume, compliance rate, and failure rate.
Quality bar:
- A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.
docs/milestones.md
+3 -1
View File
@@ -77,8 +77,10 @@ Status: In progress
Goal: convert raw DMARC data into useful operational reporting.
Planned:
Delivered:
- Dashboard trend charts for volume, compliance rate, and failure rate.
Planned:
- Top sender/source reports with pass/fail breakdowns.
- Per-domain report timeline and daily rollups.
- Exportable reports for a selected domain and date range.
docs/todo.md
+4 -4
View File
@@ -128,16 +128,16 @@ This file tracks the specific implementation tasks for each milestone of the DMA
## Milestone 4: Dashboard Enhancements
### Data Visualization
- [ ] Integrate Chart.js library
- [ ] Create time-series charts for DMARC compliance
- [ ] Add volume charts for email traffic
- [x] Integrate Chart.js library
- [x] Create time-series charts for DMARC compliance
- [x] Add volume charts for email traffic
- [ ] Implement sender breakdown visualizations
- [ ] Create policy distribution charts
### Dashboard Widgets
- [ ] Create compliance rate summary widget
- [ ] Add enforcement rate widget
- [ ] Implement email volume trends widget
- [x] Implement email volume trends widget
- [ ] Create top sender sources widget
- [ ] Add alert status summary (for later integration)