From ccc634a10b3826eccc9c3bf9db8b64a9c6e772ab Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 22 May 2026 21:12:43 +0200 Subject: [PATCH] feat: add domain daily rollups --- backend/app/api/api_v1/endpoints/domains.py | 31 ++++++++-- backend/app/templates/domain_details.html | 60 +++++++++++++++++++- backend/app/tests/test_dashboard_timeline.py | 16 ++++++ docs/development/roadmap.md | 2 +- docs/milestones.md | 2 +- docs/todo.md | 2 +- 6 files changed, 102 insertions(+), 11 deletions(-) diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py index 5fa0453..87aa23a 100644 --- a/backend/app/api/api_v1/endpoints/domains.py +++ b/backend/app/api/api_v1/endpoints/domains.py @@ -67,7 +67,12 @@ class TimelinePoint(BaseModel): """Data point for compliance timeline""" date: str + total: int + volume: int + passed: int + failed: int compliance_rate: float + failure_rate: float class ReportEntry(BaseModel): @@ -464,18 +469,34 @@ def _build_compliance_timeline(store: ReportStore, domain: str) -> List[Timeline continue if date_str not in daily_data: - daily_data[date_str] = {"total": 0, "passed": 0} + daily_data[date_str] = {"total": 0, "passed": 0, "failed": 0} summary = report.get("summary", {}) - daily_data[date_str]["total"] += summary.get("total_count", 0) - daily_data[date_str]["passed"] += summary.get("passed_count", 0) + total = summary.get("total_count", 0) + passed = summary.get("passed_count", 0) + failed = summary.get("failed_count", max(0, total - passed)) + daily_data[date_str]["total"] += total + daily_data[date_str]["passed"] += passed + daily_data[date_str]["failed"] += failed # Convert to timeline points sorted by date timeline = [] for date_str in sorted(daily_data.keys()): data = daily_data[date_str] - rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0 - timeline.append(TimelinePoint(date=date_str, compliance_rate=rate)) + total = data["total"] + compliance_rate = round((data["passed"] / total) * 100, 1) if total > 0 else 0.0 + failure_rate = round((data["failed"] / total) * 100, 1) if total > 0 else 0.0 + timeline.append( + TimelinePoint( + date=date_str, + total=total, + volume=total, + passed=data["passed"], + failed=data["failed"], + compliance_rate=compliance_rate, + failure_rate=failure_rate, + ) + ) return timeline diff --git a/backend/app/templates/domain_details.html b/backend/app/templates/domain_details.html index 5eb73c9..fbb38a0 100644 --- a/backend/app/templates/domain_details.html +++ b/backend/app/templates/domain_details.html @@ -116,7 +116,7 @@ {% call card_header() %} {% call card_title() %}Compliance Over Time{% endcall %} {% call card_description() %} - DMARC pass rate for the past 30 days + Daily DMARC volume, pass rate, and failure rate {% endcall %} {% endcall %} {% call card_content() %} @@ -636,7 +636,7 @@ function domainDetailsApp(domainId) { }, initComplianceChart(timelineData) { - if (!timelineData) return; + if (!timelineData || !timelineData.length) return; const ctx = document.getElementById('compliance-chart').getContext('2d'); @@ -646,6 +646,8 @@ function domainDetailsApp(domainId) { const labels = timelineData.map(item => item.date); const complianceData = timelineData.map(item => item.compliance_rate); + const failureData = timelineData.map(item => item.failure_rate || 0); + const volumeData = timelineData.map(item => item.volume || item.total || 0); // Calculate the threshold line data (recommended 98% for policy advancement) const thresholdData = Array(labels.length).fill(98); @@ -658,6 +660,7 @@ function domainDetailsApp(domainId) { { label: 'Compliance Rate', data: complianceData, + yAxisID: 'yRate', borderColor: 'rgb(59, 130, 246)', // blue-500 backgroundColor: 'rgba(59, 130, 246, 0.1)', tension: 0.4, @@ -666,9 +669,33 @@ function domainDetailsApp(domainId) { pointRadius: 3, pointHoverRadius: 5 }, + { + label: 'Failure Rate', + data: failureData, + yAxisID: 'yRate', + borderColor: 'rgb(220, 38, 38)', + backgroundColor: 'rgba(220, 38, 38, 0.08)', + tension: 0.4, + fill: false, + pointBackgroundColor: 'rgb(220, 38, 38)', + pointRadius: 3, + pointHoverRadius: 5 + }, + { + label: 'Message Volume', + type: 'bar', + data: volumeData, + yAxisID: 'yVolume', + backgroundColor: 'rgba(107, 114, 128, 0.22)', + borderColor: 'rgba(107, 114, 128, 0.5)', + borderWidth: 1, + borderRadius: 4, + maxBarThickness: 28 + }, { label: 'Recommended Threshold (98%)', data: thresholdData, + yAxisID: 'yRate', borderColor: 'rgba(220, 38, 38, 0.6)', // red-600 with opacity borderDash: [5, 5], pointRadius: 0, @@ -682,7 +709,9 @@ function domainDetailsApp(domainId) { responsive: true, maintainAspectRatio: false, scales: { - y: { + yRate: { + type: 'linear', + position: 'left', beginAtZero: false, min: Math.max(0, Math.min(...complianceData) - 10), // Dynamic min value max: 100, @@ -700,6 +729,24 @@ function domainDetailsApp(domainId) { color: 'rgba(0, 0, 0, 0.05)' } }, + yVolume: { + type: 'linear', + position: 'right', + beginAtZero: true, + ticks: { + precision: 0 + }, + title: { + display: true, + text: 'Messages', + font: { + weight: 'bold' + } + }, + grid: { + drawOnChartArea: false + } + }, x: { title: { display: true, @@ -728,6 +775,12 @@ function domainDetailsApp(domainId) { if (context.dataset.label === 'Compliance Rate') { return `Compliance: ${context.parsed.y}%`; } + if (context.dataset.label === 'Failure Rate') { + return `Failures: ${context.parsed.y}%`; + } + if (context.dataset.label === 'Message Volume') { + return `Messages: ${context.parsed.y}`; + } return context.dataset.label; }, title: function(context) { @@ -747,6 +800,7 @@ function domainDetailsApp(domainId) { annotations: { box1: { type: 'box', + yScaleID: 'yRate', yMin: 90, yMax: 100, backgroundColor: 'rgba(34, 197, 94, 0.05)', diff --git a/backend/app/tests/test_dashboard_timeline.py b/backend/app/tests/test_dashboard_timeline.py index a245f4e..3559546 100644 --- a/backend/app/tests/test_dashboard_timeline.py +++ b/backend/app/tests/test_dashboard_timeline.py @@ -42,6 +42,11 @@ class TestComplianceTimeline: data = response.json() timeline = data["compliance_timeline"] assert len(timeline) >= 1 + assert timeline[0]["total"] == 10 + assert timeline[0]["volume"] == 10 + assert timeline[0]["passed"] == 8 + assert timeline[0]["failed"] == 2 + assert timeline[0]["failure_rate"] == 20.0 def test_timeline_uses_real_dates(self, client: TestClient): """Timeline dates should come from actual report begin_dates.""" @@ -108,7 +113,11 @@ class TestComplianceTimeline: timeline = _build_compliance_timeline(store, "isodate.com") assert len(timeline) == 1 assert timeline[0].date == "2020-08-15" + assert timeline[0].total == 10 + assert timeline[0].passed == 9 + assert timeline[0].failed == 1 assert timeline[0].compliance_rate == 90.0 + assert timeline[0].failure_rate == 10.0 class TestBuildComplianceTimelineMultipleReports: @@ -126,8 +135,12 @@ class TestBuildComplianceTimelineMultipleReports: assert len(timeline) == 1 assert timeline[0]["date"] == "2020-08-15" + assert timeline[0]["total"] == 20 + assert timeline[0]["passed"] == 14 + assert timeline[0]["failed"] == 6 # Aggregated: 14 passed out of 20 total = 70% assert timeline[0]["compliance_rate"] == 70.0 + assert timeline[0]["failure_rate"] == 30.0 def test_reports_on_different_days(self, client: TestClient): """Reports on different days should produce separate timeline points.""" @@ -142,6 +155,9 @@ class TestBuildComplianceTimelineMultipleReports: assert len(timeline) == 2 # Sorted by date assert timeline[0]["date"] == "2020-08-15" + assert timeline[0]["total"] == 10 assert timeline[0]["compliance_rate"] == 100.0 assert timeline[1]["date"] == "2020-08-16" + assert timeline[1]["total"] == 10 assert timeline[1]["compliance_rate"] == 50.0 + assert timeline[1]["failure_rate"] == 50.0 diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index 1762381..09baab9 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -41,7 +41,6 @@ Implementation note: Objective: turn parsed DMARC data into administrator-friendly reports. Priority tasks: -- Add per-domain daily rollups. - Add "what changed" summaries for newly observed senders and sudden compliance drops. - Add exportable reports for a domain and date range. - Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns. @@ -49,6 +48,7 @@ Priority tasks: Delivered: - Dashboard time-series charts show daily mail volume, compliance rate, and failure rate. - Top sending sources show DMARC, SPF, and DKIM pass/fail breakdowns on the dashboard. +- Per-domain timelines include daily volume, pass, fail, compliance-rate, and failure-rate rollups. Quality bar: - A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next. diff --git a/docs/milestones.md b/docs/milestones.md index 86a7265..5475e7b 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -80,9 +80,9 @@ Goal: convert raw DMARC data into useful operational reporting. Delivered: - Dashboard trend charts for volume, compliance rate, and failure rate. - Top sender/source reports with pass/fail breakdowns. +- Per-domain report timeline and daily rollups. Planned: -- Per-domain report timeline and daily rollups. - Exportable reports for a selected domain and date range. - Clear recommendations for common cases: unknown source, SPF-only pass, DKIM-only pass, full fail, and policy not enforced. diff --git a/docs/todo.md b/docs/todo.md index e715a5d..85b5f2b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -148,7 +148,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA - [ ] Implement data comparison features ### Meaningful Reports -- [ ] Add per-domain daily rollups +- [x] Add per-domain daily rollups - [x] Add sender/source pass/fail totals - [ ] Add newly observed source detection - [ ] Add exportable domain reports