Merge pull request #107 from christianlouis/codex/domain-daily-rollups

feat: add domain daily rollups

backend/app/api/api_v1/endpoints/domains.py

@@ -67,7 +67,12 @@ class TimelinePoint(BaseModel):
     """Data point for compliance timeline"""
 
     date: str
+    total: int
+    volume: int
+    passed: int
+    failed: int
     compliance_rate: float
+    failure_rate: float
 
 
 class ReportEntry(BaseModel):
@@ -464,18 +469,34 @@ def _build_compliance_timeline(store: ReportStore, domain: str) -> List[Timeline
             continue
 
         if date_str not in daily_data:
-            daily_data[date_str] = {"total": 0, "passed": 0}
+            daily_data[date_str] = {"total": 0, "passed": 0, "failed": 0}
 
         summary = report.get("summary", {})
-        daily_data[date_str]["total"] += summary.get("total_count", 0)
+        total = summary.get("total_count", 0)
-        daily_data[date_str]["passed"] += summary.get("passed_count", 0)
+        passed = summary.get("passed_count", 0)
+        failed = summary.get("failed_count", max(0, total - passed))
+        daily_data[date_str]["total"] += total
+        daily_data[date_str]["passed"] += passed
+        daily_data[date_str]["failed"] += failed
 
     # Convert to timeline points sorted by date
     timeline = []
     for date_str in sorted(daily_data.keys()):
         data = daily_data[date_str]
-        rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0
+        total = data["total"]
-        timeline.append(TimelinePoint(date=date_str, compliance_rate=rate))
+        compliance_rate = round((data["passed"] / total) * 100, 1) if total > 0 else 0.0
+        failure_rate = round((data["failed"] / total) * 100, 1) if total > 0 else 0.0
+        timeline.append(
+            TimelinePoint(
+                date=date_str,
+                total=total,
+                volume=total,
+                passed=data["passed"],
+                failed=data["failed"],
+                compliance_rate=compliance_rate,
+                failure_rate=failure_rate,
+            )
+        )
 
     return timeline
 

backend/app/templates/domain_details.html

@@ -116,7 +116,7 @@
             {% call card_header() %}
                 {% call card_title() %}Compliance Over Time{% endcall %}
                 {% call card_description() %}
-                    DMARC pass rate for the past 30 days
+                    Daily DMARC volume, pass rate, and failure rate
                 {% endcall %}
             {% endcall %}
             {% call card_content() %}
@@ -636,7 +636,7 @@ function domainDetailsApp(domainId) {
         },
         
         initComplianceChart(timelineData) {
-            if (!timelineData) return;
+            if (!timelineData || !timelineData.length) return;
             
             const ctx = document.getElementById('compliance-chart').getContext('2d');
             
@@ -646,6 +646,8 @@ function domainDetailsApp(domainId) {
             
             const labels = timelineData.map(item => item.date);
             const complianceData = timelineData.map(item => item.compliance_rate);
+            const failureData = timelineData.map(item => item.failure_rate || 0);
+            const volumeData = timelineData.map(item => item.volume || item.total || 0);
             
             // Calculate the threshold line data (recommended 98% for policy advancement)
             const thresholdData = Array(labels.length).fill(98);
@@ -658,6 +660,7 @@ function domainDetailsApp(domainId) {
                         {
                             label: 'Compliance Rate',
                             data: complianceData,
+                            yAxisID: 'yRate',
                             borderColor: 'rgb(59, 130, 246)', // blue-500
                             backgroundColor: 'rgba(59, 130, 246, 0.1)',
                             tension: 0.4,
@@ -666,9 +669,33 @@ function domainDetailsApp(domainId) {
                             pointRadius: 3,
                             pointHoverRadius: 5
                         },
+                        {
+                            label: 'Failure Rate',
+                            data: failureData,
+                            yAxisID: 'yRate',
+                            borderColor: 'rgb(220, 38, 38)',
+                            backgroundColor: 'rgba(220, 38, 38, 0.08)',
+                            tension: 0.4,
+                            fill: false,
+                            pointBackgroundColor: 'rgb(220, 38, 38)',
+                            pointRadius: 3,
+                            pointHoverRadius: 5
+                        },
+                        {
+                            label: 'Message Volume',
+                            type: 'bar',
+                            data: volumeData,
+                            yAxisID: 'yVolume',
+                            backgroundColor: 'rgba(107, 114, 128, 0.22)',
+                            borderColor: 'rgba(107, 114, 128, 0.5)',
+                            borderWidth: 1,
+                            borderRadius: 4,
+                            maxBarThickness: 28
+                        },
                         {
                             label: 'Recommended Threshold (98%)',
                             data: thresholdData,
+                            yAxisID: 'yRate',
                             borderColor: 'rgba(220, 38, 38, 0.6)', // red-600 with opacity
                             borderDash: [5, 5],
                             pointRadius: 0,
@@ -682,7 +709,9 @@ function domainDetailsApp(domainId) {
                     responsive: true,
                     maintainAspectRatio: false,
                     scales: {
-                        y: {
+                        yRate: {
+                            type: 'linear',
+                            position: 'left',
                             beginAtZero: false,
                             min: Math.max(0, Math.min(...complianceData) - 10), // Dynamic min value
                             max: 100,
@@ -700,6 +729,24 @@ function domainDetailsApp(domainId) {
                                 color: 'rgba(0, 0, 0, 0.05)'
                             }
                         },
+                        yVolume: {
+                            type: 'linear',
+                            position: 'right',
+                            beginAtZero: true,
+                            ticks: {
+                                precision: 0
+                            },
+                            title: {
+                                display: true,
+                                text: 'Messages',
+                                font: {
+                                    weight: 'bold'
+                                }
+                            },
+                            grid: {
+                                drawOnChartArea: false
+                            }
+                        },
                         x: {
                             title: {
                                 display: true,
@@ -728,6 +775,12 @@ function domainDetailsApp(domainId) {
                                     if (context.dataset.label === 'Compliance Rate') {
                                         return `Compliance: ${context.parsed.y}%`;
                                     }
+                                    if (context.dataset.label === 'Failure Rate') {
+                                        return `Failures: ${context.parsed.y}%`;
+                                    }
+                                    if (context.dataset.label === 'Message Volume') {
+                                        return `Messages: ${context.parsed.y}`;
+                                    }
                                     return context.dataset.label;
                                 },
                                 title: function(context) {
@@ -747,6 +800,7 @@ function domainDetailsApp(domainId) {
                             annotations: {
                                 box1: {
                                     type: 'box',
+                                    yScaleID: 'yRate',
                                     yMin: 90,
                                     yMax: 100,
                                     backgroundColor: 'rgba(34, 197, 94, 0.05)',

backend/app/tests/test_dashboard_timeline.py

@@ -42,6 +42,11 @@ class TestComplianceTimeline:
         data = response.json()
         timeline = data["compliance_timeline"]
         assert len(timeline) >= 1
+        assert timeline[0]["total"] == 10
+        assert timeline[0]["volume"] == 10
+        assert timeline[0]["passed"] == 8
+        assert timeline[0]["failed"] == 2
+        assert timeline[0]["failure_rate"] == 20.0
 
     def test_timeline_uses_real_dates(self, client: TestClient):
         """Timeline dates should come from actual report begin_dates."""
@@ -108,7 +113,11 @@ class TestComplianceTimeline:
         timeline = _build_compliance_timeline(store, "isodate.com")
         assert len(timeline) == 1
         assert timeline[0].date == "2020-08-15"
+        assert timeline[0].total == 10
+        assert timeline[0].passed == 9
+        assert timeline[0].failed == 1
         assert timeline[0].compliance_rate == 90.0
+        assert timeline[0].failure_rate == 10.0
 
 
 class TestBuildComplianceTimelineMultipleReports:
@@ -126,8 +135,12 @@ class TestBuildComplianceTimelineMultipleReports:
 
         assert len(timeline) == 1
         assert timeline[0]["date"] == "2020-08-15"
+        assert timeline[0]["total"] == 20
+        assert timeline[0]["passed"] == 14
+        assert timeline[0]["failed"] == 6
         # Aggregated: 14 passed out of 20 total = 70%
         assert timeline[0]["compliance_rate"] == 70.0
+        assert timeline[0]["failure_rate"] == 30.0
 
     def test_reports_on_different_days(self, client: TestClient):
         """Reports on different days should produce separate timeline points."""
@@ -142,6 +155,9 @@ class TestBuildComplianceTimelineMultipleReports:
         assert len(timeline) == 2
         # Sorted by date
         assert timeline[0]["date"] == "2020-08-15"
+        assert timeline[0]["total"] == 10
         assert timeline[0]["compliance_rate"] == 100.0
         assert timeline[1]["date"] == "2020-08-16"
+        assert timeline[1]["total"] == 10
         assert timeline[1]["compliance_rate"] == 50.0
+        assert timeline[1]["failure_rate"] == 50.0

docs/development/roadmap.md

@@ -41,7 +41,6 @@ Implementation note:
 Objective: turn parsed DMARC data into administrator-friendly reports.
 
 Priority tasks:
-- Add per-domain daily rollups.
 - Add "what changed" summaries for newly observed senders and sudden compliance drops.
 - Add exportable reports for a domain and date range.
 - Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns.
@@ -49,6 +48,7 @@ Priority tasks:
 Delivered:
 - Dashboard time-series charts show daily mail volume, compliance rate, and failure rate.
 - Top sending sources show DMARC, SPF, and DKIM pass/fail breakdowns on the dashboard.
+- Per-domain timelines include daily volume, pass, fail, compliance-rate, and failure-rate rollups.
 
 Quality bar:
 - A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.

docs/milestones.md

@@ -80,9 +80,9 @@ Goal: convert raw DMARC data into useful operational reporting.
 Delivered:
 - Dashboard trend charts for volume, compliance rate, and failure rate.
 - Top sender/source reports with pass/fail breakdowns.
+- Per-domain report timeline and daily rollups.
 
 Planned:
-- Per-domain report timeline and daily rollups.
 - Exportable reports for a selected domain and date range.
 - Clear recommendations for common cases: unknown source, SPF-only pass, DKIM-only pass, full fail, and policy not enforced.
 

docs/todo.md

@@ -148,7 +148,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA
 - [ ] Implement data comparison features
 
 ### Meaningful Reports
-- [ ] Add per-domain daily rollups
+- [x] Add per-domain daily rollups
 - [x] Add sender/source pass/fail totals
 - [ ] Add newly observed source detection
 - [ ] Add exportable domain reports