gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #107 from christianlouis/codex/domain-daily-rollups

Browse Source 
feat: add domain daily rollups
This commit is contained in:
Christian Krakau-Louis
2026-05-22 21:15:36 +02:00
committed by GitHub
parent b1ff08590f ccc634a10b
commit a15b9fe5d3
6 changed files with 102 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/api_v1/endpoints/domains.py
+26 -5
View File
@@ -67,7 +67,12 @@ class TimelinePoint(BaseModel):
"""Data point for compliance timeline"""
date: str
total: int
volume: int
passed: int
failed: int
compliance_rate: float
failure_rate: float
class ReportEntry(BaseModel):
@@ -464,18 +469,34 @@ def _build_compliance_timeline(store: ReportStore, domain: str) -> List[Timeline
continue
if date_str not in daily_data:
daily_data[date_str] = {"total": 0, "passed": 0}
daily_data[date_str] = {"total": 0, "passed": 0, "failed": 0}
summary = report.get("summary", {})
daily_data[date_str]["total"] += summary.get("total_count", 0)
daily_data[date_str]["passed"] += summary.get("passed_count", 0)
total = summary.get("total_count", 0)
passed = summary.get("passed_count", 0)
failed = summary.get("failed_count", max(0, total - passed))
daily_data[date_str]["total"] += total
daily_data[date_str]["passed"] += passed
daily_data[date_str]["failed"] += failed
# Convert to timeline points sorted by date
timeline = []
for date_str in sorted(daily_data.keys()):
data = daily_data[date_str]
rate = round((data["passed"] / data["total"]) * 100, 1) if data["total"] > 0 else 0.0
timeline.append(TimelinePoint(date=date_str, compliance_rate=rate))
total = data["total"]
compliance_rate = round((data["passed"] / total) * 100, 1) if total > 0 else 0.0
failure_rate = round((data["failed"] / total) * 100, 1) if total > 0 else 0.0
timeline.append(
TimelinePoint(
date=date_str,
total=total,
volume=total,
passed=data["passed"],
failed=data["failed"],
compliance_rate=compliance_rate,
failure_rate=failure_rate,
)
)
return timeline
backend/app/templates/domain_details.html
+57 -3
View File
@@ -116,7 +116,7 @@
{% call card_header() %}
{% call card_title() %}Compliance Over Time{% endcall %}
{% call card_description() %}
DMARC pass rate for the past 30 days
Daily DMARC volume, pass rate, and failure rate
{% endcall %}
{% endcall %}
{% call card_content() %}
@@ -636,7 +636,7 @@ function domainDetailsApp(domainId) {
},
initComplianceChart(timelineData) {
if (!timelineData) return;
if (!timelineData || !timelineData.length) return;
const ctx = document.getElementById('compliance-chart').getContext('2d');
@@ -646,6 +646,8 @@ function domainDetailsApp(domainId) {
const labels = timelineData.map(item => item.date);
const complianceData = timelineData.map(item => item.compliance_rate);
const failureData = timelineData.map(item => item.failure_rate || 0);
const volumeData = timelineData.map(item => item.volume || item.total || 0);
// Calculate the threshold line data (recommended 98% for policy advancement)
const thresholdData = Array(labels.length).fill(98);
@@ -658,6 +660,7 @@ function domainDetailsApp(domainId) {
{
label: 'Compliance Rate',
data: complianceData,
yAxisID: 'yRate',
borderColor: 'rgb(59, 130, 246)', // blue-500
backgroundColor: 'rgba(59, 130, 246, 0.1)',
tension: 0.4,
@@ -666,9 +669,33 @@ function domainDetailsApp(domainId) {
pointRadius: 3,
pointHoverRadius: 5
},
{
label: 'Failure Rate',
data: failureData,
yAxisID: 'yRate',
borderColor: 'rgb(220, 38, 38)',
backgroundColor: 'rgba(220, 38, 38, 0.08)',
tension: 0.4,
fill: false,
pointBackgroundColor: 'rgb(220, 38, 38)',
pointRadius: 3,
pointHoverRadius: 5
},
{
label: 'Message Volume',
type: 'bar',
data: volumeData,
yAxisID: 'yVolume',
backgroundColor: 'rgba(107, 114, 128, 0.22)',
borderColor: 'rgba(107, 114, 128, 0.5)',
borderWidth: 1,
borderRadius: 4,
maxBarThickness: 28
},
{
label: 'Recommended Threshold (98%)',
data: thresholdData,
yAxisID: 'yRate',
borderColor: 'rgba(220, 38, 38, 0.6)', // red-600 with opacity
borderDash: [5, 5],
pointRadius: 0,
@@ -682,7 +709,9 @@ function domainDetailsApp(domainId) {
responsive: true,
maintainAspectRatio: false,
scales: {
y: {
yRate: {
type: 'linear',
position: 'left',
beginAtZero: false,
min: Math.max(0, Math.min(...complianceData) - 10), // Dynamic min value
max: 100,
@@ -700,6 +729,24 @@ function domainDetailsApp(domainId) {
color: 'rgba(0, 0, 0, 0.05)'
}
},
yVolume: {
type: 'linear',
position: 'right',
beginAtZero: true,
ticks: {
precision: 0
},
title: {
display: true,
text: 'Messages',
font: {
weight: 'bold'
}
},
grid: {
drawOnChartArea: false
}
},
x: {
title: {
display: true,
@@ -728,6 +775,12 @@ function domainDetailsApp(domainId) {
if (context.dataset.label === 'Compliance Rate') {
return `Compliance: ${context.parsed.y}%`;
}
if (context.dataset.label === 'Failure Rate') {
return `Failures: ${context.parsed.y}%`;
}
if (context.dataset.label === 'Message Volume') {
return `Messages: ${context.parsed.y}`;
}
return context.dataset.label;
},
title: function(context) {
@@ -747,6 +800,7 @@ function domainDetailsApp(domainId) {
annotations: {
box1: {
type: 'box',
yScaleID: 'yRate',
yMin: 90,
yMax: 100,
backgroundColor: 'rgba(34, 197, 94, 0.05)',
backend/app/tests/test_dashboard_timeline.py
+16
View File
@@ -42,6 +42,11 @@ class TestComplianceTimeline:
data = response.json()
timeline = data["compliance_timeline"]
assert len(timeline) >= 1
assert timeline[0]["total"] == 10
assert timeline[0]["volume"] == 10
assert timeline[0]["passed"] == 8
assert timeline[0]["failed"] == 2
assert timeline[0]["failure_rate"] == 20.0
def test_timeline_uses_real_dates(self, client: TestClient):
"""Timeline dates should come from actual report begin_dates."""
@@ -108,7 +113,11 @@ class TestComplianceTimeline:
timeline = _build_compliance_timeline(store, "isodate.com")
assert len(timeline) == 1
assert timeline[0].date == "2020-08-15"
assert timeline[0].total == 10
assert timeline[0].passed == 9
assert timeline[0].failed == 1
assert timeline[0].compliance_rate == 90.0
assert timeline[0].failure_rate == 10.0
class TestBuildComplianceTimelineMultipleReports:
@@ -126,8 +135,12 @@ class TestBuildComplianceTimelineMultipleReports:
assert len(timeline) == 1
assert timeline[0]["date"] == "2020-08-15"
assert timeline[0]["total"] == 20
assert timeline[0]["passed"] == 14
assert timeline[0]["failed"] == 6
# Aggregated: 14 passed out of 20 total = 70%
assert timeline[0]["compliance_rate"] == 70.0
assert timeline[0]["failure_rate"] == 30.0
def test_reports_on_different_days(self, client: TestClient):
"""Reports on different days should produce separate timeline points."""
@@ -142,6 +155,9 @@ class TestBuildComplianceTimelineMultipleReports:
assert len(timeline) == 2
# Sorted by date
assert timeline[0]["date"] == "2020-08-15"
assert timeline[0]["total"] == 10
assert timeline[0]["compliance_rate"] == 100.0
assert timeline[1]["date"] == "2020-08-16"
assert timeline[1]["total"] == 10
assert timeline[1]["compliance_rate"] == 50.0
assert timeline[1]["failure_rate"] == 50.0
docs/development/roadmap.md
+1 -1
View File
@@ -41,7 +41,6 @@ Implementation note:
Objective: turn parsed DMARC data into administrator-friendly reports.
Priority tasks:
- Add per-domain daily rollups.
- Add "what changed" summaries for newly observed senders and sudden compliance drops.
- Add exportable reports for a domain and date range.
- Add actionable recommendations for common SPF, DKIM, and DMARC failure patterns.
@@ -49,6 +48,7 @@ Priority tasks:
Delivered:
- Dashboard time-series charts show daily mail volume, compliance rate, and failure rate.
- Top sending sources show DMARC, SPF, and DKIM pass/fail breakdowns on the dashboard.
- Per-domain timelines include daily volume, pass, fail, compliance-rate, and failure-rate rollups.
Quality bar:
- A domain owner can understand who sends mail as their domain, which sources fail, and what to fix next.
docs/milestones.md
+1 -1
View File
@@ -80,9 +80,9 @@ Goal: convert raw DMARC data into useful operational reporting.
Delivered:
- Dashboard trend charts for volume, compliance rate, and failure rate.
- Top sender/source reports with pass/fail breakdowns.
- Per-domain report timeline and daily rollups.
Planned:
- Per-domain report timeline and daily rollups.
- Exportable reports for a selected domain and date range.
- Clear recommendations for common cases: unknown source, SPF-only pass, DKIM-only pass, full fail, and policy not enforced.
docs/todo.md
+1 -1
View File
@@ -148,7 +148,7 @@ This file tracks the specific implementation tasks for each milestone of the DMA
- [ ] Implement data comparison features
### Meaningful Reports
- [ ] Add per-domain daily rollups
- [x] Add per-domain daily rollups
- [x] Add sender/source pass/fail totals
- [ ] Add newly observed source detection
- [ ] Add exportable domain reports