Christian Krakau-Louis
1630 lines
55 KiB
Python
1630 lines
55 KiB
Python
import asyncio
|
|
import csv
|
|
import io
|
|
import ipaddress
|
|
import logging
|
|
from datetime import date, datetime, timezone
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
|
|
from fastapi.responses import Response
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.database import get_db
|
|
from app.core.security import require_admin_auth
|
|
from app.models.domain import Domain
|
|
from app.services.cloudflare_dns import (
|
|
analyze_dns_records,
|
|
discover_cloudflare_zones,
|
|
get_zone_for_domain,
|
|
import_cloudflare_domains,
|
|
list_dns_record_changes,
|
|
sync_dns_record_changes,
|
|
)
|
|
from app.services.dns_cache import resolve_domain_dns_cached
|
|
from app.services.dns_resolver import (
|
|
DomainDNSResult,
|
|
extract_dmarc_policy,
|
|
get_default_provider,
|
|
)
|
|
from app.services.mta_sts import MTAStsResult, check_mta_sts_cached
|
|
from app.services.report_persistence import (
|
|
delete_persisted_domain,
|
|
hydrate_report_store_from_db,
|
|
)
|
|
from app.services.report_store import ReportStore
|
|
from app.utils.domain_validator import validate_domain_config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
DOMAIN_SELECTOR_LOOKUP_CHUNK_SIZE = 500
|
|
|
|
|
|
class DomainBase(BaseModel):
|
|
"""Base Domain schema"""
|
|
|
|
name: str
|
|
description: Optional[str] = None
|
|
policy: Optional[str] = None
|
|
|
|
|
|
class DomainResponse(DomainBase):
|
|
"""Domain response schema"""
|
|
|
|
reports_count: int = 0
|
|
emails_count: int = 0
|
|
compliance_rate: float = 0.0
|
|
|
|
|
|
class DomainCreate(BaseModel):
|
|
"""Payload for creating a monitored domain."""
|
|
|
|
name: str
|
|
description: Optional[str] = None
|
|
dkim_selectors: Optional[List[str]] = None
|
|
|
|
|
|
class DomainStatsResponse(BaseModel):
|
|
"""Domain statistics for the domain details page"""
|
|
|
|
complianceRate: float
|
|
totalEmails: int
|
|
failedEmails: int
|
|
reportCount: int
|
|
|
|
|
|
class DNSRecordResponse(BaseModel):
|
|
"""DNS record information for a domain"""
|
|
|
|
dmarc: bool
|
|
dmarcRecord: Optional[str] = None
|
|
spf: bool
|
|
spfRecord: Optional[str] = None
|
|
dkim: bool
|
|
dkimSelectors: List[str] = []
|
|
cached: bool = False
|
|
checkedAt: Optional[str] = None
|
|
|
|
|
|
class DNSHealthEvidence(BaseModel):
|
|
"""Evidence backing a DNS health recommendation."""
|
|
|
|
label: str
|
|
value: str
|
|
href: str
|
|
|
|
|
|
class DNSHealthCheck(BaseModel):
|
|
"""Single DNS health check result."""
|
|
|
|
key: str
|
|
label: str
|
|
status: str
|
|
message: str
|
|
evidence: List[DNSHealthEvidence] = Field(default_factory=list)
|
|
|
|
|
|
class DNSHealthRecommendation(BaseModel):
|
|
"""Actionable DNS or enforcement recommendation."""
|
|
|
|
type: str
|
|
severity: str
|
|
title: str
|
|
detail: str
|
|
action: str
|
|
evidence: List[DNSHealthEvidence] = Field(default_factory=list)
|
|
|
|
|
|
class DNSHealthResponse(BaseModel):
|
|
"""Evidence-linked DNS health summary for a domain."""
|
|
|
|
status: str
|
|
policy: str
|
|
compliance_rate: float
|
|
total_emails: int
|
|
failed_emails: int
|
|
checks: List[DNSHealthCheck]
|
|
recommendations: List[DNSHealthRecommendation]
|
|
|
|
|
|
class MTAStsResponse(BaseModel):
|
|
"""MTA-STS posture result for a domain."""
|
|
|
|
status: str
|
|
dns_record: Optional[str] = None
|
|
policy_url: Optional[str] = None
|
|
policy_text: Optional[str] = None
|
|
mode: Optional[str] = None
|
|
max_age: Optional[int] = None
|
|
mx: List[str] = Field(default_factory=list)
|
|
errors: List[str] = Field(default_factory=list)
|
|
warnings: List[str] = Field(default_factory=list)
|
|
cached: bool = False
|
|
checked_at: Optional[str] = None
|
|
|
|
|
|
class CloudflareZoneResponse(BaseModel):
|
|
"""Cloudflare zone available for import."""
|
|
|
|
id: str
|
|
name: str
|
|
status: Optional[str] = None
|
|
account_name: Optional[str] = None
|
|
imported: bool = False
|
|
|
|
|
|
class CloudflareImportRequest(BaseModel):
|
|
"""Optional list of Cloudflare domains to import."""
|
|
|
|
domains: Optional[List[str]] = None
|
|
|
|
|
|
class CloudflareImportResponse(BaseModel):
|
|
"""Cloudflare domain import summary."""
|
|
|
|
imported: List[str]
|
|
existing: List[str]
|
|
skipped: List[str]
|
|
total_discovered: int
|
|
|
|
|
|
class CloudflareDNSAnalysisResponse(BaseModel):
|
|
"""Cloudflare-managed DNS analysis and recent change details."""
|
|
|
|
zone: Dict[str, Any]
|
|
records: List[Dict[str, Any]]
|
|
checks: Dict[str, Any]
|
|
suggestions: List[Dict[str, str]]
|
|
changes: List[Dict[str, Any]]
|
|
history: List[Dict[str, Any]]
|
|
|
|
|
|
class DNSChangeHistoryResponse(BaseModel):
|
|
"""Recent DNS record changes for a domain."""
|
|
|
|
history: List[Dict[str, Any]]
|
|
|
|
|
|
class TimelinePoint(BaseModel):
|
|
"""Data point for compliance timeline"""
|
|
|
|
date: str
|
|
total: int
|
|
volume: int
|
|
passed: int
|
|
failed: int
|
|
compliance_rate: float
|
|
failure_rate: float
|
|
|
|
|
|
class ReportEntry(BaseModel):
|
|
"""Summary of a DMARC report"""
|
|
|
|
id: str
|
|
org_name: str
|
|
begin_date: int
|
|
end_date: int
|
|
total_emails: int
|
|
pass_rate: float
|
|
policy: str
|
|
|
|
|
|
class SourceRecommendation(BaseModel):
|
|
"""Actionable recommendation for a sending source"""
|
|
|
|
type: str
|
|
severity: str
|
|
title: str
|
|
detail: str
|
|
action: str
|
|
|
|
|
|
class SourceEntry(BaseModel):
|
|
"""Summary of a sending source"""
|
|
|
|
ip: str
|
|
count: int
|
|
spf: str
|
|
dkim: str
|
|
dmarc: str
|
|
disposition: str
|
|
spf_pass_count: int = 0
|
|
spf_fail_count: int = 0
|
|
dkim_pass_count: int = 0
|
|
dkim_fail_count: int = 0
|
|
dmarc_pass_count: int = 0
|
|
dmarc_fail_count: int = 0
|
|
disposition_counts: Dict[str, int] = Field(default_factory=dict)
|
|
hostname: Optional[str] = None
|
|
spf_fix_hint: Optional[str] = None
|
|
recommendations: List[SourceRecommendation] = Field(default_factory=list)
|
|
|
|
|
|
class DomainReportsResponse(BaseModel):
|
|
"""Domain reports with compliance timeline"""
|
|
|
|
reports: List[ReportEntry]
|
|
compliance_timeline: List[TimelinePoint]
|
|
|
|
|
|
class DomainSourcesResponse(BaseModel):
|
|
"""Domain sending sources"""
|
|
|
|
sources: List[SourceEntry]
|
|
|
|
|
|
class DomainSummaryResponse(BaseModel):
|
|
"""Domain summary for dashboard"""
|
|
|
|
total_domains: int
|
|
total_emails: int
|
|
overall_pass_rate: float
|
|
reports_processed: int
|
|
domains: List[Dict[str, Any]]
|
|
|
|
|
|
class SelectorRequest(BaseModel):
|
|
"""Request body for adding a DKIM selector"""
|
|
|
|
selector: str = Field(..., min_length=1, description="DKIM selector name")
|
|
|
|
|
|
def _get_selectors_from_reports(store: "ReportStore", domain: str) -> List[str]:
|
|
"""Extract DKIM selectors seen in stored DMARC reports for *domain*.
|
|
|
|
DMARC aggregate report records include DKIM auth results that carry the
|
|
selector used by the sending server. Collecting these gives us a set of
|
|
real-world selectors to verify against live DNS, in addition to any
|
|
manually configured selectors.
|
|
"""
|
|
selectors: List[str] = []
|
|
for report in store.get_domain_reports(domain):
|
|
for record in report.get("records", []):
|
|
dkim_entries = record.get("dkim") or []
|
|
for dkim_entry in dkim_entries:
|
|
if not isinstance(dkim_entry, dict):
|
|
continue
|
|
sel = dkim_entry.get("selector", "").strip()
|
|
if sel and sel not in selectors:
|
|
selectors.append(sel)
|
|
return selectors
|
|
|
|
|
|
def _get_domain_selectors_from_db(db: Session, domain_name: str) -> List[str]:
|
|
"""Return the manually configured DKIM selectors for *domain_name* from the DB."""
|
|
domain_db = db.query(Domain).filter(Domain.name == domain_name).first()
|
|
if domain_db and domain_db.dkim_selectors:
|
|
return [s.strip() for s in domain_db.dkim_selectors.split(",") if s.strip()]
|
|
return []
|
|
|
|
|
|
def _get_domain_selectors_map_from_db(db: Session, domain_names: List[str]) -> Dict[str, List[str]]:
|
|
"""Return manually configured DKIM selectors for all requested domains."""
|
|
if not domain_names:
|
|
return {}
|
|
|
|
unique_names = list(dict.fromkeys(domain_names))
|
|
selectors_by_domain: Dict[str, List[str]] = {}
|
|
for index in range(0, len(unique_names), DOMAIN_SELECTOR_LOOKUP_CHUNK_SIZE):
|
|
chunk = unique_names[index : index + DOMAIN_SELECTOR_LOOKUP_CHUNK_SIZE]
|
|
rows = db.query(Domain.name, Domain.dkim_selectors).filter(Domain.name.in_(chunk)).all()
|
|
for name, selectors in rows:
|
|
selectors_by_domain[name] = [
|
|
selector.strip() for selector in (selectors or "").split(",") if selector.strip()
|
|
]
|
|
return selectors_by_domain
|
|
|
|
|
|
def _policy_enforcement_suggestions(
|
|
dmarc_policy: Optional[str],
|
|
summary: Dict[str, Any],
|
|
) -> List[Dict[str, str]]:
|
|
"""Suggest policy enforcement when report history supports moving beyond monitoring."""
|
|
if dmarc_policy != "none":
|
|
return []
|
|
total_count = int(summary.get("total_count", 0) or 0)
|
|
compliance_rate = float(summary.get("compliance_rate", 0.0) or 0.0)
|
|
if total_count >= 100 and compliance_rate >= 98.0:
|
|
return [
|
|
{
|
|
"type": "policy_enforcement_ready",
|
|
"severity": "info",
|
|
"message": (
|
|
"Recent reports show very high DMARC compliance. Consider moving from "
|
|
"p=none to p=quarantine with a limited pct value."
|
|
),
|
|
}
|
|
]
|
|
if total_count >= 100 and compliance_rate >= 90.0:
|
|
return [
|
|
{
|
|
"type": "policy_enforcement_review",
|
|
"severity": "info",
|
|
"message": (
|
|
"DMARC compliance is trending high. Review remaining failures before "
|
|
"moving the domain policy beyond p=none."
|
|
),
|
|
}
|
|
]
|
|
return []
|
|
|
|
|
|
def _normalize_domain_name(name: str) -> str:
|
|
return name.strip().strip(".").lower()
|
|
|
|
|
|
def _domain_names_for_summary(db: Session, store: ReportStore) -> List[str]:
|
|
report_domains = store.get_domains()
|
|
stored_domains = [
|
|
name
|
|
for (name,) in db.query(Domain.name)
|
|
.filter(Domain.active == True) # noqa: E712
|
|
.order_by(Domain.name)
|
|
.all()
|
|
]
|
|
return list(dict.fromkeys(stored_domains + report_domains))
|
|
|
|
|
|
def _domain_exists(db: Session, store: ReportStore, domain_name: str) -> bool:
|
|
return domain_name in store.get_domains() or bool(
|
|
db.query(Domain.id).filter(Domain.name == domain_name).first()
|
|
)
|
|
|
|
|
|
def _record_evidence(
|
|
label: str, value: Optional[str], href: str = "#dns-records"
|
|
) -> DNSHealthEvidence:
|
|
return DNSHealthEvidence(label=label, value=value or "Not found", href=href)
|
|
|
|
|
|
def _summary_evidence(
|
|
label: str, value: object, href: str = "#compliance-chart"
|
|
) -> DNSHealthEvidence:
|
|
return DNSHealthEvidence(label=label, value=str(value), href=href)
|
|
|
|
|
|
def _dns_check(
|
|
key: str,
|
|
label: str,
|
|
present: bool,
|
|
present_message: str,
|
|
missing_message: str,
|
|
evidence: List[DNSHealthEvidence],
|
|
) -> DNSHealthCheck:
|
|
return DNSHealthCheck(
|
|
key=key,
|
|
label=label,
|
|
status="pass" if present else "fail",
|
|
message=present_message if present else missing_message,
|
|
evidence=evidence,
|
|
)
|
|
|
|
|
|
def _enforcement_recommendation(
|
|
policy: str,
|
|
summary: Dict[str, Any],
|
|
) -> DNSHealthRecommendation:
|
|
total = int(summary.get("total_count", 0) or 0)
|
|
failed = int(summary.get("failed_count", 0) or 0)
|
|
compliance = float(summary.get("compliance_rate", 0.0) or 0.0)
|
|
evidence = [
|
|
_summary_evidence("Policy", f"p={policy}", "#dns-records"),
|
|
_summary_evidence("Total messages", total),
|
|
_summary_evidence("Compliance", f"{compliance}%"),
|
|
_summary_evidence("Failed messages", failed, "#sending-sources"),
|
|
]
|
|
if policy != "none":
|
|
return DNSHealthRecommendation(
|
|
type="policy_already_enforced",
|
|
severity="info",
|
|
title="DMARC policy is already enforced",
|
|
detail="This domain is already beyond monitoring mode.",
|
|
action="Continue watching failure trends before tightening further.",
|
|
evidence=evidence,
|
|
)
|
|
if total < 100:
|
|
return DNSHealthRecommendation(
|
|
type="policy_needs_more_data",
|
|
severity="warning",
|
|
title="Collect more report volume before enforcement",
|
|
detail="DMARQ needs at least 100 observed messages before recommending quarantine.",
|
|
action="Keep p=none until more aggregate reports arrive.",
|
|
evidence=evidence,
|
|
)
|
|
if compliance >= 98.0 and failed <= max(2, int(total * 0.02)):
|
|
return DNSHealthRecommendation(
|
|
type="policy_enforcement_ready",
|
|
severity="info",
|
|
title="Ready to plan quarantine",
|
|
detail="Recent report volume is high and failures are low enough to plan enforcement.",
|
|
action="Move gradually: set p=quarantine with a low pct value, then watch failures.",
|
|
evidence=evidence,
|
|
)
|
|
if compliance >= 90.0:
|
|
return DNSHealthRecommendation(
|
|
type="policy_enforcement_review",
|
|
severity="warning",
|
|
title="Close remaining failures before enforcement",
|
|
detail="Compliance is improving, but failures still need review before policy changes.",
|
|
action="Review failing sources and SPF/DKIM alignment before changing p=none.",
|
|
evidence=evidence,
|
|
)
|
|
return DNSHealthRecommendation(
|
|
type="policy_not_ready",
|
|
severity="error",
|
|
title="Not ready for enforcement",
|
|
detail="Current DMARC compliance is too low for a safe policy change.",
|
|
action="Fix unauthenticated or unknown senders before moving beyond p=none.",
|
|
evidence=evidence,
|
|
)
|
|
|
|
|
|
def _mta_sts_check(result: MTAStsResult) -> DNSHealthCheck:
|
|
evidence = [
|
|
_record_evidence("MTA-STS TXT", result.dns_record, "#dns-records"),
|
|
_record_evidence("Policy URL", result.policy_url, "#mta-sts-posture"),
|
|
]
|
|
if result.mode:
|
|
evidence.append(_record_evidence("Mode", result.mode, "#mta-sts-posture"))
|
|
if result.mx:
|
|
evidence.append(_record_evidence("MX patterns", ", ".join(result.mx), "#mta-sts-posture"))
|
|
message = (
|
|
"MTA-STS DNS and HTTPS policy are valid."
|
|
if result.status == "pass"
|
|
else (result.errors[0] if result.errors else "MTA-STS posture needs attention.")
|
|
)
|
|
return DNSHealthCheck(
|
|
key="mta_sts",
|
|
label="MTA-STS",
|
|
status=result.status,
|
|
message=message,
|
|
evidence=evidence,
|
|
)
|
|
|
|
|
|
def _mta_sts_recommendation(result: MTAStsResult) -> Optional[DNSHealthRecommendation]:
|
|
if result.status == "pass" and not result.warnings:
|
|
return None
|
|
severity = "warning" if result.status == "pass" else "error"
|
|
title = "MTA-STS policy needs review" if result.status == "pass" else "Publish MTA-STS"
|
|
detail = (
|
|
"; ".join(result.warnings)
|
|
if result.status == "pass"
|
|
else "; ".join(result.errors or ["MTA-STS is not configured or is not valid."])
|
|
)
|
|
action = (
|
|
"Move the policy to mode: enforce once MX coverage is confirmed."
|
|
if result.status == "pass"
|
|
else "Publish _mta-sts TXT and a valid HTTPS policy at the well-known URL."
|
|
)
|
|
return DNSHealthRecommendation(
|
|
type="mta_sts_review" if result.status == "pass" else "missing_mta_sts",
|
|
severity=severity,
|
|
title=title,
|
|
detail=detail,
|
|
action=action,
|
|
evidence=_mta_sts_check(result).evidence,
|
|
)
|
|
|
|
|
|
@router.get("/summary", response_model=DomainSummaryResponse)
|
|
async def get_domains_summary(db: Session = Depends(get_db)):
|
|
"""
|
|
Get summary statistics for all domains, formatted for the dashboard.
|
|
|
|
Performs cached DNS lookups for each domain and includes the results
|
|
(DMARC/SPF/DKIM status and live DMARC policy) in the per-domain entries.
|
|
A per-domain timeout of 10 s prevents slow DNS responses from blocking the
|
|
page load.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = _domain_names_for_summary(db, store)
|
|
summaries = store.get_all_domain_summaries()
|
|
|
|
# Perform DNS checks for all domains, reusing fresh cached results.
|
|
provider = get_default_provider(db)
|
|
manual_selectors_by_domain = _get_domain_selectors_map_from_db(db, domains)
|
|
|
|
async def _dns_for_domain(domain_name: str) -> DomainDNSResult:
|
|
manual_selectors = manual_selectors_by_domain.get(domain_name, [])
|
|
report_selectors = _get_selectors_from_reports(store, domain_name)
|
|
combined = list(dict.fromkeys(manual_selectors + report_selectors))
|
|
try:
|
|
result, cached, checked_at = await asyncio.wait_for(
|
|
resolve_domain_dns_cached(db, provider, domain_name, selectors=combined),
|
|
timeout=10.0,
|
|
)
|
|
result.cached = cached # type: ignore[attr-defined]
|
|
result.checked_at = checked_at # type: ignore[attr-defined]
|
|
return result
|
|
except (asyncio.TimeoutError, LookupError, OSError) as exc:
|
|
logger.warning("DNS check failed for %s: %s", domain_name, exc)
|
|
return DomainDNSResult()
|
|
|
|
dns_results = []
|
|
for domain_name in domains:
|
|
dns_results.append(await _dns_for_domain(domain_name))
|
|
|
|
# Calculate overall statistics
|
|
total_domains = len(domains)
|
|
total_emails = 0
|
|
total_passed = 0
|
|
total_reports = 0
|
|
|
|
domains_list = []
|
|
|
|
for domain_name, dns in zip(domains, dns_results):
|
|
summary = summaries.get(domain_name, {})
|
|
total_emails += summary.get("total_count", 0)
|
|
total_passed += summary.get("passed_count", 0)
|
|
total_reports += summary.get("reports_processed", 0)
|
|
|
|
# Prefer live DNS policy; fall back to policy seen in reports
|
|
live_policy = extract_dmarc_policy(dns.dmarc_record)
|
|
reported_policy = summary.get("policy", {})
|
|
if isinstance(reported_policy, dict):
|
|
reported_policy = reported_policy.get("p")
|
|
dmarc_policy = live_policy or reported_policy or "none"
|
|
|
|
# Format domain data for frontend
|
|
domains_list.append(
|
|
{
|
|
"id": domain_name,
|
|
"domain_name": domain_name,
|
|
"total_emails": summary.get("total_count", 0),
|
|
"passed_count": summary.get("passed_count", 0),
|
|
"failed_count": summary.get("failed_count", 0),
|
|
"pass_rate": summary.get("compliance_rate", 0),
|
|
"report_count": summary.get("reports_processed", 0),
|
|
# Real DNS status
|
|
"dmarc_status": dns.dmarc,
|
|
"dmarc_policy": dmarc_policy,
|
|
"spf_status": dns.spf,
|
|
"dkim_status": dns.dkim,
|
|
"dns_cached": getattr(dns, "cached", False),
|
|
"dns_checked_at": (
|
|
getattr(dns, "checked_at", None).isoformat()
|
|
if getattr(dns, "checked_at", None)
|
|
else None
|
|
),
|
|
}
|
|
)
|
|
|
|
# Calculate overall pass rate
|
|
overall_pass_rate = 0
|
|
if total_emails > 0:
|
|
overall_pass_rate = round((total_passed / total_emails) * 100, 1)
|
|
|
|
return DomainSummaryResponse(
|
|
total_domains=total_domains,
|
|
total_emails=total_emails,
|
|
overall_pass_rate=overall_pass_rate,
|
|
reports_processed=total_reports,
|
|
domains=domains_list,
|
|
)
|
|
|
|
|
|
@router.get("/domains", response_model=List[DomainResponse])
|
|
async def read_domains(db: Session = Depends(get_db)):
|
|
"""
|
|
Retrieve domains with their statistics.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = _domain_names_for_summary(db, store)
|
|
summaries = store.get_all_domain_summaries()
|
|
stored = {
|
|
domain.name: domain for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
|
|
}
|
|
|
|
result = []
|
|
for domain_name in domains:
|
|
summary = summaries.get(domain_name, {})
|
|
stored_domain = stored.get(domain_name)
|
|
domain_response = DomainResponse(
|
|
name=domain_name,
|
|
description=stored_domain.description if stored_domain else None,
|
|
policy=(
|
|
summary.get("policy")
|
|
or (stored_domain.dmarc_policy if stored_domain else None)
|
|
or "unknown"
|
|
),
|
|
reports_count=summary.get("reports_processed", 0),
|
|
emails_count=summary.get("total_count", 0),
|
|
compliance_rate=summary.get("compliance_rate", 0.0),
|
|
)
|
|
result.append(domain_response)
|
|
|
|
return result
|
|
|
|
|
|
@router.post("/domains", response_model=DomainResponse, status_code=status.HTTP_201_CREATED)
|
|
async def create_domain(
|
|
payload: DomainCreate,
|
|
db: Session = Depends(get_db),
|
|
_auth: dict = Depends(require_admin_auth),
|
|
):
|
|
"""Create a monitored domain before any DMARC reports have arrived."""
|
|
name = _normalize_domain_name(payload.name)
|
|
validation = validate_domain_config({"name": name, "description": payload.description or ""})
|
|
if not validation["valid"]:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=validation["errors"],
|
|
)
|
|
existing = db.query(Domain).filter(Domain.name == name).first()
|
|
if existing:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail="Domain is already monitored",
|
|
)
|
|
|
|
selectors = ",".join(
|
|
selector.strip()
|
|
for selector in payload.dkim_selectors or []
|
|
if selector and selector.strip()
|
|
)
|
|
domain = Domain(
|
|
name=name,
|
|
description=payload.description,
|
|
dkim_selectors=selectors or None,
|
|
active=True,
|
|
verified=False,
|
|
)
|
|
db.add(domain)
|
|
try:
|
|
db.commit()
|
|
except IntegrityError as exc:
|
|
db.rollback()
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail="Domain is already monitored",
|
|
) from exc
|
|
db.refresh(domain)
|
|
return DomainResponse(
|
|
name=domain.name,
|
|
description=domain.description,
|
|
policy=domain.dmarc_policy or "unknown",
|
|
)
|
|
|
|
|
|
@router.get("/domains/{domain_name}", response_model=DomainResponse)
|
|
async def read_domain(domain_name: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Get statistics for a specific domain.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
stored_domain = db.query(Domain).filter(Domain.name == domain_name).first()
|
|
|
|
if domain_name not in domains and stored_domain is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
summary = store.get_domain_summary(domain_name)
|
|
|
|
return DomainResponse(
|
|
name=domain_name,
|
|
description=stored_domain.description if stored_domain else None,
|
|
policy=(
|
|
summary.get("policy")
|
|
or (stored_domain.dmarc_policy if stored_domain else None)
|
|
or "unknown"
|
|
),
|
|
reports_count=summary.get("reports_processed", 0),
|
|
emails_count=summary.get("total_count", 0),
|
|
compliance_rate=summary.get("compliance_rate", 0.0),
|
|
)
|
|
|
|
|
|
# New endpoints for domain details page
|
|
|
|
|
|
@router.get("/{domain_id}/stats", response_model=DomainStatsResponse)
|
|
async def get_domain_stats(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Get detailed statistics for a specific domain
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = _domain_names_for_summary(db, store)
|
|
|
|
if domain_id not in domains:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
summary = store.get_domain_summary(domain_id)
|
|
total_count = summary.get("total_count", 0)
|
|
passed_count = summary.get("passed_count", 0)
|
|
failed_count = total_count - passed_count
|
|
compliance_rate = summary.get("compliance_rate", 0.0)
|
|
reports_processed = summary.get("reports_processed", 0)
|
|
|
|
return DomainStatsResponse(
|
|
complianceRate=compliance_rate,
|
|
totalEmails=total_count,
|
|
failedEmails=failed_count,
|
|
reportCount=reports_processed,
|
|
)
|
|
|
|
|
|
@router.get("/{domain_id}/dns", response_model=DNSRecordResponse)
|
|
async def get_domain_dns_records(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
refresh: bool = Query(False, title="Refresh cached DNS result"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Get DNS records for a specific domain using live DNS lookups.
|
|
|
|
Manual selectors (stored in the database) are checked first, followed by
|
|
selectors observed in stored DMARC reports, with common well-known
|
|
selectors used as a final fallback.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
|
|
if not _domain_exists(db, store, domain_id):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
manual_selectors = _get_domain_selectors_from_db(db, domain_id)
|
|
report_selectors = _get_selectors_from_reports(store, domain_id)
|
|
combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors))
|
|
|
|
provider = get_default_provider(db)
|
|
result, cached, checked_at = await resolve_domain_dns_cached(
|
|
db,
|
|
provider,
|
|
domain_id,
|
|
selectors=combined_selectors,
|
|
refresh=refresh,
|
|
)
|
|
|
|
return DNSRecordResponse(
|
|
dmarc=result.dmarc,
|
|
dmarcRecord=result.dmarc_record,
|
|
spf=result.spf,
|
|
spfRecord=result.spf_record,
|
|
dkim=result.dkim,
|
|
dkimSelectors=result.dkim_selectors,
|
|
cached=cached,
|
|
checkedAt=checked_at.isoformat(),
|
|
)
|
|
|
|
|
|
@router.get("/{domain_id}/dns/health", response_model=DNSHealthResponse)
|
|
async def get_domain_dns_health(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
refresh: bool = Query(False, title="Refresh cached DNS result"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Return evidence-linked DNS health and enforcement readiness guidance."""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
if not _domain_exists(db, store, domain_id):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
manual_selectors = _get_domain_selectors_from_db(db, domain_id)
|
|
report_selectors = _get_selectors_from_reports(store, domain_id)
|
|
combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors))
|
|
|
|
provider = get_default_provider(db)
|
|
result, _, _ = await resolve_domain_dns_cached(
|
|
db,
|
|
provider,
|
|
domain_id,
|
|
selectors=combined_selectors,
|
|
refresh=refresh,
|
|
)
|
|
mta_sts_result, _, _ = await check_mta_sts_cached(
|
|
db,
|
|
provider,
|
|
domain_id,
|
|
refresh=refresh,
|
|
)
|
|
summary = store.get_domain_summary(domain_id)
|
|
policy = extract_dmarc_policy(result.dmarc_record) or "none"
|
|
checks = [
|
|
_dns_check(
|
|
"dmarc",
|
|
"DMARC",
|
|
result.dmarc,
|
|
"DMARC record is published.",
|
|
"No DMARC record was found.",
|
|
[_record_evidence("DMARC TXT", result.dmarc_record)],
|
|
),
|
|
_dns_check(
|
|
"spf",
|
|
"SPF",
|
|
result.spf,
|
|
"SPF record is published.",
|
|
"No SPF record was found at the domain root.",
|
|
[_record_evidence("SPF TXT", result.spf_record)],
|
|
),
|
|
_dns_check(
|
|
"dkim",
|
|
"DKIM",
|
|
result.dkim,
|
|
"At least one DKIM selector resolved.",
|
|
"No DKIM record was found for configured or observed selectors.",
|
|
[
|
|
_record_evidence(
|
|
"Selectors checked",
|
|
", ".join(combined_selectors or result.selectors_checked or []),
|
|
),
|
|
_record_evidence("DKIM TXT", result.dkim_record),
|
|
],
|
|
),
|
|
_mta_sts_check(mta_sts_result),
|
|
]
|
|
recommendations: List[DNSHealthRecommendation] = []
|
|
for check in checks:
|
|
if check.status == "fail" and check.key != "mta_sts":
|
|
recommendations.append(
|
|
DNSHealthRecommendation(
|
|
type=f"missing_{check.key}",
|
|
severity="error" if check.key == "dmarc" else "warning",
|
|
title=f"{check.label} needs attention",
|
|
detail=check.message,
|
|
action=f"Publish or repair the {check.label} DNS record, then refresh DNS health.",
|
|
evidence=check.evidence,
|
|
)
|
|
)
|
|
recommendations.append(_enforcement_recommendation(policy, summary))
|
|
mta_sts_recommendation = _mta_sts_recommendation(mta_sts_result)
|
|
if mta_sts_recommendation:
|
|
recommendations.append(mta_sts_recommendation)
|
|
|
|
failed_checks = sum(1 for check in checks if check.status == "fail")
|
|
health_status = (
|
|
"healthy" if failed_checks == 0 else "degraded" if failed_checks < 3 else "critical"
|
|
)
|
|
return DNSHealthResponse(
|
|
status=health_status,
|
|
policy=policy,
|
|
compliance_rate=float(summary.get("compliance_rate", 0.0) or 0.0),
|
|
total_emails=int(summary.get("total_count", 0) or 0),
|
|
failed_emails=int(summary.get("failed_count", 0) or 0),
|
|
checks=checks,
|
|
recommendations=recommendations,
|
|
)
|
|
|
|
|
|
@router.get("/{domain_id}/dns/mta-sts", response_model=MTAStsResponse)
|
|
async def get_domain_mta_sts(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
refresh: bool = Query(False, title="Refresh cached MTA-STS result"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Return cached MTA-STS DNS and HTTPS policy posture for a domain."""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
if not _domain_exists(db, store, domain_id):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
result, cached, checked_at = await check_mta_sts_cached(
|
|
db,
|
|
get_default_provider(db),
|
|
domain_id,
|
|
refresh=refresh,
|
|
)
|
|
return MTAStsResponse(
|
|
status=result.status,
|
|
dns_record=result.dns_record,
|
|
policy_url=result.policy_url,
|
|
policy_text=result.policy_text,
|
|
mode=result.mode,
|
|
max_age=result.max_age,
|
|
mx=result.mx,
|
|
errors=result.errors,
|
|
warnings=result.warnings,
|
|
cached=cached,
|
|
checked_at=checked_at.isoformat(),
|
|
)
|
|
|
|
|
|
@router.get("/cloudflare/discover", response_model=List[CloudflareZoneResponse])
|
|
async def discover_cloudflare_domains(db: Session = Depends(get_db)):
|
|
"""Discover active Cloudflare zones visible to the configured API token."""
|
|
try:
|
|
return await discover_cloudflare_zones(db)
|
|
except LookupError as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=str(exc),
|
|
) from exc
|
|
|
|
|
|
@router.post("/cloudflare/import", response_model=CloudflareImportResponse)
|
|
async def import_cloudflare_domain_zones(
|
|
payload: CloudflareImportRequest,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Import selected, or all, Cloudflare zones as monitored domains."""
|
|
try:
|
|
return await import_cloudflare_domains(db, requested_domains=payload.domains)
|
|
except LookupError as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=str(exc),
|
|
) from exc
|
|
|
|
|
|
@router.get("/{domain_id}/dns/cloudflare", response_model=CloudflareDNSAnalysisResponse)
|
|
async def get_cloudflare_domain_dns_analysis(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Analyze Cloudflare-managed DNS records and persist detected changes."""
|
|
try:
|
|
zone_data = await get_zone_for_domain(db, domain_id)
|
|
except LookupError as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=str(exc),
|
|
) from exc
|
|
|
|
records = zone_data["records"]
|
|
changes = sync_dns_record_changes(
|
|
db,
|
|
domain=domain_id,
|
|
zone_id=zone_data["id"],
|
|
records=records,
|
|
)
|
|
analysis = analyze_dns_records(domain_id, records)
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
analysis["suggestions"].extend(
|
|
_policy_enforcement_suggestions(
|
|
analysis["checks"].get("dmarc_policy"),
|
|
store.get_domain_summary(domain_id),
|
|
)
|
|
)
|
|
history = list_dns_record_changes(db, domain_id)
|
|
return CloudflareDNSAnalysisResponse(
|
|
zone={"id": zone_data["id"], "name": zone_data["name"]},
|
|
records=analysis["records"],
|
|
checks=analysis["checks"],
|
|
suggestions=analysis["suggestions"],
|
|
changes=changes,
|
|
history=history,
|
|
)
|
|
|
|
|
|
@router.get("/{domain_id}/dns/history", response_model=DNSChangeHistoryResponse)
|
|
async def get_domain_dns_change_history(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
limit: int = Query(50, title="Maximum number of change events to return"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Return recent provider-backed DNS record changes for a domain."""
|
|
return DNSChangeHistoryResponse(history=list_dns_record_changes(db, domain_id, limit=limit))
|
|
|
|
|
|
@router.get("/{domain_id}/reports", response_model=DomainReportsResponse)
|
|
async def get_domain_reports(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
limit: int = Query(10, title="Maximum number of reports to return"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Get recent DMARC reports for a specific domain, along with compliance timeline
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
|
|
if domain_id not in domains:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
# Get reports for this domain
|
|
reports = store.get_domain_reports(domain_id, limit=limit)
|
|
|
|
# Generate report entries
|
|
report_entries = []
|
|
for report in reports:
|
|
policy_val = report.get("policy", "none")
|
|
if isinstance(policy_val, dict):
|
|
policy_val = policy_val.get("p", "none")
|
|
report_entries.append(
|
|
ReportEntry(
|
|
id=report.get("report_id", "unknown"),
|
|
org_name=report.get("org_name", "Unknown Organization"),
|
|
begin_date=report.get("begin_timestamp", 0),
|
|
end_date=report.get("end_timestamp", 0),
|
|
total_emails=report.get("total_count", 0),
|
|
pass_rate=report.get("pass_rate", 0.0),
|
|
policy=policy_val,
|
|
)
|
|
)
|
|
|
|
# Build compliance timeline from actual report data
|
|
timeline = _build_compliance_timeline(store, domain_id)
|
|
|
|
return DomainReportsResponse(reports=report_entries, compliance_timeline=timeline)
|
|
|
|
|
|
@router.get("/{domain_id}/reports/export")
|
|
async def export_domain_reports(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
start_date: Optional[date] = Query(None, title="Start date for exported reports"),
|
|
end_date: Optional[date] = Query(None, title="End date for exported reports"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Export DMARC report summaries for a specific domain as CSV.
|
|
"""
|
|
if start_date and end_date and start_date > end_date:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="start_date must be on or before end_date",
|
|
)
|
|
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
|
|
if domain_id not in store.get_domains():
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
reports = [
|
|
report
|
|
for report in store.get_domain_reports(domain_id, limit=10000)
|
|
if _report_in_export_range(report, start_date, end_date)
|
|
]
|
|
|
|
output = io.StringIO()
|
|
writer = csv.writer(output)
|
|
writer.writerow(
|
|
[
|
|
"domain",
|
|
"report_id",
|
|
"org_name",
|
|
"begin_date",
|
|
"end_date",
|
|
"total_emails",
|
|
"passed",
|
|
"failed",
|
|
"pass_rate",
|
|
"policy",
|
|
"subdomain_policy",
|
|
"non_subdomain_policy",
|
|
"adkim",
|
|
"aspf",
|
|
"failure_options",
|
|
"testing",
|
|
"discovery_method",
|
|
"schema_version",
|
|
"report_variant",
|
|
"generator",
|
|
]
|
|
)
|
|
|
|
for report in reports:
|
|
summary = report.get("summary", {})
|
|
total = int(summary.get("total_count", report.get("total_count", 0)) or 0)
|
|
passed = int(summary.get("passed_count", report.get("passed_count", 0)) or 0)
|
|
failed = int(summary.get("failed_count", report.get("failed_count", 0)) or 0)
|
|
policy = report.get("policy", "none")
|
|
policy_parts = policy if isinstance(policy, dict) else {}
|
|
if isinstance(policy, dict):
|
|
policy = policy.get("p", "none")
|
|
writer.writerow(
|
|
[
|
|
domain_id,
|
|
report.get("report_id", "unknown"),
|
|
report.get("org_name", "Unknown Organization"),
|
|
_format_report_date(report.get("begin_timestamp") or report.get("begin_date")),
|
|
_format_report_date(report.get("end_timestamp") or report.get("end_date")),
|
|
total,
|
|
passed,
|
|
failed,
|
|
report.get("pass_rate", 0.0),
|
|
policy,
|
|
policy_parts.get("sp", ""),
|
|
policy_parts.get("np", ""),
|
|
policy_parts.get("adkim", ""),
|
|
policy_parts.get("aspf", ""),
|
|
policy_parts.get("fo", ""),
|
|
policy_parts.get("testing", ""),
|
|
policy_parts.get("discovery_method", ""),
|
|
report.get("schema_version", ""),
|
|
report.get("variant", ""),
|
|
report.get("generator", ""),
|
|
]
|
|
)
|
|
|
|
filename = f"{domain_id.replace('/', '_')}-dmarc-reports.csv"
|
|
return Response(
|
|
content=output.getvalue(),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
|
)
|
|
|
|
|
|
def _build_compliance_timeline(store: ReportStore, domain: str) -> List[TimelinePoint]:
|
|
"""
|
|
Build a compliance timeline from actual report data stored in ReportStore.
|
|
|
|
Groups reports by date and calculates the pass rate per day to provide
|
|
real historical trend data for the compliance chart.
|
|
"""
|
|
all_reports = store.get_domain_reports(domain)
|
|
|
|
# Aggregate report data by date
|
|
daily_data: Dict[str, Dict[str, int]] = {}
|
|
for report in all_reports:
|
|
# Use begin_date to determine the day of this report
|
|
begin = report.get("begin_date", 0)
|
|
if isinstance(begin, (int, float)) and begin > 0:
|
|
date_str = datetime.fromtimestamp(begin, tz=timezone.utc).strftime("%Y-%m-%d")
|
|
elif isinstance(begin, str):
|
|
# Handle ISO-format strings
|
|
try:
|
|
date_str = datetime.fromisoformat(begin).strftime("%Y-%m-%d")
|
|
except (ValueError, TypeError):
|
|
continue
|
|
else:
|
|
continue
|
|
|
|
if date_str not in daily_data:
|
|
daily_data[date_str] = {"total": 0, "passed": 0, "failed": 0}
|
|
|
|
summary = report.get("summary", {})
|
|
total = summary.get("total_count", 0)
|
|
passed = summary.get("passed_count", 0)
|
|
failed = summary.get("failed_count", max(0, total - passed))
|
|
daily_data[date_str]["total"] += total
|
|
daily_data[date_str]["passed"] += passed
|
|
daily_data[date_str]["failed"] += failed
|
|
|
|
# Convert to timeline points sorted by date
|
|
timeline = []
|
|
for date_str in sorted(daily_data.keys()):
|
|
data = daily_data[date_str]
|
|
total = data["total"]
|
|
compliance_rate = round((data["passed"] / total) * 100, 1) if total > 0 else 0.0
|
|
failure_rate = round((data["failed"] / total) * 100, 1) if total > 0 else 0.0
|
|
timeline.append(
|
|
TimelinePoint(
|
|
date=date_str,
|
|
total=total,
|
|
volume=total,
|
|
passed=data["passed"],
|
|
failed=data["failed"],
|
|
compliance_rate=compliance_rate,
|
|
failure_rate=failure_rate,
|
|
)
|
|
)
|
|
|
|
return timeline
|
|
|
|
|
|
def _report_in_export_range(
|
|
report: Dict[str, Any], start_date: Optional[date], end_date: Optional[date]
|
|
) -> bool:
|
|
report_date = _report_date(report.get("begin_timestamp") or report.get("begin_date"))
|
|
if report_date is None:
|
|
return False
|
|
if start_date and report_date < start_date:
|
|
return False
|
|
if end_date and report_date > end_date:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _report_date(value: Any) -> Optional[date]:
|
|
if isinstance(value, (int, float)) and value > 0:
|
|
return datetime.fromtimestamp(value, tz=timezone.utc).date()
|
|
if isinstance(value, str):
|
|
try:
|
|
return datetime.fromisoformat(value).date()
|
|
except (ValueError, TypeError):
|
|
return None
|
|
return None
|
|
|
|
|
|
def _format_report_date(value: Any) -> str:
|
|
report_date = _report_date(value)
|
|
return report_date.isoformat() if report_date else ""
|
|
|
|
|
|
def _spf_fix_hint(ip: str, spf_result: str, failed_count: int = 0) -> Optional[str]:
|
|
"""Return a copy-paste SPF mechanism (e.g. ``ip4:1.2.3.4``) for a failing IP.
|
|
|
|
Returns ``None`` when SPF did not fail or when *ip* is not a valid address.
|
|
"""
|
|
if spf_result != "fail" and failed_count <= 0:
|
|
return None
|
|
try:
|
|
addr = ipaddress.ip_address(ip)
|
|
prefix = "ip6" if isinstance(addr, ipaddress.IPv6Address) else "ip4"
|
|
return f"{prefix}:{ip}"
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _source_recommendations(
|
|
ip: str,
|
|
source: Dict[str, Any],
|
|
hostname: Optional[str],
|
|
spf_fix_hint: Optional[str],
|
|
) -> List[SourceRecommendation]:
|
|
"""Build clear next steps for common DMARC source patterns."""
|
|
spf_result = source.get("spf_result", "unknown")
|
|
dkim_result = source.get("dkim_result", "unknown")
|
|
dmarc_result = source.get("dmarc_result") or (
|
|
"pass" if spf_result == "pass" or dkim_result == "pass" else "fail"
|
|
)
|
|
disposition = source.get("disposition", "none")
|
|
disposition_counts = source.get("disposition_counts", {}) or {}
|
|
dmarc_failed = source.get("dmarc_fail_count", 0) > 0 or dmarc_result == "fail"
|
|
dmarc_passed = source.get("dmarc_pass_count", 0) > 0 or dmarc_result == "pass"
|
|
|
|
recommendations: List[SourceRecommendation] = []
|
|
|
|
if not hostname and dmarc_failed:
|
|
recommendations.append(
|
|
SourceRecommendation(
|
|
type="unknown_source",
|
|
severity="warning",
|
|
title="Unknown sending source",
|
|
detail=(
|
|
"No reverse DNS name was found for this IP, so treat it as unrecognized "
|
|
"until you confirm who owns it."
|
|
),
|
|
action=(
|
|
"Confirm whether this server should send mail for this domain before "
|
|
"authorizing it in SPF or DKIM."
|
|
),
|
|
)
|
|
)
|
|
|
|
if (
|
|
spf_result == "pass"
|
|
and dkim_result in {"fail", "mixed", "unknown", "none"}
|
|
and dmarc_passed
|
|
):
|
|
recommendations.append(
|
|
SourceRecommendation(
|
|
type="spf_only_pass",
|
|
severity="info",
|
|
title="SPF-only DMARC pass",
|
|
detail="DMARC is passing through SPF, but DKIM is not reliably passing for this source.",
|
|
action=(
|
|
"Enable DKIM signing for this sending service so messages keep passing "
|
|
"if SPF alignment changes."
|
|
),
|
|
)
|
|
)
|
|
|
|
if (
|
|
dkim_result == "pass"
|
|
and spf_result in {"fail", "mixed", "unknown", "none"}
|
|
and dmarc_passed
|
|
):
|
|
action = "Authorize this service in SPF, or confirm SPF is intentionally handled elsewhere."
|
|
if spf_fix_hint:
|
|
action = f"Add {spf_fix_hint} to your SPF record if this service is legitimate."
|
|
recommendations.append(
|
|
SourceRecommendation(
|
|
type="dkim_only_pass",
|
|
severity="info",
|
|
title="DKIM-only DMARC pass",
|
|
detail="DMARC is passing through DKIM, but SPF is not reliably passing for this source.",
|
|
action=action,
|
|
)
|
|
)
|
|
|
|
if spf_result == "fail" and dkim_result == "fail" and dmarc_failed:
|
|
action = (
|
|
"Do not authorize this source until you confirm it is legitimate; then configure "
|
|
"both SPF authorization and DKIM signing."
|
|
)
|
|
if spf_fix_hint:
|
|
action = f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service."
|
|
recommendations.append(
|
|
SourceRecommendation(
|
|
type="full_fail",
|
|
severity="error",
|
|
title="Full DMARC failure",
|
|
detail="Neither SPF nor DKIM is passing, so this mail fails DMARC.",
|
|
action=action,
|
|
)
|
|
)
|
|
|
|
if dmarc_failed and (disposition == "none" or disposition_counts.get("none", 0) > 0):
|
|
recommendations.append(
|
|
SourceRecommendation(
|
|
type="policy_not_enforced",
|
|
severity="warning",
|
|
title="Policy not enforced",
|
|
detail="Some failed mail was accepted because the applied DMARC disposition was none.",
|
|
action=(
|
|
"After legitimate sources are passing consistently, move the domain policy "
|
|
"toward quarantine or reject."
|
|
),
|
|
)
|
|
)
|
|
|
|
return recommendations
|
|
|
|
|
|
async def _safe_ptr_lookup(provider: Any, ip: str, timeout: float = 3.0) -> Optional[str]:
|
|
"""Perform a PTR lookup for *ip*, returning ``None`` on any error or timeout."""
|
|
try:
|
|
ipaddress.ip_address(ip) # validate before making a DNS query
|
|
except ValueError:
|
|
return None
|
|
try:
|
|
return await asyncio.wait_for(provider.lookup_ptr(ip), timeout=timeout)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
@router.get("/{domain_id}/sources", response_model=DomainSourcesResponse)
|
|
async def get_domain_sources(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
days: int = Query(30, title="Number of days to look back"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Get sending sources for a specific domain, including reverse-DNS hostnames
|
|
and SPF fix hints for sources that fail authentication.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
|
|
if domain_id not in domains:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
sources = store.get_domain_sources(domain_id, days=days)
|
|
provider = get_default_provider(db)
|
|
|
|
ips = [s.get("source_ip", "unknown") for s in sources]
|
|
hostnames = await asyncio.gather(*[_safe_ptr_lookup(provider, ip) for ip in ips])
|
|
|
|
source_entries = []
|
|
for source, hostname in zip(sources, hostnames):
|
|
ip = source.get("source_ip", "unknown")
|
|
spf_result = source.get("spf_result", "unknown")
|
|
dkim_result = source.get("dkim_result", "unknown")
|
|
spf_fix_hint = _spf_fix_hint(ip, spf_result, source.get("spf_fail_count", 0))
|
|
source_entries.append(
|
|
SourceEntry(
|
|
ip=ip,
|
|
count=source.get("count", 0),
|
|
spf=spf_result,
|
|
dkim=dkim_result,
|
|
dmarc=source.get("dmarc_result")
|
|
or ("pass" if spf_result == "pass" or dkim_result == "pass" else "fail"),
|
|
disposition=source.get("disposition", "none"),
|
|
spf_pass_count=source.get("spf_pass_count", 0),
|
|
spf_fail_count=source.get("spf_fail_count", 0),
|
|
dkim_pass_count=source.get("dkim_pass_count", 0),
|
|
dkim_fail_count=source.get("dkim_fail_count", 0),
|
|
dmarc_pass_count=source.get("dmarc_pass_count", 0),
|
|
dmarc_fail_count=source.get("dmarc_fail_count", 0),
|
|
disposition_counts=source.get("disposition_counts", {}),
|
|
hostname=hostname,
|
|
spf_fix_hint=spf_fix_hint,
|
|
recommendations=_source_recommendations(ip, source, hostname, spf_fix_hint),
|
|
)
|
|
)
|
|
|
|
return DomainSourcesResponse(sources=source_entries)
|
|
|
|
|
|
@router.get("/{domain_id}/selectors")
|
|
async def get_domain_selectors(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Return the manually configured DKIM selectors for a domain.
|
|
|
|
The response includes both ``selectors`` (manually configured, can be
|
|
deleted) and ``report_selectors`` (automatically discovered from received
|
|
DMARC reports, read-only).
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
if domain_id not in store.get_domains():
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
manual = _get_domain_selectors_from_db(db, domain_id)
|
|
report = _get_selectors_from_reports(store, domain_id)
|
|
# Only include in report_selectors those not already in the manual list
|
|
auto = [s for s in report if s not in manual]
|
|
return {"selectors": manual, "report_selectors": auto}
|
|
|
|
|
|
@router.post("/{domain_id}/selectors", status_code=status.HTTP_201_CREATED)
|
|
async def add_domain_selector(
|
|
selector_data: SelectorRequest,
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Add a DKIM selector to the manual list for a domain.
|
|
|
|
The selector is persisted in the ``Domain`` database row so that it will
|
|
be used in all subsequent DNS checks, even if it has not yet appeared in
|
|
any received DMARC report.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
if domain_id not in store.get_domains():
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
selector = selector_data.selector.strip()
|
|
if not selector:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="Selector must not be empty",
|
|
)
|
|
|
|
domain_db = db.query(Domain).filter(Domain.name == domain_id).first()
|
|
if not domain_db:
|
|
domain_db = Domain(name=domain_id)
|
|
db.add(domain_db)
|
|
|
|
existing = [s.strip() for s in (domain_db.dkim_selectors or "").split(",") if s.strip()]
|
|
if selector not in existing:
|
|
existing.append(selector)
|
|
domain_db.dkim_selectors = ",".join(existing)
|
|
db.commit()
|
|
|
|
return {"selectors": existing}
|
|
|
|
|
|
@router.delete("/{domain_id}/selectors/{selector}", status_code=status.HTTP_200_OK)
|
|
async def delete_domain_selector(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
selector: str = Path(..., title="The DKIM selector to remove"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Remove a manually configured DKIM selector from a domain."""
|
|
domain_db = db.query(Domain).filter(Domain.name == domain_id).first()
|
|
if not domain_db:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
existing = [s.strip() for s in (domain_db.dkim_selectors or "").split(",") if s.strip()]
|
|
if selector not in existing:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Selector '{selector}' not found",
|
|
)
|
|
|
|
existing.remove(selector)
|
|
domain_db.dkim_selectors = ",".join(existing)
|
|
db.commit()
|
|
|
|
return {"selectors": existing}
|
|
|
|
|
|
@router.delete("/{domain_id}", status_code=status.HTTP_204_NO_CONTENT)
|
|
async def delete_domain(
|
|
domain_id: str = Path(..., title="The domain ID or name"),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Delete a domain and all associated data.
|
|
This performs a full cleanup of all reports and records related to this domain.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
|
|
if domain_id not in domains:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Domain not found",
|
|
)
|
|
|
|
# Perform deletion with cleanup
|
|
deleted_from_db = delete_persisted_domain(db, domain_id)
|
|
if deleted_from_db:
|
|
db.commit()
|
|
deleted = store.delete_domain_with_cleanup(domain_id) or deleted_from_db
|
|
|
|
if not deleted:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail="Failed to delete domain",
|
|
)
|
|
|
|
# Return 204 No Content on success
|
|
return None
|
|
|
|
|
|
@router.get("/search", response_model=List[DomainResponse])
|
|
async def search_domains(
|
|
q: Optional[str] = Query(None, title="Search query for domain name or description"),
|
|
policy: Optional[str] = Query(None, title="Filter by DMARC policy"),
|
|
page: int = Query(1, title="Page number", ge=1),
|
|
limit: int = Query(10, title="Number of domains per page", ge=1, le=100),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Search domains with filtering and pagination.
|
|
This supports searching by domain name/description and filtering by DMARC policy.
|
|
|
|
Args:
|
|
q: Optional search query for domain name or description
|
|
policy: Optional filter by DMARC policy (none, quarantine, reject)
|
|
page: Page number (1-based)
|
|
limit: Number of domains per page (max 100)
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
summaries = store.get_all_domain_summaries()
|
|
|
|
# Apply search filter if provided
|
|
filtered_domains = []
|
|
for domain_name in domains:
|
|
summary = summaries.get(domain_name, {})
|
|
|
|
# Skip domain if it doesn't match the search query
|
|
if q and q.lower() not in domain_name.lower():
|
|
continue
|
|
|
|
# Skip domain if it doesn't match the policy filter
|
|
if policy and summary.get("policy") != policy:
|
|
continue
|
|
|
|
# Domain passed all filters
|
|
filtered_domains.append(
|
|
{
|
|
"name": domain_name,
|
|
"description": "", # No description in in-memory store
|
|
"policy": summary.get("policy", "unknown"),
|
|
"reports_count": summary.get("reports_processed", 0),
|
|
"emails_count": summary.get("total_count", 0),
|
|
"compliance_rate": summary.get("compliance_rate", 0.0),
|
|
}
|
|
)
|
|
|
|
# Apply pagination
|
|
start_idx = (page - 1) * limit
|
|
end_idx = start_idx + limit
|
|
paginated_domains = filtered_domains[start_idx:end_idx]
|
|
|
|
return [DomainResponse(**domain) for domain in paginated_domains]
|