feat: add cached DNS checks

This commit is contained in:
Christian Krakau-Louis
2026-05-22 23:30:12 +02:00
parent a39b273419
commit 77fd0f9552
11 changed files with 278 additions and 16 deletions
+1
View File
@@ -21,6 +21,7 @@ if database_url:
config.set_main_option("sqlalchemy.url", _make_sync_db_url(database_url)) config.set_main_option("sqlalchemy.url", _make_sync_db_url(database_url))
import app.models.alert # noqa: E402, F401 import app.models.alert # noqa: E402, F401
import app.models.dns_cache # noqa: E402, F401
import app.models.domain # noqa: E402, F401 import app.models.domain # noqa: E402, F401
import app.models.mail_source # noqa: E402, F401 import app.models.mail_source # noqa: E402, F401
import app.models.mail_source_import # noqa: E402, F401 import app.models.mail_source_import # noqa: E402, F401
@@ -0,0 +1,60 @@
"""add dns cache
Revision ID: b8c9d0e1f2a3
Revises: a7b8c9d0e1f2
Create Date: 2026-05-22 23:28:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "b8c9d0e1f2a3"
down_revision: Union[str, Sequence[str], None] = "a7b8c9d0e1f2"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Create a DNS result cache table."""
op.create_table(
"dns_cache",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("domain", sa.String(), nullable=False),
sa.Column("provider", sa.String(), nullable=False),
sa.Column("selectors_key", sa.String(length=64), nullable=False),
sa.Column("result_json", sa.Text(), nullable=False),
sa.Column("checked_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"),
)
op.create_index(op.f("ix_dns_cache_id"), "dns_cache", ["id"], unique=False)
op.create_index(op.f("ix_dns_cache_domain"), "dns_cache", ["domain"], unique=False)
op.create_index(op.f("ix_dns_cache_provider"), "dns_cache", ["provider"], unique=False)
op.create_index(
op.f("ix_dns_cache_selectors_key"),
"dns_cache",
["selectors_key"],
unique=False,
)
op.create_index(op.f("ix_dns_cache_checked_at"), "dns_cache", ["checked_at"], unique=False)
op.create_index(
"ix_dns_cache_domain_checked",
"dns_cache",
["domain", "checked_at"],
unique=False,
)
def downgrade() -> None:
"""Drop the DNS result cache table."""
op.drop_index("ix_dns_cache_domain_checked", table_name="dns_cache")
op.drop_index(op.f("ix_dns_cache_checked_at"), table_name="dns_cache")
op.drop_index(op.f("ix_dns_cache_selectors_key"), table_name="dns_cache")
op.drop_index(op.f("ix_dns_cache_provider"), table_name="dns_cache")
op.drop_index(op.f("ix_dns_cache_domain"), table_name="dns_cache")
op.drop_index(op.f("ix_dns_cache_id"), table_name="dns_cache")
op.drop_table("dns_cache")
+43 -14
View File
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
from app.core.database import get_db from app.core.database import get_db
from app.models.domain import Domain from app.models.domain import Domain
from app.services.dns_cache import resolve_domain_dns_cached
from app.services.dns_resolver import ( from app.services.dns_resolver import (
DomainDNSResult, DomainDNSResult,
extract_dmarc_policy, extract_dmarc_policy,
@@ -64,6 +65,8 @@ class DNSRecordResponse(BaseModel):
spfRecord: Optional[str] = None spfRecord: Optional[str] = None
dkim: bool dkim: bool
dkimSelectors: List[str] = [] dkimSelectors: List[str] = []
cached: bool = False
checkedAt: Optional[str] = None
class TimelinePoint(BaseModel): class TimelinePoint(BaseModel):
@@ -201,17 +204,17 @@ async def get_domains_summary(db: Session = Depends(get_db)):
""" """
Get summary statistics for all domains, formatted for the dashboard. Get summary statistics for all domains, formatted for the dashboard.
Performs live DNS lookups for each domain concurrently and includes the Performs cached DNS lookups for each domain and includes the results
results (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain entries.
entries. A per-domain timeout of 10 s prevents slow DNS responses from A per-domain timeout of 10 s prevents slow DNS responses from blocking the
blocking the page load. page load.
""" """
store = ReportStore.get_instance() store = ReportStore.get_instance()
hydrate_report_store_from_db(db, store) hydrate_report_store_from_db(db, store)
domains = store.get_domains() domains = store.get_domains()
summaries = store.get_all_domain_summaries() summaries = store.get_all_domain_summaries()
# Perform DNS checks concurrently for all domains # Perform DNS checks for all domains, reusing fresh cached results.
provider = get_default_provider() provider = get_default_provider()
manual_selectors_by_domain = _get_domain_selectors_map_from_db(db, domains) manual_selectors_by_domain = _get_domain_selectors_map_from_db(db, domains)
@@ -220,15 +223,20 @@ async def get_domains_summary(db: Session = Depends(get_db)):
report_selectors = _get_selectors_from_reports(store, domain_name) report_selectors = _get_selectors_from_reports(store, domain_name)
combined = list(dict.fromkeys(manual_selectors + report_selectors)) combined = list(dict.fromkeys(manual_selectors + report_selectors))
try: try:
return await asyncio.wait_for( result, cached, checked_at = await asyncio.wait_for(
provider.check_domain(domain_name, selectors=combined), resolve_domain_dns_cached(db, provider, domain_name, selectors=combined),
timeout=10.0, timeout=10.0,
) )
result.cached = cached # type: ignore[attr-defined]
result.checked_at = checked_at # type: ignore[attr-defined]
return result
except (asyncio.TimeoutError, LookupError, OSError) as exc: except (asyncio.TimeoutError, LookupError, OSError) as exc:
logger.warning("DNS check failed for %s: %s", domain_name, exc) logger.warning("DNS check failed for %s: %s", domain_name, exc)
return DomainDNSResult() return DomainDNSResult()
dns_results = await asyncio.gather(*[_dns_for_domain(d) for d in domains]) dns_results = []
for domain_name in domains:
dns_results.append(await _dns_for_domain(domain_name))
# Calculate overall statistics # Calculate overall statistics
total_domains = len(domains) total_domains = len(domains)
@@ -266,6 +274,12 @@ async def get_domains_summary(db: Session = Depends(get_db)):
"dmarc_policy": dmarc_policy, "dmarc_policy": dmarc_policy,
"spf_status": dns.spf, "spf_status": dns.spf,
"dkim_status": dns.dkim, "dkim_status": dns.dkim,
"dns_cached": getattr(dns, "cached", False),
"dns_checked_at": (
getattr(dns, "checked_at", None).isoformat()
if getattr(dns, "checked_at", None)
else None
),
} }
) )
@@ -375,6 +389,7 @@ async def get_domain_stats(
@router.get("/{domain_id}/dns", response_model=DNSRecordResponse) @router.get("/{domain_id}/dns", response_model=DNSRecordResponse)
async def get_domain_dns_records( async def get_domain_dns_records(
domain_id: str = Path(..., title="The domain ID or name"), domain_id: str = Path(..., title="The domain ID or name"),
refresh: bool = Query(False, title="Refresh cached DNS result"),
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
""" """
@@ -399,7 +414,13 @@ async def get_domain_dns_records(
combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors)) combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors))
provider = get_default_provider() provider = get_default_provider()
result = await provider.check_domain(domain_id, selectors=combined_selectors) result, cached, checked_at = await resolve_domain_dns_cached(
db,
provider,
domain_id,
selectors=combined_selectors,
refresh=refresh,
)
return DNSRecordResponse( return DNSRecordResponse(
dmarc=result.dmarc, dmarc=result.dmarc,
@@ -408,6 +429,8 @@ async def get_domain_dns_records(
spfRecord=result.spf_record, spfRecord=result.spf_record,
dkim=result.dkim, dkim=result.dkim,
dkimSelectors=result.dkim_selectors, dkimSelectors=result.dkim_selectors,
cached=cached,
checkedAt=checked_at.isoformat(),
) )
@@ -674,7 +697,11 @@ def _source_recommendations(
) )
) )
if spf_result == "pass" and dkim_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed: if (
spf_result == "pass"
and dkim_result in {"fail", "mixed", "unknown", "none"}
and dmarc_passed
):
recommendations.append( recommendations.append(
SourceRecommendation( SourceRecommendation(
type="spf_only_pass", type="spf_only_pass",
@@ -688,7 +715,11 @@ def _source_recommendations(
) )
) )
if dkim_result == "pass" and spf_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed: if (
dkim_result == "pass"
and spf_result in {"fail", "mixed", "unknown", "none"}
and dmarc_passed
):
action = "Authorize this service in SPF, or confirm SPF is intentionally handled elsewhere." action = "Authorize this service in SPF, or confirm SPF is intentionally handled elsewhere."
if spf_fix_hint: if spf_fix_hint:
action = f"Add {spf_fix_hint} to your SPF record if this service is legitimate." action = f"Add {spf_fix_hint} to your SPF record if this service is legitimate."
@@ -708,9 +739,7 @@ def _source_recommendations(
"both SPF authorization and DKIM signing." "both SPF authorization and DKIM signing."
) )
if spf_fix_hint: if spf_fix_hint:
action = ( action = f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service."
f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service."
)
recommendations.append( recommendations.append(
SourceRecommendation( SourceRecommendation(
type="full_fail", type="full_fail",
+1
View File
@@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates from fastapi.templating import Jinja2Templates
import app.models.alert # noqa: F401 ensure AlertHistory table is registered import app.models.alert # noqa: F401 ensure AlertHistory table is registered
import app.models.dns_cache # noqa: F401 ensure DNSCache table is registered
import app.models.domain # noqa: F401 ensure Domain/UserDomain tables are registered import app.models.domain # noqa: F401 ensure Domain/UserDomain tables are registered
import app.models.mail_source_import # noqa: F401 ensure import history table is registered import app.models.mail_source_import # noqa: F401 ensure import history table is registered
import app.models.report # noqa: F401 ensure DMARCReport/ReportRecord tables are registered import app.models.report # noqa: F401 ensure DMARCReport/ReportRecord tables are registered
+30
View File
@@ -0,0 +1,30 @@
from datetime import UTC, datetime
from sqlalchemy import Column, DateTime, Index, Integer, String, Text, UniqueConstraint
from app.core.database import Base
def _utcnow_naive() -> datetime:
return datetime.now(UTC).replace(tzinfo=None)
class DNSCache(Base):
"""Cached DNS authentication result for a domain and selector set."""
__tablename__ = "dns_cache"
id = Column(Integer, primary_key=True, index=True)
domain = Column(String, nullable=False, index=True)
provider = Column(String, nullable=False, index=True)
selectors_key = Column(String(64), nullable=False, index=True)
result_json = Column(Text, nullable=False)
checked_at = Column(DateTime, default=_utcnow_naive, nullable=False, index=True)
__table_args__ = (
UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"),
Index("ix_dns_cache_domain_checked", "domain", "checked_at"),
)
def __repr__(self):
return f"<DNSCache {self.domain} provider={self.provider}>"
+91
View File
@@ -0,0 +1,91 @@
"""Database-backed DNS result cache."""
from __future__ import annotations
import hashlib
import json
from dataclasses import asdict
from datetime import UTC, datetime, timedelta
from typing import List, Tuple
from sqlalchemy.orm import Session
from app.models.dns_cache import DNSCache
from app.services.dns_resolver import BaseDNSProvider, DomainDNSResult
DEFAULT_DNS_CACHE_TTL_SECONDS = 900
def _utcnow_naive() -> datetime:
return datetime.now(UTC).replace(tzinfo=None)
def _selectors_key(selectors: List[str]) -> str:
payload = json.dumps(list(dict.fromkeys(selectors or [])), separators=(",", ":"))
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _result_to_json(result: DomainDNSResult) -> str:
return json.dumps(asdict(result), sort_keys=True, separators=(",", ":"))
def _result_from_json(value: str) -> DomainDNSResult:
data = json.loads(value)
return DomainDNSResult(
dmarc=bool(data.get("dmarc")),
dmarc_record=data.get("dmarc_record"),
spf=bool(data.get("spf")),
spf_record=data.get("spf_record"),
dkim=bool(data.get("dkim")),
dkim_selectors=list(data.get("dkim_selectors") or []),
dkim_record=data.get("dkim_record"),
selectors_checked=list(data.get("selectors_checked") or []),
)
def _is_fresh(row: DNSCache, ttl_seconds: int, now: datetime) -> bool:
return row.checked_at >= now - timedelta(seconds=ttl_seconds)
async def resolve_domain_dns_cached(
db: Session,
provider: BaseDNSProvider,
domain: str,
*,
selectors: List[str],
ttl_seconds: int = DEFAULT_DNS_CACHE_TTL_SECONDS,
refresh: bool = False,
) -> Tuple[DomainDNSResult, bool, datetime]:
"""Resolve DNS for a domain, reusing a fresh cached result when available."""
now = _utcnow_naive()
provider_name = provider.__class__.__name__
selectors_key = _selectors_key(selectors)
row = (
db.query(DNSCache)
.filter(
DNSCache.domain == domain,
DNSCache.provider == provider_name,
DNSCache.selectors_key == selectors_key,
)
.first()
)
if row and not refresh and _is_fresh(row, ttl_seconds, now):
return _result_from_json(row.result_json), True, row.checked_at
result = await provider.check_domain(domain, selectors=selectors)
if row is None:
row = DNSCache(
domain=domain,
provider=provider_name,
selectors_key=selectors_key,
result_json=_result_to_json(result),
checked_at=now,
)
db.add(row)
else:
row.result_json = _result_to_json(result)
row.checked_at = now
db.commit()
db.refresh(row)
return result, False, row.checked_at
+1
View File
@@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool from sqlalchemy.pool import StaticPool
import app.models.alert # noqa: F401 # pylint: disable=unused-import import app.models.alert # noqa: F401 # pylint: disable=unused-import
import app.models.dns_cache # noqa: F401 # pylint: disable=unused-import
import app.models.domain # noqa: F401 # pylint: disable=unused-import import app.models.domain # noqa: F401 # pylint: disable=unused-import
import app.models.mail_source as _mail_source_model # noqa: F401 # pylint: disable=unused-import import app.models.mail_source as _mail_source_model # noqa: F401 # pylint: disable=unused-import
import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-import import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-import
+38
View File
@@ -15,6 +15,7 @@ from fastapi.testclient import TestClient
from app.api.api_v1.endpoints import domains as domains_endpoint from app.api.api_v1.endpoints import domains as domains_endpoint
from app.api.api_v1.endpoints.domains import _spf_fix_hint from app.api.api_v1.endpoints.domains import _spf_fix_hint
from app.models.dns_cache import DNSCache
from app.models.domain import Domain from app.models.domain import Domain
from app.services.dns_resolver import DomainDNSResult from app.services.dns_resolver import DomainDNSResult
from app.services.report_store import ReportStore from app.services.report_store import ReportStore
@@ -254,6 +255,43 @@ def test_dns_endpoint_returns_real_data(client: TestClient):
assert data["spf"] is True assert data["spf"] is True
assert data["dkim"] is True assert data["dkim"] is True
assert "p=none" in data["dmarcRecord"] assert "p=none" in data["dmarcRecord"]
assert data["cached"] is False
assert data["checkedAt"] is not None
def test_dns_endpoint_uses_cached_result(client: TestClient, db_session):
"""Repeated DNS checks reuse a fresh cached result."""
mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT))
with patch(
"app.api.api_v1.endpoints.domains.get_default_provider",
return_value=mock_provider,
):
first = client.get(f"/api/v1/domains/{DOMAIN}/dns")
second = client.get(f"/api/v1/domains/{DOMAIN}/dns")
assert first.status_code == 200
assert second.status_code == 200
assert first.json()["cached"] is False
assert second.json()["cached"] is True
assert mock_provider.check_domain.await_count == 1
assert db_session.query(DNSCache).count() == 1
def test_dns_endpoint_refresh_bypasses_cache(client: TestClient):
"""The refresh query parameter forces a new DNS lookup."""
mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT))
with patch(
"app.api.api_v1.endpoints.domains.get_default_provider",
return_value=mock_provider,
):
client.get(f"/api/v1/domains/{DOMAIN}/dns")
refreshed = client.get(f"/api/v1/domains/{DOMAIN}/dns?refresh=true")
assert refreshed.status_code == 200
assert refreshed.json()["cached"] is False
assert mock_provider.check_domain.await_count == 2
def test_dns_endpoint_uses_manual_selectors(client: TestClient): def test_dns_endpoint_uses_manual_selectors(client: TestClient):
+8
View File
@@ -109,6 +109,14 @@ policy if long-term storage size matters.
| `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` | | `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` |
| `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` | | `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` |
### DNS Result Cache
DMARC, SPF, and DKIM DNS checks are cached in the database-backed `dns_cache`
table for 15 minutes per domain, DNS provider, and DKIM selector set. Domain DNS
API responses include whether the result came from cache and when it was
checked. Use `?refresh=true` on the domain DNS endpoint to bypass a fresh cache
entry for operational rechecks.
### Advanced Configuration ### Advanced Configuration
| Variable | Description | Default | Example | | Variable | Description | Default | Example |
+4 -2
View File
@@ -128,9 +128,11 @@ Status: Planned
Goal: connect report findings with DNS configuration guidance. Goal: connect report findings with DNS configuration guidance.
Planned: Delivered:
- DMARC/SPF/DKIM DNS checks with cached results. - DMARC/SPF/DKIM DNS checks with database-backed cached results.
- DKIM selector discovery from report data. - DKIM selector discovery from report data.
Planned:
- Per-domain DNS health summary. - Per-domain DNS health summary.
- Suggestions for moving from `p=none` to enforcement when compliance supports it. - Suggestions for moving from `p=none` to enforcement when compliance supports it.
- Optional Cloudflare read-only integration for DNS record inspection. - Optional Cloudflare read-only integration for DNS record inspection.
+1
View File
@@ -168,6 +168,7 @@ Status: Complete for the delivered reporting milestone. Alert-specific dashboard
- [x] Add alert rules for new sender source, compliance drop, DMARC failures above threshold, and missing reports - [x] Add alert rules for new sender source, compliance drop, DMARC failures above threshold, and missing reports
- [x] Add daily and weekly summary notifications - [x] Add daily and weekly summary notifications
- [x] Add alert history - [x] Add alert history
- [x] Add cached DMARC/SPF/DKIM DNS checks and report-discovered DKIM selectors
- [ ] DNS health guidance and Cloudflare read-only inspection - [ ] DNS health guidance and Cloudflare read-only inspection
- [ ] Guided setup and operator health pages - [ ] Guided setup and operator health pages
- [ ] Forensic/RUF report support - [ ] Forensic/RUF report support