From 77fd0f95526539a0eead19dc18d5d3226fb8ba37 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 22 May 2026 23:30:12 +0200 Subject: [PATCH] feat: add cached DNS checks --- backend/alembic/env.py | 1 + .../versions/b8c9d0e1f2a3_add_dns_cache.py | 60 ++++++++++++ backend/app/api/api_v1/endpoints/domains.py | 57 +++++++++--- backend/app/main.py | 1 + backend/app/models/dns_cache.py | 30 ++++++ backend/app/services/dns_cache.py | 91 +++++++++++++++++++ backend/app/tests/conftest.py | 1 + backend/app/tests/test_dns_endpoints.py | 38 ++++++++ docs/deployment/configuration.md | 8 ++ docs/milestones.md | 6 +- docs/todo.md | 1 + 11 files changed, 278 insertions(+), 16 deletions(-) create mode 100644 backend/alembic/versions/b8c9d0e1f2a3_add_dns_cache.py create mode 100644 backend/app/models/dns_cache.py create mode 100644 backend/app/services/dns_cache.py diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 5c65b3e..f7281e9 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -21,6 +21,7 @@ if database_url: config.set_main_option("sqlalchemy.url", _make_sync_db_url(database_url)) import app.models.alert # noqa: E402, F401 +import app.models.dns_cache # noqa: E402, F401 import app.models.domain # noqa: E402, F401 import app.models.mail_source # noqa: E402, F401 import app.models.mail_source_import # noqa: E402, F401 diff --git a/backend/alembic/versions/b8c9d0e1f2a3_add_dns_cache.py b/backend/alembic/versions/b8c9d0e1f2a3_add_dns_cache.py new file mode 100644 index 0000000..32ba67f --- /dev/null +++ b/backend/alembic/versions/b8c9d0e1f2a3_add_dns_cache.py @@ -0,0 +1,60 @@ +"""add dns cache + +Revision ID: b8c9d0e1f2a3 +Revises: a7b8c9d0e1f2 +Create Date: 2026-05-22 23:28:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b8c9d0e1f2a3" +down_revision: Union[str, Sequence[str], None] = "a7b8c9d0e1f2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create a DNS result cache table.""" + op.create_table( + "dns_cache", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("domain", sa.String(), nullable=False), + sa.Column("provider", sa.String(), nullable=False), + sa.Column("selectors_key", sa.String(length=64), nullable=False), + sa.Column("result_json", sa.Text(), nullable=False), + sa.Column("checked_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"), + ) + op.create_index(op.f("ix_dns_cache_id"), "dns_cache", ["id"], unique=False) + op.create_index(op.f("ix_dns_cache_domain"), "dns_cache", ["domain"], unique=False) + op.create_index(op.f("ix_dns_cache_provider"), "dns_cache", ["provider"], unique=False) + op.create_index( + op.f("ix_dns_cache_selectors_key"), + "dns_cache", + ["selectors_key"], + unique=False, + ) + op.create_index(op.f("ix_dns_cache_checked_at"), "dns_cache", ["checked_at"], unique=False) + op.create_index( + "ix_dns_cache_domain_checked", + "dns_cache", + ["domain", "checked_at"], + unique=False, + ) + + +def downgrade() -> None: + """Drop the DNS result cache table.""" + op.drop_index("ix_dns_cache_domain_checked", table_name="dns_cache") + op.drop_index(op.f("ix_dns_cache_checked_at"), table_name="dns_cache") + op.drop_index(op.f("ix_dns_cache_selectors_key"), table_name="dns_cache") + op.drop_index(op.f("ix_dns_cache_provider"), table_name="dns_cache") + op.drop_index(op.f("ix_dns_cache_domain"), table_name="dns_cache") + op.drop_index(op.f("ix_dns_cache_id"), table_name="dns_cache") + op.drop_table("dns_cache") diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py index dcef5f4..cfec2ca 100644 --- a/backend/app/api/api_v1/endpoints/domains.py +++ b/backend/app/api/api_v1/endpoints/domains.py @@ -13,6 +13,7 @@ from sqlalchemy.orm import Session from app.core.database import get_db from app.models.domain import Domain +from app.services.dns_cache import resolve_domain_dns_cached from app.services.dns_resolver import ( DomainDNSResult, extract_dmarc_policy, @@ -64,6 +65,8 @@ class DNSRecordResponse(BaseModel): spfRecord: Optional[str] = None dkim: bool dkimSelectors: List[str] = [] + cached: bool = False + checkedAt: Optional[str] = None class TimelinePoint(BaseModel): @@ -201,17 +204,17 @@ async def get_domains_summary(db: Session = Depends(get_db)): """ Get summary statistics for all domains, formatted for the dashboard. - Performs live DNS lookups for each domain concurrently and includes the - results (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain - entries. A per-domain timeout of 10 s prevents slow DNS responses from - blocking the page load. + Performs cached DNS lookups for each domain and includes the results + (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain entries. + A per-domain timeout of 10 s prevents slow DNS responses from blocking the + page load. """ store = ReportStore.get_instance() hydrate_report_store_from_db(db, store) domains = store.get_domains() summaries = store.get_all_domain_summaries() - # Perform DNS checks concurrently for all domains + # Perform DNS checks for all domains, reusing fresh cached results. provider = get_default_provider() manual_selectors_by_domain = _get_domain_selectors_map_from_db(db, domains) @@ -220,15 +223,20 @@ async def get_domains_summary(db: Session = Depends(get_db)): report_selectors = _get_selectors_from_reports(store, domain_name) combined = list(dict.fromkeys(manual_selectors + report_selectors)) try: - return await asyncio.wait_for( - provider.check_domain(domain_name, selectors=combined), + result, cached, checked_at = await asyncio.wait_for( + resolve_domain_dns_cached(db, provider, domain_name, selectors=combined), timeout=10.0, ) + result.cached = cached # type: ignore[attr-defined] + result.checked_at = checked_at # type: ignore[attr-defined] + return result except (asyncio.TimeoutError, LookupError, OSError) as exc: logger.warning("DNS check failed for %s: %s", domain_name, exc) return DomainDNSResult() - dns_results = await asyncio.gather(*[_dns_for_domain(d) for d in domains]) + dns_results = [] + for domain_name in domains: + dns_results.append(await _dns_for_domain(domain_name)) # Calculate overall statistics total_domains = len(domains) @@ -266,6 +274,12 @@ async def get_domains_summary(db: Session = Depends(get_db)): "dmarc_policy": dmarc_policy, "spf_status": dns.spf, "dkim_status": dns.dkim, + "dns_cached": getattr(dns, "cached", False), + "dns_checked_at": ( + getattr(dns, "checked_at", None).isoformat() + if getattr(dns, "checked_at", None) + else None + ), } ) @@ -375,6 +389,7 @@ async def get_domain_stats( @router.get("/{domain_id}/dns", response_model=DNSRecordResponse) async def get_domain_dns_records( domain_id: str = Path(..., title="The domain ID or name"), + refresh: bool = Query(False, title="Refresh cached DNS result"), db: Session = Depends(get_db), ): """ @@ -399,7 +414,13 @@ async def get_domain_dns_records( combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors)) provider = get_default_provider() - result = await provider.check_domain(domain_id, selectors=combined_selectors) + result, cached, checked_at = await resolve_domain_dns_cached( + db, + provider, + domain_id, + selectors=combined_selectors, + refresh=refresh, + ) return DNSRecordResponse( dmarc=result.dmarc, @@ -408,6 +429,8 @@ async def get_domain_dns_records( spfRecord=result.spf_record, dkim=result.dkim, dkimSelectors=result.dkim_selectors, + cached=cached, + checkedAt=checked_at.isoformat(), ) @@ -674,7 +697,11 @@ def _source_recommendations( ) ) - if spf_result == "pass" and dkim_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed: + if ( + spf_result == "pass" + and dkim_result in {"fail", "mixed", "unknown", "none"} + and dmarc_passed + ): recommendations.append( SourceRecommendation( type="spf_only_pass", @@ -688,7 +715,11 @@ def _source_recommendations( ) ) - if dkim_result == "pass" and spf_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed: + if ( + dkim_result == "pass" + and spf_result in {"fail", "mixed", "unknown", "none"} + and dmarc_passed + ): action = "Authorize this service in SPF, or confirm SPF is intentionally handled elsewhere." if spf_fix_hint: action = f"Add {spf_fix_hint} to your SPF record if this service is legitimate." @@ -708,9 +739,7 @@ def _source_recommendations( "both SPF authorization and DKIM signing." ) if spf_fix_hint: - action = ( - f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service." - ) + action = f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service." recommendations.append( SourceRecommendation( type="full_fail", diff --git a/backend/app/main.py b/backend/app/main.py index 0b08498..20ff084 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates import app.models.alert # noqa: F401 – ensure AlertHistory table is registered +import app.models.dns_cache # noqa: F401 – ensure DNSCache table is registered import app.models.domain # noqa: F401 – ensure Domain/UserDomain tables are registered import app.models.mail_source_import # noqa: F401 – ensure import history table is registered import app.models.report # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered diff --git a/backend/app/models/dns_cache.py b/backend/app/models/dns_cache.py new file mode 100644 index 0000000..f2580d0 --- /dev/null +++ b/backend/app/models/dns_cache.py @@ -0,0 +1,30 @@ +from datetime import UTC, datetime + +from sqlalchemy import Column, DateTime, Index, Integer, String, Text, UniqueConstraint + +from app.core.database import Base + + +def _utcnow_naive() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +class DNSCache(Base): + """Cached DNS authentication result for a domain and selector set.""" + + __tablename__ = "dns_cache" + + id = Column(Integer, primary_key=True, index=True) + domain = Column(String, nullable=False, index=True) + provider = Column(String, nullable=False, index=True) + selectors_key = Column(String(64), nullable=False, index=True) + result_json = Column(Text, nullable=False) + checked_at = Column(DateTime, default=_utcnow_naive, nullable=False, index=True) + + __table_args__ = ( + UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"), + Index("ix_dns_cache_domain_checked", "domain", "checked_at"), + ) + + def __repr__(self): + return f"" diff --git a/backend/app/services/dns_cache.py b/backend/app/services/dns_cache.py new file mode 100644 index 0000000..70b0943 --- /dev/null +++ b/backend/app/services/dns_cache.py @@ -0,0 +1,91 @@ +"""Database-backed DNS result cache.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict +from datetime import UTC, datetime, timedelta +from typing import List, Tuple + +from sqlalchemy.orm import Session + +from app.models.dns_cache import DNSCache +from app.services.dns_resolver import BaseDNSProvider, DomainDNSResult + +DEFAULT_DNS_CACHE_TTL_SECONDS = 900 + + +def _utcnow_naive() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +def _selectors_key(selectors: List[str]) -> str: + payload = json.dumps(list(dict.fromkeys(selectors or [])), separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _result_to_json(result: DomainDNSResult) -> str: + return json.dumps(asdict(result), sort_keys=True, separators=(",", ":")) + + +def _result_from_json(value: str) -> DomainDNSResult: + data = json.loads(value) + return DomainDNSResult( + dmarc=bool(data.get("dmarc")), + dmarc_record=data.get("dmarc_record"), + spf=bool(data.get("spf")), + spf_record=data.get("spf_record"), + dkim=bool(data.get("dkim")), + dkim_selectors=list(data.get("dkim_selectors") or []), + dkim_record=data.get("dkim_record"), + selectors_checked=list(data.get("selectors_checked") or []), + ) + + +def _is_fresh(row: DNSCache, ttl_seconds: int, now: datetime) -> bool: + return row.checked_at >= now - timedelta(seconds=ttl_seconds) + + +async def resolve_domain_dns_cached( + db: Session, + provider: BaseDNSProvider, + domain: str, + *, + selectors: List[str], + ttl_seconds: int = DEFAULT_DNS_CACHE_TTL_SECONDS, + refresh: bool = False, +) -> Tuple[DomainDNSResult, bool, datetime]: + """Resolve DNS for a domain, reusing a fresh cached result when available.""" + now = _utcnow_naive() + provider_name = provider.__class__.__name__ + selectors_key = _selectors_key(selectors) + row = ( + db.query(DNSCache) + .filter( + DNSCache.domain == domain, + DNSCache.provider == provider_name, + DNSCache.selectors_key == selectors_key, + ) + .first() + ) + + if row and not refresh and _is_fresh(row, ttl_seconds, now): + return _result_from_json(row.result_json), True, row.checked_at + + result = await provider.check_domain(domain, selectors=selectors) + if row is None: + row = DNSCache( + domain=domain, + provider=provider_name, + selectors_key=selectors_key, + result_json=_result_to_json(result), + checked_at=now, + ) + db.add(row) + else: + row.result_json = _result_to_json(result) + row.checked_at = now + db.commit() + db.refresh(row) + return result, False, row.checked_at diff --git a/backend/app/tests/conftest.py b/backend/app/tests/conftest.py index 294e025..915d41c 100644 --- a/backend/app/tests/conftest.py +++ b/backend/app/tests/conftest.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool import app.models.alert # noqa: F401 # pylint: disable=unused-import +import app.models.dns_cache # noqa: F401 # pylint: disable=unused-import import app.models.domain # noqa: F401 # pylint: disable=unused-import import app.models.mail_source as _mail_source_model # noqa: F401 # pylint: disable=unused-import import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-import diff --git a/backend/app/tests/test_dns_endpoints.py b/backend/app/tests/test_dns_endpoints.py index e78a5c6..ff11acc 100644 --- a/backend/app/tests/test_dns_endpoints.py +++ b/backend/app/tests/test_dns_endpoints.py @@ -15,6 +15,7 @@ from fastapi.testclient import TestClient from app.api.api_v1.endpoints import domains as domains_endpoint from app.api.api_v1.endpoints.domains import _spf_fix_hint +from app.models.dns_cache import DNSCache from app.models.domain import Domain from app.services.dns_resolver import DomainDNSResult from app.services.report_store import ReportStore @@ -254,6 +255,43 @@ def test_dns_endpoint_returns_real_data(client: TestClient): assert data["spf"] is True assert data["dkim"] is True assert "p=none" in data["dmarcRecord"] + assert data["cached"] is False + assert data["checkedAt"] is not None + + +def test_dns_endpoint_uses_cached_result(client: TestClient, db_session): + """Repeated DNS checks reuse a fresh cached result.""" + mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT)) + + with patch( + "app.api.api_v1.endpoints.domains.get_default_provider", + return_value=mock_provider, + ): + first = client.get(f"/api/v1/domains/{DOMAIN}/dns") + second = client.get(f"/api/v1/domains/{DOMAIN}/dns") + + assert first.status_code == 200 + assert second.status_code == 200 + assert first.json()["cached"] is False + assert second.json()["cached"] is True + assert mock_provider.check_domain.await_count == 1 + assert db_session.query(DNSCache).count() == 1 + + +def test_dns_endpoint_refresh_bypasses_cache(client: TestClient): + """The refresh query parameter forces a new DNS lookup.""" + mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT)) + + with patch( + "app.api.api_v1.endpoints.domains.get_default_provider", + return_value=mock_provider, + ): + client.get(f"/api/v1/domains/{DOMAIN}/dns") + refreshed = client.get(f"/api/v1/domains/{DOMAIN}/dns?refresh=true") + + assert refreshed.status_code == 200 + assert refreshed.json()["cached"] is False + assert mock_provider.check_domain.await_count == 2 def test_dns_endpoint_uses_manual_selectors(client: TestClient): diff --git a/docs/deployment/configuration.md b/docs/deployment/configuration.md index 094b4f6..3659703 100644 --- a/docs/deployment/configuration.md +++ b/docs/deployment/configuration.md @@ -109,6 +109,14 @@ policy if long-term storage size matters. | `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` | | `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` | +### DNS Result Cache + +DMARC, SPF, and DKIM DNS checks are cached in the database-backed `dns_cache` +table for 15 minutes per domain, DNS provider, and DKIM selector set. Domain DNS +API responses include whether the result came from cache and when it was +checked. Use `?refresh=true` on the domain DNS endpoint to bypass a fresh cache +entry for operational rechecks. + ### Advanced Configuration | Variable | Description | Default | Example | diff --git a/docs/milestones.md b/docs/milestones.md index bac2a45..d91ac7b 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -128,9 +128,11 @@ Status: Planned Goal: connect report findings with DNS configuration guidance. -Planned: -- DMARC/SPF/DKIM DNS checks with cached results. +Delivered: +- DMARC/SPF/DKIM DNS checks with database-backed cached results. - DKIM selector discovery from report data. + +Planned: - Per-domain DNS health summary. - Suggestions for moving from `p=none` to enforcement when compliance supports it. - Optional Cloudflare read-only integration for DNS record inspection. diff --git a/docs/todo.md b/docs/todo.md index 26e2252..52ca2ec 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -168,6 +168,7 @@ Status: Complete for the delivered reporting milestone. Alert-specific dashboard - [x] Add alert rules for new sender source, compliance drop, DMARC failures above threshold, and missing reports - [x] Add daily and weekly summary notifications - [x] Add alert history +- [x] Add cached DMARC/SPF/DKIM DNS checks and report-discovered DKIM selectors - [ ] DNS health guidance and Cloudflare read-only inspection - [ ] Guided setup and operator health pages - [ ] Forensic/RUF report support