Merge pull request #123 from christianlouis/codex/dns-result-cache

[codex] add cached DNS checks
2026-05-22 23:33:16 +02:00
parent a39b273419 77fd0f9552
commit 55a5faa7e6
11 changed files with 278 additions and 16 deletions
@@ -21,6 +21,7 @@ if database_url:
    config.set_main_option("sqlalchemy.url", _make_sync_db_url(database_url))

 import app.models.alert  # noqa: E402, F401
+import app.models.dns_cache  # noqa: E402, F401
 import app.models.domain  # noqa: E402, F401
 import app.models.mail_source  # noqa: E402, F401
 import app.models.mail_source_import  # noqa: E402, F401
@@ -0,0 +1,60 @@
+"""add dns cache
+
+Revision ID: b8c9d0e1f2a3
+Revises: a7b8c9d0e1f2
+Create Date: 2026-05-22 23:28:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b8c9d0e1f2a3"
+down_revision: Union[str, Sequence[str], None] = "a7b8c9d0e1f2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Create a DNS result cache table."""
+    op.create_table(
+        "dns_cache",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("domain", sa.String(), nullable=False),
+        sa.Column("provider", sa.String(), nullable=False),
+        sa.Column("selectors_key", sa.String(length=64), nullable=False),
+        sa.Column("result_json", sa.Text(), nullable=False),
+        sa.Column("checked_at", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"),
+    )
+    op.create_index(op.f("ix_dns_cache_id"), "dns_cache", ["id"], unique=False)
+    op.create_index(op.f("ix_dns_cache_domain"), "dns_cache", ["domain"], unique=False)
+    op.create_index(op.f("ix_dns_cache_provider"), "dns_cache", ["provider"], unique=False)
+    op.create_index(
+        op.f("ix_dns_cache_selectors_key"),
+        "dns_cache",
+        ["selectors_key"],
+        unique=False,
+    )
+    op.create_index(op.f("ix_dns_cache_checked_at"), "dns_cache", ["checked_at"], unique=False)
+    op.create_index(
+        "ix_dns_cache_domain_checked",
+        "dns_cache",
+        ["domain", "checked_at"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    """Drop the DNS result cache table."""
+    op.drop_index("ix_dns_cache_domain_checked", table_name="dns_cache")
+    op.drop_index(op.f("ix_dns_cache_checked_at"), table_name="dns_cache")
+    op.drop_index(op.f("ix_dns_cache_selectors_key"), table_name="dns_cache")
+    op.drop_index(op.f("ix_dns_cache_provider"), table_name="dns_cache")
+    op.drop_index(op.f("ix_dns_cache_domain"), table_name="dns_cache")
+    op.drop_index(op.f("ix_dns_cache_id"), table_name="dns_cache")
+    op.drop_table("dns_cache")
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session

 from app.core.database import get_db
 from app.models.domain import Domain
+from app.services.dns_cache import resolve_domain_dns_cached
 from app.services.dns_resolver import (
    DomainDNSResult,
    extract_dmarc_policy,
@@ -64,6 +65,8 @@ class DNSRecordResponse(BaseModel):
    spfRecord: Optional[str] = None
    dkim: bool
    dkimSelectors: List[str] = []
+    cached: bool = False
+    checkedAt: Optional[str] = None


 class TimelinePoint(BaseModel):
@@ -201,17 +204,17 @@ async def get_domains_summary(db: Session = Depends(get_db)):
    """
    Get summary statistics for all domains, formatted for the dashboard.

-    Performs live DNS lookups for each domain concurrently and includes the
-    results (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain
-    entries.  A per-domain timeout of 10 s prevents slow DNS responses from
-    blocking the page load.
+    Performs cached DNS lookups for each domain and includes the results
+    (DMARC/SPF/DKIM status and live DMARC policy) in the per-domain entries.
+    A per-domain timeout of 10 s prevents slow DNS responses from blocking the
+    page load.
    """
    store = ReportStore.get_instance()
    hydrate_report_store_from_db(db, store)
    domains = store.get_domains()
    summaries = store.get_all_domain_summaries()

-    # Perform DNS checks concurrently for all domains
+    # Perform DNS checks for all domains, reusing fresh cached results.
    provider = get_default_provider()
    manual_selectors_by_domain = _get_domain_selectors_map_from_db(db, domains)

@@ -220,15 +223,20 @@ async def get_domains_summary(db: Session = Depends(get_db)):
        report_selectors = _get_selectors_from_reports(store, domain_name)
        combined = list(dict.fromkeys(manual_selectors + report_selectors))
        try:
-            return await asyncio.wait_for(
-                provider.check_domain(domain_name, selectors=combined),
+            result, cached, checked_at = await asyncio.wait_for(
+                resolve_domain_dns_cached(db, provider, domain_name, selectors=combined),
                timeout=10.0,
            )
+            result.cached = cached  # type: ignore[attr-defined]
+            result.checked_at = checked_at  # type: ignore[attr-defined]
+            return result
        except (asyncio.TimeoutError, LookupError, OSError) as exc:
            logger.warning("DNS check failed for %s: %s", domain_name, exc)
            return DomainDNSResult()

-    dns_results = await asyncio.gather(*[_dns_for_domain(d) for d in domains])
+    dns_results = []
+    for domain_name in domains:
+        dns_results.append(await _dns_for_domain(domain_name))

    # Calculate overall statistics
    total_domains = len(domains)
@@ -266,6 +274,12 @@ async def get_domains_summary(db: Session = Depends(get_db)):
                "dmarc_policy": dmarc_policy,
                "spf_status": dns.spf,
                "dkim_status": dns.dkim,
+                "dns_cached": getattr(dns, "cached", False),
+                "dns_checked_at": (
+                    getattr(dns, "checked_at", None).isoformat()
+                    if getattr(dns, "checked_at", None)
+                    else None
+                ),
            }
        )

@@ -375,6 +389,7 @@ async def get_domain_stats(
@router.get("/{domain_id}/dns", response_model=DNSRecordResponse)
 async def get_domain_dns_records(
    domain_id: str = Path(..., title="The domain ID or name"),
+    refresh: bool = Query(False, title="Refresh cached DNS result"),
    db: Session = Depends(get_db),
 ):
    """
@@ -399,7 +414,13 @@ async def get_domain_dns_records(
    combined_selectors = list(dict.fromkeys(manual_selectors + report_selectors))

    provider = get_default_provider()
-    result = await provider.check_domain(domain_id, selectors=combined_selectors)
+    result, cached, checked_at = await resolve_domain_dns_cached(
+        db,
+        provider,
+        domain_id,
+        selectors=combined_selectors,
+        refresh=refresh,
+    )

    return DNSRecordResponse(
        dmarc=result.dmarc,
@@ -408,6 +429,8 @@ async def get_domain_dns_records(
        spfRecord=result.spf_record,
        dkim=result.dkim,
        dkimSelectors=result.dkim_selectors,
+        cached=cached,
+        checkedAt=checked_at.isoformat(),
    )


@@ -674,7 +697,11 @@ def _source_recommendations(
            )
        )

-    if spf_result == "pass" and dkim_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed:
+    if (
+        spf_result == "pass"
+        and dkim_result in {"fail", "mixed", "unknown", "none"}
+        and dmarc_passed
+    ):
        recommendations.append(
            SourceRecommendation(
                type="spf_only_pass",
@@ -688,7 +715,11 @@ def _source_recommendations(
            )
        )

-    if dkim_result == "pass" and spf_result in {"fail", "mixed", "unknown", "none"} and dmarc_passed:
+    if (
+        dkim_result == "pass"
+        and spf_result in {"fail", "mixed", "unknown", "none"}
+        and dmarc_passed
+    ):
        action = "Authorize this service in SPF, or confirm SPF is intentionally handled elsewhere."
        if spf_fix_hint:
            action = f"Add {spf_fix_hint} to your SPF record if this service is legitimate."
@@ -708,9 +739,7 @@ def _source_recommendations(
            "both SPF authorization and DKIM signing."
        )
        if spf_fix_hint:
-            action = (
-                f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service."
-            )
+            action = f"If legitimate, add {spf_fix_hint} to SPF and enable DKIM signing for this service."
        recommendations.append(
            SourceRecommendation(
                type="full_fail",
@@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates

 import app.models.alert  # noqa: F401 – ensure AlertHistory table is registered
+import app.models.dns_cache  # noqa: F401 – ensure DNSCache table is registered
 import app.models.domain  # noqa: F401 – ensure Domain/UserDomain tables are registered
 import app.models.mail_source_import  # noqa: F401 – ensure import history table is registered
 import app.models.report  # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered
@@ -0,0 +1,30 @@
+from datetime import UTC, datetime
+
+from sqlalchemy import Column, DateTime, Index, Integer, String, Text, UniqueConstraint
+
+from app.core.database import Base
+
+
+def _utcnow_naive() -> datetime:
+    return datetime.now(UTC).replace(tzinfo=None)
+
+
+class DNSCache(Base):
+    """Cached DNS authentication result for a domain and selector set."""
+
+    __tablename__ = "dns_cache"
+
+    id = Column(Integer, primary_key=True, index=True)
+    domain = Column(String, nullable=False, index=True)
+    provider = Column(String, nullable=False, index=True)
+    selectors_key = Column(String(64), nullable=False, index=True)
+    result_json = Column(Text, nullable=False)
+    checked_at = Column(DateTime, default=_utcnow_naive, nullable=False, index=True)
+
+    __table_args__ = (
+        UniqueConstraint("domain", "provider", "selectors_key", name="uq_dns_cache_lookup"),
+        Index("ix_dns_cache_domain_checked", "domain", "checked_at"),
+    )
+
+    def __repr__(self):
+        return f"<DNSCache {self.domain} provider={self.provider}>"
@@ -0,0 +1,91 @@
+"""Database-backed DNS result cache."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict
+from datetime import UTC, datetime, timedelta
+from typing import List, Tuple
+
+from sqlalchemy.orm import Session
+
+from app.models.dns_cache import DNSCache
+from app.services.dns_resolver import BaseDNSProvider, DomainDNSResult
+
+DEFAULT_DNS_CACHE_TTL_SECONDS = 900
+
+
+def _utcnow_naive() -> datetime:
+    return datetime.now(UTC).replace(tzinfo=None)
+
+
+def _selectors_key(selectors: List[str]) -> str:
+    payload = json.dumps(list(dict.fromkeys(selectors or [])), separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _result_to_json(result: DomainDNSResult) -> str:
+    return json.dumps(asdict(result), sort_keys=True, separators=(",", ":"))
+
+
+def _result_from_json(value: str) -> DomainDNSResult:
+    data = json.loads(value)
+    return DomainDNSResult(
+        dmarc=bool(data.get("dmarc")),
+        dmarc_record=data.get("dmarc_record"),
+        spf=bool(data.get("spf")),
+        spf_record=data.get("spf_record"),
+        dkim=bool(data.get("dkim")),
+        dkim_selectors=list(data.get("dkim_selectors") or []),
+        dkim_record=data.get("dkim_record"),
+        selectors_checked=list(data.get("selectors_checked") or []),
+    )
+
+
+def _is_fresh(row: DNSCache, ttl_seconds: int, now: datetime) -> bool:
+    return row.checked_at >= now - timedelta(seconds=ttl_seconds)
+
+
+async def resolve_domain_dns_cached(
+    db: Session,
+    provider: BaseDNSProvider,
+    domain: str,
+    *,
+    selectors: List[str],
+    ttl_seconds: int = DEFAULT_DNS_CACHE_TTL_SECONDS,
+    refresh: bool = False,
+) -> Tuple[DomainDNSResult, bool, datetime]:
+    """Resolve DNS for a domain, reusing a fresh cached result when available."""
+    now = _utcnow_naive()
+    provider_name = provider.__class__.__name__
+    selectors_key = _selectors_key(selectors)
+    row = (
+        db.query(DNSCache)
+        .filter(
+            DNSCache.domain == domain,
+            DNSCache.provider == provider_name,
+            DNSCache.selectors_key == selectors_key,
+        )
+        .first()
+    )
+
+    if row and not refresh and _is_fresh(row, ttl_seconds, now):
+        return _result_from_json(row.result_json), True, row.checked_at
+
+    result = await provider.check_domain(domain, selectors=selectors)
+    if row is None:
+        row = DNSCache(
+            domain=domain,
+            provider=provider_name,
+            selectors_key=selectors_key,
+            result_json=_result_to_json(result),
+            checked_at=now,
+        )
+        db.add(row)
+    else:
+        row.result_json = _result_to_json(result)
+        row.checked_at = now
+    db.commit()
+    db.refresh(row)
+    return result, False, row.checked_at
@@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy.pool import StaticPool

 import app.models.alert  # noqa: F401  # pylint: disable=unused-import
+import app.models.dns_cache  # noqa: F401  # pylint: disable=unused-import
 import app.models.domain  # noqa: F401  # pylint: disable=unused-import
 import app.models.mail_source as _mail_source_model  # noqa: F401  # pylint: disable=unused-import
 import app.models.mail_source_import  # noqa: F401  # pylint: disable=unused-import
@@ -15,6 +15,7 @@ from fastapi.testclient import TestClient

 from app.api.api_v1.endpoints import domains as domains_endpoint
 from app.api.api_v1.endpoints.domains import _spf_fix_hint
+from app.models.dns_cache import DNSCache
 from app.models.domain import Domain
 from app.services.dns_resolver import DomainDNSResult
 from app.services.report_store import ReportStore
@@ -254,6 +255,43 @@ def test_dns_endpoint_returns_real_data(client: TestClient):
    assert data["spf"] is True
    assert data["dkim"] is True
    assert "p=none" in data["dmarcRecord"]
+    assert data["cached"] is False
+    assert data["checkedAt"] is not None
+
+
+def test_dns_endpoint_uses_cached_result(client: TestClient, db_session):
+    """Repeated DNS checks reuse a fresh cached result."""
+    mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT))
+
+    with patch(
+        "app.api.api_v1.endpoints.domains.get_default_provider",
+        return_value=mock_provider,
+    ):
+        first = client.get(f"/api/v1/domains/{DOMAIN}/dns")
+        second = client.get(f"/api/v1/domains/{DOMAIN}/dns")
+
+    assert first.status_code == 200
+    assert second.status_code == 200
+    assert first.json()["cached"] is False
+    assert second.json()["cached"] is True
+    assert mock_provider.check_domain.await_count == 1
+    assert db_session.query(DNSCache).count() == 1
+
+
+def test_dns_endpoint_refresh_bypasses_cache(client: TestClient):
+    """The refresh query parameter forces a new DNS lookup."""
+    mock_provider = AsyncMock(check_domain=AsyncMock(return_value=MOCK_DNS_RESULT))
+
+    with patch(
+        "app.api.api_v1.endpoints.domains.get_default_provider",
+        return_value=mock_provider,
+    ):
+        client.get(f"/api/v1/domains/{DOMAIN}/dns")
+        refreshed = client.get(f"/api/v1/domains/{DOMAIN}/dns?refresh=true")
+
+    assert refreshed.status_code == 200
+    assert refreshed.json()["cached"] is False
+    assert mock_provider.check_domain.await_count == 2


 def test_dns_endpoint_uses_manual_selectors(client: TestClient):
@@ -109,6 +109,14 @@ policy if long-term storage size matters.
 | `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` |
 | `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` |

+### DNS Result Cache
+
+DMARC, SPF, and DKIM DNS checks are cached in the database-backed `dns_cache`
+table for 15 minutes per domain, DNS provider, and DKIM selector set. Domain DNS
+API responses include whether the result came from cache and when it was
+checked. Use `?refresh=true` on the domain DNS endpoint to bypass a fresh cache
+entry for operational rechecks.
+
 ### Advanced Configuration

 | Variable | Description | Default | Example |
@@ -128,9 +128,11 @@ Status: Planned

 Goal: connect report findings with DNS configuration guidance.

-Planned:
- DMARC/SPF/DKIM DNS checks with cached results.
+Delivered:
+- DMARC/SPF/DKIM DNS checks with database-backed cached results.
 - DKIM selector discovery from report data.
+
+Planned:
 - Per-domain DNS health summary.
 - Suggestions for moving from `p=none` to enforcement when compliance supports it.
 - Optional Cloudflare read-only integration for DNS record inspection.
@@ -168,6 +168,7 @@ Status: Complete for the delivered reporting milestone. Alert-specific dashboard
 - [x] Add alert rules for new sender source, compliance drop, DMARC failures above threshold, and missing reports
 - [x] Add daily and weekly summary notifications
 - [x] Add alert history
+- [x] Add cached DMARC/SPF/DKIM DNS checks and report-discovered DKIM selectors
 - [ ] DNS health guidance and Cloudflare read-only inspection
 - [ ] Guided setup and operator health pages
 - [ ] Forensic/RUF report support