feat: add operations health and domain setup polish

This commit is contained in:
Christian Krakau-Louis
2026-05-23 11:37:10 +02:00
parent ef506ca654
commit beae6e7469
18 changed files with 622 additions and 42 deletions
+1
View File
@@ -130,6 +130,7 @@ celerybeat.pid
# Environments # Environments
.env .env
.venv .venv
.pipcache/
env/ env/
venv/ venv/
ENV/ ENV/
+100 -6
View File
@@ -9,9 +9,11 @@ from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Path, Query, status from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
from fastapi.responses import Response from fastapi.responses import Response
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.database import get_db from app.core.database import get_db
from app.core.security import require_admin_auth
from app.models.domain import Domain from app.models.domain import Domain
from app.services.cloudflare_dns import ( from app.services.cloudflare_dns import (
analyze_dns_records, analyze_dns_records,
@@ -32,6 +34,7 @@ from app.services.report_persistence import (
hydrate_report_store_from_db, hydrate_report_store_from_db,
) )
from app.services.report_store import ReportStore from app.services.report_store import ReportStore
from app.utils.domain_validator import validate_domain_config
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -55,6 +58,14 @@ class DomainResponse(DomainBase):
compliance_rate: float = 0.0 compliance_rate: float = 0.0
class DomainCreate(BaseModel):
"""Payload for creating a monitored domain."""
name: str
description: Optional[str] = None
dkim_selectors: Optional[List[str]] = None
class DomainStatsResponse(BaseModel): class DomainStatsResponse(BaseModel):
"""Domain statistics for the domain details page""" """Domain statistics for the domain details page"""
@@ -283,6 +294,22 @@ def _policy_enforcement_suggestions(
return [] return []
def _normalize_domain_name(name: str) -> str:
return name.strip().strip(".").lower()
def _domain_names_for_summary(db: Session, store: ReportStore) -> List[str]:
report_domains = store.get_domains()
stored_domains = [
name
for (name,) in db.query(Domain.name)
.filter(Domain.active == True) # noqa: E712
.order_by(Domain.name)
.all()
]
return list(dict.fromkeys(stored_domains + report_domains))
@router.get("/summary", response_model=DomainSummaryResponse) @router.get("/summary", response_model=DomainSummaryResponse)
async def get_domains_summary(db: Session = Depends(get_db)): async def get_domains_summary(db: Session = Depends(get_db)):
""" """
@@ -295,7 +322,7 @@ async def get_domains_summary(db: Session = Depends(get_db)):
""" """
store = ReportStore.get_instance() store = ReportStore.get_instance()
hydrate_report_store_from_db(db, store) hydrate_report_store_from_db(db, store)
domains = store.get_domains() domains = _domain_names_for_summary(db, store)
summaries = store.get_all_domain_summaries() summaries = store.get_all_domain_summaries()
# Perform DNS checks for all domains, reusing fresh cached results. # Perform DNS checks for all domains, reusing fresh cached results.
@@ -385,19 +412,28 @@ async def get_domains_summary(db: Session = Depends(get_db)):
async def read_domains(db: Session = Depends(get_db)): async def read_domains(db: Session = Depends(get_db)):
""" """
Retrieve domains with their statistics. Retrieve domains with their statistics.
For Milestone 1, this simply returns domains from the in-memory store.
""" """
store = ReportStore.get_instance() store = ReportStore.get_instance()
hydrate_report_store_from_db(db, store) hydrate_report_store_from_db(db, store)
domains = store.get_domains() domains = _domain_names_for_summary(db, store)
summaries = store.get_all_domain_summaries() summaries = store.get_all_domain_summaries()
stored = {
domain.name: domain
for domain in db.query(Domain).filter(Domain.name.in_(domains)).all()
}
result = [] result = []
for domain_name in domains: for domain_name in domains:
summary = summaries.get(domain_name, {}) summary = summaries.get(domain_name, {})
stored_domain = stored.get(domain_name)
domain_response = DomainResponse( domain_response = DomainResponse(
name=domain_name, name=domain_name,
policy=summary.get("policy", "unknown"), description=stored_domain.description if stored_domain else None,
policy=(
summary.get("policy")
or (stored_domain.dmarc_policy if stored_domain else None)
or "unknown"
),
reports_count=summary.get("reports_processed", 0), reports_count=summary.get("reports_processed", 0),
emails_count=summary.get("total_count", 0), emails_count=summary.get("total_count", 0),
compliance_rate=summary.get("compliance_rate", 0.0), compliance_rate=summary.get("compliance_rate", 0.0),
@@ -407,6 +443,58 @@ async def read_domains(db: Session = Depends(get_db)):
return result return result
@router.post("/domains", response_model=DomainResponse, status_code=status.HTTP_201_CREATED)
async def create_domain(
payload: DomainCreate,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
):
"""Create a monitored domain before any DMARC reports have arrived."""
name = _normalize_domain_name(payload.name)
validation = validate_domain_config(
{"name": name, "description": payload.description or ""}
)
if not validation["valid"]:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=validation["errors"],
)
existing = db.query(Domain).filter(Domain.name == name).first()
if existing:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Domain is already monitored",
)
selectors = ",".join(
selector.strip()
for selector in payload.dkim_selectors or []
if selector and selector.strip()
)
domain = Domain(
name=name,
description=payload.description,
dkim_selectors=selectors or None,
active=True,
verified=False,
)
db.add(domain)
try:
db.commit()
except IntegrityError as exc:
db.rollback()
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Domain is already monitored",
) from exc
db.refresh(domain)
return DomainResponse(
name=domain.name,
description=domain.description,
policy=domain.dmarc_policy or "unknown",
)
@router.get("/domains/{domain_name}", response_model=DomainResponse) @router.get("/domains/{domain_name}", response_model=DomainResponse)
async def read_domain(domain_name: str, db: Session = Depends(get_db)): async def read_domain(domain_name: str, db: Session = Depends(get_db)):
""" """
@@ -415,8 +503,9 @@ async def read_domain(domain_name: str, db: Session = Depends(get_db)):
store = ReportStore.get_instance() store = ReportStore.get_instance()
hydrate_report_store_from_db(db, store) hydrate_report_store_from_db(db, store)
domains = store.get_domains() domains = store.get_domains()
stored_domain = db.query(Domain).filter(Domain.name == domain_name).first()
if domain_name not in domains: if domain_name not in domains and stored_domain is None:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND,
detail="Domain not found", detail="Domain not found",
@@ -426,7 +515,12 @@ async def read_domain(domain_name: str, db: Session = Depends(get_db)):
return DomainResponse( return DomainResponse(
name=domain_name, name=domain_name,
policy=summary.get("policy", "unknown"), description=stored_domain.description if stored_domain else None,
policy=(
summary.get("policy")
or (stored_domain.dmarc_policy if stored_domain else None)
or "unknown"
),
reports_count=summary.get("reports_processed", 0), reports_count=summary.get("reports_processed", 0),
emails_count=summary.get("total_count", 0), emails_count=summary.get("total_count", 0),
compliance_rate=summary.get("compliance_rate", 0.0), compliance_rate=summary.get("compliance_rate", 0.0),
+100 -1
View File
@@ -1,6 +1,14 @@
from fastapi import APIRouter from fastapi import APIRouter, Depends
from sqlalchemy import func, text
from sqlalchemy.orm import Session
from app.api.api_v1.endpoints.setup import setup_status from app.api.api_v1.endpoints.setup import setup_status
from app.core.database import get_db
from app.core.security import require_admin_auth
from app.models.mail_source import MailSource
from app.models.mail_source_import import MailSourceImport
from app.models.report import DMARCReport
from app.services.runtime_status import get_scheduler_status
router = APIRouter() router = APIRouter()
@@ -17,3 +25,94 @@ async def health_check():
"service": "dmarq", "service": "dmarq",
"is_setup_complete": setup_status["is_setup_complete"], "is_setup_complete": setup_status["is_setup_complete"],
} }
def _iso(value):
return value.isoformat() if value else None
@router.get("/health/operations", status_code=200)
async def operations_health(
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
):
"""Return operational health details for the web health page."""
database = {"ok": True, "detail": "Connected"}
try:
db.execute(text("SELECT 1"))
except Exception as exc: # pylint: disable=broad-exception-caught
database = {"ok": False, "detail": str(exc)}
enabled_sources = 0
total_sources = 0
report_count = 0
latest_report = None
latest_import = None
latest_successful_import = None
if database["ok"]:
enabled_sources = (
db.query(func.count(MailSource.id))
.filter(MailSource.enabled == True) # noqa: E712
.scalar()
)
total_sources = db.query(func.count(MailSource.id)).scalar()
report_count = db.query(func.count(DMARCReport.id)).scalar()
latest_report = db.query(func.max(DMARCReport.processed_at)).scalar()
latest_import = (
db.query(MailSourceImport)
.order_by(MailSourceImport.finished_at.desc(), MailSourceImport.id.desc())
.first()
)
latest_successful_import = (
db.query(MailSourceImport)
.filter(MailSourceImport.status.in_(["success", "warning"]))
.order_by(MailSourceImport.finished_at.desc(), MailSourceImport.id.desc())
.first()
)
scheduler = get_scheduler_status()
status = "ok"
checks = []
if not database["ok"]:
status = "degraded"
checks.append("Database connectivity failed.")
if total_sources and enabled_sources == 0:
status = "degraded"
checks.append("All mail sources are disabled.")
if scheduler.get("last_error"):
status = "degraded"
checks.append("The scheduler reported a recent error.")
return {
"status": status,
"service": "dmarq",
"database": database,
"scheduler": {
**scheduler,
"enabled_sources": int(enabled_sources or 0),
"total_sources": int(total_sources or 0),
},
"imports": {
"latest": {
"status": latest_import.status,
"trigger": latest_import.trigger,
"reports_found": latest_import.reports_found,
"finished_at": _iso(latest_import.finished_at),
}
if latest_import
else None,
"latest_successful": {
"status": latest_successful_import.status,
"trigger": latest_successful_import.trigger,
"reports_found": latest_successful_import.reports_found,
"finished_at": _iso(latest_successful_import.finished_at),
}
if latest_successful_import
else None,
},
"reports": {
"count": int(report_count or 0),
"latest_processed_at": _iso(latest_report),
},
"checks": checks,
}
+10
View File
@@ -7,6 +7,8 @@ from sqlalchemy.orm import Session
from app.core.database import get_db from app.core.database import get_db
from app.core.security import api_key_header, require_admin_auth, security_bearer from app.core.security import api_key_header, require_admin_auth, security_bearer
from app.models.domain import Domain
from app.models.mail_source import MailSource
from app.models.setting import Setting from app.models.setting import Setting
router = APIRouter() router = APIRouter()
@@ -89,6 +91,9 @@ class SetupStatusResponse(BaseModel):
is_setup_complete: bool is_setup_complete: bool
app_name: str app_name: str
total_domains: int = 0
total_mail_sources: int = 0
enabled_mail_sources: int = 0
class AdminSetupRequest(BaseModel): class AdminSetupRequest(BaseModel):
@@ -125,6 +130,11 @@ async def get_setup_status(db: Session = Depends(get_db)):
return SetupStatusResponse( return SetupStatusResponse(
is_setup_complete=current_status["is_setup_complete"], is_setup_complete=current_status["is_setup_complete"],
app_name=current_status["app_name"], app_name=current_status["app_name"],
total_domains=db.query(Domain.id).count(),
total_mail_sources=db.query(MailSource.id).count(),
enabled_mail_sources=db.query(MailSource.id)
.filter(MailSource.enabled == True) # noqa: E712
.count(),
) )
+30 -5
View File
@@ -2,6 +2,7 @@ import asyncio
import logging import logging
import os import os
from datetime import datetime from datetime import datetime
from typing import List, Optional
from fastapi import Depends, FastAPI, Query, Request from fastapi import Depends, FastAPI, Query, Request
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
@@ -24,12 +25,20 @@ from app.core.security import add_api_key, generate_api_key, require_admin_auth
from app.core.startup_checks import run_startup_checks from app.core.startup_checks import run_startup_checks
from app.middleware.auth import AuthRedirectMiddleware from app.middleware.auth import AuthRedirectMiddleware
from app.middleware.security import SecurityHeadersMiddleware from app.middleware.security import SecurityHeadersMiddleware
from app.models.domain import Domain
from app.models.mail_source import MailSource # noqa: F401 ensure table is registered from app.models.mail_source import MailSource # noqa: F401 ensure table is registered
from app.services.gmail_client import GmailClient from app.services.gmail_client import GmailClient
from app.services.imap_client import IMAPClient from app.services.imap_client import IMAPClient
from app.services.import_history import record_import_attempt from app.services.import_history import record_import_attempt
from app.services.report_persistence import hydrate_report_store_from_db from app.services.report_persistence import hydrate_report_store_from_db
from app.services.report_store import ReportStore from app.services.report_store import ReportStore
from app.services.runtime_status import (
mark_scheduler_cycle_started,
mark_scheduler_error,
mark_scheduler_started,
mark_scheduler_stopped,
mark_scheduler_success,
)
from app.services.summary_notifications import send_due_scheduled_summaries from app.services.summary_notifications import send_due_scheduled_summaries
# Set up logging # Set up logging
@@ -203,7 +212,7 @@ def _send_due_summary_notifications() -> None:
def _next_sleep_seconds( def _next_sleep_seconds(
min_sleep: int = 60, enabled_sources: list[MailSource] | None = None min_sleep: int = 60, enabled_sources: Optional[List[MailSource]] = None
) -> int: ) -> int:
"""Return how many seconds to sleep until the next polling cycle.""" """Return how many seconds to sleep until the next polling cycle."""
try: try:
@@ -226,11 +235,14 @@ async def scheduled_imap_polling():
try: try:
while True: while True:
logger.info("Starting scheduled IMAP polling for DMARC reports") logger.info("Starting scheduled IMAP polling for DMARC reports")
mark_scheduler_cycle_started()
try: try:
enabled_sources = _poll_all_enabled_sources() enabled_sources = _poll_all_enabled_sources()
_send_due_summary_notifications() _send_due_summary_notifications()
mark_scheduler_success()
except Exception as e: # pylint: disable=broad-exception-caught except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error in IMAP polling task: %s", str(e)) logger.error("Error in IMAP polling task: %s", str(e))
mark_scheduler_error(e)
enabled_sources = None enabled_sources = None
try: try:
@@ -243,6 +255,7 @@ async def scheduled_imap_polling():
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("IMAP polling task cancelled") logger.info("IMAP polling task cancelled")
mark_scheduler_stopped()
def _migrate_imap_env_vars_to_db() -> None: def _migrate_imap_env_vars_to_db() -> None:
@@ -407,6 +420,7 @@ def create_app() -> FastAPI:
# Start background polling task (iterates over DB-enabled mail sources) # Start background polling task (iterates over DB-enabled mail sources)
logger.info("Starting IMAP polling background task") logger.info("Starting IMAP polling background task")
mark_scheduler_started()
background_task = asyncio.create_task(scheduled_imap_polling()) background_task = asyncio.create_task(scheduled_imap_polling())
@application.on_event("shutdown") @application.on_event("shutdown")
@@ -419,6 +433,7 @@ def create_app() -> FastAPI:
await background_task await background_task
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
mark_scheduler_stopped()
return application return application
@@ -484,17 +499,18 @@ async def domain_details(request: Request, domain_id: str):
db = SessionLocal() db = SessionLocal()
try: try:
hydrate_report_store_from_db(db, store) hydrate_report_store_from_db(db, store)
stored_domain = db.query(Domain).filter(Domain.name == domain_id).first()
finally: finally:
db.close() db.close()
known_domains = store.get_domains() known_domains = store.get_domains()
if domain_id not in known_domains: if domain_id not in known_domains and stored_domain is None:
# Domain not found, redirect to domains list # Domain not found, redirect to domains list
return templates.TemplateResponse( return templates.TemplateResponse(
request, "domains.html", {"error": f"Domain {domain_id} not found"} request, "domains.html", {"error": f"Domain {domain_id} not found"}
) )
domain_summary = store.get_domain_summary(domain_id) domain_summary = store.get_domain_summary(domain_id) if domain_id in known_domains else {}
return templates.TemplateResponse( return templates.TemplateResponse(
request, request,
@@ -503,8 +519,12 @@ async def domain_details(request: Request, domain_id: str):
"domain_id": domain_id, "domain_id": domain_id,
"domain": { "domain": {
"name": domain_id, "name": domain_id,
"description": "", # Add description if available "description": stored_domain.description if stored_domain else "",
"policy": domain_summary.get("policy", "unknown"), "policy": (
domain_summary.get("policy")
or (stored_domain.dmarc_policy if stored_domain else None)
or "unknown"
),
}, },
}, },
) )
@@ -550,6 +570,11 @@ async def mail_sources_page(request: Request):
return templates.TemplateResponse(request, "mail_sources.html") return templates.TemplateResponse(request, "mail_sources.html")
@app.get("/operations", response_class=HTMLResponse)
async def operations_page(request: Request):
return templates.TemplateResponse(request, "operations.html")
@app.get("/upload", response_class=HTMLResponse) @app.get("/upload", response_class=HTMLResponse)
async def upload_page(request: Request): async def upload_page(request: Request):
return templates.TemplateResponse(request, "upload.html") return templates.TemplateResponse(request, "upload.html")
+2 -2
View File
@@ -1,4 +1,4 @@
from datetime import UTC, datetime from datetime import datetime, timezone
from sqlalchemy import Boolean, Column, DateTime, Index, Integer, String, Text, UniqueConstraint from sqlalchemy import Boolean, Column, DateTime, Index, Integer, String, Text, UniqueConstraint
@@ -6,7 +6,7 @@ from app.core.database import Base
def _utcnow_naive() -> datetime: def _utcnow_naive() -> datetime:
return datetime.now(UTC).replace(tzinfo=None) return datetime.now(timezone.utc).replace(tzinfo=None)
class DNSCache(Base): class DNSCache(Base):
+2 -2
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import hashlib import hashlib
import json import json
from dataclasses import dataclass from dataclasses import dataclass
from datetime import UTC, datetime from datetime import datetime, timezone
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -33,7 +33,7 @@ class CloudflareCredentials:
def _utcnow_naive() -> datetime: def _utcnow_naive() -> datetime:
return datetime.now(UTC).replace(tzinfo=None) return datetime.now(timezone.utc).replace(tzinfo=None)
def _plain_setting_value(db: Session, key: str) -> Optional[str]: def _plain_setting_value(db: Session, key: str) -> Optional[str]:
+2 -2
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import hashlib import hashlib
import json import json
from dataclasses import asdict from dataclasses import asdict
from datetime import UTC, datetime, timedelta from datetime import datetime, timedelta, timezone
from typing import List, Tuple from typing import List, Tuple
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -17,7 +17,7 @@ DEFAULT_DNS_CACHE_TTL_SECONDS = 900
def _utcnow_naive() -> datetime: def _utcnow_naive() -> datetime:
return datetime.now(UTC).replace(tzinfo=None) return datetime.now(timezone.utc).replace(tzinfo=None)
def _selectors_key(selectors: List[str]) -> str: def _selectors_key(selectors: List[str]) -> str:
+7 -7
View File
@@ -392,7 +392,7 @@ class IMAPClient:
) )
@staticmethod @staticmethod
def _append_detail(stats: dict | None, **detail: str) -> None: def _append_detail(stats: Optional[Dict[str, Any]], **detail: str) -> None:
"""Append a compact attachment/message outcome to the import stats.""" """Append a compact attachment/message outcome to the import stats."""
if stats is None: if stats is None:
return return
@@ -405,8 +405,8 @@ class IMAPClient:
report: Dict[str, Any], report: Dict[str, Any],
*, *,
filename: str, filename: str,
stats: dict | None, stats: Optional[Dict[str, Any]],
message_id: str | None, message_id: Optional[str],
) -> bool: ) -> bool:
domain = report.get("domain", "unknown") domain = report.get("domain", "unknown")
report_id = report.get("report_id", "") report_id = report.get("report_id", "")
@@ -445,8 +445,8 @@ class IMAPClient:
part: email.message.Message, part: email.message.Message,
*, *,
filename: str, filename: str,
stats: dict | None, stats: Optional[Dict[str, Any]],
message_id: str | None, message_id: Optional[str],
) -> bool: ) -> bool:
try: try:
content = part.get_payload(decode=True) content = part.get_payload(decode=True)
@@ -487,8 +487,8 @@ class IMAPClient:
def _process_attachments( def _process_attachments(
self, self,
msg: email.message.Message, msg: email.message.Message,
stats: dict | None = None, stats: Optional[Dict[str, Any]] = None,
message_id: str | None = None, message_id: Optional[str] = None,
) -> int: ) -> int:
""" """
Process email attachments that might be DMARC reports Process email attachments that might be DMARC reports
+1 -1
View File
@@ -191,7 +191,7 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
} }
def hydrate_report_store_from_db(db: Session, store: ReportStore | None = None) -> int: def hydrate_report_store_from_db(db: Session, store: Optional[ReportStore] = None) -> int:
"""Load persisted reports into ReportStore when the database has report rows.""" """Load persisted reports into ReportStore when the database has report rows."""
report_count = db.query(DMARCReport.id).count() report_count = db.query(DMARCReport.id).count()
if report_count == 0: if report_count == 0:
+65
View File
@@ -0,0 +1,65 @@
"""Small in-process status tracker for scheduler health."""
from datetime import datetime
from typing import Any, Dict, Optional
_scheduler_state: Dict[str, Any] = {
"running": False,
"started_at": None,
"stopped_at": None,
"last_cycle_started_at": None,
"last_success_at": None,
"last_error_at": None,
"last_error": None,
}
def _iso(value: Optional[datetime]) -> Optional[str]:
return value.isoformat() if value else None
def mark_scheduler_started() -> None:
now = datetime.utcnow()
_scheduler_state.update(
{
"running": True,
"started_at": now,
"stopped_at": None,
"last_error": None,
}
)
def mark_scheduler_stopped() -> None:
_scheduler_state.update({"running": False, "stopped_at": datetime.utcnow()})
def mark_scheduler_cycle_started() -> None:
_scheduler_state["last_cycle_started_at"] = datetime.utcnow()
def mark_scheduler_success() -> None:
_scheduler_state.update({"last_success_at": datetime.utcnow(), "last_error": None})
def mark_scheduler_error(error: object) -> None:
text = str(error)
_scheduler_state.update(
{
"last_error_at": datetime.utcnow(),
"last_error": text[:300],
}
)
def get_scheduler_status() -> Dict[str, Any]:
return {
"running": bool(_scheduler_state["running"]),
"started_at": _iso(_scheduler_state["started_at"]),
"stopped_at": _iso(_scheduler_state["stopped_at"]),
"last_cycle_started_at": _iso(_scheduler_state["last_cycle_started_at"]),
"last_success_at": _iso(_scheduler_state["last_success_at"]),
"last_error_at": _iso(_scheduler_state["last_error_at"]),
"last_error": _scheduler_state["last_error"],
}
+84 -3
View File
@@ -21,9 +21,9 @@
{% call card_header() %} {% call card_header() %}
<div class="flex items-center justify-between"> <div class="flex items-center justify-between">
{% call card_title() %}Monitored Domains{% endcall %} {% call card_title() %}Monitored Domains{% endcall %}
{% call button(variant="outline", size="sm") %} <button type="button" class="btn btn-outline btn-sm" @click="openCreate = true">
<span class="mr-1">+</span> Add Domain <span class="mr-1">+</span> Add Domain
{% endcall %} </button>
</div> </div>
{% call card_description() %} {% call card_description() %}
Domains currently being monitored for DMARC compliance Domains currently being monitored for DMARC compliance
@@ -91,6 +91,40 @@
{% endcall %} {% endcall %}
{% endcall %} {% endcall %}
</div> </div>
<dialog class="modal" :open="openCreate">
<div class="modal-box max-w-2xl">
<h2 class="text-xl font-semibold">Add monitored domain</h2>
<p class="mt-1 text-sm text-base-content/70">Track DNS health and future DMARC reports before the first report arrives.</p>
<form class="mt-5 space-y-4" @submit.prevent="createDomain">
<label class="form-control">
<span class="label-text font-medium">Domain</span>
<input x-model.trim="newDomain.name" type="text" required placeholder="example.com" class="input input-bordered">
</label>
<label class="form-control">
<span class="label-text font-medium">Description</span>
<textarea x-model.trim="newDomain.description" rows="3" class="textarea textarea-bordered" placeholder="Production mail domain"></textarea>
</label>
<label class="form-control">
<span class="label-text font-medium">DKIM selectors</span>
<input x-model.trim="newDomain.dkim_selectors" type="text" placeholder="default, google, selector1" class="input input-bordered">
</label>
<div x-show="createError" role="alert" class="alert alert-error py-3">
<span x-text="createError"></span>
</div>
<div class="modal-action">
<button type="button" class="btn btn-ghost" @click="closeCreate" :disabled="saving">Cancel</button>
<button type="submit" class="btn btn-primary" :disabled="saving">
<span x-show="saving" class="loading loading-spinner loading-xs"></span>
Add domain
</button>
</div>
</form>
</div>
<form method="dialog" class="modal-backdrop">
<button type="button" @click="closeCreate">close</button>
</form>
</dialog>
</div> </div>
{% endblock %} {% endblock %}
@@ -99,6 +133,14 @@
function domainsApp() { function domainsApp() {
return { return {
domains: [], domains: [],
openCreate: false,
saving: false,
createError: '',
newDomain: {
name: '',
description: '',
dkim_selectors: '',
},
init() { init() {
// Fetch domains from server // Fetch domains from server
@@ -128,7 +170,46 @@ function domainsApp() {
} catch (error) { } catch (error) {
console.error('Error fetching domains:', error); console.error('Error fetching domains:', error);
} }
} },
closeCreate() {
this.openCreate = false;
this.createError = '';
this.newDomain = { name: '', description: '', dkim_selectors: '' };
},
async createDomain() {
this.saving = true;
this.createError = '';
try {
const selectors = this.newDomain.dkim_selectors
.split(',')
.map((selector) => selector.trim())
.filter(Boolean);
const response = await fetch('/api/v1/domains/domains', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: this.newDomain.name,
description: this.newDomain.description || null,
dkim_selectors: selectors,
}),
});
if (!response.ok) {
const data = await response.json().catch(() => ({}));
const detail = typeof data.detail === 'string'
? data.detail
: Object.values(data.detail || {}).join(', ');
throw new Error(detail || 'Domain could not be added.');
}
this.closeCreate();
await this.fetchDomains();
} catch (error) {
this.createError = error.message || 'Domain could not be added.';
} finally {
this.saving = false;
}
},
} }
} }
</script> </script>
+1
View File
@@ -40,6 +40,7 @@
<li><a href="/reports">Reports</a></li> <li><a href="/reports">Reports</a></li>
<li><a href="/upload">Upload</a></li> <li><a href="/upload">Upload</a></li>
<li><a href="/mail-sources">Mail Sources</a></li> <li><a href="/mail-sources">Mail Sources</a></li>
<li><a href="/operations">Health</a></li>
<li><a href="/settings">Settings</a></li> <li><a href="/settings">Settings</a></li>
</ul> </ul>
<!-- User menu --> <!-- User menu -->
+148
View File
@@ -0,0 +1,148 @@
{% extends "layouts/base.html" %}
{% block title %}DMARQ - Health{% endblock %}
{% block content %}
<div class="mx-auto max-w-6xl space-y-6" x-data="operationsHealth()" x-init="load()">
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
<div>
<h1 class="text-2xl font-bold">System Health</h1>
<p class="mt-1 text-sm text-base-content/70">Operational status for imports, scheduler activity, and storage.</p>
</div>
<button class="btn btn-outline btn-sm" @click="load" :disabled="loading">
<span x-show="loading" class="loading loading-spinner loading-xs"></span>
Refresh
</button>
</div>
<div x-show="error" role="alert" class="alert alert-error">
<span x-text="error"></span>
</div>
<div class="grid gap-4 md:grid-cols-4">
<div class="card bg-base-100 shadow">
<div class="card-body">
<div class="text-sm text-base-content/60">Overall</div>
<div class="text-2xl font-semibold capitalize" :class="health.status === 'ok' ? 'text-success' : 'text-warning'" x-text="health.status || 'loading'"></div>
</div>
</div>
<div class="card bg-base-100 shadow">
<div class="card-body">
<div class="text-sm text-base-content/60">Database</div>
<div class="text-2xl font-semibold" :class="health.database?.ok ? 'text-success' : 'text-error'" x-text="health.database?.ok ? 'Connected' : 'Failed'"></div>
</div>
</div>
<div class="card bg-base-100 shadow">
<div class="card-body">
<div class="text-sm text-base-content/60">Mail Sources</div>
<div class="text-2xl font-semibold" x-text="`${health.scheduler?.enabled_sources || 0}/${health.scheduler?.total_sources || 0}`"></div>
</div>
</div>
<div class="card bg-base-100 shadow">
<div class="card-body">
<div class="text-sm text-base-content/60">Reports</div>
<div class="text-2xl font-semibold" x-text="health.reports?.count || 0"></div>
</div>
</div>
</div>
<div class="grid gap-6 lg:grid-cols-2">
<section class="card bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">Scheduler</h2>
<dl class="grid gap-3 text-sm sm:grid-cols-2">
<div>
<dt class="text-base-content/60">State</dt>
<dd class="font-medium" x-text="health.scheduler?.running ? 'Running' : 'Stopped'"></dd>
</div>
<div>
<dt class="text-base-content/60">Last cycle</dt>
<dd class="font-medium" x-text="formatDate(health.scheduler?.last_cycle_started_at)"></dd>
</div>
<div>
<dt class="text-base-content/60">Last successful cycle</dt>
<dd class="font-medium" x-text="formatDate(health.scheduler?.last_success_at)"></dd>
</div>
<div>
<dt class="text-base-content/60">Last error</dt>
<dd class="font-medium" x-text="health.scheduler?.last_error || 'None'"></dd>
</div>
</dl>
</div>
</section>
<section class="card bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">Imports</h2>
<dl class="grid gap-3 text-sm sm:grid-cols-2">
<div>
<dt class="text-base-content/60">Last completed</dt>
<dd class="font-medium" x-text="formatImport(health.imports?.latest)"></dd>
</div>
<div>
<dt class="text-base-content/60">Last successful</dt>
<dd class="font-medium" x-text="formatImport(health.imports?.latest_successful)"></dd>
</div>
<div>
<dt class="text-base-content/60">Latest report</dt>
<dd class="font-medium" x-text="formatDate(health.reports?.latest_processed_at)"></dd>
</div>
<div>
<dt class="text-base-content/60">Database detail</dt>
<dd class="font-medium" x-text="health.database?.detail || 'Unknown'"></dd>
</div>
</dl>
</div>
</section>
</div>
<section class="card bg-base-100 shadow" x-show="health.checks?.length">
<div class="card-body">
<h2 class="card-title">Attention</h2>
<ul class="list-disc space-y-2 pl-5 text-sm">
<template x-for="check in health.checks" :key="check">
<li x-text="check"></li>
</template>
</ul>
</div>
</section>
</div>
{% endblock %}
{% block scripts %}
<script>
function operationsHealth() {
return {
loading: false,
error: '',
health: {},
async load() {
this.loading = true;
this.error = '';
try {
const response = await fetch('/api/v1/health/operations');
if (!response.ok) {
throw new Error('Health details could not be loaded.');
}
this.health = await response.json();
} catch (error) {
this.error = error.message || 'Health details could not be loaded.';
} finally {
this.loading = false;
}
},
formatDate(value) {
if (!value) return 'Not recorded';
return new Date(value).toLocaleString();
},
formatImport(value) {
if (!value) return 'Not recorded';
return `${value.status} (${value.reports_found} reports) at ${this.formatDate(value.finished_at)}`;
},
};
}
</script>
{% endblock %}
+18 -4
View File
@@ -166,19 +166,27 @@
<div class="card bg-base-100 shadow"> <div class="card bg-base-100 shadow">
<div class="card-body"> <div class="card-body">
<h2 class="card-title text-lg">Next steps</h2> <h2 class="card-title text-lg">Setup checklist</h2>
<ul class="space-y-3 text-sm text-base-content/70"> <ul class="space-y-3 text-sm text-base-content/70">
<li class="flex gap-2"> <li class="flex gap-2">
<span class="mt-1 h-2 w-2 rounded-full bg-primary"></span> <span class="mt-1 h-2 w-2 rounded-full bg-primary"></span>
<span>Add a mail source from the Mail Sources page.</span> <span>
<a href="/domains" class="link link-primary font-medium">Add monitored domains</a>
<span class="ml-1" x-text="`(${totalDomains} configured)`"></span>
</span>
</li> </li>
<li class="flex gap-2"> <li class="flex gap-2">
<span class="mt-1 h-2 w-2 rounded-full bg-secondary"></span> <span class="mt-1 h-2 w-2 rounded-full bg-secondary"></span>
<span>Import DMARC reports from Gmail, IMAP, upload, or webhook.</span> <span>
<a href="/mail-sources" class="link link-primary font-medium">Connect a mailbox</a>
<span class="ml-1" x-text="`(${enabledMailSources}/${totalMailSources} enabled)`"></span>
</span>
</li> </li>
<li class="flex gap-2"> <li class="flex gap-2">
<span class="mt-1 h-2 w-2 rounded-full bg-accent"></span> <span class="mt-1 h-2 w-2 rounded-full bg-accent"></span>
<span>Use DNS health checks to review SPF, DKIM, and DMARC records.</span> <span>
<a href="/operations" class="link link-primary font-medium">Check system health</a>
</span>
</li> </li>
</ul> </ul>
</div> </div>
@@ -196,6 +204,9 @@
complete: false, complete: false,
error: '', error: '',
message: '', message: '',
totalDomains: 0,
totalMailSources: 0,
enabledMailSources: 0,
steps: [ steps: [
{ id: 1, title: 'Admin', detail: 'Contact details' }, { id: 1, title: 'Admin', detail: 'Contact details' },
{ id: 2, title: 'System', detail: 'Name and URL' }, { id: 2, title: 'System', detail: 'Name and URL' },
@@ -222,6 +233,9 @@
this.system.app_name = status.app_name || this.system.app_name; this.system.app_name = status.app_name || this.system.app_name;
this.complete = Boolean(status.is_setup_complete); this.complete = Boolean(status.is_setup_complete);
this.currentStep = this.complete ? 3 : 1; this.currentStep = this.complete ? 3 : 1;
this.totalDomains = status.total_domains || 0;
this.totalMailSources = status.total_mail_sources || 0;
this.enabledMailSources = status.enabled_mail_sources || 0;
} catch (err) { } catch (err) {
this.error = err.message || 'Could not load setup status.'; this.error = err.message || 'Could not load setup status.';
} finally { } finally {
+36
View File
@@ -18,6 +18,42 @@ def test_domains_empty(client: TestClient):
assert data == [] assert data == []
def test_create_domain_without_reports(authed_client: TestClient):
"""A monitored domain can be created before its first report arrives."""
response = authed_client.post(
"/api/v1/domains/domains",
json={"name": "Example.COM.", "description": "Primary mail domain"},
)
assert response.status_code == 201
assert response.json()["name"] == "example.com"
list_response = authed_client.get("/api/v1/domains/domains")
assert list_response.status_code == 200
assert list_response.json()[0]["name"] == "example.com"
assert list_response.json()[0]["reports_count"] == 0
def test_create_domain_rejects_duplicates(authed_client: TestClient):
"""Creating the same monitored domain twice returns a conflict."""
first = authed_client.post("/api/v1/domains/domains", json={"name": "example.com"})
second = authed_client.post("/api/v1/domains/domains", json={"name": "EXAMPLE.com"})
assert first.status_code == 201
assert second.status_code == 409
def test_operations_health_endpoint(authed_client: TestClient):
"""Detailed health includes database, scheduler, import, and report sections."""
response = authed_client.get("/api/v1/health/operations")
assert response.status_code == 200
data = response.json()
assert data["service"] == "dmarq"
assert data["database"]["ok"] is True
assert "scheduler" in data
assert "imports" in data
assert "reports" in data
def test_reports_upload_invalid_extension(client: TestClient): def test_reports_upload_invalid_extension(client: TestClient):
"""Test that uploading a file with an unsupported extension returns 400.""" """Test that uploading a file with an unsupported extension returns 400."""
response = client.post( response = client.post(
+2 -1
View File
@@ -3,6 +3,7 @@ from email.header import Header
from email.mime.application import MIMEApplication from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from typing import Optional
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
@@ -60,7 +61,7 @@ def _raw_email_with_report() -> bytes:
def _raw_email_with_attachment( def _raw_email_with_attachment(
filename: str | None, filename: Optional[str],
content: bytes, content: bytes,
subtype: str = "octet-stream", subtype: str = "octet-stream",
subject: str = "DMARC report", subject: str = "DMARC report",
+10 -5
View File
@@ -144,15 +144,20 @@ Exit criteria:
## Milestone 9: Setup and Operations Polish ## Milestone 9: Setup and Operations Polish
Status: Planned Status: In Progress
Goal: make first-run setup, maintenance, and troubleshooting straightforward. Goal: make first-run setup, maintenance, and troubleshooting straightforward.
Planned: Delivered:
- Guided setup flow for domains and mail sources. - First-run setup status is persisted in the database instead of only memory.
- Setup now surfaces a checklist for monitored domains, enabled mail sources, and system health.
- Monitored domains can be created before any DMARC report has arrived.
- Domain summaries and detail pages include manually configured domains with no report history yet.
- Health page and API show database connectivity, scheduler state, report totals, latest import, and latest successful import.
Remaining:
- Better mailbox test output and recovery suggestions. - Better mailbox test output and recovery suggestions.
- Health page for scheduler status, last successful import, and database connectivity. - Operator documentation refresh for Docker Compose and manual deployments.
- Operator documentation for Docker Compose and manual deployments.
Exit criteria: Exit criteria:
- A new user can deploy DMARQ, connect a mailbox, and confirm the system is healthy without reading code. - A new user can deploy DMARQ, connect a mailbox, and confirm the system is healthy without reading code.