From beae6e7469582aaa7225ff78b5ec9aa32cff42c9 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Sat, 23 May 2026 11:37:10 +0200 Subject: [PATCH] feat: add operations health and domain setup polish --- .gitignore | 3 +- backend/app/api/api_v1/endpoints/domains.py | 106 +++++++++++++- backend/app/api/api_v1/endpoints/health.py | 101 ++++++++++++- backend/app/api/api_v1/endpoints/setup.py | 10 ++ backend/app/main.py | 35 ++++- backend/app/models/dns_cache.py | 4 +- backend/app/services/cloudflare_dns.py | 4 +- backend/app/services/dns_cache.py | 4 +- backend/app/services/imap_client.py | 14 +- backend/app/services/report_persistence.py | 2 +- backend/app/services/runtime_status.py | 65 +++++++++ backend/app/templates/domains.html | 89 +++++++++++- backend/app/templates/layouts/base.html | 3 +- backend/app/templates/operations.html | 148 ++++++++++++++++++++ backend/app/templates/setup.html | 22 ++- backend/app/tests/test_api.py | 36 +++++ backend/app/tests/test_webhook.py | 3 +- docs/milestones.md | 15 +- 18 files changed, 622 insertions(+), 42 deletions(-) create mode 100644 backend/app/services/runtime_status.py create mode 100644 backend/app/templates/operations.html diff --git a/.gitignore b/.gitignore index 700d3a3..8f54720 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,7 @@ celerybeat.pid # Environments .env .venv +.pipcache/ env/ venv/ ENV/ @@ -216,4 +217,4 @@ temp/ *.backup # Docker override -docker-compose.override.yml \ No newline at end of file +docker-compose.override.yml diff --git a/backend/app/api/api_v1/endpoints/domains.py b/backend/app/api/api_v1/endpoints/domains.py index 0607251..a6b64c1 100644 --- a/backend/app/api/api_v1/endpoints/domains.py +++ b/backend/app/api/api_v1/endpoints/domains.py @@ -9,9 +9,11 @@ from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException, Path, Query, status from fastapi.responses import Response from pydantic import BaseModel, Field +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app.core.database import get_db +from app.core.security import require_admin_auth from app.models.domain import Domain from app.services.cloudflare_dns import ( analyze_dns_records, @@ -32,6 +34,7 @@ from app.services.report_persistence import ( hydrate_report_store_from_db, ) from app.services.report_store import ReportStore +from app.utils.domain_validator import validate_domain_config logger = logging.getLogger(__name__) @@ -55,6 +58,14 @@ class DomainResponse(DomainBase): compliance_rate: float = 0.0 +class DomainCreate(BaseModel): + """Payload for creating a monitored domain.""" + + name: str + description: Optional[str] = None + dkim_selectors: Optional[List[str]] = None + + class DomainStatsResponse(BaseModel): """Domain statistics for the domain details page""" @@ -283,6 +294,22 @@ def _policy_enforcement_suggestions( return [] +def _normalize_domain_name(name: str) -> str: + return name.strip().strip(".").lower() + + +def _domain_names_for_summary(db: Session, store: ReportStore) -> List[str]: + report_domains = store.get_domains() + stored_domains = [ + name + for (name,) in db.query(Domain.name) + .filter(Domain.active == True) # noqa: E712 + .order_by(Domain.name) + .all() + ] + return list(dict.fromkeys(stored_domains + report_domains)) + + @router.get("/summary", response_model=DomainSummaryResponse) async def get_domains_summary(db: Session = Depends(get_db)): """ @@ -295,7 +322,7 @@ async def get_domains_summary(db: Session = Depends(get_db)): """ store = ReportStore.get_instance() hydrate_report_store_from_db(db, store) - domains = store.get_domains() + domains = _domain_names_for_summary(db, store) summaries = store.get_all_domain_summaries() # Perform DNS checks for all domains, reusing fresh cached results. @@ -385,19 +412,28 @@ async def get_domains_summary(db: Session = Depends(get_db)): async def read_domains(db: Session = Depends(get_db)): """ Retrieve domains with their statistics. - For Milestone 1, this simply returns domains from the in-memory store. """ store = ReportStore.get_instance() hydrate_report_store_from_db(db, store) - domains = store.get_domains() + domains = _domain_names_for_summary(db, store) summaries = store.get_all_domain_summaries() + stored = { + domain.name: domain + for domain in db.query(Domain).filter(Domain.name.in_(domains)).all() + } result = [] for domain_name in domains: summary = summaries.get(domain_name, {}) + stored_domain = stored.get(domain_name) domain_response = DomainResponse( name=domain_name, - policy=summary.get("policy", "unknown"), + description=stored_domain.description if stored_domain else None, + policy=( + summary.get("policy") + or (stored_domain.dmarc_policy if stored_domain else None) + or "unknown" + ), reports_count=summary.get("reports_processed", 0), emails_count=summary.get("total_count", 0), compliance_rate=summary.get("compliance_rate", 0.0), @@ -407,6 +443,58 @@ async def read_domains(db: Session = Depends(get_db)): return result +@router.post("/domains", response_model=DomainResponse, status_code=status.HTTP_201_CREATED) +async def create_domain( + payload: DomainCreate, + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Create a monitored domain before any DMARC reports have arrived.""" + name = _normalize_domain_name(payload.name) + validation = validate_domain_config( + {"name": name, "description": payload.description or ""} + ) + if not validation["valid"]: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=validation["errors"], + ) + existing = db.query(Domain).filter(Domain.name == name).first() + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Domain is already monitored", + ) + + selectors = ",".join( + selector.strip() + for selector in payload.dkim_selectors or [] + if selector and selector.strip() + ) + domain = Domain( + name=name, + description=payload.description, + dkim_selectors=selectors or None, + active=True, + verified=False, + ) + db.add(domain) + try: + db.commit() + except IntegrityError as exc: + db.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Domain is already monitored", + ) from exc + db.refresh(domain) + return DomainResponse( + name=domain.name, + description=domain.description, + policy=domain.dmarc_policy or "unknown", + ) + + @router.get("/domains/{domain_name}", response_model=DomainResponse) async def read_domain(domain_name: str, db: Session = Depends(get_db)): """ @@ -415,8 +503,9 @@ async def read_domain(domain_name: str, db: Session = Depends(get_db)): store = ReportStore.get_instance() hydrate_report_store_from_db(db, store) domains = store.get_domains() + stored_domain = db.query(Domain).filter(Domain.name == domain_name).first() - if domain_name not in domains: + if domain_name not in domains and stored_domain is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Domain not found", @@ -426,7 +515,12 @@ async def read_domain(domain_name: str, db: Session = Depends(get_db)): return DomainResponse( name=domain_name, - policy=summary.get("policy", "unknown"), + description=stored_domain.description if stored_domain else None, + policy=( + summary.get("policy") + or (stored_domain.dmarc_policy if stored_domain else None) + or "unknown" + ), reports_count=summary.get("reports_processed", 0), emails_count=summary.get("total_count", 0), compliance_rate=summary.get("compliance_rate", 0.0), diff --git a/backend/app/api/api_v1/endpoints/health.py b/backend/app/api/api_v1/endpoints/health.py index a721120..e9c73a3 100644 --- a/backend/app/api/api_v1/endpoints/health.py +++ b/backend/app/api/api_v1/endpoints/health.py @@ -1,6 +1,14 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends +from sqlalchemy import func, text +from sqlalchemy.orm import Session from app.api.api_v1.endpoints.setup import setup_status +from app.core.database import get_db +from app.core.security import require_admin_auth +from app.models.mail_source import MailSource +from app.models.mail_source_import import MailSourceImport +from app.models.report import DMARCReport +from app.services.runtime_status import get_scheduler_status router = APIRouter() @@ -17,3 +25,94 @@ async def health_check(): "service": "dmarq", "is_setup_complete": setup_status["is_setup_complete"], } + + +def _iso(value): + return value.isoformat() if value else None + + +@router.get("/health/operations", status_code=200) +async def operations_health( + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Return operational health details for the web health page.""" + database = {"ok": True, "detail": "Connected"} + try: + db.execute(text("SELECT 1")) + except Exception as exc: # pylint: disable=broad-exception-caught + database = {"ok": False, "detail": str(exc)} + + enabled_sources = 0 + total_sources = 0 + report_count = 0 + latest_report = None + latest_import = None + latest_successful_import = None + if database["ok"]: + enabled_sources = ( + db.query(func.count(MailSource.id)) + .filter(MailSource.enabled == True) # noqa: E712 + .scalar() + ) + total_sources = db.query(func.count(MailSource.id)).scalar() + report_count = db.query(func.count(DMARCReport.id)).scalar() + latest_report = db.query(func.max(DMARCReport.processed_at)).scalar() + latest_import = ( + db.query(MailSourceImport) + .order_by(MailSourceImport.finished_at.desc(), MailSourceImport.id.desc()) + .first() + ) + latest_successful_import = ( + db.query(MailSourceImport) + .filter(MailSourceImport.status.in_(["success", "warning"])) + .order_by(MailSourceImport.finished_at.desc(), MailSourceImport.id.desc()) + .first() + ) + + scheduler = get_scheduler_status() + status = "ok" + checks = [] + if not database["ok"]: + status = "degraded" + checks.append("Database connectivity failed.") + if total_sources and enabled_sources == 0: + status = "degraded" + checks.append("All mail sources are disabled.") + if scheduler.get("last_error"): + status = "degraded" + checks.append("The scheduler reported a recent error.") + + return { + "status": status, + "service": "dmarq", + "database": database, + "scheduler": { + **scheduler, + "enabled_sources": int(enabled_sources or 0), + "total_sources": int(total_sources or 0), + }, + "imports": { + "latest": { + "status": latest_import.status, + "trigger": latest_import.trigger, + "reports_found": latest_import.reports_found, + "finished_at": _iso(latest_import.finished_at), + } + if latest_import + else None, + "latest_successful": { + "status": latest_successful_import.status, + "trigger": latest_successful_import.trigger, + "reports_found": latest_successful_import.reports_found, + "finished_at": _iso(latest_successful_import.finished_at), + } + if latest_successful_import + else None, + }, + "reports": { + "count": int(report_count or 0), + "latest_processed_at": _iso(latest_report), + }, + "checks": checks, + } diff --git a/backend/app/api/api_v1/endpoints/setup.py b/backend/app/api/api_v1/endpoints/setup.py index 11b3bf1..502a30a 100644 --- a/backend/app/api/api_v1/endpoints/setup.py +++ b/backend/app/api/api_v1/endpoints/setup.py @@ -7,6 +7,8 @@ from sqlalchemy.orm import Session from app.core.database import get_db from app.core.security import api_key_header, require_admin_auth, security_bearer +from app.models.domain import Domain +from app.models.mail_source import MailSource from app.models.setting import Setting router = APIRouter() @@ -89,6 +91,9 @@ class SetupStatusResponse(BaseModel): is_setup_complete: bool app_name: str + total_domains: int = 0 + total_mail_sources: int = 0 + enabled_mail_sources: int = 0 class AdminSetupRequest(BaseModel): @@ -125,6 +130,11 @@ async def get_setup_status(db: Session = Depends(get_db)): return SetupStatusResponse( is_setup_complete=current_status["is_setup_complete"], app_name=current_status["app_name"], + total_domains=db.query(Domain.id).count(), + total_mail_sources=db.query(MailSource.id).count(), + enabled_mail_sources=db.query(MailSource.id) + .filter(MailSource.enabled == True) # noqa: E712 + .count(), ) diff --git a/backend/app/main.py b/backend/app/main.py index bfc004d..cec4dcd 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,6 +2,7 @@ import asyncio import logging import os from datetime import datetime +from typing import List, Optional from fastapi import Depends, FastAPI, Query, Request from fastapi.middleware.cors import CORSMiddleware @@ -24,12 +25,20 @@ from app.core.security import add_api_key, generate_api_key, require_admin_auth from app.core.startup_checks import run_startup_checks from app.middleware.auth import AuthRedirectMiddleware from app.middleware.security import SecurityHeadersMiddleware +from app.models.domain import Domain from app.models.mail_source import MailSource # noqa: F401 – ensure table is registered from app.services.gmail_client import GmailClient from app.services.imap_client import IMAPClient from app.services.import_history import record_import_attempt from app.services.report_persistence import hydrate_report_store_from_db from app.services.report_store import ReportStore +from app.services.runtime_status import ( + mark_scheduler_cycle_started, + mark_scheduler_error, + mark_scheduler_started, + mark_scheduler_stopped, + mark_scheduler_success, +) from app.services.summary_notifications import send_due_scheduled_summaries # Set up logging @@ -203,7 +212,7 @@ def _send_due_summary_notifications() -> None: def _next_sleep_seconds( - min_sleep: int = 60, enabled_sources: list[MailSource] | None = None + min_sleep: int = 60, enabled_sources: Optional[List[MailSource]] = None ) -> int: """Return how many seconds to sleep until the next polling cycle.""" try: @@ -226,11 +235,14 @@ async def scheduled_imap_polling(): try: while True: logger.info("Starting scheduled IMAP polling for DMARC reports") + mark_scheduler_cycle_started() try: enabled_sources = _poll_all_enabled_sources() _send_due_summary_notifications() + mark_scheduler_success() except Exception as e: # pylint: disable=broad-exception-caught logger.error("Error in IMAP polling task: %s", str(e)) + mark_scheduler_error(e) enabled_sources = None try: @@ -243,6 +255,7 @@ async def scheduled_imap_polling(): except asyncio.CancelledError: logger.info("IMAP polling task cancelled") + mark_scheduler_stopped() def _migrate_imap_env_vars_to_db() -> None: @@ -407,6 +420,7 @@ def create_app() -> FastAPI: # Start background polling task (iterates over DB-enabled mail sources) logger.info("Starting IMAP polling background task") + mark_scheduler_started() background_task = asyncio.create_task(scheduled_imap_polling()) @application.on_event("shutdown") @@ -419,6 +433,7 @@ def create_app() -> FastAPI: await background_task except asyncio.CancelledError: pass + mark_scheduler_stopped() return application @@ -484,17 +499,18 @@ async def domain_details(request: Request, domain_id: str): db = SessionLocal() try: hydrate_report_store_from_db(db, store) + stored_domain = db.query(Domain).filter(Domain.name == domain_id).first() finally: db.close() known_domains = store.get_domains() - if domain_id not in known_domains: + if domain_id not in known_domains and stored_domain is None: # Domain not found, redirect to domains list return templates.TemplateResponse( request, "domains.html", {"error": f"Domain {domain_id} not found"} ) - domain_summary = store.get_domain_summary(domain_id) + domain_summary = store.get_domain_summary(domain_id) if domain_id in known_domains else {} return templates.TemplateResponse( request, @@ -503,8 +519,12 @@ async def domain_details(request: Request, domain_id: str): "domain_id": domain_id, "domain": { "name": domain_id, - "description": "", # Add description if available - "policy": domain_summary.get("policy", "unknown"), + "description": stored_domain.description if stored_domain else "", + "policy": ( + domain_summary.get("policy") + or (stored_domain.dmarc_policy if stored_domain else None) + or "unknown" + ), }, }, ) @@ -550,6 +570,11 @@ async def mail_sources_page(request: Request): return templates.TemplateResponse(request, "mail_sources.html") +@app.get("/operations", response_class=HTMLResponse) +async def operations_page(request: Request): + return templates.TemplateResponse(request, "operations.html") + + @app.get("/upload", response_class=HTMLResponse) async def upload_page(request: Request): return templates.TemplateResponse(request, "upload.html") diff --git a/backend/app/models/dns_cache.py b/backend/app/models/dns_cache.py index fcb992b..c230d6e 100644 --- a/backend/app/models/dns_cache.py +++ b/backend/app/models/dns_cache.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import datetime, timezone from sqlalchemy import Boolean, Column, DateTime, Index, Integer, String, Text, UniqueConstraint @@ -6,7 +6,7 @@ from app.core.database import Base def _utcnow_naive() -> datetime: - return datetime.now(UTC).replace(tzinfo=None) + return datetime.now(timezone.utc).replace(tzinfo=None) class DNSCache(Base): diff --git a/backend/app/services/cloudflare_dns.py b/backend/app/services/cloudflare_dns.py index 75833fa..b7e0465 100644 --- a/backend/app/services/cloudflare_dns.py +++ b/backend/app/services/cloudflare_dns.py @@ -5,7 +5,7 @@ from __future__ import annotations import hashlib import json from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import datetime, timezone from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session @@ -33,7 +33,7 @@ class CloudflareCredentials: def _utcnow_naive() -> datetime: - return datetime.now(UTC).replace(tzinfo=None) + return datetime.now(timezone.utc).replace(tzinfo=None) def _plain_setting_value(db: Session, key: str) -> Optional[str]: diff --git a/backend/app/services/dns_cache.py b/backend/app/services/dns_cache.py index 70b0943..042f045 100644 --- a/backend/app/services/dns_cache.py +++ b/backend/app/services/dns_cache.py @@ -5,7 +5,7 @@ from __future__ import annotations import hashlib import json from dataclasses import asdict -from datetime import UTC, datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List, Tuple from sqlalchemy.orm import Session @@ -17,7 +17,7 @@ DEFAULT_DNS_CACHE_TTL_SECONDS = 900 def _utcnow_naive() -> datetime: - return datetime.now(UTC).replace(tzinfo=None) + return datetime.now(timezone.utc).replace(tzinfo=None) def _selectors_key(selectors: List[str]) -> str: diff --git a/backend/app/services/imap_client.py b/backend/app/services/imap_client.py index 99e31da..d83ce35 100644 --- a/backend/app/services/imap_client.py +++ b/backend/app/services/imap_client.py @@ -392,7 +392,7 @@ class IMAPClient: ) @staticmethod - def _append_detail(stats: dict | None, **detail: str) -> None: + def _append_detail(stats: Optional[Dict[str, Any]], **detail: str) -> None: """Append a compact attachment/message outcome to the import stats.""" if stats is None: return @@ -405,8 +405,8 @@ class IMAPClient: report: Dict[str, Any], *, filename: str, - stats: dict | None, - message_id: str | None, + stats: Optional[Dict[str, Any]], + message_id: Optional[str], ) -> bool: domain = report.get("domain", "unknown") report_id = report.get("report_id", "") @@ -445,8 +445,8 @@ class IMAPClient: part: email.message.Message, *, filename: str, - stats: dict | None, - message_id: str | None, + stats: Optional[Dict[str, Any]], + message_id: Optional[str], ) -> bool: try: content = part.get_payload(decode=True) @@ -487,8 +487,8 @@ class IMAPClient: def _process_attachments( self, msg: email.message.Message, - stats: dict | None = None, - message_id: str | None = None, + stats: Optional[Dict[str, Any]] = None, + message_id: Optional[str] = None, ) -> int: """ Process email attachments that might be DMARC reports diff --git a/backend/app/services/report_persistence.py b/backend/app/services/report_persistence.py index 2b5769a..d386c83 100644 --- a/backend/app/services/report_persistence.py +++ b/backend/app/services/report_persistence.py @@ -191,7 +191,7 @@ def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]: } -def hydrate_report_store_from_db(db: Session, store: ReportStore | None = None) -> int: +def hydrate_report_store_from_db(db: Session, store: Optional[ReportStore] = None) -> int: """Load persisted reports into ReportStore when the database has report rows.""" report_count = db.query(DMARCReport.id).count() if report_count == 0: diff --git a/backend/app/services/runtime_status.py b/backend/app/services/runtime_status.py new file mode 100644 index 0000000..17a4956 --- /dev/null +++ b/backend/app/services/runtime_status.py @@ -0,0 +1,65 @@ +"""Small in-process status tracker for scheduler health.""" + +from datetime import datetime +from typing import Any, Dict, Optional + + +_scheduler_state: Dict[str, Any] = { + "running": False, + "started_at": None, + "stopped_at": None, + "last_cycle_started_at": None, + "last_success_at": None, + "last_error_at": None, + "last_error": None, +} + + +def _iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if value else None + + +def mark_scheduler_started() -> None: + now = datetime.utcnow() + _scheduler_state.update( + { + "running": True, + "started_at": now, + "stopped_at": None, + "last_error": None, + } + ) + + +def mark_scheduler_stopped() -> None: + _scheduler_state.update({"running": False, "stopped_at": datetime.utcnow()}) + + +def mark_scheduler_cycle_started() -> None: + _scheduler_state["last_cycle_started_at"] = datetime.utcnow() + + +def mark_scheduler_success() -> None: + _scheduler_state.update({"last_success_at": datetime.utcnow(), "last_error": None}) + + +def mark_scheduler_error(error: object) -> None: + text = str(error) + _scheduler_state.update( + { + "last_error_at": datetime.utcnow(), + "last_error": text[:300], + } + ) + + +def get_scheduler_status() -> Dict[str, Any]: + return { + "running": bool(_scheduler_state["running"]), + "started_at": _iso(_scheduler_state["started_at"]), + "stopped_at": _iso(_scheduler_state["stopped_at"]), + "last_cycle_started_at": _iso(_scheduler_state["last_cycle_started_at"]), + "last_success_at": _iso(_scheduler_state["last_success_at"]), + "last_error_at": _iso(_scheduler_state["last_error_at"]), + "last_error": _scheduler_state["last_error"], + } diff --git a/backend/app/templates/domains.html b/backend/app/templates/domains.html index 1152f0f..4f19726 100644 --- a/backend/app/templates/domains.html +++ b/backend/app/templates/domains.html @@ -21,9 +21,9 @@ {% call card_header() %}
{% call card_title() %}Monitored Domains{% endcall %} - {% call button(variant="outline", size="sm") %} +
{% call card_description() %} Domains currently being monitored for DMARC compliance @@ -91,6 +91,40 @@ {% endcall %} {% endcall %} + + +
+

Add monitored domain

+

Track DNS health and future DMARC reports before the first report arrives.

+
+ + + + +
+ + +
+
+
+
+ +
+ {% endblock %} @@ -99,6 +133,14 @@ function domainsApp() { return { domains: [], + openCreate: false, + saving: false, + createError: '', + newDomain: { + name: '', + description: '', + dkim_selectors: '', + }, init() { // Fetch domains from server @@ -128,8 +170,47 @@ function domainsApp() { } catch (error) { console.error('Error fetching domains:', error); } - } + }, + + closeCreate() { + this.openCreate = false; + this.createError = ''; + this.newDomain = { name: '', description: '', dkim_selectors: '' }; + }, + + async createDomain() { + this.saving = true; + this.createError = ''; + try { + const selectors = this.newDomain.dkim_selectors + .split(',') + .map((selector) => selector.trim()) + .filter(Boolean); + const response = await fetch('/api/v1/domains/domains', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: this.newDomain.name, + description: this.newDomain.description || null, + dkim_selectors: selectors, + }), + }); + if (!response.ok) { + const data = await response.json().catch(() => ({})); + const detail = typeof data.detail === 'string' + ? data.detail + : Object.values(data.detail || {}).join(', '); + throw new Error(detail || 'Domain could not be added.'); + } + this.closeCreate(); + await this.fetchDomains(); + } catch (error) { + this.createError = error.message || 'Domain could not be added.'; + } finally { + this.saving = false; + } + }, } } -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/backend/app/templates/layouts/base.html b/backend/app/templates/layouts/base.html index 4928bab..e699513 100644 --- a/backend/app/templates/layouts/base.html +++ b/backend/app/templates/layouts/base.html @@ -40,6 +40,7 @@
  • Reports
  • Upload
  • Mail Sources
    • +
  • Health
  • Settings
    • @@ -122,4 +123,4 @@ }); - \ No newline at end of file + diff --git a/backend/app/templates/operations.html b/backend/app/templates/operations.html new file mode 100644 index 0000000..23fd639 --- /dev/null +++ b/backend/app/templates/operations.html @@ -0,0 +1,148 @@ +{% extends "layouts/base.html" %} + +{% block title %}DMARQ - Health{% endblock %} + +{% block content %} +
    +
    +
    +

    System Health

    +

    Operational status for imports, scheduler activity, and storage.

    +
    + +
    + + + +
    +
    +
    +
    Overall
    +
    +
    +
    +
    +
    +
    Database
    +
    +
    +
    +
    +
    +
    Mail Sources
    +
    +
    +
    +
    +
    +
    Reports
    +
    +
    +
    +
    + +
    +
    +
    +

    Scheduler

    +
    +
    +
    State
    +
    +
    +
    +
    Last cycle
    +
    +
    +
    +
    Last successful cycle
    +
    +
    +
    +
    Last error
    +
    +
    +
    +
    +
    + +
    +
    +

    Imports

    +
    +
    +
    Last completed
    +
    +
    +
    +
    Last successful
    +
    +
    +
    +
    Latest report
    +
    +
    +
    +
    Database detail
    +
    +
    +
    +
    +
    +
    + +
    +
    +

    Attention

    +
      + +
    +
    +
    +
    +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/backend/app/templates/setup.html b/backend/app/templates/setup.html index dac8b12..554f590 100644 --- a/backend/app/templates/setup.html +++ b/backend/app/templates/setup.html @@ -166,19 +166,27 @@
    -

    Next steps

    +

    Setup checklist

    @@ -196,6 +204,9 @@ complete: false, error: '', message: '', + totalDomains: 0, + totalMailSources: 0, + enabledMailSources: 0, steps: [ { id: 1, title: 'Admin', detail: 'Contact details' }, { id: 2, title: 'System', detail: 'Name and URL' }, @@ -222,6 +233,9 @@ this.system.app_name = status.app_name || this.system.app_name; this.complete = Boolean(status.is_setup_complete); this.currentStep = this.complete ? 3 : 1; + this.totalDomains = status.total_domains || 0; + this.totalMailSources = status.total_mail_sources || 0; + this.enabledMailSources = status.enabled_mail_sources || 0; } catch (err) { this.error = err.message || 'Could not load setup status.'; } finally { diff --git a/backend/app/tests/test_api.py b/backend/app/tests/test_api.py index 0b6cef1..b57b147 100644 --- a/backend/app/tests/test_api.py +++ b/backend/app/tests/test_api.py @@ -18,6 +18,42 @@ def test_domains_empty(client: TestClient): assert data == [] +def test_create_domain_without_reports(authed_client: TestClient): + """A monitored domain can be created before its first report arrives.""" + response = authed_client.post( + "/api/v1/domains/domains", + json={"name": "Example.COM.", "description": "Primary mail domain"}, + ) + assert response.status_code == 201 + assert response.json()["name"] == "example.com" + + list_response = authed_client.get("/api/v1/domains/domains") + assert list_response.status_code == 200 + assert list_response.json()[0]["name"] == "example.com" + assert list_response.json()[0]["reports_count"] == 0 + + +def test_create_domain_rejects_duplicates(authed_client: TestClient): + """Creating the same monitored domain twice returns a conflict.""" + first = authed_client.post("/api/v1/domains/domains", json={"name": "example.com"}) + second = authed_client.post("/api/v1/domains/domains", json={"name": "EXAMPLE.com"}) + + assert first.status_code == 201 + assert second.status_code == 409 + + +def test_operations_health_endpoint(authed_client: TestClient): + """Detailed health includes database, scheduler, import, and report sections.""" + response = authed_client.get("/api/v1/health/operations") + assert response.status_code == 200 + data = response.json() + assert data["service"] == "dmarq" + assert data["database"]["ok"] is True + assert "scheduler" in data + assert "imports" in data + assert "reports" in data + + def test_reports_upload_invalid_extension(client: TestClient): """Test that uploading a file with an unsupported extension returns 400.""" response = client.post( diff --git a/backend/app/tests/test_webhook.py b/backend/app/tests/test_webhook.py index ec46f81..ebecf4a 100644 --- a/backend/app/tests/test_webhook.py +++ b/backend/app/tests/test_webhook.py @@ -3,6 +3,7 @@ from email.header import Header from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from typing import Optional from unittest.mock import patch import pytest @@ -60,7 +61,7 @@ def _raw_email_with_report() -> bytes: def _raw_email_with_attachment( - filename: str | None, + filename: Optional[str], content: bytes, subtype: str = "octet-stream", subject: str = "DMARC report", diff --git a/docs/milestones.md b/docs/milestones.md index 0bbbfe3..1f2e5d5 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -144,15 +144,20 @@ Exit criteria: ## Milestone 9: Setup and Operations Polish -Status: Planned +Status: In Progress Goal: make first-run setup, maintenance, and troubleshooting straightforward. -Planned: -- Guided setup flow for domains and mail sources. +Delivered: +- First-run setup status is persisted in the database instead of only memory. +- Setup now surfaces a checklist for monitored domains, enabled mail sources, and system health. +- Monitored domains can be created before any DMARC report has arrived. +- Domain summaries and detail pages include manually configured domains with no report history yet. +- Health page and API show database connectivity, scheduler state, report totals, latest import, and latest successful import. + +Remaining: - Better mailbox test output and recovery suggestions. -- Health page for scheduler status, last successful import, and database connectivity. -- Operator documentation for Docker Compose and manual deployments. +- Operator documentation refresh for Docker Compose and manual deployments. Exit criteria: - A new user can deploy DMARQ, connect a mailbox, and confirm the system is healthy without reading code.