diff --git a/backend/alembic/versions/f7a8b9c0d1e2_add_tls_reports.py b/backend/alembic/versions/f7a8b9c0d1e2_add_tls_reports.py new file mode 100644 index 0000000..361b03c --- /dev/null +++ b/backend/alembic/versions/f7a8b9c0d1e2_add_tls_reports.py @@ -0,0 +1,128 @@ +"""add tls reports + +Revision ID: f7a8b9c0d1e2 +Revises: c4d5e6f7a8b9 +Create Date: 2026-05-23 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "f7a8b9c0d1e2" +down_revision: Union[str, Sequence[str], None] = "c4d5e6f7a8b9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create privacy-conscious SMTP TLS report storage.""" + op.create_table( + "tls_reports", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("domain_id", sa.Integer(), nullable=True), + sa.Column("report_id", sa.String(), nullable=False), + sa.Column("org_name", sa.String(), nullable=True), + sa.Column("contact_info", sa.String(), nullable=True), + sa.Column("policy_domain", sa.String(), nullable=False), + sa.Column("policy_type", sa.String(), nullable=True), + sa.Column("begin_date", sa.DateTime(), nullable=True), + sa.Column("end_date", sa.DateTime(), nullable=True), + sa.Column("total_successful_sessions", sa.Integer(), nullable=False), + sa.Column("total_failure_sessions", sa.Integer(), nullable=False), + sa.Column("raw_policy", sa.Text(), nullable=True), + sa.Column("processed_at", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["domain_id"], ["domains.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("report_id", "policy_domain", name="uq_tls_reports_report_domain"), + ) + op.create_index(op.f("ix_tls_reports_id"), "tls_reports", ["id"]) + op.create_index(op.f("ix_tls_reports_domain_id"), "tls_reports", ["domain_id"]) + op.create_index(op.f("ix_tls_reports_report_id"), "tls_reports", ["report_id"]) + op.create_index(op.f("ix_tls_reports_org_name"), "tls_reports", ["org_name"]) + op.create_index(op.f("ix_tls_reports_policy_domain"), "tls_reports", ["policy_domain"]) + op.create_index(op.f("ix_tls_reports_policy_type"), "tls_reports", ["policy_type"]) + op.create_index(op.f("ix_tls_reports_begin_date"), "tls_reports", ["begin_date"]) + op.create_index(op.f("ix_tls_reports_end_date"), "tls_reports", ["end_date"]) + op.create_index(op.f("ix_tls_reports_processed_at"), "tls_reports", ["processed_at"]) + op.create_index( + "ix_tls_reports_domain_dates", + "tls_reports", + ["domain_id", "begin_date", "end_date"], + ) + op.create_index( + "ix_tls_reports_policy_domain_dates", + "tls_reports", + ["policy_domain", "begin_date", "end_date"], + ) + + op.create_table( + "tls_report_failures", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("report_id", sa.Integer(), nullable=False), + sa.Column("result_type", sa.String(), nullable=False), + sa.Column("failed_session_count", sa.Integer(), nullable=False), + sa.Column("sending_mta_ip", sa.String(), nullable=True), + sa.Column("receiving_mx_hostname", sa.String(), nullable=True), + sa.Column("receiving_mx_helo", sa.String(), nullable=True), + sa.Column("receiving_ip", sa.String(), nullable=True), + sa.Column("failure_reason_code", sa.String(), nullable=True), + sa.Column("additional_information", sa.Text(), nullable=True), + sa.ForeignKeyConstraint(["report_id"], ["tls_reports.id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_tls_report_failures_id"), "tls_report_failures", ["id"]) + op.create_index(op.f("ix_tls_report_failures_report_id"), "tls_report_failures", ["report_id"]) + op.create_index( + op.f("ix_tls_report_failures_result_type"), "tls_report_failures", ["result_type"] + ) + op.create_index( + op.f("ix_tls_report_failures_sending_mta_ip"), + "tls_report_failures", + ["sending_mta_ip"], + ) + op.create_index( + op.f("ix_tls_report_failures_receiving_mx_hostname"), + "tls_report_failures", + ["receiving_mx_hostname"], + ) + op.create_index( + "ix_tls_report_failures_result_count", + "tls_report_failures", + ["result_type", "failed_session_count"], + ) + op.create_index( + "ix_tls_report_failures_report_result", + "tls_report_failures", + ["report_id", "result_type"], + ) + + +def downgrade() -> None: + """Drop SMTP TLS report storage.""" + op.drop_index("ix_tls_report_failures_report_result", table_name="tls_report_failures") + op.drop_index("ix_tls_report_failures_result_count", table_name="tls_report_failures") + op.drop_index( + op.f("ix_tls_report_failures_receiving_mx_hostname"), + table_name="tls_report_failures", + ) + op.drop_index(op.f("ix_tls_report_failures_sending_mta_ip"), table_name="tls_report_failures") + op.drop_index(op.f("ix_tls_report_failures_result_type"), table_name="tls_report_failures") + op.drop_index(op.f("ix_tls_report_failures_report_id"), table_name="tls_report_failures") + op.drop_index(op.f("ix_tls_report_failures_id"), table_name="tls_report_failures") + op.drop_table("tls_report_failures") + op.drop_index("ix_tls_reports_policy_domain_dates", table_name="tls_reports") + op.drop_index("ix_tls_reports_domain_dates", table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_processed_at"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_end_date"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_begin_date"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_policy_type"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_policy_domain"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_org_name"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_report_id"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_domain_id"), table_name="tls_reports") + op.drop_index(op.f("ix_tls_reports_id"), table_name="tls_reports") + op.drop_table("tls_reports") diff --git a/backend/app/api/api_v1/api.py b/backend/app/api/api_v1/api.py index 5d49f23..db7bffd 100644 --- a/backend/app/api/api_v1/api.py +++ b/backend/app/api/api_v1/api.py @@ -11,6 +11,7 @@ from app.api.api_v1.endpoints import ( settings, setup, stats, + tls_reports, webhook, ) @@ -27,4 +28,5 @@ api_router.include_router(imap.router, prefix="/imap", tags=["imap"]) api_router.include_router(stats.router, prefix="/stats", tags=["stats"]) api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["mail-sources"]) api_router.include_router(settings.router, prefix="/settings", tags=["settings"]) +api_router.include_router(tls_reports.router, prefix="/tls-reports", tags=["tls-reports"]) api_router.include_router(webhook.router, prefix="/webhook", tags=["webhook"]) diff --git a/backend/app/api/api_v1/endpoints/tls_reports.py b/backend/app/api/api_v1/endpoints/tls_reports.py new file mode 100644 index 0000000..e3d205f --- /dev/null +++ b/backend/app/api/api_v1/endpoints/tls_reports.py @@ -0,0 +1,189 @@ +import logging +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session, selectinload + +from app.core.database import get_db +from app.core.security import require_admin_auth +from app.models.domain import Domain +from app.models.report import TLSReport +from app.services.tls_report_parser import MAX_TLS_REPORT_SIZE, TLSReportParser +from app.services.tls_report_persistence import ( + TLS_REPORT_PRIVACY_CONTROLS, + save_tls_report, + summarize_tls_reports, + tls_report_to_dict, +) + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +class TLSFailureResponse(BaseModel): + result_type: str + failed_session_count: int + sending_mta_ip: Optional[str] = None + receiving_mx_hostname: Optional[str] = None + receiving_mx_helo: Optional[str] = None + receiving_ip: Optional[str] = None + failure_reason_code: Optional[str] = None + additional_information: Optional[str] = None + + +class TLSReportResponse(BaseModel): + id: int + report_id: str + domain: Optional[str] = None + org_name: Optional[str] = None + contact_info: Optional[str] = None + policy_domain: str + policy_type: Optional[str] = None + begin_date: Optional[str] = None + end_date: Optional[str] = None + total_successful_sessions: int + total_failure_sessions: int + processed_at: Optional[str] = None + failures: List[TLSFailureResponse] = Field(default_factory=list) + + +class TLSReportListResponse(BaseModel): + total: int + page: int + page_size: int + total_pages: int + reports: List[TLSReportResponse] + privacy: Dict[str, Any] + + +class TLSReportUploadResponse(BaseModel): + success: bool + report_id: str + policies_created: int + policies_skipped: int + duplicate: bool = False + message: str + privacy: Dict[str, Any] + + +class TLSSummaryResponse(BaseModel): + domain: Optional[str] = None + days: int + totals: Dict[str, Any] + trends: List[Dict[str, Any]] = Field(default_factory=list) + top_failures: List[Dict[str, Any]] = Field(default_factory=list) + affected_domains: List[Dict[str, Any]] = Field(default_factory=list) + privacy: Dict[str, Any] + + +def _validate_upload(file: UploadFile, content: bytes) -> None: + filename = file.filename or "" + if not filename: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Filename is required") + if len(content) == 0: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="File is empty") + if len(content) > MAX_TLS_REPORT_SIZE: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail="File too large", + ) + if not filename.lower().endswith((".json", ".json.gz", ".gzip", ".zip")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid file type. Upload a TLS report as .json, .json.gz, or .zip.", + ) + + +def _filtered_tls_query(db: Session, *, domain: Optional[str] = None): + query = db.query(TLSReport).options( + selectinload(TLSReport.domain), selectinload(TLSReport.failures) + ) + if domain: + normalized = domain.lower().strip(".") + query = query.outerjoin(Domain).filter( + (Domain.name == normalized) | (TLSReport.policy_domain == normalized) + ) + return query + + +@router.post("/upload", response_model=TLSReportUploadResponse) +async def upload_tls_report( + file: UploadFile = File(...), + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Upload and store an SMTP TLS Reporting aggregate.""" + try: + content = await file.read() + _validate_upload(file, content) + parsed = TLSReportParser.parse_file(content, file.filename or "") + result = save_tls_report(db, parsed) + db.commit() + return TLSReportUploadResponse( + success=True, + report_id=parsed["report_id"], + policies_created=result["created"], + policies_skipped=result["skipped"], + duplicate=result["created"] == 0 and result["skipped"] > 0, + message=( + "TLS report imported." + if result["created"] + else "TLS report had already been imported." + ), + privacy=TLS_REPORT_PRIVACY_CONTROLS, + ) + except HTTPException: + raise + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc) or "Invalid TLS report format.", + ) from exc + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("Unexpected TLS report upload failure for %s: %s", file.filename, exc) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error processing TLS report.", + ) from exc + + +@router.get("", response_model=TLSReportListResponse) +async def list_tls_reports( + domain: Optional[str] = Query(default=None), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=50, ge=1, le=200), + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """List stored SMTP TLS reports, newest first.""" + query = _filtered_tls_query(db, domain=domain) + total = query.count() + rows = ( + query.order_by(TLSReport.begin_date.desc().nullslast(), TLSReport.id.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + total_pages = (total + page_size - 1) // page_size if total else 0 + return TLSReportListResponse( + total=total, + page=page, + page_size=page_size, + total_pages=total_pages, + reports=[TLSReportResponse(**tls_report_to_dict(row)) for row in rows], + privacy=TLS_REPORT_PRIVACY_CONTROLS, + ) + + +@router.get("/summary", response_model=TLSSummaryResponse) +async def tls_report_summary( + domain: Optional[str] = Query(default=None), + days: int = Query(default=30, ge=1, le=365), + limit: int = Query(default=10, ge=1, le=50), + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Summarize TLS reports into trends and top failure causes.""" + return TLSSummaryResponse(**summarize_tls_reports(db, domain=domain, days=days, limit=limit)) diff --git a/backend/app/main.py b/backend/app/main.py index 74b22dc..b73022c 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -631,6 +631,12 @@ async def forensic_reports(request: Request): return templates.TemplateResponse(request, "forensic_reports.html") +@app.get("/tls-reports", response_class=HTMLResponse) +async def tls_reports(request: Request): + """View SMTP TLS reporting posture summaries.""" + return templates.TemplateResponse(request, "tls_reports.html") + + @app.get("/forensics/{report_id}", response_class=HTMLResponse) async def forensic_report_detail(request: Request, report_id: int): """View detailed information for a specific forensic report.""" diff --git a/backend/app/models/domain.py b/backend/app/models/domain.py index a2b2736..efe7085 100644 --- a/backend/app/models/domain.py +++ b/backend/app/models/domain.py @@ -34,6 +34,7 @@ class Domain(Base): forensic_reports = relationship( "ForensicReport", back_populates="domain", cascade="all, delete-orphan" ) + tls_reports = relationship("TLSReport", back_populates="domain", cascade="all, delete-orphan") user_domains = relationship("UserDomain", back_populates="domain", cascade="all, delete-orphan") # Indexes for common queries diff --git a/backend/app/models/report.py b/backend/app/models/report.py index b1db09c..7486924 100644 --- a/backend/app/models/report.py +++ b/backend/app/models/report.py @@ -151,3 +151,67 @@ class ForensicReport(Base): def __repr__(self): return f"" + + +class TLSReport(Base): + """SMTP TLS reporting (TLS-RPT) aggregate report.""" + + __tablename__ = "tls_reports" + + id = Column(Integer, primary_key=True, index=True) + domain_id = Column(Integer, ForeignKey("domains.id"), nullable=True, index=True) + + report_id = Column(String, nullable=False, index=True) + org_name = Column(String, nullable=True, index=True) + contact_info = Column(String, nullable=True) + policy_domain = Column(String, nullable=False, index=True) + policy_type = Column(String, nullable=True, index=True) + begin_date = Column(DateTime, nullable=True, index=True) + end_date = Column(DateTime, nullable=True, index=True) + total_successful_sessions = Column(Integer, nullable=False, default=0) + total_failure_sessions = Column(Integer, nullable=False, default=0) + raw_policy = Column(Text, nullable=True) + processed_at = Column(DateTime, default=datetime.utcnow, index=True) + + domain = relationship("Domain", back_populates="tls_reports") + failures = relationship( + "TLSReportFailure", + back_populates="report", + cascade="all, delete-orphan", + ) + + __table_args__ = ( + UniqueConstraint("report_id", "policy_domain", name="uq_tls_reports_report_domain"), + Index("ix_tls_reports_domain_dates", "domain_id", "begin_date", "end_date"), + Index("ix_tls_reports_policy_domain_dates", "policy_domain", "begin_date", "end_date"), + ) + + def __repr__(self): + return f"" + + +class TLSReportFailure(Base): + """Grouped TLS-RPT failure detail without message-level identifiers.""" + + __tablename__ = "tls_report_failures" + + id = Column(Integer, primary_key=True, index=True) + report_id = Column(Integer, ForeignKey("tls_reports.id"), nullable=False, index=True) + result_type = Column(String, nullable=False, index=True) + failed_session_count = Column(Integer, nullable=False, default=0) + sending_mta_ip = Column(String, nullable=True, index=True) + receiving_mx_hostname = Column(String, nullable=True, index=True) + receiving_mx_helo = Column(String, nullable=True) + receiving_ip = Column(String, nullable=True) + failure_reason_code = Column(String, nullable=True) + additional_information = Column(Text, nullable=True) + + report = relationship("TLSReport", back_populates="failures") + + __table_args__ = ( + Index("ix_tls_report_failures_result_count", "result_type", "failed_session_count"), + Index("ix_tls_report_failures_report_result", "report_id", "result_type"), + ) + + def __repr__(self): + return f"" diff --git a/backend/app/services/tls_report_parser.py b/backend/app/services/tls_report_parser.py new file mode 100644 index 0000000..3770881 --- /dev/null +++ b/backend/app/services/tls_report_parser.py @@ -0,0 +1,168 @@ +"""Parser for SMTP TLS Reporting (TLS-RPT) JSON aggregates.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import zipfile +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +MAX_TLS_REPORT_SIZE = 10 * 1024 * 1024 +MAX_TLS_UNCOMPRESSED_SIZE = 100 * 1024 * 1024 +MAX_TLS_FILES_IN_ARCHIVE = 10 + + +def _clean(value: Any) -> str: + return " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split()) + + +def _safe_int(value: Any) -> int: + try: + return max(int(value or 0), 0) + except (TypeError, ValueError): + return 0 + + +def _parse_datetime(value: Any) -> Optional[datetime]: + text = _clean(value) + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=None) + + +def _extract_json_from_zip(file_content: bytes) -> Optional[bytes]: + try: + with zipfile.ZipFile(io.BytesIO(file_content)) as archive: + files = archive.infolist() + if len(files) > MAX_TLS_FILES_IN_ARCHIVE: + raise ValueError("TLS report archive contains too many files") + if sum(item.file_size for item in files) > MAX_TLS_UNCOMPRESSED_SIZE: + raise ValueError("TLS report archive is too large after decompression") + for item in files: + if item.filename.lower().endswith(".json"): + if item.file_size > MAX_TLS_UNCOMPRESSED_SIZE: + raise ValueError("TLS report JSON is too large after decompression") + return archive.read(item.filename) + except zipfile.BadZipFile: + return None + return None + + +def _extract_json_content(file_content: bytes, filename: str) -> bytes: + lower = filename.lower() + if lower.endswith(".zip"): + extracted = _extract_json_from_zip(file_content) + if extracted is not None: + return extracted + if lower.endswith((".gz", ".gzip")): + try: + return gzip.decompress(file_content) + except gzip.BadGzipFile as exc: + raise ValueError("Invalid gzip TLS report") from exc + if lower.endswith(".json"): + return file_content + raise ValueError("Invalid TLS report file type. Upload .json, .json.gz, or .zip.") + + +def _policy_domain(policy: Dict[str, Any]) -> str: + return _clean(policy.get("policy-domain")).lower().strip(".") + + +def _normalize_failure(detail: Dict[str, Any]) -> Dict[str, Any]: + return { + "result_type": _clean(detail.get("result-type") or "unknown").lower(), + "failed_session_count": _safe_int(detail.get("failed-session-count")), + "sending_mta_ip": _clean(detail.get("sending-mta-ip")), + "receiving_mx_hostname": _clean(detail.get("receiving-mx-hostname")).lower(), + "receiving_mx_helo": _clean(detail.get("receiving-mx-helo")), + "receiving_ip": _clean(detail.get("receiving-ip")), + "failure_reason_code": _clean(detail.get("failure-reason-code")), + "additional_information": _clean(detail.get("additional-information")), + } + + +def _load_payload(json_content: bytes) -> Dict[str, Any]: + try: + payload = json.loads(json_content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("TLS report is not valid JSON") from exc + if not isinstance(payload, dict): + raise ValueError("TLS report JSON must be an object") + return payload + + +def _normalize_policy(item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + policy = item.get("policy") or {} + summary = item.get("summary") or {} + if not isinstance(policy, dict) or not isinstance(summary, dict): + return None + domain = _policy_domain(policy) + if not domain: + return None + failures = [ + _normalize_failure(detail) + for detail in item.get("failure-details") or [] + if isinstance(detail, dict) + ] + return { + "policy_domain": domain, + "policy_type": _clean(policy.get("policy-type")).lower(), + "policy": policy, + "total_successful_sessions": _safe_int(summary.get("total-successful-session-count")), + "total_failure_sessions": _safe_int(summary.get("total-failure-session-count")), + "failures": failures, + } + + +def _normalize_policies(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + policies = [] + for item in payload.get("policies") or []: + if not isinstance(item, dict): + continue + policy = _normalize_policy(item) + if policy is not None: + policies.append(policy) + if not policies: + raise ValueError("TLS report does not contain any policy-domain entries") + return policies + + +class TLSReportParser: + """Parse TLS-RPT JSON while retaining only aggregate posture data.""" + + @staticmethod + def parse_file(file_content: bytes, filename: str) -> Dict[str, Any]: + """Parse a TLS-RPT JSON, gzip, or zip attachment into normalized dictionaries.""" + if len(file_content) > MAX_TLS_REPORT_SIZE: + raise ValueError("TLS report is too large") + if not file_content: + raise ValueError("TLS report is empty") + + json_content = _extract_json_content(file_content, filename) + if len(json_content) > MAX_TLS_UNCOMPRESSED_SIZE: + raise ValueError("TLS report is too large after decompression") + + payload = _load_payload(json_content) + date_range = payload.get("date-range") or {} + policies = _normalize_policies(payload) + + report_id = _clean(payload.get("report-id")) + if not report_id: + report_id = "tlsrpt-" + hashlib.sha256(json_content).hexdigest()[:24] + return { + "report_id": report_id, + "org_name": _clean(payload.get("organization-name")), + "contact_info": _clean(payload.get("contact-info")), + "begin_date": _parse_datetime(date_range.get("start-datetime")), + "end_date": _parse_datetime(date_range.get("end-datetime")), + "policies": policies, + } diff --git a/backend/app/services/tls_report_persistence.py b/backend/app/services/tls_report_persistence.py new file mode 100644 index 0000000..68db3b9 --- /dev/null +++ b/backend/app/services/tls_report_persistence.py @@ -0,0 +1,336 @@ +"""Persistence and summarization helpers for SMTP TLS reports.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from datetime import datetime, timedelta +from typing import Any, Dict, Iterable, List, Optional + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, selectinload + +from app.models.domain import Domain +from app.models.report import TLSReport, TLSReportFailure +from app.utils.domain_validator import DomainValidationError, validate_domain + + +TLS_REPORT_PRIVACY_CONTROLS = { + "retention": ( + "TLS reports store aggregate session counts, reporting organization metadata, " + "policy domains, and grouped TLS failure details." + ), + "stored_fields": [ + "report id", + "reporting organization", + "contact info", + "policy domain", + "policy type", + "report date range", + "successful and failed session counts", + "grouped result type and failed-session count", + "sending MTA IP when supplied by the reporter", + "receiving MX host/HELO/IP when supplied by the reporter", + "failure reason code and additional grouped diagnostic text", + ], + "not_stored": [ + "message bodies", + "message subjects", + "sender or recipient addresses", + "recipient local-parts", + "raw uploaded attachments", + "mailbox credentials or source message identifiers", + ], +} + + +def tls_report_exists(db: Session, report_id: str, policy_domain: str) -> bool: + """Return True when a TLS report policy entry already exists.""" + normalized_report_id = str(report_id or "").strip() + normalized_domain = str(policy_domain or "").strip().lower().strip(".") + if not normalized_report_id or not normalized_domain: + return False + return ( + db.query(TLSReport.id) + .filter( + TLSReport.report_id == normalized_report_id, + TLSReport.policy_domain == normalized_domain, + ) + .first() + is not None + ) + + +def _domain_for_report(db: Session, domain_name: Optional[str]) -> Optional[Domain]: + if not domain_name: + return None + normalized = domain_name.lower().strip(".") + is_valid, _, error_code = validate_domain(normalized, check_dns=False) + if not is_valid and error_code != DomainValidationError.DNS_RESOLUTION_FAILED: + return None + + domain = db.query(Domain).filter(Domain.name == normalized).first() + if domain is None: + domain = Domain(name=normalized) + db.add(domain) + db.flush() + return domain + + +def _save_policy_report( + db: Session, + parsed_report: Dict[str, Any], + policy: Dict[str, Any], +) -> tuple[Optional[TLSReport], bool]: + report_id = str(parsed_report.get("report_id") or "").strip() + policy_domain = str(policy.get("policy_domain") or "").strip().lower().strip(".") + if not report_id or not policy_domain: + return None, False + + existing = ( + db.query(TLSReport) + .filter( + TLSReport.report_id == report_id, + TLSReport.policy_domain == policy_domain, + ) + .first() + ) + if existing is not None: + return existing, False + + domain = _domain_for_report(db, policy_domain) + row = TLSReport( + domain_id=domain.id if domain else None, + report_id=report_id, + org_name=parsed_report.get("org_name"), + contact_info=parsed_report.get("contact_info"), + policy_domain=policy_domain, + policy_type=policy.get("policy_type"), + begin_date=parsed_report.get("begin_date"), + end_date=parsed_report.get("end_date"), + total_successful_sessions=policy.get("total_successful_sessions") or 0, + total_failure_sessions=policy.get("total_failure_sessions") or 0, + raw_policy=json.dumps(policy.get("policy") or {}, sort_keys=True), + ) + for failure in policy.get("failures") or []: + row.failures.append( + TLSReportFailure( + result_type=failure.get("result_type") or "unknown", + failed_session_count=failure.get("failed_session_count") or 0, + sending_mta_ip=failure.get("sending_mta_ip") or None, + receiving_mx_hostname=failure.get("receiving_mx_hostname") or None, + receiving_mx_helo=failure.get("receiving_mx_helo") or None, + receiving_ip=failure.get("receiving_ip") or None, + failure_reason_code=failure.get("failure_reason_code") or None, + additional_information=failure.get("additional_information") or None, + ) + ) + + db.add(row) + try: + db.flush() + except IntegrityError: + db.rollback() + existing = ( + db.query(TLSReport) + .filter( + TLSReport.report_id == report_id, + TLSReport.policy_domain == policy_domain, + ) + .first() + ) + if existing is not None: + return existing, False + raise + return row, True + + +def save_tls_report(db: Session, parsed_report: Dict[str, Any]) -> Dict[str, Any]: + """Persist parsed TLS report policy entries. + + One TLS-RPT JSON can carry multiple policy domains. Each policy is stored + independently so partial duplicate imports can still add newly seen domains. + The caller owns the transaction. + """ + rows: List[TLSReport] = [] + created = 0 + skipped = 0 + for policy in parsed_report.get("policies") or []: + row, was_created = _save_policy_report(db, parsed_report, policy) + if row is None: + skipped += 1 + continue + rows.append(row) + if was_created: + created += 1 + else: + skipped += 1 + return {"rows": rows, "created": created, "skipped": skipped} + + +def tls_report_to_dict(row: TLSReport) -> Dict[str, Any]: + """Convert a TLS report row to an API-safe dictionary.""" + return { + "id": row.id, + "report_id": row.report_id, + "domain": row.domain.name if row.domain else row.policy_domain, + "org_name": row.org_name, + "contact_info": row.contact_info, + "policy_domain": row.policy_domain, + "policy_type": row.policy_type, + "begin_date": row.begin_date.isoformat() if row.begin_date else None, + "end_date": row.end_date.isoformat() if row.end_date else None, + "total_successful_sessions": row.total_successful_sessions, + "total_failure_sessions": row.total_failure_sessions, + "processed_at": row.processed_at.isoformat() if row.processed_at else None, + "failures": [ + { + "result_type": failure.result_type, + "failed_session_count": failure.failed_session_count, + "sending_mta_ip": failure.sending_mta_ip, + "receiving_mx_hostname": failure.receiving_mx_hostname, + "receiving_mx_helo": failure.receiving_mx_helo, + "receiving_ip": failure.receiving_ip, + "failure_reason_code": failure.failure_reason_code, + "additional_information": failure.additional_information, + } + for failure in row.failures + ], + } + + +def _row_day(row: TLSReport) -> str: + basis = row.begin_date or row.end_date or row.processed_at or datetime.utcnow() + return basis.date().isoformat() + + +def _report_rows( + db: Session, + *, + domain: Optional[str] = None, + days: int = 30, +) -> Iterable[TLSReport]: + cutoff = datetime.utcnow() - timedelta(days=days) + query = db.query(TLSReport).options( + selectinload(TLSReport.domain), selectinload(TLSReport.failures) + ) + query = query.filter( + (TLSReport.begin_date >= cutoff) + | (TLSReport.end_date >= cutoff) + | (TLSReport.processed_at >= cutoff) + ) + if domain: + normalized = domain.lower().strip(".") + query = query.outerjoin(Domain).filter( + (Domain.name == normalized) | (TLSReport.policy_domain == normalized) + ) + return query.order_by(TLSReport.begin_date.desc().nullslast(), TLSReport.id.desc()).all() + + +def summarize_tls_reports( + db: Session, + *, + domain: Optional[str] = None, + days: int = 30, + limit: int = 10, +) -> Dict[str, Any]: + """Summarize TLS reports into trends and actionable failure groupings.""" + rows = list(_report_rows(db, domain=domain, days=days)) + + totals = { + "reports": len(rows), + "successful_sessions": sum(row.total_successful_sessions or 0 for row in rows), + "failed_sessions": sum(row.total_failure_sessions or 0 for row in rows), + } + session_total = totals["successful_sessions"] + totals["failed_sessions"] + totals["failure_rate"] = (totals["failed_sessions"] / session_total) if session_total else 0.0 + + trend_map: Dict[str, Dict[str, Any]] = defaultdict( + lambda: {"date": "", "reports": 0, "successful_sessions": 0, "failed_sessions": 0} + ) + domain_map: Dict[str, Dict[str, Any]] = defaultdict( + lambda: { + "domain": "", + "reports": 0, + "successful_sessions": 0, + "failed_sessions": 0, + "top_failure": None, + } + ) + failure_map: Dict[str, Dict[str, Any]] = defaultdict( + lambda: { + "result_type": "", + "failed_sessions": 0, + "reports": set(), + "affected_domains": set(), + "receiving_mx_hostnames": set(), + "reason_codes": set(), + } + ) + + for row in rows: + day = _row_day(row) + trend = trend_map[day] + trend["date"] = day + trend["reports"] += 1 + trend["successful_sessions"] += row.total_successful_sessions or 0 + trend["failed_sessions"] += row.total_failure_sessions or 0 + + domain_summary = domain_map[row.policy_domain] + domain_summary["domain"] = row.policy_domain + domain_summary["reports"] += 1 + domain_summary["successful_sessions"] += row.total_successful_sessions or 0 + domain_summary["failed_sessions"] += row.total_failure_sessions or 0 + + top_for_row = None + for failure in row.failures: + result_type = failure.result_type or "unknown" + item = failure_map[result_type] + item["result_type"] = result_type + item["failed_sessions"] += failure.failed_session_count or 0 + item["reports"].add(row.report_id) + item["affected_domains"].add(row.policy_domain) + if failure.receiving_mx_hostname: + item["receiving_mx_hostnames"].add(failure.receiving_mx_hostname) + if failure.failure_reason_code: + item["reason_codes"].add(failure.failure_reason_code) + if ( + top_for_row is None + or (failure.failed_session_count or 0) > top_for_row.failed_session_count + ): + top_for_row = failure + if top_for_row is not None: + domain_summary["top_failure"] = top_for_row.result_type + + trends = [trend_map[key] for key in sorted(trend_map)] + affected_domains = [] + for item in domain_map.values(): + domain_sessions = item["successful_sessions"] + item["failed_sessions"] + item["failure_rate"] = item["failed_sessions"] / domain_sessions if domain_sessions else 0.0 + affected_domains.append(item) + + top_failures = [] + for item in failure_map.values(): + top_failures.append( + { + "result_type": item["result_type"], + "failed_sessions": item["failed_sessions"], + "report_count": len(item["reports"]), + "affected_domains": sorted(item["affected_domains"]), + "receiving_mx_hostnames": sorted(item["receiving_mx_hostnames"])[:5], + "reason_codes": sorted(item["reason_codes"])[:5], + } + ) + + top_failures.sort(key=lambda item: item["failed_sessions"], reverse=True) + affected_domains.sort(key=lambda item: item["failed_sessions"], reverse=True) + + return { + "domain": domain, + "days": days, + "totals": totals, + "trends": trends, + "top_failures": top_failures[:limit], + "affected_domains": affected_domains[:limit], + "privacy": TLS_REPORT_PRIVACY_CONTROLS, + } diff --git a/backend/app/templates/layouts/base.html b/backend/app/templates/layouts/base.html index 4d6ae64..ba76cd3 100644 --- a/backend/app/templates/layouts/base.html +++ b/backend/app/templates/layouts/base.html @@ -39,6 +39,7 @@
  • Domains
  • Reports
  • Forensics
    • +
  • TLS Reports
  • Upload
  • Mail Sources
  • Health
    • diff --git a/backend/app/templates/tls_reports.html b/backend/app/templates/tls_reports.html new file mode 100644 index 0000000..3c6fd50 --- /dev/null +++ b/backend/app/templates/tls_reports.html @@ -0,0 +1,281 @@ +{% extends "layouts/base.html" %} +{% from "components/ui/card.html" import card, card_header, card_title, card_description, card_content %} +{% from "components/ui/table.html" import table, thead, tbody, tr, th, td %} + +{% block title %}DMARQ - TLS Reports{% endblock %} + +{% block content %} +
    +
    +
    +

    TLS Reports

    +

    SMTP TLS delivery trends and failure causes

    +
    +
    + + +
    +
    + +
    + {% call card() %} + {% call card_header() %}{% call card_title() %}Reports{% endcall %}{% endcall %} + {% call card_content() %} +
    + {% endcall %} + {% endcall %} + {% call card() %} + {% call card_header() %}{% call card_title() %}Successful Sessions{% endcall %}{% endcall %} + {% call card_content() %} +
    + {% endcall %} + {% endcall %} + {% call card() %} + {% call card_header() %}{% call card_title() %}Failed Sessions{% endcall %}{% endcall %} + {% call card_content() %} +
    + {% endcall %} + {% endcall %} + {% call card() %} + {% call card_header() %}{% call card_title() %}Failure Rate{% endcall %}{% endcall %} + {% call card_content() %} +
    + {% endcall %} + {% endcall %} +
    + +
    + {% call card() %} + {% call card_header() %} +
    +
    + {% call card_title() %}Failure Trends{% endcall %} + {% call card_description() %}Daily TLS session results from imported TLS-RPT files{% endcall %} +
    + +
    + {% endcall %} + {% call card_content() %} + + + +
    + +
    + {% endcall %} + {% endcall %} + + {% call card() %} + {% call card_header() %} + {% call card_title() %}Import TLS Report{% endcall %} + {% call card_description() %}.json, .json.gz, or .zip{% endcall %} + {% endcall %} + {% call card_content() %} +
    + + +

    +
    + {% endcall %} + {% endcall %} +
    + +
    + {% call card() %} + {% call card_header() %} + {% call card_title() %}Top Failure Causes{% endcall %} + {% call card_description() %}Grouped by TLS-RPT result type{% endcall %} + {% endcall %} + {% call card_content() %} + {% call table() %} + {% call thead() %} + {% call tr() %} + {% call th() %}Cause{% endcall %} + {% call th("text-right") %}Failed Sessions{% endcall %} + {% call th() %}Domains{% endcall %} + {% call th() %}MX Hosts{% endcall %} + {% endcall %} + {% endcall %} + {% call tbody() %} + + + {% endcall %} + {% endcall %} + {% endcall %} + {% endcall %} + + {% call card() %} + {% call card_header() %} + {% call card_title() %}Affected Domains{% endcall %} + {% call card_description() %}Domains with TLS-RPT activity in the selected window{% endcall %} + {% endcall %} + {% call card_content() %} + {% call table() %} + {% call thead() %} + {% call tr() %} + {% call th() %}Domain{% endcall %} + {% call th("text-right") %}Reports{% endcall %} + {% call th("text-right") %}Failed{% endcall %} + {% call th("text-right") %}Rate{% endcall %} + {% endcall %} + {% endcall %} + {% call tbody() %} + + + {% endcall %} + {% endcall %} + {% endcall %} + {% endcall %} +
    + + {% call card() %} + {% call card_header() %} + {% call card_title() %}Stored Data{% endcall %} + {% call card_description() %}Current TLS reporting retention and privacy controls{% endcall %} + {% endcall %} + {% call card_content() %} +
    +

    +
    +

    Stored

    +
      + +
    +
    +
    +

    Not Stored

    +
      + +
    +
    +
    + {% endcall %} + {% endcall %} +
    +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/backend/app/tests/test_tls_report_parser.py b/backend/app/tests/test_tls_report_parser.py new file mode 100644 index 0000000..93f23f0 --- /dev/null +++ b/backend/app/tests/test_tls_report_parser.py @@ -0,0 +1,99 @@ +import gzip +import io +import json +import zipfile + +import pytest + +from app.services.tls_report_parser import TLSReportParser + + +SAMPLE_TLS_REPORT = { + "organization-name": "Example Reporter", + "date-range": { + "start-datetime": "2026-05-20T00:00:00Z", + "end-datetime": "2026-05-20T23:59:59Z", + }, + "contact-info": "tlsrpt@example-reporter.test", + "report-id": "tls-report-20260520", + "policies": [ + { + "policy": { + "policy-type": "sts", + "policy-string": ["version: STSv1", "mode: enforce"], + "policy-domain": "Example.com.", + "mx-host": ["mx.example.com"], + }, + "summary": { + "total-successful-session-count": 125, + "total-failure-session-count": 7, + }, + "failure-details": [ + { + "result-type": "certificate-expired", + "sending-mta-ip": "203.0.113.9", + "receiving-mx-hostname": "MX.EXAMPLE.COM", + "failed-session-count": 7, + "failure-reason-code": "tls", + "additional-information": "certificate expired", + } + ], + } + ], +} + + +def sample_tls_report_bytes(report=None): + return json.dumps(report or SAMPLE_TLS_REPORT).encode("utf-8") + + +def test_parse_tls_report_json_normalizes_policy_and_failures(): + parsed = TLSReportParser.parse_file(sample_tls_report_bytes(), "tls-report.json") + + assert parsed["report_id"] == "tls-report-20260520" + assert parsed["org_name"] == "Example Reporter" + assert parsed["begin_date"].isoformat() == "2026-05-20T00:00:00" + assert parsed["policies"][0]["policy_domain"] == "example.com" + assert parsed["policies"][0]["policy_type"] == "sts" + assert parsed["policies"][0]["total_failure_sessions"] == 7 + assert parsed["policies"][0]["failures"][0]["result_type"] == "certificate-expired" + assert parsed["policies"][0]["failures"][0]["receiving_mx_hostname"] == "mx.example.com" + + +def test_parse_tls_report_gzip(): + compressed = gzip.compress(sample_tls_report_bytes()) + + parsed = TLSReportParser.parse_file(compressed, "tls-report.json.gz") + + assert parsed["report_id"] == "tls-report-20260520" + assert parsed["policies"][0]["total_successful_sessions"] == 125 + + +def test_parse_tls_report_zip(): + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zip_file: + zip_file.writestr("nested/tls-report.json", sample_tls_report_bytes()) + + parsed = TLSReportParser.parse_file(archive.getvalue(), "tls-report.zip") + + assert parsed["policies"][0]["policy_domain"] == "example.com" + + +def test_parse_tls_report_generates_stable_id_when_missing(): + report = dict(SAMPLE_TLS_REPORT) + report.pop("report-id") + + parsed = TLSReportParser.parse_file(sample_tls_report_bytes(report), "tls-report.json") + + assert parsed["report_id"].startswith("tlsrpt-") + assert len(parsed["report_id"]) == 31 + + +def test_parse_tls_report_rejects_missing_policies(): + with pytest.raises(ValueError, match="policy-domain"): + TLSReportParser.parse_file(b'{"policies":[]}', "tls-report.json") + + +def test_parse_tls_report_rejects_invalid_extension(): + with pytest.raises(ValueError, match="Invalid TLS report file type"): + TLSReportParser.parse_file(sample_tls_report_bytes(), "tls-report.txt") diff --git a/backend/app/tests/test_tls_reports_api.py b/backend/app/tests/test_tls_reports_api.py new file mode 100644 index 0000000..15b9182 --- /dev/null +++ b/backend/app/tests/test_tls_reports_api.py @@ -0,0 +1,152 @@ +from fastapi import HTTPException +from fastapi.testclient import TestClient + +from app.api.api_v1.endpoints import tls_reports as tls_endpoint +from app.models.domain import Domain +from app.models.report import TLSReport, TLSReportFailure +from app.services.tls_report_parser import TLSReportParser +from app.services.tls_report_persistence import ( + save_tls_report, + summarize_tls_reports, + tls_report_exists, +) +from app.tests.test_tls_report_parser import sample_tls_report_bytes + + +def test_upload_tls_report_persists_policy_and_failure_details(authed_client, db_session): + response = authed_client.post( + "/api/v1/tls-reports/upload", + files={"file": ("tls-report.json", sample_tls_report_bytes(), "application/json")}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["policies_created"] == 1 + assert data["policies_skipped"] == 0 + assert "message bodies" in data["privacy"]["not_stored"] + + report = db_session.query(TLSReport).one() + assert report.policy_domain == "example.com" + assert report.total_failure_sessions == 7 + assert db_session.query(Domain).filter(Domain.name == "example.com").count() == 1 + failure = db_session.query(TLSReportFailure).one() + assert failure.result_type == "certificate-expired" + assert failure.failed_session_count == 7 + + +def test_upload_tls_report_marks_duplicates_without_double_counting(authed_client, db_session): + files = {"file": ("tls-report.json", sample_tls_report_bytes(), "application/json")} + assert authed_client.post("/api/v1/tls-reports/upload", files=files).status_code == 200 + + response = authed_client.post( + "/api/v1/tls-reports/upload", + files={"file": ("tls-report.json", sample_tls_report_bytes(), "application/json")}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["duplicate"] is True + assert data["policies_created"] == 0 + assert data["policies_skipped"] == 1 + assert db_session.query(TLSReport).count() == 1 + + +def test_tls_report_summary_groups_failures_and_domains(authed_client): + authed_client.post( + "/api/v1/tls-reports/upload", + files={"file": ("tls-report.json", sample_tls_report_bytes(), "application/json")}, + ) + + response = authed_client.get("/api/v1/tls-reports/summary?domain=example.com&days=30") + + assert response.status_code == 200 + data = response.json() + assert data["totals"]["reports"] == 1 + assert data["totals"]["failed_sessions"] == 7 + assert data["top_failures"][0]["result_type"] == "certificate-expired" + assert data["top_failures"][0]["affected_domains"] == ["example.com"] + assert data["affected_domains"][0]["failure_rate"] > 0 + assert "sender or recipient addresses" in data["privacy"]["not_stored"] + + +def test_list_tls_reports_filters_by_domain(authed_client, db_session): + parsed = TLSReportParser.parse_file(sample_tls_report_bytes(), "tls-report.json") + second = dict(parsed) + second["report_id"] = "tls-report-second" + second["policies"] = [dict(parsed["policies"][0], policy_domain="example.net")] + save_tls_report(db_session, parsed) + save_tls_report(db_session, second) + db_session.commit() + + response = authed_client.get("/api/v1/tls-reports?domain=example.net") + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["reports"][0]["policy_domain"] == "example.net" + + +def test_tls_report_summary_empty_response_includes_privacy_controls(authed_client): + response = authed_client.get("/api/v1/tls-reports/summary") + + assert response.status_code == 200 + data = response.json() + assert data["totals"]["reports"] == 0 + assert data["top_failures"] == [] + assert "raw uploaded attachments" in data["privacy"]["not_stored"] + + +def test_tls_report_persistence_helpers(db_session): + parsed = TLSReportParser.parse_file(sample_tls_report_bytes(), "tls-report.json") + + result = save_tls_report(db_session, parsed) + db_session.commit() + + assert result["created"] == 1 + assert tls_report_exists(db_session, "tls-report-20260520", "example.com") + summary = summarize_tls_reports(db_session, domain="example.com") + assert summary["totals"]["successful_sessions"] == 125 + + +def test_upload_tls_report_rejects_invalid_file_type(authed_client): + response = authed_client.post( + "/api/v1/tls-reports/upload", + files={"file": ("tls-report.txt", sample_tls_report_bytes(), "text/plain")}, + ) + + assert response.status_code == 400 + + +def test_validate_upload_rejects_missing_name_and_large_file(): + missing_name = type("Upload", (), {"filename": ""})() + too_large = type("Upload", (), {"filename": "report.json"})() + + try: + tls_endpoint._validate_upload(missing_name, b"content") + except HTTPException as exc: + assert exc.status_code == 400 + else: + raise AssertionError("Expected missing filename to be rejected") + + try: + tls_endpoint._validate_upload( + too_large, + b"x" * (tls_endpoint.MAX_TLS_REPORT_SIZE + 1), + ) + except HTTPException as exc: + assert exc.status_code == 413 + else: + raise AssertionError("Expected large file to be rejected") + + +def test_tls_report_html_page_renders(): + from app.core.logto import SESSION_COOKIE, create_session_token # noqa: PLC0415 + from app.main import app as main_app # noqa: PLC0415 + + cookies = {SESSION_COOKIE: create_session_token(user_id=1)} + with TestClient(main_app) as client: + response = client.get("/tls-reports", cookies=cookies) + + assert response.status_code == 200 + assert "TLS Reports" in response.text diff --git a/docs/index.md b/docs/index.md index 3a7875b..1b9dafa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,4 +25,6 @@ For installation instructions, check the [Docker Setup](deployment/docker.md) or For Microsoft 365 setup, see [Microsoft 365 Mail Sources](user_guide/microsoft365.md). +For SMTP TLS reporting imports and privacy controls, see [TLS Reports](user_guide/tls_reports.md). + For aggregate-report parser support, known edge cases, and fixture guidance, see [DMARC Aggregate Format Compatibility](reference/dmarc-compatibility.md). diff --git a/docs/milestones.md b/docs/milestones.md index 35de81a..bf9c688 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -220,7 +220,7 @@ Goal: turn DMARQ into a broader email authentication posture console (still priv Planned: - MTA-STS posture: delivered cached `_mta-sts` TXT checks, HTTPS policy validation, domain-detail evidence, and operator guidance for missing, invalid, or non-enforcing policies. Optional helper tooling remains a future enhancement. -- TLS reporting posture: ingest and summarize TLS report data (where available) with actionable failure grouping. +- TLS reporting posture: delivered authenticated TLS-RPT upload for `.json`, `.json.gz`, and `.zip` attachments; duplicate-safe persistence by report ID and policy domain; daily session trends; top failure-cause grouping; affected-domain summaries; and explicit privacy controls that avoid storing message content or recipient data. - BIMI posture: record validation + readiness checks + operator guidance. - Extended DNS checks that support the posture surface (e.g., MX/BIMI; optional DANE/TLSA where relevant). diff --git a/docs/reference/api.md b/docs/reference/api.md index ac866c6..4c43002 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -259,6 +259,38 @@ Uploads a new DMARC report for processing. } ``` +### TLS Reports + +#### Upload TLS Report + +``` +POST /tls-reports/upload +``` + +Uploads an SMTP TLS Reporting aggregate attachment. Supported file types are +`.json`, `.json.gz`, and `.zip`. + +**Example Response:** +```json +{ + "success": true, + "report_id": "tls-report-20260520", + "policies_created": 1, + "policies_skipped": 0, + "duplicate": false, + "message": "TLS report imported." +} +``` + +#### Summarize TLS Reports + +``` +GET /tls-reports/summary?domain=example.com&days=30 +``` + +Returns aggregate TLS trends, top failure causes, affected domains, and the +privacy controls for stored TLS-RPT data. + ### Statistics #### Compliance Summary @@ -413,4 +445,4 @@ Supported events: - `report.processed` - When a new report is processed - `compliance.threshold` - When compliance falls below threshold - `domain.added` - When a domain is added -- `domain.removed` - When a domain is removed \ No newline at end of file +- `domain.removed` - When a domain is removed diff --git a/docs/user_guide/tls_reports.md b/docs/user_guide/tls_reports.md new file mode 100644 index 0000000..6ff578c --- /dev/null +++ b/docs/user_guide/tls_reports.md @@ -0,0 +1,53 @@ +# TLS Reports + +DMARQ can import SMTP TLS Reporting (TLS-RPT) JSON aggregates and summarize +delivery security failures alongside the existing DMARC posture views. + +## Importing TLS Reports + +Open **TLS Reports** and upload a `.json`, `.json.gz`, or `.zip` TLS-RPT +attachment. A single file can contain multiple policy domains; DMARQ stores +each policy-domain entry independently and skips duplicates by `report-id` plus +policy domain. + +Imported reports appear in the same page as: + +- daily successful and failed TLS session trends +- top TLS failure causes grouped by `result-type` +- affected policy domains with failure rates +- receiving MX hostnames and reason codes when reporters include them + +## API + +TLS reporting endpoints require the same admin authentication as other +operational endpoints: + +- `POST /api/v1/tls-reports/upload` imports one TLS-RPT attachment. +- `GET /api/v1/tls-reports` lists stored TLS report policy entries. +- `GET /api/v1/tls-reports/summary?domain=example.com&days=30` returns trends, + top failure causes, and affected domains. + +## Retention and Privacy + +DMARQ stores only aggregate TLS reporting posture data: + +- report ID +- reporting organization and contact info +- policy domain and policy type +- report date range +- successful and failed session counts +- grouped failure result type and failed-session count +- sending MTA IP, receiving MX hostname, HELO, or IP when supplied by the report +- failure reason code and grouped diagnostic text + +DMARQ does not store: + +- message bodies +- message subjects +- sender or recipient addresses +- recipient local-parts +- raw uploaded attachments +- mailbox credentials or source message identifiers + +Use normal database backup and retention processes to control how long imported +TLS-RPT aggregates remain available. diff --git a/mkdocs.yml b/mkdocs.yml index f2f8e01..783f58a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Dashboard: user_guide/dashboard.md - Managing Domains: user_guide/domains.md - DMARC Reports: user_guide/reports.md + - TLS Reports: user_guide/tls_reports.md - IMAP Integration: user_guide/imap.md - Settings: user_guide/settings.md - Installation: @@ -74,4 +75,4 @@ markdown_extensions: - attr_list - md_in_html - toc: - permalink: true \ No newline at end of file + permalink: true