From 1f954c9e083e8706661cd69d67ffbef16ab4bf47 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Sat, 23 May 2026 12:52:54 +0200 Subject: [PATCH] feat: add forensic report parsing storage --- TODO.md | 7 +- .../d1e2f3a4b5c6_add_forensic_reports.py | 88 ++++++++ backend/app/api/api_v1/api.py | 2 + backend/app/api/api_v1/endpoints/forensics.py | 172 +++++++++++++++ backend/app/api/api_v1/endpoints/imap.py | 2 + .../app/api/api_v1/endpoints/mail_sources.py | 11 +- backend/app/main.py | 12 +- backend/app/models/domain.py | 3 + backend/app/models/report.py | 49 ++++- backend/app/services/forensic_parser.py | 197 ++++++++++++++++++ backend/app/services/forensic_persistence.py | 103 +++++++++ backend/app/services/gmail_client.py | 84 +++++++- backend/app/services/imap_client.py | 95 +++++++++ backend/app/tests/test_forensic_parser.py | 77 +++++++ backend/app/tests/test_forensics_api.py | 56 +++++ backend/app/tests/test_gmail_client.py | 25 ++- backend/app/tests/test_imap_client.py | 30 ++- docs/milestones.md | 9 +- 18 files changed, 1005 insertions(+), 17 deletions(-) create mode 100644 backend/alembic/versions/d1e2f3a4b5c6_add_forensic_reports.py create mode 100644 backend/app/api/api_v1/endpoints/forensics.py create mode 100644 backend/app/services/forensic_parser.py create mode 100644 backend/app/services/forensic_persistence.py create mode 100644 backend/app/tests/test_forensic_parser.py create mode 100644 backend/app/tests/test_forensics_api.py diff --git a/TODO.md b/TODO.md index 452fb1e..a7ba915 100644 --- a/TODO.md +++ b/TODO.md @@ -63,9 +63,10 @@ have no working implementation in the codebase yet. ### Forensic Reports (RFC 6591) - **Documented in**: README.md ("Forensic Reports: Analyze failure samples (RFC 6591 support)") -- **Current state**: The DMARC parser (`backend/app/services/dmarc_parser.py`) only - handles aggregate reports. There is no forensic report parsing, UI, or storage. -- [ ] Forensic report parsing +- **Current state**: Aggregate and forensic reports are now parsed separately. Forensic + reports are stored in dedicated database rows and surfaced through authenticated APIs + without affecting aggregate compliance statistics. +- [x] Forensic report parsing - [ ] Failure sample analysis - [ ] PII redaction options - [ ] Detailed authentication failure views diff --git a/backend/alembic/versions/d1e2f3a4b5c6_add_forensic_reports.py b/backend/alembic/versions/d1e2f3a4b5c6_add_forensic_reports.py new file mode 100644 index 0000000..71f9422 --- /dev/null +++ b/backend/alembic/versions/d1e2f3a4b5c6_add_forensic_reports.py @@ -0,0 +1,88 @@ +"""add forensic reports + +Revision ID: d1e2f3a4b5c6 +Revises: c0d1e2f3a4b5 +Create Date: 2026-05-23 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "d1e2f3a4b5c6" +down_revision: Union[str, Sequence[str], None] = "c0d1e2f3a4b5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create DMARC forensic/failure report storage.""" + op.create_table( + "forensic_reports", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("domain_id", sa.Integer(), nullable=True), + sa.Column("report_id", sa.String(), nullable=False), + sa.Column("source_email", sa.String(), nullable=True), + sa.Column("feedback_type", sa.String(), nullable=True), + sa.Column("user_agent", sa.String(), nullable=True), + sa.Column("version", sa.String(), nullable=True), + sa.Column("reported_domain", sa.String(), nullable=True), + sa.Column("source_ip", sa.String(), nullable=True), + sa.Column("auth_failure", sa.String(), nullable=True), + sa.Column("delivery_result", sa.String(), nullable=True), + sa.Column("arrival_date", sa.DateTime(), nullable=True), + sa.Column("authentication_results", sa.Text(), nullable=True), + sa.Column("original_mail_from", sa.String(), nullable=True), + sa.Column("original_from", sa.String(), nullable=True), + sa.Column("original_to", sa.String(), nullable=True), + sa.Column("original_subject", sa.String(), nullable=True), + sa.Column("original_message_id", sa.String(), nullable=True), + sa.Column("original_date", sa.String(), nullable=True), + sa.Column("feedback_headers", sa.Text(), nullable=True), + sa.Column("processed_at", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["domain_id"], ["domains.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("report_id", name="uq_forensic_reports_report_id"), + ) + op.create_index(op.f("ix_forensic_reports_id"), "forensic_reports", ["id"]) + op.create_index(op.f("ix_forensic_reports_domain_id"), "forensic_reports", ["domain_id"]) + op.create_index(op.f("ix_forensic_reports_report_id"), "forensic_reports", ["report_id"]) + op.create_index( + op.f("ix_forensic_reports_feedback_type"), "forensic_reports", ["feedback_type"] + ) + op.create_index( + op.f("ix_forensic_reports_reported_domain"), "forensic_reports", ["reported_domain"] + ) + op.create_index(op.f("ix_forensic_reports_source_ip"), "forensic_reports", ["source_ip"]) + op.create_index(op.f("ix_forensic_reports_auth_failure"), "forensic_reports", ["auth_failure"]) + op.create_index(op.f("ix_forensic_reports_arrival_date"), "forensic_reports", ["arrival_date"]) + op.create_index(op.f("ix_forensic_reports_processed_at"), "forensic_reports", ["processed_at"]) + op.create_index( + "ix_forensic_reports_domain_arrival", + "forensic_reports", + ["domain_id", "arrival_date"], + ) + op.create_index( + "ix_forensic_reports_failure_source", + "forensic_reports", + ["auth_failure", "source_ip"], + ) + + +def downgrade() -> None: + """Drop DMARC forensic/failure report storage.""" + op.drop_index("ix_forensic_reports_failure_source", table_name="forensic_reports") + op.drop_index("ix_forensic_reports_domain_arrival", table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_processed_at"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_arrival_date"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_auth_failure"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_source_ip"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_reported_domain"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_feedback_type"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_report_id"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_domain_id"), table_name="forensic_reports") + op.drop_index(op.f("ix_forensic_reports_id"), table_name="forensic_reports") + op.drop_table("forensic_reports") diff --git a/backend/app/api/api_v1/api.py b/backend/app/api/api_v1/api.py index 6c8a518..5d49f23 100644 --- a/backend/app/api/api_v1/api.py +++ b/backend/app/api/api_v1/api.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from app.api.api_v1.endpoints import ( auth, domains, + forensics, health, imap, mail_sources, @@ -20,6 +21,7 @@ api_router.include_router(auth.router, prefix="/auth", tags=["auth"]) api_router.include_router(health.router, tags=["health"]) api_router.include_router(domains.router, prefix="/domains", tags=["domains"]) api_router.include_router(reports.router, prefix="/reports", tags=["reports"]) +api_router.include_router(forensics.router, prefix="/forensics", tags=["forensics"]) api_router.include_router(setup.router, prefix="/setup", tags=["setup"]) api_router.include_router(imap.router, prefix="/imap", tags=["imap"]) api_router.include_router(stats.router, prefix="/stats", tags=["stats"]) diff --git a/backend/app/api/api_v1/endpoints/forensics.py b/backend/app/api/api_v1/endpoints/forensics.py new file mode 100644 index 0000000..3c92e49 --- /dev/null +++ b/backend/app/api/api_v1/endpoints/forensics.py @@ -0,0 +1,172 @@ +import logging +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session, selectinload + +from app.core.database import get_db +from app.core.security import require_admin_auth +from app.models.domain import Domain +from app.models.report import ForensicReport +from app.services.forensic_parser import ForensicParser, MAX_FORENSIC_REPORT_SIZE +from app.services.forensic_persistence import ( + forensic_report_exists, + forensic_report_to_dict, + save_forensic_report, +) + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +class ForensicReportResponse(BaseModel): + id: int + report_id: str + domain: Optional[str] = None + reported_domain: Optional[str] = None + source_email: Optional[str] = None + feedback_type: Optional[str] = None + user_agent: Optional[str] = None + version: Optional[str] = None + source_ip: Optional[str] = None + auth_failure: Optional[str] = None + delivery_result: Optional[str] = None + arrival_date: Optional[str] = None + authentication_results: Optional[str] = None + original_mail_from: Optional[str] = None + original_from: Optional[str] = None + original_to: Optional[str] = None + original_subject: Optional[str] = None + original_message_id: Optional[str] = None + original_date: Optional[str] = None + feedback_headers: Dict[str, Any] = Field(default_factory=dict) + processed_at: Optional[str] = None + + +class ForensicListResponse(BaseModel): + total: int + page: int + page_size: int + total_pages: int + reports: List[ForensicReportResponse] + + +class ForensicUploadResponse(BaseModel): + success: bool + report_id: str + domain: Optional[str] = None + message: str + duplicate: bool = False + + +def _validate_upload(file: UploadFile, content: bytes) -> None: + filename = file.filename or "" + if not filename: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Filename is required") + if len(content) == 0: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="File is empty") + if len(content) > MAX_FORENSIC_REPORT_SIZE: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File too large" + ) + if not filename.lower().endswith((".eml", ".txt")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid file type. Upload a forensic report email as .eml or .txt.", + ) + + +@router.post("/upload", response_model=ForensicUploadResponse) +async def upload_forensic_report( + file: UploadFile = File(...), + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Upload and store a DMARC forensic/failure report email.""" + try: + content = await file.read() + _validate_upload(file, content) + parsed = ForensicParser.parse_bytes(content) + if forensic_report_exists(db, parsed["report_id"]): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Forensic report has already been uploaded.", + ) + + row, _created = save_forensic_report(db, parsed) + db.commit() + db.refresh(row) + return ForensicUploadResponse( + success=True, + report_id=row.report_id, + domain=row.reported_domain, + message="Forensic report processed successfully.", + ) + except HTTPException: + raise + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid forensic report format.", + ) from exc + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("Unexpected forensic upload failure for %s: %s", file.filename, exc) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error processing forensic report.", + ) from exc + + +@router.get("", response_model=ForensicListResponse) +async def list_forensic_reports( + domain: Optional[str] = Query(default=None), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=50, ge=1, le=200), + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """List stored forensic reports, newest first.""" + query = db.query(ForensicReport).options(selectinload(ForensicReport.domain)) + if domain: + normalized = domain.lower() + query = query.outerjoin(Domain).filter( + (Domain.name == normalized) | (ForensicReport.reported_domain == normalized) + ) + + total = query.count() + rows = ( + query.order_by(ForensicReport.arrival_date.desc().nullslast(), ForensicReport.id.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + total_pages = (total + page_size - 1) // page_size if total else 0 + return ForensicListResponse( + total=total, + page=page, + page_size=page_size, + total_pages=total_pages, + reports=[ForensicReportResponse(**forensic_report_to_dict(row)) for row in rows], + ) + + +@router.get("/{report_id}", response_model=ForensicReportResponse) +async def get_forensic_report( + report_id: int, + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +): + """Return one stored forensic report by numeric ID.""" + row = ( + db.query(ForensicReport) + .options(selectinload(ForensicReport.domain)) + .filter(ForensicReport.id == report_id) + .first() + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Forensic report not found" + ) + return ForensicReportResponse(**forensic_report_to_dict(row)) diff --git a/backend/app/api/api_v1/endpoints/imap.py b/backend/app/api/api_v1/endpoints/imap.py index 920581f..ff5a31e 100644 --- a/backend/app/api/api_v1/endpoints/imap.py +++ b/backend/app/api/api_v1/endpoints/imap.py @@ -108,6 +108,8 @@ async def fetch_imap_reports( "success": results["success"], "processed_emails": results["processed"], "reports_found": results["reports_found"], + "forensic_reports_found": int(results.get("forensic_reports_found", 0)), + "duplicate_forensic_reports": int(results.get("duplicate_forensic_reports", 0)), "new_domains": results["new_domains"], "errors": results["errors"] if "errors" in results and results["errors"] else None, "timestamp": datetime.now().isoformat(), diff --git a/backend/app/api/api_v1/endpoints/mail_sources.py b/backend/app/api/api_v1/endpoints/mail_sources.py index fa197c1..602700b 100644 --- a/backend/app/api/api_v1/endpoints/mail_sources.py +++ b/backend/app/api/api_v1/endpoints/mail_sources.py @@ -392,6 +392,8 @@ def _fetch_response(source: MailSource, results: Dict[str, Any]) -> Dict[str, An "processed": int(results.get("processed", 0)), "reports_found": int(results.get("reports_found", 0)), "duplicate_reports": int(results.get("duplicate_reports", 0)), + "forensic_reports_found": int(results.get("forensic_reports_found", 0)), + "duplicate_forensic_reports": int(results.get("duplicate_forensic_reports", 0)), "new_domains": [str(d) for d in results.get("new_domains", [])], "error_count": len(results.get("errors", [])), "timestamp": datetime.now().isoformat(), @@ -555,10 +557,12 @@ async def fetch_mail_source( source = _get_source_or_404(source_id, db) results = _fetch_source(source, db, days) logger.info( - "Manual fetch for source id=%d: processed=%d reports_found=%d duplicates=%d", + "Manual fetch for source id=%d: processed=%d reports_found=%d " + "forensic_reports_found=%d duplicates=%d", int(source_id), int(results.get("processed", 0)), int(results.get("reports_found", 0)), + int(results.get("forensic_reports_found", 0)), int(results.get("duplicate_reports", 0)), ) for err in results.get("errors", []): @@ -980,10 +984,11 @@ async def gmail_fetch_reports( db.commit() logger.info( - "Gmail fetch for source id=%d: processed=%d reports_found=%d", + "Gmail fetch for source id=%d: processed=%d reports_found=%d forensic_reports_found=%d", int(source_id), int(results.get("processed", 0)), int(results.get("reports_found", 0)), + int(results.get("forensic_reports_found", 0)), ) for err in results.get("errors", []): @@ -997,6 +1002,8 @@ async def gmail_fetch_reports( "success": bool(results.get("success", False)), "processed": int(results.get("processed", 0)), "reports_found": int(results.get("reports_found", 0)), + "forensic_reports_found": int(results.get("forensic_reports_found", 0)), + "duplicate_forensic_reports": int(results.get("duplicate_forensic_reports", 0)), "new_domains": [str(d) for d in results.get("new_domains", [])], "error_count": len(results.get("errors", [])), "timestamp": datetime.now().isoformat(), diff --git a/backend/app/main.py b/backend/app/main.py index cec4dcd..abe8326 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -80,10 +80,12 @@ def _poll_single_imap_source(source: MailSource) -> None: if results["success"]: logger.info( - "IMAP polling (source id=%d): %s emails processed, %s reports found", + "IMAP polling (source id=%d): %s emails processed, %s aggregate reports found, " + "%s forensic reports found", source.id, results["processed"], results["reports_found"], + results.get("forensic_reports_found", 0), ) if results["new_domains"]: logger.info("New domains found: %s", ", ".join(results["new_domains"])) @@ -143,10 +145,12 @@ def _poll_single_gmail_source(source: MailSource) -> None: if results["success"]: logger.info( - "Gmail polling (source id=%d): %s emails processed, %s reports found", + "Gmail polling (source id=%d): %s emails processed, %s aggregate reports found, " + "%s forensic reports found", source.id, results["processed"], results["reports_found"], + results.get("forensic_reports_found", 0), ) if results["new_domains"]: logger.info("New domains found: %s", ", ".join(results["new_domains"])) @@ -616,6 +620,8 @@ def _trigger_poll_imap_source(source: MailSource, db, days: int = 7) -> dict: "success": results["success"], "processed": results.get("processed", 0), "reports_found": results.get("reports_found", 0), + "forensic_reports_found": results.get("forensic_reports_found", 0), + "duplicate_forensic_reports": results.get("duplicate_forensic_reports", 0), "new_domains": results.get("new_domains", []), } @@ -654,6 +660,8 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict: "success": results["success"], "processed": results.get("processed", 0), "reports_found": results.get("reports_found", 0), + "forensic_reports_found": results.get("forensic_reports_found", 0), + "duplicate_forensic_reports": results.get("duplicate_forensic_reports", 0), "new_domains": results.get("new_domains", []), } diff --git a/backend/app/models/domain.py b/backend/app/models/domain.py index 448d373..a2b2736 100644 --- a/backend/app/models/domain.py +++ b/backend/app/models/domain.py @@ -31,6 +31,9 @@ class Domain(Base): # Relationships reports = relationship("DMARCReport", back_populates="domain", cascade="all, delete-orphan") + forensic_reports = relationship( + "ForensicReport", back_populates="domain", cascade="all, delete-orphan" + ) user_domains = relationship("UserDomain", back_populates="domain", cascade="all, delete-orphan") # Indexes for common queries diff --git a/backend/app/models/report.py b/backend/app/models/report.py index e8df9ac..addab6f 100644 --- a/backend/app/models/report.py +++ b/backend/app/models/report.py @@ -1,6 +1,6 @@ from datetime import datetime -from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, String, Text +from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import relationship from app.core.database import Base @@ -90,3 +90,50 @@ class ReportRecord(Base): def __repr__(self): return f"" + + +class ForensicReport(Base): + """DMARC forensic/failure report model (RFC 6591 / ARF).""" + + __tablename__ = "forensic_reports" + + id = Column(Integer, primary_key=True, index=True) + domain_id = Column(Integer, ForeignKey("domains.id"), nullable=True, index=True) + + # Report metadata + report_id = Column(String, nullable=False, index=True) + source_email = Column(String, nullable=True) + feedback_type = Column(String, nullable=True, index=True) + user_agent = Column(String, nullable=True) + version = Column(String, nullable=True) + + # DMARC failure fields + reported_domain = Column(String, nullable=True, index=True) + source_ip = Column(String, nullable=True, index=True) + auth_failure = Column(String, nullable=True, index=True) + delivery_result = Column(String, nullable=True) + arrival_date = Column(DateTime, nullable=True, index=True) + authentication_results = Column(Text, nullable=True) + + # Redacted original-message metadata. Never store original body content here. + original_mail_from = Column(String, nullable=True) + original_from = Column(String, nullable=True) + original_to = Column(String, nullable=True) + original_subject = Column(String, nullable=True) + original_message_id = Column(String, nullable=True) + original_date = Column(String, nullable=True) + + # Sanitized parser details for operators/debugging. + feedback_headers = Column(Text, nullable=True) + processed_at = Column(DateTime, default=datetime.utcnow, index=True) + + domain = relationship("Domain", back_populates="forensic_reports") + + __table_args__ = ( + UniqueConstraint("report_id", name="uq_forensic_reports_report_id"), + Index("ix_forensic_reports_domain_arrival", "domain_id", "arrival_date"), + Index("ix_forensic_reports_failure_source", "auth_failure", "source_ip"), + ) + + def __repr__(self): + return f"" diff --git a/backend/app/services/forensic_parser.py b/backend/app/services/forensic_parser.py new file mode 100644 index 0000000..3c801d9 --- /dev/null +++ b/backend/app/services/forensic_parser.py @@ -0,0 +1,197 @@ +import hashlib +import json +import re +from datetime import datetime +from email import message_from_bytes +from email.message import Message +from email.parser import Parser +from email.utils import getaddresses, parsedate_to_datetime +from typing import Any, Dict, Optional + + +MAX_FORENSIC_REPORT_SIZE = 10 * 1024 * 1024 + +_EMAIL_RE = re.compile(r"\b([A-Z0-9._%+-]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b", re.IGNORECASE) +_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b") + + +def _coerce_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + +def _clean(value: Any, *, redact: bool = True) -> str: + text = " ".join(_coerce_text(value).replace("\r", " ").replace("\n", " ").split()) + return redact_text(text) if redact else text + + +def redact_text(value: str) -> str: + """Redact email local-parts and long opaque tokens from forensic metadata.""" + + def _redact_email(match: re.Match[str]) -> str: + local = match.group(1) + domain = match.group(2) + prefix = local[:2] if len(local) > 2 else local[:1] + return f"{prefix}***@{domain.lower()}" + + redacted = _EMAIL_RE.sub(_redact_email, value) + return _LONG_TOKEN_RE.sub("[redacted-token]", redacted) + + +def _header(msg: Optional[Message], name: str, *, redact: bool = True) -> str: + return _clean(msg.get(name, "") if msg is not None else "", redact=redact) + + +def _payload_text(part: Message) -> str: + payload = part.get_payload(decode=True) + if payload is not None: + charset = part.get_content_charset() or "utf-8" + return payload.decode(charset, errors="replace") + payload_value = part.get_payload() + if isinstance(payload_value, list): + return "" + return _coerce_text(payload_value) + + +def _message_part_payload(part: Message) -> Optional[Message]: + payload = part.get_payload() + if isinstance(payload, list) and payload: + return payload[0] + return None + + +def _parse_feedback_headers(text: str) -> Message: + return Parser().parsestr(text or "") + + +def _domain_from_address(value: str) -> str: + addresses = getaddresses([value]) + for _, addr in addresses: + if "@" in addr: + return addr.rsplit("@", 1)[-1].lower() + return "" + + +def _parse_datetime(value: str) -> Optional[datetime]: + if not value: + return None + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError, IndexError): + return None + if parsed is None: + return None + return parsed.replace(tzinfo=None) + + +def _message_id_hash(value: str) -> str: + cleaned = _clean(value, redact=False) + if not cleaned: + return "" + return hashlib.sha256(cleaned.encode("utf-8")).hexdigest()[:24] + + +class ForensicParser: + """Parse DMARC forensic/failure report emails without retaining message bodies.""" + + @staticmethod + def is_forensic_report(msg: Message) -> bool: + if msg.get_content_type() == "multipart/report": + report_type = (msg.get_param("report-type") or "").lower() + if report_type == "feedback-report": + return True + + for part in msg.walk(): + content_type = part.get_content_type().lower() + if content_type == "message/feedback-report": + return True + if content_type == "text/rfc822-headers" and "dmarc" in _payload_text(part).lower(): + return True + + subject = _header(msg, "Subject", redact=False).lower() + return "dmarc" in subject and any( + term in subject for term in ("failure", "forensic", "ruf") + ) + + @classmethod + def parse_bytes( + cls, content: bytes, *, message_id_hint: Optional[str] = None + ) -> Dict[str, Any]: + if len(content) > MAX_FORENSIC_REPORT_SIZE: + raise ValueError("Forensic report is too large") + if not content: + raise ValueError("Forensic report is empty") + + msg = message_from_bytes(content) + if not cls.is_forensic_report(msg): + raise ValueError("Email is not a DMARC forensic report") + + feedback = None + original_headers = None + + for part in msg.walk(): + content_type = part.get_content_type().lower() + if content_type == "message/feedback-report": + feedback = _message_part_payload(part) or _parse_feedback_headers( + _payload_text(part) + ) + elif content_type == "text/rfc822-headers": + original_headers = _parse_feedback_headers(_payload_text(part)) + elif content_type == "message/rfc822" and original_headers is None: + original_headers = _message_part_payload(part) + + feedback = feedback or msg + reported_domain = ( + _header(feedback, "Reported-Domain", redact=False) + or _header(feedback, "DKIM-Domain", redact=False) + or _domain_from_address(_header(feedback, "Original-Mail-From", redact=False)) + or _domain_from_address(_header(original_headers, "From", redact=False)) + ).lower() + source_ip = _header(feedback, "Source-IP", redact=False) + auth_failure = _header(feedback, "Auth-Failure", redact=False) + original_message_id = _header(original_headers, "Message-ID", redact=False) + top_message_id = _header(msg, "Message-ID", redact=False) + + report_id = ( + _clean(message_id_hint, redact=False) + or _message_id_hash(top_message_id) + or _message_id_hash(original_message_id) + or hashlib.sha256(content).hexdigest()[:24] + ) + if not report_id.startswith("ruf-"): + report_id = f"ruf-{report_id}" + + source_email = _header(msg, "From") + arrival_date = _parse_datetime(_header(feedback, "Arrival-Date", redact=False)) + + details = { + "identity_alignment": _header(feedback, "Identity-Alignment", redact=False), + "dkim_domain": _header(feedback, "DKIM-Domain", redact=False), + "spf_dns": _header(feedback, "SPF-DNS", redact=False), + "reported_uri": _header(feedback, "Reported-URI"), + } + details = {key: value for key, value in details.items() if value} + + return { + "report_id": report_id, + "source_email": source_email, + "feedback_type": _header(feedback, "Feedback-Type", redact=False) or "auth-failure", + "user_agent": _header(feedback, "User-Agent"), + "version": _header(feedback, "Version", redact=False), + "reported_domain": reported_domain, + "source_ip": source_ip, + "auth_failure": auth_failure, + "delivery_result": _header(feedback, "Delivery-Result", redact=False), + "arrival_date": arrival_date, + "authentication_results": _header(feedback, "Authentication-Results"), + "original_mail_from": _header(feedback, "Original-Mail-From"), + "original_from": _header(original_headers, "From"), + "original_to": _header(original_headers, "To"), + "original_subject": _header(original_headers, "Subject"), + "original_message_id": _message_id_hash(original_message_id), + "original_date": _header(original_headers, "Date", redact=False), + "feedback_headers": json.dumps(details, sort_keys=True) if details else None, + } diff --git a/backend/app/services/forensic_persistence.py b/backend/app/services/forensic_persistence.py new file mode 100644 index 0000000..bb80f95 --- /dev/null +++ b/backend/app/services/forensic_persistence.py @@ -0,0 +1,103 @@ +import json +from typing import Any, Dict, Optional + +from sqlalchemy.orm import Session + +from app.models.domain import Domain +from app.models.report import ForensicReport +from app.utils.domain_validator import DomainValidationError, validate_domain + + +def forensic_report_exists(db: Session, report_id: str) -> bool: + """Return True when a forensic report ID is already persisted.""" + if not report_id: + return False + return ( + db.query(ForensicReport.id).filter(ForensicReport.report_id == report_id).first() + is not None + ) + + +def _domain_for_report(db: Session, domain_name: Optional[str]) -> Optional[Domain]: + if not domain_name: + return None + normalized = domain_name.lower().strip(".") + is_valid, _, error_code = validate_domain(normalized, check_dns=False) + if not is_valid and error_code != DomainValidationError.DNS_RESOLUTION_FAILED: + return None + + domain = db.query(Domain).filter(Domain.name == normalized).first() + if domain is None: + domain = Domain(name=normalized) + db.add(domain) + db.flush() + return domain + + +def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicReport, bool]: + """Persist a parsed forensic report. + + Returns ``(row, created)``. The caller owns the transaction and should + commit after related work has completed. + """ + report_id = str(report.get("report_id") or "") + existing = db.query(ForensicReport).filter(ForensicReport.report_id == report_id).first() + if existing is not None: + return existing, False + + domain = _domain_for_report(db, report.get("reported_domain")) + feedback_headers = report.get("feedback_headers") + if isinstance(feedback_headers, dict): + feedback_headers = json.dumps(feedback_headers, sort_keys=True) + + row = ForensicReport( + domain_id=domain.id if domain else None, + report_id=report_id, + source_email=report.get("source_email"), + feedback_type=report.get("feedback_type"), + user_agent=report.get("user_agent"), + version=report.get("version"), + reported_domain=report.get("reported_domain"), + source_ip=report.get("source_ip"), + auth_failure=report.get("auth_failure"), + delivery_result=report.get("delivery_result"), + arrival_date=report.get("arrival_date"), + authentication_results=report.get("authentication_results"), + original_mail_from=report.get("original_mail_from"), + original_from=report.get("original_from"), + original_to=report.get("original_to"), + original_subject=report.get("original_subject"), + original_message_id=report.get("original_message_id"), + original_date=report.get("original_date"), + feedback_headers=feedback_headers, + ) + db.add(row) + db.flush() + return row, True + + +def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]: + """Convert a forensic report row to an API-safe dictionary.""" + return { + "id": row.id, + "report_id": row.report_id, + "domain": row.domain.name if row.domain else row.reported_domain, + "reported_domain": row.reported_domain, + "source_email": row.source_email, + "feedback_type": row.feedback_type, + "user_agent": row.user_agent, + "version": row.version, + "source_ip": row.source_ip, + "auth_failure": row.auth_failure, + "delivery_result": row.delivery_result, + "arrival_date": row.arrival_date.isoformat() if row.arrival_date else None, + "authentication_results": row.authentication_results, + "original_mail_from": row.original_mail_from, + "original_from": row.original_from, + "original_to": row.original_to, + "original_subject": row.original_subject, + "original_message_id": row.original_message_id, + "original_date": row.original_date, + "feedback_headers": json.loads(row.feedback_headers) if row.feedback_headers else {}, + "processed_at": row.processed_at.isoformat() if row.processed_at else None, + } diff --git a/backend/app/services/gmail_client.py b/backend/app/services/gmail_client.py index 7dd9396..404b62b 100644 --- a/backend/app/services/gmail_client.py +++ b/backend/app/services/gmail_client.py @@ -21,6 +21,8 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError from app.services.dmarc_parser import DMARCParser +from app.services.forensic_parser import ForensicParser +from app.services.forensic_persistence import forensic_report_exists, save_forensic_report from app.services.report_persistence import report_exists, save_parsed_report from app.services.report_store import ReportStore @@ -47,8 +49,8 @@ GMAIL_SCOPES = [ # --------------------------------------------------------------------------- DMARC_GMAIL_QUERY = ( - "has:attachment " - "(filename:zip OR filename:gz OR filename:xml) " + "((has:attachment (filename:zip OR filename:gz OR filename:xml)) " + 'OR subject:"DMARC failure" OR subject:"failure report" OR subject:forensic OR subject:ruf) ' "(subject:dmarc OR subject:report OR subject:rua OR subject:submitter " 'OR subject:"aggregate report" OR subject:"domain report" ' 'OR subject:"report domain" OR from:dmarc OR from:dmarc-noreply ' @@ -210,7 +212,9 @@ class GmailClient: "success": True, "processed": 0, "reports_found": 0, + "forensic_reports_found": 0, "duplicate_reports": 0, + "duplicate_forensic_reports": 0, "new_domains": [], "errors": [], "new_ingested_ids": [], @@ -302,7 +306,9 @@ class GmailClient: @staticmethod def _append_detail(stats: dict, **detail: str) -> None: """Append a compact attachment/message outcome to the import stats.""" - stats.setdefault("details", []).append({key: value for key, value in detail.items() if value}) + stats.setdefault("details", []).append( + {key: value for key, value in detail.items() if value} + ) def _process_message(self, service, msg_id: str, stats: dict) -> int: """ @@ -327,6 +333,8 @@ class GmailClient: raw_bytes = base64.urlsafe_b64decode(msg_data.get("raw", "")) msg = email.message_from_bytes(raw_bytes) + if ForensicParser.is_forensic_report(msg): + return 1 if self._process_forensic_message(raw_bytes, stats, message_id=msg_id) else 0 return self._process_attachments(msg, stats, message_id=msg_id) @staticmethod @@ -370,6 +378,76 @@ class GmailClient: self.report_store.add_report(report) return True + def _process_forensic_message( + self, + raw_bytes: bytes, + stats: dict, + message_id: Optional[str] = None, + ) -> bool: + """Parse and persist one DMARC forensic report message.""" + try: + report = ForensicParser.parse_bytes(raw_bytes, message_id_hint=message_id) + report_id = str(report.get("report_id", "")) + domain = str(report.get("reported_domain") or "unknown") + + if self.db is None: + self._append_detail( + stats, + status="skipped", + reason="forensic_report_requires_database", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + + if forensic_report_exists(self.db, report_id): + stats["duplicate_forensic_reports"] = stats.get("duplicate_forensic_reports", 0) + 1 + self._append_detail( + stats, + status="duplicate", + reason="duplicate_forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + + _row, created = save_forensic_report(self.db, report) + if not created: + stats["duplicate_forensic_reports"] = stats.get("duplicate_forensic_reports", 0) + 1 + self._append_detail( + stats, + status="duplicate", + reason="duplicate_forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + + stats["forensic_reports_found"] = stats.get("forensic_reports_found", 0) + 1 + self._append_detail( + stats, + status="imported", + reason="forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return True + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("Failed to parse Gmail forensic report %s: %s", message_id, exc) + stats["errors"].append(f"Failed to parse forensic report {message_id}: {exc}") + self._append_detail( + stats, + status="error", + reason="forensic_parse_failed", + message_id=message_id, + error=str(exc), + ) + return False + def _process_attachments( self, msg: email.message.Message, diff --git a/backend/app/services/imap_client.py b/backend/app/services/imap_client.py index bb58a3b..1ea6966 100644 --- a/backend/app/services/imap_client.py +++ b/backend/app/services/imap_client.py @@ -7,6 +7,8 @@ from typing import Any, Dict, Optional, Tuple from app.core.config import get_settings from app.services.dmarc_parser import DMARCParser +from app.services.forensic_parser import ForensicParser +from app.services.forensic_persistence import forensic_report_exists, save_forensic_report from app.services.report_persistence import report_exists, save_parsed_report from app.services.report_store import ReportStore @@ -202,6 +204,19 @@ class IMAPClient: raw_email = msg_data[0][1] msg = email.message_from_bytes(raw_email) + if ForensicParser.is_forensic_report(msg): + imported = self._process_forensic_email( + raw_email, + stats=stats, + message_id=message_id, + ) + mail.store(email_id, "+FLAGS", "\\Seen") + if self.delete_emails and imported: + mail.store(email_id, "+FLAGS", "\\Deleted") + stats["deleted"] = stats.get("deleted", 0) + 1 + stats["processed"] += 1 + return + if self._is_dmarc_report_email(msg): reports_found = self._process_attachments(msg, stats, message_id=message_id) stats["reports_found"] += reports_found @@ -243,8 +258,10 @@ class IMAPClient: "success": True, "processed": 0, "reports_found": 0, + "forensic_reports_found": 0, "deleted": 0, "duplicate_reports": 0, + "duplicate_forensic_reports": 0, "new_domains": [], "errors": [], "details": [], @@ -472,6 +489,84 @@ class IMAPClient: ) return True + def _process_forensic_email( + self, + raw_email: bytes, + *, + stats: Optional[Dict[str, Any]], + message_id: Optional[str], + ) -> bool: + try: + report = ForensicParser.parse_bytes(raw_email) + report_id = str(report.get("report_id", "")) + domain = str(report.get("reported_domain") or "unknown") + + if self.db is not None and forensic_report_exists(self.db, report_id): + if stats is not None: + stats["duplicate_forensic_reports"] = ( + stats.get("duplicate_forensic_reports", 0) + 1 + ) + self._append_detail( + stats, + status="duplicate", + reason="duplicate_forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + + if self.db is None: + self._append_detail( + stats, + status="skipped", + reason="forensic_report_requires_database", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + + _row, created = save_forensic_report(self.db, report) + if created: + if stats is not None: + stats["forensic_reports_found"] = stats.get("forensic_reports_found", 0) + 1 + self._append_detail( + stats, + status="imported", + reason="forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return True + + if stats is not None: + stats["duplicate_forensic_reports"] = stats.get("duplicate_forensic_reports", 0) + 1 + self._append_detail( + stats, + status="duplicate", + reason="duplicate_forensic_report", + message_id=message_id, + domain=domain, + report_id=report_id, + ) + return False + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("Error processing forensic report email %s: %s", message_id, exc) + if stats is not None: + stats.setdefault("errors", []).append( + f"Failed to parse forensic report {message_id}: {exc}" + ) + self._append_detail( + stats, + status="error", + reason="forensic_parse_failed", + message_id=message_id, + error=str(exc), + ) + return False + def _process_dmarc_attachment( self, part: email.message.Message, diff --git a/backend/app/tests/test_forensic_parser.py b/backend/app/tests/test_forensic_parser.py new file mode 100644 index 0000000..041317c --- /dev/null +++ b/backend/app/tests/test_forensic_parser.py @@ -0,0 +1,77 @@ +import email + +from app.services.forensic_parser import ForensicParser + + +SAMPLE_FORENSIC_EMAIL = b"""\ +From: DMARC Reporter +To: postmaster@example.com +Subject: DMARC Failure Report for example.com +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; boundary="ruf-boundary" + +--ruf-boundary +Content-Type: text/plain; charset=utf-8 + +This is a DMARC failure report. + +--ruf-boundary +Content-Type: message/feedback-report + +Feedback-Type: auth-failure +User-Agent: Example Reporter +Version: 1 +Original-Mail-From: alice@example.com +Arrival-Date: Fri, 22 May 2026 10:15:00 +0000 +Source-IP: 203.0.113.8 +Reported-Domain: example.com +Authentication-Results: mx.example.net; dkim=fail header.d=example.com; spf=pass +Auth-Failure: dkim +Delivery-Result: reject + +--ruf-boundary +Content-Type: text/rfc822-headers + +From: Alice Sender +To: Bob Receiver +Subject: Customer renewal token abcdefghijklmnopqrstuvwxyz123456 +Message-ID: +Date: Fri, 22 May 2026 10:14:55 +0000 + +--ruf-boundary-- +""" + + +def test_detects_forensic_report_email(): + msg = email.message_from_bytes(SAMPLE_FORENSIC_EMAIL) + + assert ForensicParser.is_forensic_report(msg) is True + + +def test_parse_forensic_email_redacts_and_extracts_failure_fields(): + parsed = ForensicParser.parse_bytes(SAMPLE_FORENSIC_EMAIL) + + assert parsed["report_id"].startswith("ruf-") + assert parsed["reported_domain"] == "example.com" + assert parsed["source_ip"] == "203.0.113.8" + assert parsed["auth_failure"] == "dkim" + assert parsed["delivery_result"] == "reject" + assert parsed["arrival_date"].year == 2026 + assert parsed["original_mail_from"] == "al***@example.com" + assert "al***@example.com" in parsed["original_from"] + assert "bo***@example.net" in parsed["original_to"] + assert "[redacted-token]" in parsed["original_subject"] + assert parsed["original_message_id"] + assert "original-message@example.com" not in parsed["original_message_id"] + + +def test_non_forensic_email_is_rejected(): + content = b"From: sender@example.com\r\nSubject: hello\r\n\r\nplain email" + + try: + ForensicParser.parse_bytes(content) + except ValueError as exc: + assert "forensic" in str(exc) + else: + raise AssertionError("Expected parser to reject non-forensic email") diff --git a/backend/app/tests/test_forensics_api.py b/backend/app/tests/test_forensics_api.py new file mode 100644 index 0000000..6e3058b --- /dev/null +++ b/backend/app/tests/test_forensics_api.py @@ -0,0 +1,56 @@ +from app.models.report import ForensicReport +from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL + + +def test_upload_forensic_report_persists_redacted_metadata(authed_client, db_session): + response = authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["domain"] == "example.com" + assert db_session.query(ForensicReport).count() == 1 + + +def test_upload_forensic_report_rejects_duplicates(authed_client): + files = {"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")} + assert authed_client.post("/api/v1/forensics/upload", files=files).status_code == 200 + + response = authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")}, + ) + + assert response.status_code == 409 + + +def test_list_and_detail_forensic_reports(authed_client): + authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")}, + ) + + list_response = authed_client.get("/api/v1/forensics?domain=example.com") + assert list_response.status_code == 200 + list_data = list_response.json() + assert list_data["total"] == 1 + item = list_data["reports"][0] + assert item["source_ip"] == "203.0.113.8" + assert item["auth_failure"] == "dkim" + assert item["original_message_id"] != "" + + detail_response = authed_client.get(f"/api/v1/forensics/{item['id']}") + assert detail_response.status_code == 200 + assert detail_response.json()["reported_domain"] == "example.com" + + +def test_upload_forensic_report_rejects_aggregate_xml(authed_client): + response = authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.xml", b"", "application/xml")}, + ) + + assert response.status_code == 400 diff --git a/backend/app/tests/test_gmail_client.py b/backend/app/tests/test_gmail_client.py index f9e6dfd..14e0eed 100644 --- a/backend/app/tests/test_gmail_client.py +++ b/backend/app/tests/test_gmail_client.py @@ -19,10 +19,11 @@ from unittest.mock import MagicMock, patch import pytest -from app.models.report import DMARCReport +from app.models.report import DMARCReport, ForensicReport from app.services.gmail_client import GmailClient from app.services.report_store import ReportStore from app.tests.test_data import SAMPLE_XML +from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL # --------------------------------------------------------------------------- # Helpers @@ -442,6 +443,28 @@ class TestProcessMessage: assert stats["details"][0]["status"] == "error" assert stats["details"][0]["message_id"] == "bad-id" + def test_forensic_report_is_processed_separately(self, db_session): + client = _make_client(db=db_session) + service = MagicMock() + service.users.return_value.messages.return_value.get.return_value.execute.return_value = { + "raw": _b64_raw(SAMPLE_FORENSIC_EMAIL) + } + + stats = { + "reports_found": 0, + "forensic_reports_found": 0, + "duplicate_forensic_reports": 0, + "errors": [], + } + count = client._process_message(service, "msg-forensic", stats) + + assert count == 1 + assert stats["reports_found"] == 0 + assert stats["forensic_reports_found"] == 1 + assert stats["details"][0]["reason"] == "forensic_report" + assert db_session.query(DMARCReport).count() == 0 + assert db_session.query(ForensicReport).count() == 1 + # =========================================================================== # _process_attachments diff --git a/backend/app/tests/test_imap_client.py b/backend/app/tests/test_imap_client.py index dceca58..4ab69ac 100644 --- a/backend/app/tests/test_imap_client.py +++ b/backend/app/tests/test_imap_client.py @@ -17,9 +17,10 @@ from zipfile import ZipFile import pytest -from app.models.report import DMARCReport +from app.models.report import DMARCReport, ForensicReport from app.services.imap_client import IMAPClient from app.services.report_store import ReportStore +from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL # --------------------------------------------------------------------------- # Helpers @@ -587,7 +588,7 @@ class TestProcessAttachments: class TestProcessSingleEmail: - def _make_client(self): + def _make_client(self, db=None): with patch("app.services.imap_client.get_settings") as mock_settings: mock_settings.return_value = MagicMock( IMAP_SERVER="imap.example.com", @@ -595,7 +596,7 @@ class TestProcessSingleEmail: IMAP_USERNAME="u", IMAP_PASSWORD="p", ) - return IMAPClient() + return IMAPClient(db=db) def test_processes_valid_dmarc_email(self): client = self._make_client() @@ -616,6 +617,29 @@ class TestProcessSingleEmail: assert stats["reports_found"] == 1 assert stats["details"][0]["status"] == "imported" + def test_processes_forensic_report_without_aggregate_count(self, db_session): + client = self._make_client(db=db_session) + mock_mail = MagicMock() + mock_mail.fetch.return_value = ("OK", [(b"1", SAMPLE_FORENSIC_EMAIL)]) + mock_mail.store.return_value = ("OK", None) + + stats = { + "processed": 0, + "reports_found": 0, + "forensic_reports_found": 0, + "deleted": 0, + "errors": [], + } + client._process_single_email(mock_mail, b"1", stats) + + assert stats["processed"] == 1 + assert stats["reports_found"] == 0 + assert stats["forensic_reports_found"] == 1 + assert stats["details"][0]["reason"] == "forensic_report" + assert db_session.query(DMARCReport).count() == 0 + assert db_session.query(ForensicReport).count() == 1 + assert ReportStore.get_instance().get_domains() == [] + def test_fetch_error_skips_email(self): client = self._make_client() mock_mail = MagicMock() diff --git a/docs/milestones.md b/docs/milestones.md index 2fa6ad4..4f7e79e 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -163,15 +163,20 @@ Exit criteria: ## Milestone 10: Forensic Report Support -Status: Backlog +Status: In Progress Goal: support DMARC RUF/forensic reports for individual failure investigation. -Planned: +Delivered: - Detect forensic report messages. - Parse safe metadata from ARF/attached email formats. - Store minimal incident details with privacy controls. +- Keep forensic reports out of aggregate report statistics and ReportStore rollups. +- Expose authenticated forensic upload/list/detail APIs. + +Planned: - Add a dedicated forensic report view. +- Add configurable redaction controls and richer failure investigation workflows. Exit criteria: - A security analyst can inspect individual failure reports without mixing them into aggregate statistics.