feat: add forensic redaction controls

backend/app/services/forensic_parser.py

@@ -1,6 +1,5 @@
 import hashlib
 import json
-import re
 from datetime import datetime, timezone
 from email import message_from_bytes
 from email.message import Message
@@ -8,12 +7,15 @@ from email.parser import Parser
 from email.utils import getaddresses, parsedate_to_datetime
 from typing import Any, Dict, Optional
 
+from app.services.forensic_redaction import (
+    ForensicRedactionPolicy,
+    normalize_forensic_redaction_policy,
+    redact_forensic_text,
+)
+
 
 MAX_FORENSIC_REPORT_SIZE = 10 * 1024 * 1024
 
-_EMAIL_RE = re.compile(r"\b([A-Z0-9._%+-]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b", re.IGNORECASE)
-_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b")
-
 
 def _coerce_text(value: Any) -> str:
     if value is None:
@@ -23,26 +25,37 @@ def _coerce_text(value: Any) -> str:
     return str(value)
 
 
-def _clean(value: Any, *, redact: bool = True) -> str:
+def _clean(
+    value: Any,
+    *,
+    redact: bool = True,
+    redaction_policy: Optional[ForensicRedactionPolicy] = None,
+) -> str:
     text = " ".join(_coerce_text(value).replace("\r", " ").replace("\n", " ").split())
-    return redact_text(text) if redact else text
+    return redact_text(text, redaction_policy=redaction_policy) if redact else text
 
 
-def redact_text(value: str) -> str:
+def redact_text(
+    value: str,
+    *,
+    redaction_policy: Optional[ForensicRedactionPolicy] = None,
+) -> str:
     """Redact email local-parts and long opaque tokens from forensic metadata."""
-
-    def _redact_email(match: re.Match[str]) -> str:
-        local = match.group(1)
-        domain = match.group(2)
-        prefix = local[:2] if len(local) > 2 else local[:1]
-        return f"{prefix}***@{domain.lower()}"
-
-    redacted = _EMAIL_RE.sub(_redact_email, value)
-    return _LONG_TOKEN_RE.sub("[redacted-token]", redacted)
+    return redact_forensic_text(value, redaction_policy)
 
 
-def _header(msg: Optional[Message], name: str, *, redact: bool = True) -> str:
-    return _clean(msg.get(name, "") if msg is not None else "", redact=redact)
+def _header(
+    msg: Optional[Message],
+    name: str,
+    *,
+    redact: bool = True,
+    redaction_policy: Optional[ForensicRedactionPolicy] = None,
+) -> str:
+    return _clean(
+        msg.get(name, "") if msg is not None else "",
+        redact=redact,
+        redaction_policy=redaction_policy,
+    )
 
 
 def _payload_text(part: Message) -> str:
@@ -120,7 +133,11 @@ class ForensicParser:
 
     @classmethod
     def parse_bytes(
-        cls, content: bytes, *, message_id_hint: Optional[str] = None
+        cls,
+        content: bytes,
+        *,
+        message_id_hint: Optional[str] = None,
+        redaction_policy: Optional[ForensicRedactionPolicy] = None,
     ) -> Dict[str, Any]:
         if len(content) > MAX_FORENSIC_REPORT_SIZE:
             raise ValueError("Forensic report is too large")
@@ -131,6 +148,7 @@ class ForensicParser:
         if not cls.is_forensic_report(msg):
             raise ValueError("Email is not a DMARC forensic report")
 
+        redaction_policy = normalize_forensic_redaction_policy(redaction_policy)
         feedback = None
         original_headers = None
 
@@ -166,14 +184,18 @@ class ForensicParser:
         if not report_id.startswith("ruf-"):
             report_id = f"ruf-{report_id}"
 
-        source_email = _header(msg, "From")
+        source_email = _header(msg, "From", redaction_policy=redaction_policy)
         arrival_date = _parse_datetime(_header(feedback, "Arrival-Date", redact=False))
 
         details = {
             "identity_alignment": _header(feedback, "Identity-Alignment", redact=False),
             "dkim_domain": _header(feedback, "DKIM-Domain", redact=False),
             "spf_dns": _header(feedback, "SPF-DNS", redact=False),
-            "reported_uri": _header(feedback, "Reported-URI"),
+            "reported_uri": _header(
+                feedback,
+                "Reported-URI",
+                redaction_policy=redaction_policy,
+            ),
         }
         details = {key: value for key, value in details.items() if value}
 
@@ -181,18 +203,34 @@ class ForensicParser:
             "report_id": report_id,
             "source_email": source_email,
             "feedback_type": _header(feedback, "Feedback-Type", redact=False) or "auth-failure",
-            "user_agent": _header(feedback, "User-Agent"),
+            "user_agent": _header(feedback, "User-Agent", redaction_policy=redaction_policy),
             "version": _header(feedback, "Version", redact=False),
             "reported_domain": reported_domain,
             "source_ip": source_ip,
             "auth_failure": auth_failure,
             "delivery_result": _header(feedback, "Delivery-Result", redact=False),
             "arrival_date": arrival_date,
-            "authentication_results": _header(feedback, "Authentication-Results"),
-            "original_mail_from": _header(feedback, "Original-Mail-From"),
-            "original_from": _header(original_headers, "From"),
-            "original_to": _header(original_headers, "To"),
-            "original_subject": _header(original_headers, "Subject"),
+            "authentication_results": _header(
+                feedback,
+                "Authentication-Results",
+                redaction_policy=redaction_policy,
+            ),
+            "original_mail_from": _header(
+                feedback,
+                "Original-Mail-From",
+                redaction_policy=redaction_policy,
+            ),
+            "original_from": _header(
+                original_headers,
+                "From",
+                redaction_policy=redaction_policy,
+            ),
+            "original_to": _header(original_headers, "To", redaction_policy=redaction_policy),
+            "original_subject": _header(
+                original_headers,
+                "Subject",
+                redaction_policy=redaction_policy,
+            ),
             "original_message_id": _message_id_hash(original_message_id),
             "original_date": _header(original_headers, "Date", redact=False),
             "feedback_headers": json.dumps(details, sort_keys=True) if details else None,

backend/app/services/forensic_persistence.py

@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
 
 from app.models.domain import Domain
 from app.models.report import ForensicReport
+from app.services.forensic_redaction import ForensicRedactionPolicy, redact_forensic_value
 from app.utils.domain_validator import DomainValidationError, validate_domain
 
 
@@ -87,7 +88,23 @@ def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicR
     return row, True
 
 
-def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
+_REDACTABLE_RESPONSE_FIELDS = {
+    "source_email",
+    "user_agent",
+    "authentication_results",
+    "original_mail_from",
+    "original_from",
+    "original_to",
+    "original_subject",
+    "feedback_headers",
+}
+
+
+def forensic_report_to_dict(
+    row: ForensicReport,
+    *,
+    redaction_policy: Optional[ForensicRedactionPolicy] = None,
+) -> Dict[str, Any]:
     """Convert a forensic report row to an API-safe dictionary."""
     feedback_headers = {}
     if row.feedback_headers:
@@ -96,7 +113,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
         except (json.JSONDecodeError, TypeError):
             feedback_headers = {}
 
-    return {
+    result = {
         "id": row.id,
         "report_id": row.report_id,
         "domain": row.domain.name if row.domain else row.reported_domain,
@@ -119,3 +136,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
         "feedback_headers": feedback_headers,
         "processed_at": row.processed_at.isoformat() if row.processed_at else None,
     }
+    if redaction_policy is not None:
+        for field in _REDACTABLE_RESPONSE_FIELDS:
+            result[field] = redact_forensic_value(result.get(field), redaction_policy)
+    return result

backend/app/services/forensic_redaction.py

@@ -0,0 +1,106 @@
+import re
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from sqlalchemy.orm import Session
+
+from app.models.setting import Setting
+
+
+FORENSIC_REDACTION_MODE_KEY = "forensics.redaction_mode"
+FORENSIC_REDACT_LONG_TOKENS_KEY = "forensics.redact_long_tokens_enabled"
+DEFAULT_FORENSIC_REDACTION_MODE = "balanced"
+FORENSIC_REDACTION_MODES = {"balanced", "domain_only", "strict"}
+
+_EMAIL_RE = re.compile(
+    r"\b([A-Z0-9._%+\-*]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b",
+    re.IGNORECASE,
+)
+_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b")
+
+
+@dataclass(frozen=True)
+class ForensicRedactionPolicy:
+    """Privacy policy for forensic report metadata."""
+
+    mode: str = DEFAULT_FORENSIC_REDACTION_MODE
+    redact_long_tokens: bool = True
+
+
+def _truthy(value: Optional[str], *, default: bool = True) -> bool:
+    if value is None:
+        return default
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def normalize_forensic_redaction_policy(
+    policy: Optional[ForensicRedactionPolicy] = None,
+    *,
+    mode: Optional[str] = None,
+    redact_long_tokens: Optional[bool] = None,
+) -> ForensicRedactionPolicy:
+    requested_mode = (mode if mode is not None else policy.mode if policy else "").strip().lower()
+    if requested_mode not in FORENSIC_REDACTION_MODES:
+        requested_mode = DEFAULT_FORENSIC_REDACTION_MODE
+    return ForensicRedactionPolicy(
+        mode=requested_mode,
+        redact_long_tokens=(
+            policy.redact_long_tokens
+            if redact_long_tokens is None and policy is not None
+            else bool(True if redact_long_tokens is None else redact_long_tokens)
+        ),
+    )
+
+
+def get_forensic_redaction_policy(db: Optional[Session]) -> ForensicRedactionPolicy:
+    """Load the current forensic redaction policy from persisted settings."""
+    if db is None:
+        return ForensicRedactionPolicy()
+
+    rows = (
+        db.query(Setting.key, Setting.value)
+        .filter(Setting.key.in_([FORENSIC_REDACTION_MODE_KEY, FORENSIC_REDACT_LONG_TOKENS_KEY]))
+        .all()
+    )
+    values = {key: value for key, value in rows}
+    return normalize_forensic_redaction_policy(
+        mode=values.get(FORENSIC_REDACTION_MODE_KEY),
+        redact_long_tokens=_truthy(values.get(FORENSIC_REDACT_LONG_TOKENS_KEY), default=True),
+    )
+
+
+def redact_forensic_text(
+    value: str,
+    policy: Optional[ForensicRedactionPolicy] = None,
+) -> str:
+    """Redact forensic metadata according to the selected privacy policy."""
+    policy = normalize_forensic_redaction_policy(policy)
+
+    def _redact_email(match: re.Match[str]) -> str:
+        local = match.group(1)
+        domain = match.group(2).lower()
+        if policy.mode == "strict":
+            return "[redacted-email]"
+        if policy.mode == "domain_only":
+            return f"***@{domain}"
+        prefix = local[:2] if len(local) > 2 else local[:1]
+        return f"{prefix}***@{domain}"
+
+    redacted = _EMAIL_RE.sub(_redact_email, value)
+    if policy.redact_long_tokens:
+        redacted = _LONG_TOKEN_RE.sub("[redacted-token]", redacted)
+    return redacted
+
+
+def redact_forensic_value(
+    value: Any,
+    policy: Optional[ForensicRedactionPolicy] = None,
+) -> Any:
+    """Redact strings inside a response value while preserving container shape."""
+    if isinstance(value, str):
+        return redact_forensic_text(value, policy)
+    if isinstance(value, dict):
+        return {key: redact_forensic_value(item, policy) for key, item in value.items()}
+    if isinstance(value, list):
+        return [redact_forensic_value(item, policy) for item in value]
+    return value

backend/app/services/gmail_client.py

@@ -23,6 +23,7 @@ from googleapiclient.errors import HttpError
 from app.services.dmarc_parser import DMARCParser
 from app.services.forensic_parser import ForensicParser
 from app.services.forensic_persistence import forensic_report_exists, save_forensic_report
+from app.services.forensic_redaction import get_forensic_redaction_policy
 from app.services.report_persistence import report_exists, save_parsed_report
 from app.services.report_store import ReportStore
 
@@ -385,7 +386,11 @@ class GmailClient:
     ) -> int:
         """Parse and persist one DMARC forensic report message."""
         try:
-            report = ForensicParser.parse_bytes(raw_bytes, message_id_hint=message_id)
+            report = ForensicParser.parse_bytes(
+                raw_bytes,
+                message_id_hint=message_id,
+                redaction_policy=get_forensic_redaction_policy(self.db),
+            )
             report_id = str(report.get("report_id", ""))
             domain = str(report.get("reported_domain") or "unknown")
 

backend/app/services/imap_client.py

@@ -9,6 +9,7 @@ from app.core.config import get_settings
 from app.services.dmarc_parser import DMARCParser
 from app.services.forensic_parser import ForensicParser
 from app.services.forensic_persistence import forensic_report_exists, save_forensic_report
+from app.services.forensic_redaction import get_forensic_redaction_policy
 from app.services.report_persistence import report_exists, save_parsed_report
 from app.services.report_store import ReportStore
 
@@ -497,7 +498,10 @@ class IMAPClient:
         message_id: Optional[str],
     ) -> bool:
         try:
-            report = ForensicParser.parse_bytes(raw_email)
+            report = ForensicParser.parse_bytes(
+                raw_email,
+                redaction_policy=get_forensic_redaction_policy(self.db),
+            )
             report_id = str(report.get("report_id", ""))
             domain = str(report.get("reported_domain") or "unknown")