diff --git a/TODO.md b/TODO.md index a7ba915..b1e090d 100644 --- a/TODO.md +++ b/TODO.md @@ -65,10 +65,11 @@ have no working implementation in the codebase yet. - **Documented in**: README.md ("Forensic Reports: Analyze failure samples (RFC 6591 support)") - **Current state**: Aggregate and forensic reports are now parsed separately. Forensic reports are stored in dedicated database rows and surfaced through authenticated APIs - without affecting aggregate compliance statistics. + without affecting aggregate compliance statistics. Operators can configure forensic + email-address and token redaction under Settings. - [x] Forensic report parsing - [ ] Failure sample analysis -- [ ] PII redaction options +- [x] PII redaction options - [ ] Detailed authentication failure views ### User Authentication & Multi-User Support diff --git a/backend/app/api/api_v1/endpoints/forensics.py b/backend/app/api/api_v1/endpoints/forensics.py index 33fdf2e..38691e8 100644 --- a/backend/app/api/api_v1/endpoints/forensics.py +++ b/backend/app/api/api_v1/endpoints/forensics.py @@ -15,6 +15,7 @@ from app.services.forensic_persistence import ( forensic_report_to_dict, save_forensic_report, ) +from app.services.forensic_redaction import get_forensic_redaction_policy logger = logging.getLogger(__name__) @@ -88,7 +89,8 @@ async def upload_forensic_report( try: content = await file.read() _validate_upload(file, content) - parsed = ForensicParser.parse_bytes(content) + redaction_policy = get_forensic_redaction_policy(db) + parsed = ForensicParser.parse_bytes(content, redaction_policy=redaction_policy) if forensic_report_exists(db, parsed["report_id"]): raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -148,12 +150,18 @@ async def list_forensic_reports( .all() ) total_pages = (total + page_size - 1) // page_size if total else 0 + redaction_policy = get_forensic_redaction_policy(db) return ForensicListResponse( total=total, page=page, page_size=page_size, total_pages=total_pages, - reports=[ForensicReportResponse(**forensic_report_to_dict(row)) for row in rows], + reports=[ + ForensicReportResponse( + **forensic_report_to_dict(row, redaction_policy=redaction_policy) + ) + for row in rows + ], ) @@ -174,4 +182,5 @@ async def get_forensic_report( raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Forensic report not found" ) - return ForensicReportResponse(**forensic_report_to_dict(row)) + redaction_policy = get_forensic_redaction_policy(db) + return ForensicReportResponse(**forensic_report_to_dict(row, redaction_policy=redaction_policy)) diff --git a/backend/app/api/api_v1/endpoints/settings.py b/backend/app/api/api_v1/endpoints/settings.py index 65ff56d..b3d7180 100644 --- a/backend/app/api/api_v1/endpoints/settings.py +++ b/backend/app/api/api_v1/endpoints/settings.py @@ -8,6 +8,7 @@ in the ``settings`` database table. Settings are organised into categories: - ``dmarc`` – Default DMARC policy, percentage, etc. - ``dns`` – Default DNS resolver, Cloudflare DoH toggle. - ``cloudflare`` – Cloudflare API token and Zone ID. +- ``forensics`` – Forensic report privacy and retention controls. - ``notifications`` – Future alerting/notification settings. """ @@ -121,6 +122,21 @@ SETTING_DEFAULTS: List[Dict[str, Any]] = [ "value_type": "string", "category": "cloudflare", }, + # ── Forensics ──────────────────────────────────────────────────────────── + { + "key": "forensics.redaction_mode", + "value": "balanced", + "description": "Forensic report email-address redaction mode: balanced, domain_only, or strict", + "value_type": "string", + "category": "forensics", + }, + { + "key": "forensics.redact_long_tokens_enabled", + "value": "true", + "description": "Redact long opaque tokens in forensic report metadata", + "value_type": "boolean", + "category": "forensics", + }, # ── Notifications ───────────────────────────────────────────────────────── { "key": "notifications.apprise_enabled", @@ -312,7 +328,7 @@ def _audit_value_for_setting(key: str, value: Optional[str]) -> Optional[str]: def _should_audit_setting(key: str) -> bool: - return key.startswith("notifications.") + return key.startswith(("notifications.", "forensics.")) def _audit_setting_change( diff --git a/backend/app/services/forensic_parser.py b/backend/app/services/forensic_parser.py index 68273ea..117d4bb 100644 --- a/backend/app/services/forensic_parser.py +++ b/backend/app/services/forensic_parser.py @@ -1,6 +1,5 @@ import hashlib import json -import re from datetime import datetime, timezone from email import message_from_bytes from email.message import Message @@ -8,12 +7,15 @@ from email.parser import Parser from email.utils import getaddresses, parsedate_to_datetime from typing import Any, Dict, Optional +from app.services.forensic_redaction import ( + ForensicRedactionPolicy, + normalize_forensic_redaction_policy, + redact_forensic_text, +) + MAX_FORENSIC_REPORT_SIZE = 10 * 1024 * 1024 -_EMAIL_RE = re.compile(r"\b([A-Z0-9._%+-]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b", re.IGNORECASE) -_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b") - def _coerce_text(value: Any) -> str: if value is None: @@ -23,26 +25,37 @@ def _coerce_text(value: Any) -> str: return str(value) -def _clean(value: Any, *, redact: bool = True) -> str: +def _clean( + value: Any, + *, + redact: bool = True, + redaction_policy: Optional[ForensicRedactionPolicy] = None, +) -> str: text = " ".join(_coerce_text(value).replace("\r", " ").replace("\n", " ").split()) - return redact_text(text) if redact else text + return redact_text(text, redaction_policy=redaction_policy) if redact else text -def redact_text(value: str) -> str: +def redact_text( + value: str, + *, + redaction_policy: Optional[ForensicRedactionPolicy] = None, +) -> str: """Redact email local-parts and long opaque tokens from forensic metadata.""" - - def _redact_email(match: re.Match[str]) -> str: - local = match.group(1) - domain = match.group(2) - prefix = local[:2] if len(local) > 2 else local[:1] - return f"{prefix}***@{domain.lower()}" - - redacted = _EMAIL_RE.sub(_redact_email, value) - return _LONG_TOKEN_RE.sub("[redacted-token]", redacted) + return redact_forensic_text(value, redaction_policy) -def _header(msg: Optional[Message], name: str, *, redact: bool = True) -> str: - return _clean(msg.get(name, "") if msg is not None else "", redact=redact) +def _header( + msg: Optional[Message], + name: str, + *, + redact: bool = True, + redaction_policy: Optional[ForensicRedactionPolicy] = None, +) -> str: + return _clean( + msg.get(name, "") if msg is not None else "", + redact=redact, + redaction_policy=redaction_policy, + ) def _payload_text(part: Message) -> str: @@ -120,7 +133,11 @@ class ForensicParser: @classmethod def parse_bytes( - cls, content: bytes, *, message_id_hint: Optional[str] = None + cls, + content: bytes, + *, + message_id_hint: Optional[str] = None, + redaction_policy: Optional[ForensicRedactionPolicy] = None, ) -> Dict[str, Any]: if len(content) > MAX_FORENSIC_REPORT_SIZE: raise ValueError("Forensic report is too large") @@ -131,6 +148,7 @@ class ForensicParser: if not cls.is_forensic_report(msg): raise ValueError("Email is not a DMARC forensic report") + redaction_policy = normalize_forensic_redaction_policy(redaction_policy) feedback = None original_headers = None @@ -166,14 +184,18 @@ class ForensicParser: if not report_id.startswith("ruf-"): report_id = f"ruf-{report_id}" - source_email = _header(msg, "From") + source_email = _header(msg, "From", redaction_policy=redaction_policy) arrival_date = _parse_datetime(_header(feedback, "Arrival-Date", redact=False)) details = { "identity_alignment": _header(feedback, "Identity-Alignment", redact=False), "dkim_domain": _header(feedback, "DKIM-Domain", redact=False), "spf_dns": _header(feedback, "SPF-DNS", redact=False), - "reported_uri": _header(feedback, "Reported-URI"), + "reported_uri": _header( + feedback, + "Reported-URI", + redaction_policy=redaction_policy, + ), } details = {key: value for key, value in details.items() if value} @@ -181,18 +203,34 @@ class ForensicParser: "report_id": report_id, "source_email": source_email, "feedback_type": _header(feedback, "Feedback-Type", redact=False) or "auth-failure", - "user_agent": _header(feedback, "User-Agent"), + "user_agent": _header(feedback, "User-Agent", redaction_policy=redaction_policy), "version": _header(feedback, "Version", redact=False), "reported_domain": reported_domain, "source_ip": source_ip, "auth_failure": auth_failure, "delivery_result": _header(feedback, "Delivery-Result", redact=False), "arrival_date": arrival_date, - "authentication_results": _header(feedback, "Authentication-Results"), - "original_mail_from": _header(feedback, "Original-Mail-From"), - "original_from": _header(original_headers, "From"), - "original_to": _header(original_headers, "To"), - "original_subject": _header(original_headers, "Subject"), + "authentication_results": _header( + feedback, + "Authentication-Results", + redaction_policy=redaction_policy, + ), + "original_mail_from": _header( + feedback, + "Original-Mail-From", + redaction_policy=redaction_policy, + ), + "original_from": _header( + original_headers, + "From", + redaction_policy=redaction_policy, + ), + "original_to": _header(original_headers, "To", redaction_policy=redaction_policy), + "original_subject": _header( + original_headers, + "Subject", + redaction_policy=redaction_policy, + ), "original_message_id": _message_id_hash(original_message_id), "original_date": _header(original_headers, "Date", redact=False), "feedback_headers": json.dumps(details, sort_keys=True) if details else None, diff --git a/backend/app/services/forensic_persistence.py b/backend/app/services/forensic_persistence.py index d9b2b08..e8fecf6 100644 --- a/backend/app/services/forensic_persistence.py +++ b/backend/app/services/forensic_persistence.py @@ -6,6 +6,7 @@ from sqlalchemy.orm import Session from app.models.domain import Domain from app.models.report import ForensicReport +from app.services.forensic_redaction import ForensicRedactionPolicy, redact_forensic_value from app.utils.domain_validator import DomainValidationError, validate_domain @@ -87,7 +88,23 @@ def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicR return row, True -def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]: +_REDACTABLE_RESPONSE_FIELDS = { + "source_email", + "user_agent", + "authentication_results", + "original_mail_from", + "original_from", + "original_to", + "original_subject", + "feedback_headers", +} + + +def forensic_report_to_dict( + row: ForensicReport, + *, + redaction_policy: Optional[ForensicRedactionPolicy] = None, +) -> Dict[str, Any]: """Convert a forensic report row to an API-safe dictionary.""" feedback_headers = {} if row.feedback_headers: @@ -96,7 +113,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]: except (json.JSONDecodeError, TypeError): feedback_headers = {} - return { + result = { "id": row.id, "report_id": row.report_id, "domain": row.domain.name if row.domain else row.reported_domain, @@ -119,3 +136,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]: "feedback_headers": feedback_headers, "processed_at": row.processed_at.isoformat() if row.processed_at else None, } + if redaction_policy is not None: + for field in _REDACTABLE_RESPONSE_FIELDS: + result[field] = redact_forensic_value(result.get(field), redaction_policy) + return result diff --git a/backend/app/services/forensic_redaction.py b/backend/app/services/forensic_redaction.py new file mode 100644 index 0000000..51e981c --- /dev/null +++ b/backend/app/services/forensic_redaction.py @@ -0,0 +1,106 @@ +import re +from dataclasses import dataclass +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from app.models.setting import Setting + + +FORENSIC_REDACTION_MODE_KEY = "forensics.redaction_mode" +FORENSIC_REDACT_LONG_TOKENS_KEY = "forensics.redact_long_tokens_enabled" +DEFAULT_FORENSIC_REDACTION_MODE = "balanced" +FORENSIC_REDACTION_MODES = {"balanced", "domain_only", "strict"} + +_EMAIL_RE = re.compile( + r"\b([A-Z0-9._%+\-*]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b", + re.IGNORECASE, +) +_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b") + + +@dataclass(frozen=True) +class ForensicRedactionPolicy: + """Privacy policy for forensic report metadata.""" + + mode: str = DEFAULT_FORENSIC_REDACTION_MODE + redact_long_tokens: bool = True + + +def _truthy(value: Optional[str], *, default: bool = True) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def normalize_forensic_redaction_policy( + policy: Optional[ForensicRedactionPolicy] = None, + *, + mode: Optional[str] = None, + redact_long_tokens: Optional[bool] = None, +) -> ForensicRedactionPolicy: + requested_mode = (mode if mode is not None else policy.mode if policy else "").strip().lower() + if requested_mode not in FORENSIC_REDACTION_MODES: + requested_mode = DEFAULT_FORENSIC_REDACTION_MODE + return ForensicRedactionPolicy( + mode=requested_mode, + redact_long_tokens=( + policy.redact_long_tokens + if redact_long_tokens is None and policy is not None + else bool(True if redact_long_tokens is None else redact_long_tokens) + ), + ) + + +def get_forensic_redaction_policy(db: Optional[Session]) -> ForensicRedactionPolicy: + """Load the current forensic redaction policy from persisted settings.""" + if db is None: + return ForensicRedactionPolicy() + + rows = ( + db.query(Setting.key, Setting.value) + .filter(Setting.key.in_([FORENSIC_REDACTION_MODE_KEY, FORENSIC_REDACT_LONG_TOKENS_KEY])) + .all() + ) + values = {key: value for key, value in rows} + return normalize_forensic_redaction_policy( + mode=values.get(FORENSIC_REDACTION_MODE_KEY), + redact_long_tokens=_truthy(values.get(FORENSIC_REDACT_LONG_TOKENS_KEY), default=True), + ) + + +def redact_forensic_text( + value: str, + policy: Optional[ForensicRedactionPolicy] = None, +) -> str: + """Redact forensic metadata according to the selected privacy policy.""" + policy = normalize_forensic_redaction_policy(policy) + + def _redact_email(match: re.Match[str]) -> str: + local = match.group(1) + domain = match.group(2).lower() + if policy.mode == "strict": + return "[redacted-email]" + if policy.mode == "domain_only": + return f"***@{domain}" + prefix = local[:2] if len(local) > 2 else local[:1] + return f"{prefix}***@{domain}" + + redacted = _EMAIL_RE.sub(_redact_email, value) + if policy.redact_long_tokens: + redacted = _LONG_TOKEN_RE.sub("[redacted-token]", redacted) + return redacted + + +def redact_forensic_value( + value: Any, + policy: Optional[ForensicRedactionPolicy] = None, +) -> Any: + """Redact strings inside a response value while preserving container shape.""" + if isinstance(value, str): + return redact_forensic_text(value, policy) + if isinstance(value, dict): + return {key: redact_forensic_value(item, policy) for key, item in value.items()} + if isinstance(value, list): + return [redact_forensic_value(item, policy) for item in value] + return value diff --git a/backend/app/services/gmail_client.py b/backend/app/services/gmail_client.py index b219d4c..5738523 100644 --- a/backend/app/services/gmail_client.py +++ b/backend/app/services/gmail_client.py @@ -23,6 +23,7 @@ from googleapiclient.errors import HttpError from app.services.dmarc_parser import DMARCParser from app.services.forensic_parser import ForensicParser from app.services.forensic_persistence import forensic_report_exists, save_forensic_report +from app.services.forensic_redaction import get_forensic_redaction_policy from app.services.report_persistence import report_exists, save_parsed_report from app.services.report_store import ReportStore @@ -385,7 +386,11 @@ class GmailClient: ) -> int: """Parse and persist one DMARC forensic report message.""" try: - report = ForensicParser.parse_bytes(raw_bytes, message_id_hint=message_id) + report = ForensicParser.parse_bytes( + raw_bytes, + message_id_hint=message_id, + redaction_policy=get_forensic_redaction_policy(self.db), + ) report_id = str(report.get("report_id", "")) domain = str(report.get("reported_domain") or "unknown") diff --git a/backend/app/services/imap_client.py b/backend/app/services/imap_client.py index 1ea6966..9263fb9 100644 --- a/backend/app/services/imap_client.py +++ b/backend/app/services/imap_client.py @@ -9,6 +9,7 @@ from app.core.config import get_settings from app.services.dmarc_parser import DMARCParser from app.services.forensic_parser import ForensicParser from app.services.forensic_persistence import forensic_report_exists, save_forensic_report +from app.services.forensic_redaction import get_forensic_redaction_policy from app.services.report_persistence import report_exists, save_parsed_report from app.services.report_store import ReportStore @@ -497,7 +498,10 @@ class IMAPClient: message_id: Optional[str], ) -> bool: try: - report = ForensicParser.parse_bytes(raw_email) + report = ForensicParser.parse_bytes( + raw_email, + redaction_policy=get_forensic_redaction_policy(self.db), + ) report_id = str(report.get("report_id", "")) domain = str(report.get("reported_domain") or "unknown") diff --git a/backend/app/templates/settings.html b/backend/app/templates/settings.html index 654d7ce..d0ffb9e 100644 --- a/backend/app/templates/settings.html +++ b/backend/app/templates/settings.html @@ -262,6 +262,48 @@ {% endcall %} {% endcall %} + + {% call card() %} + {% call card_header() %} + {% call card_title() %}Forensic Reports{% endcall %} + {% call card_description() %}Control privacy for forensic report metadata{% endcall %} + {% endcall %} + {% call card_content() %} +
+
+
+ + +
+
+ +
+
+ +
+ +
+
+ {% endcall %} + {% endcall %} + {% call card() %} {% call card_header() %} diff --git a/backend/app/tests/test_forensic_parser.py b/backend/app/tests/test_forensic_parser.py index 54eca27..4185a9e 100644 --- a/backend/app/tests/test_forensic_parser.py +++ b/backend/app/tests/test_forensic_parser.py @@ -12,6 +12,7 @@ from app.services.forensic_parser import ( _message_part_payload, _payload_text, ) +from app.services.forensic_redaction import ForensicRedactionPolicy SAMPLE_FORENSIC_EMAIL = b"""\ @@ -77,6 +78,28 @@ def test_parse_forensic_email_redacts_and_extracts_failure_fields(): assert "original-message@example.com" not in parsed["original_message_id"] +def test_parse_forensic_email_supports_stricter_redaction_policies(): + domain_only = ForensicParser.parse_bytes( + SAMPLE_FORENSIC_EMAIL, + redaction_policy=ForensicRedactionPolicy(mode="domain_only"), + ) + strict = ForensicParser.parse_bytes( + SAMPLE_FORENSIC_EMAIL, + redaction_policy=ForensicRedactionPolicy(mode="strict"), + ) + token_visible = ForensicParser.parse_bytes( + SAMPLE_FORENSIC_EMAIL, + redaction_policy=ForensicRedactionPolicy(redact_long_tokens=False), + ) + + assert domain_only["original_mail_from"] == "***@example.com" + assert "***@example.com" in domain_only["original_from"] + assert domain_only["source_email"] == "DMARC Reporter <***@example.net>" + assert strict["original_mail_from"] == "[redacted-email]" + assert strict["source_email"] == "DMARC Reporter <[redacted-email]>" + assert "abcdefghijklmnopqrstuvwxyz123456" in token_visible["original_subject"] + + def test_non_forensic_email_is_rejected(): content = b"From: sender@example.com\r\nSubject: hello\r\n\r\nplain email" diff --git a/backend/app/tests/test_forensics_api.py b/backend/app/tests/test_forensics_api.py index d968dc6..db14178 100644 --- a/backend/app/tests/test_forensics_api.py +++ b/backend/app/tests/test_forensics_api.py @@ -10,6 +10,7 @@ from app.services.forensic_persistence import ( forensic_report_to_dict, save_forensic_report, ) +from app.services.forensic_redaction import ForensicRedactionPolicy from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL @@ -62,6 +63,36 @@ def test_list_and_detail_forensic_reports(authed_client): assert detail_response.json()["reported_domain"] == "example.com" +def test_forensic_api_applies_configured_redaction_policy(authed_client, db_session): + authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")}, + ) + authed_client.put("/api/v1/settings/forensics.redaction_mode", json={"value": "strict"}) + + list_response = authed_client.get("/api/v1/forensics?domain=example.com") + + assert list_response.status_code == 200 + item = list_response.json()["reports"][0] + assert item["original_mail_from"] == "[redacted-email]" + assert item["source_email"] == "DMARC Reporter <[redacted-email]>" + stored = db_session.query(ForensicReport).one() + assert stored.original_mail_from == "al***@example.com" + + +def test_upload_forensic_report_uses_configured_redaction_policy(authed_client, db_session): + authed_client.put("/api/v1/settings/forensics.redaction_mode", json={"value": "domain_only"}) + + response = authed_client.post( + "/api/v1/forensics/upload", + files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")}, + ) + + assert response.status_code == 200 + report = db_session.query(ForensicReport).one() + assert report.original_mail_from == "***@example.com" + + def test_upload_forensic_report_rejects_aggregate_xml(authed_client): response = authed_client.post( "/api/v1/forensics/upload", @@ -156,6 +187,11 @@ def test_save_forensic_report_duplicate_and_invalid_domain_paths(db_session): assert forensic_report_exists(db_session, parsed["report_id"]) is True assert forensic_report_exists(db_session, "") is False assert forensic_report_to_dict(first)["feedback_headers"] == {"identity_alignment": "dkim"} + strict = forensic_report_to_dict( + first, + redaction_policy=ForensicRedactionPolicy(mode="strict"), + ) + assert strict["original_mail_from"] == "[redacted-email]" first.feedback_headers = "{not-json" assert forensic_report_to_dict(first)["feedback_headers"] == {} diff --git a/backend/app/tests/test_gmail_client.py b/backend/app/tests/test_gmail_client.py index a274e06..69830ad 100644 --- a/backend/app/tests/test_gmail_client.py +++ b/backend/app/tests/test_gmail_client.py @@ -20,6 +20,7 @@ from unittest.mock import MagicMock, patch import pytest from app.models.report import DMARCReport, ForensicReport +from app.models.setting import Setting from app.services.gmail_client import GmailClient, RETRYABLE_MESSAGE_FAILURE from app.services.report_store import ReportStore from app.tests.test_data import SAMPLE_XML @@ -465,6 +466,32 @@ class TestProcessMessage: assert db_session.query(DMARCReport).count() == 0 assert db_session.query(ForensicReport).count() == 1 + def test_forensic_report_uses_configured_redaction_policy(self, db_session): + setting = ( + db_session.query(Setting).filter(Setting.key == "forensics.redaction_mode").first() + ) + if setting is None: + setting = Setting( + key="forensics.redaction_mode", + value_type="string", + category="forensics", + ) + db_session.add(setting) + setting.value = "strict" + db_session.commit() + client = _make_client(db=db_session) + stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []} + + count = client._process_forensic_message( + SAMPLE_FORENSIC_EMAIL, + stats, + message_id="msg-forensic", + ) + + assert count == 1 + report = db_session.query(ForensicReport).one() + assert report.original_mail_from == "[redacted-email]" + def test_forensic_report_without_database_is_skipped(self): client = _make_client() stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []} diff --git a/backend/app/tests/test_imap_client.py b/backend/app/tests/test_imap_client.py index a0bc6b9..4529e74 100644 --- a/backend/app/tests/test_imap_client.py +++ b/backend/app/tests/test_imap_client.py @@ -18,6 +18,7 @@ from zipfile import ZipFile import pytest from app.models.report import DMARCReport, ForensicReport +from app.models.setting import Setting from app.services.imap_client import IMAPClient from app.services.report_store import ReportStore from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL @@ -641,6 +642,27 @@ class TestProcessSingleEmail: assert db_session.query(ForensicReport).count() == 1 assert ReportStore.get_instance().get_domains() == [] + def test_forensic_report_uses_configured_redaction_policy(self, db_session): + setting = ( + db_session.query(Setting).filter(Setting.key == "forensics.redaction_mode").first() + ) + if setting is None: + setting = Setting( + key="forensics.redaction_mode", + value_type="string", + category="forensics", + ) + db_session.add(setting) + setting.value = "domain_only" + db_session.commit() + client = self._make_client(db=db_session) + stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []} + + assert client._process_forensic_email(SAMPLE_FORENSIC_EMAIL, stats=stats, message_id="1") + + report = db_session.query(ForensicReport).one() + assert report.original_mail_from == "***@example.com" + def test_forensic_report_without_database_is_skipped(self): client = self._make_client() diff --git a/backend/app/tests/test_settings.py b/backend/app/tests/test_settings.py index 9e1e11f..ceb3ff2 100644 --- a/backend/app/tests/test_settings.py +++ b/backend/app/tests/test_settings.py @@ -123,6 +123,8 @@ class TestSettingsAPI: assert "notifications.summary_weekday_utc" in keys assert "notifications.min_send_interval_minutes" in keys assert "notifications.redact_pii_enabled" in keys + assert "forensics.redaction_mode" in keys + assert "forensics.redact_long_tokens_enabled" in keys def test_list_settings_filter_by_category(self, authed_client: TestClient): """GET /api/v1/settings?category=dmarc returns only dmarc settings.""" diff --git a/docs/deployment/configuration.md b/docs/deployment/configuration.md index 13a20fb..4d550d5 100644 --- a/docs/deployment/configuration.md +++ b/docs/deployment/configuration.md @@ -101,6 +101,14 @@ from API responses and encrypted at rest with the application `SECRET_KEY`. | `notifications.summary_send_hour_utc` | UTC hour for scheduled summaries | `8` | `7` | | `notifications.summary_weekday_utc` | UTC weekday for weekly summaries, where 0 is Monday | `0` | `4` | +Forensic report privacy controls are configured under **Settings** > +**Forensic Reports**. + +| Setting | Description | Default | Example | +|---------|-------------|---------|---------| +| `forensics.redaction_mode` | Email-address redaction in forensic metadata: `balanced`, `domain_only`, or `strict` | `balanced` | `strict` | +| `forensics.redact_long_tokens_enabled` | Redact long opaque tokens in forensic metadata | `true` | `true` | + Alert history is stored in the database-backed `alert_history` table. Notification and alert-rule configuration changes are stored in `alert_configuration_audit` with secret values sanitized. Current retention is diff --git a/docs/milestones.md b/docs/milestones.md index 4f7e79e..662edc2 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -171,12 +171,13 @@ Delivered: - Detect forensic report messages. - Parse safe metadata from ARF/attached email formats. - Store minimal incident details with privacy controls. +- Configure forensic report redaction for balanced, domain-only, and strict views. - Keep forensic reports out of aggregate report statistics and ReportStore rollups. - Expose authenticated forensic upload/list/detail APIs. Planned: - Add a dedicated forensic report view. -- Add configurable redaction controls and richer failure investigation workflows. +- Add richer failure investigation workflows. Exit criteria: - A security analyst can inspect individual failure reports without mixing them into aggregate statistics.