feat: add forensic redaction controls
This commit is contained in:
@@ -65,10 +65,11 @@ have no working implementation in the codebase yet.
|
||||
- **Documented in**: README.md ("Forensic Reports: Analyze failure samples (RFC 6591 support)")
|
||||
- **Current state**: Aggregate and forensic reports are now parsed separately. Forensic
|
||||
reports are stored in dedicated database rows and surfaced through authenticated APIs
|
||||
without affecting aggregate compliance statistics.
|
||||
without affecting aggregate compliance statistics. Operators can configure forensic
|
||||
email-address and token redaction under Settings.
|
||||
- [x] Forensic report parsing
|
||||
- [ ] Failure sample analysis
|
||||
- [ ] PII redaction options
|
||||
- [x] PII redaction options
|
||||
- [ ] Detailed authentication failure views
|
||||
|
||||
### User Authentication & Multi-User Support
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.services.forensic_persistence import (
|
||||
forensic_report_to_dict,
|
||||
save_forensic_report,
|
||||
)
|
||||
from app.services.forensic_redaction import get_forensic_redaction_policy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -88,7 +89,8 @@ async def upload_forensic_report(
|
||||
try:
|
||||
content = await file.read()
|
||||
_validate_upload(file, content)
|
||||
parsed = ForensicParser.parse_bytes(content)
|
||||
redaction_policy = get_forensic_redaction_policy(db)
|
||||
parsed = ForensicParser.parse_bytes(content, redaction_policy=redaction_policy)
|
||||
if forensic_report_exists(db, parsed["report_id"]):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
@@ -148,12 +150,18 @@ async def list_forensic_reports(
|
||||
.all()
|
||||
)
|
||||
total_pages = (total + page_size - 1) // page_size if total else 0
|
||||
redaction_policy = get_forensic_redaction_policy(db)
|
||||
return ForensicListResponse(
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total_pages=total_pages,
|
||||
reports=[ForensicReportResponse(**forensic_report_to_dict(row)) for row in rows],
|
||||
reports=[
|
||||
ForensicReportResponse(
|
||||
**forensic_report_to_dict(row, redaction_policy=redaction_policy)
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -174,4 +182,5 @@ async def get_forensic_report(
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Forensic report not found"
|
||||
)
|
||||
return ForensicReportResponse(**forensic_report_to_dict(row))
|
||||
redaction_policy = get_forensic_redaction_policy(db)
|
||||
return ForensicReportResponse(**forensic_report_to_dict(row, redaction_policy=redaction_policy))
|
||||
|
||||
@@ -8,6 +8,7 @@ in the ``settings`` database table. Settings are organised into categories:
|
||||
- ``dmarc`` – Default DMARC policy, percentage, etc.
|
||||
- ``dns`` – Default DNS resolver, Cloudflare DoH toggle.
|
||||
- ``cloudflare`` – Cloudflare API token and Zone ID.
|
||||
- ``forensics`` – Forensic report privacy and retention controls.
|
||||
- ``notifications`` – Future alerting/notification settings.
|
||||
"""
|
||||
|
||||
@@ -121,6 +122,21 @@ SETTING_DEFAULTS: List[Dict[str, Any]] = [
|
||||
"value_type": "string",
|
||||
"category": "cloudflare",
|
||||
},
|
||||
# ── Forensics ────────────────────────────────────────────────────────────
|
||||
{
|
||||
"key": "forensics.redaction_mode",
|
||||
"value": "balanced",
|
||||
"description": "Forensic report email-address redaction mode: balanced, domain_only, or strict",
|
||||
"value_type": "string",
|
||||
"category": "forensics",
|
||||
},
|
||||
{
|
||||
"key": "forensics.redact_long_tokens_enabled",
|
||||
"value": "true",
|
||||
"description": "Redact long opaque tokens in forensic report metadata",
|
||||
"value_type": "boolean",
|
||||
"category": "forensics",
|
||||
},
|
||||
# ── Notifications ─────────────────────────────────────────────────────────
|
||||
{
|
||||
"key": "notifications.apprise_enabled",
|
||||
@@ -312,7 +328,7 @@ def _audit_value_for_setting(key: str, value: Optional[str]) -> Optional[str]:
|
||||
|
||||
|
||||
def _should_audit_setting(key: str) -> bool:
|
||||
return key.startswith("notifications.")
|
||||
return key.startswith(("notifications.", "forensics."))
|
||||
|
||||
|
||||
def _audit_setting_change(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from email import message_from_bytes
|
||||
from email.message import Message
|
||||
@@ -8,12 +7,15 @@ from email.parser import Parser
|
||||
from email.utils import getaddresses, parsedate_to_datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from app.services.forensic_redaction import (
|
||||
ForensicRedactionPolicy,
|
||||
normalize_forensic_redaction_policy,
|
||||
redact_forensic_text,
|
||||
)
|
||||
|
||||
|
||||
MAX_FORENSIC_REPORT_SIZE = 10 * 1024 * 1024
|
||||
|
||||
_EMAIL_RE = re.compile(r"\b([A-Z0-9._%+-]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b", re.IGNORECASE)
|
||||
_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b")
|
||||
|
||||
|
||||
def _coerce_text(value: Any) -> str:
|
||||
if value is None:
|
||||
@@ -23,26 +25,37 @@ def _coerce_text(value: Any) -> str:
|
||||
return str(value)
|
||||
|
||||
|
||||
def _clean(value: Any, *, redact: bool = True) -> str:
|
||||
def _clean(
|
||||
value: Any,
|
||||
*,
|
||||
redact: bool = True,
|
||||
redaction_policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> str:
|
||||
text = " ".join(_coerce_text(value).replace("\r", " ").replace("\n", " ").split())
|
||||
return redact_text(text) if redact else text
|
||||
return redact_text(text, redaction_policy=redaction_policy) if redact else text
|
||||
|
||||
|
||||
def redact_text(value: str) -> str:
|
||||
def redact_text(
|
||||
value: str,
|
||||
*,
|
||||
redaction_policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> str:
|
||||
"""Redact email local-parts and long opaque tokens from forensic metadata."""
|
||||
|
||||
def _redact_email(match: re.Match[str]) -> str:
|
||||
local = match.group(1)
|
||||
domain = match.group(2)
|
||||
prefix = local[:2] if len(local) > 2 else local[:1]
|
||||
return f"{prefix}***@{domain.lower()}"
|
||||
|
||||
redacted = _EMAIL_RE.sub(_redact_email, value)
|
||||
return _LONG_TOKEN_RE.sub("[redacted-token]", redacted)
|
||||
return redact_forensic_text(value, redaction_policy)
|
||||
|
||||
|
||||
def _header(msg: Optional[Message], name: str, *, redact: bool = True) -> str:
|
||||
return _clean(msg.get(name, "") if msg is not None else "", redact=redact)
|
||||
def _header(
|
||||
msg: Optional[Message],
|
||||
name: str,
|
||||
*,
|
||||
redact: bool = True,
|
||||
redaction_policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> str:
|
||||
return _clean(
|
||||
msg.get(name, "") if msg is not None else "",
|
||||
redact=redact,
|
||||
redaction_policy=redaction_policy,
|
||||
)
|
||||
|
||||
|
||||
def _payload_text(part: Message) -> str:
|
||||
@@ -120,7 +133,11 @@ class ForensicParser:
|
||||
|
||||
@classmethod
|
||||
def parse_bytes(
|
||||
cls, content: bytes, *, message_id_hint: Optional[str] = None
|
||||
cls,
|
||||
content: bytes,
|
||||
*,
|
||||
message_id_hint: Optional[str] = None,
|
||||
redaction_policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if len(content) > MAX_FORENSIC_REPORT_SIZE:
|
||||
raise ValueError("Forensic report is too large")
|
||||
@@ -131,6 +148,7 @@ class ForensicParser:
|
||||
if not cls.is_forensic_report(msg):
|
||||
raise ValueError("Email is not a DMARC forensic report")
|
||||
|
||||
redaction_policy = normalize_forensic_redaction_policy(redaction_policy)
|
||||
feedback = None
|
||||
original_headers = None
|
||||
|
||||
@@ -166,14 +184,18 @@ class ForensicParser:
|
||||
if not report_id.startswith("ruf-"):
|
||||
report_id = f"ruf-{report_id}"
|
||||
|
||||
source_email = _header(msg, "From")
|
||||
source_email = _header(msg, "From", redaction_policy=redaction_policy)
|
||||
arrival_date = _parse_datetime(_header(feedback, "Arrival-Date", redact=False))
|
||||
|
||||
details = {
|
||||
"identity_alignment": _header(feedback, "Identity-Alignment", redact=False),
|
||||
"dkim_domain": _header(feedback, "DKIM-Domain", redact=False),
|
||||
"spf_dns": _header(feedback, "SPF-DNS", redact=False),
|
||||
"reported_uri": _header(feedback, "Reported-URI"),
|
||||
"reported_uri": _header(
|
||||
feedback,
|
||||
"Reported-URI",
|
||||
redaction_policy=redaction_policy,
|
||||
),
|
||||
}
|
||||
details = {key: value for key, value in details.items() if value}
|
||||
|
||||
@@ -181,18 +203,34 @@ class ForensicParser:
|
||||
"report_id": report_id,
|
||||
"source_email": source_email,
|
||||
"feedback_type": _header(feedback, "Feedback-Type", redact=False) or "auth-failure",
|
||||
"user_agent": _header(feedback, "User-Agent"),
|
||||
"user_agent": _header(feedback, "User-Agent", redaction_policy=redaction_policy),
|
||||
"version": _header(feedback, "Version", redact=False),
|
||||
"reported_domain": reported_domain,
|
||||
"source_ip": source_ip,
|
||||
"auth_failure": auth_failure,
|
||||
"delivery_result": _header(feedback, "Delivery-Result", redact=False),
|
||||
"arrival_date": arrival_date,
|
||||
"authentication_results": _header(feedback, "Authentication-Results"),
|
||||
"original_mail_from": _header(feedback, "Original-Mail-From"),
|
||||
"original_from": _header(original_headers, "From"),
|
||||
"original_to": _header(original_headers, "To"),
|
||||
"original_subject": _header(original_headers, "Subject"),
|
||||
"authentication_results": _header(
|
||||
feedback,
|
||||
"Authentication-Results",
|
||||
redaction_policy=redaction_policy,
|
||||
),
|
||||
"original_mail_from": _header(
|
||||
feedback,
|
||||
"Original-Mail-From",
|
||||
redaction_policy=redaction_policy,
|
||||
),
|
||||
"original_from": _header(
|
||||
original_headers,
|
||||
"From",
|
||||
redaction_policy=redaction_policy,
|
||||
),
|
||||
"original_to": _header(original_headers, "To", redaction_policy=redaction_policy),
|
||||
"original_subject": _header(
|
||||
original_headers,
|
||||
"Subject",
|
||||
redaction_policy=redaction_policy,
|
||||
),
|
||||
"original_message_id": _message_id_hash(original_message_id),
|
||||
"original_date": _header(original_headers, "Date", redact=False),
|
||||
"feedback_headers": json.dumps(details, sort_keys=True) if details else None,
|
||||
|
||||
@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.domain import Domain
|
||||
from app.models.report import ForensicReport
|
||||
from app.services.forensic_redaction import ForensicRedactionPolicy, redact_forensic_value
|
||||
from app.utils.domain_validator import DomainValidationError, validate_domain
|
||||
|
||||
|
||||
@@ -87,7 +88,23 @@ def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicR
|
||||
return row, True
|
||||
|
||||
|
||||
def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
|
||||
_REDACTABLE_RESPONSE_FIELDS = {
|
||||
"source_email",
|
||||
"user_agent",
|
||||
"authentication_results",
|
||||
"original_mail_from",
|
||||
"original_from",
|
||||
"original_to",
|
||||
"original_subject",
|
||||
"feedback_headers",
|
||||
}
|
||||
|
||||
|
||||
def forensic_report_to_dict(
|
||||
row: ForensicReport,
|
||||
*,
|
||||
redaction_policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert a forensic report row to an API-safe dictionary."""
|
||||
feedback_headers = {}
|
||||
if row.feedback_headers:
|
||||
@@ -96,7 +113,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
feedback_headers = {}
|
||||
|
||||
return {
|
||||
result = {
|
||||
"id": row.id,
|
||||
"report_id": row.report_id,
|
||||
"domain": row.domain.name if row.domain else row.reported_domain,
|
||||
@@ -119,3 +136,7 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
|
||||
"feedback_headers": feedback_headers,
|
||||
"processed_at": row.processed_at.isoformat() if row.processed_at else None,
|
||||
}
|
||||
if redaction_policy is not None:
|
||||
for field in _REDACTABLE_RESPONSE_FIELDS:
|
||||
result[field] = redact_forensic_value(result.get(field), redaction_policy)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.setting import Setting
|
||||
|
||||
|
||||
FORENSIC_REDACTION_MODE_KEY = "forensics.redaction_mode"
|
||||
FORENSIC_REDACT_LONG_TOKENS_KEY = "forensics.redact_long_tokens_enabled"
|
||||
DEFAULT_FORENSIC_REDACTION_MODE = "balanced"
|
||||
FORENSIC_REDACTION_MODES = {"balanced", "domain_only", "strict"}
|
||||
|
||||
_EMAIL_RE = re.compile(
|
||||
r"\b([A-Z0-9._%+\-*]{1,64})@([A-Z0-9.-]+\.[A-Z]{2,})\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LONG_TOKEN_RE = re.compile(r"\b[A-Za-z0-9_./+=-]{28,}\b")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ForensicRedactionPolicy:
|
||||
"""Privacy policy for forensic report metadata."""
|
||||
|
||||
mode: str = DEFAULT_FORENSIC_REDACTION_MODE
|
||||
redact_long_tokens: bool = True
|
||||
|
||||
|
||||
def _truthy(value: Optional[str], *, default: bool = True) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def normalize_forensic_redaction_policy(
|
||||
policy: Optional[ForensicRedactionPolicy] = None,
|
||||
*,
|
||||
mode: Optional[str] = None,
|
||||
redact_long_tokens: Optional[bool] = None,
|
||||
) -> ForensicRedactionPolicy:
|
||||
requested_mode = (mode if mode is not None else policy.mode if policy else "").strip().lower()
|
||||
if requested_mode not in FORENSIC_REDACTION_MODES:
|
||||
requested_mode = DEFAULT_FORENSIC_REDACTION_MODE
|
||||
return ForensicRedactionPolicy(
|
||||
mode=requested_mode,
|
||||
redact_long_tokens=(
|
||||
policy.redact_long_tokens
|
||||
if redact_long_tokens is None and policy is not None
|
||||
else bool(True if redact_long_tokens is None else redact_long_tokens)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_forensic_redaction_policy(db: Optional[Session]) -> ForensicRedactionPolicy:
|
||||
"""Load the current forensic redaction policy from persisted settings."""
|
||||
if db is None:
|
||||
return ForensicRedactionPolicy()
|
||||
|
||||
rows = (
|
||||
db.query(Setting.key, Setting.value)
|
||||
.filter(Setting.key.in_([FORENSIC_REDACTION_MODE_KEY, FORENSIC_REDACT_LONG_TOKENS_KEY]))
|
||||
.all()
|
||||
)
|
||||
values = {key: value for key, value in rows}
|
||||
return normalize_forensic_redaction_policy(
|
||||
mode=values.get(FORENSIC_REDACTION_MODE_KEY),
|
||||
redact_long_tokens=_truthy(values.get(FORENSIC_REDACT_LONG_TOKENS_KEY), default=True),
|
||||
)
|
||||
|
||||
|
||||
def redact_forensic_text(
|
||||
value: str,
|
||||
policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> str:
|
||||
"""Redact forensic metadata according to the selected privacy policy."""
|
||||
policy = normalize_forensic_redaction_policy(policy)
|
||||
|
||||
def _redact_email(match: re.Match[str]) -> str:
|
||||
local = match.group(1)
|
||||
domain = match.group(2).lower()
|
||||
if policy.mode == "strict":
|
||||
return "[redacted-email]"
|
||||
if policy.mode == "domain_only":
|
||||
return f"***@{domain}"
|
||||
prefix = local[:2] if len(local) > 2 else local[:1]
|
||||
return f"{prefix}***@{domain}"
|
||||
|
||||
redacted = _EMAIL_RE.sub(_redact_email, value)
|
||||
if policy.redact_long_tokens:
|
||||
redacted = _LONG_TOKEN_RE.sub("[redacted-token]", redacted)
|
||||
return redacted
|
||||
|
||||
|
||||
def redact_forensic_value(
|
||||
value: Any,
|
||||
policy: Optional[ForensicRedactionPolicy] = None,
|
||||
) -> Any:
|
||||
"""Redact strings inside a response value while preserving container shape."""
|
||||
if isinstance(value, str):
|
||||
return redact_forensic_text(value, policy)
|
||||
if isinstance(value, dict):
|
||||
return {key: redact_forensic_value(item, policy) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [redact_forensic_value(item, policy) for item in value]
|
||||
return value
|
||||
@@ -23,6 +23,7 @@ from googleapiclient.errors import HttpError
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.forensic_parser import ForensicParser
|
||||
from app.services.forensic_persistence import forensic_report_exists, save_forensic_report
|
||||
from app.services.forensic_redaction import get_forensic_redaction_policy
|
||||
from app.services.report_persistence import report_exists, save_parsed_report
|
||||
from app.services.report_store import ReportStore
|
||||
|
||||
@@ -385,7 +386,11 @@ class GmailClient:
|
||||
) -> int:
|
||||
"""Parse and persist one DMARC forensic report message."""
|
||||
try:
|
||||
report = ForensicParser.parse_bytes(raw_bytes, message_id_hint=message_id)
|
||||
report = ForensicParser.parse_bytes(
|
||||
raw_bytes,
|
||||
message_id_hint=message_id,
|
||||
redaction_policy=get_forensic_redaction_policy(self.db),
|
||||
)
|
||||
report_id = str(report.get("report_id", ""))
|
||||
domain = str(report.get("reported_domain") or "unknown")
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from app.core.config import get_settings
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.forensic_parser import ForensicParser
|
||||
from app.services.forensic_persistence import forensic_report_exists, save_forensic_report
|
||||
from app.services.forensic_redaction import get_forensic_redaction_policy
|
||||
from app.services.report_persistence import report_exists, save_parsed_report
|
||||
from app.services.report_store import ReportStore
|
||||
|
||||
@@ -497,7 +498,10 @@ class IMAPClient:
|
||||
message_id: Optional[str],
|
||||
) -> bool:
|
||||
try:
|
||||
report = ForensicParser.parse_bytes(raw_email)
|
||||
report = ForensicParser.parse_bytes(
|
||||
raw_email,
|
||||
redaction_policy=get_forensic_redaction_policy(self.db),
|
||||
)
|
||||
report_id = str(report.get("report_id", ""))
|
||||
domain = str(report.get("reported_domain") or "unknown")
|
||||
|
||||
|
||||
@@ -262,6 +262,48 @@
|
||||
{% endcall %}
|
||||
{% endcall %}
|
||||
|
||||
<!-- ── Forensics ──────────────────────────────────────────────────────── -->
|
||||
{% call card() %}
|
||||
{% call card_header() %}
|
||||
{% call card_title() %}Forensic Reports{% endcall %}
|
||||
{% call card_description() %}Control privacy for forensic report metadata{% endcall %}
|
||||
{% endcall %}
|
||||
{% call card_content() %}
|
||||
<form @submit.prevent="saveCategory('forensics')" class="space-y-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Email Redaction</span></label>
|
||||
<select x-model="s['forensics.redaction_mode']"
|
||||
class="select select-bordered w-full">
|
||||
<option value="balanced">Balanced</option>
|
||||
<option value="domain_only">Domain only</option>
|
||||
<option value="strict">Strict</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-control">
|
||||
<label class="label cursor-pointer justify-start gap-3">
|
||||
<input type="checkbox"
|
||||
:checked="s['forensics.redact_long_tokens_enabled'] === 'true'"
|
||||
@change="s['forensics.redact_long_tokens_enabled'] = $event.target.checked ? 'true' : 'false'"
|
||||
class="checkbox checkbox-primary" />
|
||||
<span class="label-text font-medium">Redact long tokens</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end">
|
||||
<button type="submit" class="btn btn-default btn-md" :disabled="saving">
|
||||
<template x-if="!saving">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="mr-2"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"></path><polyline points="17 21 17 13 7 13 7 21"></polyline><polyline points="7 3 7 8 15 8"></polyline></svg>
|
||||
</template>
|
||||
<template x-if="saving"><span class="loading loading-spinner loading-xs mr-2"></span></template>
|
||||
Save Forensic Settings
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
{% endcall %}
|
||||
{% endcall %}
|
||||
|
||||
<!-- ── Notifications ──────────────────────────────────────────────────── -->
|
||||
{% call card() %}
|
||||
{% call card_header() %}
|
||||
|
||||
@@ -12,6 +12,7 @@ from app.services.forensic_parser import (
|
||||
_message_part_payload,
|
||||
_payload_text,
|
||||
)
|
||||
from app.services.forensic_redaction import ForensicRedactionPolicy
|
||||
|
||||
|
||||
SAMPLE_FORENSIC_EMAIL = b"""\
|
||||
@@ -77,6 +78,28 @@ def test_parse_forensic_email_redacts_and_extracts_failure_fields():
|
||||
assert "original-message@example.com" not in parsed["original_message_id"]
|
||||
|
||||
|
||||
def test_parse_forensic_email_supports_stricter_redaction_policies():
|
||||
domain_only = ForensicParser.parse_bytes(
|
||||
SAMPLE_FORENSIC_EMAIL,
|
||||
redaction_policy=ForensicRedactionPolicy(mode="domain_only"),
|
||||
)
|
||||
strict = ForensicParser.parse_bytes(
|
||||
SAMPLE_FORENSIC_EMAIL,
|
||||
redaction_policy=ForensicRedactionPolicy(mode="strict"),
|
||||
)
|
||||
token_visible = ForensicParser.parse_bytes(
|
||||
SAMPLE_FORENSIC_EMAIL,
|
||||
redaction_policy=ForensicRedactionPolicy(redact_long_tokens=False),
|
||||
)
|
||||
|
||||
assert domain_only["original_mail_from"] == "***@example.com"
|
||||
assert "***@example.com" in domain_only["original_from"]
|
||||
assert domain_only["source_email"] == "DMARC Reporter <***@example.net>"
|
||||
assert strict["original_mail_from"] == "[redacted-email]"
|
||||
assert strict["source_email"] == "DMARC Reporter <[redacted-email]>"
|
||||
assert "abcdefghijklmnopqrstuvwxyz123456" in token_visible["original_subject"]
|
||||
|
||||
|
||||
def test_non_forensic_email_is_rejected():
|
||||
content = b"From: sender@example.com\r\nSubject: hello\r\n\r\nplain email"
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.services.forensic_persistence import (
|
||||
forensic_report_to_dict,
|
||||
save_forensic_report,
|
||||
)
|
||||
from app.services.forensic_redaction import ForensicRedactionPolicy
|
||||
from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL
|
||||
|
||||
|
||||
@@ -62,6 +63,36 @@ def test_list_and_detail_forensic_reports(authed_client):
|
||||
assert detail_response.json()["reported_domain"] == "example.com"
|
||||
|
||||
|
||||
def test_forensic_api_applies_configured_redaction_policy(authed_client, db_session):
|
||||
authed_client.post(
|
||||
"/api/v1/forensics/upload",
|
||||
files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")},
|
||||
)
|
||||
authed_client.put("/api/v1/settings/forensics.redaction_mode", json={"value": "strict"})
|
||||
|
||||
list_response = authed_client.get("/api/v1/forensics?domain=example.com")
|
||||
|
||||
assert list_response.status_code == 200
|
||||
item = list_response.json()["reports"][0]
|
||||
assert item["original_mail_from"] == "[redacted-email]"
|
||||
assert item["source_email"] == "DMARC Reporter <[redacted-email]>"
|
||||
stored = db_session.query(ForensicReport).one()
|
||||
assert stored.original_mail_from == "al***@example.com"
|
||||
|
||||
|
||||
def test_upload_forensic_report_uses_configured_redaction_policy(authed_client, db_session):
|
||||
authed_client.put("/api/v1/settings/forensics.redaction_mode", json={"value": "domain_only"})
|
||||
|
||||
response = authed_client.post(
|
||||
"/api/v1/forensics/upload",
|
||||
files={"file": ("report.eml", SAMPLE_FORENSIC_EMAIL, "message/rfc822")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
report = db_session.query(ForensicReport).one()
|
||||
assert report.original_mail_from == "***@example.com"
|
||||
|
||||
|
||||
def test_upload_forensic_report_rejects_aggregate_xml(authed_client):
|
||||
response = authed_client.post(
|
||||
"/api/v1/forensics/upload",
|
||||
@@ -156,6 +187,11 @@ def test_save_forensic_report_duplicate_and_invalid_domain_paths(db_session):
|
||||
assert forensic_report_exists(db_session, parsed["report_id"]) is True
|
||||
assert forensic_report_exists(db_session, "") is False
|
||||
assert forensic_report_to_dict(first)["feedback_headers"] == {"identity_alignment": "dkim"}
|
||||
strict = forensic_report_to_dict(
|
||||
first,
|
||||
redaction_policy=ForensicRedactionPolicy(mode="strict"),
|
||||
)
|
||||
assert strict["original_mail_from"] == "[redacted-email]"
|
||||
|
||||
first.feedback_headers = "{not-json"
|
||||
assert forensic_report_to_dict(first)["feedback_headers"] == {}
|
||||
|
||||
@@ -20,6 +20,7 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from app.models.report import DMARCReport, ForensicReport
|
||||
from app.models.setting import Setting
|
||||
from app.services.gmail_client import GmailClient, RETRYABLE_MESSAGE_FAILURE
|
||||
from app.services.report_store import ReportStore
|
||||
from app.tests.test_data import SAMPLE_XML
|
||||
@@ -465,6 +466,32 @@ class TestProcessMessage:
|
||||
assert db_session.query(DMARCReport).count() == 0
|
||||
assert db_session.query(ForensicReport).count() == 1
|
||||
|
||||
def test_forensic_report_uses_configured_redaction_policy(self, db_session):
|
||||
setting = (
|
||||
db_session.query(Setting).filter(Setting.key == "forensics.redaction_mode").first()
|
||||
)
|
||||
if setting is None:
|
||||
setting = Setting(
|
||||
key="forensics.redaction_mode",
|
||||
value_type="string",
|
||||
category="forensics",
|
||||
)
|
||||
db_session.add(setting)
|
||||
setting.value = "strict"
|
||||
db_session.commit()
|
||||
client = _make_client(db=db_session)
|
||||
stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []}
|
||||
|
||||
count = client._process_forensic_message(
|
||||
SAMPLE_FORENSIC_EMAIL,
|
||||
stats,
|
||||
message_id="msg-forensic",
|
||||
)
|
||||
|
||||
assert count == 1
|
||||
report = db_session.query(ForensicReport).one()
|
||||
assert report.original_mail_from == "[redacted-email]"
|
||||
|
||||
def test_forensic_report_without_database_is_skipped(self):
|
||||
client = _make_client()
|
||||
stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []}
|
||||
|
||||
@@ -18,6 +18,7 @@ from zipfile import ZipFile
|
||||
import pytest
|
||||
|
||||
from app.models.report import DMARCReport, ForensicReport
|
||||
from app.models.setting import Setting
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.report_store import ReportStore
|
||||
from app.tests.test_forensic_parser import SAMPLE_FORENSIC_EMAIL
|
||||
@@ -641,6 +642,27 @@ class TestProcessSingleEmail:
|
||||
assert db_session.query(ForensicReport).count() == 1
|
||||
assert ReportStore.get_instance().get_domains() == []
|
||||
|
||||
def test_forensic_report_uses_configured_redaction_policy(self, db_session):
|
||||
setting = (
|
||||
db_session.query(Setting).filter(Setting.key == "forensics.redaction_mode").first()
|
||||
)
|
||||
if setting is None:
|
||||
setting = Setting(
|
||||
key="forensics.redaction_mode",
|
||||
value_type="string",
|
||||
category="forensics",
|
||||
)
|
||||
db_session.add(setting)
|
||||
setting.value = "domain_only"
|
||||
db_session.commit()
|
||||
client = self._make_client(db=db_session)
|
||||
stats = {"forensic_reports_found": 0, "duplicate_forensic_reports": 0, "errors": []}
|
||||
|
||||
assert client._process_forensic_email(SAMPLE_FORENSIC_EMAIL, stats=stats, message_id="1")
|
||||
|
||||
report = db_session.query(ForensicReport).one()
|
||||
assert report.original_mail_from == "***@example.com"
|
||||
|
||||
def test_forensic_report_without_database_is_skipped(self):
|
||||
client = self._make_client()
|
||||
|
||||
|
||||
@@ -123,6 +123,8 @@ class TestSettingsAPI:
|
||||
assert "notifications.summary_weekday_utc" in keys
|
||||
assert "notifications.min_send_interval_minutes" in keys
|
||||
assert "notifications.redact_pii_enabled" in keys
|
||||
assert "forensics.redaction_mode" in keys
|
||||
assert "forensics.redact_long_tokens_enabled" in keys
|
||||
|
||||
def test_list_settings_filter_by_category(self, authed_client: TestClient):
|
||||
"""GET /api/v1/settings?category=dmarc returns only dmarc settings."""
|
||||
|
||||
@@ -101,6 +101,14 @@ from API responses and encrypted at rest with the application `SECRET_KEY`.
|
||||
| `notifications.summary_send_hour_utc` | UTC hour for scheduled summaries | `8` | `7` |
|
||||
| `notifications.summary_weekday_utc` | UTC weekday for weekly summaries, where 0 is Monday | `0` | `4` |
|
||||
|
||||
Forensic report privacy controls are configured under **Settings** >
|
||||
**Forensic Reports**.
|
||||
|
||||
| Setting | Description | Default | Example |
|
||||
|---------|-------------|---------|---------|
|
||||
| `forensics.redaction_mode` | Email-address redaction in forensic metadata: `balanced`, `domain_only`, or `strict` | `balanced` | `strict` |
|
||||
| `forensics.redact_long_tokens_enabled` | Redact long opaque tokens in forensic metadata | `true` | `true` |
|
||||
|
||||
Alert history is stored in the database-backed `alert_history` table.
|
||||
Notification and alert-rule configuration changes are stored in
|
||||
`alert_configuration_audit` with secret values sanitized. Current retention is
|
||||
|
||||
+2
-1
@@ -171,12 +171,13 @@ Delivered:
|
||||
- Detect forensic report messages.
|
||||
- Parse safe metadata from ARF/attached email formats.
|
||||
- Store minimal incident details with privacy controls.
|
||||
- Configure forensic report redaction for balanced, domain-only, and strict views.
|
||||
- Keep forensic reports out of aggregate report statistics and ReportStore rollups.
|
||||
- Expose authenticated forensic upload/list/detail APIs.
|
||||
|
||||
Planned:
|
||||
- Add a dedicated forensic report view.
|
||||
- Add configurable redaction controls and richer failure investigation workflows.
|
||||
- Add richer failure investigation workflows.
|
||||
|
||||
Exit criteria:
|
||||
- A security analyst can inspect individual failure reports without mixing them into aggregate statistics.
|
||||
|
||||
Reference in New Issue
Block a user