feat: add TLS reporting posture summaries
This commit is contained in:
Christian Krakau-Louis
@@ -0,0 +1,168 @@
|
||||
"""Parser for SMTP TLS Reporting (TLS-RPT) JSON aggregates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
MAX_TLS_REPORT_SIZE = 10 * 1024 * 1024
|
||||
MAX_TLS_UNCOMPRESSED_SIZE = 100 * 1024 * 1024
|
||||
MAX_TLS_FILES_IN_ARCHIVE = 10
|
||||
|
||||
|
||||
def _clean(value: Any) -> str:
|
||||
return " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int:
|
||||
try:
|
||||
return max(int(value or 0), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> Optional[datetime]:
|
||||
text = _clean(value)
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.astimezone(timezone.utc)
|
||||
return parsed.replace(tzinfo=None)
|
||||
|
||||
|
||||
def _extract_json_from_zip(file_content: bytes) -> Optional[bytes]:
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(file_content)) as archive:
|
||||
files = archive.infolist()
|
||||
if len(files) > MAX_TLS_FILES_IN_ARCHIVE:
|
||||
raise ValueError("TLS report archive contains too many files")
|
||||
if sum(item.file_size for item in files) > MAX_TLS_UNCOMPRESSED_SIZE:
|
||||
raise ValueError("TLS report archive is too large after decompression")
|
||||
for item in files:
|
||||
if item.filename.lower().endswith(".json"):
|
||||
if item.file_size > MAX_TLS_UNCOMPRESSED_SIZE:
|
||||
raise ValueError("TLS report JSON is too large after decompression")
|
||||
return archive.read(item.filename)
|
||||
except zipfile.BadZipFile:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_json_content(file_content: bytes, filename: str) -> bytes:
|
||||
lower = filename.lower()
|
||||
if lower.endswith(".zip"):
|
||||
extracted = _extract_json_from_zip(file_content)
|
||||
if extracted is not None:
|
||||
return extracted
|
||||
if lower.endswith((".gz", ".gzip")):
|
||||
try:
|
||||
return gzip.decompress(file_content)
|
||||
except gzip.BadGzipFile as exc:
|
||||
raise ValueError("Invalid gzip TLS report") from exc
|
||||
if lower.endswith(".json"):
|
||||
return file_content
|
||||
raise ValueError("Invalid TLS report file type. Upload .json, .json.gz, or .zip.")
|
||||
|
||||
|
||||
def _policy_domain(policy: Dict[str, Any]) -> str:
|
||||
return _clean(policy.get("policy-domain")).lower().strip(".")
|
||||
|
||||
|
||||
def _normalize_failure(detail: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"result_type": _clean(detail.get("result-type") or "unknown").lower(),
|
||||
"failed_session_count": _safe_int(detail.get("failed-session-count")),
|
||||
"sending_mta_ip": _clean(detail.get("sending-mta-ip")),
|
||||
"receiving_mx_hostname": _clean(detail.get("receiving-mx-hostname")).lower(),
|
||||
"receiving_mx_helo": _clean(detail.get("receiving-mx-helo")),
|
||||
"receiving_ip": _clean(detail.get("receiving-ip")),
|
||||
"failure_reason_code": _clean(detail.get("failure-reason-code")),
|
||||
"additional_information": _clean(detail.get("additional-information")),
|
||||
}
|
||||
|
||||
|
||||
def _load_payload(json_content: bytes) -> Dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(json_content.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("TLS report is not valid JSON") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("TLS report JSON must be an object")
|
||||
return payload
|
||||
|
||||
|
||||
def _normalize_policy(item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
policy = item.get("policy") or {}
|
||||
summary = item.get("summary") or {}
|
||||
if not isinstance(policy, dict) or not isinstance(summary, dict):
|
||||
return None
|
||||
domain = _policy_domain(policy)
|
||||
if not domain:
|
||||
return None
|
||||
failures = [
|
||||
_normalize_failure(detail)
|
||||
for detail in item.get("failure-details") or []
|
||||
if isinstance(detail, dict)
|
||||
]
|
||||
return {
|
||||
"policy_domain": domain,
|
||||
"policy_type": _clean(policy.get("policy-type")).lower(),
|
||||
"policy": policy,
|
||||
"total_successful_sessions": _safe_int(summary.get("total-successful-session-count")),
|
||||
"total_failure_sessions": _safe_int(summary.get("total-failure-session-count")),
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_policies(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
policies = []
|
||||
for item in payload.get("policies") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
policy = _normalize_policy(item)
|
||||
if policy is not None:
|
||||
policies.append(policy)
|
||||
if not policies:
|
||||
raise ValueError("TLS report does not contain any policy-domain entries")
|
||||
return policies
|
||||
|
||||
|
||||
class TLSReportParser:
|
||||
"""Parse TLS-RPT JSON while retaining only aggregate posture data."""
|
||||
|
||||
@staticmethod
|
||||
def parse_file(file_content: bytes, filename: str) -> Dict[str, Any]:
|
||||
"""Parse a TLS-RPT JSON, gzip, or zip attachment into normalized dictionaries."""
|
||||
if len(file_content) > MAX_TLS_REPORT_SIZE:
|
||||
raise ValueError("TLS report is too large")
|
||||
if not file_content:
|
||||
raise ValueError("TLS report is empty")
|
||||
|
||||
json_content = _extract_json_content(file_content, filename)
|
||||
if len(json_content) > MAX_TLS_UNCOMPRESSED_SIZE:
|
||||
raise ValueError("TLS report is too large after decompression")
|
||||
|
||||
payload = _load_payload(json_content)
|
||||
date_range = payload.get("date-range") or {}
|
||||
policies = _normalize_policies(payload)
|
||||
|
||||
report_id = _clean(payload.get("report-id"))
|
||||
if not report_id:
|
||||
report_id = "tlsrpt-" + hashlib.sha256(json_content).hexdigest()[:24]
|
||||
return {
|
||||
"report_id": report_id,
|
||||
"org_name": _clean(payload.get("organization-name")),
|
||||
"contact_info": _clean(payload.get("contact-info")),
|
||||
"begin_date": _parse_datetime(date_range.get("start-datetime")),
|
||||
"end_date": _parse_datetime(date_range.get("end-datetime")),
|
||||
"policies": policies,
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
"""Persistence and summarization helpers for SMTP TLS reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from app.models.domain import Domain
|
||||
from app.models.report import TLSReport, TLSReportFailure
|
||||
from app.utils.domain_validator import DomainValidationError, validate_domain
|
||||
|
||||
|
||||
TLS_REPORT_PRIVACY_CONTROLS = {
|
||||
"retention": (
|
||||
"TLS reports store aggregate session counts, reporting organization metadata, "
|
||||
"policy domains, and grouped TLS failure details."
|
||||
),
|
||||
"stored_fields": [
|
||||
"report id",
|
||||
"reporting organization",
|
||||
"contact info",
|
||||
"policy domain",
|
||||
"policy type",
|
||||
"report date range",
|
||||
"successful and failed session counts",
|
||||
"grouped result type and failed-session count",
|
||||
"sending MTA IP when supplied by the reporter",
|
||||
"receiving MX host/HELO/IP when supplied by the reporter",
|
||||
"failure reason code and additional grouped diagnostic text",
|
||||
],
|
||||
"not_stored": [
|
||||
"message bodies",
|
||||
"message subjects",
|
||||
"sender or recipient addresses",
|
||||
"recipient local-parts",
|
||||
"raw uploaded attachments",
|
||||
"mailbox credentials or source message identifiers",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def tls_report_exists(db: Session, report_id: str, policy_domain: str) -> bool:
|
||||
"""Return True when a TLS report policy entry already exists."""
|
||||
normalized_report_id = str(report_id or "").strip()
|
||||
normalized_domain = str(policy_domain or "").strip().lower().strip(".")
|
||||
if not normalized_report_id or not normalized_domain:
|
||||
return False
|
||||
return (
|
||||
db.query(TLSReport.id)
|
||||
.filter(
|
||||
TLSReport.report_id == normalized_report_id,
|
||||
TLSReport.policy_domain == normalized_domain,
|
||||
)
|
||||
.first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _domain_for_report(db: Session, domain_name: Optional[str]) -> Optional[Domain]:
|
||||
if not domain_name:
|
||||
return None
|
||||
normalized = domain_name.lower().strip(".")
|
||||
is_valid, _, error_code = validate_domain(normalized, check_dns=False)
|
||||
if not is_valid and error_code != DomainValidationError.DNS_RESOLUTION_FAILED:
|
||||
return None
|
||||
|
||||
domain = db.query(Domain).filter(Domain.name == normalized).first()
|
||||
if domain is None:
|
||||
domain = Domain(name=normalized)
|
||||
db.add(domain)
|
||||
db.flush()
|
||||
return domain
|
||||
|
||||
|
||||
def _save_policy_report(
|
||||
db: Session,
|
||||
parsed_report: Dict[str, Any],
|
||||
policy: Dict[str, Any],
|
||||
) -> tuple[Optional[TLSReport], bool]:
|
||||
report_id = str(parsed_report.get("report_id") or "").strip()
|
||||
policy_domain = str(policy.get("policy_domain") or "").strip().lower().strip(".")
|
||||
if not report_id or not policy_domain:
|
||||
return None, False
|
||||
|
||||
existing = (
|
||||
db.query(TLSReport)
|
||||
.filter(
|
||||
TLSReport.report_id == report_id,
|
||||
TLSReport.policy_domain == policy_domain,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
|
||||
domain = _domain_for_report(db, policy_domain)
|
||||
row = TLSReport(
|
||||
domain_id=domain.id if domain else None,
|
||||
report_id=report_id,
|
||||
org_name=parsed_report.get("org_name"),
|
||||
contact_info=parsed_report.get("contact_info"),
|
||||
policy_domain=policy_domain,
|
||||
policy_type=policy.get("policy_type"),
|
||||
begin_date=parsed_report.get("begin_date"),
|
||||
end_date=parsed_report.get("end_date"),
|
||||
total_successful_sessions=policy.get("total_successful_sessions") or 0,
|
||||
total_failure_sessions=policy.get("total_failure_sessions") or 0,
|
||||
raw_policy=json.dumps(policy.get("policy") or {}, sort_keys=True),
|
||||
)
|
||||
for failure in policy.get("failures") or []:
|
||||
row.failures.append(
|
||||
TLSReportFailure(
|
||||
result_type=failure.get("result_type") or "unknown",
|
||||
failed_session_count=failure.get("failed_session_count") or 0,
|
||||
sending_mta_ip=failure.get("sending_mta_ip") or None,
|
||||
receiving_mx_hostname=failure.get("receiving_mx_hostname") or None,
|
||||
receiving_mx_helo=failure.get("receiving_mx_helo") or None,
|
||||
receiving_ip=failure.get("receiving_ip") or None,
|
||||
failure_reason_code=failure.get("failure_reason_code") or None,
|
||||
additional_information=failure.get("additional_information") or None,
|
||||
)
|
||||
)
|
||||
|
||||
db.add(row)
|
||||
try:
|
||||
db.flush()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
existing = (
|
||||
db.query(TLSReport)
|
||||
.filter(
|
||||
TLSReport.report_id == report_id,
|
||||
TLSReport.policy_domain == policy_domain,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
raise
|
||||
return row, True
|
||||
|
||||
|
||||
def save_tls_report(db: Session, parsed_report: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Persist parsed TLS report policy entries.
|
||||
|
||||
One TLS-RPT JSON can carry multiple policy domains. Each policy is stored
|
||||
independently so partial duplicate imports can still add newly seen domains.
|
||||
The caller owns the transaction.
|
||||
"""
|
||||
rows: List[TLSReport] = []
|
||||
created = 0
|
||||
skipped = 0
|
||||
for policy in parsed_report.get("policies") or []:
|
||||
row, was_created = _save_policy_report(db, parsed_report, policy)
|
||||
if row is None:
|
||||
skipped += 1
|
||||
continue
|
||||
rows.append(row)
|
||||
if was_created:
|
||||
created += 1
|
||||
else:
|
||||
skipped += 1
|
||||
return {"rows": rows, "created": created, "skipped": skipped}
|
||||
|
||||
|
||||
def tls_report_to_dict(row: TLSReport) -> Dict[str, Any]:
|
||||
"""Convert a TLS report row to an API-safe dictionary."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"report_id": row.report_id,
|
||||
"domain": row.domain.name if row.domain else row.policy_domain,
|
||||
"org_name": row.org_name,
|
||||
"contact_info": row.contact_info,
|
||||
"policy_domain": row.policy_domain,
|
||||
"policy_type": row.policy_type,
|
||||
"begin_date": row.begin_date.isoformat() if row.begin_date else None,
|
||||
"end_date": row.end_date.isoformat() if row.end_date else None,
|
||||
"total_successful_sessions": row.total_successful_sessions,
|
||||
"total_failure_sessions": row.total_failure_sessions,
|
||||
"processed_at": row.processed_at.isoformat() if row.processed_at else None,
|
||||
"failures": [
|
||||
{
|
||||
"result_type": failure.result_type,
|
||||
"failed_session_count": failure.failed_session_count,
|
||||
"sending_mta_ip": failure.sending_mta_ip,
|
||||
"receiving_mx_hostname": failure.receiving_mx_hostname,
|
||||
"receiving_mx_helo": failure.receiving_mx_helo,
|
||||
"receiving_ip": failure.receiving_ip,
|
||||
"failure_reason_code": failure.failure_reason_code,
|
||||
"additional_information": failure.additional_information,
|
||||
}
|
||||
for failure in row.failures
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _row_day(row: TLSReport) -> str:
|
||||
basis = row.begin_date or row.end_date or row.processed_at or datetime.utcnow()
|
||||
return basis.date().isoformat()
|
||||
|
||||
|
||||
def _report_rows(
|
||||
db: Session,
|
||||
*,
|
||||
domain: Optional[str] = None,
|
||||
days: int = 30,
|
||||
) -> Iterable[TLSReport]:
|
||||
cutoff = datetime.utcnow() - timedelta(days=days)
|
||||
query = db.query(TLSReport).options(
|
||||
selectinload(TLSReport.domain), selectinload(TLSReport.failures)
|
||||
)
|
||||
query = query.filter(
|
||||
(TLSReport.begin_date >= cutoff)
|
||||
| (TLSReport.end_date >= cutoff)
|
||||
| (TLSReport.processed_at >= cutoff)
|
||||
)
|
||||
if domain:
|
||||
normalized = domain.lower().strip(".")
|
||||
query = query.outerjoin(Domain).filter(
|
||||
(Domain.name == normalized) | (TLSReport.policy_domain == normalized)
|
||||
)
|
||||
return query.order_by(TLSReport.begin_date.desc().nullslast(), TLSReport.id.desc()).all()
|
||||
|
||||
|
||||
def summarize_tls_reports(
|
||||
db: Session,
|
||||
*,
|
||||
domain: Optional[str] = None,
|
||||
days: int = 30,
|
||||
limit: int = 10,
|
||||
) -> Dict[str, Any]:
|
||||
"""Summarize TLS reports into trends and actionable failure groupings."""
|
||||
rows = list(_report_rows(db, domain=domain, days=days))
|
||||
|
||||
totals = {
|
||||
"reports": len(rows),
|
||||
"successful_sessions": sum(row.total_successful_sessions or 0 for row in rows),
|
||||
"failed_sessions": sum(row.total_failure_sessions or 0 for row in rows),
|
||||
}
|
||||
session_total = totals["successful_sessions"] + totals["failed_sessions"]
|
||||
totals["failure_rate"] = (totals["failed_sessions"] / session_total) if session_total else 0.0
|
||||
|
||||
trend_map: Dict[str, Dict[str, Any]] = defaultdict(
|
||||
lambda: {"date": "", "reports": 0, "successful_sessions": 0, "failed_sessions": 0}
|
||||
)
|
||||
domain_map: Dict[str, Dict[str, Any]] = defaultdict(
|
||||
lambda: {
|
||||
"domain": "",
|
||||
"reports": 0,
|
||||
"successful_sessions": 0,
|
||||
"failed_sessions": 0,
|
||||
"top_failure": None,
|
||||
}
|
||||
)
|
||||
failure_map: Dict[str, Dict[str, Any]] = defaultdict(
|
||||
lambda: {
|
||||
"result_type": "",
|
||||
"failed_sessions": 0,
|
||||
"reports": set(),
|
||||
"affected_domains": set(),
|
||||
"receiving_mx_hostnames": set(),
|
||||
"reason_codes": set(),
|
||||
}
|
||||
)
|
||||
|
||||
for row in rows:
|
||||
day = _row_day(row)
|
||||
trend = trend_map[day]
|
||||
trend["date"] = day
|
||||
trend["reports"] += 1
|
||||
trend["successful_sessions"] += row.total_successful_sessions or 0
|
||||
trend["failed_sessions"] += row.total_failure_sessions or 0
|
||||
|
||||
domain_summary = domain_map[row.policy_domain]
|
||||
domain_summary["domain"] = row.policy_domain
|
||||
domain_summary["reports"] += 1
|
||||
domain_summary["successful_sessions"] += row.total_successful_sessions or 0
|
||||
domain_summary["failed_sessions"] += row.total_failure_sessions or 0
|
||||
|
||||
top_for_row = None
|
||||
for failure in row.failures:
|
||||
result_type = failure.result_type or "unknown"
|
||||
item = failure_map[result_type]
|
||||
item["result_type"] = result_type
|
||||
item["failed_sessions"] += failure.failed_session_count or 0
|
||||
item["reports"].add(row.report_id)
|
||||
item["affected_domains"].add(row.policy_domain)
|
||||
if failure.receiving_mx_hostname:
|
||||
item["receiving_mx_hostnames"].add(failure.receiving_mx_hostname)
|
||||
if failure.failure_reason_code:
|
||||
item["reason_codes"].add(failure.failure_reason_code)
|
||||
if (
|
||||
top_for_row is None
|
||||
or (failure.failed_session_count or 0) > top_for_row.failed_session_count
|
||||
):
|
||||
top_for_row = failure
|
||||
if top_for_row is not None:
|
||||
domain_summary["top_failure"] = top_for_row.result_type
|
||||
|
||||
trends = [trend_map[key] for key in sorted(trend_map)]
|
||||
affected_domains = []
|
||||
for item in domain_map.values():
|
||||
domain_sessions = item["successful_sessions"] + item["failed_sessions"]
|
||||
item["failure_rate"] = item["failed_sessions"] / domain_sessions if domain_sessions else 0.0
|
||||
affected_domains.append(item)
|
||||
|
||||
top_failures = []
|
||||
for item in failure_map.values():
|
||||
top_failures.append(
|
||||
{
|
||||
"result_type": item["result_type"],
|
||||
"failed_sessions": item["failed_sessions"],
|
||||
"report_count": len(item["reports"]),
|
||||
"affected_domains": sorted(item["affected_domains"]),
|
||||
"receiving_mx_hostnames": sorted(item["receiving_mx_hostnames"])[:5],
|
||||
"reason_codes": sorted(item["reason_codes"])[:5],
|
||||
}
|
||||
)
|
||||
|
||||
top_failures.sort(key=lambda item: item["failed_sessions"], reverse=True)
|
||||
affected_domains.sort(key=lambda item: item["failed_sessions"], reverse=True)
|
||||
|
||||
return {
|
||||
"domain": domain,
|
||||
"days": days,
|
||||
"totals": totals,
|
||||
"trends": trends,
|
||||
"top_failures": top_failures[:limit],
|
||||
"affected_domains": affected_domains[:limit],
|
||||
"privacy": TLS_REPORT_PRIVACY_CONTROLS,
|
||||
}
|
||||
Reference in New Issue
Block a user