Christian Krakau-Louis
237 lines
7.8 KiB
Python
237 lines
7.8 KiB
Python
import json
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from sqlalchemy.orm import Session, selectinload
|
|
|
|
from app.models.domain import Domain
|
|
from app.models.report import DMARCReport, ReportRecord
|
|
from app.services.report_store import ReportStore
|
|
|
|
|
|
def _parse_timestamp(value: Any) -> int:
|
|
"""Return a Unix timestamp from an int-like or ISO date value."""
|
|
if value in (None, ""):
|
|
return 0
|
|
if isinstance(value, (int, float)):
|
|
return int(value)
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
try:
|
|
return int(datetime.fromisoformat(str(value)).timestamp())
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _iso_from_timestamp(value: int) -> str:
|
|
if not value:
|
|
return ""
|
|
return datetime.fromtimestamp(value).isoformat()
|
|
|
|
|
|
def _loads_json_list(value: Optional[str]) -> Optional[List[Dict[str, Any]]]:
|
|
if not value:
|
|
return None
|
|
try:
|
|
decoded = json.loads(value)
|
|
except (json.JSONDecodeError, TypeError):
|
|
return None
|
|
return decoded if isinstance(decoded, list) else None
|
|
|
|
|
|
def _policy_parts(report: Dict[str, Any]) -> Dict[str, Any]:
|
|
policy = report.get("policy") or {}
|
|
if isinstance(policy, str):
|
|
return {"p": policy, "sp": "", "pct": "100"}
|
|
if not isinstance(policy, dict):
|
|
return {"p": "none", "sp": "", "pct": "100"}
|
|
return {
|
|
"p": policy.get("p", "none"),
|
|
"sp": policy.get("sp", ""),
|
|
"pct": str(policy.get("pct", "100")),
|
|
"adkim": policy.get("adkim") or report.get("adkim"),
|
|
"aspf": policy.get("aspf") or report.get("aspf"),
|
|
}
|
|
|
|
|
|
def report_exists(db: Session, domain_name: str, report_id: str) -> bool:
|
|
"""Return True when the domain/report ID pair is already persisted."""
|
|
if not report_id:
|
|
return False
|
|
return (
|
|
db.query(DMARCReport.id)
|
|
.join(Domain, DMARCReport.domain_id == Domain.id)
|
|
.filter(Domain.name == domain_name, DMARCReport.report_id == report_id)
|
|
.first()
|
|
is not None
|
|
)
|
|
|
|
|
|
def save_parsed_report(db: Session, report: Dict[str, Any]) -> tuple[DMARCReport, bool]:
|
|
"""Persist a parsed DMARC report and its records.
|
|
|
|
Returns ``(row, created)``. The caller owns the transaction and should
|
|
commit after all related work has completed.
|
|
"""
|
|
domain_name = report.get("domain") or "unknown"
|
|
report_id = report.get("report_id") or ""
|
|
policy = _policy_parts(report)
|
|
|
|
domain = db.query(Domain).filter(Domain.name == domain_name).first()
|
|
if domain is None:
|
|
domain = Domain(name=domain_name, dmarc_policy=policy["p"])
|
|
db.add(domain)
|
|
db.flush()
|
|
elif policy.get("p"):
|
|
domain.dmarc_policy = policy["p"]
|
|
|
|
existing = (
|
|
db.query(DMARCReport)
|
|
.filter(DMARCReport.domain_id == domain.id, DMARCReport.report_id == report_id)
|
|
.first()
|
|
)
|
|
if existing is not None:
|
|
return existing, False
|
|
|
|
begin_ts = _parse_timestamp(report.get("begin_timestamp") or report.get("begin_date"))
|
|
end_ts = _parse_timestamp(report.get("end_timestamp") or report.get("end_date"))
|
|
pct = _parse_timestamp(policy.get("pct")) or 100
|
|
|
|
db_report = DMARCReport(
|
|
domain_id=domain.id,
|
|
report_id=report_id,
|
|
org_name=report.get("org_name") or "",
|
|
begin_date=begin_ts,
|
|
end_date=end_ts,
|
|
source_email=report.get("email") or report.get("source_email"),
|
|
policy=policy["p"],
|
|
subdomain_policy=policy.get("sp") or None,
|
|
adkim=policy.get("adkim") or None,
|
|
aspf=policy.get("aspf") or None,
|
|
percentage=pct,
|
|
)
|
|
db.add(db_report)
|
|
db.flush()
|
|
|
|
for record in report.get("records", []):
|
|
db.add(
|
|
ReportRecord(
|
|
report_id=db_report.id,
|
|
source_ip=record.get("source_ip") or "unknown",
|
|
count=int(record.get("count") or 0),
|
|
disposition=record.get("disposition") or "none",
|
|
dkim=record.get("dkim_result") or record.get("dkim") or "unknown",
|
|
spf=record.get("spf_result") or record.get("spf") or "unknown",
|
|
header_from=record.get("header_from"),
|
|
envelope_from=record.get("envelope_from"),
|
|
dkim_auth_details=(
|
|
json.dumps(record.get("dkim")) if isinstance(record.get("dkim"), list) else None
|
|
),
|
|
spf_auth_details=(
|
|
json.dumps(record.get("spf")) if isinstance(record.get("spf"), list) else None
|
|
),
|
|
)
|
|
)
|
|
|
|
return db_report, True
|
|
|
|
|
|
def persisted_report_to_dict(report: DMARCReport) -> Dict[str, Any]:
|
|
"""Convert persisted report rows into the parsed-report shape used by the UI."""
|
|
records: List[Dict[str, Any]] = []
|
|
total_count = 0
|
|
passed_count = 0
|
|
|
|
for record in report.records:
|
|
count = int(record.count or 0)
|
|
dkim_result = record.dkim or "unknown"
|
|
spf_result = record.spf or "unknown"
|
|
total_count += count
|
|
if dkim_result == "pass" or spf_result == "pass":
|
|
passed_count += count
|
|
|
|
records.append(
|
|
{
|
|
"source_ip": record.source_ip,
|
|
"count": count,
|
|
"disposition": record.disposition or "none",
|
|
"dkim_result": dkim_result,
|
|
"spf_result": spf_result,
|
|
"header_from": record.header_from or "",
|
|
"dkim": _loads_json_list(record.dkim_auth_details) or [],
|
|
"spf": _loads_json_list(record.spf_auth_details) or [],
|
|
}
|
|
)
|
|
|
|
failed_count = total_count - passed_count
|
|
pass_rate = round(passed_count / total_count * 100, 1) if total_count > 0 else 0.0
|
|
return {
|
|
"domain": report.domain.name if report.domain else "unknown",
|
|
"report_id": report.report_id,
|
|
"org_name": report.org_name,
|
|
"email": report.source_email or "",
|
|
"begin_date": _iso_from_timestamp(report.begin_date),
|
|
"end_date": _iso_from_timestamp(report.end_date),
|
|
"begin_timestamp": report.begin_date,
|
|
"end_timestamp": report.end_date,
|
|
"policy": {
|
|
"p": report.policy or "none",
|
|
"sp": report.subdomain_policy or "",
|
|
"pct": str(report.percentage or 100),
|
|
},
|
|
"records": records,
|
|
"summary": {
|
|
"total_count": total_count,
|
|
"passed_count": passed_count,
|
|
"failed_count": failed_count,
|
|
"pass_rate": pass_rate,
|
|
},
|
|
}
|
|
|
|
|
|
def hydrate_report_store_from_db(db: Session, store: Optional[ReportStore] = None) -> int:
|
|
"""Load persisted reports into ReportStore when the database has report rows."""
|
|
report_count = db.query(DMARCReport.id).count()
|
|
if report_count == 0:
|
|
return 0
|
|
|
|
store = store or ReportStore.get_instance()
|
|
store.clear()
|
|
reports = (
|
|
db.query(DMARCReport)
|
|
.options(
|
|
selectinload(DMARCReport.domain),
|
|
selectinload(DMARCReport.records),
|
|
)
|
|
.order_by(DMARCReport.end_date.desc())
|
|
.all()
|
|
)
|
|
for report in reports:
|
|
store.add_report(persisted_report_to_dict(report))
|
|
return len(reports)
|
|
|
|
|
|
def delete_persisted_report(db: Session, domain_name: str, report_id: str) -> bool:
|
|
"""Delete a persisted report by domain/report ID."""
|
|
report = (
|
|
db.query(DMARCReport)
|
|
.join(Domain, DMARCReport.domain_id == Domain.id)
|
|
.filter(Domain.name == domain_name, DMARCReport.report_id == report_id)
|
|
.first()
|
|
)
|
|
if report is None:
|
|
return False
|
|
db.delete(report)
|
|
return True
|
|
|
|
|
|
def delete_persisted_domain(db: Session, domain_name: str) -> bool:
|
|
"""Delete a domain row and cascaded report data."""
|
|
domain = db.query(Domain).filter(Domain.name == domain_name).first()
|
|
if domain is None:
|
|
return False
|
|
db.delete(domain)
|
|
return True
|