Christian Krakau-Louis
559 lines
18 KiB
Python
559 lines
18 KiB
Python
import logging
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.database import get_db
|
|
from app.services.dmarc_parser import DMARCParser
|
|
from app.services.report_persistence import (
|
|
delete_persisted_report,
|
|
hydrate_report_store_from_db,
|
|
report_exists,
|
|
save_parsed_report,
|
|
)
|
|
from app.services.report_store import ReportStore
|
|
from app.utils.domain_validator import DomainValidationError, validate_domain
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Try to import python-magic for MIME type detection
|
|
try:
|
|
import magic
|
|
|
|
HAS_MAGIC = True
|
|
except ImportError:
|
|
HAS_MAGIC = False
|
|
logger.warning("python-magic not installed. MIME type validation will be skipped.")
|
|
|
|
router = APIRouter()
|
|
|
|
# Security: Allowed MIME types for DMARC report uploads
|
|
ALLOWED_MIME_TYPES = {
|
|
"text/xml",
|
|
"application/xml",
|
|
"application/zip",
|
|
"application/x-zip-compressed",
|
|
"application/gzip",
|
|
"application/x-gzip",
|
|
"application/octet-stream", # Sometimes zip/gzip are detected as this
|
|
}
|
|
|
|
# Security: Allowed file extensions
|
|
ALLOWED_EXTENSIONS = {".xml", ".zip", ".gz", ".gzip"}
|
|
|
|
|
|
def _validate_mime_type(file_content: bytes) -> None:
|
|
"""Validate the MIME type of the uploaded file using python-magic.
|
|
|
|
No-ops silently when python-magic is unavailable.
|
|
Raises HTTPException on a disallowed MIME type.
|
|
"""
|
|
if not HAS_MAGIC:
|
|
logger.debug("MIME type validation skipped (python-magic not available)")
|
|
return
|
|
try:
|
|
mime_type = magic.from_buffer(file_content, mime=True)
|
|
if mime_type not in ALLOWED_MIME_TYPES:
|
|
logger.warning("Rejected file with MIME type: %s", mime_type)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Invalid file type. File must be XML, ZIP, or GZIP format.",
|
|
)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e: # pylint: disable=broad-exception-caught
|
|
# If magic fails, log but continue (fallback to extension check)
|
|
logger.warning("MIME type detection failed: %s", str(e))
|
|
|
|
|
|
def _validate_upload_file(file: UploadFile, file_content: bytes) -> None:
|
|
"""Run all pre-parse validation checks on an uploaded file.
|
|
|
|
Raises HTTPException for any validation failure.
|
|
"""
|
|
# Security: Validate filename is provided
|
|
if not file.filename:
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Filename is required")
|
|
|
|
# Security: Validate file extension
|
|
file_ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
|
|
if file_ext not in ALLOWED_EXTENSIONS:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"Invalid file type. Allowed types: {', '.join(ALLOWED_EXTENSIONS)}",
|
|
)
|
|
|
|
# Security: Validate file is not empty
|
|
if len(file_content) == 0:
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="File is empty")
|
|
|
|
# Security: Validate MIME type (if python-magic is available)
|
|
_validate_mime_type(file_content)
|
|
|
|
|
|
def _handle_upload_value_error(filename: str, error_message: str) -> None:
|
|
"""Translate a parser ValueError into a sanitized HTTPException.
|
|
|
|
Always raises — never returns.
|
|
"""
|
|
logger.error("ValueError processing report %s: %s", filename, error_message)
|
|
if "too large" in error_message.lower():
|
|
raise HTTPException(
|
|
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File too large"
|
|
)
|
|
if "zip bomb" in error_message.lower():
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid archive file")
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid report format")
|
|
|
|
|
|
class UploadResponse(BaseModel):
|
|
"""Response model for report upload"""
|
|
|
|
success: bool
|
|
domain: str
|
|
message: str
|
|
processed_records: int = 0 # Added this field to track processed records
|
|
|
|
|
|
class DomainSummary(BaseModel):
|
|
"""Domain summary response model"""
|
|
|
|
domain: str
|
|
total_count: int
|
|
passed_count: int
|
|
failed_count: int
|
|
reports_processed: int
|
|
compliance_rate: float
|
|
|
|
|
|
class ReportSummary(BaseModel):
|
|
"""DMARC report summary model"""
|
|
|
|
report_id: str
|
|
org_name: str
|
|
begin_date: str
|
|
end_date: str
|
|
total_count: int
|
|
passed_count: int
|
|
failed_count: int
|
|
|
|
|
|
class AllReportsItem(BaseModel):
|
|
"""Single report item for the cross-domain reports list"""
|
|
|
|
report_id: str
|
|
domain: str
|
|
org_name: str
|
|
begin_date: str
|
|
end_date: str
|
|
total_count: int
|
|
passed_count: int
|
|
failed_count: int
|
|
pass_rate: float
|
|
|
|
|
|
class PaginatedReportResponse(BaseModel):
|
|
"""Paginated reports response model"""
|
|
|
|
total: int
|
|
page: int
|
|
page_size: int
|
|
total_pages: int
|
|
reports: List[ReportSummary]
|
|
|
|
|
|
@router.post("/upload", response_model=UploadResponse)
|
|
async def upload_report(file: UploadFile = File(...), db: Session = Depends(get_db)):
|
|
"""
|
|
Upload and process a DMARC aggregate report file (XML, ZIP, or GZIP)
|
|
|
|
Security:
|
|
- File type validation (extension and MIME type)
|
|
- File size limits enforced in parser
|
|
- Zip bomb protection
|
|
- Sanitized error messages
|
|
"""
|
|
try:
|
|
# Read content first so validators can inspect it
|
|
file_content = await file.read()
|
|
_validate_upload_file(file, file_content)
|
|
|
|
# Parse the report
|
|
parser = DMARCParser()
|
|
report = parser.parse_file(file_content, file.filename)
|
|
|
|
# Security: Validate domain from report
|
|
domain = report.get("domain", "")
|
|
if not domain:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Report does not contain a valid domain",
|
|
)
|
|
|
|
# Validate domain format (not DNS resolution to avoid external calls)
|
|
is_valid, error_msg, error_code = validate_domain(domain, check_dns=False)
|
|
if not is_valid and error_code != DomainValidationError.DNS_RESOLUTION_FAILED:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"Invalid domain in report: {error_msg}",
|
|
)
|
|
|
|
# Check for duplicate report before storing
|
|
store = ReportStore.get_instance()
|
|
report_id = report.get("report_id", "")
|
|
if report_id and (
|
|
store.has_report(domain, report_id) or report_exists(db, domain, report_id)
|
|
):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail=(
|
|
f"Report '{report_id}' for domain '{domain}' has already been uploaded. "
|
|
"Duplicate reports are not stored to keep statistics accurate."
|
|
),
|
|
)
|
|
|
|
# Store the report
|
|
save_parsed_report(db, report)
|
|
db.commit()
|
|
store.add_report(report)
|
|
|
|
processed_records = report.get("summary", {}).get("total_count", 0)
|
|
|
|
return UploadResponse(
|
|
success=True,
|
|
domain=domain,
|
|
message=f"Report processed successfully for domain {domain}",
|
|
processed_records=processed_records,
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except ValueError as e:
|
|
# Security: Sanitize error messages from parser
|
|
_handle_upload_value_error(file.filename, str(e))
|
|
except Exception as e: # pylint: disable=broad-exception-caught
|
|
# Security: Don't expose internal errors to client
|
|
logger.error("Unexpected error processing report %s: %s", file.filename, str(e))
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail="Error processing report. Please contact support if this persists.",
|
|
) from e
|
|
|
|
|
|
@router.get("", response_model=List[AllReportsItem])
|
|
async def get_all_reports(db: Session = Depends(get_db)):
|
|
"""
|
|
Get all DMARC reports across all domains, sorted by end_date descending.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
domains = store.get_domains()
|
|
|
|
all_reports: List[AllReportsItem] = []
|
|
for domain in domains:
|
|
domain_reports = store.get_domain_reports(domain)
|
|
for report in domain_reports:
|
|
summary = report.get("summary", {})
|
|
total = summary.get("total_count", 0)
|
|
passed = summary.get("passed_count", 0)
|
|
pass_rate = round(passed / total * 100, 1) if total > 0 else 0.0
|
|
all_reports.append(
|
|
AllReportsItem(
|
|
report_id=report.get("report_id", ""),
|
|
domain=domain,
|
|
org_name=report.get("org_name", ""),
|
|
begin_date=str(report.get("begin_date", "")),
|
|
end_date=str(report.get("end_date", "")),
|
|
total_count=total,
|
|
passed_count=passed,
|
|
failed_count=summary.get("failed_count", 0),
|
|
pass_rate=pass_rate,
|
|
)
|
|
)
|
|
|
|
# end_date is stored in ISO 8601 format (YYYY-MM-DDTHH:MM:SS), so lexicographic
|
|
# sorting produces correct chronological order.
|
|
all_reports.sort(key=lambda r: r.end_date, reverse=True)
|
|
return all_reports
|
|
|
|
|
|
@router.get("/domains", response_model=List[str])
|
|
async def get_domains(db: Session = Depends(get_db)):
|
|
"""
|
|
Get list of all domains with reports
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
return store.get_domains()
|
|
|
|
|
|
@router.get("/domain/{domain}/summary", response_model=DomainSummary)
|
|
async def get_domain_summary(domain: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Get summary statistics for a specific domain
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
summary = store.get_domain_summary(domain)
|
|
|
|
if not summary:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
|
)
|
|
|
|
return DomainSummary(domain=domain, **summary)
|
|
|
|
|
|
@router.get("/summary", response_model=List[DomainSummary])
|
|
async def get_all_summaries(db: Session = Depends(get_db)):
|
|
"""
|
|
Get summary statistics for all domains
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
all_summaries = store.get_all_domain_summaries()
|
|
|
|
return [DomainSummary(domain=domain, **summary) for domain, summary in all_summaries.items()]
|
|
|
|
|
|
@router.get("/domain/{domain}/reports", response_model=List[ReportSummary])
|
|
async def get_domain_reports(domain: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Get all reports for a specific domain
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
reports = store.get_domain_reports(domain)
|
|
|
|
if not reports:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
|
)
|
|
|
|
return [
|
|
ReportSummary(
|
|
report_id=report.get("report_id", ""),
|
|
org_name=report.get("org_name", ""),
|
|
begin_date=report.get("begin_date", ""),
|
|
end_date=report.get("end_date", ""),
|
|
total_count=report.get("summary", {}).get("total_count", 0),
|
|
passed_count=report.get("summary", {}).get("passed_count", 0),
|
|
failed_count=report.get("summary", {}).get("failed_count", 0),
|
|
)
|
|
for report in reports
|
|
]
|
|
|
|
|
|
@router.get("/domain/{domain}/reports/paginated", response_model=PaginatedReportResponse)
|
|
async def get_domain_reports_paginated(
|
|
domain: str,
|
|
page: int = 1,
|
|
page_size: int = 10,
|
|
sort_by: str = "end_date",
|
|
sort_order: str = "desc",
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""
|
|
Get paginated reports for a specific domain with sorting options
|
|
|
|
Args:
|
|
domain: Domain name
|
|
page: Page number (1-based)
|
|
page_size: Number of reports per page
|
|
sort_by: Field to sort by (report_id, org_name, begin_date, end_date, total_count)
|
|
sort_order: Sort order (asc or desc)
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
all_reports = store.get_domain_reports(domain)
|
|
|
|
if not all_reports:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
|
)
|
|
|
|
# Apply sorting
|
|
valid_sort_fields = ["report_id", "org_name", "begin_date", "end_date", "total_count"]
|
|
sort_field = sort_by if sort_by in valid_sort_fields else "end_date"
|
|
|
|
if sort_field == "total_count":
|
|
all_reports.sort(
|
|
key=lambda r: r.get("summary", {}).get("total_count", 0),
|
|
reverse=sort_order == "desc",
|
|
)
|
|
else:
|
|
all_reports.sort(key=lambda r: r.get(sort_field, ""), reverse=sort_order == "desc")
|
|
|
|
# Apply pagination
|
|
total = len(all_reports)
|
|
total_pages = (total + page_size - 1) // page_size
|
|
start_idx = (page - 1) * page_size
|
|
end_idx = start_idx + page_size
|
|
paginated_reports = all_reports[start_idx:end_idx]
|
|
|
|
# Format reports
|
|
report_entries = [
|
|
ReportSummary(
|
|
report_id=report.get("report_id", ""),
|
|
org_name=report.get("org_name", ""),
|
|
begin_date=report.get("begin_date", ""),
|
|
end_date=report.get("end_date", ""),
|
|
total_count=report.get("summary", {}).get("total_count", 0),
|
|
passed_count=report.get("summary", {}).get("passed_count", 0),
|
|
failed_count=report.get("summary", {}).get("failed_count", 0),
|
|
)
|
|
for report in paginated_reports
|
|
]
|
|
|
|
return PaginatedReportResponse(
|
|
total=total, page=page, page_size=page_size, total_pages=total_pages, reports=report_entries
|
|
)
|
|
|
|
|
|
class DeleteReportResponse(BaseModel):
|
|
"""Response model for report deletion"""
|
|
|
|
success: bool
|
|
message: str
|
|
|
|
|
|
@router.delete(
|
|
"/domain/{domain}/reports/{report_id}",
|
|
response_model=DeleteReportResponse,
|
|
)
|
|
async def delete_report(domain: str, report_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Delete a single DMARC report for a domain.
|
|
|
|
Removes the report from the store and recomputes all domain statistics so
|
|
that aggregated numbers remain accurate after deletion.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
deleted_from_db = delete_persisted_report(db, domain, report_id)
|
|
if deleted_from_db:
|
|
db.commit()
|
|
deleted = store.delete_report(domain, report_id) or deleted_from_db
|
|
|
|
if not deleted:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Report '{report_id}' not found for domain '{domain}'.",
|
|
)
|
|
|
|
return DeleteReportResponse(
|
|
success=True,
|
|
message=f"Report '{report_id}' for domain '{domain}' deleted successfully.",
|
|
)
|
|
|
|
|
|
class ReportRecordDetail(BaseModel):
|
|
"""Detailed record from a DMARC report"""
|
|
|
|
source_ip: str
|
|
count: int
|
|
disposition: str
|
|
dkim_result: str
|
|
spf_result: str
|
|
header_from: str
|
|
spf: Optional[List[Dict[str, Any]]] = None
|
|
dkim: Optional[List[Dict[str, Any]]] = None
|
|
|
|
|
|
class ReportPolicyDetail(BaseModel):
|
|
"""Published policy from a DMARC report"""
|
|
|
|
p: str
|
|
sp: str = ""
|
|
pct: str = "100"
|
|
|
|
|
|
class ReportSummaryDetail(BaseModel):
|
|
"""Summary statistics for a DMARC report"""
|
|
|
|
total_count: int
|
|
passed_count: int
|
|
failed_count: int
|
|
pass_rate: float
|
|
|
|
|
|
class ReportDetail(BaseModel):
|
|
"""Full detail of a single DMARC report"""
|
|
|
|
report_id: str
|
|
org_name: str
|
|
email: str
|
|
domain: str
|
|
begin_date: str
|
|
end_date: str
|
|
begin_timestamp: int
|
|
end_timestamp: int
|
|
policy: ReportPolicyDetail
|
|
records: List[ReportRecordDetail]
|
|
summary: ReportSummaryDetail
|
|
|
|
|
|
@router.get("/{report_id}", response_model=ReportDetail)
|
|
async def get_report_by_id(report_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Get full details for a single DMARC report by its report ID.
|
|
"""
|
|
store = ReportStore.get_instance()
|
|
hydrate_report_store_from_db(db, store)
|
|
report = store.get_report_by_id(report_id)
|
|
|
|
if report is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Report '{report_id}' not found.",
|
|
)
|
|
|
|
# Normalize the policy field
|
|
policy_val = report.get("policy", {})
|
|
if isinstance(policy_val, str):
|
|
policy_val = {"p": policy_val, "sp": "", "pct": "100"}
|
|
policy_detail = ReportPolicyDetail(
|
|
p=policy_val.get("p", "none"),
|
|
sp=policy_val.get("sp", ""),
|
|
pct=str(policy_val.get("pct", "100")),
|
|
)
|
|
|
|
# Normalize records
|
|
record_details = []
|
|
for rec in report.get("records", []):
|
|
record_details.append(
|
|
ReportRecordDetail(
|
|
source_ip=rec.get("source_ip", ""),
|
|
count=rec.get("count", 0),
|
|
disposition=rec.get("disposition", "none"),
|
|
dkim_result=rec.get("dkim_result", ""),
|
|
spf_result=rec.get("spf_result", ""),
|
|
header_from=rec.get("header_from", ""),
|
|
spf=rec.get("spf") if isinstance(rec.get("spf"), list) else None,
|
|
dkim=rec.get("dkim") if isinstance(rec.get("dkim"), list) else None,
|
|
)
|
|
)
|
|
|
|
raw_summary = report.get("summary", {})
|
|
summary_detail = ReportSummaryDetail(
|
|
total_count=raw_summary.get("total_count", 0),
|
|
passed_count=raw_summary.get("passed_count", 0),
|
|
failed_count=raw_summary.get("failed_count", 0),
|
|
pass_rate=raw_summary.get("pass_rate", 0.0),
|
|
)
|
|
|
|
return ReportDetail(
|
|
report_id=report.get("report_id", ""),
|
|
org_name=report.get("org_name", ""),
|
|
email=report.get("email", ""),
|
|
domain=report.get("domain", ""),
|
|
begin_date=str(report.get("begin_date", "")),
|
|
end_date=str(report.get("end_date", "")),
|
|
begin_timestamp=report.get("begin_timestamp", 0),
|
|
end_timestamp=report.get("end_timestamp", 0),
|
|
policy=policy_detail,
|
|
records=record_details,
|
|
summary=summary_detail,
|
|
)
|