Fix code formatting and linting issues
- Auto-format all Python files with black and isort - Remove unused imports with autoflake - Fix flake8 issues (missing newlines, blank lines, etc.) - Fix nonlocal/global scope issues in main.py - Fix security.py import order (E402) - Remove f-string without placeholders - Add nosec comment for intentional exception handling - Fix test imports to match refactored DMARCParser API Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -1,33 +1,41 @@
|
||||
from typing import List, Optional, Dict, Any
|
||||
from datetime import datetime, timedelta
|
||||
from fastapi import APIRouter, HTTPException, status, Path, Query
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.services.report_store import ReportStore
|
||||
from fastapi import APIRouter, HTTPException, Path, Query, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class DomainBase(BaseModel):
|
||||
"""Base Domain schema"""
|
||||
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
policy: Optional[str] = None
|
||||
|
||||
|
||||
class DomainResponse(DomainBase):
|
||||
"""Domain response schema"""
|
||||
|
||||
reports_count: int = 0
|
||||
emails_count: int = 0
|
||||
compliance_rate: float = 0.0
|
||||
|
||||
|
||||
class DomainStatsResponse(BaseModel):
|
||||
"""Domain statistics for the domain details page"""
|
||||
|
||||
complianceRate: float
|
||||
totalEmails: int
|
||||
failedEmails: int
|
||||
reportCount: int
|
||||
|
||||
|
||||
class DNSRecordResponse(BaseModel):
|
||||
"""DNS record information for a domain"""
|
||||
|
||||
dmarc: bool
|
||||
dmarcRecord: Optional[str] = None
|
||||
spf: bool
|
||||
@@ -35,13 +43,17 @@ class DNSRecordResponse(BaseModel):
|
||||
dkim: bool
|
||||
dkimSelectors: Optional[str] = None
|
||||
|
||||
|
||||
class TimelinePoint(BaseModel):
|
||||
"""Data point for compliance timeline"""
|
||||
|
||||
date: str
|
||||
compliance_rate: float
|
||||
|
||||
|
||||
class ReportEntry(BaseModel):
|
||||
"""Summary of a DMARC report"""
|
||||
|
||||
id: str
|
||||
org_name: str
|
||||
begin_date: int
|
||||
@@ -50,8 +62,10 @@ class ReportEntry(BaseModel):
|
||||
pass_rate: float
|
||||
policy: str
|
||||
|
||||
|
||||
class SourceEntry(BaseModel):
|
||||
"""Summary of a sending source"""
|
||||
|
||||
ip: str
|
||||
count: int
|
||||
spf: str
|
||||
@@ -59,23 +73,30 @@ class SourceEntry(BaseModel):
|
||||
dmarc: str
|
||||
disposition: str
|
||||
|
||||
|
||||
class DomainReportsResponse(BaseModel):
|
||||
"""Domain reports with compliance timeline"""
|
||||
|
||||
reports: List[ReportEntry]
|
||||
compliance_timeline: List[TimelinePoint]
|
||||
|
||||
|
||||
class DomainSourcesResponse(BaseModel):
|
||||
"""Domain sending sources"""
|
||||
|
||||
sources: List[SourceEntry]
|
||||
|
||||
|
||||
class DomainSummaryResponse(BaseModel):
|
||||
"""Domain summary for dashboard"""
|
||||
|
||||
total_domains: int
|
||||
total_emails: int
|
||||
overall_pass_rate: float
|
||||
reports_processed: int
|
||||
domains: List[Dict[str, Any]]
|
||||
|
||||
|
||||
@router.get("/summary", response_model=DomainSummaryResponse)
|
||||
async def get_domains_summary():
|
||||
"""
|
||||
@@ -84,45 +105,48 @@ async def get_domains_summary():
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
summaries = store.get_all_domain_summaries()
|
||||
|
||||
|
||||
# Calculate overall statistics
|
||||
total_domains = len(domains)
|
||||
total_emails = 0
|
||||
total_passed = 0
|
||||
total_reports = 0
|
||||
|
||||
|
||||
domains_list = []
|
||||
|
||||
|
||||
for domain_name in domains:
|
||||
summary = summaries.get(domain_name, {})
|
||||
total_emails += summary.get("total_count", 0)
|
||||
total_passed += summary.get("passed_count", 0)
|
||||
total_reports += summary.get("reports_processed", 0)
|
||||
|
||||
|
||||
# Format domain data for frontend
|
||||
domains_list.append({
|
||||
"id": domain_name, # Using the domain name as ID for now
|
||||
"domain_name": domain_name,
|
||||
"total_emails": summary.get("total_count", 0),
|
||||
"passed_count": summary.get("passed_count", 0),
|
||||
"failed_count": summary.get("failed_count", 0),
|
||||
"pass_rate": summary.get("compliance_rate", 0),
|
||||
"report_count": summary.get("reports_processed", 0)
|
||||
})
|
||||
|
||||
domains_list.append(
|
||||
{
|
||||
"id": domain_name, # Using the domain name as ID for now
|
||||
"domain_name": domain_name,
|
||||
"total_emails": summary.get("total_count", 0),
|
||||
"passed_count": summary.get("passed_count", 0),
|
||||
"failed_count": summary.get("failed_count", 0),
|
||||
"pass_rate": summary.get("compliance_rate", 0),
|
||||
"report_count": summary.get("reports_processed", 0),
|
||||
}
|
||||
)
|
||||
|
||||
# Calculate overall pass rate
|
||||
overall_pass_rate = 0
|
||||
if total_emails > 0:
|
||||
overall_pass_rate = round((total_passed / total_emails) * 100, 1)
|
||||
|
||||
|
||||
return DomainSummaryResponse(
|
||||
total_domains=total_domains,
|
||||
total_emails=total_emails,
|
||||
overall_pass_rate=overall_pass_rate,
|
||||
reports_processed=total_reports,
|
||||
domains=domains_list
|
||||
domains=domains_list,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/domains", response_model=List[DomainResponse])
|
||||
async def read_domains():
|
||||
"""
|
||||
@@ -132,7 +156,7 @@ async def read_domains():
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
summaries = store.get_all_domain_summaries()
|
||||
|
||||
|
||||
result = []
|
||||
for domain_name in domains:
|
||||
summary = summaries.get(domain_name, {})
|
||||
@@ -141,12 +165,13 @@ async def read_domains():
|
||||
policy=summary.get("policy", "unknown"),
|
||||
reports_count=summary.get("reports_processed", 0),
|
||||
emails_count=summary.get("total_count", 0),
|
||||
compliance_rate=summary.get("compliance_rate", 0.0)
|
||||
compliance_rate=summary.get("compliance_rate", 0.0),
|
||||
)
|
||||
result.append(domain_response)
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/domains/{domain_name}", response_model=DomainResponse)
|
||||
async def read_domain(domain_name: str):
|
||||
"""
|
||||
@@ -154,25 +179,27 @@ async def read_domain(domain_name: str):
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
if domain_name not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
summary = store.get_domain_summary(domain_name)
|
||||
|
||||
|
||||
return DomainResponse(
|
||||
name=domain_name,
|
||||
policy=summary.get("policy", "unknown"),
|
||||
reports_count=summary.get("reports_processed", 0),
|
||||
emails_count=summary.get("total_count", 0),
|
||||
compliance_rate=summary.get("compliance_rate", 0.0)
|
||||
compliance_rate=summary.get("compliance_rate", 0.0),
|
||||
)
|
||||
|
||||
|
||||
# New endpoints for domain details page
|
||||
|
||||
|
||||
@router.get("/{domain_id}/stats", response_model=DomainStatsResponse)
|
||||
async def get_domain_stats(domain_id: str = Path(..., title="The domain ID or name")):
|
||||
"""
|
||||
@@ -180,43 +207,44 @@ async def get_domain_stats(domain_id: str = Path(..., title="The domain ID or na
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
# For Milestone 1, domain_id is simply the domain name
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
summary = store.get_domain_summary(domain_id)
|
||||
total_count = summary.get("total_count", 0)
|
||||
passed_count = summary.get("passed_count", 0)
|
||||
failed_count = total_count - passed_count
|
||||
compliance_rate = summary.get("compliance_rate", 0.0)
|
||||
reports_processed = summary.get("reports_processed", 0)
|
||||
|
||||
|
||||
return DomainStatsResponse(
|
||||
complianceRate=compliance_rate,
|
||||
totalEmails=total_count,
|
||||
failedEmails=failed_count,
|
||||
reportCount=reports_processed
|
||||
reportCount=reports_processed,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{domain_id}/dns", response_model=DNSRecordResponse)
|
||||
async def get_domain_dns_records(domain_id: str = Path(..., title="The domain ID or name")):
|
||||
"""
|
||||
Get DNS records for a specific domain. For Milestone 1,
|
||||
Get DNS records for a specific domain. For Milestone 1,
|
||||
this returns mock data since DNS integration is part of a future milestone.
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
# For Milestone 1, return mock DNS record data
|
||||
# In a future milestone, this will be replaced with actual DNS lookups
|
||||
return DNSRecordResponse(
|
||||
@@ -225,97 +253,101 @@ async def get_domain_dns_records(domain_id: str = Path(..., title="The domain ID
|
||||
spf=True,
|
||||
spfRecord="v=spf1 include:_spf.google.com include:spf.protection.outlook.com -all",
|
||||
dkim=True,
|
||||
dkimSelectors="selector1, selector2"
|
||||
dkimSelectors="selector1, selector2",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{domain_id}/reports", response_model=DomainReportsResponse)
|
||||
async def get_domain_reports(
|
||||
domain_id: str = Path(..., title="The domain ID or name"),
|
||||
limit: int = Query(10, title="Maximum number of reports to return")
|
||||
limit: int = Query(10, title="Maximum number of reports to return"),
|
||||
):
|
||||
"""
|
||||
Get recent DMARC reports for a specific domain, along with compliance timeline
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
# Get reports for this domain
|
||||
reports = store.get_domain_reports(domain_id, limit=limit)
|
||||
|
||||
|
||||
# Generate report entries
|
||||
report_entries = []
|
||||
for report in reports:
|
||||
report_entries.append(ReportEntry(
|
||||
id=report.get("report_id", "unknown"),
|
||||
org_name=report.get("org_name", "Unknown Organization"),
|
||||
begin_date=report.get("begin_date", 0),
|
||||
end_date=report.get("end_date", 0),
|
||||
total_emails=report.get("total_count", 0),
|
||||
pass_rate=report.get("pass_rate", 0.0),
|
||||
policy=report.get("policy", "none")
|
||||
))
|
||||
|
||||
report_entries.append(
|
||||
ReportEntry(
|
||||
id=report.get("report_id", "unknown"),
|
||||
org_name=report.get("org_name", "Unknown Organization"),
|
||||
begin_date=report.get("begin_date", 0),
|
||||
end_date=report.get("end_date", 0),
|
||||
total_emails=report.get("total_count", 0),
|
||||
pass_rate=report.get("pass_rate", 0.0),
|
||||
policy=report.get("policy", "none"),
|
||||
)
|
||||
)
|
||||
|
||||
# Generate compliance timeline (last 30 days)
|
||||
timeline = []
|
||||
for i in range(30, 0, -1):
|
||||
date = datetime.now() - timedelta(days=i)
|
||||
date_str = date.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
# For Milestone 1, generate some mock data with variation
|
||||
# In future milestone, this will use actual historical data
|
||||
import random
|
||||
|
||||
compliance_rate = random.uniform(80, 100)
|
||||
|
||||
timeline.append(TimelinePoint(
|
||||
date=date_str,
|
||||
compliance_rate=round(compliance_rate, 1)
|
||||
))
|
||||
|
||||
return DomainReportsResponse(
|
||||
reports=report_entries,
|
||||
compliance_timeline=timeline
|
||||
)
|
||||
|
||||
timeline.append(TimelinePoint(date=date_str, compliance_rate=round(compliance_rate, 1)))
|
||||
|
||||
return DomainReportsResponse(reports=report_entries, compliance_timeline=timeline)
|
||||
|
||||
|
||||
@router.get("/{domain_id}/sources", response_model=DomainSourcesResponse)
|
||||
async def get_domain_sources(
|
||||
domain_id: str = Path(..., title="The domain ID or name"),
|
||||
days: int = Query(30, title="Number of days to look back")
|
||||
days: int = Query(30, title="Number of days to look back"),
|
||||
):
|
||||
"""
|
||||
Get sending sources for a specific domain
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
# Get sending sources for this domain
|
||||
sources = store.get_domain_sources(domain_id, days=days)
|
||||
|
||||
|
||||
source_entries = []
|
||||
for source in sources:
|
||||
source_entries.append(SourceEntry(
|
||||
ip=source.get("source_ip", "unknown"),
|
||||
count=source.get("count", 0),
|
||||
spf=source.get("spf_result", "unknown"),
|
||||
dkim=source.get("dkim_result", "unknown"),
|
||||
dmarc="pass" if source.get("spf_result") == "pass" or source.get("dkim_result") == "pass" else "fail",
|
||||
disposition=source.get("disposition", "none")
|
||||
))
|
||||
|
||||
return DomainSourcesResponse(
|
||||
sources=source_entries
|
||||
)
|
||||
source_entries.append(
|
||||
SourceEntry(
|
||||
ip=source.get("source_ip", "unknown"),
|
||||
count=source.get("count", 0),
|
||||
spf=source.get("spf_result", "unknown"),
|
||||
dkim=source.get("dkim_result", "unknown"),
|
||||
dmarc=(
|
||||
"pass"
|
||||
if source.get("spf_result") == "pass" or source.get("dkim_result") == "pass"
|
||||
else "fail"
|
||||
),
|
||||
disposition=source.get("disposition", "none"),
|
||||
)
|
||||
)
|
||||
|
||||
return DomainSourcesResponse(sources=source_entries)
|
||||
|
||||
|
||||
@router.delete("/{domain_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_domain(domain_id: str = Path(..., title="The domain ID or name")):
|
||||
@@ -325,36 +357,37 @@ async def delete_domain(domain_id: str = Path(..., title="The domain ID or name"
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
|
||||
# Perform deletion with cleanup
|
||||
deleted = store.delete_domain_with_cleanup(domain_id)
|
||||
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete domain",
|
||||
)
|
||||
|
||||
|
||||
# Return 204 No Content on success
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/search", response_model=List[DomainResponse])
|
||||
async def search_domains(
|
||||
q: Optional[str] = Query(None, title="Search query for domain name or description"),
|
||||
policy: Optional[str] = Query(None, title="Filter by DMARC policy"),
|
||||
page: int = Query(1, title="Page number", ge=1),
|
||||
limit: int = Query(10, title="Number of domains per page", ge=1, le=100)
|
||||
limit: int = Query(10, title="Number of domains per page", ge=1, le=100),
|
||||
):
|
||||
"""
|
||||
Search domains with filtering and pagination.
|
||||
This supports searching by domain name/description and filtering by DMARC policy.
|
||||
|
||||
|
||||
Args:
|
||||
q: Optional search query for domain name or description
|
||||
policy: Optional filter by DMARC policy (none, quarantine, reject)
|
||||
@@ -364,33 +397,35 @@ async def search_domains(
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
summaries = store.get_all_domain_summaries()
|
||||
|
||||
|
||||
# Apply search filter if provided
|
||||
filtered_domains = []
|
||||
for domain_name in domains:
|
||||
summary = summaries.get(domain_name, {})
|
||||
|
||||
|
||||
# Skip domain if it doesn't match the search query
|
||||
if q and q.lower() not in domain_name.lower():
|
||||
continue
|
||||
|
||||
|
||||
# Skip domain if it doesn't match the policy filter
|
||||
if policy and summary.get("policy") != policy:
|
||||
continue
|
||||
|
||||
|
||||
# Domain passed all filters
|
||||
filtered_domains.append({
|
||||
"name": domain_name,
|
||||
"description": "", # No description in in-memory store
|
||||
"policy": summary.get("policy", "unknown"),
|
||||
"reports_count": summary.get("reports_processed", 0),
|
||||
"emails_count": summary.get("total_count", 0),
|
||||
"compliance_rate": summary.get("compliance_rate", 0.0)
|
||||
})
|
||||
|
||||
filtered_domains.append(
|
||||
{
|
||||
"name": domain_name,
|
||||
"description": "", # No description in in-memory store
|
||||
"policy": summary.get("policy", "unknown"),
|
||||
"reports_count": summary.get("reports_processed", 0),
|
||||
"emails_count": summary.get("total_count", 0),
|
||||
"compliance_rate": summary.get("compliance_rate", 0.0),
|
||||
}
|
||||
)
|
||||
|
||||
# Apply pagination
|
||||
start_idx = (page - 1) * limit
|
||||
end_idx = start_idx + limit
|
||||
paginated_domains = filtered_domains[start_idx:end_idx]
|
||||
|
||||
return [DomainResponse(**domain) for domain in paginated_domains]
|
||||
|
||||
return [DomainResponse(**domain) for domain in paginated_domains]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from app.api.api_v1.endpoints.setup import setup_status
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.api_v1.endpoints.setup import setup_status
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/health", status_code=200)
|
||||
async def health_check():
|
||||
"""
|
||||
@@ -14,5 +14,5 @@ async def health_check():
|
||||
"status": "ok",
|
||||
"version": "0.1.0",
|
||||
"service": "dmarq",
|
||||
"is_setup_complete": setup_status["is_setup_complete"]
|
||||
}
|
||||
"is_setup_complete": setup_status["is_setup_complete"],
|
||||
}
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
from typing import Dict, Any
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.core.security import require_admin_auth
|
||||
from app.services.imap_client import IMAPClient
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@router.post("/test-connection")
|
||||
async def test_imap_connection(
|
||||
auth: dict = Depends(require_admin_auth),
|
||||
@@ -16,11 +17,11 @@ async def test_imap_connection(
|
||||
port: int = 993,
|
||||
username: str = None,
|
||||
password: str = None,
|
||||
ssl: bool = True
|
||||
ssl: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Test connection to an IMAP server and gather mailbox statistics
|
||||
|
||||
|
||||
Security: Requires authentication (X-API-Key or Bearer token)
|
||||
Note: Credentials should be passed in request body, not query params
|
||||
"""
|
||||
@@ -29,18 +30,13 @@ async def test_imap_connection(
|
||||
logger.warning("IMAP credentials passed as query parameters - this is insecure")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Credentials should be passed in request body, not query parameters"
|
||||
detail="Credentials should be passed in request body, not query parameters",
|
||||
)
|
||||
|
||||
imap_client = IMAPClient(
|
||||
server=server,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
|
||||
imap_client = IMAPClient(server=server, port=port, username=username, password=password)
|
||||
|
||||
success, message, stats = imap_client.test_connection()
|
||||
|
||||
|
||||
return {
|
||||
"success": success,
|
||||
"message": message,
|
||||
@@ -48,7 +44,7 @@ async def test_imap_connection(
|
||||
"unread_count": stats.get("unread_count", 0),
|
||||
"dmarc_count": stats.get("dmarc_count", 0),
|
||||
"available_mailboxes": stats.get("available_mailboxes", []),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@@ -57,48 +53,44 @@ async def fetch_imap_reports(
|
||||
background_tasks: BackgroundTasks,
|
||||
auth: dict = Depends(require_admin_auth),
|
||||
days: int = 7,
|
||||
delete_emails: bool = False
|
||||
delete_emails: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch DMARC reports from the configured IMAP mailbox
|
||||
|
||||
|
||||
Security: Requires authentication (X-API-Key or Bearer token)
|
||||
"""
|
||||
# Security: Validate parameters
|
||||
if days < 1 or days > 365:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Days parameter must be between 1 and 365"
|
||||
)
|
||||
|
||||
raise HTTPException(status_code=400, detail="Days parameter must be between 1 and 365")
|
||||
|
||||
imap_client = IMAPClient(delete_emails=delete_emails)
|
||||
|
||||
|
||||
# Run in background if it might take a while
|
||||
if days > 14:
|
||||
background_tasks.add_task(imap_client.fetch_reports, days)
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Background task started to fetch {days} days of reports",
|
||||
"timestamp": datetime.now().isoformat()
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
# Otherwise run immediately
|
||||
try:
|
||||
results = imap_client.fetch_reports(days=days)
|
||||
|
||||
|
||||
return {
|
||||
"success": results["success"],
|
||||
"processed_emails": results["processed"],
|
||||
"reports_found": results["reports_found"],
|
||||
"new_domains": results["new_domains"],
|
||||
"errors": results["errors"] if "errors" in results and results["errors"] else None,
|
||||
"timestamp": datetime.now().isoformat()
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching IMAP reports: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to fetch reports. Check server logs for details."
|
||||
status_code=500, detail="Failed to fetch reports. Check server logs for details."
|
||||
)
|
||||
|
||||
|
||||
@@ -106,18 +98,18 @@ async def fetch_imap_reports(
|
||||
async def get_imap_status(auth: dict = Depends(require_admin_auth)) -> Dict[str, Any]:
|
||||
"""
|
||||
Get the current status of IMAP polling background processes
|
||||
|
||||
|
||||
Security: Requires authentication (X-API-Key or Bearer token)
|
||||
"""
|
||||
# In a real implementation this would check a persistent store
|
||||
# or a global variable tracking the status of background tasks
|
||||
# For now, returning simplified status
|
||||
|
||||
|
||||
return {
|
||||
"is_running": True, # In a real app, check if the background task is running
|
||||
"last_check": None, # In production, track actual last check time
|
||||
"next_check": None, # In production, calculate based on polling interval
|
||||
"messages_processed": 0, # In production, track actual messages processed
|
||||
"reports_found": 0, # In production, track reports found
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
"reports_found": 0, # In production, track reports found
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
from typing import Dict, List, Any
|
||||
from fastapi import APIRouter, File, HTTPException, UploadFile, status
|
||||
from pydantic import BaseModel
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.report_store import ReportStore
|
||||
from app.utils.domain_validator import validate_domain, DomainValidationError
|
||||
from app.utils.domain_validator import DomainValidationError, validate_domain
|
||||
from fastapi import APIRouter, File, HTTPException, UploadFile, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Try to import python-magic for MIME type detection
|
||||
try:
|
||||
import magic
|
||||
|
||||
HAS_MAGIC = True
|
||||
except ImportError:
|
||||
HAS_MAGIC = False
|
||||
@@ -21,27 +22,31 @@ router = APIRouter()
|
||||
|
||||
# Security: Allowed MIME types for DMARC report uploads
|
||||
ALLOWED_MIME_TYPES = {
|
||||
'text/xml',
|
||||
'application/xml',
|
||||
'application/zip',
|
||||
'application/x-zip-compressed',
|
||||
'application/gzip',
|
||||
'application/x-gzip',
|
||||
'application/octet-stream' # Sometimes zip/gzip are detected as this
|
||||
"text/xml",
|
||||
"application/xml",
|
||||
"application/zip",
|
||||
"application/x-zip-compressed",
|
||||
"application/gzip",
|
||||
"application/x-gzip",
|
||||
"application/octet-stream", # Sometimes zip/gzip are detected as this
|
||||
}
|
||||
|
||||
# Security: Allowed file extensions
|
||||
ALLOWED_EXTENSIONS = {'.xml', '.zip', '.gz', '.gzip'}
|
||||
ALLOWED_EXTENSIONS = {".xml", ".zip", ".gz", ".gzip"}
|
||||
|
||||
|
||||
class UploadResponse(BaseModel):
|
||||
"""Response model for report upload"""
|
||||
|
||||
success: bool
|
||||
domain: str
|
||||
message: str
|
||||
processed_records: int = 0 # Added this field to track processed records
|
||||
|
||||
|
||||
class DomainSummary(BaseModel):
|
||||
"""Domain summary response model"""
|
||||
|
||||
domain: str
|
||||
total_count: int
|
||||
passed_count: int
|
||||
@@ -49,8 +54,10 @@ class DomainSummary(BaseModel):
|
||||
reports_processed: int
|
||||
compliance_rate: float
|
||||
|
||||
|
||||
class ReportSummary(BaseModel):
|
||||
"""DMARC report summary model"""
|
||||
|
||||
report_id: str
|
||||
org_name: str
|
||||
begin_date: str
|
||||
@@ -59,19 +66,22 @@ class ReportSummary(BaseModel):
|
||||
passed_count: int
|
||||
failed_count: int
|
||||
|
||||
|
||||
class PaginatedReportResponse(BaseModel):
|
||||
"""Paginated reports response model"""
|
||||
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
total_pages: int
|
||||
reports: List[ReportSummary]
|
||||
|
||||
|
||||
@router.post("/upload", response_model=UploadResponse)
|
||||
async def upload_report(file: UploadFile = File(...)):
|
||||
"""
|
||||
Upload and process a DMARC aggregate report file (XML, ZIP, or GZIP)
|
||||
|
||||
|
||||
Security:
|
||||
- File type validation (extension and MIME type)
|
||||
- File size limits enforced in parser
|
||||
@@ -82,28 +92,24 @@ async def upload_report(file: UploadFile = File(...)):
|
||||
# Security: Validate filename is provided
|
||||
if not file.filename:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Filename is required"
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Filename is required"
|
||||
)
|
||||
|
||||
|
||||
# Security: Validate file extension
|
||||
file_ext = '.' + file.filename.rsplit('.', 1)[-1].lower() if '.' in file.filename else ''
|
||||
file_ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
|
||||
if file_ext not in ALLOWED_EXTENSIONS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid file type. Allowed types: {', '.join(ALLOWED_EXTENSIONS)}"
|
||||
detail=f"Invalid file type. Allowed types: {', '.join(ALLOWED_EXTENSIONS)}",
|
||||
)
|
||||
|
||||
|
||||
# Read the file content
|
||||
file_content = await file.read()
|
||||
|
||||
|
||||
# Security: Validate file is not empty
|
||||
if len(file_content) == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File is empty"
|
||||
)
|
||||
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="File is empty")
|
||||
|
||||
# Security: Validate MIME type using python-magic (if available)
|
||||
if HAS_MAGIC:
|
||||
try:
|
||||
@@ -112,48 +118,48 @@ async def upload_report(file: UploadFile = File(...)):
|
||||
logger.warning(f"Rejected file with MIME type: {mime_type}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid file type. File must be XML, ZIP, or GZIP format."
|
||||
detail="Invalid file type. File must be XML, ZIP, or GZIP format.",
|
||||
)
|
||||
except Exception as e:
|
||||
# If magic fails, log but continue (fallback to extension check)
|
||||
logger.warning(f"MIME type detection failed: {str(e)}")
|
||||
else:
|
||||
logger.debug("MIME type validation skipped (python-magic not available)")
|
||||
|
||||
|
||||
# Parse the report
|
||||
parser = DMARCParser()
|
||||
report = parser.parse_file(file_content, file.filename)
|
||||
|
||||
|
||||
# Security: Validate domain from report
|
||||
domain = report.get("domain", "")
|
||||
if not domain:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Report does not contain a valid domain"
|
||||
detail="Report does not contain a valid domain",
|
||||
)
|
||||
|
||||
|
||||
# Validate domain format (not DNS resolution to avoid external calls)
|
||||
is_valid, error_msg, error_code = validate_domain(domain, check_dns=False)
|
||||
if not is_valid and error_code != DomainValidationError.DNS_RESOLUTION_FAILED:
|
||||
# Allow domains that fail DNS resolution but have valid format
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid domain in report: {error_msg}"
|
||||
detail=f"Invalid domain in report: {error_msg}",
|
||||
)
|
||||
|
||||
|
||||
# Store the report
|
||||
store = ReportStore.get_instance()
|
||||
store.add_report(report)
|
||||
|
||||
|
||||
processed_records = report.get("summary", {}).get("total_count", 0)
|
||||
|
||||
|
||||
return UploadResponse(
|
||||
success=True,
|
||||
domain=domain,
|
||||
message=f"Report processed successfully for domain {domain}",
|
||||
processed_records=processed_records
|
||||
processed_records=processed_records,
|
||||
)
|
||||
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions as-is
|
||||
raise
|
||||
@@ -165,27 +171,25 @@ async def upload_report(file: UploadFile = File(...)):
|
||||
# Return sanitized message
|
||||
if "too large" in error_message.lower():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail="File too large"
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File too large"
|
||||
)
|
||||
elif "zip bomb" in error_message.lower():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid archive file"
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid archive file"
|
||||
)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid report format"
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid report format"
|
||||
)
|
||||
except Exception as e:
|
||||
# Security: Don't expose internal errors to client
|
||||
logger.error(f"Unexpected error processing report {file.filename}: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Error processing report. Please contact support if this persists."
|
||||
detail="Error processing report. Please contact support if this persists.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/domains", response_model=List[str])
|
||||
async def get_domains():
|
||||
"""
|
||||
@@ -194,6 +198,7 @@ async def get_domains():
|
||||
store = ReportStore.get_instance()
|
||||
return store.get_domains()
|
||||
|
||||
|
||||
@router.get("/domain/{domain}/summary", response_model=DomainSummary)
|
||||
async def get_domain_summary(domain: str):
|
||||
"""
|
||||
@@ -201,17 +206,14 @@ async def get_domain_summary(domain: str):
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
summary = store.get_domain_summary(domain)
|
||||
|
||||
|
||||
if not summary:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"No reports found for domain {domain}"
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
||||
)
|
||||
|
||||
return DomainSummary(
|
||||
domain=domain,
|
||||
**summary
|
||||
)
|
||||
|
||||
return DomainSummary(domain=domain, **summary)
|
||||
|
||||
|
||||
@router.get("/summary", response_model=List[DomainSummary])
|
||||
async def get_all_summaries():
|
||||
@@ -220,11 +222,9 @@ async def get_all_summaries():
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
all_summaries = store.get_all_domain_summaries()
|
||||
|
||||
return [
|
||||
DomainSummary(domain=domain, **summary)
|
||||
for domain, summary in all_summaries.items()
|
||||
]
|
||||
|
||||
return [DomainSummary(domain=domain, **summary) for domain, summary in all_summaries.items()]
|
||||
|
||||
|
||||
@router.get("/domain/{domain}/reports", response_model=List[ReportSummary])
|
||||
async def get_domain_reports(domain: str):
|
||||
@@ -233,13 +233,12 @@ async def get_domain_reports(domain: str):
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
reports = store.get_domain_reports(domain)
|
||||
|
||||
|
||||
if not reports:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"No reports found for domain {domain}"
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
||||
)
|
||||
|
||||
|
||||
return [
|
||||
ReportSummary(
|
||||
report_id=report.get("report_id", ""),
|
||||
@@ -248,22 +247,23 @@ async def get_domain_reports(domain: str):
|
||||
end_date=report.get("end_date", ""),
|
||||
total_count=report.get("summary", {}).get("total_count", 0),
|
||||
passed_count=report.get("summary", {}).get("passed_count", 0),
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0)
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0),
|
||||
)
|
||||
for report in reports
|
||||
]
|
||||
|
||||
|
||||
@router.get("/domain/{domain}/reports/paginated", response_model=PaginatedReportResponse)
|
||||
async def get_domain_reports_paginated(
|
||||
domain: str,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
sort_by: str = "end_date",
|
||||
sort_order: str = "desc"
|
||||
sort_order: str = "desc",
|
||||
):
|
||||
"""
|
||||
Get paginated reports for a specific domain with sorting options
|
||||
|
||||
|
||||
Args:
|
||||
domain: Domain name
|
||||
page: Page number (1-based)
|
||||
@@ -273,35 +273,30 @@ async def get_domain_reports_paginated(
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
all_reports = store.get_domain_reports(domain)
|
||||
|
||||
|
||||
if not all_reports:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"No reports found for domain {domain}"
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail=f"No reports found for domain {domain}"
|
||||
)
|
||||
|
||||
|
||||
# Apply sorting
|
||||
valid_sort_fields = ["report_id", "org_name", "begin_date", "end_date", "total_count"]
|
||||
sort_field = sort_by if sort_by in valid_sort_fields else "end_date"
|
||||
|
||||
|
||||
if sort_field == "total_count":
|
||||
all_reports.sort(
|
||||
key=lambda r: r.get("summary", {}).get("total_count", 0),
|
||||
reverse=(sort_order == "desc")
|
||||
key=lambda r: r.get("summary", {}).get("total_count", 0), reverse=(sort_order == "desc")
|
||||
)
|
||||
else:
|
||||
all_reports.sort(
|
||||
key=lambda r: r.get(sort_field, ""),
|
||||
reverse=(sort_order == "desc")
|
||||
)
|
||||
|
||||
all_reports.sort(key=lambda r: r.get(sort_field, ""), reverse=(sort_order == "desc"))
|
||||
|
||||
# Apply pagination
|
||||
total = len(all_reports)
|
||||
total_pages = (total + page_size - 1) // page_size
|
||||
start_idx = (page - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
paginated_reports = all_reports[start_idx:end_idx]
|
||||
|
||||
|
||||
# Format reports
|
||||
report_entries = [
|
||||
ReportSummary(
|
||||
@@ -311,15 +306,11 @@ async def get_domain_reports_paginated(
|
||||
end_date=report.get("end_date", ""),
|
||||
total_count=report.get("summary", {}).get("total_count", 0),
|
||||
passed_count=report.get("summary", {}).get("passed_count", 0),
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0)
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0),
|
||||
)
|
||||
for report in paginated_reports
|
||||
]
|
||||
|
||||
|
||||
return PaginatedReportResponse(
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total_pages=total_pages,
|
||||
reports=report_entries
|
||||
)
|
||||
total=total, page=page, page_size=page_size, total_pages=total_pages, reports=report_entries
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from pydantic import BaseModel, EmailStr
|
||||
from typing import Dict, Optional
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -11,30 +11,37 @@ setup_status = {
|
||||
"app_name": "DMARQ",
|
||||
}
|
||||
|
||||
|
||||
class SetupStatusResponse(BaseModel):
|
||||
"""Setup status response"""
|
||||
|
||||
is_setup_complete: bool
|
||||
app_name: str
|
||||
|
||||
|
||||
class AdminSetupRequest(BaseModel):
|
||||
"""Admin user setup request body"""
|
||||
|
||||
email: EmailStr
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class SystemConfigRequest(BaseModel):
|
||||
"""System configuration setup request body"""
|
||||
|
||||
app_name: str
|
||||
base_url: str
|
||||
|
||||
|
||||
@router.get("/status", response_model=SetupStatusResponse)
|
||||
async def get_setup_status():
|
||||
"""Get the current setup status"""
|
||||
return SetupStatusResponse(
|
||||
is_setup_complete=setup_status["is_setup_complete"],
|
||||
app_name=setup_status["app_name"]
|
||||
is_setup_complete=setup_status["is_setup_complete"], app_name=setup_status["app_name"]
|
||||
)
|
||||
|
||||
|
||||
@router.post("/admin", status_code=201)
|
||||
async def setup_admin(request: AdminSetupRequest):
|
||||
"""
|
||||
@@ -43,15 +50,15 @@ async def setup_admin(request: AdminSetupRequest):
|
||||
"""
|
||||
if setup_status["is_setup_complete"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Setup already completed"
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Setup already completed"
|
||||
)
|
||||
|
||||
|
||||
# Store admin email
|
||||
setup_status["admin_email"] = request.email
|
||||
|
||||
|
||||
return {"message": "Admin user setup completed"}
|
||||
|
||||
|
||||
@router.post("/system", status_code=200)
|
||||
async def setup_system(request: SystemConfigRequest):
|
||||
"""
|
||||
@@ -61,5 +68,5 @@ async def setup_system(request: SystemConfigRequest):
|
||||
# Store app name
|
||||
setup_status["app_name"] = request.app_name
|
||||
setup_status["is_setup_complete"] = True
|
||||
|
||||
return {"message": "System settings saved successfully"}
|
||||
|
||||
return {"message": "System settings saved successfully"}
|
||||
|
||||
@@ -1,75 +1,77 @@
|
||||
from typing import Dict, Any, List, Optional
|
||||
from fastapi import APIRouter, Depends, Query, Path
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.utils.stats_summarizer import StatsSummarizer
|
||||
from fastapi import APIRouter, Depends, Path, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/dashboard")
|
||||
async def get_dashboard_statistics(
|
||||
db: Session = Depends(get_db),
|
||||
force_refresh: bool = Query(False, title="Force refresh of statistics"),
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics")
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics"),
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get optimized statistics for the dashboard using cached data when possible.
|
||||
This endpoint provides efficient access to statistics for large datasets.
|
||||
|
||||
|
||||
Args:
|
||||
force_refresh: If True, invalidate cache and recalculate statistics
|
||||
period_days: Period in days for time-based statistics (default: 30)
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary with dashboard statistics
|
||||
"""
|
||||
# Initialize statistics summarizer
|
||||
stats_summarizer = StatsSummarizer()
|
||||
|
||||
|
||||
# If force refresh, invalidate cache
|
||||
if force_refresh:
|
||||
stats_summarizer.invalidate_cache()
|
||||
|
||||
|
||||
# Get statistics (from cache or calculate if needed)
|
||||
stats = stats_summarizer.calculate_summary_statistics(db)
|
||||
|
||||
|
||||
# Add version and timestamp
|
||||
stats["api_version"] = "1.0"
|
||||
stats["period_days"] = period_days
|
||||
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
@router.get("/domain/{domain_id}")
|
||||
async def get_domain_statistics(
|
||||
domain_id: str = Path(..., title="The domain ID or name"),
|
||||
db: Session = Depends(get_db),
|
||||
force_refresh: bool = Query(False, title="Force refresh of statistics"),
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics")
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics"),
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get optimized statistics for a specific domain using cached data when possible.
|
||||
|
||||
|
||||
Args:
|
||||
domain_id: The domain ID or name
|
||||
force_refresh: If True, invalidate cache and recalculate statistics
|
||||
period_days: Period in days for time-based statistics (default: 30)
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary with domain statistics
|
||||
"""
|
||||
# Initialize statistics summarizer
|
||||
stats_summarizer = StatsSummarizer()
|
||||
|
||||
|
||||
# If force refresh, invalidate domain cache
|
||||
if force_refresh:
|
||||
stats_summarizer.invalidate_cache(domain_id)
|
||||
|
||||
|
||||
# Get domain statistics (from cache or calculate if needed)
|
||||
stats = stats_summarizer.calculate_summary_statistics(db, domain_id)
|
||||
|
||||
|
||||
# Add version and timestamp
|
||||
stats["api_version"] = "1.0"
|
||||
stats["period_days"] = period_days
|
||||
|
||||
return stats
|
||||
|
||||
return stats
|
||||
|
||||
Reference in New Issue
Block a user