Add comprehensive documentation for DMARQ, including user guides, deployment instructions, and feature descriptions
- Created main documentation index and user guide with sections on getting started, dashboard overview, managing domains, and reports. - Added detailed deployment guide for Docker and manual installation. - Included user-friendly explanations of DMARC, its benefits, and how to manage domains and reports. - Implemented visual assets for dashboard, domains, IMAP, and reports. - Established requirements for documentation build using MkDocs and Material theme. - Integrated navigation structure for easy access to all documentation sections.
This commit is contained in:
Christian Krakau-Louis
@@ -1,6 +1,6 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.api_v1.endpoints import domains, health, reports, setup, imap
|
||||
from app.api.api_v1.endpoints import domains, health, reports, setup, imap, stats
|
||||
|
||||
api_router = APIRouter()
|
||||
|
||||
@@ -9,4 +9,5 @@ api_router.include_router(health.router, tags=["health"])
|
||||
api_router.include_router(domains.router, prefix="/domains", tags=["domains"])
|
||||
api_router.include_router(reports.router, prefix="/reports", tags=["reports"])
|
||||
api_router.include_router(setup.router, prefix="/setup", tags=["setup"])
|
||||
api_router.include_router(imap.router, prefix="/imap", tags=["imap"])
|
||||
api_router.include_router(imap.router, prefix="/imap", tags=["imap"])
|
||||
api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
|
||||
@@ -315,4 +315,82 @@ async def get_domain_sources(
|
||||
|
||||
return DomainSourcesResponse(
|
||||
sources=source_entries
|
||||
)
|
||||
)
|
||||
|
||||
@router.delete("/{domain_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_domain(domain_id: str = Path(..., title="The domain ID or name")):
|
||||
"""
|
||||
Delete a domain and all associated data.
|
||||
This performs a full cleanup of all reports and records related to this domain.
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
|
||||
if domain_id not in domains:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Domain not found",
|
||||
)
|
||||
|
||||
# Perform deletion with cleanup
|
||||
deleted = store.delete_domain_with_cleanup(domain_id)
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete domain",
|
||||
)
|
||||
|
||||
# Return 204 No Content on success
|
||||
return None
|
||||
|
||||
@router.get("/search", response_model=List[DomainResponse])
|
||||
async def search_domains(
|
||||
q: Optional[str] = Query(None, title="Search query for domain name or description"),
|
||||
policy: Optional[str] = Query(None, title="Filter by DMARC policy"),
|
||||
page: int = Query(1, title="Page number", ge=1),
|
||||
limit: int = Query(10, title="Number of domains per page", ge=1, le=100)
|
||||
):
|
||||
"""
|
||||
Search domains with filtering and pagination.
|
||||
This supports searching by domain name/description and filtering by DMARC policy.
|
||||
|
||||
Args:
|
||||
q: Optional search query for domain name or description
|
||||
policy: Optional filter by DMARC policy (none, quarantine, reject)
|
||||
page: Page number (1-based)
|
||||
limit: Number of domains per page (max 100)
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
domains = store.get_domains()
|
||||
summaries = store.get_all_domain_summaries()
|
||||
|
||||
# Apply search filter if provided
|
||||
filtered_domains = []
|
||||
for domain_name in domains:
|
||||
summary = summaries.get(domain_name, {})
|
||||
|
||||
# Skip domain if it doesn't match the search query
|
||||
if q and q.lower() not in domain_name.lower():
|
||||
continue
|
||||
|
||||
# Skip domain if it doesn't match the policy filter
|
||||
if policy and summary.get("policy") != policy:
|
||||
continue
|
||||
|
||||
# Domain passed all filters
|
||||
filtered_domains.append({
|
||||
"name": domain_name,
|
||||
"description": "", # No description in in-memory store
|
||||
"policy": summary.get("policy", "unknown"),
|
||||
"reports_count": summary.get("reports_processed", 0),
|
||||
"emails_count": summary.get("total_count", 0),
|
||||
"compliance_rate": summary.get("compliance_rate", 0.0)
|
||||
})
|
||||
|
||||
# Apply pagination
|
||||
start_idx = (page - 1) * limit
|
||||
end_idx = start_idx + limit
|
||||
paginated_domains = filtered_domains[start_idx:end_idx]
|
||||
|
||||
return [DomainResponse(**domain) for domain in paginated_domains]
|
||||
@@ -11,10 +11,11 @@ async def test_imap_connection(
|
||||
server: str = None,
|
||||
port: int = 993,
|
||||
username: str = None,
|
||||
password: str = None
|
||||
password: str = None,
|
||||
ssl: bool = True
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Test connection to an IMAP server
|
||||
Test connection to an IMAP server and gather mailbox statistics
|
||||
"""
|
||||
imap_client = IMAPClient(
|
||||
server=server,
|
||||
@@ -23,11 +24,15 @@ async def test_imap_connection(
|
||||
password=password
|
||||
)
|
||||
|
||||
success, message = imap_client.test_connection()
|
||||
success, message, stats = imap_client.test_connection()
|
||||
|
||||
return {
|
||||
"success": success,
|
||||
"message": message,
|
||||
"message_count": stats.get("message_count", 0),
|
||||
"unread_count": stats.get("unread_count", 0),
|
||||
"dmarc_count": stats.get("dmarc_count", 0),
|
||||
"available_mailboxes": stats.get("available_mailboxes", []),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
@@ -62,4 +67,55 @@ async def fetch_imap_reports(
|
||||
"new_domains": results["new_domains"],
|
||||
"errors": results["errors"] if "errors" in results and results["errors"] else None,
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def get_imap_status() -> Dict[str, Any]:
|
||||
"""
|
||||
Get the current status of IMAP polling background processes
|
||||
"""
|
||||
# In a real implementation this would check a persistent store
|
||||
# or a global variable tracking the status of background tasks
|
||||
# For now, returning mock data as this is MVP
|
||||
|
||||
# Get the last check time if available
|
||||
last_check_time = None
|
||||
try:
|
||||
# In a production app, this would be stored in database
|
||||
# For MVP, using a simple file-based approach
|
||||
import os
|
||||
status_file = os.path.join(os.path.dirname(__file__), "../../../../../tmp/imap_last_check.txt")
|
||||
if os.path.exists(status_file):
|
||||
with open(status_file, "r") as f:
|
||||
last_check_time = f.read().strip()
|
||||
except:
|
||||
pass
|
||||
|
||||
# If status file doesn't exist, create the directory
|
||||
try:
|
||||
os.makedirs(os.path.dirname(os.path.join(os.path.dirname(__file__), "../../../../../tmp")), exist_ok=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
# For demonstration purposes, update the last check time to now
|
||||
# In a real app, this would be updated by the background process
|
||||
try:
|
||||
with open(os.path.join(os.path.dirname(__file__), "../../../../../tmp/imap_last_check.txt"), "w") as f:
|
||||
now = datetime.now().isoformat()
|
||||
f.write(now)
|
||||
# If there was no previous check time, set it to now
|
||||
if not last_check_time:
|
||||
last_check_time = now
|
||||
except:
|
||||
pass
|
||||
|
||||
# Return the status
|
||||
return {
|
||||
"is_running": True, # In a real app, check if the background task is running
|
||||
"last_check": last_check_time,
|
||||
"next_check": None, # In production, this would be calculated based on polling interval
|
||||
"messages_processed": 0, # In production, this would track actual messages processed
|
||||
"reports_found": 0, # In production, this would track reports found
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
@@ -33,6 +33,14 @@ class ReportSummary(BaseModel):
|
||||
passed_count: int
|
||||
failed_count: int
|
||||
|
||||
class PaginatedReportResponse(BaseModel):
|
||||
"""Paginated reports response model"""
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
total_pages: int
|
||||
reports: List[ReportSummary]
|
||||
|
||||
@router.post("/upload", response_model=UploadResponse)
|
||||
async def upload_report(file: UploadFile = File(...)):
|
||||
"""
|
||||
@@ -132,4 +140,75 @@ async def get_domain_reports(domain: str):
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0)
|
||||
)
|
||||
for report in reports
|
||||
]
|
||||
]
|
||||
|
||||
@router.get("/domain/{domain}/reports/paginated", response_model=PaginatedReportResponse)
|
||||
async def get_domain_reports_paginated(
|
||||
domain: str,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
sort_by: str = "end_date",
|
||||
sort_order: str = "desc"
|
||||
):
|
||||
"""
|
||||
Get paginated reports for a specific domain with sorting options
|
||||
|
||||
Args:
|
||||
domain: Domain name
|
||||
page: Page number (1-based)
|
||||
page_size: Number of reports per page
|
||||
sort_by: Field to sort by (report_id, org_name, begin_date, end_date, total_count)
|
||||
sort_order: Sort order (asc or desc)
|
||||
"""
|
||||
store = ReportStore.get_instance()
|
||||
all_reports = store.get_domain_reports(domain)
|
||||
|
||||
if not all_reports:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"No reports found for domain {domain}"
|
||||
)
|
||||
|
||||
# Apply sorting
|
||||
valid_sort_fields = ["report_id", "org_name", "begin_date", "end_date", "total_count"]
|
||||
sort_field = sort_by if sort_by in valid_sort_fields else "end_date"
|
||||
|
||||
if sort_field == "total_count":
|
||||
all_reports.sort(
|
||||
key=lambda r: r.get("summary", {}).get("total_count", 0),
|
||||
reverse=(sort_order == "desc")
|
||||
)
|
||||
else:
|
||||
all_reports.sort(
|
||||
key=lambda r: r.get(sort_field, ""),
|
||||
reverse=(sort_order == "desc")
|
||||
)
|
||||
|
||||
# Apply pagination
|
||||
total = len(all_reports)
|
||||
total_pages = (total + page_size - 1) // page_size
|
||||
start_idx = (page - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
paginated_reports = all_reports[start_idx:end_idx]
|
||||
|
||||
# Format reports
|
||||
report_entries = [
|
||||
ReportSummary(
|
||||
report_id=report.get("report_id", ""),
|
||||
org_name=report.get("org_name", ""),
|
||||
begin_date=report.get("begin_date", ""),
|
||||
end_date=report.get("end_date", ""),
|
||||
total_count=report.get("summary", {}).get("total_count", 0),
|
||||
passed_count=report.get("summary", {}).get("passed_count", 0),
|
||||
failed_count=report.get("summary", {}).get("failed_count", 0)
|
||||
)
|
||||
for report in paginated_reports
|
||||
]
|
||||
|
||||
return PaginatedReportResponse(
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total_pages=total_pages,
|
||||
reports=report_entries
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
from typing import Dict, Any, List, Optional
|
||||
from fastapi import APIRouter, Depends, Query, Path
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.utils.stats_summarizer import StatsSummarizer
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/dashboard")
|
||||
async def get_dashboard_statistics(
|
||||
db: Session = Depends(get_db),
|
||||
force_refresh: bool = Query(False, title="Force refresh of statistics"),
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics")
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get optimized statistics for the dashboard using cached data when possible.
|
||||
This endpoint provides efficient access to statistics for large datasets.
|
||||
|
||||
Args:
|
||||
force_refresh: If True, invalidate cache and recalculate statistics
|
||||
period_days: Period in days for time-based statistics (default: 30)
|
||||
|
||||
Returns:
|
||||
Dictionary with dashboard statistics
|
||||
"""
|
||||
# Initialize statistics summarizer
|
||||
stats_summarizer = StatsSummarizer()
|
||||
|
||||
# If force refresh, invalidate cache
|
||||
if force_refresh:
|
||||
stats_summarizer.invalidate_cache()
|
||||
|
||||
# Get statistics (from cache or calculate if needed)
|
||||
stats = stats_summarizer.calculate_summary_statistics(db)
|
||||
|
||||
# Add version and timestamp
|
||||
stats["api_version"] = "1.0"
|
||||
stats["period_days"] = period_days
|
||||
|
||||
return stats
|
||||
|
||||
@router.get("/domain/{domain_id}")
|
||||
async def get_domain_statistics(
|
||||
domain_id: str = Path(..., title="The domain ID or name"),
|
||||
db: Session = Depends(get_db),
|
||||
force_refresh: bool = Query(False, title="Force refresh of statistics"),
|
||||
period_days: int = Query(30, title="Period in days for time-based statistics")
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get optimized statistics for a specific domain using cached data when possible.
|
||||
|
||||
Args:
|
||||
domain_id: The domain ID or name
|
||||
force_refresh: If True, invalidate cache and recalculate statistics
|
||||
period_days: Period in days for time-based statistics (default: 30)
|
||||
|
||||
Returns:
|
||||
Dictionary with domain statistics
|
||||
"""
|
||||
# Initialize statistics summarizer
|
||||
stats_summarizer = StatsSummarizer()
|
||||
|
||||
# If force refresh, invalidate domain cache
|
||||
if force_refresh:
|
||||
stats_summarizer.invalidate_cache(domain_id)
|
||||
|
||||
# Get domain statistics (from cache or calculate if needed)
|
||||
stats = stats_summarizer.calculate_summary_statistics(db, domain_id)
|
||||
|
||||
# Add version and timestamp
|
||||
stats["api_version"] = "1.0"
|
||||
stats["period_days"] = period_days
|
||||
|
||||
return stats
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.core.database import Base
|
||||
@@ -15,25 +15,35 @@ class Domain(Base):
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String, unique=True, index=True, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
active = Column(Boolean, default=True)
|
||||
active = Column(Boolean, default=True, index=True)
|
||||
|
||||
# DMARC policy information
|
||||
dmarc_policy = Column(String, nullable=True)
|
||||
dmarc_policy = Column(String, nullable=True, index=True)
|
||||
spf_record = Column(String, nullable=True)
|
||||
dkim_selectors = Column(String, nullable=True) # Comma-separated list of DKIM selectors
|
||||
|
||||
# DNS verification status
|
||||
verified = Column(Boolean, default=False)
|
||||
verified = Column(Boolean, default=False, index=True)
|
||||
verification_token = Column(String, nullable=True)
|
||||
|
||||
# Date fields
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
reports = relationship("DMARCReport", back_populates="domain", cascade="all, delete-orphan")
|
||||
user_domains = relationship("UserDomain", back_populates="domain", cascade="all, delete-orphan")
|
||||
|
||||
# Indexes for common queries
|
||||
__table_args__ = (
|
||||
# Index for finding active and verified domains
|
||||
Index('ix_domains_active_verified', 'active', 'verified'),
|
||||
# Index for finding domains by policy
|
||||
Index('ix_domains_policy', 'dmarc_policy'),
|
||||
# Index for finding recently updated domains
|
||||
Index('ix_domains_updated', 'updated_at'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Domain {self.name}>"
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.core.database import Base
|
||||
@@ -13,30 +13,40 @@ class DMARCReport(Base):
|
||||
__tablename__ = "dmarc_reports"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
domain_id = Column(Integer, ForeignKey("domains.id"), nullable=False)
|
||||
domain_id = Column(Integer, ForeignKey("domains.id"), nullable=False, index=True)
|
||||
|
||||
# Report metadata
|
||||
report_id = Column(String, index=True, nullable=False)
|
||||
org_name = Column(String, nullable=False)
|
||||
begin_date = Column(Integer, nullable=False) # Unix timestamp
|
||||
end_date = Column(Integer, nullable=False) # Unix timestamp
|
||||
org_name = Column(String, nullable=False, index=True)
|
||||
begin_date = Column(Integer, nullable=False, index=True) # Unix timestamp
|
||||
end_date = Column(Integer, nullable=False, index=True) # Unix timestamp
|
||||
source_email = Column(String, nullable=True)
|
||||
|
||||
# Policy information
|
||||
policy = Column(String, nullable=True) # none, quarantine, reject
|
||||
policy = Column(String, nullable=True, index=True) # none, quarantine, reject
|
||||
subdomain_policy = Column(String, nullable=True)
|
||||
adkim = Column(String(1), nullable=True) # r (relaxed) or s (strict)
|
||||
aspf = Column(String(1), nullable=True) # r (relaxed) or s (strict)
|
||||
percentage = Column(Integer, nullable=True)
|
||||
|
||||
# Processing metadata
|
||||
processed_at = Column(DateTime, default=datetime.utcnow)
|
||||
processed_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
raw_data = Column(Text, nullable=True) # Original XML content (optional)
|
||||
|
||||
# Relationships
|
||||
domain = relationship("Domain", back_populates="reports")
|
||||
records = relationship("ReportRecord", back_populates="report", cascade="all, delete-orphan")
|
||||
|
||||
# Indexes for common queries
|
||||
__table_args__ = (
|
||||
# Composite index for domain and date range queries (common dashboard queries)
|
||||
Index('ix_dmarc_reports_domain_dates', 'domain_id', 'begin_date', 'end_date'),
|
||||
# Index for finding reports by policy
|
||||
Index('ix_dmarc_reports_policy', 'policy'),
|
||||
# Index for finding recent reports (dashboard statistics)
|
||||
Index('ix_dmarc_reports_processed', 'processed_at'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<DMARCReport {self.report_id} for {self.domain_id}>"
|
||||
|
||||
@@ -47,19 +57,19 @@ class ReportRecord(Base):
|
||||
__tablename__ = "report_records"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
report_id = Column(Integer, ForeignKey("dmarc_reports.id"), nullable=False)
|
||||
report_id = Column(Integer, ForeignKey("dmarc_reports.id"), nullable=False, index=True)
|
||||
|
||||
# Source information
|
||||
source_ip = Column(String, nullable=False, index=True)
|
||||
count = Column(Integer, nullable=False, default=0)
|
||||
|
||||
# Policy evaluation
|
||||
disposition = Column(String, nullable=False) # none, quarantine, reject
|
||||
dkim = Column(String, nullable=True) # pass, fail
|
||||
spf = Column(String, nullable=True) # pass, fail
|
||||
disposition = Column(String, nullable=False, index=True) # none, quarantine, reject
|
||||
dkim = Column(String, nullable=True, index=True) # pass, fail
|
||||
spf = Column(String, nullable=True, index=True) # pass, fail
|
||||
|
||||
# Identifiers
|
||||
header_from = Column(String, nullable=True)
|
||||
header_from = Column(String, nullable=True, index=True)
|
||||
envelope_from = Column(String, nullable=True)
|
||||
|
||||
# Authentication details (optional JSON fields)
|
||||
@@ -69,5 +79,13 @@ class ReportRecord(Base):
|
||||
# Relationships
|
||||
report = relationship("DMARCReport", back_populates="records")
|
||||
|
||||
# Indexes for common queries
|
||||
__table_args__ = (
|
||||
# Composite index for source IP and evaluation results (for filtering)
|
||||
Index('ix_report_records_source_auth', 'source_ip', 'dkim', 'spf'),
|
||||
# Composite index for disposition and count (for statistics)
|
||||
Index('ix_report_records_disposition', 'disposition', 'count'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<ReportRecord {self.id} ({self.source_ip})>"
|
||||
@@ -48,31 +48,82 @@ class IMAPClient:
|
||||
if not all([self.server, self.username, self.password]):
|
||||
logger.warning("IMAP credentials not fully configured")
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
def test_connection(self) -> Tuple[bool, str, Dict[str, Any]]:
|
||||
"""
|
||||
Test the IMAP connection
|
||||
Test the IMAP connection and gather basic mailbox statistics
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
Tuple of (success, message, stats)
|
||||
- success: Boolean indicating if connection was successful
|
||||
- message: String message describing the result
|
||||
- stats: Dictionary with mailbox statistics (if successful)
|
||||
"""
|
||||
if not all([self.server, self.username, self.password]):
|
||||
return False, "IMAP credentials not fully configured"
|
||||
return False, "IMAP credentials not fully configured", {}
|
||||
|
||||
try:
|
||||
# Create IMAP4 connection
|
||||
mail = imaplib.IMAP4_SSL(self.server, self.port)
|
||||
# Login
|
||||
mail.login(self.username, self.password)
|
||||
# List mailboxes
|
||||
mail.list()
|
||||
# Select inbox
|
||||
mail.select('INBOX')
|
||||
# Logout
|
||||
|
||||
# List available mailboxes
|
||||
status, mailbox_list = mail.list()
|
||||
available_mailboxes = []
|
||||
|
||||
if status == 'OK':
|
||||
for mailbox in mailbox_list:
|
||||
if isinstance(mailbox, bytes):
|
||||
try:
|
||||
# Extract mailbox name from response
|
||||
mailbox_str = mailbox.decode('utf-8')
|
||||
# Extract the mailbox name (after the last quote)
|
||||
parts = mailbox_str.split('"')
|
||||
if len(parts) > 2:
|
||||
mailbox_name = parts[-1].strip()
|
||||
if mailbox_name.startswith(' '):
|
||||
mailbox_name = mailbox_name[1:]
|
||||
available_mailboxes.append(mailbox_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Select inbox and get message count
|
||||
status, data = mail.select('INBOX')
|
||||
message_count = 0
|
||||
unread_count = 0
|
||||
|
||||
if status == 'OK':
|
||||
message_count = int(data[0])
|
||||
|
||||
# Count unread messages
|
||||
status, data = mail.search(None, 'UNSEEN')
|
||||
if status == 'OK':
|
||||
unread_count = len(data[0].split())
|
||||
|
||||
# Gather some stats about potential DMARC reports
|
||||
dmarc_count = 0
|
||||
status, data = mail.search(None, 'SUBJECT "DMARC"')
|
||||
if status == 'OK':
|
||||
dmarc_count = len(data[0].split())
|
||||
|
||||
# Close connection
|
||||
mail.close()
|
||||
mail.logout()
|
||||
return True, "Connection successful"
|
||||
|
||||
stats = {
|
||||
"message_count": message_count,
|
||||
"unread_count": unread_count,
|
||||
"dmarc_count": dmarc_count,
|
||||
"available_mailboxes": available_mailboxes,
|
||||
"server": self.server,
|
||||
"port": self.port,
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
return True, "Connection successful", stats
|
||||
except Exception as e:
|
||||
logger.error(f"IMAP connection test failed: {str(e)}")
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
return False, f"Connection failed: {str(e)}", {}
|
||||
|
||||
def fetch_reports(self, days: int = 7) -> Dict[str, Any]:
|
||||
"""
|
||||
|
||||
@@ -189,4 +189,27 @@ class ReportStore:
|
||||
"""
|
||||
self.domain_reports = {}
|
||||
self.domain_summary = {}
|
||||
self.domain_sources = {}
|
||||
self.domain_sources = {}
|
||||
|
||||
def delete_domain_with_cleanup(self, domain: str) -> bool:
|
||||
"""
|
||||
Delete a domain and all its associated data
|
||||
|
||||
Args:
|
||||
domain: Domain name to delete
|
||||
|
||||
Returns:
|
||||
True if domain was deleted, False otherwise
|
||||
"""
|
||||
if domain not in self.domain_reports:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Remove all data for this domain
|
||||
self.domain_reports.pop(domain, None)
|
||||
self.domain_summary.pop(domain, None)
|
||||
self.domain_sources.pop(domain, None)
|
||||
return True
|
||||
except Exception:
|
||||
# If any exception occurs during deletion, return False
|
||||
return False
|
||||
@@ -436,36 +436,110 @@ function domainDetailsApp(domainId) {
|
||||
const labels = timelineData.map(item => item.date);
|
||||
const complianceData = timelineData.map(item => item.compliance_rate);
|
||||
|
||||
// Calculate the threshold line data (recommended 98% for policy advancement)
|
||||
const thresholdData = Array(labels.length).fill(98);
|
||||
|
||||
this.complianceChart = new Chart(ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: labels,
|
||||
datasets: [{
|
||||
label: 'Compliance Rate',
|
||||
data: complianceData,
|
||||
borderColor: '#1A237E',
|
||||
backgroundColor: 'rgba(26, 35, 126, 0.1)',
|
||||
tension: 0.3,
|
||||
fill: true
|
||||
}]
|
||||
datasets: [
|
||||
{
|
||||
label: 'Compliance Rate',
|
||||
data: complianceData,
|
||||
borderColor: 'rgb(59, 130, 246)', // blue-500
|
||||
backgroundColor: 'rgba(59, 130, 246, 0.1)',
|
||||
tension: 0.4,
|
||||
fill: true,
|
||||
pointBackgroundColor: 'rgb(59, 130, 246)',
|
||||
pointRadius: 3,
|
||||
pointHoverRadius: 5
|
||||
},
|
||||
{
|
||||
label: 'Recommended Threshold (98%)',
|
||||
data: thresholdData,
|
||||
borderColor: 'rgba(220, 38, 38, 0.6)', // red-600 with opacity
|
||||
borderDash: [5, 5],
|
||||
pointRadius: 0,
|
||||
borderWidth: 2,
|
||||
fill: false,
|
||||
tension: 0
|
||||
}
|
||||
]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
scales: {
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
beginAtZero: false,
|
||||
min: Math.max(0, Math.min(...complianceData) - 10), // Dynamic min value
|
||||
max: 100,
|
||||
ticks: {
|
||||
callback: value => value + '%'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Compliance Rate (%)',
|
||||
font: {
|
||||
weight: 'bold'
|
||||
}
|
||||
},
|
||||
grid: {
|
||||
color: 'rgba(0, 0, 0, 0.05)'
|
||||
}
|
||||
},
|
||||
x: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Date',
|
||||
font: {
|
||||
weight: 'bold'
|
||||
}
|
||||
},
|
||||
grid: {
|
||||
display: false
|
||||
}
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
tooltip: {
|
||||
backgroundColor: 'rgba(0, 0, 0, 0.8)',
|
||||
titleFont: {
|
||||
size: 13
|
||||
},
|
||||
bodyFont: {
|
||||
size: 12
|
||||
},
|
||||
padding: 10,
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
return context.parsed.y + '%';
|
||||
if (context.dataset.label === 'Compliance Rate') {
|
||||
return `Compliance: ${context.parsed.y}%`;
|
||||
}
|
||||
return context.dataset.label;
|
||||
},
|
||||
title: function(context) {
|
||||
return `Date: ${context[0].label}`;
|
||||
}
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
display: true,
|
||||
position: 'top',
|
||||
labels: {
|
||||
usePointStyle: true,
|
||||
padding: 15
|
||||
}
|
||||
},
|
||||
annotation: {
|
||||
annotations: {
|
||||
box1: {
|
||||
type: 'box',
|
||||
yMin: 90,
|
||||
yMax: 100,
|
||||
backgroundColor: 'rgba(34, 197, 94, 0.05)',
|
||||
borderWidth: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Utilities for DMARQ application.
|
||||
"""
|
||||
@@ -0,0 +1,70 @@
|
||||
import re
|
||||
import socket
|
||||
from typing import Dict, Tuple, Union, Optional
|
||||
|
||||
|
||||
def validate_domain(domain_name: str) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Validates a domain name for format and resolvability.
|
||||
|
||||
Args:
|
||||
domain_name: The domain name to validate
|
||||
|
||||
Returns:
|
||||
Tuple containing (is_valid, error_message)
|
||||
- is_valid: Boolean indicating if domain is valid
|
||||
- error_message: String with error message if not valid, None if valid
|
||||
"""
|
||||
# Check for empty domain
|
||||
if not domain_name:
|
||||
return False, "Domain name cannot be empty"
|
||||
|
||||
# Check domain format with regex
|
||||
# This regex allows domain names with alphanumeric characters, hyphens,
|
||||
# and periods as separators. It enforces proper domain structure.
|
||||
domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]$'
|
||||
if not re.match(domain_pattern, domain_name):
|
||||
return False, "Invalid domain format"
|
||||
|
||||
# Check if domain exists by attempting to resolve DNS
|
||||
try:
|
||||
socket.gethostbyname(domain_name)
|
||||
return True, None
|
||||
except socket.gaierror:
|
||||
# We could consider this valid if we don't require DNS resolution,
|
||||
# but since DMARC requires valid DNS, we'll mark it as warning
|
||||
return False, "Domain could not be resolved (DNS lookup failed)"
|
||||
|
||||
|
||||
def validate_domain_config(domain_data: Dict) -> Dict[str, Union[bool, str]]:
|
||||
"""
|
||||
Validates domain configuration data for creating or updating domains.
|
||||
|
||||
Args:
|
||||
domain_data: Dictionary with domain configuration
|
||||
|
||||
Returns:
|
||||
Dictionary with validation results containing:
|
||||
- valid: Boolean indicating if configuration is valid
|
||||
- errors: Dict of field-specific errors
|
||||
"""
|
||||
errors = {}
|
||||
|
||||
# Validate domain name
|
||||
if "name" in domain_data:
|
||||
is_valid, error_msg = validate_domain(domain_data["name"])
|
||||
if not is_valid:
|
||||
errors["name"] = error_msg
|
||||
else:
|
||||
errors["name"] = "Domain name is required"
|
||||
|
||||
# Validate description (optional but with max length)
|
||||
if "description" in domain_data and domain_data["description"]:
|
||||
if len(domain_data["description"]) > 255:
|
||||
errors["description"] = "Description is too long (max 255 characters)"
|
||||
|
||||
# Return validation results
|
||||
return {
|
||||
"valid": len(errors) == 0,
|
||||
"errors": errors
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any, Optional
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
|
||||
# Setup logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class StatsSummarizer:
|
||||
"""
|
||||
Utility class for summarizing and caching dashboard statistics
|
||||
to improve performance with large datasets.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: str = None):
|
||||
"""
|
||||
Initialize the stats summarizer with optional cache directory
|
||||
|
||||
Args:
|
||||
cache_dir: Directory to store cached statistics (defaults to tmp/stats)
|
||||
"""
|
||||
if cache_dir is None:
|
||||
# Default cache directory is tmp/stats under the project root
|
||||
self.cache_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "tmp", "stats")
|
||||
else:
|
||||
self.cache_dir = cache_dir
|
||||
|
||||
# Create cache directory if it doesn't exist
|
||||
os.makedirs(self.cache_dir, exist_ok=True)
|
||||
|
||||
def get_cached_summary(self, domain_id: Optional[str] = None, max_age_minutes: int = 60) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get cached summary statistics if available and not too old
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID to get domain-specific stats
|
||||
If None, gets global summary
|
||||
max_age_minutes: Maximum age of cache in minutes
|
||||
|
||||
Returns:
|
||||
Cached statistics or None if not available or too old
|
||||
"""
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
|
||||
try:
|
||||
if not os.path.exists(cache_file):
|
||||
return None
|
||||
|
||||
# Check file modification time
|
||||
mtime = os.path.getmtime(cache_file)
|
||||
file_age = datetime.now() - datetime.fromtimestamp(mtime)
|
||||
|
||||
# If cache is too old, return None
|
||||
if file_age > timedelta(minutes=max_age_minutes):
|
||||
return None
|
||||
|
||||
# Read cache file
|
||||
with open(cache_file, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error reading cache file {cache_file}: {str(e)}")
|
||||
return None
|
||||
|
||||
def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Save summary statistics to cache
|
||||
|
||||
Args:
|
||||
stats: Dictionary of statistics to cache
|
||||
domain_id: Optional domain ID for domain-specific stats
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
|
||||
try:
|
||||
# Add timestamp
|
||||
stats["cached_at"] = datetime.now().isoformat()
|
||||
|
||||
# Write to cache file
|
||||
with open(cache_file, 'w') as f:
|
||||
json.dump(stats, f)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing cache file {cache_file}: {str(e)}")
|
||||
return False
|
||||
|
||||
def invalidate_cache(self, domain_id: Optional[str] = None) -> None:
|
||||
"""
|
||||
Invalidate cache for a domain or all domains
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID to invalidate specific domain cache
|
||||
If None, invalidates global summary cache
|
||||
"""
|
||||
if domain_id is None:
|
||||
# Invalidate all caches
|
||||
cache_file = self._get_cache_filename()
|
||||
if os.path.exists(cache_file):
|
||||
os.remove(cache_file)
|
||||
else:
|
||||
# Invalidate specific domain cache
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
if os.path.exists(cache_file):
|
||||
os.remove(cache_file)
|
||||
|
||||
def _get_cache_filename(self, domain_id: Optional[str] = None) -> str:
|
||||
"""
|
||||
Get the filename for a cache file
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID for domain-specific cache
|
||||
|
||||
Returns:
|
||||
Path to the cache file
|
||||
"""
|
||||
if domain_id is None:
|
||||
return os.path.join(self.cache_dir, "global_summary.json")
|
||||
else:
|
||||
# Sanitize domain_id to use as filename
|
||||
safe_domain = domain_id.replace(".", "_").replace("/", "_")
|
||||
return os.path.join(self.cache_dir, f"domain_{safe_domain}.json")
|
||||
|
||||
def calculate_summary_statistics(self, db, domain_id: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Calculate summary statistics from the database
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
domain_id: Optional domain ID to calculate domain-specific stats
|
||||
|
||||
Returns:
|
||||
Dictionary with summary statistics
|
||||
"""
|
||||
# In a real implementation, this would query the database
|
||||
# using SQLAlchemy models and calculate statistics
|
||||
# For now, we'll return mock statistics
|
||||
|
||||
# First check if we have cached stats
|
||||
cached_stats = self.get_cached_summary(domain_id)
|
||||
if cached_stats:
|
||||
return cached_stats
|
||||
|
||||
# If no cached stats, calculate from database
|
||||
# In a real implementation, this would be done with SQL queries
|
||||
# optimized for performance with large datasets
|
||||
|
||||
# For now, mock statistics
|
||||
if domain_id is None:
|
||||
# Global statistics
|
||||
stats = {
|
||||
"total_domains": 5,
|
||||
"total_emails": 1250,
|
||||
"compliant_emails": 1100,
|
||||
"compliance_rate": 88.0,
|
||||
"reports_processed": 25,
|
||||
"top_sources": [
|
||||
{"ip": "192.168.1.1", "count": 150},
|
||||
{"ip": "10.0.0.1", "count": 120},
|
||||
{"ip": "172.16.0.1", "count": 100}
|
||||
],
|
||||
"compliance_trend": [
|
||||
{"date": "2025-04-13", "rate": 85.5},
|
||||
{"date": "2025-04-14", "rate": 86.2},
|
||||
{"date": "2025-04-15", "rate": 86.8},
|
||||
{"date": "2025-04-16", "rate": 87.3},
|
||||
{"date": "2025-04-17", "rate": 87.9},
|
||||
{"date": "2025-04-18", "rate": 88.4},
|
||||
{"date": "2025-04-19", "rate": 88.0}
|
||||
]
|
||||
}
|
||||
else:
|
||||
# Domain-specific statistics
|
||||
stats = {
|
||||
"domain": domain_id,
|
||||
"total_emails": 250,
|
||||
"compliant_emails": 220,
|
||||
"compliance_rate": 88.0,
|
||||
"reports_processed": 5,
|
||||
"sources": [
|
||||
{"ip": "192.168.1.1", "count": 100, "spf": "pass", "dkim": "pass"},
|
||||
{"ip": "10.0.0.1", "count": 80, "spf": "pass", "dkim": "fail"},
|
||||
{"ip": "172.16.0.1", "count": 70, "spf": "fail", "dkim": "pass"}
|
||||
],
|
||||
"compliance_trend": [
|
||||
{"date": "2025-04-13", "rate": 85.0},
|
||||
{"date": "2025-04-14", "rate": 86.0},
|
||||
{"date": "2025-04-15", "rate": 87.0},
|
||||
{"date": "2025-04-16", "rate": 87.5},
|
||||
{"date": "2025-04-17", "rate": 88.0},
|
||||
{"date": "2025-04-18", "rate": 88.5},
|
||||
{"date": "2025-04-19", "rate": 88.0}
|
||||
]
|
||||
}
|
||||
|
||||
# Cache the statistics
|
||||
self.save_summary(stats, domain_id)
|
||||
|
||||
return stats
|
||||
Reference in New Issue
Block a user