import json import logging import os from datetime import datetime, timedelta from typing import Any, Dict, Optional # Setup logger logger = logging.getLogger(__name__) class StatsSummarizer: """ Utility class for summarizing and caching dashboard statistics to improve performance with large datasets. """ def __init__(self, cache_dir: str = None): """ Initialize the stats summarizer with optional cache directory Args: cache_dir: Directory to store cached statistics (defaults to tmp/stats) """ if cache_dir is None: # Default cache directory is tmp/stats under the project root self.cache_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "tmp", "stats", ) else: self.cache_dir = cache_dir # Create cache directory if it doesn't exist os.makedirs(self.cache_dir, exist_ok=True) def get_cached_summary( self, domain_id: Optional[str] = None, max_age_minutes: int = 60 ) -> Optional[Dict[str, Any]]: """ Get cached summary statistics if available and not too old Args: domain_id: Optional domain ID to get domain-specific stats If None, gets global summary max_age_minutes: Maximum age of cache in minutes Returns: Cached statistics or None if not available or too old """ cache_file = self._get_cache_filename(domain_id) try: if not os.path.exists(cache_file): return None # Check file modification time mtime = os.path.getmtime(cache_file) file_age = datetime.now() - datetime.fromtimestamp(mtime) # If cache is too old, return None if file_age > timedelta(minutes=max_age_minutes): return None # Read cache file with open(cache_file, "r") as f: return json.load(f) except Exception as e: logger.warning(f"Error reading cache file {cache_file}: {str(e)}") return None def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool: """ Save summary statistics to cache Args: stats: Dictionary of statistics to cache domain_id: Optional domain ID for domain-specific stats Returns: True if save was successful, False otherwise """ cache_file = self._get_cache_filename(domain_id) try: # Add timestamp stats["cached_at"] = datetime.now().isoformat() # Write to cache file with open(cache_file, "w") as f: json.dump(stats, f) return True except Exception as e: logger.error(f"Error writing cache file {cache_file}: {str(e)}") return False def invalidate_cache(self, domain_id: Optional[str] = None) -> None: """ Invalidate cache for a domain or all domains Args: domain_id: Optional domain ID to invalidate specific domain cache If None, invalidates global summary cache """ if domain_id is None: # Invalidate all caches cache_file = self._get_cache_filename() if os.path.exists(cache_file): os.remove(cache_file) else: # Invalidate specific domain cache cache_file = self._get_cache_filename(domain_id) if os.path.exists(cache_file): os.remove(cache_file) def _get_cache_filename(self, domain_id: Optional[str] = None) -> str: """ Get the filename for a cache file Args: domain_id: Optional domain ID for domain-specific cache Returns: Path to the cache file """ if domain_id is None: return os.path.join(self.cache_dir, "global_summary.json") else: # Sanitize domain_id to use as filename safe_domain = domain_id.replace(".", "_").replace("/", "_") return os.path.join(self.cache_dir, f"domain_{safe_domain}.json") def calculate_summary_statistics(self, db, domain_id: Optional[str] = None) -> Dict[str, Any]: """ Calculate summary statistics from the database Args: db: Database session domain_id: Optional domain ID to calculate domain-specific stats Returns: Dictionary with summary statistics """ # In a real implementation, this would query the database # using SQLAlchemy models and calculate statistics # For now, we'll return mock statistics # First check if we have cached stats cached_stats = self.get_cached_summary(domain_id) if cached_stats: return cached_stats # If no cached stats, calculate from database # In a real implementation, this would be done with SQL queries # optimized for performance with large datasets # For now, mock statistics if domain_id is None: # Global statistics stats = { "total_domains": 5, "total_emails": 1250, "compliant_emails": 1100, "compliance_rate": 88.0, "reports_processed": 25, "top_sources": [ {"ip": "192.168.1.1", "count": 150}, {"ip": "10.0.0.1", "count": 120}, {"ip": "172.16.0.1", "count": 100}, ], "compliance_trend": [ {"date": "2025-04-13", "rate": 85.5}, {"date": "2025-04-14", "rate": 86.2}, {"date": "2025-04-15", "rate": 86.8}, {"date": "2025-04-16", "rate": 87.3}, {"date": "2025-04-17", "rate": 87.9}, {"date": "2025-04-18", "rate": 88.4}, {"date": "2025-04-19", "rate": 88.0}, ], } else: # Domain-specific statistics stats = { "domain": domain_id, "total_emails": 250, "compliant_emails": 220, "compliance_rate": 88.0, "reports_processed": 5, "sources": [ {"ip": "192.168.1.1", "count": 100, "spf": "pass", "dkim": "pass"}, {"ip": "10.0.0.1", "count": 80, "spf": "pass", "dkim": "fail"}, {"ip": "172.16.0.1", "count": 70, "spf": "fail", "dkim": "pass"}, ], "compliance_trend": [ {"date": "2025-04-13", "rate": 85.0}, {"date": "2025-04-14", "rate": 86.0}, {"date": "2025-04-15", "rate": 87.0}, {"date": "2025-04-16", "rate": 87.5}, {"date": "2025-04-17", "rate": 88.0}, {"date": "2025-04-18", "rate": 88.5}, {"date": "2025-04-19", "rate": 88.0}, ], } # Cache the statistics self.save_summary(stats, domain_id) return stats