Fix code formatting and linting issues
- Auto-format all Python files with black and isort - Remove unused imports with autoflake - Fix flake8 issues (missing newlines, blank lines, etc.) - Fix nonlocal/global scope issues in main.py - Fix security.py import order (E402) - Remove f-string without placeholders - Add nosec comment for intentional exception handling - Fix test imports to match refactored DMARCParser API Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -1,97 +1,104 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any, Optional
|
||||
import logging
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# Setup logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StatsSummarizer:
|
||||
"""
|
||||
Utility class for summarizing and caching dashboard statistics
|
||||
to improve performance with large datasets.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, cache_dir: str = None):
|
||||
"""
|
||||
Initialize the stats summarizer with optional cache directory
|
||||
|
||||
|
||||
Args:
|
||||
cache_dir: Directory to store cached statistics (defaults to tmp/stats)
|
||||
"""
|
||||
if cache_dir is None:
|
||||
# Default cache directory is tmp/stats under the project root
|
||||
self.cache_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "tmp", "stats")
|
||||
self.cache_dir = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
|
||||
"tmp",
|
||||
"stats",
|
||||
)
|
||||
else:
|
||||
self.cache_dir = cache_dir
|
||||
|
||||
|
||||
# Create cache directory if it doesn't exist
|
||||
os.makedirs(self.cache_dir, exist_ok=True)
|
||||
|
||||
def get_cached_summary(self, domain_id: Optional[str] = None, max_age_minutes: int = 60) -> Optional[Dict[str, Any]]:
|
||||
|
||||
def get_cached_summary(
|
||||
self, domain_id: Optional[str] = None, max_age_minutes: int = 60
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get cached summary statistics if available and not too old
|
||||
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID to get domain-specific stats
|
||||
If None, gets global summary
|
||||
max_age_minutes: Maximum age of cache in minutes
|
||||
|
||||
|
||||
Returns:
|
||||
Cached statistics or None if not available or too old
|
||||
"""
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
|
||||
|
||||
try:
|
||||
if not os.path.exists(cache_file):
|
||||
return None
|
||||
|
||||
|
||||
# Check file modification time
|
||||
mtime = os.path.getmtime(cache_file)
|
||||
file_age = datetime.now() - datetime.fromtimestamp(mtime)
|
||||
|
||||
|
||||
# If cache is too old, return None
|
||||
if file_age > timedelta(minutes=max_age_minutes):
|
||||
return None
|
||||
|
||||
|
||||
# Read cache file
|
||||
with open(cache_file, 'r') as f:
|
||||
with open(cache_file, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error reading cache file {cache_file}: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
def save_summary(self, stats: Dict[str, Any], domain_id: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Save summary statistics to cache
|
||||
|
||||
|
||||
Args:
|
||||
stats: Dictionary of statistics to cache
|
||||
domain_id: Optional domain ID for domain-specific stats
|
||||
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
|
||||
|
||||
try:
|
||||
# Add timestamp
|
||||
stats["cached_at"] = datetime.now().isoformat()
|
||||
|
||||
|
||||
# Write to cache file
|
||||
with open(cache_file, 'w') as f:
|
||||
with open(cache_file, "w") as f:
|
||||
json.dump(stats, f)
|
||||
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing cache file {cache_file}: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def invalidate_cache(self, domain_id: Optional[str] = None) -> None:
|
||||
"""
|
||||
Invalidate cache for a domain or all domains
|
||||
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID to invalidate specific domain cache
|
||||
If None, invalidates global summary cache
|
||||
@@ -106,14 +113,14 @@ class StatsSummarizer:
|
||||
cache_file = self._get_cache_filename(domain_id)
|
||||
if os.path.exists(cache_file):
|
||||
os.remove(cache_file)
|
||||
|
||||
|
||||
def _get_cache_filename(self, domain_id: Optional[str] = None) -> str:
|
||||
"""
|
||||
Get the filename for a cache file
|
||||
|
||||
|
||||
Args:
|
||||
domain_id: Optional domain ID for domain-specific cache
|
||||
|
||||
|
||||
Returns:
|
||||
Path to the cache file
|
||||
"""
|
||||
@@ -123,31 +130,31 @@ class StatsSummarizer:
|
||||
# Sanitize domain_id to use as filename
|
||||
safe_domain = domain_id.replace(".", "_").replace("/", "_")
|
||||
return os.path.join(self.cache_dir, f"domain_{safe_domain}.json")
|
||||
|
||||
|
||||
def calculate_summary_statistics(self, db, domain_id: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Calculate summary statistics from the database
|
||||
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
domain_id: Optional domain ID to calculate domain-specific stats
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary with summary statistics
|
||||
"""
|
||||
# In a real implementation, this would query the database
|
||||
# using SQLAlchemy models and calculate statistics
|
||||
# For now, we'll return mock statistics
|
||||
|
||||
|
||||
# First check if we have cached stats
|
||||
cached_stats = self.get_cached_summary(domain_id)
|
||||
if cached_stats:
|
||||
return cached_stats
|
||||
|
||||
|
||||
# If no cached stats, calculate from database
|
||||
# In a real implementation, this would be done with SQL queries
|
||||
# optimized for performance with large datasets
|
||||
|
||||
|
||||
# For now, mock statistics
|
||||
if domain_id is None:
|
||||
# Global statistics
|
||||
@@ -160,7 +167,7 @@ class StatsSummarizer:
|
||||
"top_sources": [
|
||||
{"ip": "192.168.1.1", "count": 150},
|
||||
{"ip": "10.0.0.1", "count": 120},
|
||||
{"ip": "172.16.0.1", "count": 100}
|
||||
{"ip": "172.16.0.1", "count": 100},
|
||||
],
|
||||
"compliance_trend": [
|
||||
{"date": "2025-04-13", "rate": 85.5},
|
||||
@@ -169,8 +176,8 @@ class StatsSummarizer:
|
||||
{"date": "2025-04-16", "rate": 87.3},
|
||||
{"date": "2025-04-17", "rate": 87.9},
|
||||
{"date": "2025-04-18", "rate": 88.4},
|
||||
{"date": "2025-04-19", "rate": 88.0}
|
||||
]
|
||||
{"date": "2025-04-19", "rate": 88.0},
|
||||
],
|
||||
}
|
||||
else:
|
||||
# Domain-specific statistics
|
||||
@@ -183,7 +190,7 @@ class StatsSummarizer:
|
||||
"sources": [
|
||||
{"ip": "192.168.1.1", "count": 100, "spf": "pass", "dkim": "pass"},
|
||||
{"ip": "10.0.0.1", "count": 80, "spf": "pass", "dkim": "fail"},
|
||||
{"ip": "172.16.0.1", "count": 70, "spf": "fail", "dkim": "pass"}
|
||||
{"ip": "172.16.0.1", "count": 70, "spf": "fail", "dkim": "pass"},
|
||||
],
|
||||
"compliance_trend": [
|
||||
{"date": "2025-04-13", "rate": 85.0},
|
||||
@@ -192,11 +199,11 @@ class StatsSummarizer:
|
||||
{"date": "2025-04-16", "rate": 87.5},
|
||||
{"date": "2025-04-17", "rate": 88.0},
|
||||
{"date": "2025-04-18", "rate": 88.5},
|
||||
{"date": "2025-04-19", "rate": 88.0}
|
||||
]
|
||||
{"date": "2025-04-19", "rate": 88.0},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# Cache the statistics
|
||||
self.save_summary(stats, domain_id)
|
||||
|
||||
return stats
|
||||
|
||||
return stats
|
||||
|
||||
Reference in New Issue
Block a user