fix: harden forensic report ingestion

backend/app/services/forensic_parser.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 from email import message_from_bytes
 from email.message import Message
 from email.parser import Parser
@@ -84,6 +84,8 @@ def _parse_datetime(value: str) -> Optional[datetime]:
         return None
     if parsed is None:
         return None
+    if parsed.tzinfo is not None:
+        parsed = parsed.astimezone(timezone.utc)
     return parsed.replace(tzinfo=None)
 
 

backend/app/services/forensic_persistence.py

@@ -1,6 +1,7 @@
 import json
 from typing import Any, Dict, Optional
 
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from app.models.domain import Domain
@@ -10,7 +11,7 @@ from app.utils.domain_validator import DomainValidationError, validate_domain
 
 def forensic_report_exists(db: Session, report_id: str) -> bool:
     """Return True when a forensic report ID is already persisted."""
-    if not report_id:
+    if not str(report_id or "").strip():
         return False
     return (
         db.query(ForensicReport.id).filter(ForensicReport.report_id == report_id).first()
@@ -40,7 +41,10 @@ def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicR
     Returns ``(row, created)``. The caller owns the transaction and should
     commit after related work has completed.
     """
-    report_id = str(report.get("report_id") or "")
+    report_id = str(report.get("report_id") or "").strip()
+    if not report_id:
+        raise ValueError("Forensic report_id is required")
+
     existing = db.query(ForensicReport).filter(ForensicReport.report_id == report_id).first()
     if existing is not None:
         return existing, False
@@ -72,12 +76,26 @@ def save_forensic_report(db: Session, report: Dict[str, Any]) -> tuple[ForensicR
         feedback_headers=feedback_headers,
     )
     db.add(row)
-    db.flush()
+    try:
+        db.flush()
+    except IntegrityError:
+        db.rollback()
+        existing = db.query(ForensicReport).filter(ForensicReport.report_id == report_id).first()
+        if existing is not None:
+            return existing, False
+        raise
     return row, True
 
 
 def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
     """Convert a forensic report row to an API-safe dictionary."""
+    feedback_headers = {}
+    if row.feedback_headers:
+        try:
+            feedback_headers = json.loads(row.feedback_headers)
+        except (json.JSONDecodeError, TypeError):
+            feedback_headers = {}
+
     return {
         "id": row.id,
         "report_id": row.report_id,
@@ -98,6 +116,6 @@ def forensic_report_to_dict(row: ForensicReport) -> Dict[str, Any]:
         "original_subject": row.original_subject,
         "original_message_id": row.original_message_id,
         "original_date": row.original_date,
-        "feedback_headers": json.loads(row.feedback_headers) if row.feedback_headers else {},
+        "feedback_headers": feedback_headers,
         "processed_at": row.processed_at.isoformat() if row.processed_at else None,
     }

backend/app/services/gmail_client.py

@@ -59,6 +59,7 @@ DMARC_GMAIL_QUERY = (
 
 # How many message results to fetch per API page
 _PAGE_SIZE = 100
+RETRYABLE_MESSAGE_FAILURE = -1
 
 
 class GmailClient:
@@ -247,11 +248,9 @@ class GmailClient:
 
             stats["processed"] += 1
             found = self._process_message(service, msg_id, stats)
-            if found > 0:
-                stats["new_ingested_ids"].append(msg_id)
-                self.already_ingested_ids.append(msg_id)
-            else:
-                # Track it anyway so we don't re-examine it next run
+            if found >= 0:
+                # Track it even when no report is found so we don't re-examine
+                # unrelated messages on every poll. Retryable failures return -1.
                 stats["new_ingested_ids"].append(msg_id)
                 self.already_ingested_ids.append(msg_id)
 
@@ -334,7 +333,7 @@ class GmailClient:
         raw_bytes = base64.urlsafe_b64decode(msg_data.get("raw", ""))
         msg = email.message_from_bytes(raw_bytes)
         if ForensicParser.is_forensic_report(msg):
-            return 1 if self._process_forensic_message(raw_bytes, stats, message_id=msg_id) else 0
+            return self._process_forensic_message(raw_bytes, stats, message_id=msg_id)
         return self._process_attachments(msg, stats, message_id=msg_id)
 
     @staticmethod
@@ -383,7 +382,7 @@ class GmailClient:
         raw_bytes: bytes,
         stats: dict,
         message_id: Optional[str] = None,
-    ) -> bool:
+    ) -> int:
         """Parse and persist one DMARC forensic report message."""
         try:
             report = ForensicParser.parse_bytes(raw_bytes, message_id_hint=message_id)
@@ -399,7 +398,7 @@ class GmailClient:
                     domain=domain,
                     report_id=report_id,
                 )
-                return False
+                return RETRYABLE_MESSAGE_FAILURE
 
             if forensic_report_exists(self.db, report_id):
                 stats["duplicate_forensic_reports"] = stats.get("duplicate_forensic_reports", 0) + 1
@@ -411,7 +410,7 @@ class GmailClient:
                     domain=domain,
                     report_id=report_id,
                 )
-                return False
+                return 0
 
             _row, created = save_forensic_report(self.db, report)
             if not created:
@@ -424,7 +423,7 @@ class GmailClient:
                     domain=domain,
                     report_id=report_id,
                 )
-                return False
+                return 0
 
             stats["forensic_reports_found"] = stats.get("forensic_reports_found", 0) + 1
             self._append_detail(
@@ -435,10 +434,12 @@ class GmailClient:
                 domain=domain,
                 report_id=report_id,
             )
-            return True
+            return 1
         except Exception as exc:  # pylint: disable=broad-exception-caught
             logger.error("Failed to parse Gmail forensic report %s: %s", message_id, exc)
-            stats["errors"].append(f"Failed to parse forensic report {message_id}: {exc}")
+            stats.setdefault("errors", []).append(
+                f"Failed to parse forensic report {message_id}: {exc}"
+            )
             self._append_detail(
                 stats,
                 status="error",
@@ -446,7 +447,7 @@ class GmailClient:
                 message_id=message_id,
                 error=str(exc),
             )
-            return False
+            return RETRYABLE_MESSAGE_FAILURE
 
     def _process_attachments(
         self,