Merge pull request #98 from christianlouis/codex/mail-source-import-history

feat: add mail source import history

backend/alembic/env.py

@@ -25,6 +25,8 @@ if database_url:
 # Import all models so that autogenerate can detect them
 from app.core.database import Base  # noqa: E402
 import app.models.domain  # noqa: E402, F401
+import app.models.mail_source  # noqa: E402, F401
+import app.models.mail_source_import  # noqa: E402, F401
 import app.models.report  # noqa: E402, F401
 import app.models.setting  # noqa: E402, F401
 import app.models.user  # noqa: E402, F401

backend/alembic/versions/e5f6a7b8c9d0_add_mail_source_imports.py

@@ -0,0 +1,73 @@
+"""add mail source import history
+
+Revision ID: e5f6a7b8c9d0
+Revises: d4e5f6a7b8c9
+Create Date: 2026-05-22 19:40:00.000000
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "e5f6a7b8c9d0"
+down_revision: Union[str, Sequence[str], None] = "d4e5f6a7b8c9"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Create the mail_source_imports table."""
+    op.create_table(
+        "mail_source_imports",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("mail_source_id", sa.Integer(), nullable=False),
+        sa.Column("trigger", sa.String(), nullable=False, server_default="manual"),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column("processed", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("reports_found", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("duplicate_reports", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("error_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("new_domains", sa.Text(), nullable=True),
+        sa.Column("errors", sa.Text(), nullable=True),
+        sa.Column("started_at", sa.DateTime(), nullable=False),
+        sa.Column("finished_at", sa.DateTime(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
+        sa.ForeignKeyConstraint(["mail_source_id"], ["mail_sources.id"]),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_mail_source_imports_id"), "mail_source_imports", ["id"])
+    op.create_index(
+        op.f("ix_mail_source_imports_mail_source_id"),
+        "mail_source_imports",
+        ["mail_source_id"],
+    )
+    op.create_index(
+        op.f("ix_mail_source_imports_status"),
+        "mail_source_imports",
+        ["status"],
+    )
+    op.create_index(
+        op.f("ix_mail_source_imports_started_at"),
+        "mail_source_imports",
+        ["started_at"],
+    )
+    op.create_index(
+        op.f("ix_mail_source_imports_finished_at"),
+        "mail_source_imports",
+        ["finished_at"],
+    )
+
+
+def downgrade() -> None:
+    """Drop the mail_source_imports table."""
+    op.drop_index(op.f("ix_mail_source_imports_finished_at"), table_name="mail_source_imports")
+    op.drop_index(op.f("ix_mail_source_imports_started_at"), table_name="mail_source_imports")
+    op.drop_index(op.f("ix_mail_source_imports_status"), table_name="mail_source_imports")
+    op.drop_index(
+        op.f("ix_mail_source_imports_mail_source_id"),
+        table_name="mail_source_imports",
+    )
+    op.drop_index(op.f("ix_mail_source_imports_id"), table_name="mail_source_imports")
+    op.drop_table("mail_source_imports")

backend/app/api/api_v1/endpoints/mail_sources.py

@@ -7,6 +7,7 @@ persisting anything.  Gmail API sources additionally have OAuth2 helper
 endpoints (authorize-url, callback, fetch).
 """
 
+import json
 import logging
 from datetime import datetime
 from typing import Any, Dict, List, Optional
@@ -18,8 +19,10 @@ from sqlalchemy.orm import Session
 from app.core.database import get_db
 from app.core.security import require_admin_auth
 from app.models.mail_source import MailSource
+from app.models.mail_source_import import MailSourceImport
 from app.services.gmail_client import GmailClient
 from app.services.imap_client import IMAPClient
+from app.services.import_history import record_import_attempt
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -105,6 +108,24 @@ class GmailCallbackRequest(BaseModel):
     redirect_uri: str
 
 
+class MailSourceImportResponse(BaseModel):
+    """Sanitized import-history entry for a mail source."""
+
+    id: int
+    mail_source_id: int
+    trigger: str
+    status: str
+    processed: int
+    reports_found: int
+    duplicate_reports: int
+    error_count: int
+    new_domains: List[str]
+    errors: List[str]
+    started_at: datetime
+    finished_at: datetime
+    created_at: datetime
+
+
 # ---------------------------------------------------------------------------
 # Helper
 # ---------------------------------------------------------------------------
@@ -149,6 +170,38 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
     )
 
 
+def _decode_json_list(value: Optional[str]) -> List[str]:
+    """Decode a JSON list stored on import history rows."""
+    if not value:
+        return []
+    try:
+        decoded = json.loads(value)
+    except (json.JSONDecodeError, TypeError):
+        return []
+    if not isinstance(decoded, list):
+        return []
+    return [str(item) for item in decoded]
+
+
+def _import_to_response(row: MailSourceImport) -> MailSourceImportResponse:
+    """Convert an import-history ORM row to an API response."""
+    return MailSourceImportResponse(
+        id=row.id,
+        mail_source_id=row.mail_source_id,
+        trigger=row.trigger,
+        status=row.status,
+        processed=row.processed,
+        reports_found=row.reports_found,
+        duplicate_reports=row.duplicate_reports,
+        error_count=row.error_count,
+        new_domains=_decode_json_list(row.new_domains),
+        errors=_decode_json_list(row.errors),
+        started_at=row.started_at,
+        finished_at=row.finished_at,
+        created_at=row.created_at,
+    )
+
+
 # ---------------------------------------------------------------------------
 # Routes
 # ---------------------------------------------------------------------------
@@ -205,6 +258,26 @@ async def get_mail_source(
     return _source_to_response(source)
 
 
+@router.get("/{source_id}/imports", response_model=List[MailSourceImportResponse])
+async def list_mail_source_imports(
+    source_id: int,
+    limit: int = 20,
+    db: Session = Depends(get_db),
+    _auth: dict = Depends(require_admin_auth),
+) -> List[MailSourceImportResponse]:
+    """Return recent sanitized import attempts for one mail source."""
+    _get_source_or_404(source_id, db)
+    safe_limit = min(max(limit, 1), 100)
+    rows = (
+        db.query(MailSourceImport)
+        .filter(MailSourceImport.mail_source_id == source_id)
+        .order_by(MailSourceImport.started_at.desc(), MailSourceImport.id.desc())
+        .limit(safe_limit)
+        .all()
+    )
+    return [_import_to_response(row) for row in rows]
+
+
 @router.put("/{source_id}", response_model=MailSourceResponse)
 async def update_mail_source(
     source_id: int,
@@ -608,6 +681,7 @@ async def gmail_fetch_reports(
         already_ingested_ids=already,
     )
 
+    started_at = datetime.utcnow()
     results = client.fetch_reports()
 
     # Persist updated ingested IDs and any refreshed tokens
@@ -622,6 +696,7 @@ async def gmail_fetch_reports(
             source.gmail_refresh_token = refreshed["refresh_token"]
 
     source.last_checked = datetime.utcnow()
+    record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
     db.commit()
 
     logger.info(

backend/app/main.py

@@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 import app.models.domain  # noqa: F401 – ensure Domain/UserDomain tables are registered
+import app.models.mail_source_import  # noqa: F401 – ensure import history table is registered
 import app.models.report  # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered
 import app.models.setting  # noqa: F401 – ensure Setting table is registered
 import app.models.user  # noqa: F401 – ensure User table is registered
@@ -22,6 +23,7 @@ from app.middleware.security import SecurityHeadersMiddleware
 from app.models.mail_source import MailSource  # noqa: F401 – ensure table is registered
 from app.services.gmail_client import GmailClient
 from app.services.imap_client import IMAPClient
+from app.services.import_history import record_import_attempt
 from app.services.report_store import ReportStore
 
 # Set up logging
@@ -45,6 +47,7 @@ def _poll_single_imap_source(source: MailSource) -> None:
         password=source.password,
         delete_emails=False,
     )
+    started_at = datetime.utcnow()
     results = imap_client.fetch_reports(days=9999)
 
     db = SessionLocal()
@@ -52,6 +55,7 @@ def _poll_single_imap_source(source: MailSource) -> None:
         src = db.query(MailSource).get(source.id)
         if src:
             src.last_checked = datetime.utcnow()
+            record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
             db.commit()
     finally:
         db.close()
@@ -95,6 +99,7 @@ def _poll_single_gmail_source(source: MailSource) -> None:
         already_ingested_ids=already,
     )
 
+    started_at = datetime.utcnow()
     results = client.fetch_reports()
 
     db = SessionLocal()
@@ -112,6 +117,7 @@ def _poll_single_gmail_source(source: MailSource) -> None:
                     src.gmail_refresh_token = refreshed["refresh_token"]
 
             src.last_checked = datetime.utcnow()
+            record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
             db.commit()
     finally:
         db.close()
@@ -517,9 +523,11 @@ def _trigger_poll_imap_source(source: MailSource, db) -> dict:
         password=source.password,
         delete_emails=False,
     )
+    started_at = datetime.utcnow()
     results = imap_client.fetch_reports(days=7)
     last_check_time = datetime.now()
     source.last_checked = datetime.utcnow()
+    record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
     db.commit()
     return {
         "source_id": source.id,
@@ -543,6 +551,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
         refresh_token=source.gmail_refresh_token or "",
         already_ingested_ids=already,
     )
+    started_at = datetime.utcnow()
     results = gmail_client.fetch_reports()
     last_check_time = datetime.now()
 
@@ -555,6 +564,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
         if "refresh_token" in refreshed:
             source.gmail_refresh_token = refreshed["refresh_token"]
     source.last_checked = datetime.utcnow()
+    record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
     db.commit()
     return {
         "source_id": source.id,

backend/app/models/mail_source.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text
+from sqlalchemy.orm import relationship
 
 from app.core.database import Base
 
@@ -63,5 +64,11 @@ class MailSource(Base):
     created_at = Column(DateTime, default=datetime.utcnow)
     updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
 
+    imports = relationship(
+        "MailSourceImport",
+        back_populates="mail_source",
+        cascade="all, delete-orphan",
+    )
+
     def __repr__(self):
         return f"<MailSource id={self.id} name={self.name!r} method={self.method!r}>"

backend/app/models/mail_source_import.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import relationship
+
+from app.core.database import Base
+
+
+class MailSourceImport(Base):
+    """Sanitized audit record for one mail source import attempt."""
+
+    __tablename__ = "mail_source_imports"
+
+    id = Column(Integer, primary_key=True, index=True)
+    mail_source_id = Column(Integer, ForeignKey("mail_sources.id"), nullable=False, index=True)
+
+    trigger = Column(String, nullable=False, default="manual")
+    status = Column(String, nullable=False, index=True)
+
+    processed = Column(Integer, nullable=False, default=0)
+    reports_found = Column(Integer, nullable=False, default=0)
+    duplicate_reports = Column(Integer, nullable=False, default=0)
+    error_count = Column(Integer, nullable=False, default=0)
+
+    new_domains = Column(Text, nullable=True)
+    errors = Column(Text, nullable=True)
+
+    started_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
+    finished_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    mail_source = relationship("MailSource", back_populates="imports")
+
+    def __repr__(self):
+        return (
+            f"<MailSourceImport id={self.id} source={self.mail_source_id} "
+            f"status={self.status!r}>"
+        )

backend/app/services/gmail_client.py

@@ -207,6 +207,7 @@ class GmailClient:
             "success": True,
             "processed": 0,
             "reports_found": 0,
+            "duplicate_reports": 0,
             "new_domains": [],
             "errors": [],
             "new_ingested_ids": [],
@@ -365,6 +366,8 @@ class GmailClient:
                 if self._store_report_if_new(report):
                     stats["reports_found"] += 1
                     reports_found += 1
+                else:
+                    stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1
             except Exception as exc:  # pylint: disable=broad-exception-caught
                 logger.error("Failed to parse DMARC attachment %s: %s", filename, exc)
                 stats["errors"].append(f"Failed to parse {filename}: {exc}")

backend/app/services/imap_client.py

@@ -146,7 +146,7 @@ class IMAPClient:
             msg = email.message_from_bytes(raw_email)
 
             if self._is_dmarc_report_email(msg):
-                reports_found = self._process_attachments(msg)
+                reports_found = self._process_attachments(msg, stats)
                 stats["reports_found"] += reports_found
 
                 # Mark email as read (and optionally delete)
@@ -178,6 +178,7 @@ class IMAPClient:
             "success": True,
             "processed": 0,
             "reports_found": 0,
+            "duplicate_reports": 0,
             "new_domains": [],
             "errors": [],
         }
@@ -350,7 +351,7 @@ class IMAPClient:
 
         return False
 
-    def _process_attachments(self, msg: email.message.Message) -> int:
+    def _process_attachments(self, msg: email.message.Message, stats: dict | None = None) -> int:
         """
         Process email attachments that might be DMARC reports
 
@@ -394,6 +395,10 @@ class IMAPClient:
                                     report_id,
                                     domain,
                                 )
+                                if stats is not None:
+                                    stats["duplicate_reports"] = (
+                                        stats.get("duplicate_reports", 0) + 1
+                                    )
                                 continue
 
                             # Add the report to the store

backend/app/services/import_history.py

@@ -0,0 +1,54 @@
+import json
+from datetime import datetime
+from typing import Any, Dict, Iterable, Optional
+
+from sqlalchemy.orm import Session
+
+from app.models.mail_source import MailSource
+from app.models.mail_source_import import MailSourceImport
+
+MAX_STORED_ERRORS = 10
+MAX_ERROR_LENGTH = 500
+
+
+def _sanitize_error(value: object) -> str:
+    """Return a compact, log-safe error string for storage and UI display."""
+    text = str(value).replace("\r", "").replace("\n", " ").strip()
+    if len(text) > MAX_ERROR_LENGTH:
+        return text[: MAX_ERROR_LENGTH - 1] + "..."
+    return text
+
+
+def _json_list(values: Optional[Iterable[Any]]) -> str:
+    return json.dumps([str(value) for value in values or []])
+
+
+def record_import_attempt(
+    db: Session,
+    source: MailSource,
+    results: Dict[str, Any],
+    *,
+    started_at: datetime,
+    trigger: str,
+) -> MailSourceImport:
+    """Persist a sanitized summary of a mail source import attempt."""
+    result_errors = list(results.get("errors") or [])
+    errors = [_sanitize_error(error) for error in result_errors[:MAX_STORED_ERRORS]]
+    success = bool(results.get("success", False))
+    status = "success" if success and not errors else "warning" if success else "failed"
+
+    attempt = MailSourceImport(
+        mail_source_id=source.id,
+        trigger=trigger,
+        status=status,
+        processed=int(results.get("processed", 0) or 0),
+        reports_found=int(results.get("reports_found", 0) or 0),
+        duplicate_reports=int(results.get("duplicate_reports", 0) or 0),
+        error_count=len(result_errors),
+        new_domains=_json_list(results.get("new_domains", [])),
+        errors=json.dumps(errors),
+        started_at=started_at,
+        finished_at=datetime.utcnow(),
+    )
+    db.add(attempt)
+    return attempt

backend/app/tests/conftest.py

@@ -8,6 +8,7 @@ from sqlalchemy.pool import StaticPool
 
 import app.models.domain  # noqa: F401  # pylint: disable=unused-import
 import app.models.mail_source as _mail_source_model  # noqa: F401  # pylint: disable=unused-import
+import app.models.mail_source_import  # noqa: F401  # pylint: disable=unused-import
 import app.models.report  # noqa: F401  # pylint: disable=unused-import
 import app.models.setting  # noqa: F401  # pylint: disable=unused-import
 import app.models.user  # noqa: F401  # pylint: disable=unused-import

backend/app/tests/test_mail_sources.py

@@ -11,6 +11,7 @@ from fastapi.testclient import TestClient
 from sqlalchemy.orm import Session
 
 from app.models.mail_source import MailSource
+from app.models.mail_source_import import MailSourceImport
 
 
 class TestMailSourceModel:
@@ -75,6 +76,36 @@ class TestMailSourceModel:
         assert len(all_sources) == 3
 
 
+class TestMailSourceImportModel:
+    """Unit tests for persisted mail source import history."""
+
+    def test_create_import_history_row(self, db_session: Session):
+        source = MailSource(name="History Source", method="GMAIL_API")
+        db_session.add(source)
+        db_session.commit()
+        db_session.refresh(source)
+
+        row = MailSourceImport(
+            mail_source_id=source.id,
+            trigger="manual",
+            status="warning",
+            processed=3,
+            reports_found=2,
+            duplicate_reports=1,
+            error_count=1,
+            new_domains='["example.com"]',
+            errors='["bad attachment"]',
+        )
+        db_session.add(row)
+        db_session.commit()
+        db_session.refresh(row)
+
+        assert row.id is not None
+        assert row.mail_source_id == source.id
+        assert row.duplicate_reports == 1
+        assert row.mail_source.name == "History Source"
+
+
 class TestMailSourcesAPI:
     """Integration tests for /api/v1/mail-sources endpoints (no auth)."""
 
@@ -219,6 +250,42 @@ class TestMailSourcesAPIAuthed:
         assert resp.json()["id"] == source_id
         assert resp.json()["name"] == "Get Test"
 
+    def test_list_import_history(self, authed_client: TestClient, db_session: Session):
+        create_resp = authed_client.post(
+            "/api/v1/mail-sources", json={"name": "History API", "method": "IMAP"}
+        )
+        source_id = create_resp.json()["id"]
+
+        db_session.add(
+            MailSourceImport(
+                mail_source_id=source_id,
+                trigger="manual",
+                status="warning",
+                processed=2,
+                reports_found=1,
+                duplicate_reports=1,
+                error_count=1,
+                new_domains='["example.com"]',
+                errors='["sanitized error"]',
+            )
+        )
+        db_session.commit()
+
+        resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/imports")
+
+        assert resp.status_code == 200
+        data = resp.json()
+        assert len(data) == 1
+        assert data[0]["mail_source_id"] == source_id
+        assert data[0]["status"] == "warning"
+        assert data[0]["duplicate_reports"] == 1
+        assert data[0]["new_domains"] == ["example.com"]
+        assert data[0]["errors"] == ["sanitized error"]
+
+    def test_list_import_history_unknown_source_returns_404(self, authed_client: TestClient):
+        resp = authed_client.get("/api/v1/mail-sources/99999/imports")
+        assert resp.status_code == 404
+
     def test_get_nonexistent_source_returns_404(self, authed_client: TestClient):
         resp = authed_client.get("/api/v1/mail-sources/99999")
         assert resp.status_code == 404

docs/development/roadmap.md

@@ -12,7 +12,7 @@ Complete:
 - Upload validation and archive safety checks.
 - IMAP mailbox ingestion.
 - Gmail OAuth ingestion.
-- Persistent database models and migrations.
+- Persistent database models and migrations for domains, reports, records, settings, users, and mail sources.
 - Domain, report, settings, and mail source APIs.
 - Dashboard and domain detail views.
 - Logto auth integration and local development auth-disabled mode.
@@ -23,13 +23,30 @@ Recently improved:
 - Gmail import now uses the same parser path as uploads and IMAP.
 - Gmail and IMAP imports now skip duplicate domain/report IDs.
 - Tests cover Google-style DMARC ZIP attachment imports.
+- Mail source imports now create sanitized import-history records for manual and scheduled polls.
 
-## Active Milestone: Reporting Quality and Import Confidence
+Important gap:
+- Parsed DMARC report data is still served primarily from the in-memory `ReportStore`. The database schema exists, but report upload/import paths and dashboard read paths must be completed before the persistence milestone can be called done.
+
+## Active Milestone: Finish Report Persistence
+
+Objective: complete the database-backed report storage promised by Milestone 3.
+
+Priority tasks:
+- Add a report persistence service that converts parsed DMARC report dicts into `Domain`, `DMARCReport`, and `ReportRecord` rows.
+- Load or query persisted reports for dashboard, domain, and report endpoints.
+- Add duplicate report detection against the database.
+- Keep tests covering upload, Gmail import, IMAP import, and restart-style reload behavior.
+
+Quality bar:
+- Uploading or importing a report survives application restart and remains visible in report/domain endpoints.
+
+## Next Milestone: Reporting Quality and Import Confidence
 
 Objective: make mailbox imports auditable and make report totals trustworthy.
 
 Priority tasks:
-- Persist import attempts with message ID, source, attachment filename, outcome, and sanitized error details.
+- Expand import attempts with message ID, source, attachment filename, outcome, and sanitized error details.
 - Show import history on mail source detail pages.
 - Report duplicate skips separately from parse failures.
 - Add backfill controls for Gmail and IMAP sources.
@@ -40,7 +57,7 @@ Quality bar:
 - Parse failures must be visible and actionable.
 - The user should be able to tell whether a mail source is healthy without reading logs.
 
-## Next Milestone: Meaningful Reports
+## Following Milestone: Meaningful Reports
 
 Objective: turn parsed DMARC data into administrator-friendly reports.
 

docs/milestones.md

@@ -29,19 +29,25 @@ Delivered:
 - Duplicate report protection for both Gmail and IMAP imports.
 - Background polling and manual poll hooks for configured mail sources.
 
-## Milestone 3: Persistence, Domain Management, and Auth Foundation - Complete
+## Milestone 3: Database Foundation, Domain Management, and Auth Foundation - In Progress
 
-Status: Complete
+Status: In progress
 
 Delivered:
 - SQLAlchemy models and Alembic migrations.
 - SQLite/PostgreSQL-compatible database configuration.
 - Domain management APIs and UI.
-- Report and source persistence.
+- Report and source database models.
 - Settings and mail source persistence.
 - Logto-based auth integration plus an explicit local development auth-disabled mode.
 - Security middleware, safer default secret generation, and security-focused tests.
 
+Remaining before this milestone is complete:
+- Persist parsed DMARC reports and report records through the database-backed models.
+- Load or query persisted reports after restart so dashboards do not depend on process memory.
+- Move report/domain summary endpoints from the in-memory `ReportStore` to database queries.
+- Keep duplicate report detection consistent across upload, IMAP, and Gmail after persistence is enabled.
+
 ## Milestone 4: Reporting Quality and Import Confidence - In Progress
 
 Status: In progress
@@ -52,10 +58,11 @@ Recently delivered:
 - Gmail import now handles real inbox metadata patterns and Google-style ZIP filenames.
 - Gmail/IMAP imports skip duplicate report IDs to avoid inflated totals.
 - Tests now cover a real Google-style ZIP attachment path rather than only mocked parser behavior.
+- Mail source imports now persist sanitized import-history records for manual and scheduled polls.
 
 Next tasks:
+- Finish Milestone 3 report persistence before expanding report features.
 - Add per-import result details: skipped duplicates, parse failures, unsupported attachments, and imported report IDs.
-- Store failed import attempts with enough metadata to retry or diagnose them without exposing secrets.
 - Add a UI import history view for each mail source.
 - Add mailbox search controls for date range/backfill without requiring code changes.
 - Improve source aggregation so each sender IP keeps pass/fail totals instead of only the latest result.

docs/todo.md

@@ -88,8 +88,8 @@ This file tracks the specific implementation tasks for each milestone of the DMA
 ### Duplicate and Error Handling
 - [x] Skip duplicate domain/report IDs during Gmail imports
 - [x] Skip duplicate domain/report IDs during IMAP imports
-- [ ] Persist sanitized import errors for UI review
+- [x] Persist sanitized import errors for API/UI review
-- [ ] Count duplicate skips separately from parse failures
+- [x] Count duplicate skips separately from parse failures
 - [ ] Add retry/backfill controls per mail source
 
 ## Milestone 3: Database Integration
@@ -101,11 +101,14 @@ This file tracks the specific implementation tasks for each milestone of the DMA
 - [x] Implement data access layer
 
 ### Model Migration
-- [x] Convert in-memory models to database models
+- [ ] Convert report ingestion and dashboard reads from in-memory storage to database-backed storage
 - [x] Create Domain table
 - [x] Create AggregateReport table
 - [x] Create ReportRecord table for sender details
 - [x] Implement relationships between models
+- [ ] Persist parsed upload reports to `dmarc_reports` and `report_records`
+- [ ] Persist parsed Gmail/IMAP reports to `dmarc_reports` and `report_records`
+- [ ] Load/query persisted reports after app restart
 
 ### Domain Management
 - [x] Create UI for adding/editing domains