diff --git a/backend/alembic/env.py b/backend/alembic/env.py index b21e3f5..2dc0526 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -25,6 +25,8 @@ if database_url: # Import all models so that autogenerate can detect them from app.core.database import Base # noqa: E402 import app.models.domain # noqa: E402, F401 +import app.models.mail_source # noqa: E402, F401 +import app.models.mail_source_import # noqa: E402, F401 import app.models.report # noqa: E402, F401 import app.models.setting # noqa: E402, F401 import app.models.user # noqa: E402, F401 diff --git a/backend/alembic/versions/e5f6a7b8c9d0_add_mail_source_imports.py b/backend/alembic/versions/e5f6a7b8c9d0_add_mail_source_imports.py new file mode 100644 index 0000000..cdb5fcc --- /dev/null +++ b/backend/alembic/versions/e5f6a7b8c9d0_add_mail_source_imports.py @@ -0,0 +1,73 @@ +"""add mail source import history + +Revision ID: e5f6a7b8c9d0 +Revises: d4e5f6a7b8c9 +Create Date: 2026-05-22 19:40:00.000000 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e5f6a7b8c9d0" +down_revision: Union[str, Sequence[str], None] = "d4e5f6a7b8c9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the mail_source_imports table.""" + op.create_table( + "mail_source_imports", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mail_source_id", sa.Integer(), nullable=False), + sa.Column("trigger", sa.String(), nullable=False, server_default="manual"), + sa.Column("status", sa.String(), nullable=False), + sa.Column("processed", sa.Integer(), nullable=False, server_default="0"), + sa.Column("reports_found", sa.Integer(), nullable=False, server_default="0"), + sa.Column("duplicate_reports", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("new_domains", sa.Text(), nullable=True), + sa.Column("errors", sa.Text(), nullable=True), + sa.Column("started_at", sa.DateTime(), nullable=False), + sa.Column("finished_at", sa.DateTime(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint(["mail_source_id"], ["mail_sources.id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_mail_source_imports_id"), "mail_source_imports", ["id"]) + op.create_index( + op.f("ix_mail_source_imports_mail_source_id"), + "mail_source_imports", + ["mail_source_id"], + ) + op.create_index( + op.f("ix_mail_source_imports_status"), + "mail_source_imports", + ["status"], + ) + op.create_index( + op.f("ix_mail_source_imports_started_at"), + "mail_source_imports", + ["started_at"], + ) + op.create_index( + op.f("ix_mail_source_imports_finished_at"), + "mail_source_imports", + ["finished_at"], + ) + + +def downgrade() -> None: + """Drop the mail_source_imports table.""" + op.drop_index(op.f("ix_mail_source_imports_finished_at"), table_name="mail_source_imports") + op.drop_index(op.f("ix_mail_source_imports_started_at"), table_name="mail_source_imports") + op.drop_index(op.f("ix_mail_source_imports_status"), table_name="mail_source_imports") + op.drop_index( + op.f("ix_mail_source_imports_mail_source_id"), + table_name="mail_source_imports", + ) + op.drop_index(op.f("ix_mail_source_imports_id"), table_name="mail_source_imports") + op.drop_table("mail_source_imports") diff --git a/backend/app/api/api_v1/endpoints/mail_sources.py b/backend/app/api/api_v1/endpoints/mail_sources.py index 7d1bed5..31a4c6e 100644 --- a/backend/app/api/api_v1/endpoints/mail_sources.py +++ b/backend/app/api/api_v1/endpoints/mail_sources.py @@ -7,6 +7,7 @@ persisting anything. Gmail API sources additionally have OAuth2 helper endpoints (authorize-url, callback, fetch). """ +import json import logging from datetime import datetime from typing import Any, Dict, List, Optional @@ -18,8 +19,10 @@ from sqlalchemy.orm import Session from app.core.database import get_db from app.core.security import require_admin_auth from app.models.mail_source import MailSource +from app.models.mail_source_import import MailSourceImport from app.services.gmail_client import GmailClient from app.services.imap_client import IMAPClient +from app.services.import_history import record_import_attempt router = APIRouter() logger = logging.getLogger(__name__) @@ -105,6 +108,24 @@ class GmailCallbackRequest(BaseModel): redirect_uri: str +class MailSourceImportResponse(BaseModel): + """Sanitized import-history entry for a mail source.""" + + id: int + mail_source_id: int + trigger: str + status: str + processed: int + reports_found: int + duplicate_reports: int + error_count: int + new_domains: List[str] + errors: List[str] + started_at: datetime + finished_at: datetime + created_at: datetime + + # --------------------------------------------------------------------------- # Helper # --------------------------------------------------------------------------- @@ -149,6 +170,38 @@ def _source_to_response(source: MailSource) -> MailSourceResponse: ) +def _decode_json_list(value: Optional[str]) -> List[str]: + """Decode a JSON list stored on import history rows.""" + if not value: + return [] + try: + decoded = json.loads(value) + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(decoded, list): + return [] + return [str(item) for item in decoded] + + +def _import_to_response(row: MailSourceImport) -> MailSourceImportResponse: + """Convert an import-history ORM row to an API response.""" + return MailSourceImportResponse( + id=row.id, + mail_source_id=row.mail_source_id, + trigger=row.trigger, + status=row.status, + processed=row.processed, + reports_found=row.reports_found, + duplicate_reports=row.duplicate_reports, + error_count=row.error_count, + new_domains=_decode_json_list(row.new_domains), + errors=_decode_json_list(row.errors), + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + ) + + # --------------------------------------------------------------------------- # Routes # --------------------------------------------------------------------------- @@ -205,6 +258,26 @@ async def get_mail_source( return _source_to_response(source) +@router.get("/{source_id}/imports", response_model=List[MailSourceImportResponse]) +async def list_mail_source_imports( + source_id: int, + limit: int = 20, + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +) -> List[MailSourceImportResponse]: + """Return recent sanitized import attempts for one mail source.""" + _get_source_or_404(source_id, db) + safe_limit = min(max(limit, 1), 100) + rows = ( + db.query(MailSourceImport) + .filter(MailSourceImport.mail_source_id == source_id) + .order_by(MailSourceImport.started_at.desc(), MailSourceImport.id.desc()) + .limit(safe_limit) + .all() + ) + return [_import_to_response(row) for row in rows] + + @router.put("/{source_id}", response_model=MailSourceResponse) async def update_mail_source( source_id: int, @@ -608,6 +681,7 @@ async def gmail_fetch_reports( already_ingested_ids=already, ) + started_at = datetime.utcnow() results = client.fetch_reports() # Persist updated ingested IDs and any refreshed tokens @@ -622,6 +696,7 @@ async def gmail_fetch_reports( source.gmail_refresh_token = refreshed["refresh_token"] source.last_checked = datetime.utcnow() + record_import_attempt(db, source, results, started_at=started_at, trigger="manual") db.commit() logger.info( diff --git a/backend/app/main.py b/backend/app/main.py index 8fb7929..05b6199 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates import app.models.domain # noqa: F401 – ensure Domain/UserDomain tables are registered +import app.models.mail_source_import # noqa: F401 – ensure import history table is registered import app.models.report # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered import app.models.setting # noqa: F401 – ensure Setting table is registered import app.models.user # noqa: F401 – ensure User table is registered @@ -22,6 +23,7 @@ from app.middleware.security import SecurityHeadersMiddleware from app.models.mail_source import MailSource # noqa: F401 – ensure table is registered from app.services.gmail_client import GmailClient from app.services.imap_client import IMAPClient +from app.services.import_history import record_import_attempt from app.services.report_store import ReportStore # Set up logging @@ -45,6 +47,7 @@ def _poll_single_imap_source(source: MailSource) -> None: password=source.password, delete_emails=False, ) + started_at = datetime.utcnow() results = imap_client.fetch_reports(days=9999) db = SessionLocal() @@ -52,6 +55,7 @@ def _poll_single_imap_source(source: MailSource) -> None: src = db.query(MailSource).get(source.id) if src: src.last_checked = datetime.utcnow() + record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled") db.commit() finally: db.close() @@ -95,6 +99,7 @@ def _poll_single_gmail_source(source: MailSource) -> None: already_ingested_ids=already, ) + started_at = datetime.utcnow() results = client.fetch_reports() db = SessionLocal() @@ -112,6 +117,7 @@ def _poll_single_gmail_source(source: MailSource) -> None: src.gmail_refresh_token = refreshed["refresh_token"] src.last_checked = datetime.utcnow() + record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled") db.commit() finally: db.close() @@ -517,9 +523,11 @@ def _trigger_poll_imap_source(source: MailSource, db) -> dict: password=source.password, delete_emails=False, ) + started_at = datetime.utcnow() results = imap_client.fetch_reports(days=7) last_check_time = datetime.now() source.last_checked = datetime.utcnow() + record_import_attempt(db, source, results, started_at=started_at, trigger="manual") db.commit() return { "source_id": source.id, @@ -543,6 +551,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict: refresh_token=source.gmail_refresh_token or "", already_ingested_ids=already, ) + started_at = datetime.utcnow() results = gmail_client.fetch_reports() last_check_time = datetime.now() @@ -555,6 +564,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict: if "refresh_token" in refreshed: source.gmail_refresh_token = refreshed["refresh_token"] source.last_checked = datetime.utcnow() + record_import_attempt(db, source, results, started_at=started_at, trigger="manual") db.commit() return { "source_id": source.id, diff --git a/backend/app/models/mail_source.py b/backend/app/models/mail_source.py index 70e3961..1e75907 100644 --- a/backend/app/models/mail_source.py +++ b/backend/app/models/mail_source.py @@ -1,6 +1,7 @@ from datetime import datetime from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text +from sqlalchemy.orm import relationship from app.core.database import Base @@ -63,5 +64,11 @@ class MailSource(Base): created_at = Column(DateTime, default=datetime.utcnow) updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + imports = relationship( + "MailSourceImport", + back_populates="mail_source", + cascade="all, delete-orphan", + ) + def __repr__(self): return f"" diff --git a/backend/app/models/mail_source_import.py b/backend/app/models/mail_source_import.py new file mode 100644 index 0000000..c9911b8 --- /dev/null +++ b/backend/app/models/mail_source_import.py @@ -0,0 +1,38 @@ +from datetime import datetime + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import relationship + +from app.core.database import Base + + +class MailSourceImport(Base): + """Sanitized audit record for one mail source import attempt.""" + + __tablename__ = "mail_source_imports" + + id = Column(Integer, primary_key=True, index=True) + mail_source_id = Column(Integer, ForeignKey("mail_sources.id"), nullable=False, index=True) + + trigger = Column(String, nullable=False, default="manual") + status = Column(String, nullable=False, index=True) + + processed = Column(Integer, nullable=False, default=0) + reports_found = Column(Integer, nullable=False, default=0) + duplicate_reports = Column(Integer, nullable=False, default=0) + error_count = Column(Integer, nullable=False, default=0) + + new_domains = Column(Text, nullable=True) + errors = Column(Text, nullable=True) + + started_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True) + finished_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + mail_source = relationship("MailSource", back_populates="imports") + + def __repr__(self): + return ( + f"" + ) diff --git a/backend/app/services/gmail_client.py b/backend/app/services/gmail_client.py index 3e49ad0..34fb157 100644 --- a/backend/app/services/gmail_client.py +++ b/backend/app/services/gmail_client.py @@ -207,6 +207,7 @@ class GmailClient: "success": True, "processed": 0, "reports_found": 0, + "duplicate_reports": 0, "new_domains": [], "errors": [], "new_ingested_ids": [], @@ -365,6 +366,8 @@ class GmailClient: if self._store_report_if_new(report): stats["reports_found"] += 1 reports_found += 1 + else: + stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1 except Exception as exc: # pylint: disable=broad-exception-caught logger.error("Failed to parse DMARC attachment %s: %s", filename, exc) stats["errors"].append(f"Failed to parse {filename}: {exc}") diff --git a/backend/app/services/imap_client.py b/backend/app/services/imap_client.py index fcd2130..5c0b91c 100644 --- a/backend/app/services/imap_client.py +++ b/backend/app/services/imap_client.py @@ -146,7 +146,7 @@ class IMAPClient: msg = email.message_from_bytes(raw_email) if self._is_dmarc_report_email(msg): - reports_found = self._process_attachments(msg) + reports_found = self._process_attachments(msg, stats) stats["reports_found"] += reports_found # Mark email as read (and optionally delete) @@ -178,6 +178,7 @@ class IMAPClient: "success": True, "processed": 0, "reports_found": 0, + "duplicate_reports": 0, "new_domains": [], "errors": [], } @@ -350,7 +351,7 @@ class IMAPClient: return False - def _process_attachments(self, msg: email.message.Message) -> int: + def _process_attachments(self, msg: email.message.Message, stats: dict | None = None) -> int: """ Process email attachments that might be DMARC reports @@ -394,6 +395,10 @@ class IMAPClient: report_id, domain, ) + if stats is not None: + stats["duplicate_reports"] = ( + stats.get("duplicate_reports", 0) + 1 + ) continue # Add the report to the store diff --git a/backend/app/services/import_history.py b/backend/app/services/import_history.py new file mode 100644 index 0000000..bac1df6 --- /dev/null +++ b/backend/app/services/import_history.py @@ -0,0 +1,54 @@ +import json +from datetime import datetime +from typing import Any, Dict, Iterable, Optional + +from sqlalchemy.orm import Session + +from app.models.mail_source import MailSource +from app.models.mail_source_import import MailSourceImport + +MAX_STORED_ERRORS = 10 +MAX_ERROR_LENGTH = 500 + + +def _sanitize_error(value: object) -> str: + """Return a compact, log-safe error string for storage and UI display.""" + text = str(value).replace("\r", "").replace("\n", " ").strip() + if len(text) > MAX_ERROR_LENGTH: + return text[: MAX_ERROR_LENGTH - 1] + "..." + return text + + +def _json_list(values: Optional[Iterable[Any]]) -> str: + return json.dumps([str(value) for value in values or []]) + + +def record_import_attempt( + db: Session, + source: MailSource, + results: Dict[str, Any], + *, + started_at: datetime, + trigger: str, +) -> MailSourceImport: + """Persist a sanitized summary of a mail source import attempt.""" + result_errors = list(results.get("errors") or []) + errors = [_sanitize_error(error) for error in result_errors[:MAX_STORED_ERRORS]] + success = bool(results.get("success", False)) + status = "success" if success and not errors else "warning" if success else "failed" + + attempt = MailSourceImport( + mail_source_id=source.id, + trigger=trigger, + status=status, + processed=int(results.get("processed", 0) or 0), + reports_found=int(results.get("reports_found", 0) or 0), + duplicate_reports=int(results.get("duplicate_reports", 0) or 0), + error_count=len(result_errors), + new_domains=_json_list(results.get("new_domains", [])), + errors=json.dumps(errors), + started_at=started_at, + finished_at=datetime.utcnow(), + ) + db.add(attempt) + return attempt diff --git a/backend/app/tests/conftest.py b/backend/app/tests/conftest.py index 87eaafa..d1c146f 100644 --- a/backend/app/tests/conftest.py +++ b/backend/app/tests/conftest.py @@ -8,6 +8,7 @@ from sqlalchemy.pool import StaticPool import app.models.domain # noqa: F401 # pylint: disable=unused-import import app.models.mail_source as _mail_source_model # noqa: F401 # pylint: disable=unused-import +import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-import import app.models.report # noqa: F401 # pylint: disable=unused-import import app.models.setting # noqa: F401 # pylint: disable=unused-import import app.models.user # noqa: F401 # pylint: disable=unused-import diff --git a/backend/app/tests/test_mail_sources.py b/backend/app/tests/test_mail_sources.py index 1705ec9..e754c1b 100644 --- a/backend/app/tests/test_mail_sources.py +++ b/backend/app/tests/test_mail_sources.py @@ -11,6 +11,7 @@ from fastapi.testclient import TestClient from sqlalchemy.orm import Session from app.models.mail_source import MailSource +from app.models.mail_source_import import MailSourceImport class TestMailSourceModel: @@ -75,6 +76,36 @@ class TestMailSourceModel: assert len(all_sources) == 3 +class TestMailSourceImportModel: + """Unit tests for persisted mail source import history.""" + + def test_create_import_history_row(self, db_session: Session): + source = MailSource(name="History Source", method="GMAIL_API") + db_session.add(source) + db_session.commit() + db_session.refresh(source) + + row = MailSourceImport( + mail_source_id=source.id, + trigger="manual", + status="warning", + processed=3, + reports_found=2, + duplicate_reports=1, + error_count=1, + new_domains='["example.com"]', + errors='["bad attachment"]', + ) + db_session.add(row) + db_session.commit() + db_session.refresh(row) + + assert row.id is not None + assert row.mail_source_id == source.id + assert row.duplicate_reports == 1 + assert row.mail_source.name == "History Source" + + class TestMailSourcesAPI: """Integration tests for /api/v1/mail-sources endpoints (no auth).""" @@ -219,6 +250,42 @@ class TestMailSourcesAPIAuthed: assert resp.json()["id"] == source_id assert resp.json()["name"] == "Get Test" + def test_list_import_history(self, authed_client: TestClient, db_session: Session): + create_resp = authed_client.post( + "/api/v1/mail-sources", json={"name": "History API", "method": "IMAP"} + ) + source_id = create_resp.json()["id"] + + db_session.add( + MailSourceImport( + mail_source_id=source_id, + trigger="manual", + status="warning", + processed=2, + reports_found=1, + duplicate_reports=1, + error_count=1, + new_domains='["example.com"]', + errors='["sanitized error"]', + ) + ) + db_session.commit() + + resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/imports") + + assert resp.status_code == 200 + data = resp.json() + assert len(data) == 1 + assert data[0]["mail_source_id"] == source_id + assert data[0]["status"] == "warning" + assert data[0]["duplicate_reports"] == 1 + assert data[0]["new_domains"] == ["example.com"] + assert data[0]["errors"] == ["sanitized error"] + + def test_list_import_history_unknown_source_returns_404(self, authed_client: TestClient): + resp = authed_client.get("/api/v1/mail-sources/99999/imports") + assert resp.status_code == 404 + def test_get_nonexistent_source_returns_404(self, authed_client: TestClient): resp = authed_client.get("/api/v1/mail-sources/99999") assert resp.status_code == 404 diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index 7d65c0f..4a89769 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -12,7 +12,7 @@ Complete: - Upload validation and archive safety checks. - IMAP mailbox ingestion. - Gmail OAuth ingestion. -- Persistent database models and migrations. +- Persistent database models and migrations for domains, reports, records, settings, users, and mail sources. - Domain, report, settings, and mail source APIs. - Dashboard and domain detail views. - Logto auth integration and local development auth-disabled mode. @@ -23,13 +23,30 @@ Recently improved: - Gmail import now uses the same parser path as uploads and IMAP. - Gmail and IMAP imports now skip duplicate domain/report IDs. - Tests cover Google-style DMARC ZIP attachment imports. +- Mail source imports now create sanitized import-history records for manual and scheduled polls. -## Active Milestone: Reporting Quality and Import Confidence +Important gap: +- Parsed DMARC report data is still served primarily from the in-memory `ReportStore`. The database schema exists, but report upload/import paths and dashboard read paths must be completed before the persistence milestone can be called done. + +## Active Milestone: Finish Report Persistence + +Objective: complete the database-backed report storage promised by Milestone 3. + +Priority tasks: +- Add a report persistence service that converts parsed DMARC report dicts into `Domain`, `DMARCReport`, and `ReportRecord` rows. +- Load or query persisted reports for dashboard, domain, and report endpoints. +- Add duplicate report detection against the database. +- Keep tests covering upload, Gmail import, IMAP import, and restart-style reload behavior. + +Quality bar: +- Uploading or importing a report survives application restart and remains visible in report/domain endpoints. + +## Next Milestone: Reporting Quality and Import Confidence Objective: make mailbox imports auditable and make report totals trustworthy. Priority tasks: -- Persist import attempts with message ID, source, attachment filename, outcome, and sanitized error details. +- Expand import attempts with message ID, source, attachment filename, outcome, and sanitized error details. - Show import history on mail source detail pages. - Report duplicate skips separately from parse failures. - Add backfill controls for Gmail and IMAP sources. @@ -40,7 +57,7 @@ Quality bar: - Parse failures must be visible and actionable. - The user should be able to tell whether a mail source is healthy without reading logs. -## Next Milestone: Meaningful Reports +## Following Milestone: Meaningful Reports Objective: turn parsed DMARC data into administrator-friendly reports. diff --git a/docs/milestones.md b/docs/milestones.md index 4136416..04b8cb2 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -29,19 +29,25 @@ Delivered: - Duplicate report protection for both Gmail and IMAP imports. - Background polling and manual poll hooks for configured mail sources. -## Milestone 3: Persistence, Domain Management, and Auth Foundation - Complete +## Milestone 3: Database Foundation, Domain Management, and Auth Foundation - In Progress -Status: Complete +Status: In progress Delivered: - SQLAlchemy models and Alembic migrations. - SQLite/PostgreSQL-compatible database configuration. - Domain management APIs and UI. -- Report and source persistence. +- Report and source database models. - Settings and mail source persistence. - Logto-based auth integration plus an explicit local development auth-disabled mode. - Security middleware, safer default secret generation, and security-focused tests. +Remaining before this milestone is complete: +- Persist parsed DMARC reports and report records through the database-backed models. +- Load or query persisted reports after restart so dashboards do not depend on process memory. +- Move report/domain summary endpoints from the in-memory `ReportStore` to database queries. +- Keep duplicate report detection consistent across upload, IMAP, and Gmail after persistence is enabled. + ## Milestone 4: Reporting Quality and Import Confidence - In Progress Status: In progress @@ -52,10 +58,11 @@ Recently delivered: - Gmail import now handles real inbox metadata patterns and Google-style ZIP filenames. - Gmail/IMAP imports skip duplicate report IDs to avoid inflated totals. - Tests now cover a real Google-style ZIP attachment path rather than only mocked parser behavior. +- Mail source imports now persist sanitized import-history records for manual and scheduled polls. Next tasks: +- Finish Milestone 3 report persistence before expanding report features. - Add per-import result details: skipped duplicates, parse failures, unsupported attachments, and imported report IDs. -- Store failed import attempts with enough metadata to retry or diagnose them without exposing secrets. - Add a UI import history view for each mail source. - Add mailbox search controls for date range/backfill without requiring code changes. - Improve source aggregation so each sender IP keeps pass/fail totals instead of only the latest result. diff --git a/docs/todo.md b/docs/todo.md index 43f8610..6631dd1 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -88,8 +88,8 @@ This file tracks the specific implementation tasks for each milestone of the DMA ### Duplicate and Error Handling - [x] Skip duplicate domain/report IDs during Gmail imports - [x] Skip duplicate domain/report IDs during IMAP imports -- [ ] Persist sanitized import errors for UI review -- [ ] Count duplicate skips separately from parse failures +- [x] Persist sanitized import errors for API/UI review +- [x] Count duplicate skips separately from parse failures - [ ] Add retry/backfill controls per mail source ## Milestone 3: Database Integration @@ -101,11 +101,14 @@ This file tracks the specific implementation tasks for each milestone of the DMA - [x] Implement data access layer ### Model Migration -- [x] Convert in-memory models to database models +- [ ] Convert report ingestion and dashboard reads from in-memory storage to database-backed storage - [x] Create Domain table - [x] Create AggregateReport table - [x] Create ReportRecord table for sender details - [x] Implement relationships between models +- [ ] Persist parsed upload reports to `dmarc_reports` and `report_records` +- [ ] Persist parsed Gmail/IMAP reports to `dmarc_reports` and `report_records` +- [ ] Load/query persisted reports after app restart ### Domain Management - [x] Create UI for adding/editing domains