gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add mail source import history

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-05-22 19:29:49 +02:00
parent 07b2ea400e
commit 4fc69602b3
14 changed files with 375 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/api_v1/endpoints/mail_sources.py
+75
View File
@@ -7,6 +7,7 @@ persisting anything. Gmail API sources additionally have OAuth2 helper
endpoints (authorize-url, callback, fetch).
"""
import json
import logging
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -18,8 +19,10 @@ from sqlalchemy.orm import Session
from app.core.database import get_db
from app.core.security import require_admin_auth
from app.models.mail_source import MailSource
from app.models.mail_source_import import MailSourceImport
from app.services.gmail_client import GmailClient
from app.services.imap_client import IMAPClient
from app.services.import_history import record_import_attempt
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -105,6 +108,24 @@ class GmailCallbackRequest(BaseModel):
redirect_uri: str
class MailSourceImportResponse(BaseModel):
"""Sanitized import-history entry for a mail source."""
id: int
mail_source_id: int
trigger: str
status: str
processed: int
reports_found: int
duplicate_reports: int
error_count: int
new_domains: List[str]
errors: List[str]
started_at: datetime
finished_at: datetime
created_at: datetime
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
@@ -149,6 +170,38 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
)
def _decode_json_list(value: Optional[str]) -> List[str]:
"""Decode a JSON list stored on import history rows."""
if not value:
return []
try:
decoded = json.loads(value)
except (json.JSONDecodeError, TypeError):
return []
if not isinstance(decoded, list):
return []
return [str(item) for item in decoded]
def _import_to_response(row: MailSourceImport) -> MailSourceImportResponse:
"""Convert an import-history ORM row to an API response."""
return MailSourceImportResponse(
id=row.id,
mail_source_id=row.mail_source_id,
trigger=row.trigger,
status=row.status,
processed=row.processed,
reports_found=row.reports_found,
duplicate_reports=row.duplicate_reports,
error_count=row.error_count,
new_domains=_decode_json_list(row.new_domains),
errors=_decode_json_list(row.errors),
started_at=row.started_at,
finished_at=row.finished_at,
created_at=row.created_at,
)
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
@@ -205,6 +258,26 @@ async def get_mail_source(
return _source_to_response(source)
@router.get("/{source_id}/imports", response_model=List[MailSourceImportResponse])
async def list_mail_source_imports(
source_id: int,
limit: int = 20,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> List[MailSourceImportResponse]:
"""Return recent sanitized import attempts for one mail source."""
_get_source_or_404(source_id, db)
safe_limit = min(max(limit, 1), 100)
rows = (
db.query(MailSourceImport)
.filter(MailSourceImport.mail_source_id == source_id)
.order_by(MailSourceImport.started_at.desc(), MailSourceImport.id.desc())
.limit(safe_limit)
.all()
)
return [_import_to_response(row) for row in rows]
@router.put("/{source_id}", response_model=MailSourceResponse)
async def update_mail_source(
source_id: int,
@@ -608,6 +681,7 @@ async def gmail_fetch_reports(
already_ingested_ids=already,
)
started_at = datetime.utcnow()
results = client.fetch_reports()
# Persist updated ingested IDs and any refreshed tokens
@@ -622,6 +696,7 @@ async def gmail_fetch_reports(
source.gmail_refresh_token = refreshed["refresh_token"]
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
db.commit()
logger.info(
backend/app/main.py
+10
View File
@@ -10,6 +10,7 @@ from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
import app.models.domain # noqa: F401 – ensure Domain/UserDomain tables are registered
import app.models.mail_source_import # noqa: F401 – ensure import history table is registered
import app.models.report # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered
import app.models.setting # noqa: F401 – ensure Setting table is registered
import app.models.user # noqa: F401 – ensure User table is registered
@@ -22,6 +23,7 @@ from app.middleware.security import SecurityHeadersMiddleware
from app.models.mail_source import MailSource # noqa: F401 – ensure table is registered
from app.services.gmail_client import GmailClient
from app.services.imap_client import IMAPClient
from app.services.import_history import record_import_attempt
from app.services.report_store import ReportStore
# Set up logging
@@ -45,6 +47,7 @@ def _poll_single_imap_source(source: MailSource) -> None:
password=source.password,
delete_emails=False,
)
started_at = datetime.utcnow()
results = imap_client.fetch_reports(days=9999)
db = SessionLocal()
@@ -52,6 +55,7 @@ def _poll_single_imap_source(source: MailSource) -> None:
src = db.query(MailSource).get(source.id)
if src:
src.last_checked = datetime.utcnow()
record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
db.commit()
finally:
db.close()
@@ -95,6 +99,7 @@ def _poll_single_gmail_source(source: MailSource) -> None:
already_ingested_ids=already,
)
started_at = datetime.utcnow()
results = client.fetch_reports()
db = SessionLocal()
@@ -112,6 +117,7 @@ def _poll_single_gmail_source(source: MailSource) -> None:
src.gmail_refresh_token = refreshed["refresh_token"]
src.last_checked = datetime.utcnow()
record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
db.commit()
finally:
db.close()
@@ -517,9 +523,11 @@ def _trigger_poll_imap_source(source: MailSource, db) -> dict:
password=source.password,
delete_emails=False,
)
started_at = datetime.utcnow()
results = imap_client.fetch_reports(days=7)
last_check_time = datetime.now()
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
db.commit()
return {
"source_id": source.id,
@@ -543,6 +551,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
refresh_token=source.gmail_refresh_token or "",
already_ingested_ids=already,
)
started_at = datetime.utcnow()
results = gmail_client.fetch_reports()
last_check_time = datetime.now()
@@ -555,6 +564,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
if "refresh_token" in refreshed:
source.gmail_refresh_token = refreshed["refresh_token"]
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
db.commit()
return {
"source_id": source.id,
backend/app/models/mail_source.py
+7
View File
@@ -1,6 +1,7 @@
from datetime import datetime
from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text
from sqlalchemy.orm import relationship
from app.core.database import Base
@@ -63,5 +64,11 @@ class MailSource(Base):
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
imports = relationship(
"MailSourceImport",
back_populates="mail_source",
cascade="all, delete-orphan",
)
def __repr__(self):
return f"<MailSource id={self.id} name={self.name!r} method={self.method!r}>"
backend/app/models/mail_source_import.py
+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
from sqlalchemy.orm import relationship
from app.core.database import Base
class MailSourceImport(Base):
"""Sanitized audit record for one mail source import attempt."""
__tablename__ = "mail_source_imports"
id = Column(Integer, primary_key=True, index=True)
mail_source_id = Column(Integer, ForeignKey("mail_sources.id"), nullable=False, index=True)
trigger = Column(String, nullable=False, default="manual")
status = Column(String, nullable=False, index=True)
processed = Column(Integer, nullable=False, default=0)
reports_found = Column(Integer, nullable=False, default=0)
duplicate_reports = Column(Integer, nullable=False, default=0)
error_count = Column(Integer, nullable=False, default=0)
new_domains = Column(Text, nullable=True)
errors = Column(Text, nullable=True)
started_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
finished_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
mail_source = relationship("MailSource", back_populates="imports")
def __repr__(self):
return (
f"<MailSourceImport id={self.id} source={self.mail_source_id} "
f"status={self.status!r}>"
)
backend/app/services/gmail_client.py
+3
View File
@@ -207,6 +207,7 @@ class GmailClient:
"success": True,
"processed": 0,
"reports_found": 0,
"duplicate_reports": 0,
"new_domains": [],
"errors": [],
"new_ingested_ids": [],
@@ -365,6 +366,8 @@ class GmailClient:
if self._store_report_if_new(report):
stats["reports_found"] += 1
reports_found += 1
else:
stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error("Failed to parse DMARC attachment %s: %s", filename, exc)
stats["errors"].append(f"Failed to parse {filename}: {exc}")
backend/app/services/imap_client.py
+7 -2
View File
@@ -146,7 +146,7 @@ class IMAPClient:
msg = email.message_from_bytes(raw_email)
if self._is_dmarc_report_email(msg):
reports_found = self._process_attachments(msg)
reports_found = self._process_attachments(msg, stats)
stats["reports_found"] += reports_found
# Mark email as read (and optionally delete)
@@ -178,6 +178,7 @@ class IMAPClient:
"success": True,
"processed": 0,
"reports_found": 0,
"duplicate_reports": 0,
"new_domains": [],
"errors": [],
}
@@ -350,7 +351,7 @@ class IMAPClient:
return False
def _process_attachments(self, msg: email.message.Message) -> int:
def _process_attachments(self, msg: email.message.Message, stats: dict | None = None) -> int:
"""
Process email attachments that might be DMARC reports
@@ -394,6 +395,10 @@ class IMAPClient:
report_id,
domain,
)
if stats is not None:
stats["duplicate_reports"] = (
stats.get("duplicate_reports", 0) + 1
)
continue
# Add the report to the store
backend/app/services/import_history.py
+54
View File
@@ -0,0 +1,54 @@
import json
from datetime import datetime
from typing import Any, Dict, Iterable, Optional
from sqlalchemy.orm import Session
from app.models.mail_source import MailSource
from app.models.mail_source_import import MailSourceImport
MAX_STORED_ERRORS = 10
MAX_ERROR_LENGTH = 500
def _sanitize_error(value: object) -> str:
"""Return a compact, log-safe error string for storage and UI display."""
text = str(value).replace("\r", "").replace("\n", " ").strip()
if len(text) > MAX_ERROR_LENGTH:
return text[: MAX_ERROR_LENGTH - 1] + "..."
return text
def _json_list(values: Optional[Iterable[Any]]) -> str:
return json.dumps([str(value) for value in values or []])
def record_import_attempt(
db: Session,
source: MailSource,
results: Dict[str, Any],
*,
started_at: datetime,
trigger: str,
) -> MailSourceImport:
"""Persist a sanitized summary of a mail source import attempt."""
result_errors = list(results.get("errors") or [])
errors = [_sanitize_error(error) for error in result_errors[:MAX_STORED_ERRORS]]
success = bool(results.get("success", False))
status = "success" if success and not errors else "warning" if success else "failed"
attempt = MailSourceImport(
mail_source_id=source.id,
trigger=trigger,
status=status,
processed=int(results.get("processed", 0) or 0),
reports_found=int(results.get("reports_found", 0) or 0),
duplicate_reports=int(results.get("duplicate_reports", 0) or 0),
error_count=len(result_errors),
new_domains=_json_list(results.get("new_domains", [])),
errors=json.dumps(errors),
started_at=started_at,
finished_at=datetime.utcnow(),
)
db.add(attempt)
return attempt
backend/app/tests/conftest.py
+1
View File
@@ -8,6 +8,7 @@ from sqlalchemy.pool import StaticPool
import app.models.domain # noqa: F401 # pylint: disable=unused-import
import app.models.mail_source as _mail_source_model # noqa: F401 # pylint: disable=unused-import
import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-import
import app.models.report # noqa: F401 # pylint: disable=unused-import
import app.models.setting # noqa: F401 # pylint: disable=unused-import
import app.models.user # noqa: F401 # pylint: disable=unused-import
backend/app/tests/test_mail_sources.py
+67
View File
@@ -11,6 +11,7 @@ from fastapi.testclient import TestClient
from sqlalchemy.orm import Session
from app.models.mail_source import MailSource
from app.models.mail_source_import import MailSourceImport
class TestMailSourceModel:
@@ -75,6 +76,36 @@ class TestMailSourceModel:
assert len(all_sources) == 3
class TestMailSourceImportModel:
"""Unit tests for persisted mail source import history."""
def test_create_import_history_row(self, db_session: Session):
source = MailSource(name="History Source", method="GMAIL_API")
db_session.add(source)
db_session.commit()
db_session.refresh(source)
row = MailSourceImport(
mail_source_id=source.id,
trigger="manual",
status="warning",
processed=3,
reports_found=2,
duplicate_reports=1,
error_count=1,
new_domains='["example.com"]',
errors='["bad attachment"]',
)
db_session.add(row)
db_session.commit()
db_session.refresh(row)
assert row.id is not None
assert row.mail_source_id == source.id
assert row.duplicate_reports == 1
assert row.mail_source.name == "History Source"
class TestMailSourcesAPI:
"""Integration tests for /api/v1/mail-sources endpoints (no auth)."""
@@ -219,6 +250,42 @@ class TestMailSourcesAPIAuthed:
assert resp.json()["id"] == source_id
assert resp.json()["name"] == "Get Test"
def test_list_import_history(self, authed_client: TestClient, db_session: Session):
create_resp = authed_client.post(
"/api/v1/mail-sources", json={"name": "History API", "method": "IMAP"}
)
source_id = create_resp.json()["id"]
db_session.add(
MailSourceImport(
mail_source_id=source_id,
trigger="manual",
status="warning",
processed=2,
reports_found=1,
duplicate_reports=1,
error_count=1,
new_domains='["example.com"]',
errors='["sanitized error"]',
)
)
db_session.commit()
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/imports")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 1
assert data[0]["mail_source_id"] == source_id
assert data[0]["status"] == "warning"
assert data[0]["duplicate_reports"] == 1
assert data[0]["new_domains"] == ["example.com"]
assert data[0]["errors"] == ["sanitized error"]
def test_list_import_history_unknown_source_returns_404(self, authed_client: TestClient):
resp = authed_client.get("/api/v1/mail-sources/99999/imports")
assert resp.status_code == 404
def test_get_nonexistent_source_returns_404(self, authed_client: TestClient):
resp = authed_client.get("/api/v1/mail-sources/99999")
assert resp.status_code == 404