feat: add Microsoft 365 Graph mail source

This commit is contained in:
Christian Krakau-Louis
2026-05-23 14:49:54 +02:00
parent 08af80fa0e
commit 5caefb06db
12 changed files with 2379 additions and 21 deletions
@@ -0,0 +1,39 @@
"""Add Microsoft 365 Graph mail source fields.
Revision ID: f3a4b5c6d7e8
Revises: e2f3a4b5c6d7
Create Date: 2026-05-23 00:00:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "f3a4b5c6d7e8"
down_revision: Union[str, Sequence[str], None] = "e2f3a4b5c6d7"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("mail_sources", sa.Column("m365_tenant_id", sa.String(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_client_id", sa.String(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_client_secret", sa.Text(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_access_token", sa.Text(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_refresh_token", sa.Text(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_mailbox", sa.String(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_email", sa.String(), nullable=True))
op.add_column("mail_sources", sa.Column("m365_ingested_ids", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("mail_sources", "m365_ingested_ids")
op.drop_column("mail_sources", "m365_email")
op.drop_column("mail_sources", "m365_mailbox")
op.drop_column("mail_sources", "m365_refresh_token")
op.drop_column("mail_sources", "m365_access_token")
op.drop_column("mail_sources", "m365_client_secret")
op.drop_column("mail_sources", "m365_client_id")
op.drop_column("mail_sources", "m365_tenant_id")
+384 -11
View File
@@ -3,8 +3,8 @@ Mail Sources API endpoints.
Provides CRUD operations for MailSource objects stored in the database, plus
a *test-connection* action that validates the supplied credentials without
persisting anything. Gmail API sources additionally have OAuth2 helper
endpoints (authorize-url, callback, fetch).
persisting anything. Gmail API and Microsoft 365 sources additionally have
OAuth2 helper endpoints (authorize-url, callback, fetch).
"""
import json
@@ -24,6 +24,7 @@ from app.models.mail_source_import import MailSourceImport
from app.services.gmail_client import GmailClient
from app.services.imap_client import IMAPClient
from app.services.import_history import record_import_attempt
from app.services.microsoft_graph_client import MicrosoftGraphClient
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -38,7 +39,7 @@ class MailSourceBase(BaseModel):
"""Fields shared by create and update payloads."""
name: str
method: str = "IMAP" # IMAP | POP3 | GMAIL_API
method: str = "IMAP" # IMAP | POP3 | GMAIL_API | M365_GRAPH
server: Optional[str] = None
port: int = 993
username: Optional[str] = None
@@ -50,6 +51,11 @@ class MailSourceBase(BaseModel):
# Gmail API OAuth2 fields (only relevant when method == GMAIL_API)
gmail_client_id: Optional[str] = None
gmail_client_secret: Optional[str] = None
# Microsoft 365 Graph OAuth2 fields (only relevant when method == M365_GRAPH)
m365_tenant_id: Optional[str] = "common"
m365_client_id: Optional[str] = None
m365_client_secret: Optional[str] = None
m365_mailbox: Optional[str] = None
class MailSourceCreate(MailSourceBase):
@@ -71,6 +77,10 @@ class MailSourceUpdate(BaseModel):
enabled: Optional[bool] = None
gmail_client_id: Optional[str] = None
gmail_client_secret: Optional[str] = None
m365_tenant_id: Optional[str] = None
m365_client_id: Optional[str] = None
m365_client_secret: Optional[str] = None
m365_mailbox: Optional[str] = None
class MailSourceResponse(MailSourceBase):
@@ -86,6 +96,9 @@ class MailSourceResponse(MailSourceBase):
gmail_email: Optional[str] = None
# Indicate whether OAuth tokens are present (without exposing them)
gmail_connected: bool = False
# Microsoft 365: show the authorised account and token state, but not tokens
m365_email: Optional[str] = None
m365_connected: bool = False
class Config:
from_attributes = True
@@ -109,6 +122,13 @@ class GmailCallbackRequest(BaseModel):
redirect_uri: str
class M365CallbackRequest(BaseModel):
"""Payload for the Microsoft 365 OAuth2 callback endpoint."""
code: str
redirect_uri: str
class MailSourceImportResponse(BaseModel):
"""Sanitized import-history entry for a mail source."""
@@ -151,7 +171,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
"auth_required": {
"summary": "The mailbox has not been connected yet.",
"recovery_steps": [
"Use the Connect Gmail action to complete authorization.",
"Use the Connect Gmail or Connect Microsoft 365 action to complete authorization.",
"Confirm the authorized mailbox is the one that receives DMARC aggregate reports.",
],
},
@@ -159,7 +179,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
"summary": "The saved authorization is expired, revoked, or no longer accepted.",
"recovery_steps": [
"Reconnect the mailbox from Mail Sources.",
"If your provider shows a consent screen, approve Gmail read-only access again.",
"If your provider shows a consent screen, approve read-only mailbox access again.",
],
},
"authentication": {
@@ -173,7 +193,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
"summary": "The account is connected but does not have enough mailbox access.",
"recovery_steps": [
"Grant read access for the mailbox that receives DMARC reports.",
"For Gmail, reconnect and approve the requested Gmail read-only scope.",
"For OAuth sources, reconnect and approve the requested read-only mail scope.",
],
},
"connectivity": {
@@ -200,7 +220,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
"missing_config": {
"summary": "Required connection settings are missing.",
"recovery_steps": [
"Fill in the server, username, and password or complete Gmail authorization.",
"Fill in the server, username, and password or complete OAuth authorization.",
"Save the source before running stored-source tests.",
],
},
@@ -227,12 +247,23 @@ def _diagnostic_category(message: str, details: Optional[object] = None) -> str:
if any(term in text for term in ("not yet authorised", "not yet authorized", "complete oauth")):
return "auth_required"
if any(
term in text for term in ("expired", "revoked", "invalid_grant", "refresh token", "oauth")
term in text
for term in (
"expired",
"revoked",
"invalid_grant",
"interaction_required",
"refresh token",
"oauth",
)
):
return "auth_expired"
if any(term in text for term in ("rate", "quota", "throttl", "too many", "429")):
return "throttling"
if any(term in text for term in ("scope", "permission", "access denied", "insufficient")):
if any(
term in text
for term in ("scope", "permission", "access denied", "insufficient", "forbidden", "403")
):
return "permissions"
if any(term in text for term in ("mailbox", "folder", "select failed", "does not exist")):
return "mailbox_not_found"
@@ -308,6 +339,14 @@ def _get_source_or_404(source_id: int, db: Session) -> MailSource:
return source
def _safe_attr(source: MailSource, name: str, default: Any = None) -> Any:
"""Read optional source attributes without letting test doubles invent fields."""
value = getattr(source, name, default)
if value.__class__.__module__.startswith("unittest.mock"):
return default
return value
def _source_to_response(source: MailSource) -> MailSourceResponse:
"""Convert ORM row to response schema, masking the stored password."""
return MailSourceResponse(
@@ -329,6 +368,12 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
gmail_client_secret="**redacted**" if source.gmail_client_secret else None,
gmail_email=source.gmail_email,
gmail_connected=bool(source.gmail_access_token),
m365_tenant_id=_safe_attr(source, "m365_tenant_id", "common") or "common",
m365_client_id=_safe_attr(source, "m365_client_id"),
m365_client_secret=("**redacted**" if _safe_attr(source, "m365_client_secret") else None),
m365_mailbox=_safe_attr(source, "m365_mailbox"),
m365_email=_safe_attr(source, "m365_email"),
m365_connected=bool(_safe_attr(source, "m365_access_token")),
)
@@ -437,6 +482,46 @@ def _fetch_gmail_source(source: MailSource, db: Session) -> Dict[str, Any]:
return results
def _fetch_m365_source(source: MailSource, db: Session) -> Dict[str, Any]:
"""Run one Microsoft 365 Graph import and persist source/import metadata."""
if not source.m365_access_token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Microsoft 365 account not yet authorised. Complete OAuth2 flow first.",
)
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
client = MicrosoftGraphClient(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
access_token=source.m365_access_token,
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
already_ingested_ids=already,
db=db,
)
started_at = datetime.utcnow()
results = client.fetch_reports()
if results.get("new_ingested_ids"):
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
refreshed = client.get_refreshed_tokens()
if refreshed:
source.m365_access_token = refreshed["access_token"]
if "refresh_token" in refreshed:
source.m365_refresh_token = refreshed["refresh_token"]
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
db.commit()
return results
def _fetch_imap_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
"""Run one IMAP import and persist source/import metadata."""
client = IMAPClient(
@@ -459,6 +544,8 @@ def _fetch_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
"""Dispatch a manual fetch for one configured mail source."""
if source.method == "GMAIL_API":
return _fetch_gmail_source(source, db)
if source.method == "M365_GRAPH":
return _fetch_m365_source(source, db)
if source.method == "IMAP":
return _fetch_imap_source(source, db, days)
raise HTTPException(
@@ -502,6 +589,10 @@ async def create_mail_source(
enabled=payload.enabled,
gmail_client_id=payload.gmail_client_id,
gmail_client_secret=payload.gmail_client_secret,
m365_tenant_id=payload.m365_tenant_id or "common",
m365_client_id=payload.m365_client_id,
m365_client_secret=payload.m365_client_secret,
m365_mailbox=payload.m365_mailbox,
)
db.add(source)
db.commit()
@@ -572,7 +663,7 @@ async def fetch_mail_source(
_redact_sensitive_text(err),
)
return _fetch_response(source, results)
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
@router.put("/{source_id}", response_model=MailSourceResponse)
@@ -628,7 +719,7 @@ async def toggle_mail_source(
@router.post("/{source_id}/test", response_model=Dict[str, Any])
async def test_stored_mail_source(
async def test_stored_mail_source( # noqa: C901
source_id: int,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
@@ -671,6 +762,48 @@ async def test_stored_mail_source(
details=exc,
)
if source.method == "M365_GRAPH":
if not source.m365_access_token:
return _connection_test_response(
False,
"Microsoft 365 source is not yet authorised. "
"Use the Connect Microsoft 365 button to complete OAuth2 authorisation.",
)
try:
graph_client = MicrosoftGraphClient(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
access_token=source.m365_access_token,
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
)
stats = graph_client.test_connection()
refreshed = graph_client.get_refreshed_tokens()
if refreshed:
source.m365_access_token = refreshed["access_token"]
if "refresh_token" in refreshed:
source.m365_refresh_token = refreshed["refresh_token"]
source.last_checked = datetime.utcnow()
db.commit()
return _connection_test_response(
True,
f"Microsoft 365 credentials are valid (account: {source.m365_email or 'unknown'}).",
stats=stats,
)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(
"Microsoft 365 Graph test failed for source id=%d: %s",
int(source_id),
_redact_sensitive_text(exc),
)
return _connection_test_response(
False,
"Microsoft 365 test failed. The saved authorization may need attention.",
details=exc,
)
if source.method != "IMAP":
return _connection_test_response(
False,
@@ -722,6 +855,246 @@ async def test_connection_adhoc(
return _connection_test_response(success, message, stats)
# ---------------------------------------------------------------------------
# Microsoft 365 / Graph OAuth2 routes
# ---------------------------------------------------------------------------
@router.get("/{source_id}/m365/authorize-url", response_model=Dict[str, Any])
async def m365_authorize_url(
source_id: int,
request: Request,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> Dict[str, Any]:
"""Return a Microsoft identity platform authorization URL for M365_GRAPH."""
source = _get_source_or_404(source_id, db)
if source.method != "M365_GRAPH":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This endpoint is only available for M365_GRAPH sources.",
)
if not source.m365_client_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="m365_client_id is not configured for this source.",
)
base_url = str(request.base_url).rstrip("/")
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
auth_url = MicrosoftGraphClient.build_authorization_url(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id,
redirect_uri=redirect_uri,
state=str(source_id),
)
return {"authorization_url": auth_url, "redirect_uri": redirect_uri}
@router.get("/{source_id}/m365/callback")
async def m365_oauth_callback(
source_id: int,
request: Request,
db: Session = Depends(get_db),
) -> Any:
"""Handle the Microsoft identity platform OAuth2 redirect."""
from fastapi.responses import HTMLResponse
code = request.query_params.get("code")
error = request.query_params.get("error")
if error or not code:
html = (
"<html><body><p>Microsoft 365 authorisation failed: "
f"{error or 'no code received'}. "
"You may close this window.</p></body></html>"
)
return HTMLResponse(content=html, status_code=400)
source = db.query(MailSource).filter(MailSource.id == source_id).first()
if source is None or source.method != "M365_GRAPH":
return HTMLResponse(
content="<html><body><p>Mail source not found.</p></body></html>",
status_code=404,
)
base_url = str(request.base_url).rstrip("/")
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
try:
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
code=code,
redirect_uri=redirect_uri,
)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(
"Microsoft 365 token exchange error for source id=%d: %s",
int(source_id),
_redact_sensitive_text(exc),
)
html = (
"<html><body><p>Token exchange failed. "
"Please close this window and try again.</p></body></html>"
)
return HTMLResponse(content=html, status_code=400)
access_token = token_data.get("access_token")
refresh_token = token_data.get("refresh_token")
if not access_token:
return HTMLResponse(
content="<html><body><p>No access token returned by Microsoft.</p></body></html>",
status_code=400,
)
m365_email = MicrosoftGraphClient.get_account_email(access_token)
source.m365_access_token = access_token
if refresh_token:
source.m365_refresh_token = refresh_token
if m365_email:
source.m365_email = m365_email
source.updated_at = datetime.utcnow()
db.commit()
logger.info(
"Microsoft 365 OAuth2 authorisation complete for source id=%d (account=%s)",
int(source_id),
_sanitize_for_log(m365_email or "unknown"),
)
html = (
"<html><body>"
"<p>Microsoft 365 account connected successfully"
f"{(' (' + m365_email + ')') if m365_email else ''}. "
"You may close this window.</p>"
"<script>window.close();</script>"
"</body></html>"
)
return HTMLResponse(content=html)
@router.post("/{source_id}/m365/callback", response_model=MailSourceResponse)
async def m365_oauth_callback_post(
source_id: int,
payload: M365CallbackRequest,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> MailSourceResponse:
"""Exchange a Microsoft OAuth2 authorization code for Graph tokens."""
source = _get_source_or_404(source_id, db)
if source.method != "M365_GRAPH":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This endpoint is only available for M365_GRAPH sources.",
)
try:
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
code=payload.code,
redirect_uri=payload.redirect_uri,
)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(
"Microsoft 365 token exchange error for source id=%d: %s",
int(source_id),
_redact_sensitive_text(exc),
)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=(
"Token exchange failed. Please check the Microsoft 365 "
"connection settings and try again."
),
) from exc
access_token = token_data.get("access_token")
refresh_token = token_data.get("refresh_token")
if not access_token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Microsoft did not return an access token.",
)
m365_email = MicrosoftGraphClient.get_account_email(access_token)
source.m365_access_token = access_token
if refresh_token:
source.m365_refresh_token = refresh_token
if m365_email:
source.m365_email = m365_email
source.updated_at = datetime.utcnow()
db.commit()
db.refresh(source)
logger.info(
"Microsoft 365 OAuth2 tokens saved for source id=%d (account=%s)",
int(source_id),
_sanitize_for_log(m365_email or "unknown"),
)
return _source_to_response(source)
@router.post("/{source_id}/m365/fetch", response_model=Dict[str, Any])
async def m365_fetch_reports(
source_id: int,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> Dict[str, Any]:
"""Manually trigger a Microsoft 365 Graph DMARC report fetch."""
source = _get_source_or_404(source_id, db)
if source.method != "M365_GRAPH":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This endpoint is only available for M365_GRAPH sources.",
)
results = _fetch_m365_source(source, db)
logger.info(
"Microsoft 365 fetch for source id=%d: processed=%d reports_found=%d",
int(source_id),
int(results.get("processed", 0)),
int(results.get("reports_found", 0)),
)
for err in results.get("errors", []):
logger.warning(
"Microsoft 365 fetch warning for source id=%d: %s",
int(source_id),
_redact_sensitive_text(err),
)
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
@router.delete("/{source_id}/m365/connection", status_code=status.HTTP_204_NO_CONTENT)
async def m365_disconnect(
source_id: int,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> None:
"""Clear the stored Microsoft Graph OAuth2 tokens for this source."""
source = _get_source_or_404(source_id, db)
if source.method != "M365_GRAPH":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This endpoint is only available for M365_GRAPH sources.",
)
source.m365_access_token = None
source.m365_refresh_token = None
source.m365_email = None
source.updated_at = datetime.utcnow()
db.commit()
logger.info("Microsoft 365 tokens cleared for source id=%d", int(source_id))
# ---------------------------------------------------------------------------
# Gmail API OAuth2 routes
# ---------------------------------------------------------------------------
+136 -2
View File
@@ -30,6 +30,7 @@ from app.models.mail_source import MailSource # noqa: F401 ensure table is
from app.services.gmail_client import GmailClient
from app.services.imap_client import IMAPClient
from app.services.import_history import record_import_attempt
from app.services.microsoft_graph_client import MicrosoftGraphClient
from app.services.report_persistence import hydrate_report_store_from_db
from app.services.report_store import ReportStore
from app.services.runtime_status import (
@@ -162,7 +163,74 @@ def _poll_single_gmail_source(source: MailSource) -> None:
)
def _poll_all_enabled_sources() -> list[MailSource]:
def _poll_single_m365_source(source: MailSource) -> None:
"""Fetch DMARC reports for a single M365_GRAPH mail source."""
global last_check_time # pylint: disable=global-statement
if not source.m365_access_token:
logger.info(
"Microsoft 365 polling (source id=%d): skipped OAuth2 not yet authorised",
source.id,
)
return
db = SessionLocal()
try:
src = db.query(MailSource).get(source.id)
poll_source = src or source
already = MicrosoftGraphClient.load_ingested_ids(poll_source.m365_ingested_ids)
client = MicrosoftGraphClient(
tenant_id=poll_source.m365_tenant_id or "common",
client_id=poll_source.m365_client_id or "",
client_secret=poll_source.m365_client_secret or "",
access_token=poll_source.m365_access_token,
refresh_token=poll_source.m365_refresh_token or "",
mailbox=poll_source.m365_mailbox,
folder=poll_source.folder or "INBOX",
already_ingested_ids=already,
db=db,
)
started_at = datetime.utcnow()
results = client.fetch_reports()
if src:
if results.get("new_ingested_ids"):
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
src.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
refreshed = client.get_refreshed_tokens()
if refreshed:
src.m365_access_token = refreshed["access_token"]
if "refresh_token" in refreshed:
src.m365_refresh_token = refreshed["refresh_token"]
src.last_checked = datetime.utcnow()
record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
db.commit()
finally:
db.close()
last_check_time = datetime.now()
if results["success"]:
logger.info(
"Microsoft 365 polling (source id=%d): %s emails processed, "
"%s aggregate reports found",
source.id,
results["processed"],
results["reports_found"],
)
if results["new_domains"]:
logger.info("New domains found: %s", ", ".join(results["new_domains"]))
else:
logger.error(
"Microsoft 365 polling (source id=%d) failed: %s",
source.id,
results.get("error", "Unknown error"),
)
def _poll_all_enabled_sources() -> list[MailSource]: # noqa: C901
"""Iterate over all enabled mail sources and poll each one."""
db = SessionLocal()
try:
@@ -182,6 +250,11 @@ def _poll_all_enabled_sources() -> list[MailSource]:
_poll_single_gmail_source(source)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error polling Gmail source id=%d: %s", source.id, str(e))
elif source.method == "M365_GRAPH":
try:
_poll_single_m365_source(source)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
elif source.method == "IMAP":
try:
_poll_single_imap_source(source)
@@ -682,7 +755,50 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
}
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
def _trigger_poll_m365_source(source: MailSource, db) -> dict:
"""Poll a single M365_GRAPH source and return a result dict for the API response."""
global last_check_time # pylint: disable=global-statement
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
graph_client = MicrosoftGraphClient(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
access_token=source.m365_access_token,
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
already_ingested_ids=already,
db=db,
)
started_at = datetime.utcnow()
results = graph_client.fetch_reports()
last_check_time = datetime.now()
if results.get("new_ingested_ids"):
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
refreshed = graph_client.get_refreshed_tokens()
if refreshed:
source.m365_access_token = refreshed["access_token"]
if "refresh_token" in refreshed:
source.m365_refresh_token = refreshed["refresh_token"]
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
db.commit()
return {
"source_id": source.id,
"name": source.name,
"success": results["success"],
"processed": results.get("processed", 0),
"reports_found": results.get("reports_found", 0),
"forensic_reports_found": results.get("forensic_reports_found", 0),
"duplicate_forensic_reports": results.get("duplicate_forensic_reports", 0),
"new_domains": results.get("new_domains", []),
}
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict: # noqa: C901
"""Dispatch a single mail source for the manual trigger-poll endpoint.
Returns a result/summary dict that is included in the API response.
@@ -705,6 +821,24 @@ def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
"success": False,
"error": "Failed to poll. Check server logs for details.",
}
if source.method == "M365_GRAPH":
if not source.m365_access_token:
return {
"source_id": source.id,
"name": source.name,
"skipped": True,
"reason": "Microsoft 365 account not yet authorised",
}
try:
return _trigger_poll_m365_source(source, db)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
return {
"source_id": source.id,
"name": source.name,
"success": False,
"error": "Failed to poll. Check server logs for details.",
}
if source.method == "IMAP":
try:
return _trigger_poll_imap_source(source, db, days=days)
+45 -1
View File
@@ -18,6 +18,7 @@ class MailSource(Base):
- ``IMAP`` standard IMAP4 (over SSL/TLS or STARTTLS)
- ``POP3`` POP3 inbox (stub for future implementation)
- ``GMAIL_API`` Gmail API with OAuth 2.0
- ``M365_GRAPH`` Microsoft 365 / Exchange Online via Microsoft Graph
"""
__tablename__ = "mail_sources"
@@ -28,7 +29,7 @@ class MailSource(Base):
name = Column(String, nullable=False)
# Connection method determines which fields are used at runtime
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API | M365_GRAPH
# Connection details (used by IMAP and POP3)
server = Column(String, nullable=True)
@@ -48,6 +49,19 @@ class MailSource(Base):
# JSON-encoded list of Gmail message IDs that have already been ingested
gmail_ingested_ids = Column(Text, nullable=True, default="[]")
# Microsoft 365 / Graph OAuth2 credentials (used by M365_GRAPH method)
m365_tenant_id = Column(String, nullable=True, default="common")
m365_client_id = Column(String, nullable=True)
_m365_client_secret = Column("m365_client_secret", Text, nullable=True)
_m365_access_token = Column("m365_access_token", Text, nullable=True)
_m365_refresh_token = Column("m365_refresh_token", Text, nullable=True)
# Optional user/shared mailbox to poll. Empty means the authorised account (/me).
m365_mailbox = Column(String, nullable=True)
# Email address reported by Microsoft Graph for the authorised account.
m365_email = Column(String, nullable=True)
# JSON-encoded list of Graph message IDs that have already been ingested
m365_ingested_ids = Column(Text, nullable=True, default="[]")
# Polling behaviour
polling_interval = Column(Integer, default=60) # minutes
@@ -76,6 +90,9 @@ class MailSource(Base):
"gmail_client_secret": self._gmail_client_secret,
"gmail_access_token": self._gmail_access_token,
"gmail_refresh_token": self._gmail_refresh_token,
"m365_client_secret": self._m365_client_secret,
"m365_access_token": self._m365_access_token,
"m365_refresh_token": self._m365_refresh_token,
}
for public_name, stored_value in secret_fields.items():
@@ -120,3 +137,30 @@ class MailSource(Base):
@gmail_refresh_token.setter
def gmail_refresh_token(self, value):
self._gmail_refresh_token = encrypt_secret(value)
@property
def m365_client_secret(self):
"""Return the decrypted Microsoft 365 OAuth client secret, if present."""
return decrypt_secret(self._m365_client_secret)
@m365_client_secret.setter
def m365_client_secret(self, value):
self._m365_client_secret = encrypt_secret(value)
@property
def m365_access_token(self):
"""Return the decrypted Microsoft Graph access token, if present."""
return decrypt_secret(self._m365_access_token)
@m365_access_token.setter
def m365_access_token(self, value):
self._m365_access_token = encrypt_secret(value)
@property
def m365_refresh_token(self):
"""Return the decrypted Microsoft Graph refresh token, if present."""
return decrypt_secret(self._m365_refresh_token)
@m365_refresh_token.setter
def m365_refresh_token(self, value):
self._m365_refresh_token = encrypt_secret(value)
@@ -0,0 +1,466 @@
"""Microsoft Graph client for retrieving DMARC aggregate reports."""
import base64
import json
import logging
from typing import Any, Dict, List, Optional
from urllib.parse import quote, urlencode
import httpx
from app.services.dmarc_parser import DMARCParser
from app.services.report_persistence import report_exists, save_parsed_report
from app.services.report_store import ReportStore
logger = logging.getLogger(__name__)
GRAPH_BASE_URL = "https://graph.microsoft.com/v1.0"
LOGIN_BASE_URL = "https://login.microsoftonline.com"
M365_SCOPES = [
"offline_access",
"https://graph.microsoft.com/User.Read",
"https://graph.microsoft.com/Mail.Read",
]
_PAGE_SIZE = 100
_DMARC_SUBJECT_TERMS = (
"dmarc",
"aggregate report",
"domain report",
"report domain",
"rua",
"submitter",
)
_DMARC_SENDER_TERMS = (
"dmarc",
"reports",
"postmaster",
)
class MicrosoftGraphError(RuntimeError):
"""Raised when Microsoft Graph or the token endpoint returns a failure."""
class MicrosoftGraphClient:
"""
Retrieve DMARC aggregate reports from Microsoft 365 through Microsoft Graph.
The client uses delegated OAuth tokens and read-only Graph scopes. Messages
are never modified or deleted; already-ingested Graph message IDs are stored
by the caller to avoid reprocessing the same email.
"""
def __init__(
self,
tenant_id: str,
client_id: str,
client_secret: str,
access_token: str,
refresh_token: str,
mailbox: Optional[str] = None,
folder: str = "inbox",
already_ingested_ids: Optional[List[str]] = None,
db: Any = None,
):
self.tenant_id = tenant_id or "common"
self.client_id = client_id
self.client_secret = client_secret
self.access_token = access_token
self.refresh_token = refresh_token
self.mailbox = (mailbox or "").strip()
self.folder = folder or "inbox"
self.already_ingested_ids: List[str] = list(already_ingested_ids or [])
self.report_store = ReportStore.get_instance()
self.db = db
self._refreshed_tokens: Optional[Dict[str, str]] = None
def get_refreshed_tokens(self) -> Optional[Dict[str, str]]:
"""Return refreshed OAuth tokens, if a request had to refresh them."""
return self._refreshed_tokens
@staticmethod
def build_authorization_url(
tenant_id: str,
client_id: str,
redirect_uri: str,
state: Optional[str] = None,
) -> str:
"""Build a Microsoft identity platform authorization-code URL."""
tenant = quote(tenant_id or "common", safe="")
params: Dict[str, str] = {
"client_id": client_id,
"response_type": "code",
"redirect_uri": redirect_uri,
"response_mode": "query",
"scope": " ".join(M365_SCOPES),
"prompt": "select_account",
}
if state:
params["state"] = state
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/authorize?" + urlencode(params)
@staticmethod
def exchange_code_for_tokens(
tenant_id: str,
client_id: str,
client_secret: str,
code: str,
redirect_uri: str,
) -> Dict[str, Any]:
"""Exchange an authorization code for Microsoft Graph tokens."""
data = {
"client_id": client_id,
"client_secret": client_secret,
"code": code,
"redirect_uri": redirect_uri,
"grant_type": "authorization_code",
"scope": " ".join(M365_SCOPES),
}
resp = httpx.post(MicrosoftGraphClient._token_url(tenant_id), data=data, timeout=30)
if resp.status_code != 200:
raise MicrosoftGraphError(
f"Microsoft token exchange failed ({resp.status_code}): {resp.text}"
)
return resp.json()
@staticmethod
def get_account_email(access_token: str) -> Optional[str]:
"""Return the mailbox identity exposed by Graph /me for an access token."""
try:
resp = httpx.get(
f"{GRAPH_BASE_URL}/me",
headers={"Authorization": f"Bearer {access_token}"},
params={"$select": "mail,userPrincipalName"},
timeout=30,
)
if resp.status_code == 200:
profile = resp.json()
return profile.get("mail") or profile.get("userPrincipalName")
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error("Failed to fetch Microsoft 365 account email: %s", exc)
return None
@staticmethod
def load_ingested_ids(json_text: Optional[str]) -> List[str]:
"""Deserialize the m365_ingested_ids text column into a list."""
if not json_text:
return []
try:
decoded = json.loads(json_text)
except (json.JSONDecodeError, TypeError):
return []
return [str(item) for item in decoded] if isinstance(decoded, list) else []
@staticmethod
def dump_ingested_ids(ids: List[str]) -> str:
"""Serialize Graph message IDs for database storage."""
return json.dumps(ids)
def test_connection(self) -> Dict[str, Any]:
"""Verify that the saved delegated token can read the target mailbox."""
mailbox_path = self._mailbox_path()
data = self._request(
"GET",
f"{mailbox_path}/messages",
params={"$top": 1, "$select": "id"},
)
return {
"success": True,
"message_count": len(data.get("value", [])),
"diagnostic_detail": "Microsoft Graph mailbox read succeeded.",
}
def fetch_reports(self) -> Dict[str, Any]:
"""Fetch and ingest DMARC report attachments from Microsoft Graph."""
stats: Dict[str, Any] = {
"success": True,
"processed": 0,
"reports_found": 0,
"forensic_reports_found": 0,
"duplicate_reports": 0,
"duplicate_forensic_reports": 0,
"new_domains": [],
"errors": [],
"new_ingested_ids": [],
"details": [],
}
try:
messages = self._list_dmarc_messages()
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error("Microsoft Graph: failed to list messages: %s", exc)
return {**stats, "success": False, "error": str(exc), "errors": [str(exc)]}
domains_before = set(self.report_store.get_domains())
for message in messages:
message_id = str(message.get("id") or "")
if not message_id:
continue
if message_id in self.already_ingested_ids:
self._append_detail(
stats,
status="skipped",
reason="already_ingested_message",
message_id=message_id,
)
continue
stats["processed"] += 1
found = self._process_message(message, stats)
if found >= 0:
stats["new_ingested_ids"].append(message_id)
self.already_ingested_ids.append(message_id)
domains_after = set(self.report_store.get_domains())
stats["new_domains"] = list(domains_after - domains_before)
return stats
@staticmethod
def _token_url(tenant_id: str) -> str:
tenant = quote(tenant_id or "common", safe="")
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/token"
@staticmethod
def _append_detail(stats: dict, **detail: str) -> None:
stats.setdefault("details", []).append(
{key: value for key, value in detail.items() if value}
)
def _mailbox_path(self) -> str:
if not self.mailbox or self.mailbox.lower() == "me":
return "/me"
return f"/users/{quote(self.mailbox, safe='')}"
def _messages_path(self) -> str:
mailbox_path = self._mailbox_path()
folder = (self.folder or "").strip()
if not folder:
return f"{mailbox_path}/messages"
if folder.upper() == "INBOX":
folder = "inbox"
return f"{mailbox_path}/mailFolders/{quote(folder, safe='')}/messages"
def _headers(self) -> Dict[str, str]:
return {"Authorization": f"Bearer {self.access_token}"}
def _request(
self,
method: str,
path_or_url: str,
*,
params: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
url = path_or_url if path_or_url.startswith("http") else f"{GRAPH_BASE_URL}{path_or_url}"
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
if resp.status_code == 401 and self.refresh_token:
self._refresh_access_token()
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
if resp.status_code < 200 or resp.status_code >= 300:
raise MicrosoftGraphError(self._format_error(resp))
return resp.json() if resp.content else {}
def _refresh_access_token(self) -> None:
data = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": self.refresh_token,
"grant_type": "refresh_token",
"scope": " ".join(M365_SCOPES),
}
resp = httpx.post(self._token_url(self.tenant_id), data=data, timeout=30)
if resp.status_code != 200:
raise MicrosoftGraphError(
f"Microsoft token refresh failed ({resp.status_code}): {resp.text}"
)
token_data = resp.json()
access_token = token_data.get("access_token")
if not access_token:
raise MicrosoftGraphError("Microsoft token refresh did not return an access token.")
self.access_token = access_token
refreshed = {"access_token": access_token}
if token_data.get("refresh_token"):
self.refresh_token = token_data["refresh_token"]
refreshed["refresh_token"] = token_data["refresh_token"]
self._refreshed_tokens = refreshed
@staticmethod
def _format_error(resp: httpx.Response) -> str:
try:
payload = resp.json()
except ValueError:
payload = {}
message = payload.get("error_description")
if not message and isinstance(payload.get("error"), dict):
message = payload["error"].get("message")
code = payload["error"].get("code")
if code:
message = f"{code}: {message}" if message else code
return message or f"Microsoft Graph request failed ({resp.status_code}): {resp.text}"
@staticmethod
def _looks_like_dmarc_message(message: Dict[str, Any]) -> bool:
if not message.get("hasAttachments"):
return False
subject = str(message.get("subject") or "").lower()
sender = (
((message.get("from") or {}).get("emailAddress") or {}).get("address") or ""
).lower()
return any(term in subject for term in _DMARC_SUBJECT_TERMS) or any(
term in sender for term in _DMARC_SENDER_TERMS
)
@staticmethod
def _is_dmarc_attachment(filename: str) -> bool:
lower = filename.lower()
return (
lower.endswith(".xml")
or lower.endswith(".zip")
or lower.endswith(".gz")
or lower.endswith(".gzip")
)
def _list_dmarc_messages(self) -> List[Dict[str, Any]]:
messages: List[Dict[str, Any]] = []
url = self._messages_path()
params: Optional[Dict[str, Any]] = {
"$top": _PAGE_SIZE,
"$select": "id,subject,from,hasAttachments,receivedDateTime",
"$orderby": "receivedDateTime desc",
}
while url:
data = self._request("GET", url, params=params)
for message in data.get("value", []):
if self._looks_like_dmarc_message(message):
messages.append(message)
url = data.get("@odata.nextLink")
params = None
return messages
def _process_message(self, message: Dict[str, Any], stats: Dict[str, Any]) -> int:
message_id = str(message.get("id") or "")
try:
attachments = self._list_attachments(message_id)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error("Microsoft Graph: failed to fetch attachments for %s: %s", message_id, exc)
stats["errors"].append(f"Failed to fetch attachments for message {message_id}: {exc}")
self._append_detail(
stats,
status="error",
reason="attachment_fetch_failed",
message_id=message_id,
error=str(exc),
)
return -1
return self._process_attachments(message_id, attachments, stats)
def _list_attachments(self, message_id: str) -> List[Dict[str, Any]]:
mailbox_path = self._mailbox_path()
data = self._request(
"GET",
f"{mailbox_path}/messages/{quote(message_id, safe='')}/attachments",
)
return list(data.get("value", []))
def _store_report_if_new(self, report: Dict[str, Any]) -> bool:
domain = report.get("domain", "unknown")
report_id = report.get("report_id", "")
if report_id and (
self.report_store.has_report(domain, report_id)
or (self.db is not None and report_exists(self.db, domain, report_id))
):
logger.info("Skipping duplicate DMARC report %s for %s", report_id, domain)
return False
if self.db is not None:
save_parsed_report(self.db, report)
self.report_store.add_report(report)
return True
def _process_attachments(
self,
message_id: str,
attachments: List[Dict[str, Any]],
stats: Dict[str, Any],
) -> int:
reports_found = 0
for attachment in attachments:
filename = str(attachment.get("name") or "")
if not filename:
continue
if not self._is_dmarc_attachment(filename):
self._append_detail(
stats,
status="skipped",
reason="unsupported_attachment",
message_id=message_id,
filename=filename,
)
continue
attachment_type = str(attachment.get("@odata.type") or "").lower()
if "fileattachment" not in attachment_type:
self._append_detail(
stats,
status="skipped",
reason="unsupported_attachment_type",
message_id=message_id,
filename=filename,
)
continue
content_b64 = attachment.get("contentBytes")
if not content_b64:
self._append_detail(
stats,
status="skipped",
reason="empty_attachment",
message_id=message_id,
filename=filename,
)
continue
try:
content = base64.b64decode(content_b64)
report = DMARCParser.parse_file(content, filename)
domain = str(report.get("domain", "unknown"))
report_id = str(report.get("report_id", ""))
if self._store_report_if_new(report):
stats["reports_found"] += 1
reports_found += 1
self._append_detail(
stats,
status="imported",
message_id=message_id,
filename=filename,
domain=domain,
report_id=report_id,
)
else:
stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1
self._append_detail(
stats,
status="duplicate",
message_id=message_id,
filename=filename,
domain=domain,
report_id=report_id,
)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error("Failed to parse Graph DMARC attachment %s: %s", filename, exc)
stats["errors"].append(f"Failed to parse {filename}: {exc}")
self._append_detail(
stats,
status="error",
reason="parse_failed",
message_id=message_id,
filename=filename,
error=str(exc),
)
return reports_found
+171 -3
View File
@@ -86,8 +86,8 @@
<td>
<span class="badge badge-outline" x-text="source.method"></span>
</td>
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_email || '—') : (source.server ? source.server + ':' + source.port : '—')"></td>
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_connected ? '✅ Connected' : '⚠️ Not authorised') : (source.username || '—')"></td>
<td x-text="sourceAccountLabel(source)"></td>
<td x-text="sourceStatusLabel(source)"></td>
<td x-text="source.last_checked ? new Date(source.last_checked).toLocaleString() : 'Never'"></td>
<td>
<input
@@ -293,6 +293,7 @@
<option value="IMAP">IMAP</option>
<option value="POP3">POP3 (coming soon)</option>
<option value="GMAIL_API">Gmail API (OAuth2)</option>
<option value="M365_GRAPH">Microsoft 365 (Graph OAuth2)</option>
</select>
</div>
@@ -434,6 +435,85 @@
</div>
</template>
<!-- Microsoft 365 Graph fields -->
<template x-if="form.method === 'M365_GRAPH'">
<div class="space-y-4">
<div class="alert alert-info text-sm p-3">
<div>
<p class="font-semibold">Microsoft 365 Setup</p>
<p class="mt-1">Enter an app registration Client ID and client secret from Microsoft Entra admin center. Add this app's callback URL as a Web redirect URI, then save and click <strong>Connect Microsoft 365</strong>.</p>
</div>
</div>
<div>
<label class="label"><span class="label-text font-medium">Tenant ID</span></label>
<input type="text" x-model="form.m365_tenant_id"
placeholder="common, organizations, or tenant GUID"
class="input input-bordered w-full" />
</div>
<div>
<label class="label"><span class="label-text font-medium">Microsoft Client ID <span class="text-error">*</span></span></label>
<input type="text" x-model="form.m365_client_id"
placeholder="Application (client) ID"
class="input input-bordered w-full" />
</div>
<div>
<label class="label">
<span class="label-text font-medium">Microsoft Client Secret <span class="text-error">*</span></span>
<span class="label-text-alt text-muted-foreground" x-show="editingId">Leave blank to keep existing</span>
</label>
<div class="relative">
<input :type="showPassword ? 'text' : 'password'"
x-model="form.m365_client_secret"
class="input input-bordered w-full pr-10" />
<button type="button"
class="absolute right-2 top-3 text-muted-foreground hover:text-foreground"
x-on:click="showPassword = !showPassword">
<svg x-show="!showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
stroke-linecap="round" stroke-linejoin="round">
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
<circle cx="12" cy="12" r="3"></circle>
</svg>
<svg x-show="showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
stroke-linecap="round" stroke-linejoin="round">
<path d="M17.94 17.94A10.07 10.07 0 0 1 12 20c-7 0-11-8-11-8a18.45 18.45 0 0 1 5.06-5.94"></path>
<path d="M9.9 4.24A9.12 9.12 0 0 1 12 4c7 0 11 8 11 8a18.5 18.5 0 0 1-2.16 3.19"></path>
<line x1="1" y1="1" x2="23" y2="23"></line>
</svg>
</button>
</div>
</div>
<div>
<label class="label"><span class="label-text font-medium">Mailbox</span></label>
<input type="text" x-model="form.m365_mailbox"
placeholder="Leave blank for the authorised account"
class="input input-bordered w-full" />
</div>
<div>
<label class="label"><span class="label-text font-medium">Folder</span></label>
<input type="text" x-model="form.folder" placeholder="INBOX"
class="input input-bordered w-full" />
</div>
<template x-if="editingId && m365Connected">
<div class="alert alert-success text-sm p-3">
<span>Connected as <strong x-text="m365Email || 'Microsoft 365 account'"></strong></span>
</div>
</template>
<template x-if="editingId && !m365Connected">
<div class="alert alert-warning text-sm p-3">
<span>Not yet authorised. Save this source first, then click <strong>Connect Microsoft 365</strong>.</span>
</div>
</template>
</div>
</template>
<!-- Polling interval -->
<div>
<label class="label"><span class="label-text font-medium">Polling Interval (minutes)</span></label>
@@ -467,7 +547,7 @@
<div class="flex justify-between pt-2">
<!-- Test Connection (IMAP/POP3) or Connect Gmail button -->
<div class="flex gap-2">
<template x-if="form.method !== 'GMAIL_API'">
<template x-if="form.method !== 'GMAIL_API' && form.method !== 'M365_GRAPH'">
<button type="button" class="btn btn-outline btn-sm"
x-on:click="testAdHoc()"
:disabled="isTesting || isSaving">
@@ -492,6 +572,13 @@
Connect Gmail
</button>
</template>
<template x-if="form.method === 'M365_GRAPH' && editingId">
<button type="button" class="btn btn-outline btn-sm"
x-on:click="connectM365()"
:disabled="isTesting || isSaving">
Connect Microsoft 365
</button>
</template>
</div>
<div class="flex gap-2">
<button type="button" class="btn btn-ghost btn-sm" x-on:click="closeForm()">Cancel</button>
@@ -577,6 +664,8 @@ function mailSourcesApp() {
testResult: { message: '', success: false, diagnostic_summary: '', recovery_steps: [] },
gmailConnected: false,
gmailEmail: '',
m365Connected: false,
m365Email: '',
form: {
name: '',
@@ -591,6 +680,10 @@ function mailSourcesApp() {
enabled: true,
gmail_client_id: '',
gmail_client_secret: '',
m365_tenant_id: 'common',
m365_client_id: '',
m365_client_secret: '',
m365_mailbox: '',
},
async init() {
@@ -689,6 +782,26 @@ function mailSourcesApp() {
return value ? new Date(value).toLocaleString() : '—';
},
sourceAccountLabel(source) {
if (source.method === 'GMAIL_API') {
return source.gmail_email || '—';
}
if (source.method === 'M365_GRAPH') {
return source.m365_mailbox || source.m365_email || '—';
}
return source.server ? `${source.server}:${source.port}` : '—';
},
sourceStatusLabel(source) {
if (source.method === 'GMAIL_API') {
return source.gmail_connected ? 'Connected' : 'Not authorised';
}
if (source.method === 'M365_GRAPH') {
return source.m365_connected ? 'Connected' : 'Not authorised';
}
return source.username || '—';
},
formatList(value) {
return value && value.length ? value.join(', ') : '—';
},
@@ -738,6 +851,8 @@ function mailSourcesApp() {
this.editingId = null;
this.gmailConnected = false;
this.gmailEmail = '';
this.m365Connected = false;
this.m365Email = '';
this.form = {
name: '',
method: 'IMAP',
@@ -751,6 +866,10 @@ function mailSourcesApp() {
enabled: true,
gmail_client_id: '',
gmail_client_secret: '',
m365_tenant_id: 'common',
m365_client_id: '',
m365_client_secret: '',
m365_mailbox: '',
};
this.testResult = this.emptyTestResult();
this.showForm = true;
@@ -760,6 +879,8 @@ function mailSourcesApp() {
this.editingId = source.id;
this.gmailConnected = source.gmail_connected || false;
this.gmailEmail = source.gmail_email || '';
this.m365Connected = source.m365_connected || false;
this.m365Email = source.m365_email || '';
this.form = {
name: source.name,
method: source.method,
@@ -773,6 +894,10 @@ function mailSourcesApp() {
enabled: source.enabled !== false,
gmail_client_id: source.gmail_client_id || '',
gmail_client_secret: '', // never pre-fill client secret
m365_tenant_id: source.m365_tenant_id || 'common',
m365_client_id: source.m365_client_id || '',
m365_client_secret: '', // never pre-fill client secret
m365_mailbox: source.m365_mailbox || '',
};
this.testResult = this.emptyTestResult();
this.showForm = true;
@@ -826,6 +951,45 @@ function mailSourcesApp() {
}
},
async connectM365() {
if (!this.editingId) return;
try {
const resp = await fetch(
`/api/v1/mail-sources/${this.editingId}/m365/authorize-url`
);
if (!resp.ok) {
const err = await resp.json();
this.feedback = { message: `Error: ${err.detail || 'Failed to get authorization URL'}`, type: 'error' };
return;
}
const data = await resp.json();
const popup = window.open(
data.authorization_url,
'm365_oauth',
'width=700,height=760,scrollbars=yes'
);
const pollInterval = setInterval(async () => {
if (popup && popup.closed) {
clearInterval(pollInterval);
await this.loadSources();
const updated = this.sources.find(s => s.id === this.editingId);
if (updated) {
this.m365Connected = updated.m365_connected || false;
this.m365Email = updated.m365_email || '';
if (this.m365Connected) {
this.feedback = {
message: `Microsoft 365 connected successfully (${this.m365Email || 'account connected'}).`,
type: 'success',
};
}
}
}
}, 1000);
} catch (e) {
this.feedback = { message: `Error: ${e.message}`, type: 'error' };
}
},
async saveSource() {
this.isSaving = true;
this.feedback = { message: '', type: '' };
@@ -839,6 +1003,10 @@ function mailSourcesApp() {
if (this.editingId && !payload.gmail_client_secret) {
delete payload.gmail_client_secret;
}
// Don't send empty m365_client_secret on edit
if (this.editingId && !payload.m365_client_secret) {
delete payload.m365_client_secret;
}
const url = this.editingId
? `/api/v1/mail-sources/${this.editingId}`
+633
View File
@@ -99,6 +99,36 @@ class TestMailSourceModel:
assert is_encrypted_secret(stored.gmail_access_token)
assert is_encrypted_secret(stored.gmail_refresh_token)
def test_m365_oauth_secrets_are_encrypted_at_rest(self, db_session: Session):
source = MailSource(
name="Encrypted Microsoft 365",
method="M365_GRAPH",
m365_client_secret="client-secret",
m365_access_token="access-token",
m365_refresh_token="refresh-token",
)
db_session.add(source)
db_session.commit()
db_session.refresh(source)
stored = db_session.execute(
text(
"SELECT m365_client_secret, m365_access_token, m365_refresh_token "
"FROM mail_sources WHERE id = :id"
),
{"id": source.id},
).one()
assert source.m365_client_secret == "client-secret"
assert source.m365_access_token == "access-token"
assert source.m365_refresh_token == "refresh-token"
assert stored.m365_client_secret != "client-secret"
assert stored.m365_access_token != "access-token"
assert stored.m365_refresh_token != "refresh-token"
assert is_encrypted_secret(stored.m365_client_secret)
assert is_encrypted_secret(stored.m365_access_token)
assert is_encrypted_secret(stored.m365_refresh_token)
def test_legacy_plaintext_mail_source_secret_remains_readable(self, db_session: Session):
db_session.execute(
text(
@@ -968,6 +998,455 @@ class TestGmailAPIMailSource:
assert data["new_domains"] == ["example.com"]
class TestMicrosoft365GraphMailSource:
"""Tests for M365_GRAPH mail source creation, OAuth flow, and fetching."""
def test_create_m365_graph_source(self, authed_client: TestClient):
payload = {
"name": "My Microsoft 365",
"method": "M365_GRAPH",
"m365_tenant_id": "organizations",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
"m365_mailbox": "dmarc@example.com",
"folder": "INBOX",
"polling_interval": 30,
"enabled": True,
}
resp = authed_client.post("/api/v1/mail-sources", json=payload)
assert resp.status_code == 201
data = resp.json()
assert data["method"] == "M365_GRAPH"
assert data["m365_tenant_id"] == "organizations"
assert data["m365_client_id"] == "client-id"
assert data["m365_client_secret"] == "**redacted**"
assert data["m365_mailbox"] == "dmarc@example.com"
assert data["m365_connected"] is False
assert data["m365_email"] is None
def test_m365_source_test_no_token(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "Unauthed M365", "method": "M365_GRAPH"},
)
source_id = create_resp.json()["id"]
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
assert resp.status_code == 200
data = resp.json()
assert data["success"] is False
assert data["diagnostic_category"] == "auth_required"
assert any("Microsoft 365" in step for step in data["recovery_steps"])
def test_m365_source_test_with_valid_token(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Authed M365",
"method": "M365_GRAPH",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
},
)
source_id = create_resp.json()["id"]
mock_client = MagicMock()
mock_client.test_connection.return_value = {
"success": True,
"message_count": 1,
"diagnostic_detail": "ok",
}
mock_client.get_refreshed_tokens.return_value = None
with (
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
return_value=mock_client,
),
):
mock_source = MagicMock()
mock_source.method = "M365_GRAPH"
mock_source.m365_access_token = "valid-token"
mock_source.m365_email = "dmarc@example.com"
mock_source.m365_tenant_id = "organizations"
mock_source.m365_client_id = "client-id"
mock_source.m365_client_secret = "client-secret"
mock_source.m365_refresh_token = "refresh-token"
mock_source.m365_mailbox = None
mock_source.folder = "INBOX"
mock_get.return_value = mock_source
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
assert resp.status_code == 200
data = resp.json()
assert data["success"] is True
assert "valid" in data["message"].lower()
@pytest.mark.parametrize(
("provider_error", "category"),
[
("invalid_grant: refresh token expired", "auth_expired"),
("Forbidden: Mail.Read permission is missing", "permissions"),
("429 Too Many Requests throttling limit", "throttling"),
],
)
def test_m365_source_test_diagnostic_categories(
self, authed_client: TestClient, provider_error: str, category: str
):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "M365 Diagnostics", "method": "M365_GRAPH"},
)
source_id = create_resp.json()["id"]
mock_client = MagicMock()
mock_client.test_connection.side_effect = RuntimeError(provider_error)
with (
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
return_value=mock_client,
),
):
mock_source = MagicMock()
mock_source.method = "M365_GRAPH"
mock_source.m365_access_token = "valid-token"
mock_source.m365_email = "dmarc@example.com"
mock_source.m365_tenant_id = "organizations"
mock_source.m365_client_id = "client-id"
mock_source.m365_client_secret = "client-secret"
mock_source.m365_refresh_token = "refresh-token"
mock_source.m365_mailbox = None
mock_source.folder = "INBOX"
mock_get.return_value = mock_source
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
assert resp.status_code == 200
assert resp.json()["diagnostic_category"] == category
def test_m365_authorize_url_returns_microsoft_url(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Ready M365",
"method": "M365_GRAPH",
"m365_tenant_id": "organizations",
"m365_client_id": "client-id",
},
)
source_id = create_resp.json()["id"]
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/authorize-url")
assert resp.status_code == 200
data = resp.json()
parsed = urlparse(data["authorization_url"])
assert parsed.hostname == "login.microsoftonline.com"
assert "/organizations/oauth2/v2.0/authorize" in parsed.path
query = parse_qs(parsed.query)
assert query["client_id"] == ["client-id"]
assert "offline_access" in query["scope"][0]
assert "https://graph.microsoft.com/Mail.Read" in query["scope"][0]
def test_m365_authorize_url_validates_source(self, authed_client: TestClient):
imap_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "IMAP Source", "method": "IMAP"},
)
imap_id = imap_resp.json()["id"]
assert (
authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/authorize-url").status_code
== 400
)
m365_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "No Client M365", "method": "M365_GRAPH"},
)
m365_id = m365_resp.json()["id"]
resp = authed_client.get(f"/api/v1/mail-sources/{m365_id}/m365/authorize-url")
assert resp.status_code == 400
assert "m365_client_id" in resp.json()["detail"]
def test_m365_get_callback_handles_error_and_wrong_source(self, authed_client: TestClient):
error_resp = authed_client.get("/api/v1/mail-sources/999/m365/callback?error=denied")
assert error_resp.status_code == 400
assert "authorisation failed" in error_resp.text
imap_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "IMAP Callback", "method": "IMAP"},
)
imap_id = imap_resp.json()["id"]
wrong_resp = authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/callback?code=abc")
assert wrong_resp.status_code == 404
def test_m365_get_callback_saves_tokens(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "GET Callback M365",
"method": "M365_GRAPH",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
},
)
source_id = create_resp.json()["id"]
with (
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
return_value={"access_token": "access", "refresh_token": "refresh"},
),
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
return_value="dmarc@example.com",
),
):
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc")
assert resp.status_code == 200
assert "connected successfully" in resp.text
def test_m365_get_callback_failure_modes(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "GET Callback Failure", "method": "M365_GRAPH"},
)
source_id = create_resp.json()["id"]
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
side_effect=ValueError("bad code"),
):
token_error = authed_client.get(
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
)
assert token_error.status_code == 400
assert "Token exchange failed" in token_error.text
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
return_value={"refresh_token": "refresh"},
):
missing_access = authed_client.get(
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
)
assert missing_access.status_code == 400
assert "No access token" in missing_access.text
def test_m365_callback_post_saves_tokens(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Callback M365",
"method": "M365_GRAPH",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
},
)
source_id = create_resp.json()["id"]
with (
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
return_value={"access_token": "access", "refresh_token": "refresh"},
),
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
return_value="dmarc@example.com",
),
):
resp = authed_client.post(
f"/api/v1/mail-sources/{source_id}/m365/callback",
json={"code": "code", "redirect_uri": "https://example.com/callback"},
)
assert resp.status_code == 200
data = resp.json()
assert data["m365_connected"] is True
assert data["m365_email"] == "dmarc@example.com"
def test_m365_callback_post_failure_modes(self, authed_client: TestClient):
imap_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "IMAP Callback Post", "method": "IMAP"},
)
imap_id = imap_resp.json()["id"]
wrong_resp = authed_client.post(
f"/api/v1/mail-sources/{imap_id}/m365/callback",
json={"code": "code", "redirect_uri": "https://example.com/callback"},
)
assert wrong_resp.status_code == 400
m365_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "Bad Callback M365", "method": "M365_GRAPH"},
)
m365_id = m365_resp.json()["id"]
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
side_effect=ValueError("bad code"),
):
token_resp = authed_client.post(
f"/api/v1/mail-sources/{m365_id}/m365/callback",
json={"code": "code", "redirect_uri": "https://example.com/callback"},
)
assert token_resp.status_code == 400
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
return_value={"refresh_token": "refresh"},
):
missing_access = authed_client.post(
f"/api/v1/mail-sources/{m365_id}/m365/callback",
json={"code": "code", "redirect_uri": "https://example.com/callback"},
)
assert missing_access.status_code == 400
def test_m365_fetch_no_token_returns_400(self, authed_client: TestClient):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "No Token M365", "method": "M365_GRAPH"},
)
source_id = create_resp.json()["id"]
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
assert resp.status_code == 400
def test_m365_fetch_with_mocked_client(self, authed_client: TestClient, db_session: Session):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Fetch M365",
"method": "M365_GRAPH",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
},
)
source_id = create_resp.json()["id"]
source = db_session.get(MailSource, source_id)
source.m365_access_token = "tok"
source.m365_refresh_token = "refresh"
db_session.commit()
mock_client = MagicMock()
mock_client.fetch_reports.return_value = {
"success": True,
"processed": 3,
"reports_found": 2,
"duplicate_reports": 1,
"new_domains": ["example.com"],
"errors": [],
"new_ingested_ids": ["id1", "id2", "id3"],
}
mock_client.get_refreshed_tokens.return_value = None
with (
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
return_value=mock_client,
),
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.load_ingested_ids",
return_value=[],
),
patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.dump_ingested_ids",
return_value='["id1", "id2", "id3"]',
),
):
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/fetch")
assert resp.status_code == 200
data = resp.json()
assert data["success"] is True
assert data["processed"] == 3
assert data["reports_found"] == 2
assert data["duplicate_reports"] == 1
assert data["new_domains"] == ["example.com"]
def test_m365_specific_fetch_persists_refreshed_tokens(
self, authed_client: TestClient, db_session: Session
):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Fetch M365 Specific",
"method": "M365_GRAPH",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
},
)
source_id = create_resp.json()["id"]
source = db_session.get(MailSource, source_id)
source.m365_access_token = "old-access"
source.m365_refresh_token = "old-refresh"
db_session.commit()
mock_client = MagicMock()
mock_client.fetch_reports.return_value = {
"success": True,
"processed": 1,
"reports_found": 0,
"duplicate_reports": 0,
"new_domains": [],
"errors": ["temporary warning"],
"new_ingested_ids": [],
}
mock_client.get_refreshed_tokens.return_value = {
"access_token": "new-access",
"refresh_token": "new-refresh",
}
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
return_value=mock_client,
):
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
db_session.refresh(source)
assert resp.status_code == 200
assert resp.json()["processed"] == 1
assert source.m365_access_token == "new-access"
assert source.m365_refresh_token == "new-refresh"
def test_m365_fetch_and_disconnect_validate_method(self, authed_client: TestClient):
imap_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "IMAP M365 Actions", "method": "IMAP"},
)
source_id = imap_resp.json()["id"]
fetch_resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
disconnect_resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
assert fetch_resp.status_code == 400
assert disconnect_resp.status_code == 400
def test_m365_disconnect_clears_tokens(self, authed_client: TestClient, db_session: Session):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "Disconnect M365", "method": "M365_GRAPH"},
)
source_id = create_resp.json()["id"]
source = db_session.get(MailSource, source_id)
source.m365_access_token = "tok"
source.m365_refresh_token = "refresh"
source.m365_email = "dmarc@example.com"
db_session.commit()
resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
db_session.refresh(source)
assert resp.status_code == 204
assert source.m365_access_token is None
assert source.m365_refresh_token is None
assert source.m365_email is None
class TestManualSourceFetchEndpoint:
"""Tests for POST /api/v1/mail-sources/{source_id}/fetch."""
@@ -2035,6 +2514,79 @@ class TestTriggerPollGmailSource:
assert src.gmail_refresh_token == "new-ref"
class TestTriggerPollM365Source:
"""Unit tests for app.main._trigger_poll_m365_source."""
def _make_src(self):
src = MagicMock()
src.id = 8
src.name = "My M365"
src.m365_tenant_id = "organizations"
src.m365_client_id = "cid"
src.m365_client_secret = "csec"
src.m365_access_token = "tok"
src.m365_refresh_token = "ref"
src.m365_mailbox = "shared@example.com"
src.m365_ingested_ids = "[]"
src.folder = "INBOX"
return src
def test_returns_result_dict_on_success(self):
from app.main import _trigger_poll_m365_source
src = self._make_src()
mock_gc = MagicMock()
mock_gc.fetch_reports.return_value = {
"success": True,
"processed": 2,
"reports_found": 1,
"new_domains": ["example.com"],
"new_ingested_ids": ["id1"],
}
mock_gc.get_refreshed_tokens.return_value = None
mock_db = MagicMock()
with (
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
patch("app.main.MicrosoftGraphClient.dump_ingested_ids", return_value='["id1"]'),
):
result = _trigger_poll_m365_source(src, mock_db)
assert result["success"] is True
assert result["source_id"] == 8
assert result["processed"] == 2
assert src.m365_ingested_ids == '["id1"]'
mock_db.commit.assert_called_once()
def test_persists_refreshed_tokens(self):
from app.main import _trigger_poll_m365_source
src = self._make_src()
mock_gc = MagicMock()
mock_gc.fetch_reports.return_value = {
"success": True,
"processed": 0,
"reports_found": 0,
"new_domains": [],
"new_ingested_ids": [],
}
mock_gc.get_refreshed_tokens.return_value = {
"access_token": "new-acc",
"refresh_token": "new-ref",
}
mock_db = MagicMock()
with (
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
):
_trigger_poll_m365_source(src, mock_db)
assert src.m365_access_token == "new-acc"
assert src.m365_refresh_token == "new-ref"
class TestPollSourceForTrigger:
"""Unit tests for app.main._poll_source_for_trigger."""
@@ -2083,6 +2635,51 @@ class TestPollSourceForTrigger:
assert result["success"] is False
assert "boom" not in result.get("error", "") # raw msg not exposed
def test_m365_no_token_returns_skipped(self):
from app.main import _poll_source_for_trigger
src = MagicMock()
src.method = "M365_GRAPH"
src.m365_access_token = None
src.id = 7
src.name = "M365 no token"
result = _poll_source_for_trigger(src, MagicMock())
assert result["skipped"] is True
assert "microsoft 365" in result["reason"].lower()
def test_m365_with_token_delegates_to_trigger_poll(self):
from app.main import _poll_source_for_trigger
src = MagicMock()
src.method = "M365_GRAPH"
src.m365_access_token = "tok"
src.id = 8
src.name = "M365"
expected = {"source_id": 8, "name": "M365", "success": True}
with patch("app.main._trigger_poll_m365_source", return_value=expected) as mock_fn:
result = _poll_source_for_trigger(src, MagicMock())
assert result is expected
mock_fn.assert_called_once()
def test_m365_exception_returns_failure_dict(self):
from app.main import _poll_source_for_trigger
src = MagicMock()
src.method = "M365_GRAPH"
src.m365_access_token = "tok"
src.id = 9
src.name = "M365 exc"
with patch("app.main._trigger_poll_m365_source", side_effect=Exception("boom")):
result = _poll_source_for_trigger(src, MagicMock())
assert result["success"] is False
assert "boom" not in result.get("error", "")
def test_imap_delegates_to_trigger_poll(self):
from app.main import _poll_source_for_trigger
@@ -2147,6 +2744,25 @@ class TestPollAllEnabledSources:
mock_gmail.assert_called_once_with(src)
def test_dispatches_m365_graph_source(self):
"""M365_GRAPH sources are forwarded to _poll_single_m365_source."""
from app.main import _poll_all_enabled_sources
src = MagicMock()
src.id = 6
src.method = "M365_GRAPH"
mock_db = MagicMock()
mock_db.query.return_value.filter.return_value.all.return_value = [src]
with (
patch("app.main.SessionLocal", return_value=mock_db),
patch("app.main._poll_single_m365_source") as mock_m365,
):
_poll_all_enabled_sources()
mock_m365.assert_called_once_with(src)
def test_dispatches_imap_source(self):
"""IMAP sources are forwarded to _poll_single_imap_source."""
from app.main import _poll_all_enabled_sources
@@ -2200,6 +2816,23 @@ class TestPollAllEnabledSources:
):
_poll_all_enabled_sources() # should not raise
def test_m365_exception_is_caught(self):
"""Exception from _poll_single_m365_source must not propagate."""
from app.main import _poll_all_enabled_sources
src = MagicMock()
src.id = 7
src.method = "M365_GRAPH"
mock_db = MagicMock()
mock_db.query.return_value.filter.return_value.all.return_value = [src]
with (
patch("app.main.SessionLocal", return_value=mock_db),
patch("app.main._poll_single_m365_source", side_effect=Exception("m365 crash")),
):
_poll_all_enabled_sources() # should not raise
def test_unknown_method_skipped(self):
"""An unknown method logs a skip message and does not raise."""
from app.main import _poll_all_enabled_sources
+62
View File
@@ -86,6 +86,68 @@ def test_poll_single_imap_source_passes_configured_folder():
db.close.assert_called_once()
def test_poll_single_m365_source_persists_import_state():
from app.main import _poll_single_m365_source
source = SimpleNamespace(
id=2,
m365_access_token="tok",
m365_tenant_id="organizations",
m365_client_id="cid",
m365_client_secret="csec",
m365_refresh_token="ref",
m365_mailbox="shared@example.com",
m365_ingested_ids="[]",
folder="INBOX",
)
db_source = SimpleNamespace(**source.__dict__)
db = MagicMock()
db.query.return_value.get.return_value = db_source
results = {
"success": True,
"processed": 1,
"reports_found": 1,
"new_domains": ["example.com"],
"new_ingested_ids": ["message-1"],
}
with (
patch("app.main.SessionLocal", return_value=db),
patch("app.main.MicrosoftGraphClient") as mock_client_cls,
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
patch(
"app.main.MicrosoftGraphClient.dump_ingested_ids",
return_value='["message-1"]',
),
patch("app.main.record_import_attempt"),
):
mock_client = mock_client_cls.return_value
mock_client.fetch_reports.return_value = results
mock_client.get_refreshed_tokens.return_value = {
"access_token": "new-tok",
"refresh_token": "new-ref",
}
_poll_single_m365_source(source)
assert db_source.m365_ingested_ids == '["message-1"]'
assert db_source.m365_access_token == "new-tok"
assert db_source.m365_refresh_token == "new-ref"
db.commit.assert_called_once()
db.close.assert_called_once()
def test_poll_single_m365_source_skips_without_token():
from app.main import _poll_single_m365_source
source = SimpleNamespace(id=3, m365_access_token=None)
with patch("app.main.SessionLocal") as mock_session:
_poll_single_m365_source(source)
mock_session.assert_not_called()
@pytest.mark.asyncio
async def test_scheduled_imap_polling_sleep_exception_falls_back_then_cancels():
from app.main import scheduled_imap_polling
@@ -0,0 +1,397 @@
"""Unit tests for app.services.microsoft_graph_client.MicrosoftGraphClient."""
import base64
import zipfile
from io import BytesIO
import httpx
from app.models.report import DMARCReport
from app.services.microsoft_graph_client import MicrosoftGraphClient
from app.tests.test_data import SAMPLE_XML
def _zip_xml(xml: str = SAMPLE_XML, name: str = "report.xml") -> bytes:
buf = BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr(name, xml.encode("utf-8"))
return buf.getvalue()
def _make_client(db=None, already_ingested=None) -> MicrosoftGraphClient:
return MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
mailbox=None,
folder="INBOX",
already_ingested_ids=already_ingested or [],
db=db,
)
class TestMicrosoftGraphOAuthHelpers:
def test_build_authorization_url_includes_required_scopes(self):
url = MicrosoftGraphClient.build_authorization_url(
tenant_id="organizations",
client_id="client-id",
redirect_uri="https://example.com/callback",
state="42",
)
assert url.startswith("https://login.microsoftonline.com/organizations/")
assert "client_id=client-id" in url
assert "response_type=code" in url
assert "offline_access" in url
assert "User.Read" in url
assert "Mail.Read" in url
assert "state=42" in url
def test_exchange_code_for_tokens_posts_to_tenant_endpoint(self, monkeypatch):
def fake_post(url, data=None, timeout=None):
assert url.endswith("/contoso-tenant/oauth2/v2.0/token")
assert data["grant_type"] == "authorization_code"
assert data["code"] == "code"
return httpx.Response(200, json={"access_token": "access"})
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
result = MicrosoftGraphClient.exchange_code_for_tokens(
tenant_id="contoso-tenant",
client_id="client-id",
client_secret="client-secret",
code="code",
redirect_uri="https://example.com/callback",
)
assert result == {"access_token": "access"}
def test_exchange_code_for_tokens_raises_on_failure(self, monkeypatch):
def fake_post(url, data=None, timeout=None):
return httpx.Response(400, text="bad request")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
try:
MicrosoftGraphClient.exchange_code_for_tokens(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
code="bad",
redirect_uri="https://example.com/callback",
)
except Exception as exc:
assert "token exchange failed" in str(exc).lower()
else:
raise AssertionError("expected token exchange failure")
def test_get_account_email_prefers_mail_then_user_principal_name(self, monkeypatch):
def fake_get(url, headers=None, params=None, timeout=None):
assert url.endswith("/me")
assert params == {"$select": "mail,userPrincipalName"}
return httpx.Response(
200,
json={"mail": "", "userPrincipalName": "dmarc@example.com"},
)
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
assert MicrosoftGraphClient.get_account_email("access") == "dmarc@example.com"
def test_get_account_email_returns_none_on_error(self, monkeypatch):
def fake_get(url, headers=None, params=None, timeout=None):
raise RuntimeError("network down")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
assert MicrosoftGraphClient.get_account_email("access") is None
def test_ingested_id_helpers_tolerate_bad_json(self):
assert MicrosoftGraphClient.load_ingested_ids(None) == []
assert MicrosoftGraphClient.load_ingested_ids('["a", 2]') == ["a", "2"]
assert MicrosoftGraphClient.load_ingested_ids("{bad") == []
assert MicrosoftGraphClient.dump_ingested_ids(["a"]) == '["a"]'
class TestMicrosoftGraphFetchReports:
def test_test_connection_reads_shared_mailbox(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
assert url.endswith("/users/shared%40example.com/messages")
assert params == {"$top": 1, "$select": "id"}
return httpx.Response(200, json={"value": [{"id": "message-1"}]})
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
mailbox="shared@example.com",
folder="INBOX",
)
result = client.test_connection()
assert result["success"] is True
assert result["message_count"] == 1
def test_fetch_reports_imports_zip_attachment(self, monkeypatch, db_session):
attachment_bytes = _zip_xml()
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/me/mailFolders/inbox/messages"):
return httpx.Response(
200,
json={
"value": [
{
"id": "message-1",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
"receivedDateTime": "2026-05-23T00:00:00Z",
}
]
},
)
if url.endswith("/me/messages/message-1/attachments"):
return httpx.Response(
200,
json={
"value": [
{
"@odata.type": "#microsoft.graph.fileAttachment",
"name": "report.zip",
"contentBytes": base64.b64encode(attachment_bytes).decode(),
}
]
},
)
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client(db=db_session)
result = client.fetch_reports()
assert result["success"] is True
assert result["processed"] == 1
assert result["reports_found"] == 1
assert result["new_ingested_ids"] == ["message-1"]
assert db_session.query(DMARCReport).count() == 1
def test_fetch_reports_skips_already_ingested_message(self, monkeypatch, db_session):
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/me/mailFolders/inbox/messages"):
return httpx.Response(
200,
json={
"value": [
{
"id": "message-1",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
}
]
},
)
raise AssertionError("Already-ingested messages should not fetch attachments")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client(db=db_session, already_ingested=["message-1"])
result = client.fetch_reports()
assert result["success"] is True
assert result["processed"] == 0
assert result["reports_found"] == 0
assert result["new_ingested_ids"] == []
def test_fetch_reports_handles_empty_message_id_and_plain_messages_path(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
assert url.endswith("/me/messages")
return httpx.Response(
200,
json={
"value": [
{
"id": "",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
}
]
},
)
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
folder="",
)
client.folder = ""
result = client.fetch_reports()
assert result["success"] is True
assert result["processed"] == 0
def test_fetch_reports_keeps_message_retryable_when_attachments_fail(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/me/mailFolders/inbox/messages"):
return httpx.Response(
200,
json={
"value": [
{
"id": "message-1",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
}
]
},
)
if url.endswith("/me/messages/message-1/attachments"):
return httpx.Response(
500,
json={"error": {"code": "ServerError", "message": "temporary failure"}},
)
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client()
result = client.fetch_reports()
assert result["success"] is True
assert result["processed"] == 1
assert result["new_ingested_ids"] == []
assert result["details"][0]["reason"] == "attachment_fetch_failed"
def test_fetch_reports_records_attachment_outcomes(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/me/mailFolders/inbox/messages"):
return httpx.Response(
200,
json={
"value": [
{
"id": "message-1",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
}
]
},
)
if url.endswith("/me/messages/message-1/attachments"):
return httpx.Response(
200,
json={
"value": [
{
"@odata.type": "#microsoft.graph.fileAttachment",
"name": "notes.txt",
"contentBytes": base64.b64encode(b"not dmarc").decode(),
},
{
"@odata.type": "#microsoft.graph.itemAttachment",
"name": "report.xml",
"contentBytes": base64.b64encode(SAMPLE_XML.encode()).decode(),
},
{
"@odata.type": "#microsoft.graph.fileAttachment",
"name": "empty.xml",
"contentBytes": "",
},
{
"@odata.type": "#microsoft.graph.fileAttachment",
"name": "bad.xml",
"contentBytes": base64.b64encode(b"<not-dmarc>").decode(),
},
]
},
)
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client()
result = client.fetch_reports()
assert result["success"] is True
assert result["processed"] == 1
assert result["reports_found"] == 0
assert result["errors"]
reasons = {detail.get("reason") for detail in result["details"]}
assert {
"unsupported_attachment",
"unsupported_attachment_type",
"empty_attachment",
"parse_failed",
}.issubset(reasons)
def test_fetch_reports_refreshes_token_after_unauthorized(self, monkeypatch):
calls = {"messages": 0}
def fake_post(url, data=None, timeout=None):
assert url.endswith("/organizations/oauth2/v2.0/token")
assert data["grant_type"] == "refresh_token"
return httpx.Response(
200,
json={"access_token": "new-access", "refresh_token": "new-refresh"},
)
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/me/mailFolders/inbox/messages"):
calls["messages"] += 1
if calls["messages"] == 1:
return httpx.Response(
401,
json={
"error": {
"code": "InvalidAuthenticationToken",
"message": "Access token has expired.",
}
},
)
assert headers["Authorization"] == "Bearer new-access"
return httpx.Response(200, json={"value": []})
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client()
result = client.fetch_reports()
assert result["success"] is True
assert calls["messages"] == 2
assert client.get_refreshed_tokens() == {
"access_token": "new-access",
"refresh_token": "new-refresh",
}
def test_fetch_reports_surfaces_throttling_error(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
return httpx.Response(
429,
json={"error": {"code": "TooManyRequests", "message": "Request is throttled."}},
)
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client()
result = client.fetch_reports()
assert result["success"] is False
assert "TooManyRequests" in result["error"]
+3 -1
View File
@@ -13,7 +13,7 @@ DMARQ is a full-stack DMARC monitoring platform designed to help organizations t
- **DMARC Report Processing**: Automatically collect and parse DMARC aggregate and forensic reports
- **Interactive Dashboard**: Visualize compliance rates and authentication trends
- **DNS Health Checks**: Verify your email authentication records (SPF, DKIM, DMARC)
- **IMAP Integration**: Automatically fetch reports from your email inbox
- **Mailbox Integrations**: Automatically fetch reports from IMAP, Gmail, and Microsoft 365 inboxes
- **Alerting**: Get notified about important authentication issues
- **Easy Setup**: Web-based configuration wizard for quick onboarding
@@ -23,4 +23,6 @@ To get started with DMARQ, please see the [Getting Started](user_guide/getting_s
For installation instructions, check the [Docker Setup](deployment/docker.md) or [Manual Installation](deployment/manual.md) guides. Operators should use the [Operator Runbook](deployment/operations.md) for deployment modes, verification, upgrades, and rollback, and the [Troubleshooting Playbooks](deployment/troubleshooting.md) for ingestion, authentication, DNS, database, and notification failures. For production secrets, use [Secret Handling with 1Password](deployment/secrets.md). For database operations, use [Database Backup and Restore](deployment/backups.md). For upgrades, use the [Release Checklist](deployment/release-checklist.md).
For Microsoft 365 setup, see [Microsoft 365 Mail Sources](user_guide/microsoft365.md).
For aggregate-report parser support, known edge cases, and fixture guidance, see [DMARC Aggregate Format Compatibility](reference/dmarc-compatibility.md).
+3 -3
View File
@@ -198,14 +198,14 @@ Exit criteria:
## Milestone 12: Enterprise Mail Sources (Microsoft 365) and Connector Framework
Status: Planned
Status: In progress
Goal: make mailbox ingestion work for the most common enterprise setups without relying on IMAP.
Planned:
- Microsoft 365 mail source using OAuth (Graph) with least-privilege scopes.
- Microsoft 365 mail source using OAuth (Graph) with least-privilege scopes. Delivered for delegated `User.Read`, `Mail.Read`, and `offline_access` with encrypted token storage, manual import, scheduled polling, UI setup, and operator docs.
- Shared mailbox and folder selection support for DMARC report collection.
- Import-history parity with existing sources (auditable attachment outcomes, duplicates, parse failures).
- Import-history parity with existing sources (auditable attachment outcomes, duplicates, parse failures). Delivered for Microsoft 365 imports.
- Backfill support with safe throttling and progressive search windows.
- Secret handling mirrors existing guidance (no raw secrets in logs; 1Password-friendly).
+40
View File
@@ -0,0 +1,40 @@
# Microsoft 365 Mail Sources
DMARQ can import DMARC aggregate report attachments from Exchange Online through Microsoft Graph. Use this source type when IMAP is disabled or unavailable in a Microsoft 365 tenant.
## App Registration
Create an app registration in Microsoft Entra admin center:
- Platform: Web
- Redirect URI: `https://<your-dmarq-host>/api/v1/mail-sources/<source-id>/m365/callback`
- Delegated API permissions:
- `User.Read`
- `Mail.Read`
- `offline_access`
- Client secret: create a secret for the web app and store it in your deployment secret manager.
`Mail.Read` is the least-privilege delegated Graph permission DMARQ needs to list messages and read file attachments. `offline_access` is requested so Microsoft returns a refresh token for scheduled polling.
## DMARQ Setup
1. Open **Mail Sources**.
2. Add a source with method **Microsoft 365 (Graph OAuth2)**.
3. Enter the tenant ID (`organizations`, `common`, or a tenant GUID), client ID, and client secret.
4. Leave **Mailbox** empty to read the authorised account, or enter a user principal name for a shared/delegated mailbox that the authorised user can read.
5. Save the source.
6. Use **Connect Microsoft 365** and approve the read-only mailbox access request.
7. Run **Test connection** and **Run import now**.
## Import Behavior
DMARQ reads recent messages in the configured folder, filters for messages that look like DMARC reports, downloads Graph `fileAttachment` items, and sends `.xml`, `.zip`, `.gz`, and `.gzip` attachments through the same parser and persistence path used by upload, IMAP, and Gmail imports.
Imported Graph message IDs are stored on the mail source so scheduled polling does not reprocess the same message. Import history records processed messages, imported reports, duplicates, parse failures, and attachment-level details.
## Troubleshooting
- **Not authorised**: reconnect the source from Mail Sources.
- **Permission error**: confirm the app registration has delegated `Mail.Read` and the authorised account can read the target mailbox.
- **Throttling**: wait and retry, or increase the polling interval.
- **Mailbox/folder not found**: leave Mailbox blank for `/me`, use a valid user principal name for delegated/shared mailboxes, and keep the default `INBOX` folder unless reports are delivered elsewhere.