Merge pull request #181 from christianlouis/codex/m12-m365-graph-source
feat: add Microsoft 365 Graph mail source
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
"""Add Microsoft 365 Graph mail source fields.
|
||||
|
||||
Revision ID: f3a4b5c6d7e8
|
||||
Revises: e2f3a4b5c6d7
|
||||
Create Date: 2026-05-23 00:00:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "f3a4b5c6d7e8"
|
||||
down_revision: Union[str, Sequence[str], None] = "e2f3a4b5c6d7"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("mail_sources", sa.Column("m365_tenant_id", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_client_id", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_client_secret", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_access_token", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_refresh_token", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_mailbox", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_email", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_ingested_ids", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("mail_sources", "m365_ingested_ids")
|
||||
op.drop_column("mail_sources", "m365_email")
|
||||
op.drop_column("mail_sources", "m365_mailbox")
|
||||
op.drop_column("mail_sources", "m365_refresh_token")
|
||||
op.drop_column("mail_sources", "m365_access_token")
|
||||
op.drop_column("mail_sources", "m365_client_secret")
|
||||
op.drop_column("mail_sources", "m365_client_id")
|
||||
op.drop_column("mail_sources", "m365_tenant_id")
|
||||
@@ -3,8 +3,8 @@ Mail Sources API endpoints.
|
||||
|
||||
Provides CRUD operations for MailSource objects stored in the database, plus
|
||||
a *test-connection* action that validates the supplied credentials without
|
||||
persisting anything. Gmail API sources additionally have OAuth2 helper
|
||||
endpoints (authorize-url, callback, fetch).
|
||||
persisting anything. Gmail API and Microsoft 365 sources additionally have
|
||||
OAuth2 helper endpoints (authorize-url, callback, fetch).
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -24,6 +24,7 @@ from app.models.mail_source_import import MailSourceImport
|
||||
from app.services.gmail_client import GmailClient
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.import_history import record_import_attempt
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,7 +39,7 @@ class MailSourceBase(BaseModel):
|
||||
"""Fields shared by create and update payloads."""
|
||||
|
||||
name: str
|
||||
method: str = "IMAP" # IMAP | POP3 | GMAIL_API
|
||||
method: str = "IMAP" # IMAP | POP3 | GMAIL_API | M365_GRAPH
|
||||
server: Optional[str] = None
|
||||
port: int = 993
|
||||
username: Optional[str] = None
|
||||
@@ -50,6 +51,11 @@ class MailSourceBase(BaseModel):
|
||||
# Gmail API OAuth2 fields (only relevant when method == GMAIL_API)
|
||||
gmail_client_id: Optional[str] = None
|
||||
gmail_client_secret: Optional[str] = None
|
||||
# Microsoft 365 Graph OAuth2 fields (only relevant when method == M365_GRAPH)
|
||||
m365_tenant_id: Optional[str] = "common"
|
||||
m365_client_id: Optional[str] = None
|
||||
m365_client_secret: Optional[str] = None
|
||||
m365_mailbox: Optional[str] = None
|
||||
|
||||
|
||||
class MailSourceCreate(MailSourceBase):
|
||||
@@ -71,6 +77,10 @@ class MailSourceUpdate(BaseModel):
|
||||
enabled: Optional[bool] = None
|
||||
gmail_client_id: Optional[str] = None
|
||||
gmail_client_secret: Optional[str] = None
|
||||
m365_tenant_id: Optional[str] = None
|
||||
m365_client_id: Optional[str] = None
|
||||
m365_client_secret: Optional[str] = None
|
||||
m365_mailbox: Optional[str] = None
|
||||
|
||||
|
||||
class MailSourceResponse(MailSourceBase):
|
||||
@@ -86,6 +96,9 @@ class MailSourceResponse(MailSourceBase):
|
||||
gmail_email: Optional[str] = None
|
||||
# Indicate whether OAuth tokens are present (without exposing them)
|
||||
gmail_connected: bool = False
|
||||
# Microsoft 365: show the authorised account and token state, but not tokens
|
||||
m365_email: Optional[str] = None
|
||||
m365_connected: bool = False
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -109,6 +122,13 @@ class GmailCallbackRequest(BaseModel):
|
||||
redirect_uri: str
|
||||
|
||||
|
||||
class M365CallbackRequest(BaseModel):
|
||||
"""Payload for the Microsoft 365 OAuth2 callback endpoint."""
|
||||
|
||||
code: str
|
||||
redirect_uri: str
|
||||
|
||||
|
||||
class MailSourceImportResponse(BaseModel):
|
||||
"""Sanitized import-history entry for a mail source."""
|
||||
|
||||
@@ -151,7 +171,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"auth_required": {
|
||||
"summary": "The mailbox has not been connected yet.",
|
||||
"recovery_steps": [
|
||||
"Use the Connect Gmail action to complete authorization.",
|
||||
"Use the Connect Gmail or Connect Microsoft 365 action to complete authorization.",
|
||||
"Confirm the authorized mailbox is the one that receives DMARC aggregate reports.",
|
||||
],
|
||||
},
|
||||
@@ -159,7 +179,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"summary": "The saved authorization is expired, revoked, or no longer accepted.",
|
||||
"recovery_steps": [
|
||||
"Reconnect the mailbox from Mail Sources.",
|
||||
"If your provider shows a consent screen, approve Gmail read-only access again.",
|
||||
"If your provider shows a consent screen, approve read-only mailbox access again.",
|
||||
],
|
||||
},
|
||||
"authentication": {
|
||||
@@ -173,7 +193,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"summary": "The account is connected but does not have enough mailbox access.",
|
||||
"recovery_steps": [
|
||||
"Grant read access for the mailbox that receives DMARC reports.",
|
||||
"For Gmail, reconnect and approve the requested Gmail read-only scope.",
|
||||
"For OAuth sources, reconnect and approve the requested read-only mail scope.",
|
||||
],
|
||||
},
|
||||
"connectivity": {
|
||||
@@ -200,7 +220,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"missing_config": {
|
||||
"summary": "Required connection settings are missing.",
|
||||
"recovery_steps": [
|
||||
"Fill in the server, username, and password or complete Gmail authorization.",
|
||||
"Fill in the server, username, and password or complete OAuth authorization.",
|
||||
"Save the source before running stored-source tests.",
|
||||
],
|
||||
},
|
||||
@@ -227,12 +247,23 @@ def _diagnostic_category(message: str, details: Optional[object] = None) -> str:
|
||||
if any(term in text for term in ("not yet authorised", "not yet authorized", "complete oauth")):
|
||||
return "auth_required"
|
||||
if any(
|
||||
term in text for term in ("expired", "revoked", "invalid_grant", "refresh token", "oauth")
|
||||
term in text
|
||||
for term in (
|
||||
"expired",
|
||||
"revoked",
|
||||
"invalid_grant",
|
||||
"interaction_required",
|
||||
"refresh token",
|
||||
"oauth",
|
||||
)
|
||||
):
|
||||
return "auth_expired"
|
||||
if any(term in text for term in ("rate", "quota", "throttl", "too many", "429")):
|
||||
return "throttling"
|
||||
if any(term in text for term in ("scope", "permission", "access denied", "insufficient")):
|
||||
if any(
|
||||
term in text
|
||||
for term in ("scope", "permission", "access denied", "insufficient", "forbidden", "403")
|
||||
):
|
||||
return "permissions"
|
||||
if any(term in text for term in ("mailbox", "folder", "select failed", "does not exist")):
|
||||
return "mailbox_not_found"
|
||||
@@ -308,6 +339,14 @@ def _get_source_or_404(source_id: int, db: Session) -> MailSource:
|
||||
return source
|
||||
|
||||
|
||||
def _safe_attr(source: MailSource, name: str, default: Any = None) -> Any:
|
||||
"""Read optional source attributes without letting test doubles invent fields."""
|
||||
value = getattr(source, name, default)
|
||||
if value.__class__.__module__.startswith("unittest.mock"):
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _source_to_response(source: MailSource) -> MailSourceResponse:
|
||||
"""Convert ORM row to response schema, masking the stored password."""
|
||||
return MailSourceResponse(
|
||||
@@ -329,6 +368,12 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
|
||||
gmail_client_secret="**redacted**" if source.gmail_client_secret else None,
|
||||
gmail_email=source.gmail_email,
|
||||
gmail_connected=bool(source.gmail_access_token),
|
||||
m365_tenant_id=_safe_attr(source, "m365_tenant_id", "common") or "common",
|
||||
m365_client_id=_safe_attr(source, "m365_client_id"),
|
||||
m365_client_secret=("**redacted**" if _safe_attr(source, "m365_client_secret") else None),
|
||||
m365_mailbox=_safe_attr(source, "m365_mailbox"),
|
||||
m365_email=_safe_attr(source, "m365_email"),
|
||||
m365_connected=bool(_safe_attr(source, "m365_access_token")),
|
||||
)
|
||||
|
||||
|
||||
@@ -437,6 +482,46 @@ def _fetch_gmail_source(source: MailSource, db: Session) -> Dict[str, Any]:
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_m365_source(source: MailSource, db: Session) -> Dict[str, Any]:
|
||||
"""Run one Microsoft 365 Graph import and persist source/import metadata."""
|
||||
if not source.m365_access_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Microsoft 365 account not yet authorised. Complete OAuth2 flow first.",
|
||||
)
|
||||
|
||||
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
results = client.fetch_reports()
|
||||
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
|
||||
refreshed = client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
|
||||
source.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
|
||||
db.commit()
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_imap_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
|
||||
"""Run one IMAP import and persist source/import metadata."""
|
||||
client = IMAPClient(
|
||||
@@ -459,6 +544,8 @@ def _fetch_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
|
||||
"""Dispatch a manual fetch for one configured mail source."""
|
||||
if source.method == "GMAIL_API":
|
||||
return _fetch_gmail_source(source, db)
|
||||
if source.method == "M365_GRAPH":
|
||||
return _fetch_m365_source(source, db)
|
||||
if source.method == "IMAP":
|
||||
return _fetch_imap_source(source, db, days)
|
||||
raise HTTPException(
|
||||
@@ -502,6 +589,10 @@ async def create_mail_source(
|
||||
enabled=payload.enabled,
|
||||
gmail_client_id=payload.gmail_client_id,
|
||||
gmail_client_secret=payload.gmail_client_secret,
|
||||
m365_tenant_id=payload.m365_tenant_id or "common",
|
||||
m365_client_id=payload.m365_client_id,
|
||||
m365_client_secret=payload.m365_client_secret,
|
||||
m365_mailbox=payload.m365_mailbox,
|
||||
)
|
||||
db.add(source)
|
||||
db.commit()
|
||||
@@ -572,7 +663,7 @@ async def fetch_mail_source(
|
||||
_redact_sensitive_text(err),
|
||||
)
|
||||
|
||||
return _fetch_response(source, results)
|
||||
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
|
||||
|
||||
|
||||
@router.put("/{source_id}", response_model=MailSourceResponse)
|
||||
@@ -628,7 +719,7 @@ async def toggle_mail_source(
|
||||
|
||||
|
||||
@router.post("/{source_id}/test", response_model=Dict[str, Any])
|
||||
async def test_stored_mail_source(
|
||||
async def test_stored_mail_source( # noqa: C901
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
@@ -671,6 +762,48 @@ async def test_stored_mail_source(
|
||||
details=exc,
|
||||
)
|
||||
|
||||
if source.method == "M365_GRAPH":
|
||||
if not source.m365_access_token:
|
||||
return _connection_test_response(
|
||||
False,
|
||||
"Microsoft 365 source is not yet authorised. "
|
||||
"Use the Connect Microsoft 365 button to complete OAuth2 authorisation.",
|
||||
)
|
||||
try:
|
||||
graph_client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
)
|
||||
stats = graph_client.test_connection()
|
||||
refreshed = graph_client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
source.last_checked = datetime.utcnow()
|
||||
db.commit()
|
||||
return _connection_test_response(
|
||||
True,
|
||||
f"Microsoft 365 credentials are valid (account: {source.m365_email or 'unknown'}).",
|
||||
stats=stats,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 Graph test failed for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
return _connection_test_response(
|
||||
False,
|
||||
"Microsoft 365 test failed. The saved authorization may need attention.",
|
||||
details=exc,
|
||||
)
|
||||
|
||||
if source.method != "IMAP":
|
||||
return _connection_test_response(
|
||||
False,
|
||||
@@ -722,6 +855,246 @@ async def test_connection_adhoc(
|
||||
return _connection_test_response(success, message, stats)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Microsoft 365 / Graph OAuth2 routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/{source_id}/m365/authorize-url", response_model=Dict[str, Any])
|
||||
async def m365_authorize_url(
|
||||
source_id: int,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return a Microsoft identity platform authorization URL for M365_GRAPH."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
if not source.m365_client_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="m365_client_id is not configured for this source.",
|
||||
)
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
|
||||
auth_url = MicrosoftGraphClient.build_authorization_url(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id,
|
||||
redirect_uri=redirect_uri,
|
||||
state=str(source_id),
|
||||
)
|
||||
return {"authorization_url": auth_url, "redirect_uri": redirect_uri}
|
||||
|
||||
|
||||
@router.get("/{source_id}/m365/callback")
|
||||
async def m365_oauth_callback(
|
||||
source_id: int,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""Handle the Microsoft identity platform OAuth2 redirect."""
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
code = request.query_params.get("code")
|
||||
error = request.query_params.get("error")
|
||||
|
||||
if error or not code:
|
||||
html = (
|
||||
"<html><body><p>Microsoft 365 authorisation failed: "
|
||||
f"{error or 'no code received'}. "
|
||||
"You may close this window.</p></body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html, status_code=400)
|
||||
|
||||
source = db.query(MailSource).filter(MailSource.id == source_id).first()
|
||||
if source is None or source.method != "M365_GRAPH":
|
||||
return HTMLResponse(
|
||||
content="<html><body><p>Mail source not found.</p></body></html>",
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
|
||||
|
||||
try:
|
||||
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
code=code,
|
||||
redirect_uri=redirect_uri,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 token exchange error for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
html = (
|
||||
"<html><body><p>Token exchange failed. "
|
||||
"Please close this window and try again.</p></body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html, status_code=400)
|
||||
|
||||
access_token = token_data.get("access_token")
|
||||
refresh_token = token_data.get("refresh_token")
|
||||
if not access_token:
|
||||
return HTMLResponse(
|
||||
content="<html><body><p>No access token returned by Microsoft.</p></body></html>",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
m365_email = MicrosoftGraphClient.get_account_email(access_token)
|
||||
source.m365_access_token = access_token
|
||||
if refresh_token:
|
||||
source.m365_refresh_token = refresh_token
|
||||
if m365_email:
|
||||
source.m365_email = m365_email
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
logger.info(
|
||||
"Microsoft 365 OAuth2 authorisation complete for source id=%d (account=%s)",
|
||||
int(source_id),
|
||||
_sanitize_for_log(m365_email or "unknown"),
|
||||
)
|
||||
|
||||
html = (
|
||||
"<html><body>"
|
||||
"<p>Microsoft 365 account connected successfully"
|
||||
f"{(' (' + m365_email + ')') if m365_email else ''}. "
|
||||
"You may close this window.</p>"
|
||||
"<script>window.close();</script>"
|
||||
"</body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html)
|
||||
|
||||
|
||||
@router.post("/{source_id}/m365/callback", response_model=MailSourceResponse)
|
||||
async def m365_oauth_callback_post(
|
||||
source_id: int,
|
||||
payload: M365CallbackRequest,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> MailSourceResponse:
|
||||
"""Exchange a Microsoft OAuth2 authorization code for Graph tokens."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
try:
|
||||
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
code=payload.code,
|
||||
redirect_uri=payload.redirect_uri,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 token exchange error for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
"Token exchange failed. Please check the Microsoft 365 "
|
||||
"connection settings and try again."
|
||||
),
|
||||
) from exc
|
||||
|
||||
access_token = token_data.get("access_token")
|
||||
refresh_token = token_data.get("refresh_token")
|
||||
if not access_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Microsoft did not return an access token.",
|
||||
)
|
||||
|
||||
m365_email = MicrosoftGraphClient.get_account_email(access_token)
|
||||
source.m365_access_token = access_token
|
||||
if refresh_token:
|
||||
source.m365_refresh_token = refresh_token
|
||||
if m365_email:
|
||||
source.m365_email = m365_email
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(source)
|
||||
|
||||
logger.info(
|
||||
"Microsoft 365 OAuth2 tokens saved for source id=%d (account=%s)",
|
||||
int(source_id),
|
||||
_sanitize_for_log(m365_email or "unknown"),
|
||||
)
|
||||
return _source_to_response(source)
|
||||
|
||||
|
||||
@router.post("/{source_id}/m365/fetch", response_model=Dict[str, Any])
|
||||
async def m365_fetch_reports(
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Manually trigger a Microsoft 365 Graph DMARC report fetch."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
results = _fetch_m365_source(source, db)
|
||||
logger.info(
|
||||
"Microsoft 365 fetch for source id=%d: processed=%d reports_found=%d",
|
||||
int(source_id),
|
||||
int(results.get("processed", 0)),
|
||||
int(results.get("reports_found", 0)),
|
||||
)
|
||||
for err in results.get("errors", []):
|
||||
logger.warning(
|
||||
"Microsoft 365 fetch warning for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(err),
|
||||
)
|
||||
|
||||
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
|
||||
|
||||
|
||||
@router.delete("/{source_id}/m365/connection", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def m365_disconnect(
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> None:
|
||||
"""Clear the stored Microsoft Graph OAuth2 tokens for this source."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
source.m365_access_token = None
|
||||
source.m365_refresh_token = None
|
||||
source.m365_email = None
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
logger.info("Microsoft 365 tokens cleared for source id=%d", int(source_id))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gmail API OAuth2 routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+136
-2
@@ -30,6 +30,7 @@ from app.models.mail_source import MailSource # noqa: F401 – ensure table is
|
||||
from app.services.gmail_client import GmailClient
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.import_history import record_import_attempt
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
from app.services.report_persistence import hydrate_report_store_from_db
|
||||
from app.services.report_store import ReportStore
|
||||
from app.services.runtime_status import (
|
||||
@@ -162,7 +163,74 @@ def _poll_single_gmail_source(source: MailSource) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _poll_all_enabled_sources() -> list[MailSource]:
|
||||
def _poll_single_m365_source(source: MailSource) -> None:
|
||||
"""Fetch DMARC reports for a single M365_GRAPH mail source."""
|
||||
global last_check_time # pylint: disable=global-statement
|
||||
|
||||
if not source.m365_access_token:
|
||||
logger.info(
|
||||
"Microsoft 365 polling (source id=%d): skipped – OAuth2 not yet authorised",
|
||||
source.id,
|
||||
)
|
||||
return
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
src = db.query(MailSource).get(source.id)
|
||||
poll_source = src or source
|
||||
already = MicrosoftGraphClient.load_ingested_ids(poll_source.m365_ingested_ids)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id=poll_source.m365_tenant_id or "common",
|
||||
client_id=poll_source.m365_client_id or "",
|
||||
client_secret=poll_source.m365_client_secret or "",
|
||||
access_token=poll_source.m365_access_token,
|
||||
refresh_token=poll_source.m365_refresh_token or "",
|
||||
mailbox=poll_source.m365_mailbox,
|
||||
folder=poll_source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
results = client.fetch_reports()
|
||||
if src:
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
src.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
|
||||
refreshed = client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
src.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
src.m365_refresh_token = refreshed["refresh_token"]
|
||||
|
||||
src.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
last_check_time = datetime.now()
|
||||
|
||||
if results["success"]:
|
||||
logger.info(
|
||||
"Microsoft 365 polling (source id=%d): %s emails processed, "
|
||||
"%s aggregate reports found",
|
||||
source.id,
|
||||
results["processed"],
|
||||
results["reports_found"],
|
||||
)
|
||||
if results["new_domains"]:
|
||||
logger.info("New domains found: %s", ", ".join(results["new_domains"]))
|
||||
else:
|
||||
logger.error(
|
||||
"Microsoft 365 polling (source id=%d) failed: %s",
|
||||
source.id,
|
||||
results.get("error", "Unknown error"),
|
||||
)
|
||||
|
||||
|
||||
def _poll_all_enabled_sources() -> list[MailSource]: # noqa: C901
|
||||
"""Iterate over all enabled mail sources and poll each one."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
@@ -182,6 +250,11 @@ def _poll_all_enabled_sources() -> list[MailSource]:
|
||||
_poll_single_gmail_source(source)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Gmail source id=%d: %s", source.id, str(e))
|
||||
elif source.method == "M365_GRAPH":
|
||||
try:
|
||||
_poll_single_m365_source(source)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
|
||||
elif source.method == "IMAP":
|
||||
try:
|
||||
_poll_single_imap_source(source)
|
||||
@@ -682,7 +755,50 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
|
||||
def _trigger_poll_m365_source(source: MailSource, db) -> dict:
|
||||
"""Poll a single M365_GRAPH source and return a result dict for the API response."""
|
||||
global last_check_time # pylint: disable=global-statement
|
||||
|
||||
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
|
||||
graph_client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
started_at = datetime.utcnow()
|
||||
results = graph_client.fetch_reports()
|
||||
last_check_time = datetime.now()
|
||||
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
refreshed = graph_client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
source.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
|
||||
db.commit()
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"success": results["success"],
|
||||
"processed": results.get("processed", 0),
|
||||
"reports_found": results.get("reports_found", 0),
|
||||
"forensic_reports_found": results.get("forensic_reports_found", 0),
|
||||
"duplicate_forensic_reports": results.get("duplicate_forensic_reports", 0),
|
||||
"new_domains": results.get("new_domains", []),
|
||||
}
|
||||
|
||||
|
||||
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict: # noqa: C901
|
||||
"""Dispatch a single mail source for the manual trigger-poll endpoint.
|
||||
|
||||
Returns a result/summary dict that is included in the API response.
|
||||
@@ -705,6 +821,24 @@ def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
|
||||
"success": False,
|
||||
"error": "Failed to poll. Check server logs for details.",
|
||||
}
|
||||
if source.method == "M365_GRAPH":
|
||||
if not source.m365_access_token:
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"skipped": True,
|
||||
"reason": "Microsoft 365 account not yet authorised",
|
||||
}
|
||||
try:
|
||||
return _trigger_poll_m365_source(source, db)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"success": False,
|
||||
"error": "Failed to poll. Check server logs for details.",
|
||||
}
|
||||
if source.method == "IMAP":
|
||||
try:
|
||||
return _trigger_poll_imap_source(source, db, days=days)
|
||||
|
||||
@@ -18,6 +18,7 @@ class MailSource(Base):
|
||||
- ``IMAP`` – standard IMAP4 (over SSL/TLS or STARTTLS)
|
||||
- ``POP3`` – POP3 inbox (stub for future implementation)
|
||||
- ``GMAIL_API`` – Gmail API with OAuth 2.0
|
||||
- ``M365_GRAPH`` – Microsoft 365 / Exchange Online via Microsoft Graph
|
||||
"""
|
||||
|
||||
__tablename__ = "mail_sources"
|
||||
@@ -28,7 +29,7 @@ class MailSource(Base):
|
||||
name = Column(String, nullable=False)
|
||||
|
||||
# Connection method – determines which fields are used at runtime
|
||||
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API
|
||||
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API | M365_GRAPH
|
||||
|
||||
# Connection details (used by IMAP and POP3)
|
||||
server = Column(String, nullable=True)
|
||||
@@ -48,6 +49,19 @@ class MailSource(Base):
|
||||
# JSON-encoded list of Gmail message IDs that have already been ingested
|
||||
gmail_ingested_ids = Column(Text, nullable=True, default="[]")
|
||||
|
||||
# Microsoft 365 / Graph OAuth2 credentials (used by M365_GRAPH method)
|
||||
m365_tenant_id = Column(String, nullable=True, default="common")
|
||||
m365_client_id = Column(String, nullable=True)
|
||||
_m365_client_secret = Column("m365_client_secret", Text, nullable=True)
|
||||
_m365_access_token = Column("m365_access_token", Text, nullable=True)
|
||||
_m365_refresh_token = Column("m365_refresh_token", Text, nullable=True)
|
||||
# Optional user/shared mailbox to poll. Empty means the authorised account (/me).
|
||||
m365_mailbox = Column(String, nullable=True)
|
||||
# Email address reported by Microsoft Graph for the authorised account.
|
||||
m365_email = Column(String, nullable=True)
|
||||
# JSON-encoded list of Graph message IDs that have already been ingested
|
||||
m365_ingested_ids = Column(Text, nullable=True, default="[]")
|
||||
|
||||
# Polling behaviour
|
||||
polling_interval = Column(Integer, default=60) # minutes
|
||||
|
||||
@@ -76,6 +90,9 @@ class MailSource(Base):
|
||||
"gmail_client_secret": self._gmail_client_secret,
|
||||
"gmail_access_token": self._gmail_access_token,
|
||||
"gmail_refresh_token": self._gmail_refresh_token,
|
||||
"m365_client_secret": self._m365_client_secret,
|
||||
"m365_access_token": self._m365_access_token,
|
||||
"m365_refresh_token": self._m365_refresh_token,
|
||||
}
|
||||
|
||||
for public_name, stored_value in secret_fields.items():
|
||||
@@ -120,3 +137,30 @@ class MailSource(Base):
|
||||
@gmail_refresh_token.setter
|
||||
def gmail_refresh_token(self, value):
|
||||
self._gmail_refresh_token = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_client_secret(self):
|
||||
"""Return the decrypted Microsoft 365 OAuth client secret, if present."""
|
||||
return decrypt_secret(self._m365_client_secret)
|
||||
|
||||
@m365_client_secret.setter
|
||||
def m365_client_secret(self, value):
|
||||
self._m365_client_secret = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_access_token(self):
|
||||
"""Return the decrypted Microsoft Graph access token, if present."""
|
||||
return decrypt_secret(self._m365_access_token)
|
||||
|
||||
@m365_access_token.setter
|
||||
def m365_access_token(self, value):
|
||||
self._m365_access_token = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_refresh_token(self):
|
||||
"""Return the decrypted Microsoft Graph refresh token, if present."""
|
||||
return decrypt_secret(self._m365_refresh_token)
|
||||
|
||||
@m365_refresh_token.setter
|
||||
def m365_refresh_token(self, value):
|
||||
self._m365_refresh_token = encrypt_secret(value)
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Microsoft Graph client for retrieving DMARC aggregate reports."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.report_persistence import report_exists, save_parsed_report
|
||||
from app.services.report_store import ReportStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GRAPH_BASE_URL = "https://graph.microsoft.com/v1.0"
|
||||
LOGIN_BASE_URL = "https://login.microsoftonline.com"
|
||||
|
||||
M365_SCOPES = [
|
||||
"offline_access",
|
||||
"https://graph.microsoft.com/User.Read",
|
||||
"https://graph.microsoft.com/Mail.Read",
|
||||
]
|
||||
|
||||
_PAGE_SIZE = 100
|
||||
_DMARC_SUBJECT_TERMS = (
|
||||
"dmarc",
|
||||
"aggregate report",
|
||||
"domain report",
|
||||
"report domain",
|
||||
"rua",
|
||||
"submitter",
|
||||
)
|
||||
_DMARC_SENDER_TERMS = (
|
||||
"dmarc",
|
||||
"reports",
|
||||
"postmaster",
|
||||
)
|
||||
|
||||
|
||||
class MicrosoftGraphError(RuntimeError):
|
||||
"""Raised when Microsoft Graph or the token endpoint returns a failure."""
|
||||
|
||||
|
||||
class MicrosoftGraphClient:
|
||||
"""
|
||||
Retrieve DMARC aggregate reports from Microsoft 365 through Microsoft Graph.
|
||||
|
||||
The client uses delegated OAuth tokens and read-only Graph scopes. Messages
|
||||
are never modified or deleted; already-ingested Graph message IDs are stored
|
||||
by the caller to avoid reprocessing the same email.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
access_token: str,
|
||||
refresh_token: str,
|
||||
mailbox: Optional[str] = None,
|
||||
folder: str = "inbox",
|
||||
already_ingested_ids: Optional[List[str]] = None,
|
||||
db: Any = None,
|
||||
):
|
||||
self.tenant_id = tenant_id or "common"
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
self.access_token = access_token
|
||||
self.refresh_token = refresh_token
|
||||
self.mailbox = (mailbox or "").strip()
|
||||
self.folder = folder or "inbox"
|
||||
self.already_ingested_ids: List[str] = list(already_ingested_ids or [])
|
||||
self.report_store = ReportStore.get_instance()
|
||||
self.db = db
|
||||
self._refreshed_tokens: Optional[Dict[str, str]] = None
|
||||
|
||||
def get_refreshed_tokens(self) -> Optional[Dict[str, str]]:
|
||||
"""Return refreshed OAuth tokens, if a request had to refresh them."""
|
||||
return self._refreshed_tokens
|
||||
|
||||
@staticmethod
|
||||
def build_authorization_url(
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
redirect_uri: str,
|
||||
state: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build a Microsoft identity platform authorization-code URL."""
|
||||
tenant = quote(tenant_id or "common", safe="")
|
||||
params: Dict[str, str] = {
|
||||
"client_id": client_id,
|
||||
"response_type": "code",
|
||||
"redirect_uri": redirect_uri,
|
||||
"response_mode": "query",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
"prompt": "select_account",
|
||||
}
|
||||
if state:
|
||||
params["state"] = state
|
||||
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/authorize?" + urlencode(params)
|
||||
|
||||
@staticmethod
|
||||
def exchange_code_for_tokens(
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
code: str,
|
||||
redirect_uri: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Exchange an authorization code for Microsoft Graph tokens."""
|
||||
data = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
}
|
||||
resp = httpx.post(MicrosoftGraphClient._token_url(tenant_id), data=data, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
raise MicrosoftGraphError(
|
||||
f"Microsoft token exchange failed ({resp.status_code}): {resp.text}"
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
@staticmethod
|
||||
def get_account_email(access_token: str) -> Optional[str]:
|
||||
"""Return the mailbox identity exposed by Graph /me for an access token."""
|
||||
try:
|
||||
resp = httpx.get(
|
||||
f"{GRAPH_BASE_URL}/me",
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
params={"$select": "mail,userPrincipalName"},
|
||||
timeout=30,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
profile = resp.json()
|
||||
return profile.get("mail") or profile.get("userPrincipalName")
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Failed to fetch Microsoft 365 account email: %s", exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def load_ingested_ids(json_text: Optional[str]) -> List[str]:
|
||||
"""Deserialize the m365_ingested_ids text column into a list."""
|
||||
if not json_text:
|
||||
return []
|
||||
try:
|
||||
decoded = json.loads(json_text)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return []
|
||||
return [str(item) for item in decoded] if isinstance(decoded, list) else []
|
||||
|
||||
@staticmethod
|
||||
def dump_ingested_ids(ids: List[str]) -> str:
|
||||
"""Serialize Graph message IDs for database storage."""
|
||||
return json.dumps(ids)
|
||||
|
||||
def test_connection(self) -> Dict[str, Any]:
|
||||
"""Verify that the saved delegated token can read the target mailbox."""
|
||||
mailbox_path = self._mailbox_path()
|
||||
data = self._request(
|
||||
"GET",
|
||||
f"{mailbox_path}/messages",
|
||||
params={"$top": 1, "$select": "id"},
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"message_count": len(data.get("value", [])),
|
||||
"diagnostic_detail": "Microsoft Graph mailbox read succeeded.",
|
||||
}
|
||||
|
||||
def fetch_reports(self) -> Dict[str, Any]:
|
||||
"""Fetch and ingest DMARC report attachments from Microsoft Graph."""
|
||||
stats: Dict[str, Any] = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"reports_found": 0,
|
||||
"forensic_reports_found": 0,
|
||||
"duplicate_reports": 0,
|
||||
"duplicate_forensic_reports": 0,
|
||||
"new_domains": [],
|
||||
"errors": [],
|
||||
"new_ingested_ids": [],
|
||||
"details": [],
|
||||
}
|
||||
|
||||
try:
|
||||
messages = self._list_dmarc_messages()
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Microsoft Graph: failed to list messages: %s", exc)
|
||||
return {**stats, "success": False, "error": str(exc), "errors": [str(exc)]}
|
||||
|
||||
domains_before = set(self.report_store.get_domains())
|
||||
|
||||
for message in messages:
|
||||
message_id = str(message.get("id") or "")
|
||||
if not message_id:
|
||||
continue
|
||||
if message_id in self.already_ingested_ids:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="already_ingested_message",
|
||||
message_id=message_id,
|
||||
)
|
||||
continue
|
||||
|
||||
stats["processed"] += 1
|
||||
found = self._process_message(message, stats)
|
||||
if found >= 0:
|
||||
stats["new_ingested_ids"].append(message_id)
|
||||
self.already_ingested_ids.append(message_id)
|
||||
|
||||
domains_after = set(self.report_store.get_domains())
|
||||
stats["new_domains"] = list(domains_after - domains_before)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _token_url(tenant_id: str) -> str:
|
||||
tenant = quote(tenant_id or "common", safe="")
|
||||
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/token"
|
||||
|
||||
@staticmethod
|
||||
def _append_detail(stats: dict, **detail: str) -> None:
|
||||
stats.setdefault("details", []).append(
|
||||
{key: value for key, value in detail.items() if value}
|
||||
)
|
||||
|
||||
def _mailbox_path(self) -> str:
|
||||
if not self.mailbox or self.mailbox.lower() == "me":
|
||||
return "/me"
|
||||
return f"/users/{quote(self.mailbox, safe='')}"
|
||||
|
||||
def _messages_path(self) -> str:
|
||||
mailbox_path = self._mailbox_path()
|
||||
folder = (self.folder or "").strip()
|
||||
if not folder:
|
||||
return f"{mailbox_path}/messages"
|
||||
if folder.upper() == "INBOX":
|
||||
folder = "inbox"
|
||||
return f"{mailbox_path}/mailFolders/{quote(folder, safe='')}/messages"
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {"Authorization": f"Bearer {self.access_token}"}
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path_or_url: str,
|
||||
*,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
url = path_or_url if path_or_url.startswith("http") else f"{GRAPH_BASE_URL}{path_or_url}"
|
||||
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
|
||||
if resp.status_code == 401 and self.refresh_token:
|
||||
self._refresh_access_token()
|
||||
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
|
||||
if resp.status_code < 200 or resp.status_code >= 300:
|
||||
raise MicrosoftGraphError(self._format_error(resp))
|
||||
return resp.json() if resp.content else {}
|
||||
|
||||
def _refresh_access_token(self) -> None:
|
||||
data = {
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret,
|
||||
"refresh_token": self.refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
}
|
||||
resp = httpx.post(self._token_url(self.tenant_id), data=data, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
raise MicrosoftGraphError(
|
||||
f"Microsoft token refresh failed ({resp.status_code}): {resp.text}"
|
||||
)
|
||||
token_data = resp.json()
|
||||
access_token = token_data.get("access_token")
|
||||
if not access_token:
|
||||
raise MicrosoftGraphError("Microsoft token refresh did not return an access token.")
|
||||
self.access_token = access_token
|
||||
refreshed = {"access_token": access_token}
|
||||
if token_data.get("refresh_token"):
|
||||
self.refresh_token = token_data["refresh_token"]
|
||||
refreshed["refresh_token"] = token_data["refresh_token"]
|
||||
self._refreshed_tokens = refreshed
|
||||
|
||||
@staticmethod
|
||||
def _format_error(resp: httpx.Response) -> str:
|
||||
try:
|
||||
payload = resp.json()
|
||||
except ValueError:
|
||||
payload = {}
|
||||
message = payload.get("error_description")
|
||||
if not message and isinstance(payload.get("error"), dict):
|
||||
message = payload["error"].get("message")
|
||||
code = payload["error"].get("code")
|
||||
if code:
|
||||
message = f"{code}: {message}" if message else code
|
||||
return message or f"Microsoft Graph request failed ({resp.status_code}): {resp.text}"
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_dmarc_message(message: Dict[str, Any]) -> bool:
|
||||
if not message.get("hasAttachments"):
|
||||
return False
|
||||
subject = str(message.get("subject") or "").lower()
|
||||
sender = (
|
||||
((message.get("from") or {}).get("emailAddress") or {}).get("address") or ""
|
||||
).lower()
|
||||
return any(term in subject for term in _DMARC_SUBJECT_TERMS) or any(
|
||||
term in sender for term in _DMARC_SENDER_TERMS
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_dmarc_attachment(filename: str) -> bool:
|
||||
lower = filename.lower()
|
||||
return (
|
||||
lower.endswith(".xml")
|
||||
or lower.endswith(".zip")
|
||||
or lower.endswith(".gz")
|
||||
or lower.endswith(".gzip")
|
||||
)
|
||||
|
||||
def _list_dmarc_messages(self) -> List[Dict[str, Any]]:
|
||||
messages: List[Dict[str, Any]] = []
|
||||
url = self._messages_path()
|
||||
params: Optional[Dict[str, Any]] = {
|
||||
"$top": _PAGE_SIZE,
|
||||
"$select": "id,subject,from,hasAttachments,receivedDateTime",
|
||||
"$orderby": "receivedDateTime desc",
|
||||
}
|
||||
|
||||
while url:
|
||||
data = self._request("GET", url, params=params)
|
||||
for message in data.get("value", []):
|
||||
if self._looks_like_dmarc_message(message):
|
||||
messages.append(message)
|
||||
url = data.get("@odata.nextLink")
|
||||
params = None
|
||||
|
||||
return messages
|
||||
|
||||
def _process_message(self, message: Dict[str, Any], stats: Dict[str, Any]) -> int:
|
||||
message_id = str(message.get("id") or "")
|
||||
try:
|
||||
attachments = self._list_attachments(message_id)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Microsoft Graph: failed to fetch attachments for %s: %s", message_id, exc)
|
||||
stats["errors"].append(f"Failed to fetch attachments for message {message_id}: {exc}")
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="error",
|
||||
reason="attachment_fetch_failed",
|
||||
message_id=message_id,
|
||||
error=str(exc),
|
||||
)
|
||||
return -1
|
||||
return self._process_attachments(message_id, attachments, stats)
|
||||
|
||||
def _list_attachments(self, message_id: str) -> List[Dict[str, Any]]:
|
||||
mailbox_path = self._mailbox_path()
|
||||
data = self._request(
|
||||
"GET",
|
||||
f"{mailbox_path}/messages/{quote(message_id, safe='')}/attachments",
|
||||
)
|
||||
return list(data.get("value", []))
|
||||
|
||||
def _store_report_if_new(self, report: Dict[str, Any]) -> bool:
|
||||
domain = report.get("domain", "unknown")
|
||||
report_id = report.get("report_id", "")
|
||||
if report_id and (
|
||||
self.report_store.has_report(domain, report_id)
|
||||
or (self.db is not None and report_exists(self.db, domain, report_id))
|
||||
):
|
||||
logger.info("Skipping duplicate DMARC report %s for %s", report_id, domain)
|
||||
return False
|
||||
|
||||
if self.db is not None:
|
||||
save_parsed_report(self.db, report)
|
||||
self.report_store.add_report(report)
|
||||
return True
|
||||
|
||||
def _process_attachments(
|
||||
self,
|
||||
message_id: str,
|
||||
attachments: List[Dict[str, Any]],
|
||||
stats: Dict[str, Any],
|
||||
) -> int:
|
||||
reports_found = 0
|
||||
|
||||
for attachment in attachments:
|
||||
filename = str(attachment.get("name") or "")
|
||||
if not filename:
|
||||
continue
|
||||
if not self._is_dmarc_attachment(filename):
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="unsupported_attachment",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
attachment_type = str(attachment.get("@odata.type") or "").lower()
|
||||
if "fileattachment" not in attachment_type:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="unsupported_attachment_type",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
content_b64 = attachment.get("contentBytes")
|
||||
if not content_b64:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="empty_attachment",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
content = base64.b64decode(content_b64)
|
||||
report = DMARCParser.parse_file(content, filename)
|
||||
domain = str(report.get("domain", "unknown"))
|
||||
report_id = str(report.get("report_id", ""))
|
||||
if self._store_report_if_new(report):
|
||||
stats["reports_found"] += 1
|
||||
reports_found += 1
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="imported",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
domain=domain,
|
||||
report_id=report_id,
|
||||
)
|
||||
else:
|
||||
stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="duplicate",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
domain=domain,
|
||||
report_id=report_id,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Failed to parse Graph DMARC attachment %s: %s", filename, exc)
|
||||
stats["errors"].append(f"Failed to parse {filename}: {exc}")
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="error",
|
||||
reason="parse_failed",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
return reports_found
|
||||
@@ -86,8 +86,8 @@
|
||||
<td>
|
||||
<span class="badge badge-outline" x-text="source.method"></span>
|
||||
</td>
|
||||
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_email || '—') : (source.server ? source.server + ':' + source.port : '—')"></td>
|
||||
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_connected ? '✅ Connected' : '⚠️ Not authorised') : (source.username || '—')"></td>
|
||||
<td x-text="sourceAccountLabel(source)"></td>
|
||||
<td x-text="sourceStatusLabel(source)"></td>
|
||||
<td x-text="source.last_checked ? new Date(source.last_checked).toLocaleString() : 'Never'"></td>
|
||||
<td>
|
||||
<input
|
||||
@@ -293,6 +293,7 @@
|
||||
<option value="IMAP">IMAP</option>
|
||||
<option value="POP3">POP3 (coming soon)</option>
|
||||
<option value="GMAIL_API">Gmail API (OAuth2)</option>
|
||||
<option value="M365_GRAPH">Microsoft 365 (Graph OAuth2)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
@@ -434,6 +435,85 @@
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- Microsoft 365 Graph fields -->
|
||||
<template x-if="form.method === 'M365_GRAPH'">
|
||||
<div class="space-y-4">
|
||||
<div class="alert alert-info text-sm p-3">
|
||||
<div>
|
||||
<p class="font-semibold">Microsoft 365 Setup</p>
|
||||
<p class="mt-1">Enter an app registration Client ID and client secret from Microsoft Entra admin center. Add this app's callback URL as a Web redirect URI, then save and click <strong>Connect Microsoft 365</strong>.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Tenant ID</span></label>
|
||||
<input type="text" x-model="form.m365_tenant_id"
|
||||
placeholder="common, organizations, or tenant GUID"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Microsoft Client ID <span class="text-error">*</span></span></label>
|
||||
<input type="text" x-model="form.m365_client_id"
|
||||
placeholder="Application (client) ID"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label">
|
||||
<span class="label-text font-medium">Microsoft Client Secret <span class="text-error">*</span></span>
|
||||
<span class="label-text-alt text-muted-foreground" x-show="editingId">Leave blank to keep existing</span>
|
||||
</label>
|
||||
<div class="relative">
|
||||
<input :type="showPassword ? 'text' : 'password'"
|
||||
x-model="form.m365_client_secret"
|
||||
class="input input-bordered w-full pr-10" />
|
||||
<button type="button"
|
||||
class="absolute right-2 top-3 text-muted-foreground hover:text-foreground"
|
||||
x-on:click="showPassword = !showPassword">
|
||||
<svg x-show="!showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
|
||||
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
|
||||
<circle cx="12" cy="12" r="3"></circle>
|
||||
</svg>
|
||||
<svg x-show="showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
|
||||
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M17.94 17.94A10.07 10.07 0 0 1 12 20c-7 0-11-8-11-8a18.45 18.45 0 0 1 5.06-5.94"></path>
|
||||
<path d="M9.9 4.24A9.12 9.12 0 0 1 12 4c7 0 11 8 11 8a18.5 18.5 0 0 1-2.16 3.19"></path>
|
||||
<line x1="1" y1="1" x2="23" y2="23"></line>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Mailbox</span></label>
|
||||
<input type="text" x-model="form.m365_mailbox"
|
||||
placeholder="Leave blank for the authorised account"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Folder</span></label>
|
||||
<input type="text" x-model="form.folder" placeholder="INBOX"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<template x-if="editingId && m365Connected">
|
||||
<div class="alert alert-success text-sm p-3">
|
||||
<span>Connected as <strong x-text="m365Email || 'Microsoft 365 account'"></strong></span>
|
||||
</div>
|
||||
</template>
|
||||
<template x-if="editingId && !m365Connected">
|
||||
<div class="alert alert-warning text-sm p-3">
|
||||
<span>Not yet authorised. Save this source first, then click <strong>Connect Microsoft 365</strong>.</span>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- Polling interval -->
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Polling Interval (minutes)</span></label>
|
||||
@@ -467,7 +547,7 @@
|
||||
<div class="flex justify-between pt-2">
|
||||
<!-- Test Connection (IMAP/POP3) or Connect Gmail button -->
|
||||
<div class="flex gap-2">
|
||||
<template x-if="form.method !== 'GMAIL_API'">
|
||||
<template x-if="form.method !== 'GMAIL_API' && form.method !== 'M365_GRAPH'">
|
||||
<button type="button" class="btn btn-outline btn-sm"
|
||||
x-on:click="testAdHoc()"
|
||||
:disabled="isTesting || isSaving">
|
||||
@@ -492,6 +572,13 @@
|
||||
Connect Gmail
|
||||
</button>
|
||||
</template>
|
||||
<template x-if="form.method === 'M365_GRAPH' && editingId">
|
||||
<button type="button" class="btn btn-outline btn-sm"
|
||||
x-on:click="connectM365()"
|
||||
:disabled="isTesting || isSaving">
|
||||
Connect Microsoft 365
|
||||
</button>
|
||||
</template>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<button type="button" class="btn btn-ghost btn-sm" x-on:click="closeForm()">Cancel</button>
|
||||
@@ -577,6 +664,8 @@ function mailSourcesApp() {
|
||||
testResult: { message: '', success: false, diagnostic_summary: '', recovery_steps: [] },
|
||||
gmailConnected: false,
|
||||
gmailEmail: '',
|
||||
m365Connected: false,
|
||||
m365Email: '',
|
||||
|
||||
form: {
|
||||
name: '',
|
||||
@@ -591,6 +680,10 @@ function mailSourcesApp() {
|
||||
enabled: true,
|
||||
gmail_client_id: '',
|
||||
gmail_client_secret: '',
|
||||
m365_tenant_id: 'common',
|
||||
m365_client_id: '',
|
||||
m365_client_secret: '',
|
||||
m365_mailbox: '',
|
||||
},
|
||||
|
||||
async init() {
|
||||
@@ -689,6 +782,26 @@ function mailSourcesApp() {
|
||||
return value ? new Date(value).toLocaleString() : '—';
|
||||
},
|
||||
|
||||
sourceAccountLabel(source) {
|
||||
if (source.method === 'GMAIL_API') {
|
||||
return source.gmail_email || '—';
|
||||
}
|
||||
if (source.method === 'M365_GRAPH') {
|
||||
return source.m365_mailbox || source.m365_email || '—';
|
||||
}
|
||||
return source.server ? `${source.server}:${source.port}` : '—';
|
||||
},
|
||||
|
||||
sourceStatusLabel(source) {
|
||||
if (source.method === 'GMAIL_API') {
|
||||
return source.gmail_connected ? 'Connected' : 'Not authorised';
|
||||
}
|
||||
if (source.method === 'M365_GRAPH') {
|
||||
return source.m365_connected ? 'Connected' : 'Not authorised';
|
||||
}
|
||||
return source.username || '—';
|
||||
},
|
||||
|
||||
formatList(value) {
|
||||
return value && value.length ? value.join(', ') : '—';
|
||||
},
|
||||
@@ -738,6 +851,8 @@ function mailSourcesApp() {
|
||||
this.editingId = null;
|
||||
this.gmailConnected = false;
|
||||
this.gmailEmail = '';
|
||||
this.m365Connected = false;
|
||||
this.m365Email = '';
|
||||
this.form = {
|
||||
name: '',
|
||||
method: 'IMAP',
|
||||
@@ -751,6 +866,10 @@ function mailSourcesApp() {
|
||||
enabled: true,
|
||||
gmail_client_id: '',
|
||||
gmail_client_secret: '',
|
||||
m365_tenant_id: 'common',
|
||||
m365_client_id: '',
|
||||
m365_client_secret: '',
|
||||
m365_mailbox: '',
|
||||
};
|
||||
this.testResult = this.emptyTestResult();
|
||||
this.showForm = true;
|
||||
@@ -760,6 +879,8 @@ function mailSourcesApp() {
|
||||
this.editingId = source.id;
|
||||
this.gmailConnected = source.gmail_connected || false;
|
||||
this.gmailEmail = source.gmail_email || '';
|
||||
this.m365Connected = source.m365_connected || false;
|
||||
this.m365Email = source.m365_email || '';
|
||||
this.form = {
|
||||
name: source.name,
|
||||
method: source.method,
|
||||
@@ -773,6 +894,10 @@ function mailSourcesApp() {
|
||||
enabled: source.enabled !== false,
|
||||
gmail_client_id: source.gmail_client_id || '',
|
||||
gmail_client_secret: '', // never pre-fill client secret
|
||||
m365_tenant_id: source.m365_tenant_id || 'common',
|
||||
m365_client_id: source.m365_client_id || '',
|
||||
m365_client_secret: '', // never pre-fill client secret
|
||||
m365_mailbox: source.m365_mailbox || '',
|
||||
};
|
||||
this.testResult = this.emptyTestResult();
|
||||
this.showForm = true;
|
||||
@@ -826,6 +951,45 @@ function mailSourcesApp() {
|
||||
}
|
||||
},
|
||||
|
||||
async connectM365() {
|
||||
if (!this.editingId) return;
|
||||
try {
|
||||
const resp = await fetch(
|
||||
`/api/v1/mail-sources/${this.editingId}/m365/authorize-url`
|
||||
);
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json();
|
||||
this.feedback = { message: `Error: ${err.detail || 'Failed to get authorization URL'}`, type: 'error' };
|
||||
return;
|
||||
}
|
||||
const data = await resp.json();
|
||||
const popup = window.open(
|
||||
data.authorization_url,
|
||||
'm365_oauth',
|
||||
'width=700,height=760,scrollbars=yes'
|
||||
);
|
||||
const pollInterval = setInterval(async () => {
|
||||
if (popup && popup.closed) {
|
||||
clearInterval(pollInterval);
|
||||
await this.loadSources();
|
||||
const updated = this.sources.find(s => s.id === this.editingId);
|
||||
if (updated) {
|
||||
this.m365Connected = updated.m365_connected || false;
|
||||
this.m365Email = updated.m365_email || '';
|
||||
if (this.m365Connected) {
|
||||
this.feedback = {
|
||||
message: `Microsoft 365 connected successfully (${this.m365Email || 'account connected'}).`,
|
||||
type: 'success',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
} catch (e) {
|
||||
this.feedback = { message: `Error: ${e.message}`, type: 'error' };
|
||||
}
|
||||
},
|
||||
|
||||
async saveSource() {
|
||||
this.isSaving = true;
|
||||
this.feedback = { message: '', type: '' };
|
||||
@@ -839,6 +1003,10 @@ function mailSourcesApp() {
|
||||
if (this.editingId && !payload.gmail_client_secret) {
|
||||
delete payload.gmail_client_secret;
|
||||
}
|
||||
// Don't send empty m365_client_secret on edit
|
||||
if (this.editingId && !payload.m365_client_secret) {
|
||||
delete payload.m365_client_secret;
|
||||
}
|
||||
|
||||
const url = this.editingId
|
||||
? `/api/v1/mail-sources/${this.editingId}`
|
||||
|
||||
@@ -99,6 +99,36 @@ class TestMailSourceModel:
|
||||
assert is_encrypted_secret(stored.gmail_access_token)
|
||||
assert is_encrypted_secret(stored.gmail_refresh_token)
|
||||
|
||||
def test_m365_oauth_secrets_are_encrypted_at_rest(self, db_session: Session):
|
||||
source = MailSource(
|
||||
name="Encrypted Microsoft 365",
|
||||
method="M365_GRAPH",
|
||||
m365_client_secret="client-secret",
|
||||
m365_access_token="access-token",
|
||||
m365_refresh_token="refresh-token",
|
||||
)
|
||||
db_session.add(source)
|
||||
db_session.commit()
|
||||
db_session.refresh(source)
|
||||
|
||||
stored = db_session.execute(
|
||||
text(
|
||||
"SELECT m365_client_secret, m365_access_token, m365_refresh_token "
|
||||
"FROM mail_sources WHERE id = :id"
|
||||
),
|
||||
{"id": source.id},
|
||||
).one()
|
||||
|
||||
assert source.m365_client_secret == "client-secret"
|
||||
assert source.m365_access_token == "access-token"
|
||||
assert source.m365_refresh_token == "refresh-token"
|
||||
assert stored.m365_client_secret != "client-secret"
|
||||
assert stored.m365_access_token != "access-token"
|
||||
assert stored.m365_refresh_token != "refresh-token"
|
||||
assert is_encrypted_secret(stored.m365_client_secret)
|
||||
assert is_encrypted_secret(stored.m365_access_token)
|
||||
assert is_encrypted_secret(stored.m365_refresh_token)
|
||||
|
||||
def test_legacy_plaintext_mail_source_secret_remains_readable(self, db_session: Session):
|
||||
db_session.execute(
|
||||
text(
|
||||
@@ -968,6 +998,455 @@ class TestGmailAPIMailSource:
|
||||
assert data["new_domains"] == ["example.com"]
|
||||
|
||||
|
||||
class TestMicrosoft365GraphMailSource:
|
||||
"""Tests for M365_GRAPH mail source creation, OAuth flow, and fetching."""
|
||||
|
||||
def test_create_m365_graph_source(self, authed_client: TestClient):
|
||||
payload = {
|
||||
"name": "My Microsoft 365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_tenant_id": "organizations",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
"m365_mailbox": "dmarc@example.com",
|
||||
"folder": "INBOX",
|
||||
"polling_interval": 30,
|
||||
"enabled": True,
|
||||
}
|
||||
resp = authed_client.post("/api/v1/mail-sources", json=payload)
|
||||
assert resp.status_code == 201
|
||||
data = resp.json()
|
||||
assert data["method"] == "M365_GRAPH"
|
||||
assert data["m365_tenant_id"] == "organizations"
|
||||
assert data["m365_client_id"] == "client-id"
|
||||
assert data["m365_client_secret"] == "**redacted**"
|
||||
assert data["m365_mailbox"] == "dmarc@example.com"
|
||||
assert data["m365_connected"] is False
|
||||
assert data["m365_email"] is None
|
||||
|
||||
def test_m365_source_test_no_token(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Unauthed M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is False
|
||||
assert data["diagnostic_category"] == "auth_required"
|
||||
assert any("Microsoft 365" in step for step in data["recovery_steps"])
|
||||
|
||||
def test_m365_source_test_with_valid_token(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Authed M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.test_connection.return_value = {
|
||||
"success": True,
|
||||
"message_count": 1,
|
||||
"diagnostic_detail": "ok",
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = None
|
||||
|
||||
with (
|
||||
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
):
|
||||
mock_source = MagicMock()
|
||||
mock_source.method = "M365_GRAPH"
|
||||
mock_source.m365_access_token = "valid-token"
|
||||
mock_source.m365_email = "dmarc@example.com"
|
||||
mock_source.m365_tenant_id = "organizations"
|
||||
mock_source.m365_client_id = "client-id"
|
||||
mock_source.m365_client_secret = "client-secret"
|
||||
mock_source.m365_refresh_token = "refresh-token"
|
||||
mock_source.m365_mailbox = None
|
||||
mock_source.folder = "INBOX"
|
||||
mock_get.return_value = mock_source
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert "valid" in data["message"].lower()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_error", "category"),
|
||||
[
|
||||
("invalid_grant: refresh token expired", "auth_expired"),
|
||||
("Forbidden: Mail.Read permission is missing", "permissions"),
|
||||
("429 Too Many Requests throttling limit", "throttling"),
|
||||
],
|
||||
)
|
||||
def test_m365_source_test_diagnostic_categories(
|
||||
self, authed_client: TestClient, provider_error: str, category: str
|
||||
):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "M365 Diagnostics", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
mock_client = MagicMock()
|
||||
mock_client.test_connection.side_effect = RuntimeError(provider_error)
|
||||
|
||||
with (
|
||||
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
):
|
||||
mock_source = MagicMock()
|
||||
mock_source.method = "M365_GRAPH"
|
||||
mock_source.m365_access_token = "valid-token"
|
||||
mock_source.m365_email = "dmarc@example.com"
|
||||
mock_source.m365_tenant_id = "organizations"
|
||||
mock_source.m365_client_id = "client-id"
|
||||
mock_source.m365_client_secret = "client-secret"
|
||||
mock_source.m365_refresh_token = "refresh-token"
|
||||
mock_source.m365_mailbox = None
|
||||
mock_source.folder = "INBOX"
|
||||
mock_get.return_value = mock_source
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["diagnostic_category"] == category
|
||||
|
||||
def test_m365_authorize_url_returns_microsoft_url(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Ready M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_tenant_id": "organizations",
|
||||
"m365_client_id": "client-id",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/authorize-url")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
parsed = urlparse(data["authorization_url"])
|
||||
assert parsed.hostname == "login.microsoftonline.com"
|
||||
assert "/organizations/oauth2/v2.0/authorize" in parsed.path
|
||||
query = parse_qs(parsed.query)
|
||||
assert query["client_id"] == ["client-id"]
|
||||
assert "offline_access" in query["scope"][0]
|
||||
assert "https://graph.microsoft.com/Mail.Read" in query["scope"][0]
|
||||
|
||||
def test_m365_authorize_url_validates_source(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Source", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
assert (
|
||||
authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/authorize-url").status_code
|
||||
== 400
|
||||
)
|
||||
|
||||
m365_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "No Client M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
m365_id = m365_resp.json()["id"]
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{m365_id}/m365/authorize-url")
|
||||
assert resp.status_code == 400
|
||||
assert "m365_client_id" in resp.json()["detail"]
|
||||
|
||||
def test_m365_get_callback_handles_error_and_wrong_source(self, authed_client: TestClient):
|
||||
error_resp = authed_client.get("/api/v1/mail-sources/999/m365/callback?error=denied")
|
||||
assert error_resp.status_code == 400
|
||||
assert "authorisation failed" in error_resp.text
|
||||
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Callback", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
wrong_resp = authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/callback?code=abc")
|
||||
assert wrong_resp.status_code == 404
|
||||
|
||||
def test_m365_get_callback_saves_tokens(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "GET Callback M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"access_token": "access", "refresh_token": "refresh"},
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
|
||||
return_value="dmarc@example.com",
|
||||
),
|
||||
):
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert "connected successfully" in resp.text
|
||||
|
||||
def test_m365_get_callback_failure_modes(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "GET Callback Failure", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
side_effect=ValueError("bad code"),
|
||||
):
|
||||
token_error = authed_client.get(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
|
||||
)
|
||||
assert token_error.status_code == 400
|
||||
assert "Token exchange failed" in token_error.text
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"refresh_token": "refresh"},
|
||||
):
|
||||
missing_access = authed_client.get(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
|
||||
)
|
||||
assert missing_access.status_code == 400
|
||||
assert "No access token" in missing_access.text
|
||||
|
||||
def test_m365_callback_post_saves_tokens(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Callback M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"access_token": "access", "refresh_token": "refresh"},
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
|
||||
return_value="dmarc@example.com",
|
||||
),
|
||||
):
|
||||
resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["m365_connected"] is True
|
||||
assert data["m365_email"] == "dmarc@example.com"
|
||||
|
||||
def test_m365_callback_post_failure_modes(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Callback Post", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
wrong_resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{imap_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert wrong_resp.status_code == 400
|
||||
|
||||
m365_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Bad Callback M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
m365_id = m365_resp.json()["id"]
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
side_effect=ValueError("bad code"),
|
||||
):
|
||||
token_resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{m365_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert token_resp.status_code == 400
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"refresh_token": "refresh"},
|
||||
):
|
||||
missing_access = authed_client.post(
|
||||
f"/api/v1/mail-sources/{m365_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert missing_access.status_code == 400
|
||||
|
||||
def test_m365_fetch_no_token_returns_400(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "No Token M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_m365_fetch_with_mocked_client(self, authed_client: TestClient, db_session: Session):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Fetch M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "tok"
|
||||
source.m365_refresh_token = "refresh"
|
||||
db_session.commit()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 3,
|
||||
"reports_found": 2,
|
||||
"duplicate_reports": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"errors": [],
|
||||
"new_ingested_ids": ["id1", "id2", "id3"],
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = None
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.load_ingested_ids",
|
||||
return_value=[],
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.dump_ingested_ids",
|
||||
return_value='["id1", "id2", "id3"]',
|
||||
),
|
||||
):
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/fetch")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["processed"] == 3
|
||||
assert data["reports_found"] == 2
|
||||
assert data["duplicate_reports"] == 1
|
||||
assert data["new_domains"] == ["example.com"]
|
||||
|
||||
def test_m365_specific_fetch_persists_refreshed_tokens(
|
||||
self, authed_client: TestClient, db_session: Session
|
||||
):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Fetch M365 Specific",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "old-access"
|
||||
source.m365_refresh_token = "old-refresh"
|
||||
db_session.commit()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"reports_found": 0,
|
||||
"duplicate_reports": 0,
|
||||
"new_domains": [],
|
||||
"errors": ["temporary warning"],
|
||||
"new_ingested_ids": [],
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-access",
|
||||
"refresh_token": "new-refresh",
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
):
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
|
||||
db_session.refresh(source)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["processed"] == 1
|
||||
assert source.m365_access_token == "new-access"
|
||||
assert source.m365_refresh_token == "new-refresh"
|
||||
|
||||
def test_m365_fetch_and_disconnect_validate_method(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP M365 Actions", "method": "IMAP"},
|
||||
)
|
||||
source_id = imap_resp.json()["id"]
|
||||
|
||||
fetch_resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
disconnect_resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
|
||||
|
||||
assert fetch_resp.status_code == 400
|
||||
assert disconnect_resp.status_code == 400
|
||||
|
||||
def test_m365_disconnect_clears_tokens(self, authed_client: TestClient, db_session: Session):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Disconnect M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "tok"
|
||||
source.m365_refresh_token = "refresh"
|
||||
source.m365_email = "dmarc@example.com"
|
||||
db_session.commit()
|
||||
|
||||
resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
|
||||
|
||||
db_session.refresh(source)
|
||||
assert resp.status_code == 204
|
||||
assert source.m365_access_token is None
|
||||
assert source.m365_refresh_token is None
|
||||
assert source.m365_email is None
|
||||
|
||||
|
||||
class TestManualSourceFetchEndpoint:
|
||||
"""Tests for POST /api/v1/mail-sources/{source_id}/fetch."""
|
||||
|
||||
@@ -2035,6 +2514,79 @@ class TestTriggerPollGmailSource:
|
||||
assert src.gmail_refresh_token == "new-ref"
|
||||
|
||||
|
||||
class TestTriggerPollM365Source:
|
||||
"""Unit tests for app.main._trigger_poll_m365_source."""
|
||||
|
||||
def _make_src(self):
|
||||
src = MagicMock()
|
||||
src.id = 8
|
||||
src.name = "My M365"
|
||||
src.m365_tenant_id = "organizations"
|
||||
src.m365_client_id = "cid"
|
||||
src.m365_client_secret = "csec"
|
||||
src.m365_access_token = "tok"
|
||||
src.m365_refresh_token = "ref"
|
||||
src.m365_mailbox = "shared@example.com"
|
||||
src.m365_ingested_ids = "[]"
|
||||
src.folder = "INBOX"
|
||||
return src
|
||||
|
||||
def test_returns_result_dict_on_success(self):
|
||||
from app.main import _trigger_poll_m365_source
|
||||
|
||||
src = self._make_src()
|
||||
mock_gc = MagicMock()
|
||||
mock_gc.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 2,
|
||||
"reports_found": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"new_ingested_ids": ["id1"],
|
||||
}
|
||||
mock_gc.get_refreshed_tokens.return_value = None
|
||||
mock_db = MagicMock()
|
||||
|
||||
with (
|
||||
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
patch("app.main.MicrosoftGraphClient.dump_ingested_ids", return_value='["id1"]'),
|
||||
):
|
||||
result = _trigger_poll_m365_source(src, mock_db)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source_id"] == 8
|
||||
assert result["processed"] == 2
|
||||
assert src.m365_ingested_ids == '["id1"]'
|
||||
mock_db.commit.assert_called_once()
|
||||
|
||||
def test_persists_refreshed_tokens(self):
|
||||
from app.main import _trigger_poll_m365_source
|
||||
|
||||
src = self._make_src()
|
||||
mock_gc = MagicMock()
|
||||
mock_gc.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"reports_found": 0,
|
||||
"new_domains": [],
|
||||
"new_ingested_ids": [],
|
||||
}
|
||||
mock_gc.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-acc",
|
||||
"refresh_token": "new-ref",
|
||||
}
|
||||
mock_db = MagicMock()
|
||||
|
||||
with (
|
||||
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
):
|
||||
_trigger_poll_m365_source(src, mock_db)
|
||||
|
||||
assert src.m365_access_token == "new-acc"
|
||||
assert src.m365_refresh_token == "new-ref"
|
||||
|
||||
|
||||
class TestPollSourceForTrigger:
|
||||
"""Unit tests for app.main._poll_source_for_trigger."""
|
||||
|
||||
@@ -2083,6 +2635,51 @@ class TestPollSourceForTrigger:
|
||||
assert result["success"] is False
|
||||
assert "boom" not in result.get("error", "") # raw msg not exposed
|
||||
|
||||
def test_m365_no_token_returns_skipped(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = None
|
||||
src.id = 7
|
||||
src.name = "M365 no token"
|
||||
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result["skipped"] is True
|
||||
assert "microsoft 365" in result["reason"].lower()
|
||||
|
||||
def test_m365_with_token_delegates_to_trigger_poll(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = "tok"
|
||||
src.id = 8
|
||||
src.name = "M365"
|
||||
|
||||
expected = {"source_id": 8, "name": "M365", "success": True}
|
||||
with patch("app.main._trigger_poll_m365_source", return_value=expected) as mock_fn:
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result is expected
|
||||
mock_fn.assert_called_once()
|
||||
|
||||
def test_m365_exception_returns_failure_dict(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = "tok"
|
||||
src.id = 9
|
||||
src.name = "M365 exc"
|
||||
|
||||
with patch("app.main._trigger_poll_m365_source", side_effect=Exception("boom")):
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result["success"] is False
|
||||
assert "boom" not in result.get("error", "")
|
||||
|
||||
def test_imap_delegates_to_trigger_poll(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
@@ -2147,6 +2744,25 @@ class TestPollAllEnabledSources:
|
||||
|
||||
mock_gmail.assert_called_once_with(src)
|
||||
|
||||
def test_dispatches_m365_graph_source(self):
|
||||
"""M365_GRAPH sources are forwarded to _poll_single_m365_source."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
src = MagicMock()
|
||||
src.id = 6
|
||||
src.method = "M365_GRAPH"
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.return_value.filter.return_value.all.return_value = [src]
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=mock_db),
|
||||
patch("app.main._poll_single_m365_source") as mock_m365,
|
||||
):
|
||||
_poll_all_enabled_sources()
|
||||
|
||||
mock_m365.assert_called_once_with(src)
|
||||
|
||||
def test_dispatches_imap_source(self):
|
||||
"""IMAP sources are forwarded to _poll_single_imap_source."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
@@ -2200,6 +2816,23 @@ class TestPollAllEnabledSources:
|
||||
):
|
||||
_poll_all_enabled_sources() # should not raise
|
||||
|
||||
def test_m365_exception_is_caught(self):
|
||||
"""Exception from _poll_single_m365_source must not propagate."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
src = MagicMock()
|
||||
src.id = 7
|
||||
src.method = "M365_GRAPH"
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.return_value.filter.return_value.all.return_value = [src]
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=mock_db),
|
||||
patch("app.main._poll_single_m365_source", side_effect=Exception("m365 crash")),
|
||||
):
|
||||
_poll_all_enabled_sources() # should not raise
|
||||
|
||||
def test_unknown_method_skipped(self):
|
||||
"""An unknown method logs a skip message and does not raise."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
@@ -86,6 +86,68 @@ def test_poll_single_imap_source_passes_configured_folder():
|
||||
db.close.assert_called_once()
|
||||
|
||||
|
||||
def test_poll_single_m365_source_persists_import_state():
|
||||
from app.main import _poll_single_m365_source
|
||||
|
||||
source = SimpleNamespace(
|
||||
id=2,
|
||||
m365_access_token="tok",
|
||||
m365_tenant_id="organizations",
|
||||
m365_client_id="cid",
|
||||
m365_client_secret="csec",
|
||||
m365_refresh_token="ref",
|
||||
m365_mailbox="shared@example.com",
|
||||
m365_ingested_ids="[]",
|
||||
folder="INBOX",
|
||||
)
|
||||
db_source = SimpleNamespace(**source.__dict__)
|
||||
db = MagicMock()
|
||||
db.query.return_value.get.return_value = db_source
|
||||
results = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"reports_found": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"new_ingested_ids": ["message-1"],
|
||||
}
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=db),
|
||||
patch("app.main.MicrosoftGraphClient") as mock_client_cls,
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
patch(
|
||||
"app.main.MicrosoftGraphClient.dump_ingested_ids",
|
||||
return_value='["message-1"]',
|
||||
),
|
||||
patch("app.main.record_import_attempt"),
|
||||
):
|
||||
mock_client = mock_client_cls.return_value
|
||||
mock_client.fetch_reports.return_value = results
|
||||
mock_client.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-tok",
|
||||
"refresh_token": "new-ref",
|
||||
}
|
||||
|
||||
_poll_single_m365_source(source)
|
||||
|
||||
assert db_source.m365_ingested_ids == '["message-1"]'
|
||||
assert db_source.m365_access_token == "new-tok"
|
||||
assert db_source.m365_refresh_token == "new-ref"
|
||||
db.commit.assert_called_once()
|
||||
db.close.assert_called_once()
|
||||
|
||||
|
||||
def test_poll_single_m365_source_skips_without_token():
|
||||
from app.main import _poll_single_m365_source
|
||||
|
||||
source = SimpleNamespace(id=3, m365_access_token=None)
|
||||
|
||||
with patch("app.main.SessionLocal") as mock_session:
|
||||
_poll_single_m365_source(source)
|
||||
|
||||
mock_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scheduled_imap_polling_sleep_exception_falls_back_then_cancels():
|
||||
from app.main import scheduled_imap_polling
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Unit tests for app.services.microsoft_graph_client.MicrosoftGraphClient."""
|
||||
|
||||
import base64
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
import httpx
|
||||
|
||||
from app.models.report import DMARCReport
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
from app.tests.test_data import SAMPLE_XML
|
||||
|
||||
|
||||
def _zip_xml(xml: str = SAMPLE_XML, name: str = "report.xml") -> bytes:
|
||||
buf = BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr(name, xml.encode("utf-8"))
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_client(db=None, already_ingested=None) -> MicrosoftGraphClient:
|
||||
return MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
mailbox=None,
|
||||
folder="INBOX",
|
||||
already_ingested_ids=already_ingested or [],
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
class TestMicrosoftGraphOAuthHelpers:
|
||||
def test_build_authorization_url_includes_required_scopes(self):
|
||||
url = MicrosoftGraphClient.build_authorization_url(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
redirect_uri="https://example.com/callback",
|
||||
state="42",
|
||||
)
|
||||
|
||||
assert url.startswith("https://login.microsoftonline.com/organizations/")
|
||||
assert "client_id=client-id" in url
|
||||
assert "response_type=code" in url
|
||||
assert "offline_access" in url
|
||||
assert "User.Read" in url
|
||||
assert "Mail.Read" in url
|
||||
assert "state=42" in url
|
||||
|
||||
def test_exchange_code_for_tokens_posts_to_tenant_endpoint(self, monkeypatch):
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
assert url.endswith("/contoso-tenant/oauth2/v2.0/token")
|
||||
assert data["grant_type"] == "authorization_code"
|
||||
assert data["code"] == "code"
|
||||
return httpx.Response(200, json={"access_token": "access"})
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
|
||||
result = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id="contoso-tenant",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
code="code",
|
||||
redirect_uri="https://example.com/callback",
|
||||
)
|
||||
|
||||
assert result == {"access_token": "access"}
|
||||
|
||||
def test_exchange_code_for_tokens_raises_on_failure(self, monkeypatch):
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
return httpx.Response(400, text="bad request")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
|
||||
try:
|
||||
MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
code="bad",
|
||||
redirect_uri="https://example.com/callback",
|
||||
)
|
||||
except Exception as exc:
|
||||
assert "token exchange failed" in str(exc).lower()
|
||||
else:
|
||||
raise AssertionError("expected token exchange failure")
|
||||
|
||||
def test_get_account_email_prefers_mail_then_user_principal_name(self, monkeypatch):
|
||||
def fake_get(url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/me")
|
||||
assert params == {"$select": "mail,userPrincipalName"}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"mail": "", "userPrincipalName": "dmarc@example.com"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
|
||||
|
||||
assert MicrosoftGraphClient.get_account_email("access") == "dmarc@example.com"
|
||||
|
||||
def test_get_account_email_returns_none_on_error(self, monkeypatch):
|
||||
def fake_get(url, headers=None, params=None, timeout=None):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
|
||||
|
||||
assert MicrosoftGraphClient.get_account_email("access") is None
|
||||
|
||||
def test_ingested_id_helpers_tolerate_bad_json(self):
|
||||
assert MicrosoftGraphClient.load_ingested_ids(None) == []
|
||||
assert MicrosoftGraphClient.load_ingested_ids('["a", 2]') == ["a", "2"]
|
||||
assert MicrosoftGraphClient.load_ingested_ids("{bad") == []
|
||||
assert MicrosoftGraphClient.dump_ingested_ids(["a"]) == '["a"]'
|
||||
|
||||
|
||||
class TestMicrosoftGraphFetchReports:
|
||||
def test_test_connection_reads_shared_mailbox(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/users/shared%40example.com/messages")
|
||||
assert params == {"$top": 1, "$select": "id"}
|
||||
return httpx.Response(200, json={"value": [{"id": "message-1"}]})
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
mailbox="shared@example.com",
|
||||
folder="INBOX",
|
||||
)
|
||||
|
||||
result = client.test_connection()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["message_count"] == 1
|
||||
|
||||
def test_fetch_reports_imports_zip_attachment(self, monkeypatch, db_session):
|
||||
attachment_bytes = _zip_xml()
|
||||
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
"receivedDateTime": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "report.zip",
|
||||
"contentBytes": base64.b64encode(attachment_bytes).decode(),
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client(db=db_session)
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["reports_found"] == 1
|
||||
assert result["new_ingested_ids"] == ["message-1"]
|
||||
assert db_session.query(DMARCReport).count() == 1
|
||||
|
||||
def test_fetch_reports_skips_already_ingested_message(self, monkeypatch, db_session):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError("Already-ingested messages should not fetch attachments")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client(db=db_session, already_ingested=["message-1"])
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 0
|
||||
assert result["reports_found"] == 0
|
||||
assert result["new_ingested_ids"] == []
|
||||
|
||||
def test_fetch_reports_handles_empty_message_id_and_plain_messages_path(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/me/messages")
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
folder="",
|
||||
)
|
||||
client.folder = ""
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 0
|
||||
|
||||
def test_fetch_reports_keeps_message_retryable_when_attachments_fail(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
500,
|
||||
json={"error": {"code": "ServerError", "message": "temporary failure"}},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["new_ingested_ids"] == []
|
||||
assert result["details"][0]["reason"] == "attachment_fetch_failed"
|
||||
|
||||
def test_fetch_reports_records_attachment_outcomes(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "notes.txt",
|
||||
"contentBytes": base64.b64encode(b"not dmarc").decode(),
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.itemAttachment",
|
||||
"name": "report.xml",
|
||||
"contentBytes": base64.b64encode(SAMPLE_XML.encode()).decode(),
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "empty.xml",
|
||||
"contentBytes": "",
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "bad.xml",
|
||||
"contentBytes": base64.b64encode(b"<not-dmarc>").decode(),
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["reports_found"] == 0
|
||||
assert result["errors"]
|
||||
reasons = {detail.get("reason") for detail in result["details"]}
|
||||
assert {
|
||||
"unsupported_attachment",
|
||||
"unsupported_attachment_type",
|
||||
"empty_attachment",
|
||||
"parse_failed",
|
||||
}.issubset(reasons)
|
||||
|
||||
def test_fetch_reports_refreshes_token_after_unauthorized(self, monkeypatch):
|
||||
calls = {"messages": 0}
|
||||
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
assert url.endswith("/organizations/oauth2/v2.0/token")
|
||||
assert data["grant_type"] == "refresh_token"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"access_token": "new-access", "refresh_token": "new-refresh"},
|
||||
)
|
||||
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
calls["messages"] += 1
|
||||
if calls["messages"] == 1:
|
||||
return httpx.Response(
|
||||
401,
|
||||
json={
|
||||
"error": {
|
||||
"code": "InvalidAuthenticationToken",
|
||||
"message": "Access token has expired.",
|
||||
}
|
||||
},
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer new-access"
|
||||
return httpx.Response(200, json={"value": []})
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert calls["messages"] == 2
|
||||
assert client.get_refreshed_tokens() == {
|
||||
"access_token": "new-access",
|
||||
"refresh_token": "new-refresh",
|
||||
}
|
||||
|
||||
def test_fetch_reports_surfaces_throttling_error(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
return httpx.Response(
|
||||
429,
|
||||
json={"error": {"code": "TooManyRequests", "message": "Request is throttled."}},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is False
|
||||
assert "TooManyRequests" in result["error"]
|
||||
+3
-1
@@ -13,7 +13,7 @@ DMARQ is a full-stack DMARC monitoring platform designed to help organizations t
|
||||
- **DMARC Report Processing**: Automatically collect and parse DMARC aggregate and forensic reports
|
||||
- **Interactive Dashboard**: Visualize compliance rates and authentication trends
|
||||
- **DNS Health Checks**: Verify your email authentication records (SPF, DKIM, DMARC)
|
||||
- **IMAP Integration**: Automatically fetch reports from your email inbox
|
||||
- **Mailbox Integrations**: Automatically fetch reports from IMAP, Gmail, and Microsoft 365 inboxes
|
||||
- **Alerting**: Get notified about important authentication issues
|
||||
- **Easy Setup**: Web-based configuration wizard for quick onboarding
|
||||
|
||||
@@ -23,4 +23,6 @@ To get started with DMARQ, please see the [Getting Started](user_guide/getting_s
|
||||
|
||||
For installation instructions, check the [Docker Setup](deployment/docker.md) or [Manual Installation](deployment/manual.md) guides. Operators should use the [Operator Runbook](deployment/operations.md) for deployment modes, verification, upgrades, and rollback, and the [Troubleshooting Playbooks](deployment/troubleshooting.md) for ingestion, authentication, DNS, database, and notification failures. For production secrets, use [Secret Handling with 1Password](deployment/secrets.md). For database operations, use [Database Backup and Restore](deployment/backups.md). For upgrades, use the [Release Checklist](deployment/release-checklist.md).
|
||||
|
||||
For Microsoft 365 setup, see [Microsoft 365 Mail Sources](user_guide/microsoft365.md).
|
||||
|
||||
For aggregate-report parser support, known edge cases, and fixture guidance, see [DMARC Aggregate Format Compatibility](reference/dmarc-compatibility.md).
|
||||
|
||||
+3
-3
@@ -198,14 +198,14 @@ Exit criteria:
|
||||
|
||||
## Milestone 12: Enterprise Mail Sources (Microsoft 365) and Connector Framework
|
||||
|
||||
Status: Planned
|
||||
Status: In progress
|
||||
|
||||
Goal: make mailbox ingestion work for the most common enterprise setups without relying on IMAP.
|
||||
|
||||
Planned:
|
||||
- Microsoft 365 mail source using OAuth (Graph) with least-privilege scopes.
|
||||
- Microsoft 365 mail source using OAuth (Graph) with least-privilege scopes. Delivered for delegated `User.Read`, `Mail.Read`, and `offline_access` with encrypted token storage, manual import, scheduled polling, UI setup, and operator docs.
|
||||
- Shared mailbox and folder selection support for DMARC report collection.
|
||||
- Import-history parity with existing sources (auditable attachment outcomes, duplicates, parse failures).
|
||||
- Import-history parity with existing sources (auditable attachment outcomes, duplicates, parse failures). Delivered for Microsoft 365 imports.
|
||||
- Backfill support with safe throttling and progressive search windows.
|
||||
- Secret handling mirrors existing guidance (no raw secrets in logs; 1Password-friendly).
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Microsoft 365 Mail Sources
|
||||
|
||||
DMARQ can import DMARC aggregate report attachments from Exchange Online through Microsoft Graph. Use this source type when IMAP is disabled or unavailable in a Microsoft 365 tenant.
|
||||
|
||||
## App Registration
|
||||
|
||||
Create an app registration in Microsoft Entra admin center:
|
||||
|
||||
- Platform: Web
|
||||
- Redirect URI: `https://<your-dmarq-host>/api/v1/mail-sources/<source-id>/m365/callback`
|
||||
- Delegated API permissions:
|
||||
- `User.Read`
|
||||
- `Mail.Read`
|
||||
- `offline_access`
|
||||
- Client secret: create a secret for the web app and store it in your deployment secret manager.
|
||||
|
||||
`Mail.Read` is the least-privilege delegated Graph permission DMARQ needs to list messages and read file attachments. `offline_access` is requested so Microsoft returns a refresh token for scheduled polling.
|
||||
|
||||
## DMARQ Setup
|
||||
|
||||
1. Open **Mail Sources**.
|
||||
2. Add a source with method **Microsoft 365 (Graph OAuth2)**.
|
||||
3. Enter the tenant ID (`organizations`, `common`, or a tenant GUID), client ID, and client secret.
|
||||
4. Leave **Mailbox** empty to read the authorised account, or enter a user principal name for a shared/delegated mailbox that the authorised user can read.
|
||||
5. Save the source.
|
||||
6. Use **Connect Microsoft 365** and approve the read-only mailbox access request.
|
||||
7. Run **Test connection** and **Run import now**.
|
||||
|
||||
## Import Behavior
|
||||
|
||||
DMARQ reads recent messages in the configured folder, filters for messages that look like DMARC reports, downloads Graph `fileAttachment` items, and sends `.xml`, `.zip`, `.gz`, and `.gzip` attachments through the same parser and persistence path used by upload, IMAP, and Gmail imports.
|
||||
|
||||
Imported Graph message IDs are stored on the mail source so scheduled polling does not reprocess the same message. Import history records processed messages, imported reports, duplicates, parse failures, and attachment-level details.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Not authorised**: reconnect the source from Mail Sources.
|
||||
- **Permission error**: confirm the app registration has delegated `Mail.Read` and the authorised account can read the target mailbox.
|
||||
- **Throttling**: wait and retry, or increase the polling interval.
|
||||
- **Mailbox/folder not found**: leave Mailbox blank for `/me`, use a valid user principal name for delegated/shared mailboxes, and keep the default `INBOX` folder unless reports are delivered elsewhere.
|
||||
Reference in New Issue
Block a user