feat: add Microsoft 365 Graph mail source
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
"""Add Microsoft 365 Graph mail source fields.
|
||||
|
||||
Revision ID: f3a4b5c6d7e8
|
||||
Revises: e2f3a4b5c6d7
|
||||
Create Date: 2026-05-23 00:00:00.000000
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "f3a4b5c6d7e8"
|
||||
down_revision: Union[str, Sequence[str], None] = "e2f3a4b5c6d7"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("mail_sources", sa.Column("m365_tenant_id", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_client_id", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_client_secret", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_access_token", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_refresh_token", sa.Text(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_mailbox", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_email", sa.String(), nullable=True))
|
||||
op.add_column("mail_sources", sa.Column("m365_ingested_ids", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("mail_sources", "m365_ingested_ids")
|
||||
op.drop_column("mail_sources", "m365_email")
|
||||
op.drop_column("mail_sources", "m365_mailbox")
|
||||
op.drop_column("mail_sources", "m365_refresh_token")
|
||||
op.drop_column("mail_sources", "m365_access_token")
|
||||
op.drop_column("mail_sources", "m365_client_secret")
|
||||
op.drop_column("mail_sources", "m365_client_id")
|
||||
op.drop_column("mail_sources", "m365_tenant_id")
|
||||
@@ -3,8 +3,8 @@ Mail Sources API endpoints.
|
||||
|
||||
Provides CRUD operations for MailSource objects stored in the database, plus
|
||||
a *test-connection* action that validates the supplied credentials without
|
||||
persisting anything. Gmail API sources additionally have OAuth2 helper
|
||||
endpoints (authorize-url, callback, fetch).
|
||||
persisting anything. Gmail API and Microsoft 365 sources additionally have
|
||||
OAuth2 helper endpoints (authorize-url, callback, fetch).
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -24,6 +24,7 @@ from app.models.mail_source_import import MailSourceImport
|
||||
from app.services.gmail_client import GmailClient
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.import_history import record_import_attempt
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,7 +39,7 @@ class MailSourceBase(BaseModel):
|
||||
"""Fields shared by create and update payloads."""
|
||||
|
||||
name: str
|
||||
method: str = "IMAP" # IMAP | POP3 | GMAIL_API
|
||||
method: str = "IMAP" # IMAP | POP3 | GMAIL_API | M365_GRAPH
|
||||
server: Optional[str] = None
|
||||
port: int = 993
|
||||
username: Optional[str] = None
|
||||
@@ -50,6 +51,11 @@ class MailSourceBase(BaseModel):
|
||||
# Gmail API OAuth2 fields (only relevant when method == GMAIL_API)
|
||||
gmail_client_id: Optional[str] = None
|
||||
gmail_client_secret: Optional[str] = None
|
||||
# Microsoft 365 Graph OAuth2 fields (only relevant when method == M365_GRAPH)
|
||||
m365_tenant_id: Optional[str] = "common"
|
||||
m365_client_id: Optional[str] = None
|
||||
m365_client_secret: Optional[str] = None
|
||||
m365_mailbox: Optional[str] = None
|
||||
|
||||
|
||||
class MailSourceCreate(MailSourceBase):
|
||||
@@ -71,6 +77,10 @@ class MailSourceUpdate(BaseModel):
|
||||
enabled: Optional[bool] = None
|
||||
gmail_client_id: Optional[str] = None
|
||||
gmail_client_secret: Optional[str] = None
|
||||
m365_tenant_id: Optional[str] = None
|
||||
m365_client_id: Optional[str] = None
|
||||
m365_client_secret: Optional[str] = None
|
||||
m365_mailbox: Optional[str] = None
|
||||
|
||||
|
||||
class MailSourceResponse(MailSourceBase):
|
||||
@@ -86,6 +96,9 @@ class MailSourceResponse(MailSourceBase):
|
||||
gmail_email: Optional[str] = None
|
||||
# Indicate whether OAuth tokens are present (without exposing them)
|
||||
gmail_connected: bool = False
|
||||
# Microsoft 365: show the authorised account and token state, but not tokens
|
||||
m365_email: Optional[str] = None
|
||||
m365_connected: bool = False
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -109,6 +122,13 @@ class GmailCallbackRequest(BaseModel):
|
||||
redirect_uri: str
|
||||
|
||||
|
||||
class M365CallbackRequest(BaseModel):
|
||||
"""Payload for the Microsoft 365 OAuth2 callback endpoint."""
|
||||
|
||||
code: str
|
||||
redirect_uri: str
|
||||
|
||||
|
||||
class MailSourceImportResponse(BaseModel):
|
||||
"""Sanitized import-history entry for a mail source."""
|
||||
|
||||
@@ -151,7 +171,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"auth_required": {
|
||||
"summary": "The mailbox has not been connected yet.",
|
||||
"recovery_steps": [
|
||||
"Use the Connect Gmail action to complete authorization.",
|
||||
"Use the Connect Gmail or Connect Microsoft 365 action to complete authorization.",
|
||||
"Confirm the authorized mailbox is the one that receives DMARC aggregate reports.",
|
||||
],
|
||||
},
|
||||
@@ -159,7 +179,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"summary": "The saved authorization is expired, revoked, or no longer accepted.",
|
||||
"recovery_steps": [
|
||||
"Reconnect the mailbox from Mail Sources.",
|
||||
"If your provider shows a consent screen, approve Gmail read-only access again.",
|
||||
"If your provider shows a consent screen, approve read-only mailbox access again.",
|
||||
],
|
||||
},
|
||||
"authentication": {
|
||||
@@ -173,7 +193,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"summary": "The account is connected but does not have enough mailbox access.",
|
||||
"recovery_steps": [
|
||||
"Grant read access for the mailbox that receives DMARC reports.",
|
||||
"For Gmail, reconnect and approve the requested Gmail read-only scope.",
|
||||
"For OAuth sources, reconnect and approve the requested read-only mail scope.",
|
||||
],
|
||||
},
|
||||
"connectivity": {
|
||||
@@ -200,7 +220,7 @@ DIAGNOSTIC_COPY: Dict[str, Dict[str, Any]] = {
|
||||
"missing_config": {
|
||||
"summary": "Required connection settings are missing.",
|
||||
"recovery_steps": [
|
||||
"Fill in the server, username, and password or complete Gmail authorization.",
|
||||
"Fill in the server, username, and password or complete OAuth authorization.",
|
||||
"Save the source before running stored-source tests.",
|
||||
],
|
||||
},
|
||||
@@ -227,12 +247,23 @@ def _diagnostic_category(message: str, details: Optional[object] = None) -> str:
|
||||
if any(term in text for term in ("not yet authorised", "not yet authorized", "complete oauth")):
|
||||
return "auth_required"
|
||||
if any(
|
||||
term in text for term in ("expired", "revoked", "invalid_grant", "refresh token", "oauth")
|
||||
term in text
|
||||
for term in (
|
||||
"expired",
|
||||
"revoked",
|
||||
"invalid_grant",
|
||||
"interaction_required",
|
||||
"refresh token",
|
||||
"oauth",
|
||||
)
|
||||
):
|
||||
return "auth_expired"
|
||||
if any(term in text for term in ("rate", "quota", "throttl", "too many", "429")):
|
||||
return "throttling"
|
||||
if any(term in text for term in ("scope", "permission", "access denied", "insufficient")):
|
||||
if any(
|
||||
term in text
|
||||
for term in ("scope", "permission", "access denied", "insufficient", "forbidden", "403")
|
||||
):
|
||||
return "permissions"
|
||||
if any(term in text for term in ("mailbox", "folder", "select failed", "does not exist")):
|
||||
return "mailbox_not_found"
|
||||
@@ -308,6 +339,14 @@ def _get_source_or_404(source_id: int, db: Session) -> MailSource:
|
||||
return source
|
||||
|
||||
|
||||
def _safe_attr(source: MailSource, name: str, default: Any = None) -> Any:
|
||||
"""Read optional source attributes without letting test doubles invent fields."""
|
||||
value = getattr(source, name, default)
|
||||
if value.__class__.__module__.startswith("unittest.mock"):
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _source_to_response(source: MailSource) -> MailSourceResponse:
|
||||
"""Convert ORM row to response schema, masking the stored password."""
|
||||
return MailSourceResponse(
|
||||
@@ -329,6 +368,12 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
|
||||
gmail_client_secret="**redacted**" if source.gmail_client_secret else None,
|
||||
gmail_email=source.gmail_email,
|
||||
gmail_connected=bool(source.gmail_access_token),
|
||||
m365_tenant_id=_safe_attr(source, "m365_tenant_id", "common") or "common",
|
||||
m365_client_id=_safe_attr(source, "m365_client_id"),
|
||||
m365_client_secret=("**redacted**" if _safe_attr(source, "m365_client_secret") else None),
|
||||
m365_mailbox=_safe_attr(source, "m365_mailbox"),
|
||||
m365_email=_safe_attr(source, "m365_email"),
|
||||
m365_connected=bool(_safe_attr(source, "m365_access_token")),
|
||||
)
|
||||
|
||||
|
||||
@@ -437,6 +482,46 @@ def _fetch_gmail_source(source: MailSource, db: Session) -> Dict[str, Any]:
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_m365_source(source: MailSource, db: Session) -> Dict[str, Any]:
|
||||
"""Run one Microsoft 365 Graph import and persist source/import metadata."""
|
||||
if not source.m365_access_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Microsoft 365 account not yet authorised. Complete OAuth2 flow first.",
|
||||
)
|
||||
|
||||
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
results = client.fetch_reports()
|
||||
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
|
||||
refreshed = client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
|
||||
source.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
|
||||
db.commit()
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_imap_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
|
||||
"""Run one IMAP import and persist source/import metadata."""
|
||||
client = IMAPClient(
|
||||
@@ -459,6 +544,8 @@ def _fetch_source(source: MailSource, db: Session, days: int) -> Dict[str, Any]:
|
||||
"""Dispatch a manual fetch for one configured mail source."""
|
||||
if source.method == "GMAIL_API":
|
||||
return _fetch_gmail_source(source, db)
|
||||
if source.method == "M365_GRAPH":
|
||||
return _fetch_m365_source(source, db)
|
||||
if source.method == "IMAP":
|
||||
return _fetch_imap_source(source, db, days)
|
||||
raise HTTPException(
|
||||
@@ -502,6 +589,10 @@ async def create_mail_source(
|
||||
enabled=payload.enabled,
|
||||
gmail_client_id=payload.gmail_client_id,
|
||||
gmail_client_secret=payload.gmail_client_secret,
|
||||
m365_tenant_id=payload.m365_tenant_id or "common",
|
||||
m365_client_id=payload.m365_client_id,
|
||||
m365_client_secret=payload.m365_client_secret,
|
||||
m365_mailbox=payload.m365_mailbox,
|
||||
)
|
||||
db.add(source)
|
||||
db.commit()
|
||||
@@ -572,7 +663,7 @@ async def fetch_mail_source(
|
||||
_redact_sensitive_text(err),
|
||||
)
|
||||
|
||||
return _fetch_response(source, results)
|
||||
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
|
||||
|
||||
|
||||
@router.put("/{source_id}", response_model=MailSourceResponse)
|
||||
@@ -628,7 +719,7 @@ async def toggle_mail_source(
|
||||
|
||||
|
||||
@router.post("/{source_id}/test", response_model=Dict[str, Any])
|
||||
async def test_stored_mail_source(
|
||||
async def test_stored_mail_source( # noqa: C901
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
@@ -671,6 +762,48 @@ async def test_stored_mail_source(
|
||||
details=exc,
|
||||
)
|
||||
|
||||
if source.method == "M365_GRAPH":
|
||||
if not source.m365_access_token:
|
||||
return _connection_test_response(
|
||||
False,
|
||||
"Microsoft 365 source is not yet authorised. "
|
||||
"Use the Connect Microsoft 365 button to complete OAuth2 authorisation.",
|
||||
)
|
||||
try:
|
||||
graph_client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
)
|
||||
stats = graph_client.test_connection()
|
||||
refreshed = graph_client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
source.last_checked = datetime.utcnow()
|
||||
db.commit()
|
||||
return _connection_test_response(
|
||||
True,
|
||||
f"Microsoft 365 credentials are valid (account: {source.m365_email or 'unknown'}).",
|
||||
stats=stats,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 Graph test failed for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
return _connection_test_response(
|
||||
False,
|
||||
"Microsoft 365 test failed. The saved authorization may need attention.",
|
||||
details=exc,
|
||||
)
|
||||
|
||||
if source.method != "IMAP":
|
||||
return _connection_test_response(
|
||||
False,
|
||||
@@ -722,6 +855,246 @@ async def test_connection_adhoc(
|
||||
return _connection_test_response(success, message, stats)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Microsoft 365 / Graph OAuth2 routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/{source_id}/m365/authorize-url", response_model=Dict[str, Any])
|
||||
async def m365_authorize_url(
|
||||
source_id: int,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return a Microsoft identity platform authorization URL for M365_GRAPH."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
if not source.m365_client_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="m365_client_id is not configured for this source.",
|
||||
)
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
|
||||
auth_url = MicrosoftGraphClient.build_authorization_url(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id,
|
||||
redirect_uri=redirect_uri,
|
||||
state=str(source_id),
|
||||
)
|
||||
return {"authorization_url": auth_url, "redirect_uri": redirect_uri}
|
||||
|
||||
|
||||
@router.get("/{source_id}/m365/callback")
|
||||
async def m365_oauth_callback(
|
||||
source_id: int,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""Handle the Microsoft identity platform OAuth2 redirect."""
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
code = request.query_params.get("code")
|
||||
error = request.query_params.get("error")
|
||||
|
||||
if error or not code:
|
||||
html = (
|
||||
"<html><body><p>Microsoft 365 authorisation failed: "
|
||||
f"{error or 'no code received'}. "
|
||||
"You may close this window.</p></body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html, status_code=400)
|
||||
|
||||
source = db.query(MailSource).filter(MailSource.id == source_id).first()
|
||||
if source is None or source.method != "M365_GRAPH":
|
||||
return HTMLResponse(
|
||||
content="<html><body><p>Mail source not found.</p></body></html>",
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
redirect_uri = f"{base_url}/api/v1/mail-sources/{source_id}/m365/callback"
|
||||
|
||||
try:
|
||||
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
code=code,
|
||||
redirect_uri=redirect_uri,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 token exchange error for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
html = (
|
||||
"<html><body><p>Token exchange failed. "
|
||||
"Please close this window and try again.</p></body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html, status_code=400)
|
||||
|
||||
access_token = token_data.get("access_token")
|
||||
refresh_token = token_data.get("refresh_token")
|
||||
if not access_token:
|
||||
return HTMLResponse(
|
||||
content="<html><body><p>No access token returned by Microsoft.</p></body></html>",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
m365_email = MicrosoftGraphClient.get_account_email(access_token)
|
||||
source.m365_access_token = access_token
|
||||
if refresh_token:
|
||||
source.m365_refresh_token = refresh_token
|
||||
if m365_email:
|
||||
source.m365_email = m365_email
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
logger.info(
|
||||
"Microsoft 365 OAuth2 authorisation complete for source id=%d (account=%s)",
|
||||
int(source_id),
|
||||
_sanitize_for_log(m365_email or "unknown"),
|
||||
)
|
||||
|
||||
html = (
|
||||
"<html><body>"
|
||||
"<p>Microsoft 365 account connected successfully"
|
||||
f"{(' (' + m365_email + ')') if m365_email else ''}. "
|
||||
"You may close this window.</p>"
|
||||
"<script>window.close();</script>"
|
||||
"</body></html>"
|
||||
)
|
||||
return HTMLResponse(content=html)
|
||||
|
||||
|
||||
@router.post("/{source_id}/m365/callback", response_model=MailSourceResponse)
|
||||
async def m365_oauth_callback_post(
|
||||
source_id: int,
|
||||
payload: M365CallbackRequest,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> MailSourceResponse:
|
||||
"""Exchange a Microsoft OAuth2 authorization code for Graph tokens."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
try:
|
||||
token_data = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
code=payload.code,
|
||||
redirect_uri=payload.redirect_uri,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Microsoft 365 token exchange error for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(exc),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
"Token exchange failed. Please check the Microsoft 365 "
|
||||
"connection settings and try again."
|
||||
),
|
||||
) from exc
|
||||
|
||||
access_token = token_data.get("access_token")
|
||||
refresh_token = token_data.get("refresh_token")
|
||||
if not access_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Microsoft did not return an access token.",
|
||||
)
|
||||
|
||||
m365_email = MicrosoftGraphClient.get_account_email(access_token)
|
||||
source.m365_access_token = access_token
|
||||
if refresh_token:
|
||||
source.m365_refresh_token = refresh_token
|
||||
if m365_email:
|
||||
source.m365_email = m365_email
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(source)
|
||||
|
||||
logger.info(
|
||||
"Microsoft 365 OAuth2 tokens saved for source id=%d (account=%s)",
|
||||
int(source_id),
|
||||
_sanitize_for_log(m365_email or "unknown"),
|
||||
)
|
||||
return _source_to_response(source)
|
||||
|
||||
|
||||
@router.post("/{source_id}/m365/fetch", response_model=Dict[str, Any])
|
||||
async def m365_fetch_reports(
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Manually trigger a Microsoft 365 Graph DMARC report fetch."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
results = _fetch_m365_source(source, db)
|
||||
logger.info(
|
||||
"Microsoft 365 fetch for source id=%d: processed=%d reports_found=%d",
|
||||
int(source_id),
|
||||
int(results.get("processed", 0)),
|
||||
int(results.get("reports_found", 0)),
|
||||
)
|
||||
for err in results.get("errors", []):
|
||||
logger.warning(
|
||||
"Microsoft 365 fetch warning for source id=%d: %s",
|
||||
int(source_id),
|
||||
_redact_sensitive_text(err),
|
||||
)
|
||||
|
||||
return _fetch_response(source, results) # lgtm[py/stack-trace-exposure]
|
||||
|
||||
|
||||
@router.delete("/{source_id}/m365/connection", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def m365_disconnect(
|
||||
source_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> None:
|
||||
"""Clear the stored Microsoft Graph OAuth2 tokens for this source."""
|
||||
source = _get_source_or_404(source_id, db)
|
||||
|
||||
if source.method != "M365_GRAPH":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="This endpoint is only available for M365_GRAPH sources.",
|
||||
)
|
||||
|
||||
source.m365_access_token = None
|
||||
source.m365_refresh_token = None
|
||||
source.m365_email = None
|
||||
source.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
logger.info("Microsoft 365 tokens cleared for source id=%d", int(source_id))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gmail API OAuth2 routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+136
-2
@@ -30,6 +30,7 @@ from app.models.mail_source import MailSource # noqa: F401 – ensure table is
|
||||
from app.services.gmail_client import GmailClient
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.import_history import record_import_attempt
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
from app.services.report_persistence import hydrate_report_store_from_db
|
||||
from app.services.report_store import ReportStore
|
||||
from app.services.runtime_status import (
|
||||
@@ -162,7 +163,74 @@ def _poll_single_gmail_source(source: MailSource) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _poll_all_enabled_sources() -> list[MailSource]:
|
||||
def _poll_single_m365_source(source: MailSource) -> None:
|
||||
"""Fetch DMARC reports for a single M365_GRAPH mail source."""
|
||||
global last_check_time # pylint: disable=global-statement
|
||||
|
||||
if not source.m365_access_token:
|
||||
logger.info(
|
||||
"Microsoft 365 polling (source id=%d): skipped – OAuth2 not yet authorised",
|
||||
source.id,
|
||||
)
|
||||
return
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
src = db.query(MailSource).get(source.id)
|
||||
poll_source = src or source
|
||||
already = MicrosoftGraphClient.load_ingested_ids(poll_source.m365_ingested_ids)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id=poll_source.m365_tenant_id or "common",
|
||||
client_id=poll_source.m365_client_id or "",
|
||||
client_secret=poll_source.m365_client_secret or "",
|
||||
access_token=poll_source.m365_access_token,
|
||||
refresh_token=poll_source.m365_refresh_token or "",
|
||||
mailbox=poll_source.m365_mailbox,
|
||||
folder=poll_source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
results = client.fetch_reports()
|
||||
if src:
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
src.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
|
||||
refreshed = client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
src.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
src.m365_refresh_token = refreshed["refresh_token"]
|
||||
|
||||
src.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, src, results, started_at=started_at, trigger="scheduled")
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
last_check_time = datetime.now()
|
||||
|
||||
if results["success"]:
|
||||
logger.info(
|
||||
"Microsoft 365 polling (source id=%d): %s emails processed, "
|
||||
"%s aggregate reports found",
|
||||
source.id,
|
||||
results["processed"],
|
||||
results["reports_found"],
|
||||
)
|
||||
if results["new_domains"]:
|
||||
logger.info("New domains found: %s", ", ".join(results["new_domains"]))
|
||||
else:
|
||||
logger.error(
|
||||
"Microsoft 365 polling (source id=%d) failed: %s",
|
||||
source.id,
|
||||
results.get("error", "Unknown error"),
|
||||
)
|
||||
|
||||
|
||||
def _poll_all_enabled_sources() -> list[MailSource]: # noqa: C901
|
||||
"""Iterate over all enabled mail sources and poll each one."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
@@ -182,6 +250,11 @@ def _poll_all_enabled_sources() -> list[MailSource]:
|
||||
_poll_single_gmail_source(source)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Gmail source id=%d: %s", source.id, str(e))
|
||||
elif source.method == "M365_GRAPH":
|
||||
try:
|
||||
_poll_single_m365_source(source)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
|
||||
elif source.method == "IMAP":
|
||||
try:
|
||||
_poll_single_imap_source(source)
|
||||
@@ -682,7 +755,50 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
|
||||
def _trigger_poll_m365_source(source: MailSource, db) -> dict:
|
||||
"""Poll a single M365_GRAPH source and return a result dict for the API response."""
|
||||
global last_check_time # pylint: disable=global-statement
|
||||
|
||||
already = MicrosoftGraphClient.load_ingested_ids(source.m365_ingested_ids)
|
||||
graph_client = MicrosoftGraphClient(
|
||||
tenant_id=source.m365_tenant_id or "common",
|
||||
client_id=source.m365_client_id or "",
|
||||
client_secret=source.m365_client_secret or "",
|
||||
access_token=source.m365_access_token,
|
||||
refresh_token=source.m365_refresh_token or "",
|
||||
mailbox=source.m365_mailbox,
|
||||
folder=source.folder or "INBOX",
|
||||
already_ingested_ids=already,
|
||||
db=db,
|
||||
)
|
||||
started_at = datetime.utcnow()
|
||||
results = graph_client.fetch_reports()
|
||||
last_check_time = datetime.now()
|
||||
|
||||
if results.get("new_ingested_ids"):
|
||||
all_ids = list(dict.fromkeys(already + results["new_ingested_ids"]))
|
||||
source.m365_ingested_ids = MicrosoftGraphClient.dump_ingested_ids(all_ids)
|
||||
refreshed = graph_client.get_refreshed_tokens()
|
||||
if refreshed:
|
||||
source.m365_access_token = refreshed["access_token"]
|
||||
if "refresh_token" in refreshed:
|
||||
source.m365_refresh_token = refreshed["refresh_token"]
|
||||
source.last_checked = datetime.utcnow()
|
||||
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
|
||||
db.commit()
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"success": results["success"],
|
||||
"processed": results.get("processed", 0),
|
||||
"reports_found": results.get("reports_found", 0),
|
||||
"forensic_reports_found": results.get("forensic_reports_found", 0),
|
||||
"duplicate_forensic_reports": results.get("duplicate_forensic_reports", 0),
|
||||
"new_domains": results.get("new_domains", []),
|
||||
}
|
||||
|
||||
|
||||
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict: # noqa: C901
|
||||
"""Dispatch a single mail source for the manual trigger-poll endpoint.
|
||||
|
||||
Returns a result/summary dict that is included in the API response.
|
||||
@@ -705,6 +821,24 @@ def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
|
||||
"success": False,
|
||||
"error": "Failed to poll. Check server logs for details.",
|
||||
}
|
||||
if source.method == "M365_GRAPH":
|
||||
if not source.m365_access_token:
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"skipped": True,
|
||||
"reason": "Microsoft 365 account not yet authorised",
|
||||
}
|
||||
try:
|
||||
return _trigger_poll_m365_source(source, db)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error polling Microsoft 365 source id=%d: %s", source.id, str(e))
|
||||
return {
|
||||
"source_id": source.id,
|
||||
"name": source.name,
|
||||
"success": False,
|
||||
"error": "Failed to poll. Check server logs for details.",
|
||||
}
|
||||
if source.method == "IMAP":
|
||||
try:
|
||||
return _trigger_poll_imap_source(source, db, days=days)
|
||||
|
||||
@@ -18,6 +18,7 @@ class MailSource(Base):
|
||||
- ``IMAP`` – standard IMAP4 (over SSL/TLS or STARTTLS)
|
||||
- ``POP3`` – POP3 inbox (stub for future implementation)
|
||||
- ``GMAIL_API`` – Gmail API with OAuth 2.0
|
||||
- ``M365_GRAPH`` – Microsoft 365 / Exchange Online via Microsoft Graph
|
||||
"""
|
||||
|
||||
__tablename__ = "mail_sources"
|
||||
@@ -28,7 +29,7 @@ class MailSource(Base):
|
||||
name = Column(String, nullable=False)
|
||||
|
||||
# Connection method – determines which fields are used at runtime
|
||||
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API
|
||||
method = Column(String, nullable=False, default="IMAP") # IMAP | POP3 | GMAIL_API | M365_GRAPH
|
||||
|
||||
# Connection details (used by IMAP and POP3)
|
||||
server = Column(String, nullable=True)
|
||||
@@ -48,6 +49,19 @@ class MailSource(Base):
|
||||
# JSON-encoded list of Gmail message IDs that have already been ingested
|
||||
gmail_ingested_ids = Column(Text, nullable=True, default="[]")
|
||||
|
||||
# Microsoft 365 / Graph OAuth2 credentials (used by M365_GRAPH method)
|
||||
m365_tenant_id = Column(String, nullable=True, default="common")
|
||||
m365_client_id = Column(String, nullable=True)
|
||||
_m365_client_secret = Column("m365_client_secret", Text, nullable=True)
|
||||
_m365_access_token = Column("m365_access_token", Text, nullable=True)
|
||||
_m365_refresh_token = Column("m365_refresh_token", Text, nullable=True)
|
||||
# Optional user/shared mailbox to poll. Empty means the authorised account (/me).
|
||||
m365_mailbox = Column(String, nullable=True)
|
||||
# Email address reported by Microsoft Graph for the authorised account.
|
||||
m365_email = Column(String, nullable=True)
|
||||
# JSON-encoded list of Graph message IDs that have already been ingested
|
||||
m365_ingested_ids = Column(Text, nullable=True, default="[]")
|
||||
|
||||
# Polling behaviour
|
||||
polling_interval = Column(Integer, default=60) # minutes
|
||||
|
||||
@@ -76,6 +90,9 @@ class MailSource(Base):
|
||||
"gmail_client_secret": self._gmail_client_secret,
|
||||
"gmail_access_token": self._gmail_access_token,
|
||||
"gmail_refresh_token": self._gmail_refresh_token,
|
||||
"m365_client_secret": self._m365_client_secret,
|
||||
"m365_access_token": self._m365_access_token,
|
||||
"m365_refresh_token": self._m365_refresh_token,
|
||||
}
|
||||
|
||||
for public_name, stored_value in secret_fields.items():
|
||||
@@ -120,3 +137,30 @@ class MailSource(Base):
|
||||
@gmail_refresh_token.setter
|
||||
def gmail_refresh_token(self, value):
|
||||
self._gmail_refresh_token = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_client_secret(self):
|
||||
"""Return the decrypted Microsoft 365 OAuth client secret, if present."""
|
||||
return decrypt_secret(self._m365_client_secret)
|
||||
|
||||
@m365_client_secret.setter
|
||||
def m365_client_secret(self, value):
|
||||
self._m365_client_secret = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_access_token(self):
|
||||
"""Return the decrypted Microsoft Graph access token, if present."""
|
||||
return decrypt_secret(self._m365_access_token)
|
||||
|
||||
@m365_access_token.setter
|
||||
def m365_access_token(self, value):
|
||||
self._m365_access_token = encrypt_secret(value)
|
||||
|
||||
@property
|
||||
def m365_refresh_token(self):
|
||||
"""Return the decrypted Microsoft Graph refresh token, if present."""
|
||||
return decrypt_secret(self._m365_refresh_token)
|
||||
|
||||
@m365_refresh_token.setter
|
||||
def m365_refresh_token(self, value):
|
||||
self._m365_refresh_token = encrypt_secret(value)
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Microsoft Graph client for retrieving DMARC aggregate reports."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.report_persistence import report_exists, save_parsed_report
|
||||
from app.services.report_store import ReportStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GRAPH_BASE_URL = "https://graph.microsoft.com/v1.0"
|
||||
LOGIN_BASE_URL = "https://login.microsoftonline.com"
|
||||
|
||||
M365_SCOPES = [
|
||||
"offline_access",
|
||||
"https://graph.microsoft.com/User.Read",
|
||||
"https://graph.microsoft.com/Mail.Read",
|
||||
]
|
||||
|
||||
_PAGE_SIZE = 100
|
||||
_DMARC_SUBJECT_TERMS = (
|
||||
"dmarc",
|
||||
"aggregate report",
|
||||
"domain report",
|
||||
"report domain",
|
||||
"rua",
|
||||
"submitter",
|
||||
)
|
||||
_DMARC_SENDER_TERMS = (
|
||||
"dmarc",
|
||||
"reports",
|
||||
"postmaster",
|
||||
)
|
||||
|
||||
|
||||
class MicrosoftGraphError(RuntimeError):
|
||||
"""Raised when Microsoft Graph or the token endpoint returns a failure."""
|
||||
|
||||
|
||||
class MicrosoftGraphClient:
|
||||
"""
|
||||
Retrieve DMARC aggregate reports from Microsoft 365 through Microsoft Graph.
|
||||
|
||||
The client uses delegated OAuth tokens and read-only Graph scopes. Messages
|
||||
are never modified or deleted; already-ingested Graph message IDs are stored
|
||||
by the caller to avoid reprocessing the same email.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
access_token: str,
|
||||
refresh_token: str,
|
||||
mailbox: Optional[str] = None,
|
||||
folder: str = "inbox",
|
||||
already_ingested_ids: Optional[List[str]] = None,
|
||||
db: Any = None,
|
||||
):
|
||||
self.tenant_id = tenant_id or "common"
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
self.access_token = access_token
|
||||
self.refresh_token = refresh_token
|
||||
self.mailbox = (mailbox or "").strip()
|
||||
self.folder = folder or "inbox"
|
||||
self.already_ingested_ids: List[str] = list(already_ingested_ids or [])
|
||||
self.report_store = ReportStore.get_instance()
|
||||
self.db = db
|
||||
self._refreshed_tokens: Optional[Dict[str, str]] = None
|
||||
|
||||
def get_refreshed_tokens(self) -> Optional[Dict[str, str]]:
|
||||
"""Return refreshed OAuth tokens, if a request had to refresh them."""
|
||||
return self._refreshed_tokens
|
||||
|
||||
@staticmethod
|
||||
def build_authorization_url(
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
redirect_uri: str,
|
||||
state: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build a Microsoft identity platform authorization-code URL."""
|
||||
tenant = quote(tenant_id or "common", safe="")
|
||||
params: Dict[str, str] = {
|
||||
"client_id": client_id,
|
||||
"response_type": "code",
|
||||
"redirect_uri": redirect_uri,
|
||||
"response_mode": "query",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
"prompt": "select_account",
|
||||
}
|
||||
if state:
|
||||
params["state"] = state
|
||||
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/authorize?" + urlencode(params)
|
||||
|
||||
@staticmethod
|
||||
def exchange_code_for_tokens(
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
code: str,
|
||||
redirect_uri: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Exchange an authorization code for Microsoft Graph tokens."""
|
||||
data = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
}
|
||||
resp = httpx.post(MicrosoftGraphClient._token_url(tenant_id), data=data, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
raise MicrosoftGraphError(
|
||||
f"Microsoft token exchange failed ({resp.status_code}): {resp.text}"
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
@staticmethod
|
||||
def get_account_email(access_token: str) -> Optional[str]:
|
||||
"""Return the mailbox identity exposed by Graph /me for an access token."""
|
||||
try:
|
||||
resp = httpx.get(
|
||||
f"{GRAPH_BASE_URL}/me",
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
params={"$select": "mail,userPrincipalName"},
|
||||
timeout=30,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
profile = resp.json()
|
||||
return profile.get("mail") or profile.get("userPrincipalName")
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Failed to fetch Microsoft 365 account email: %s", exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def load_ingested_ids(json_text: Optional[str]) -> List[str]:
|
||||
"""Deserialize the m365_ingested_ids text column into a list."""
|
||||
if not json_text:
|
||||
return []
|
||||
try:
|
||||
decoded = json.loads(json_text)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return []
|
||||
return [str(item) for item in decoded] if isinstance(decoded, list) else []
|
||||
|
||||
@staticmethod
|
||||
def dump_ingested_ids(ids: List[str]) -> str:
|
||||
"""Serialize Graph message IDs for database storage."""
|
||||
return json.dumps(ids)
|
||||
|
||||
def test_connection(self) -> Dict[str, Any]:
|
||||
"""Verify that the saved delegated token can read the target mailbox."""
|
||||
mailbox_path = self._mailbox_path()
|
||||
data = self._request(
|
||||
"GET",
|
||||
f"{mailbox_path}/messages",
|
||||
params={"$top": 1, "$select": "id"},
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"message_count": len(data.get("value", [])),
|
||||
"diagnostic_detail": "Microsoft Graph mailbox read succeeded.",
|
||||
}
|
||||
|
||||
def fetch_reports(self) -> Dict[str, Any]:
|
||||
"""Fetch and ingest DMARC report attachments from Microsoft Graph."""
|
||||
stats: Dict[str, Any] = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"reports_found": 0,
|
||||
"forensic_reports_found": 0,
|
||||
"duplicate_reports": 0,
|
||||
"duplicate_forensic_reports": 0,
|
||||
"new_domains": [],
|
||||
"errors": [],
|
||||
"new_ingested_ids": [],
|
||||
"details": [],
|
||||
}
|
||||
|
||||
try:
|
||||
messages = self._list_dmarc_messages()
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Microsoft Graph: failed to list messages: %s", exc)
|
||||
return {**stats, "success": False, "error": str(exc), "errors": [str(exc)]}
|
||||
|
||||
domains_before = set(self.report_store.get_domains())
|
||||
|
||||
for message in messages:
|
||||
message_id = str(message.get("id") or "")
|
||||
if not message_id:
|
||||
continue
|
||||
if message_id in self.already_ingested_ids:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="already_ingested_message",
|
||||
message_id=message_id,
|
||||
)
|
||||
continue
|
||||
|
||||
stats["processed"] += 1
|
||||
found = self._process_message(message, stats)
|
||||
if found >= 0:
|
||||
stats["new_ingested_ids"].append(message_id)
|
||||
self.already_ingested_ids.append(message_id)
|
||||
|
||||
domains_after = set(self.report_store.get_domains())
|
||||
stats["new_domains"] = list(domains_after - domains_before)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _token_url(tenant_id: str) -> str:
|
||||
tenant = quote(tenant_id or "common", safe="")
|
||||
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/token"
|
||||
|
||||
@staticmethod
|
||||
def _append_detail(stats: dict, **detail: str) -> None:
|
||||
stats.setdefault("details", []).append(
|
||||
{key: value for key, value in detail.items() if value}
|
||||
)
|
||||
|
||||
def _mailbox_path(self) -> str:
|
||||
if not self.mailbox or self.mailbox.lower() == "me":
|
||||
return "/me"
|
||||
return f"/users/{quote(self.mailbox, safe='')}"
|
||||
|
||||
def _messages_path(self) -> str:
|
||||
mailbox_path = self._mailbox_path()
|
||||
folder = (self.folder or "").strip()
|
||||
if not folder:
|
||||
return f"{mailbox_path}/messages"
|
||||
if folder.upper() == "INBOX":
|
||||
folder = "inbox"
|
||||
return f"{mailbox_path}/mailFolders/{quote(folder, safe='')}/messages"
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {"Authorization": f"Bearer {self.access_token}"}
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path_or_url: str,
|
||||
*,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
url = path_or_url if path_or_url.startswith("http") else f"{GRAPH_BASE_URL}{path_or_url}"
|
||||
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
|
||||
if resp.status_code == 401 and self.refresh_token:
|
||||
self._refresh_access_token()
|
||||
resp = httpx.request(method, url, headers=self._headers(), params=params, timeout=30)
|
||||
if resp.status_code < 200 or resp.status_code >= 300:
|
||||
raise MicrosoftGraphError(self._format_error(resp))
|
||||
return resp.json() if resp.content else {}
|
||||
|
||||
def _refresh_access_token(self) -> None:
|
||||
data = {
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret,
|
||||
"refresh_token": self.refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
"scope": " ".join(M365_SCOPES),
|
||||
}
|
||||
resp = httpx.post(self._token_url(self.tenant_id), data=data, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
raise MicrosoftGraphError(
|
||||
f"Microsoft token refresh failed ({resp.status_code}): {resp.text}"
|
||||
)
|
||||
token_data = resp.json()
|
||||
access_token = token_data.get("access_token")
|
||||
if not access_token:
|
||||
raise MicrosoftGraphError("Microsoft token refresh did not return an access token.")
|
||||
self.access_token = access_token
|
||||
refreshed = {"access_token": access_token}
|
||||
if token_data.get("refresh_token"):
|
||||
self.refresh_token = token_data["refresh_token"]
|
||||
refreshed["refresh_token"] = token_data["refresh_token"]
|
||||
self._refreshed_tokens = refreshed
|
||||
|
||||
@staticmethod
|
||||
def _format_error(resp: httpx.Response) -> str:
|
||||
try:
|
||||
payload = resp.json()
|
||||
except ValueError:
|
||||
payload = {}
|
||||
message = payload.get("error_description")
|
||||
if not message and isinstance(payload.get("error"), dict):
|
||||
message = payload["error"].get("message")
|
||||
code = payload["error"].get("code")
|
||||
if code:
|
||||
message = f"{code}: {message}" if message else code
|
||||
return message or f"Microsoft Graph request failed ({resp.status_code}): {resp.text}"
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_dmarc_message(message: Dict[str, Any]) -> bool:
|
||||
if not message.get("hasAttachments"):
|
||||
return False
|
||||
subject = str(message.get("subject") or "").lower()
|
||||
sender = (
|
||||
((message.get("from") or {}).get("emailAddress") or {}).get("address") or ""
|
||||
).lower()
|
||||
return any(term in subject for term in _DMARC_SUBJECT_TERMS) or any(
|
||||
term in sender for term in _DMARC_SENDER_TERMS
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_dmarc_attachment(filename: str) -> bool:
|
||||
lower = filename.lower()
|
||||
return (
|
||||
lower.endswith(".xml")
|
||||
or lower.endswith(".zip")
|
||||
or lower.endswith(".gz")
|
||||
or lower.endswith(".gzip")
|
||||
)
|
||||
|
||||
def _list_dmarc_messages(self) -> List[Dict[str, Any]]:
|
||||
messages: List[Dict[str, Any]] = []
|
||||
url = self._messages_path()
|
||||
params: Optional[Dict[str, Any]] = {
|
||||
"$top": _PAGE_SIZE,
|
||||
"$select": "id,subject,from,hasAttachments,receivedDateTime",
|
||||
"$orderby": "receivedDateTime desc",
|
||||
}
|
||||
|
||||
while url:
|
||||
data = self._request("GET", url, params=params)
|
||||
for message in data.get("value", []):
|
||||
if self._looks_like_dmarc_message(message):
|
||||
messages.append(message)
|
||||
url = data.get("@odata.nextLink")
|
||||
params = None
|
||||
|
||||
return messages
|
||||
|
||||
def _process_message(self, message: Dict[str, Any], stats: Dict[str, Any]) -> int:
|
||||
message_id = str(message.get("id") or "")
|
||||
try:
|
||||
attachments = self._list_attachments(message_id)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Microsoft Graph: failed to fetch attachments for %s: %s", message_id, exc)
|
||||
stats["errors"].append(f"Failed to fetch attachments for message {message_id}: {exc}")
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="error",
|
||||
reason="attachment_fetch_failed",
|
||||
message_id=message_id,
|
||||
error=str(exc),
|
||||
)
|
||||
return -1
|
||||
return self._process_attachments(message_id, attachments, stats)
|
||||
|
||||
def _list_attachments(self, message_id: str) -> List[Dict[str, Any]]:
|
||||
mailbox_path = self._mailbox_path()
|
||||
data = self._request(
|
||||
"GET",
|
||||
f"{mailbox_path}/messages/{quote(message_id, safe='')}/attachments",
|
||||
)
|
||||
return list(data.get("value", []))
|
||||
|
||||
def _store_report_if_new(self, report: Dict[str, Any]) -> bool:
|
||||
domain = report.get("domain", "unknown")
|
||||
report_id = report.get("report_id", "")
|
||||
if report_id and (
|
||||
self.report_store.has_report(domain, report_id)
|
||||
or (self.db is not None and report_exists(self.db, domain, report_id))
|
||||
):
|
||||
logger.info("Skipping duplicate DMARC report %s for %s", report_id, domain)
|
||||
return False
|
||||
|
||||
if self.db is not None:
|
||||
save_parsed_report(self.db, report)
|
||||
self.report_store.add_report(report)
|
||||
return True
|
||||
|
||||
def _process_attachments(
|
||||
self,
|
||||
message_id: str,
|
||||
attachments: List[Dict[str, Any]],
|
||||
stats: Dict[str, Any],
|
||||
) -> int:
|
||||
reports_found = 0
|
||||
|
||||
for attachment in attachments:
|
||||
filename = str(attachment.get("name") or "")
|
||||
if not filename:
|
||||
continue
|
||||
if not self._is_dmarc_attachment(filename):
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="unsupported_attachment",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
attachment_type = str(attachment.get("@odata.type") or "").lower()
|
||||
if "fileattachment" not in attachment_type:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="unsupported_attachment_type",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
content_b64 = attachment.get("contentBytes")
|
||||
if not content_b64:
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="skipped",
|
||||
reason="empty_attachment",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
content = base64.b64decode(content_b64)
|
||||
report = DMARCParser.parse_file(content, filename)
|
||||
domain = str(report.get("domain", "unknown"))
|
||||
report_id = str(report.get("report_id", ""))
|
||||
if self._store_report_if_new(report):
|
||||
stats["reports_found"] += 1
|
||||
reports_found += 1
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="imported",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
domain=domain,
|
||||
report_id=report_id,
|
||||
)
|
||||
else:
|
||||
stats["duplicate_reports"] = stats.get("duplicate_reports", 0) + 1
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="duplicate",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
domain=domain,
|
||||
report_id=report_id,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Failed to parse Graph DMARC attachment %s: %s", filename, exc)
|
||||
stats["errors"].append(f"Failed to parse {filename}: {exc}")
|
||||
self._append_detail(
|
||||
stats,
|
||||
status="error",
|
||||
reason="parse_failed",
|
||||
message_id=message_id,
|
||||
filename=filename,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
return reports_found
|
||||
@@ -86,8 +86,8 @@
|
||||
<td>
|
||||
<span class="badge badge-outline" x-text="source.method"></span>
|
||||
</td>
|
||||
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_email || '—') : (source.server ? source.server + ':' + source.port : '—')"></td>
|
||||
<td x-text="source.method === 'GMAIL_API' ? (source.gmail_connected ? '✅ Connected' : '⚠️ Not authorised') : (source.username || '—')"></td>
|
||||
<td x-text="sourceAccountLabel(source)"></td>
|
||||
<td x-text="sourceStatusLabel(source)"></td>
|
||||
<td x-text="source.last_checked ? new Date(source.last_checked).toLocaleString() : 'Never'"></td>
|
||||
<td>
|
||||
<input
|
||||
@@ -293,6 +293,7 @@
|
||||
<option value="IMAP">IMAP</option>
|
||||
<option value="POP3">POP3 (coming soon)</option>
|
||||
<option value="GMAIL_API">Gmail API (OAuth2)</option>
|
||||
<option value="M365_GRAPH">Microsoft 365 (Graph OAuth2)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
@@ -434,6 +435,85 @@
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- Microsoft 365 Graph fields -->
|
||||
<template x-if="form.method === 'M365_GRAPH'">
|
||||
<div class="space-y-4">
|
||||
<div class="alert alert-info text-sm p-3">
|
||||
<div>
|
||||
<p class="font-semibold">Microsoft 365 Setup</p>
|
||||
<p class="mt-1">Enter an app registration Client ID and client secret from Microsoft Entra admin center. Add this app's callback URL as a Web redirect URI, then save and click <strong>Connect Microsoft 365</strong>.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Tenant ID</span></label>
|
||||
<input type="text" x-model="form.m365_tenant_id"
|
||||
placeholder="common, organizations, or tenant GUID"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Microsoft Client ID <span class="text-error">*</span></span></label>
|
||||
<input type="text" x-model="form.m365_client_id"
|
||||
placeholder="Application (client) ID"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label">
|
||||
<span class="label-text font-medium">Microsoft Client Secret <span class="text-error">*</span></span>
|
||||
<span class="label-text-alt text-muted-foreground" x-show="editingId">Leave blank to keep existing</span>
|
||||
</label>
|
||||
<div class="relative">
|
||||
<input :type="showPassword ? 'text' : 'password'"
|
||||
x-model="form.m365_client_secret"
|
||||
class="input input-bordered w-full pr-10" />
|
||||
<button type="button"
|
||||
class="absolute right-2 top-3 text-muted-foreground hover:text-foreground"
|
||||
x-on:click="showPassword = !showPassword">
|
||||
<svg x-show="!showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
|
||||
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"></path>
|
||||
<circle cx="12" cy="12" r="3"></circle>
|
||||
</svg>
|
||||
<svg x-show="showPassword" xmlns="http://www.w3.org/2000/svg" width="16" height="16"
|
||||
viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M17.94 17.94A10.07 10.07 0 0 1 12 20c-7 0-11-8-11-8a18.45 18.45 0 0 1 5.06-5.94"></path>
|
||||
<path d="M9.9 4.24A9.12 9.12 0 0 1 12 4c7 0 11 8 11 8a18.5 18.5 0 0 1-2.16 3.19"></path>
|
||||
<line x1="1" y1="1" x2="23" y2="23"></line>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Mailbox</span></label>
|
||||
<input type="text" x-model="form.m365_mailbox"
|
||||
placeholder="Leave blank for the authorised account"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Folder</span></label>
|
||||
<input type="text" x-model="form.folder" placeholder="INBOX"
|
||||
class="input input-bordered w-full" />
|
||||
</div>
|
||||
|
||||
<template x-if="editingId && m365Connected">
|
||||
<div class="alert alert-success text-sm p-3">
|
||||
<span>Connected as <strong x-text="m365Email || 'Microsoft 365 account'"></strong></span>
|
||||
</div>
|
||||
</template>
|
||||
<template x-if="editingId && !m365Connected">
|
||||
<div class="alert alert-warning text-sm p-3">
|
||||
<span>Not yet authorised. Save this source first, then click <strong>Connect Microsoft 365</strong>.</span>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- Polling interval -->
|
||||
<div>
|
||||
<label class="label"><span class="label-text font-medium">Polling Interval (minutes)</span></label>
|
||||
@@ -467,7 +547,7 @@
|
||||
<div class="flex justify-between pt-2">
|
||||
<!-- Test Connection (IMAP/POP3) or Connect Gmail button -->
|
||||
<div class="flex gap-2">
|
||||
<template x-if="form.method !== 'GMAIL_API'">
|
||||
<template x-if="form.method !== 'GMAIL_API' && form.method !== 'M365_GRAPH'">
|
||||
<button type="button" class="btn btn-outline btn-sm"
|
||||
x-on:click="testAdHoc()"
|
||||
:disabled="isTesting || isSaving">
|
||||
@@ -492,6 +572,13 @@
|
||||
Connect Gmail
|
||||
</button>
|
||||
</template>
|
||||
<template x-if="form.method === 'M365_GRAPH' && editingId">
|
||||
<button type="button" class="btn btn-outline btn-sm"
|
||||
x-on:click="connectM365()"
|
||||
:disabled="isTesting || isSaving">
|
||||
Connect Microsoft 365
|
||||
</button>
|
||||
</template>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<button type="button" class="btn btn-ghost btn-sm" x-on:click="closeForm()">Cancel</button>
|
||||
@@ -577,6 +664,8 @@ function mailSourcesApp() {
|
||||
testResult: { message: '', success: false, diagnostic_summary: '', recovery_steps: [] },
|
||||
gmailConnected: false,
|
||||
gmailEmail: '',
|
||||
m365Connected: false,
|
||||
m365Email: '',
|
||||
|
||||
form: {
|
||||
name: '',
|
||||
@@ -591,6 +680,10 @@ function mailSourcesApp() {
|
||||
enabled: true,
|
||||
gmail_client_id: '',
|
||||
gmail_client_secret: '',
|
||||
m365_tenant_id: 'common',
|
||||
m365_client_id: '',
|
||||
m365_client_secret: '',
|
||||
m365_mailbox: '',
|
||||
},
|
||||
|
||||
async init() {
|
||||
@@ -689,6 +782,26 @@ function mailSourcesApp() {
|
||||
return value ? new Date(value).toLocaleString() : '—';
|
||||
},
|
||||
|
||||
sourceAccountLabel(source) {
|
||||
if (source.method === 'GMAIL_API') {
|
||||
return source.gmail_email || '—';
|
||||
}
|
||||
if (source.method === 'M365_GRAPH') {
|
||||
return source.m365_mailbox || source.m365_email || '—';
|
||||
}
|
||||
return source.server ? `${source.server}:${source.port}` : '—';
|
||||
},
|
||||
|
||||
sourceStatusLabel(source) {
|
||||
if (source.method === 'GMAIL_API') {
|
||||
return source.gmail_connected ? 'Connected' : 'Not authorised';
|
||||
}
|
||||
if (source.method === 'M365_GRAPH') {
|
||||
return source.m365_connected ? 'Connected' : 'Not authorised';
|
||||
}
|
||||
return source.username || '—';
|
||||
},
|
||||
|
||||
formatList(value) {
|
||||
return value && value.length ? value.join(', ') : '—';
|
||||
},
|
||||
@@ -738,6 +851,8 @@ function mailSourcesApp() {
|
||||
this.editingId = null;
|
||||
this.gmailConnected = false;
|
||||
this.gmailEmail = '';
|
||||
this.m365Connected = false;
|
||||
this.m365Email = '';
|
||||
this.form = {
|
||||
name: '',
|
||||
method: 'IMAP',
|
||||
@@ -751,6 +866,10 @@ function mailSourcesApp() {
|
||||
enabled: true,
|
||||
gmail_client_id: '',
|
||||
gmail_client_secret: '',
|
||||
m365_tenant_id: 'common',
|
||||
m365_client_id: '',
|
||||
m365_client_secret: '',
|
||||
m365_mailbox: '',
|
||||
};
|
||||
this.testResult = this.emptyTestResult();
|
||||
this.showForm = true;
|
||||
@@ -760,6 +879,8 @@ function mailSourcesApp() {
|
||||
this.editingId = source.id;
|
||||
this.gmailConnected = source.gmail_connected || false;
|
||||
this.gmailEmail = source.gmail_email || '';
|
||||
this.m365Connected = source.m365_connected || false;
|
||||
this.m365Email = source.m365_email || '';
|
||||
this.form = {
|
||||
name: source.name,
|
||||
method: source.method,
|
||||
@@ -773,6 +894,10 @@ function mailSourcesApp() {
|
||||
enabled: source.enabled !== false,
|
||||
gmail_client_id: source.gmail_client_id || '',
|
||||
gmail_client_secret: '', // never pre-fill client secret
|
||||
m365_tenant_id: source.m365_tenant_id || 'common',
|
||||
m365_client_id: source.m365_client_id || '',
|
||||
m365_client_secret: '', // never pre-fill client secret
|
||||
m365_mailbox: source.m365_mailbox || '',
|
||||
};
|
||||
this.testResult = this.emptyTestResult();
|
||||
this.showForm = true;
|
||||
@@ -826,6 +951,45 @@ function mailSourcesApp() {
|
||||
}
|
||||
},
|
||||
|
||||
async connectM365() {
|
||||
if (!this.editingId) return;
|
||||
try {
|
||||
const resp = await fetch(
|
||||
`/api/v1/mail-sources/${this.editingId}/m365/authorize-url`
|
||||
);
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json();
|
||||
this.feedback = { message: `Error: ${err.detail || 'Failed to get authorization URL'}`, type: 'error' };
|
||||
return;
|
||||
}
|
||||
const data = await resp.json();
|
||||
const popup = window.open(
|
||||
data.authorization_url,
|
||||
'm365_oauth',
|
||||
'width=700,height=760,scrollbars=yes'
|
||||
);
|
||||
const pollInterval = setInterval(async () => {
|
||||
if (popup && popup.closed) {
|
||||
clearInterval(pollInterval);
|
||||
await this.loadSources();
|
||||
const updated = this.sources.find(s => s.id === this.editingId);
|
||||
if (updated) {
|
||||
this.m365Connected = updated.m365_connected || false;
|
||||
this.m365Email = updated.m365_email || '';
|
||||
if (this.m365Connected) {
|
||||
this.feedback = {
|
||||
message: `Microsoft 365 connected successfully (${this.m365Email || 'account connected'}).`,
|
||||
type: 'success',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
} catch (e) {
|
||||
this.feedback = { message: `Error: ${e.message}`, type: 'error' };
|
||||
}
|
||||
},
|
||||
|
||||
async saveSource() {
|
||||
this.isSaving = true;
|
||||
this.feedback = { message: '', type: '' };
|
||||
@@ -839,6 +1003,10 @@ function mailSourcesApp() {
|
||||
if (this.editingId && !payload.gmail_client_secret) {
|
||||
delete payload.gmail_client_secret;
|
||||
}
|
||||
// Don't send empty m365_client_secret on edit
|
||||
if (this.editingId && !payload.m365_client_secret) {
|
||||
delete payload.m365_client_secret;
|
||||
}
|
||||
|
||||
const url = this.editingId
|
||||
? `/api/v1/mail-sources/${this.editingId}`
|
||||
|
||||
@@ -99,6 +99,36 @@ class TestMailSourceModel:
|
||||
assert is_encrypted_secret(stored.gmail_access_token)
|
||||
assert is_encrypted_secret(stored.gmail_refresh_token)
|
||||
|
||||
def test_m365_oauth_secrets_are_encrypted_at_rest(self, db_session: Session):
|
||||
source = MailSource(
|
||||
name="Encrypted Microsoft 365",
|
||||
method="M365_GRAPH",
|
||||
m365_client_secret="client-secret",
|
||||
m365_access_token="access-token",
|
||||
m365_refresh_token="refresh-token",
|
||||
)
|
||||
db_session.add(source)
|
||||
db_session.commit()
|
||||
db_session.refresh(source)
|
||||
|
||||
stored = db_session.execute(
|
||||
text(
|
||||
"SELECT m365_client_secret, m365_access_token, m365_refresh_token "
|
||||
"FROM mail_sources WHERE id = :id"
|
||||
),
|
||||
{"id": source.id},
|
||||
).one()
|
||||
|
||||
assert source.m365_client_secret == "client-secret"
|
||||
assert source.m365_access_token == "access-token"
|
||||
assert source.m365_refresh_token == "refresh-token"
|
||||
assert stored.m365_client_secret != "client-secret"
|
||||
assert stored.m365_access_token != "access-token"
|
||||
assert stored.m365_refresh_token != "refresh-token"
|
||||
assert is_encrypted_secret(stored.m365_client_secret)
|
||||
assert is_encrypted_secret(stored.m365_access_token)
|
||||
assert is_encrypted_secret(stored.m365_refresh_token)
|
||||
|
||||
def test_legacy_plaintext_mail_source_secret_remains_readable(self, db_session: Session):
|
||||
db_session.execute(
|
||||
text(
|
||||
@@ -968,6 +998,455 @@ class TestGmailAPIMailSource:
|
||||
assert data["new_domains"] == ["example.com"]
|
||||
|
||||
|
||||
class TestMicrosoft365GraphMailSource:
|
||||
"""Tests for M365_GRAPH mail source creation, OAuth flow, and fetching."""
|
||||
|
||||
def test_create_m365_graph_source(self, authed_client: TestClient):
|
||||
payload = {
|
||||
"name": "My Microsoft 365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_tenant_id": "organizations",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
"m365_mailbox": "dmarc@example.com",
|
||||
"folder": "INBOX",
|
||||
"polling_interval": 30,
|
||||
"enabled": True,
|
||||
}
|
||||
resp = authed_client.post("/api/v1/mail-sources", json=payload)
|
||||
assert resp.status_code == 201
|
||||
data = resp.json()
|
||||
assert data["method"] == "M365_GRAPH"
|
||||
assert data["m365_tenant_id"] == "organizations"
|
||||
assert data["m365_client_id"] == "client-id"
|
||||
assert data["m365_client_secret"] == "**redacted**"
|
||||
assert data["m365_mailbox"] == "dmarc@example.com"
|
||||
assert data["m365_connected"] is False
|
||||
assert data["m365_email"] is None
|
||||
|
||||
def test_m365_source_test_no_token(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Unauthed M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is False
|
||||
assert data["diagnostic_category"] == "auth_required"
|
||||
assert any("Microsoft 365" in step for step in data["recovery_steps"])
|
||||
|
||||
def test_m365_source_test_with_valid_token(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Authed M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.test_connection.return_value = {
|
||||
"success": True,
|
||||
"message_count": 1,
|
||||
"diagnostic_detail": "ok",
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = None
|
||||
|
||||
with (
|
||||
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
):
|
||||
mock_source = MagicMock()
|
||||
mock_source.method = "M365_GRAPH"
|
||||
mock_source.m365_access_token = "valid-token"
|
||||
mock_source.m365_email = "dmarc@example.com"
|
||||
mock_source.m365_tenant_id = "organizations"
|
||||
mock_source.m365_client_id = "client-id"
|
||||
mock_source.m365_client_secret = "client-secret"
|
||||
mock_source.m365_refresh_token = "refresh-token"
|
||||
mock_source.m365_mailbox = None
|
||||
mock_source.folder = "INBOX"
|
||||
mock_get.return_value = mock_source
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert "valid" in data["message"].lower()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_error", "category"),
|
||||
[
|
||||
("invalid_grant: refresh token expired", "auth_expired"),
|
||||
("Forbidden: Mail.Read permission is missing", "permissions"),
|
||||
("429 Too Many Requests throttling limit", "throttling"),
|
||||
],
|
||||
)
|
||||
def test_m365_source_test_diagnostic_categories(
|
||||
self, authed_client: TestClient, provider_error: str, category: str
|
||||
):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "M365 Diagnostics", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
mock_client = MagicMock()
|
||||
mock_client.test_connection.side_effect = RuntimeError(provider_error)
|
||||
|
||||
with (
|
||||
patch("app.api.api_v1.endpoints.mail_sources._get_source_or_404") as mock_get,
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
):
|
||||
mock_source = MagicMock()
|
||||
mock_source.method = "M365_GRAPH"
|
||||
mock_source.m365_access_token = "valid-token"
|
||||
mock_source.m365_email = "dmarc@example.com"
|
||||
mock_source.m365_tenant_id = "organizations"
|
||||
mock_source.m365_client_id = "client-id"
|
||||
mock_source.m365_client_secret = "client-secret"
|
||||
mock_source.m365_refresh_token = "refresh-token"
|
||||
mock_source.m365_mailbox = None
|
||||
mock_source.folder = "INBOX"
|
||||
mock_get.return_value = mock_source
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/test")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["diagnostic_category"] == category
|
||||
|
||||
def test_m365_authorize_url_returns_microsoft_url(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Ready M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_tenant_id": "organizations",
|
||||
"m365_client_id": "client-id",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/authorize-url")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
parsed = urlparse(data["authorization_url"])
|
||||
assert parsed.hostname == "login.microsoftonline.com"
|
||||
assert "/organizations/oauth2/v2.0/authorize" in parsed.path
|
||||
query = parse_qs(parsed.query)
|
||||
assert query["client_id"] == ["client-id"]
|
||||
assert "offline_access" in query["scope"][0]
|
||||
assert "https://graph.microsoft.com/Mail.Read" in query["scope"][0]
|
||||
|
||||
def test_m365_authorize_url_validates_source(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Source", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
assert (
|
||||
authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/authorize-url").status_code
|
||||
== 400
|
||||
)
|
||||
|
||||
m365_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "No Client M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
m365_id = m365_resp.json()["id"]
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{m365_id}/m365/authorize-url")
|
||||
assert resp.status_code == 400
|
||||
assert "m365_client_id" in resp.json()["detail"]
|
||||
|
||||
def test_m365_get_callback_handles_error_and_wrong_source(self, authed_client: TestClient):
|
||||
error_resp = authed_client.get("/api/v1/mail-sources/999/m365/callback?error=denied")
|
||||
assert error_resp.status_code == 400
|
||||
assert "authorisation failed" in error_resp.text
|
||||
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Callback", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
wrong_resp = authed_client.get(f"/api/v1/mail-sources/{imap_id}/m365/callback?code=abc")
|
||||
assert wrong_resp.status_code == 404
|
||||
|
||||
def test_m365_get_callback_saves_tokens(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "GET Callback M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"access_token": "access", "refresh_token": "refresh"},
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
|
||||
return_value="dmarc@example.com",
|
||||
),
|
||||
):
|
||||
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert "connected successfully" in resp.text
|
||||
|
||||
def test_m365_get_callback_failure_modes(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "GET Callback Failure", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
side_effect=ValueError("bad code"),
|
||||
):
|
||||
token_error = authed_client.get(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
|
||||
)
|
||||
assert token_error.status_code == 400
|
||||
assert "Token exchange failed" in token_error.text
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"refresh_token": "refresh"},
|
||||
):
|
||||
missing_access = authed_client.get(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback?code=abc"
|
||||
)
|
||||
assert missing_access.status_code == 400
|
||||
assert "No access token" in missing_access.text
|
||||
|
||||
def test_m365_callback_post_saves_tokens(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Callback M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"access_token": "access", "refresh_token": "refresh"},
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.get_account_email",
|
||||
return_value="dmarc@example.com",
|
||||
),
|
||||
):
|
||||
resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{source_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["m365_connected"] is True
|
||||
assert data["m365_email"] == "dmarc@example.com"
|
||||
|
||||
def test_m365_callback_post_failure_modes(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP Callback Post", "method": "IMAP"},
|
||||
)
|
||||
imap_id = imap_resp.json()["id"]
|
||||
wrong_resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{imap_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert wrong_resp.status_code == 400
|
||||
|
||||
m365_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Bad Callback M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
m365_id = m365_resp.json()["id"]
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
side_effect=ValueError("bad code"),
|
||||
):
|
||||
token_resp = authed_client.post(
|
||||
f"/api/v1/mail-sources/{m365_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert token_resp.status_code == 400
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.exchange_code_for_tokens",
|
||||
return_value={"refresh_token": "refresh"},
|
||||
):
|
||||
missing_access = authed_client.post(
|
||||
f"/api/v1/mail-sources/{m365_id}/m365/callback",
|
||||
json={"code": "code", "redirect_uri": "https://example.com/callback"},
|
||||
)
|
||||
assert missing_access.status_code == 400
|
||||
|
||||
def test_m365_fetch_no_token_returns_400(self, authed_client: TestClient):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "No Token M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_m365_fetch_with_mocked_client(self, authed_client: TestClient, db_session: Session):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Fetch M365",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "tok"
|
||||
source.m365_refresh_token = "refresh"
|
||||
db_session.commit()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 3,
|
||||
"reports_found": 2,
|
||||
"duplicate_reports": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"errors": [],
|
||||
"new_ingested_ids": ["id1", "id2", "id3"],
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = None
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.load_ingested_ids",
|
||||
return_value=[],
|
||||
),
|
||||
patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient.dump_ingested_ids",
|
||||
return_value='["id1", "id2", "id3"]',
|
||||
),
|
||||
):
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/fetch")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["processed"] == 3
|
||||
assert data["reports_found"] == 2
|
||||
assert data["duplicate_reports"] == 1
|
||||
assert data["new_domains"] == ["example.com"]
|
||||
|
||||
def test_m365_specific_fetch_persists_refreshed_tokens(
|
||||
self, authed_client: TestClient, db_session: Session
|
||||
):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={
|
||||
"name": "Fetch M365 Specific",
|
||||
"method": "M365_GRAPH",
|
||||
"m365_client_id": "client-id",
|
||||
"m365_client_secret": "client-secret",
|
||||
},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "old-access"
|
||||
source.m365_refresh_token = "old-refresh"
|
||||
db_session.commit()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"reports_found": 0,
|
||||
"duplicate_reports": 0,
|
||||
"new_domains": [],
|
||||
"errors": ["temporary warning"],
|
||||
"new_ingested_ids": [],
|
||||
}
|
||||
mock_client.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-access",
|
||||
"refresh_token": "new-refresh",
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
|
||||
return_value=mock_client,
|
||||
):
|
||||
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
|
||||
db_session.refresh(source)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["processed"] == 1
|
||||
assert source.m365_access_token == "new-access"
|
||||
assert source.m365_refresh_token == "new-refresh"
|
||||
|
||||
def test_m365_fetch_and_disconnect_validate_method(self, authed_client: TestClient):
|
||||
imap_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "IMAP M365 Actions", "method": "IMAP"},
|
||||
)
|
||||
source_id = imap_resp.json()["id"]
|
||||
|
||||
fetch_resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/m365/fetch")
|
||||
disconnect_resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
|
||||
|
||||
assert fetch_resp.status_code == 400
|
||||
assert disconnect_resp.status_code == 400
|
||||
|
||||
def test_m365_disconnect_clears_tokens(self, authed_client: TestClient, db_session: Session):
|
||||
create_resp = authed_client.post(
|
||||
"/api/v1/mail-sources",
|
||||
json={"name": "Disconnect M365", "method": "M365_GRAPH"},
|
||||
)
|
||||
source_id = create_resp.json()["id"]
|
||||
source = db_session.get(MailSource, source_id)
|
||||
source.m365_access_token = "tok"
|
||||
source.m365_refresh_token = "refresh"
|
||||
source.m365_email = "dmarc@example.com"
|
||||
db_session.commit()
|
||||
|
||||
resp = authed_client.delete(f"/api/v1/mail-sources/{source_id}/m365/connection")
|
||||
|
||||
db_session.refresh(source)
|
||||
assert resp.status_code == 204
|
||||
assert source.m365_access_token is None
|
||||
assert source.m365_refresh_token is None
|
||||
assert source.m365_email is None
|
||||
|
||||
|
||||
class TestManualSourceFetchEndpoint:
|
||||
"""Tests for POST /api/v1/mail-sources/{source_id}/fetch."""
|
||||
|
||||
@@ -2035,6 +2514,79 @@ class TestTriggerPollGmailSource:
|
||||
assert src.gmail_refresh_token == "new-ref"
|
||||
|
||||
|
||||
class TestTriggerPollM365Source:
|
||||
"""Unit tests for app.main._trigger_poll_m365_source."""
|
||||
|
||||
def _make_src(self):
|
||||
src = MagicMock()
|
||||
src.id = 8
|
||||
src.name = "My M365"
|
||||
src.m365_tenant_id = "organizations"
|
||||
src.m365_client_id = "cid"
|
||||
src.m365_client_secret = "csec"
|
||||
src.m365_access_token = "tok"
|
||||
src.m365_refresh_token = "ref"
|
||||
src.m365_mailbox = "shared@example.com"
|
||||
src.m365_ingested_ids = "[]"
|
||||
src.folder = "INBOX"
|
||||
return src
|
||||
|
||||
def test_returns_result_dict_on_success(self):
|
||||
from app.main import _trigger_poll_m365_source
|
||||
|
||||
src = self._make_src()
|
||||
mock_gc = MagicMock()
|
||||
mock_gc.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 2,
|
||||
"reports_found": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"new_ingested_ids": ["id1"],
|
||||
}
|
||||
mock_gc.get_refreshed_tokens.return_value = None
|
||||
mock_db = MagicMock()
|
||||
|
||||
with (
|
||||
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
patch("app.main.MicrosoftGraphClient.dump_ingested_ids", return_value='["id1"]'),
|
||||
):
|
||||
result = _trigger_poll_m365_source(src, mock_db)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source_id"] == 8
|
||||
assert result["processed"] == 2
|
||||
assert src.m365_ingested_ids == '["id1"]'
|
||||
mock_db.commit.assert_called_once()
|
||||
|
||||
def test_persists_refreshed_tokens(self):
|
||||
from app.main import _trigger_poll_m365_source
|
||||
|
||||
src = self._make_src()
|
||||
mock_gc = MagicMock()
|
||||
mock_gc.fetch_reports.return_value = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"reports_found": 0,
|
||||
"new_domains": [],
|
||||
"new_ingested_ids": [],
|
||||
}
|
||||
mock_gc.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-acc",
|
||||
"refresh_token": "new-ref",
|
||||
}
|
||||
mock_db = MagicMock()
|
||||
|
||||
with (
|
||||
patch("app.main.MicrosoftGraphClient", return_value=mock_gc),
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
):
|
||||
_trigger_poll_m365_source(src, mock_db)
|
||||
|
||||
assert src.m365_access_token == "new-acc"
|
||||
assert src.m365_refresh_token == "new-ref"
|
||||
|
||||
|
||||
class TestPollSourceForTrigger:
|
||||
"""Unit tests for app.main._poll_source_for_trigger."""
|
||||
|
||||
@@ -2083,6 +2635,51 @@ class TestPollSourceForTrigger:
|
||||
assert result["success"] is False
|
||||
assert "boom" not in result.get("error", "") # raw msg not exposed
|
||||
|
||||
def test_m365_no_token_returns_skipped(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = None
|
||||
src.id = 7
|
||||
src.name = "M365 no token"
|
||||
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result["skipped"] is True
|
||||
assert "microsoft 365" in result["reason"].lower()
|
||||
|
||||
def test_m365_with_token_delegates_to_trigger_poll(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = "tok"
|
||||
src.id = 8
|
||||
src.name = "M365"
|
||||
|
||||
expected = {"source_id": 8, "name": "M365", "success": True}
|
||||
with patch("app.main._trigger_poll_m365_source", return_value=expected) as mock_fn:
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result is expected
|
||||
mock_fn.assert_called_once()
|
||||
|
||||
def test_m365_exception_returns_failure_dict(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
src = MagicMock()
|
||||
src.method = "M365_GRAPH"
|
||||
src.m365_access_token = "tok"
|
||||
src.id = 9
|
||||
src.name = "M365 exc"
|
||||
|
||||
with patch("app.main._trigger_poll_m365_source", side_effect=Exception("boom")):
|
||||
result = _poll_source_for_trigger(src, MagicMock())
|
||||
|
||||
assert result["success"] is False
|
||||
assert "boom" not in result.get("error", "")
|
||||
|
||||
def test_imap_delegates_to_trigger_poll(self):
|
||||
from app.main import _poll_source_for_trigger
|
||||
|
||||
@@ -2147,6 +2744,25 @@ class TestPollAllEnabledSources:
|
||||
|
||||
mock_gmail.assert_called_once_with(src)
|
||||
|
||||
def test_dispatches_m365_graph_source(self):
|
||||
"""M365_GRAPH sources are forwarded to _poll_single_m365_source."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
src = MagicMock()
|
||||
src.id = 6
|
||||
src.method = "M365_GRAPH"
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.return_value.filter.return_value.all.return_value = [src]
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=mock_db),
|
||||
patch("app.main._poll_single_m365_source") as mock_m365,
|
||||
):
|
||||
_poll_all_enabled_sources()
|
||||
|
||||
mock_m365.assert_called_once_with(src)
|
||||
|
||||
def test_dispatches_imap_source(self):
|
||||
"""IMAP sources are forwarded to _poll_single_imap_source."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
@@ -2200,6 +2816,23 @@ class TestPollAllEnabledSources:
|
||||
):
|
||||
_poll_all_enabled_sources() # should not raise
|
||||
|
||||
def test_m365_exception_is_caught(self):
|
||||
"""Exception from _poll_single_m365_source must not propagate."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
src = MagicMock()
|
||||
src.id = 7
|
||||
src.method = "M365_GRAPH"
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.return_value.filter.return_value.all.return_value = [src]
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=mock_db),
|
||||
patch("app.main._poll_single_m365_source", side_effect=Exception("m365 crash")),
|
||||
):
|
||||
_poll_all_enabled_sources() # should not raise
|
||||
|
||||
def test_unknown_method_skipped(self):
|
||||
"""An unknown method logs a skip message and does not raise."""
|
||||
from app.main import _poll_all_enabled_sources
|
||||
|
||||
@@ -86,6 +86,68 @@ def test_poll_single_imap_source_passes_configured_folder():
|
||||
db.close.assert_called_once()
|
||||
|
||||
|
||||
def test_poll_single_m365_source_persists_import_state():
|
||||
from app.main import _poll_single_m365_source
|
||||
|
||||
source = SimpleNamespace(
|
||||
id=2,
|
||||
m365_access_token="tok",
|
||||
m365_tenant_id="organizations",
|
||||
m365_client_id="cid",
|
||||
m365_client_secret="csec",
|
||||
m365_refresh_token="ref",
|
||||
m365_mailbox="shared@example.com",
|
||||
m365_ingested_ids="[]",
|
||||
folder="INBOX",
|
||||
)
|
||||
db_source = SimpleNamespace(**source.__dict__)
|
||||
db = MagicMock()
|
||||
db.query.return_value.get.return_value = db_source
|
||||
results = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"reports_found": 1,
|
||||
"new_domains": ["example.com"],
|
||||
"new_ingested_ids": ["message-1"],
|
||||
}
|
||||
|
||||
with (
|
||||
patch("app.main.SessionLocal", return_value=db),
|
||||
patch("app.main.MicrosoftGraphClient") as mock_client_cls,
|
||||
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
|
||||
patch(
|
||||
"app.main.MicrosoftGraphClient.dump_ingested_ids",
|
||||
return_value='["message-1"]',
|
||||
),
|
||||
patch("app.main.record_import_attempt"),
|
||||
):
|
||||
mock_client = mock_client_cls.return_value
|
||||
mock_client.fetch_reports.return_value = results
|
||||
mock_client.get_refreshed_tokens.return_value = {
|
||||
"access_token": "new-tok",
|
||||
"refresh_token": "new-ref",
|
||||
}
|
||||
|
||||
_poll_single_m365_source(source)
|
||||
|
||||
assert db_source.m365_ingested_ids == '["message-1"]'
|
||||
assert db_source.m365_access_token == "new-tok"
|
||||
assert db_source.m365_refresh_token == "new-ref"
|
||||
db.commit.assert_called_once()
|
||||
db.close.assert_called_once()
|
||||
|
||||
|
||||
def test_poll_single_m365_source_skips_without_token():
|
||||
from app.main import _poll_single_m365_source
|
||||
|
||||
source = SimpleNamespace(id=3, m365_access_token=None)
|
||||
|
||||
with patch("app.main.SessionLocal") as mock_session:
|
||||
_poll_single_m365_source(source)
|
||||
|
||||
mock_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scheduled_imap_polling_sleep_exception_falls_back_then_cancels():
|
||||
from app.main import scheduled_imap_polling
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Unit tests for app.services.microsoft_graph_client.MicrosoftGraphClient."""
|
||||
|
||||
import base64
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
import httpx
|
||||
|
||||
from app.models.report import DMARCReport
|
||||
from app.services.microsoft_graph_client import MicrosoftGraphClient
|
||||
from app.tests.test_data import SAMPLE_XML
|
||||
|
||||
|
||||
def _zip_xml(xml: str = SAMPLE_XML, name: str = "report.xml") -> bytes:
|
||||
buf = BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr(name, xml.encode("utf-8"))
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_client(db=None, already_ingested=None) -> MicrosoftGraphClient:
|
||||
return MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
mailbox=None,
|
||||
folder="INBOX",
|
||||
already_ingested_ids=already_ingested or [],
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
class TestMicrosoftGraphOAuthHelpers:
|
||||
def test_build_authorization_url_includes_required_scopes(self):
|
||||
url = MicrosoftGraphClient.build_authorization_url(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
redirect_uri="https://example.com/callback",
|
||||
state="42",
|
||||
)
|
||||
|
||||
assert url.startswith("https://login.microsoftonline.com/organizations/")
|
||||
assert "client_id=client-id" in url
|
||||
assert "response_type=code" in url
|
||||
assert "offline_access" in url
|
||||
assert "User.Read" in url
|
||||
assert "Mail.Read" in url
|
||||
assert "state=42" in url
|
||||
|
||||
def test_exchange_code_for_tokens_posts_to_tenant_endpoint(self, monkeypatch):
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
assert url.endswith("/contoso-tenant/oauth2/v2.0/token")
|
||||
assert data["grant_type"] == "authorization_code"
|
||||
assert data["code"] == "code"
|
||||
return httpx.Response(200, json={"access_token": "access"})
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
|
||||
result = MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id="contoso-tenant",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
code="code",
|
||||
redirect_uri="https://example.com/callback",
|
||||
)
|
||||
|
||||
assert result == {"access_token": "access"}
|
||||
|
||||
def test_exchange_code_for_tokens_raises_on_failure(self, monkeypatch):
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
return httpx.Response(400, text="bad request")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
|
||||
try:
|
||||
MicrosoftGraphClient.exchange_code_for_tokens(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
code="bad",
|
||||
redirect_uri="https://example.com/callback",
|
||||
)
|
||||
except Exception as exc:
|
||||
assert "token exchange failed" in str(exc).lower()
|
||||
else:
|
||||
raise AssertionError("expected token exchange failure")
|
||||
|
||||
def test_get_account_email_prefers_mail_then_user_principal_name(self, monkeypatch):
|
||||
def fake_get(url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/me")
|
||||
assert params == {"$select": "mail,userPrincipalName"}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"mail": "", "userPrincipalName": "dmarc@example.com"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
|
||||
|
||||
assert MicrosoftGraphClient.get_account_email("access") == "dmarc@example.com"
|
||||
|
||||
def test_get_account_email_returns_none_on_error(self, monkeypatch):
|
||||
def fake_get(url, headers=None, params=None, timeout=None):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.get", fake_get)
|
||||
|
||||
assert MicrosoftGraphClient.get_account_email("access") is None
|
||||
|
||||
def test_ingested_id_helpers_tolerate_bad_json(self):
|
||||
assert MicrosoftGraphClient.load_ingested_ids(None) == []
|
||||
assert MicrosoftGraphClient.load_ingested_ids('["a", 2]') == ["a", "2"]
|
||||
assert MicrosoftGraphClient.load_ingested_ids("{bad") == []
|
||||
assert MicrosoftGraphClient.dump_ingested_ids(["a"]) == '["a"]'
|
||||
|
||||
|
||||
class TestMicrosoftGraphFetchReports:
|
||||
def test_test_connection_reads_shared_mailbox(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/users/shared%40example.com/messages")
|
||||
assert params == {"$top": 1, "$select": "id"}
|
||||
return httpx.Response(200, json={"value": [{"id": "message-1"}]})
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
mailbox="shared@example.com",
|
||||
folder="INBOX",
|
||||
)
|
||||
|
||||
result = client.test_connection()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["message_count"] == 1
|
||||
|
||||
def test_fetch_reports_imports_zip_attachment(self, monkeypatch, db_session):
|
||||
attachment_bytes = _zip_xml()
|
||||
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
"receivedDateTime": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "report.zip",
|
||||
"contentBytes": base64.b64encode(attachment_bytes).decode(),
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client(db=db_session)
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["reports_found"] == 1
|
||||
assert result["new_ingested_ids"] == ["message-1"]
|
||||
assert db_session.query(DMARCReport).count() == 1
|
||||
|
||||
def test_fetch_reports_skips_already_ingested_message(self, monkeypatch, db_session):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError("Already-ingested messages should not fetch attachments")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client(db=db_session, already_ingested=["message-1"])
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 0
|
||||
assert result["reports_found"] == 0
|
||||
assert result["new_ingested_ids"] == []
|
||||
|
||||
def test_fetch_reports_handles_empty_message_id_and_plain_messages_path(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
assert url.endswith("/me/messages")
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = MicrosoftGraphClient(
|
||||
tenant_id="organizations",
|
||||
client_id="client-id",
|
||||
client_secret="client-secret",
|
||||
access_token="access-token",
|
||||
refresh_token="refresh-token",
|
||||
folder="",
|
||||
)
|
||||
client.folder = ""
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 0
|
||||
|
||||
def test_fetch_reports_keeps_message_retryable_when_attachments_fail(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
500,
|
||||
json={"error": {"code": "ServerError", "message": "temporary failure"}},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["new_ingested_ids"] == []
|
||||
assert result["details"][0]["reason"] == "attachment_fetch_failed"
|
||||
|
||||
def test_fetch_reports_records_attachment_outcomes(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"id": "message-1",
|
||||
"subject": "DMARC aggregate report",
|
||||
"from": {"emailAddress": {"address": "reports@example.net"}},
|
||||
"hasAttachments": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
if url.endswith("/me/messages/message-1/attachments"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"value": [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "notes.txt",
|
||||
"contentBytes": base64.b64encode(b"not dmarc").decode(),
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.itemAttachment",
|
||||
"name": "report.xml",
|
||||
"contentBytes": base64.b64encode(SAMPLE_XML.encode()).decode(),
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "empty.xml",
|
||||
"contentBytes": "",
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
"name": "bad.xml",
|
||||
"contentBytes": base64.b64encode(b"<not-dmarc>").decode(),
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["processed"] == 1
|
||||
assert result["reports_found"] == 0
|
||||
assert result["errors"]
|
||||
reasons = {detail.get("reason") for detail in result["details"]}
|
||||
assert {
|
||||
"unsupported_attachment",
|
||||
"unsupported_attachment_type",
|
||||
"empty_attachment",
|
||||
"parse_failed",
|
||||
}.issubset(reasons)
|
||||
|
||||
def test_fetch_reports_refreshes_token_after_unauthorized(self, monkeypatch):
|
||||
calls = {"messages": 0}
|
||||
|
||||
def fake_post(url, data=None, timeout=None):
|
||||
assert url.endswith("/organizations/oauth2/v2.0/token")
|
||||
assert data["grant_type"] == "refresh_token"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"access_token": "new-access", "refresh_token": "new-refresh"},
|
||||
)
|
||||
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
if url.endswith("/me/mailFolders/inbox/messages"):
|
||||
calls["messages"] += 1
|
||||
if calls["messages"] == 1:
|
||||
return httpx.Response(
|
||||
401,
|
||||
json={
|
||||
"error": {
|
||||
"code": "InvalidAuthenticationToken",
|
||||
"message": "Access token has expired.",
|
||||
}
|
||||
},
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer new-access"
|
||||
return httpx.Response(200, json={"value": []})
|
||||
raise AssertionError(f"Unexpected Graph request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.post", fake_post)
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is True
|
||||
assert calls["messages"] == 2
|
||||
assert client.get_refreshed_tokens() == {
|
||||
"access_token": "new-access",
|
||||
"refresh_token": "new-refresh",
|
||||
}
|
||||
|
||||
def test_fetch_reports_surfaces_throttling_error(self, monkeypatch):
|
||||
def fake_request(method, url, headers=None, params=None, timeout=None):
|
||||
return httpx.Response(
|
||||
429,
|
||||
json={"error": {"code": "TooManyRequests", "message": "Request is throttled."}},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
|
||||
client = _make_client()
|
||||
|
||||
result = client.fetch_reports()
|
||||
|
||||
assert result["success"] is False
|
||||
assert "TooManyRequests" in result["error"]
|
||||
Reference in New Issue
Block a user