From 10b0467424120383a5f6c9f43c7daed66f6eedfb Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Sat, 23 May 2026 15:25:52 +0200 Subject: [PATCH] feat: add m365 shared mailbox folder selection --- .../c4d5e6f7a8b9_add_m365_folder_selection.py | 25 ++++ .../app/api/api_v1/endpoints/mail_sources.py | 70 +++++++++++ backend/app/main.py | 2 + backend/app/models/mail_source.py | 2 + backend/app/services/import_history.py | 2 + .../app/services/microsoft_graph_client.py | 78 +++++++++++- backend/app/templates/mail_sources.html | 79 +++++++++++- backend/app/tests/test_mail_sources.py | 118 +++++++++++++++++- .../app/tests/test_microsoft_graph_client.py | 91 +++++++++++++- docs/milestones.md | 2 +- docs/user_guide/microsoft365.md | 17 ++- 11 files changed, 472 insertions(+), 14 deletions(-) create mode 100644 backend/alembic/versions/c4d5e6f7a8b9_add_m365_folder_selection.py diff --git a/backend/alembic/versions/c4d5e6f7a8b9_add_m365_folder_selection.py b/backend/alembic/versions/c4d5e6f7a8b9_add_m365_folder_selection.py new file mode 100644 index 0000000..4843492 --- /dev/null +++ b/backend/alembic/versions/c4d5e6f7a8b9_add_m365_folder_selection.py @@ -0,0 +1,25 @@ +"""Add Microsoft 365 folder selection. + +Revision ID: c4d5e6f7a8b9 +Revises: f3a4b5c6d7e8 +Create Date: 2026-05-23 00:00:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c4d5e6f7a8b9" +down_revision: Union[str, Sequence[str], None] = "f3a4b5c6d7e8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("mail_sources", sa.Column("m365_folder_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("mail_sources", "m365_folder_id") diff --git a/backend/app/api/api_v1/endpoints/mail_sources.py b/backend/app/api/api_v1/endpoints/mail_sources.py index 5a3ea3b..0490bf9 100644 --- a/backend/app/api/api_v1/endpoints/mail_sources.py +++ b/backend/app/api/api_v1/endpoints/mail_sources.py @@ -56,6 +56,7 @@ class MailSourceBase(BaseModel): m365_client_id: Optional[str] = None m365_client_secret: Optional[str] = None m365_mailbox: Optional[str] = None + m365_folder_id: Optional[str] = None class MailSourceCreate(MailSourceBase): @@ -81,6 +82,7 @@ class MailSourceUpdate(BaseModel): m365_client_id: Optional[str] = None m365_client_secret: Optional[str] = None m365_mailbox: Optional[str] = None + m365_folder_id: Optional[str] = None class MailSourceResponse(MailSourceBase): @@ -372,6 +374,7 @@ def _source_to_response(source: MailSource) -> MailSourceResponse: m365_client_id=_safe_attr(source, "m365_client_id"), m365_client_secret=("**redacted**" if _safe_attr(source, "m365_client_secret") else None), m365_mailbox=_safe_attr(source, "m365_mailbox"), + m365_folder_id=_safe_attr(source, "m365_folder_id"), m365_email=_safe_attr(source, "m365_email"), m365_connected=bool(_safe_attr(source, "m365_access_token")), ) @@ -441,6 +444,8 @@ def _fetch_response(source: MailSource, results: Dict[str, Any]) -> Dict[str, An "duplicate_forensic_reports": int(results.get("duplicate_forensic_reports", 0)), "new_domains": [str(d) for d in results.get("new_domains", [])], "error_count": len(results.get("errors", [])), + "target_mailbox": results.get("target_mailbox"), + "target_folder": results.get("target_folder"), "timestamp": datetime.now().isoformat(), } @@ -499,6 +504,7 @@ def _fetch_m365_source(source: MailSource, db: Session) -> Dict[str, Any]: refresh_token=source.m365_refresh_token or "", mailbox=source.m365_mailbox, folder=source.folder or "INBOX", + folder_id=_safe_attr(source, "m365_folder_id"), already_ingested_ids=already, db=db, ) @@ -593,6 +599,7 @@ async def create_mail_source( m365_client_id=payload.m365_client_id, m365_client_secret=payload.m365_client_secret, m365_mailbox=payload.m365_mailbox, + m365_folder_id=payload.m365_folder_id, ) db.add(source) db.commit() @@ -778,6 +785,7 @@ async def test_stored_mail_source( # noqa: C901 refresh_token=source.m365_refresh_token or "", mailbox=source.m365_mailbox, folder=source.folder or "INBOX", + folder_id=_safe_attr(source, "m365_folder_id"), ) stats = graph_client.test_connection() refreshed = graph_client.get_refreshed_tokens() @@ -892,6 +900,68 @@ async def m365_authorize_url( return {"authorization_url": auth_url, "redirect_uri": redirect_uri} +@router.get("/{source_id}/m365/folders", response_model=Dict[str, Any]) +async def m365_list_folders( + source_id: int, + db: Session = Depends(get_db), + _auth: dict = Depends(require_admin_auth), +) -> Dict[str, Any]: + """Return selectable Microsoft 365 mail folders for this source.""" + source = _get_source_or_404(source_id, db) + + if source.method != "M365_GRAPH": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="This endpoint is only available for M365_GRAPH sources.", + ) + if not source.m365_access_token: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Microsoft 365 account not yet authorised. Complete OAuth2 flow first.", + ) + + graph_client = MicrosoftGraphClient( + tenant_id=source.m365_tenant_id or "common", + client_id=source.m365_client_id or "", + client_secret=source.m365_client_secret or "", + access_token=source.m365_access_token, + refresh_token=source.m365_refresh_token or "", + mailbox=source.m365_mailbox, + folder=source.folder or "INBOX", + folder_id=_safe_attr(source, "m365_folder_id"), + ) + + try: + folders = graph_client.list_mail_folders() + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error( + "Microsoft 365 folder listing failed for source id=%d: %s", + int(source_id), + _redact_sensitive_text(exc), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Could not load Microsoft 365 folders. Confirm the authorised " + "account can read the selected mailbox." + ), + ) from exc + + refreshed = graph_client.get_refreshed_tokens() + if refreshed: + source.m365_access_token = refreshed["access_token"] + if "refresh_token" in refreshed: + source.m365_refresh_token = refreshed["refresh_token"] + db.commit() + + return { + "target_mailbox": source.m365_mailbox or source.m365_email or "authorized account", + "selected_folder_id": _safe_attr(source, "m365_folder_id"), + "selected_folder": source.folder or "INBOX", + "folders": folders, + } + + @router.get("/{source_id}/m365/callback") async def m365_oauth_callback( source_id: int, diff --git a/backend/app/main.py b/backend/app/main.py index 36f5e23..e777d8b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -187,6 +187,7 @@ def _poll_single_m365_source(source: MailSource) -> None: refresh_token=poll_source.m365_refresh_token or "", mailbox=poll_source.m365_mailbox, folder=poll_source.folder or "INBOX", + folder_id=getattr(poll_source, "m365_folder_id", None), already_ingested_ids=already, db=db, ) @@ -768,6 +769,7 @@ def _trigger_poll_m365_source(source: MailSource, db) -> dict: refresh_token=source.m365_refresh_token or "", mailbox=source.m365_mailbox, folder=source.folder or "INBOX", + folder_id=getattr(source, "m365_folder_id", None), already_ingested_ids=already, db=db, ) diff --git a/backend/app/models/mail_source.py b/backend/app/models/mail_source.py index 06376d7..440ed74 100644 --- a/backend/app/models/mail_source.py +++ b/backend/app/models/mail_source.py @@ -57,6 +57,8 @@ class MailSource(Base): _m365_refresh_token = Column("m365_refresh_token", Text, nullable=True) # Optional user/shared mailbox to poll. Empty means the authorised account (/me). m365_mailbox = Column(String, nullable=True) + # Optional Microsoft Graph mailFolder id. Empty means use ``folder`` as a well-known name. + m365_folder_id = Column(String, nullable=True) # Email address reported by Microsoft Graph for the authorised account. m365_email = Column(String, nullable=True) # JSON-encoded list of Graph message IDs that have already been ingested diff --git a/backend/app/services/import_history.py b/backend/app/services/import_history.py index b8270fe..133f894 100644 --- a/backend/app/services/import_history.py +++ b/backend/app/services/import_history.py @@ -19,6 +19,8 @@ DETAIL_FIELDS = { "filename", "domain", "report_id", + "mailbox", + "folder", "error", } diff --git a/backend/app/services/microsoft_graph_client.py b/backend/app/services/microsoft_graph_client.py index eb98773..942442b 100644 --- a/backend/app/services/microsoft_graph_client.py +++ b/backend/app/services/microsoft_graph_client.py @@ -24,6 +24,7 @@ M365_SCOPES = [ ] _PAGE_SIZE = 100 +_MAX_FOLDER_DEPTH = 5 _DMARC_SUBJECT_TERMS = ( "dmarc", "aggregate report", @@ -61,6 +62,7 @@ class MicrosoftGraphClient: refresh_token: str, mailbox: Optional[str] = None, folder: str = "inbox", + folder_id: Optional[str] = None, already_ingested_ids: Optional[List[str]] = None, db: Any = None, ): @@ -71,6 +73,7 @@ class MicrosoftGraphClient: self.refresh_token = refresh_token self.mailbox = (mailbox or "").strip() self.folder = folder or "inbox" + self.folder_id = (folder_id or "").strip() self.already_ingested_ids: List[str] = list(already_ingested_ids or []) self.report_store = ReportStore.get_instance() self.db = db @@ -160,18 +163,70 @@ class MicrosoftGraphClient: def test_connection(self) -> Dict[str, Any]: """Verify that the saved delegated token can read the target mailbox.""" - mailbox_path = self._mailbox_path() data = self._request( "GET", - f"{mailbox_path}/messages", + self._messages_path(), params={"$top": 1, "$select": "id"}, ) return { "success": True, "message_count": len(data.get("value", [])), + "target_mailbox": self._target_mailbox_label(), + "target_folder": self._target_folder_label(), "diagnostic_detail": "Microsoft Graph mailbox read succeeded.", } + def list_mail_folders(self) -> List[Dict[str, str]]: + """Return selectable mail folders for the configured mailbox.""" + folders: List[Dict[str, str]] = [] + self._collect_mail_folders( + f"{self._mailbox_path()}/mailFolders", + folders, + parent_path="", + depth=0, + ) + return folders + + def _collect_mail_folders( + self, + start_url: str, + folders: List[Dict[str, str]], + *, + parent_path: str, + depth: int, + ) -> None: + params: Optional[Dict[str, Any]] = { + "$top": _PAGE_SIZE, + "$select": "id,displayName,parentFolderId,childFolderCount", + } + url: Optional[str] = start_url + + while url: + data = self._request("GET", url, params=params) + for folder in data.get("value", []): + folder_id = str(folder.get("id") or "") + display_name = str(folder.get("displayName") or folder_id) + if not folder_id: + continue + folder_path = f"{parent_path} / {display_name}" if parent_path else display_name + folders.append( + { + "id": folder_id, + "display_name": display_name, + "path": folder_path, + "parent_folder_id": str(folder.get("parentFolderId") or ""), + } + ) + if int(folder.get("childFolderCount") or 0) > 0 and depth < _MAX_FOLDER_DEPTH: + self._collect_mail_folders( + f"{self._mailbox_path()}/mailFolders/{quote(folder_id, safe='')}/childFolders", + folders, + parent_path=folder_path, + depth=depth + 1, + ) + url = data.get("@odata.nextLink") + params = None + def fetch_reports(self) -> Dict[str, Any]: """Fetch and ingest DMARC report attachments from Microsoft Graph.""" stats: Dict[str, Any] = { @@ -185,6 +240,8 @@ class MicrosoftGraphClient: "errors": [], "new_ingested_ids": [], "details": [], + "target_mailbox": self._target_mailbox_label(), + "target_folder": self._target_folder_label(), } try: @@ -223,12 +280,23 @@ class MicrosoftGraphClient: tenant = quote(tenant_id or "common", safe="") return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/token" - @staticmethod - def _append_detail(stats: dict, **detail: str) -> None: + def _append_detail(self, stats: dict, **detail: str) -> None: + detail.setdefault("mailbox", self._target_mailbox_label()) + detail.setdefault("folder", self._target_folder_label()) stats.setdefault("details", []).append( {key: value for key, value in detail.items() if value} ) + def _target_mailbox_label(self) -> str: + return self.mailbox or "authorized account" + + def _target_folder_label(self) -> str: + if self.folder: + return self.folder + if self.folder_id: + return self.folder_id + return "All messages" + def _mailbox_path(self) -> str: if not self.mailbox or self.mailbox.lower() == "me": return "/me" @@ -236,6 +304,8 @@ class MicrosoftGraphClient: def _messages_path(self) -> str: mailbox_path = self._mailbox_path() + if self.folder_id: + return f"{mailbox_path}/mailFolders/{quote(self.folder_id, safe='')}/messages" folder = (self.folder or "").strip() if not folder: return f"{mailbox_path}/messages" diff --git a/backend/app/templates/mail_sources.html b/backend/app/templates/mail_sources.html index df091da..b079f95 100644 --- a/backend/app/templates/mail_sources.html +++ b/backend/app/templates/mail_sources.html @@ -17,7 +17,7 @@

Mail Sources

Manage inbox accounts used to automatically retrieve DMARC reports. - Multiple accounts and connection methods (IMAP, POP3, Gmail API) are supported. + Multiple accounts and connection methods (IMAP, POP3, Gmail API, Microsoft 365) are supported.

@@ -242,6 +242,8 @@
+ + • @@ -497,8 +499,26 @@
+
+ + +
+ x-on:input="form.m365_folder_id = ''" + class="input input-bordered w-full mt-2" /> +

Use a listed folder when available; otherwise enter a well-known folder such as INBOX.

+