Merge pull request #182 from christianlouis/codex/m12-m365-mailbox-folder-selection

feat: add M365 shared mailbox folder selection
This commit is contained in:
Christian Krakau-Louis
2026-05-23 15:29:43 +02:00
committed by GitHub
11 changed files with 472 additions and 14 deletions
@@ -0,0 +1,25 @@
"""Add Microsoft 365 folder selection.
Revision ID: c4d5e6f7a8b9
Revises: f3a4b5c6d7e8
Create Date: 2026-05-23 00:00:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "c4d5e6f7a8b9"
down_revision: Union[str, Sequence[str], None] = "f3a4b5c6d7e8"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("mail_sources", sa.Column("m365_folder_id", sa.String(), nullable=True))
def downgrade() -> None:
op.drop_column("mail_sources", "m365_folder_id")
@@ -56,6 +56,7 @@ class MailSourceBase(BaseModel):
m365_client_id: Optional[str] = None
m365_client_secret: Optional[str] = None
m365_mailbox: Optional[str] = None
m365_folder_id: Optional[str] = None
class MailSourceCreate(MailSourceBase):
@@ -81,6 +82,7 @@ class MailSourceUpdate(BaseModel):
m365_client_id: Optional[str] = None
m365_client_secret: Optional[str] = None
m365_mailbox: Optional[str] = None
m365_folder_id: Optional[str] = None
class MailSourceResponse(MailSourceBase):
@@ -372,6 +374,7 @@ def _source_to_response(source: MailSource) -> MailSourceResponse:
m365_client_id=_safe_attr(source, "m365_client_id"),
m365_client_secret=("**redacted**" if _safe_attr(source, "m365_client_secret") else None),
m365_mailbox=_safe_attr(source, "m365_mailbox"),
m365_folder_id=_safe_attr(source, "m365_folder_id"),
m365_email=_safe_attr(source, "m365_email"),
m365_connected=bool(_safe_attr(source, "m365_access_token")),
)
@@ -441,6 +444,8 @@ def _fetch_response(source: MailSource, results: Dict[str, Any]) -> Dict[str, An
"duplicate_forensic_reports": int(results.get("duplicate_forensic_reports", 0)),
"new_domains": [str(d) for d in results.get("new_domains", [])],
"error_count": len(results.get("errors", [])),
"target_mailbox": results.get("target_mailbox"),
"target_folder": results.get("target_folder"),
"timestamp": datetime.now().isoformat(),
}
@@ -499,6 +504,7 @@ def _fetch_m365_source(source: MailSource, db: Session) -> Dict[str, Any]:
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
folder_id=_safe_attr(source, "m365_folder_id"),
already_ingested_ids=already,
db=db,
)
@@ -593,6 +599,7 @@ async def create_mail_source(
m365_client_id=payload.m365_client_id,
m365_client_secret=payload.m365_client_secret,
m365_mailbox=payload.m365_mailbox,
m365_folder_id=payload.m365_folder_id,
)
db.add(source)
db.commit()
@@ -778,6 +785,7 @@ async def test_stored_mail_source( # noqa: C901
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
folder_id=_safe_attr(source, "m365_folder_id"),
)
stats = graph_client.test_connection()
refreshed = graph_client.get_refreshed_tokens()
@@ -892,6 +900,68 @@ async def m365_authorize_url(
return {"authorization_url": auth_url, "redirect_uri": redirect_uri}
@router.get("/{source_id}/m365/folders", response_model=Dict[str, Any])
async def m365_list_folders(
source_id: int,
db: Session = Depends(get_db),
_auth: dict = Depends(require_admin_auth),
) -> Dict[str, Any]:
"""Return selectable Microsoft 365 mail folders for this source."""
source = _get_source_or_404(source_id, db)
if source.method != "M365_GRAPH":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This endpoint is only available for M365_GRAPH sources.",
)
if not source.m365_access_token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Microsoft 365 account not yet authorised. Complete OAuth2 flow first.",
)
graph_client = MicrosoftGraphClient(
tenant_id=source.m365_tenant_id or "common",
client_id=source.m365_client_id or "",
client_secret=source.m365_client_secret or "",
access_token=source.m365_access_token,
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
folder_id=_safe_attr(source, "m365_folder_id"),
)
try:
folders = graph_client.list_mail_folders()
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(
"Microsoft 365 folder listing failed for source id=%d: %s",
int(source_id),
_redact_sensitive_text(exc),
)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=(
"Could not load Microsoft 365 folders. Confirm the authorised "
"account can read the selected mailbox."
),
) from exc
refreshed = graph_client.get_refreshed_tokens()
if refreshed:
source.m365_access_token = refreshed["access_token"]
if "refresh_token" in refreshed:
source.m365_refresh_token = refreshed["refresh_token"]
db.commit()
return {
"target_mailbox": source.m365_mailbox or source.m365_email or "authorized account",
"selected_folder_id": _safe_attr(source, "m365_folder_id"),
"selected_folder": source.folder or "INBOX",
"folders": folders,
}
@router.get("/{source_id}/m365/callback")
async def m365_oauth_callback(
source_id: int,
+2
View File
@@ -187,6 +187,7 @@ def _poll_single_m365_source(source: MailSource) -> None:
refresh_token=poll_source.m365_refresh_token or "",
mailbox=poll_source.m365_mailbox,
folder=poll_source.folder or "INBOX",
folder_id=getattr(poll_source, "m365_folder_id", None),
already_ingested_ids=already,
db=db,
)
@@ -768,6 +769,7 @@ def _trigger_poll_m365_source(source: MailSource, db) -> dict:
refresh_token=source.m365_refresh_token or "",
mailbox=source.m365_mailbox,
folder=source.folder or "INBOX",
folder_id=getattr(source, "m365_folder_id", None),
already_ingested_ids=already,
db=db,
)
+2
View File
@@ -57,6 +57,8 @@ class MailSource(Base):
_m365_refresh_token = Column("m365_refresh_token", Text, nullable=True)
# Optional user/shared mailbox to poll. Empty means the authorised account (/me).
m365_mailbox = Column(String, nullable=True)
# Optional Microsoft Graph mailFolder id. Empty means use ``folder`` as a well-known name.
m365_folder_id = Column(String, nullable=True)
# Email address reported by Microsoft Graph for the authorised account.
m365_email = Column(String, nullable=True)
# JSON-encoded list of Graph message IDs that have already been ingested
+2
View File
@@ -19,6 +19,8 @@ DETAIL_FIELDS = {
"filename",
"domain",
"report_id",
"mailbox",
"folder",
"error",
}
+74 -4
View File
@@ -24,6 +24,7 @@ M365_SCOPES = [
]
_PAGE_SIZE = 100
_MAX_FOLDER_DEPTH = 5
_DMARC_SUBJECT_TERMS = (
"dmarc",
"aggregate report",
@@ -61,6 +62,7 @@ class MicrosoftGraphClient:
refresh_token: str,
mailbox: Optional[str] = None,
folder: str = "inbox",
folder_id: Optional[str] = None,
already_ingested_ids: Optional[List[str]] = None,
db: Any = None,
):
@@ -71,6 +73,7 @@ class MicrosoftGraphClient:
self.refresh_token = refresh_token
self.mailbox = (mailbox or "").strip()
self.folder = folder or "inbox"
self.folder_id = (folder_id or "").strip()
self.already_ingested_ids: List[str] = list(already_ingested_ids or [])
self.report_store = ReportStore.get_instance()
self.db = db
@@ -160,18 +163,70 @@ class MicrosoftGraphClient:
def test_connection(self) -> Dict[str, Any]:
"""Verify that the saved delegated token can read the target mailbox."""
mailbox_path = self._mailbox_path()
data = self._request(
"GET",
f"{mailbox_path}/messages",
self._messages_path(),
params={"$top": 1, "$select": "id"},
)
return {
"success": True,
"message_count": len(data.get("value", [])),
"target_mailbox": self._target_mailbox_label(),
"target_folder": self._target_folder_label(),
"diagnostic_detail": "Microsoft Graph mailbox read succeeded.",
}
def list_mail_folders(self) -> List[Dict[str, str]]:
"""Return selectable mail folders for the configured mailbox."""
folders: List[Dict[str, str]] = []
self._collect_mail_folders(
f"{self._mailbox_path()}/mailFolders",
folders,
parent_path="",
depth=0,
)
return folders
def _collect_mail_folders(
self,
start_url: str,
folders: List[Dict[str, str]],
*,
parent_path: str,
depth: int,
) -> None:
params: Optional[Dict[str, Any]] = {
"$top": _PAGE_SIZE,
"$select": "id,displayName,parentFolderId,childFolderCount",
}
url: Optional[str] = start_url
while url:
data = self._request("GET", url, params=params)
for folder in data.get("value", []):
folder_id = str(folder.get("id") or "")
display_name = str(folder.get("displayName") or folder_id)
if not folder_id:
continue
folder_path = f"{parent_path} / {display_name}" if parent_path else display_name
folders.append(
{
"id": folder_id,
"display_name": display_name,
"path": folder_path,
"parent_folder_id": str(folder.get("parentFolderId") or ""),
}
)
if int(folder.get("childFolderCount") or 0) > 0 and depth < _MAX_FOLDER_DEPTH:
self._collect_mail_folders(
f"{self._mailbox_path()}/mailFolders/{quote(folder_id, safe='')}/childFolders",
folders,
parent_path=folder_path,
depth=depth + 1,
)
url = data.get("@odata.nextLink")
params = None
def fetch_reports(self) -> Dict[str, Any]:
"""Fetch and ingest DMARC report attachments from Microsoft Graph."""
stats: Dict[str, Any] = {
@@ -185,6 +240,8 @@ class MicrosoftGraphClient:
"errors": [],
"new_ingested_ids": [],
"details": [],
"target_mailbox": self._target_mailbox_label(),
"target_folder": self._target_folder_label(),
}
try:
@@ -223,12 +280,23 @@ class MicrosoftGraphClient:
tenant = quote(tenant_id or "common", safe="")
return f"{LOGIN_BASE_URL}/{tenant}/oauth2/v2.0/token"
@staticmethod
def _append_detail(stats: dict, **detail: str) -> None:
def _append_detail(self, stats: dict, **detail: str) -> None:
detail.setdefault("mailbox", self._target_mailbox_label())
detail.setdefault("folder", self._target_folder_label())
stats.setdefault("details", []).append(
{key: value for key, value in detail.items() if value}
)
def _target_mailbox_label(self) -> str:
return self.mailbox or "authorized account"
def _target_folder_label(self) -> str:
if self.folder:
return self.folder
if self.folder_id:
return self.folder_id
return "All messages"
def _mailbox_path(self) -> str:
if not self.mailbox or self.mailbox.lower() == "me":
return "/me"
@@ -236,6 +304,8 @@ class MicrosoftGraphClient:
def _messages_path(self) -> str:
mailbox_path = self._mailbox_path()
if self.folder_id:
return f"{mailbox_path}/mailFolders/{quote(self.folder_id, safe='')}/messages"
folder = (self.folder or "").strip()
if not folder:
return f"{mailbox_path}/messages"
+75 -4
View File
@@ -17,7 +17,7 @@
<h1 class="text-2xl font-bold">Mail Sources</h1>
<p class="text-muted-foreground mt-1">
Manage inbox accounts used to automatically retrieve DMARC reports.
Multiple accounts and connection methods (IMAP, POP3, Gmail API) are supported.
Multiple accounts and connection methods (IMAP, POP3, Gmail API, Microsoft 365) are supported.
</p>
</div>
<button class="btn btn-default btn-md" x-on:click="openAddForm()">
@@ -174,7 +174,7 @@
<div>
{% call card_title() %}Import History{% endcall %}
{% call card_description() %}
<span x-text="historySource ? historySource.name : ''"></span>
<span x-text="historySource ? `${historySource.name} • ${sourceTargetLabel(historySource)}` : ''"></span>
{% endcall %}
</div>
<button class="btn btn-ghost btn-sm" x-on:click="closeHistory()">Close</button>
@@ -242,6 +242,8 @@
<span class="font-mono" x-show="detail.report_id" x-text="detail.report_id"></span>
</div>
<div class="mt-1 text-muted-foreground">
<span x-show="detail.mailbox || detail.folder" x-text="`${detail.mailbox || 'mailbox'} / ${detail.folder || 'folder'}`"></span>
<span x-show="(detail.mailbox || detail.folder) && (detail.filename || detail.domain || detail.reason || detail.error)"></span>
<span x-show="detail.filename" x-text="detail.filename"></span>
<span x-show="detail.domain" x-text="` • ${detail.domain}`"></span>
<span x-show="detail.reason" x-text="` • ${(detail.reason || '').replaceAll('_', ' ')}`"></span>
@@ -497,8 +499,26 @@
<div>
<label class="label"><span class="label-text font-medium">Folder</span></label>
<div class="flex flex-col gap-2 md:flex-row">
<select class="select select-bordered w-full md:flex-1"
x-model="form.m365_folder_id"
x-on:change="applyM365FolderSelection($event.target.value)">
<option value="">Use folder name below</option>
<template x-for="folder in m365Folders" :key="folder.id">
<option :value="folder.id" x-text="folder.path || folder.display_name"></option>
</template>
</select>
<button type="button" class="btn btn-outline"
x-on:click="loadM365Folders()"
:disabled="!editingId || !m365Connected || m365FoldersLoading">
<span x-text="m365FoldersLoading ? 'Loading…' : 'Load folders'"></span>
</button>
</div>
<input type="text" x-model="form.folder" placeholder="INBOX"
class="input input-bordered w-full" />
x-on:input="form.m365_folder_id = ''"
class="input input-bordered w-full mt-2" />
<p class="text-xs text-muted-foreground mt-1">Use a listed folder when available; otherwise enter a well-known folder such as INBOX.</p>
<p class="text-xs text-error mt-1" x-show="m365FoldersError" x-text="m365FoldersError"></p>
</div>
<template x-if="editingId && m365Connected">
@@ -666,6 +686,9 @@ function mailSourcesApp() {
gmailEmail: '',
m365Connected: false,
m365Email: '',
m365Folders: [],
m365FoldersLoading: false,
m365FoldersError: '',
form: {
name: '',
@@ -684,6 +707,7 @@ function mailSourcesApp() {
m365_client_id: '',
m365_client_secret: '',
m365_mailbox: '',
m365_folder_id: '',
},
async init() {
@@ -787,11 +811,20 @@ function mailSourcesApp() {
return source.gmail_email || '—';
}
if (source.method === 'M365_GRAPH') {
return source.m365_mailbox || source.m365_email || '';
return source.m365_mailbox || source.m365_email || 'authorized account';
}
return source.server ? `${source.server}:${source.port}` : '—';
},
sourceTargetLabel(source) {
if (!source) return '—';
if (source.method === 'M365_GRAPH') {
const mailbox = source.m365_mailbox || source.m365_email || 'authorized account';
return `${mailbox} / ${source.folder || 'INBOX'}`;
}
return this.sourceAccountLabel(source);
},
sourceStatusLabel(source) {
if (source.method === 'GMAIL_API') {
return source.gmail_connected ? 'Connected' : 'Not authorised';
@@ -853,6 +886,8 @@ function mailSourcesApp() {
this.gmailEmail = '';
this.m365Connected = false;
this.m365Email = '';
this.m365Folders = [];
this.m365FoldersError = '';
this.form = {
name: '',
method: 'IMAP',
@@ -870,6 +905,7 @@ function mailSourcesApp() {
m365_client_id: '',
m365_client_secret: '',
m365_mailbox: '',
m365_folder_id: '',
};
this.testResult = this.emptyTestResult();
this.showForm = true;
@@ -881,6 +917,8 @@ function mailSourcesApp() {
this.gmailEmail = source.gmail_email || '';
this.m365Connected = source.m365_connected || false;
this.m365Email = source.m365_email || '';
this.m365Folders = [];
this.m365FoldersError = '';
this.form = {
name: source.name,
method: source.method,
@@ -898,6 +936,7 @@ function mailSourcesApp() {
m365_client_id: source.m365_client_id || '',
m365_client_secret: '', // never pre-fill client secret
m365_mailbox: source.m365_mailbox || '',
m365_folder_id: source.m365_folder_id || '',
};
this.testResult = this.emptyTestResult();
this.showForm = true;
@@ -906,6 +945,8 @@ function mailSourcesApp() {
closeForm() {
this.showForm = false;
this.editingId = null;
this.m365Folders = [];
this.m365FoldersError = '';
this.testResult = this.emptyTestResult();
},
@@ -976,6 +1017,7 @@ function mailSourcesApp() {
if (updated) {
this.m365Connected = updated.m365_connected || false;
this.m365Email = updated.m365_email || '';
this.form.m365_folder_id = updated.m365_folder_id || this.form.m365_folder_id || '';
if (this.m365Connected) {
this.feedback = {
message: `Microsoft 365 connected successfully (${this.m365Email || 'account connected'}).`,
@@ -990,6 +1032,35 @@ function mailSourcesApp() {
}
},
applyM365FolderSelection(folderId) {
if (!folderId) return;
const selected = this.m365Folders.find(folder => folder.id === folderId);
if (selected) {
this.form.folder = selected.display_name || 'INBOX';
}
},
async loadM365Folders() {
if (!this.editingId) return;
this.m365FoldersLoading = true;
this.m365FoldersError = '';
try {
const resp = await fetch(`/api/v1/mail-sources/${this.editingId}/m365/folders`);
const data = await resp.json();
if (!resp.ok) {
throw new Error(data.detail || 'Could not load Microsoft 365 folders');
}
this.m365Folders = data.folders || [];
if (this.m365Folders.length === 0) {
this.m365FoldersError = 'No selectable folders were returned for this mailbox.';
}
} catch (e) {
this.m365FoldersError = e.message;
} finally {
this.m365FoldersLoading = false;
}
},
async saveSource() {
this.isSaving = true;
this.feedback = { message: '', type: '' };
+117 -1
View File
@@ -246,7 +246,13 @@ class TestMailSourceImportModel:
"errors": ["x" * 600],
"details": [
"skip-me",
{"status": "imported", "filename": "a" * 400, "ignored": "secret"},
{
"status": "imported",
"filename": "a" * 400,
"mailbox": "shared@example.com",
"folder": "DMARC Reports",
"ignored": "secret",
},
],
},
started_at=datetime.utcnow(),
@@ -258,6 +264,8 @@ class TestMailSourceImportModel:
assert len(errors[0]) == 500
assert details[0]["status"] == "imported"
assert len(details[0]["filename"]) == 300
assert details[0]["mailbox"] == "shared@example.com"
assert details[0]["folder"] == "DMARC Reports"
assert "ignored" not in details[0]
def test_record_import_attempt_redacts_sensitive_values(self, db_session: Session):
@@ -1269,6 +1277,112 @@ class TestMicrosoft365GraphMailSource:
assert data["m365_connected"] is True
assert data["m365_email"] == "dmarc@example.com"
def test_m365_create_and_update_include_folder_id(
self, authed_client: TestClient, db_session: Session
):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Folder M365",
"method": "M365_GRAPH",
"folder": "DMARC Reports",
"m365_client_id": "client-id",
"m365_client_secret": "client-secret",
"m365_mailbox": "shared@example.com",
"m365_folder_id": "folder-id",
},
)
assert create_resp.status_code == 201
data = create_resp.json()
assert data["m365_mailbox"] == "shared@example.com"
assert data["m365_folder_id"] == "folder-id"
source = db_session.get(MailSource, data["id"])
assert source.m365_folder_id == "folder-id"
update_resp = authed_client.put(
f"/api/v1/mail-sources/{data['id']}",
json={"folder": "Inbox", "m365_folder_id": ""},
)
assert update_resp.status_code == 200
db_session.refresh(source)
assert update_resp.json()["m365_folder_id"] == ""
assert source.m365_folder_id == ""
def test_m365_list_folders_returns_selectable_folders(
self, authed_client: TestClient, db_session: Session
):
create_resp = authed_client.post(
"/api/v1/mail-sources",
json={
"name": "Folder List M365",
"method": "M365_GRAPH",
"folder": "DMARC Reports",
"m365_mailbox": "shared@example.com",
"m365_folder_id": "folder-id",
},
)
source_id = create_resp.json()["id"]
source = db_session.get(MailSource, source_id)
source.m365_access_token = "old-access"
source.m365_refresh_token = "old-refresh"
source.m365_email = "operator@example.com"
db_session.commit()
mock_client = MagicMock()
mock_client.list_mail_folders.return_value = [
{"id": "folder-id", "display_name": "DMARC Reports", "parent_folder_id": ""}
]
mock_client.get_refreshed_tokens.return_value = {
"access_token": "new-access",
"refresh_token": "new-refresh",
}
with patch(
"app.api.api_v1.endpoints.mail_sources.MicrosoftGraphClient",
return_value=mock_client,
) as client_cls:
resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/m365/folders")
assert resp.status_code == 200
data = resp.json()
assert data["target_mailbox"] == "shared@example.com"
assert data["selected_folder"] == "DMARC Reports"
assert data["selected_folder_id"] == "folder-id"
assert data["folders"] == [
{"id": "folder-id", "display_name": "DMARC Reports", "parent_folder_id": ""}
]
client_cls.assert_called_once()
_, kwargs = client_cls.call_args
assert kwargs["mailbox"] == "shared@example.com"
assert kwargs["folder"] == "DMARC Reports"
assert kwargs["folder_id"] == "folder-id"
db_session.refresh(source)
assert source.m365_access_token == "new-access"
assert source.m365_refresh_token == "new-refresh"
def test_m365_list_folders_requires_method_and_token(self, authed_client: TestClient):
imap_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "IMAP Folder List", "method": "IMAP"},
)
wrong_method = authed_client.get(
f"/api/v1/mail-sources/{imap_resp.json()['id']}/m365/folders"
)
assert wrong_method.status_code == 400
m365_resp = authed_client.post(
"/api/v1/mail-sources",
json={"name": "No Token Folder List", "method": "M365_GRAPH"},
)
no_token = authed_client.get(
f"/api/v1/mail-sources/{m365_resp.json()['id']}/m365/folders"
)
assert no_token.status_code == 400
def test_m365_callback_post_failure_modes(self, authed_client: TestClient):
imap_resp = authed_client.post(
"/api/v1/mail-sources",
@@ -1367,6 +1481,8 @@ class TestMicrosoft365GraphMailSource:
assert data["reports_found"] == 2
assert data["duplicate_reports"] == 1
assert data["new_domains"] == ["example.com"]
assert data["target_mailbox"] is None
assert data["target_folder"] is None
def test_m365_specific_fetch_persists_refreshed_tokens(
self, authed_client: TestClient, db_session: Session
@@ -118,7 +118,7 @@ class TestMicrosoftGraphOAuthHelpers:
class TestMicrosoftGraphFetchReports:
def test_test_connection_reads_shared_mailbox(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
assert url.endswith("/users/shared%40example.com/messages")
assert url.endswith("/users/shared%40example.com/mailFolders/inbox/messages")
assert params == {"$top": 1, "$select": "id"}
return httpx.Response(200, json={"value": [{"id": "message-1"}]})
@@ -137,6 +137,95 @@ class TestMicrosoftGraphFetchReports:
assert result["success"] is True
assert result["message_count"] == 1
assert result["target_mailbox"] == "shared@example.com"
assert result["target_folder"] == "INBOX"
def test_list_mail_folders_reads_shared_mailbox(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
assert params == {
"$top": 100,
"$select": "id,displayName,parentFolderId,childFolderCount",
}
if url.endswith("/users/shared%40example.com/mailFolders"):
return httpx.Response(
200,
json={
"value": [
{"id": "inbox-id", "displayName": "Inbox", "childFolderCount": 1},
{"id": "dmarc-id", "displayName": "DMARC Reports"},
]
},
)
if url.endswith("/users/shared%40example.com/mailFolders/inbox-id/childFolders"):
return httpx.Response(
200,
json={"value": [{"id": "nested-id", "displayName": "Nested DMARC"}]},
)
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
mailbox="shared@example.com",
)
assert client.list_mail_folders() == [
{"id": "inbox-id", "display_name": "Inbox", "path": "Inbox", "parent_folder_id": ""},
{
"id": "nested-id",
"display_name": "Nested DMARC",
"path": "Inbox / Nested DMARC",
"parent_folder_id": "",
},
{
"id": "dmarc-id",
"display_name": "DMARC Reports",
"path": "DMARC Reports",
"parent_folder_id": "",
},
]
def test_fetch_reports_uses_selected_folder_id_and_records_context(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/users/shared%40example.com/mailFolders/folder-id/messages"):
return httpx.Response(
200,
json={
"value": [
{
"id": "message-1",
"subject": "DMARC aggregate report",
"from": {"emailAddress": {"address": "reports@example.net"}},
"hasAttachments": True,
}
]
},
)
if url.endswith("/users/shared%40example.com/messages/message-1/attachments"):
return httpx.Response(200, json={"value": []})
raise AssertionError(f"Unexpected Graph request: {method} {url}")
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
mailbox="shared@example.com",
folder="DMARC Reports",
folder_id="folder-id",
)
result = client.fetch_reports()
assert result["processed"] == 1
assert result["target_mailbox"] == "shared@example.com"
assert result["target_folder"] == "DMARC Reports"
def test_fetch_reports_imports_zip_attachment(self, monkeypatch, db_session):
attachment_bytes = _zip_xml()
+1 -1
View File
@@ -204,7 +204,7 @@ Goal: make mailbox ingestion work for the most common enterprise setups without
Planned:
- Microsoft 365 mail source using OAuth (Graph) with least-privilege scopes. Delivered for delegated `User.Read`, `Mail.Read`, and `offline_access` with encrypted token storage, manual import, scheduled polling, UI setup, and operator docs.
- Shared mailbox and folder selection support for DMARC report collection.
- Shared mailbox and folder selection support for DMARC report collection. Delivered with shared mailbox targeting, Microsoft Graph folder listing, folder-id based imports, UI selection, and mailbox/folder context in import history.
- Import-history parity with existing sources (auditable attachment outcomes, duplicates, parse failures). Delivered for Microsoft 365 imports.
- Backfill support with safe throttling and progressive search windows.
- Secret handling mirrors existing guidance (no raw secrets in logs; 1Password-friendly).
+14 -3
View File
@@ -24,17 +24,28 @@ Create an app registration in Microsoft Entra admin center:
4. Leave **Mailbox** empty to read the authorised account, or enter a user principal name for a shared/delegated mailbox that the authorised user can read.
5. Save the source.
6. Use **Connect Microsoft 365** and approve the read-only mailbox access request.
7. Run **Test connection** and **Run import now**.
7. If DMARC reports are delivered to a dedicated folder, click **Load folders** and choose the folder. If folder loading is unavailable, enter a well-known folder name such as `INBOX`.
8. Run **Test connection** and **Run import now**.
## Shared Mailboxes and Folders
DMARQ uses delegated Microsoft Graph access. That means the authorised Microsoft 365 user must be able to read the mailbox and folder you configure:
- For the authorised user's own mailbox, leave **Mailbox** blank.
- For a shared mailbox, enter the shared mailbox user principal name, for example `dmarc-reports@example.com`.
- The authorised user must have mailbox or folder-level read access in Exchange Online before DMARQ can list folders or messages.
- Folder selection stores the Microsoft Graph folder id when you choose a folder from **Load folders**. This is safer for localized or renamed folders than typing a display name.
- If you type a folder manually, use a well-known Graph folder name such as `INBOX` unless your tenant has confirmed a custom folder identifier.
## Import Behavior
DMARQ reads recent messages in the configured folder, filters for messages that look like DMARC reports, downloads Graph `fileAttachment` items, and sends `.xml`, `.zip`, `.gz`, and `.gzip` attachments through the same parser and persistence path used by upload, IMAP, and Gmail imports.
Imported Graph message IDs are stored on the mail source so scheduled polling does not reprocess the same message. Import history records processed messages, imported reports, duplicates, parse failures, and attachment-level details.
Imported Graph message IDs are stored on the mail source so scheduled polling does not reprocess the same message. Import history records processed messages, imported reports, duplicates, parse failures, attachment-level details, and the mailbox/folder target used for the attempt.
## Troubleshooting
- **Not authorised**: reconnect the source from Mail Sources.
- **Permission error**: confirm the app registration has delegated `Mail.Read` and the authorised account can read the target mailbox.
- **Throttling**: wait and retry, or increase the polling interval.
- **Mailbox/folder not found**: leave Mailbox blank for `/me`, use a valid user principal name for delegated/shared mailboxes, and keep the default `INBOX` folder unless reports are delivered elsewhere.
- **Mailbox/folder not found**: leave Mailbox blank for `/me`, use a valid user principal name for delegated/shared mailboxes, confirm the authorised user has access, and reload folders after changing the target mailbox.