feat: add m365 safe backfill windows

This commit is contained in:
Christian Krakau-Louis
2026-05-23 15:42:52 +02:00
parent 78df802a98
commit 8556ea30bf
8 changed files with 122 additions and 23 deletions
+8 -4
View File
@@ -1455,6 +1455,7 @@ class TestMicrosoft365GraphMailSource:
"new_domains": ["example.com"],
"errors": [],
"new_ingested_ids": ["id1", "id2", "id3"],
"search_window_days": 30,
}
mock_client.get_refreshed_tokens.return_value = None
@@ -1472,7 +1473,7 @@ class TestMicrosoft365GraphMailSource:
return_value='["id1", "id2", "id3"]',
),
):
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/fetch")
resp = authed_client.post(f"/api/v1/mail-sources/{source_id}/fetch?days=30")
assert resp.status_code == 200
data = resp.json()
@@ -1483,6 +1484,8 @@ class TestMicrosoft365GraphMailSource:
assert data["new_domains"] == ["example.com"]
assert data["target_mailbox"] is None
assert data["target_folder"] is None
assert data["search_window_days"] == 30
mock_client.fetch_reports.assert_called_once_with(days=30)
def test_m365_specific_fetch_persists_refreshed_tokens(
self, authed_client: TestClient, db_session: Session
@@ -2667,12 +2670,13 @@ class TestTriggerPollM365Source:
patch("app.main.MicrosoftGraphClient.load_ingested_ids", return_value=[]),
patch("app.main.MicrosoftGraphClient.dump_ingested_ids", return_value='["id1"]'),
):
result = _trigger_poll_m365_source(src, mock_db)
result = _trigger_poll_m365_source(src, mock_db, days=30)
assert result["success"] is True
assert result["source_id"] == 8
assert result["processed"] == 2
assert src.m365_ingested_ids == '["id1"]'
mock_gc.fetch_reports.assert_called_once_with(days=30)
mock_db.commit.assert_called_once()
def test_persists_refreshed_tokens(self):
@@ -2776,10 +2780,10 @@ class TestPollSourceForTrigger:
expected = {"source_id": 8, "name": "M365", "success": True}
with patch("app.main._trigger_poll_m365_source", return_value=expected) as mock_fn:
result = _poll_source_for_trigger(src, MagicMock())
result = _poll_source_for_trigger(src, MagicMock(), days=30)
assert result is expected
mock_fn.assert_called_once()
assert mock_fn.call_args.kwargs["days"] == 30
def test_m365_exception_returns_failure_dict(self):
from app.main import _poll_source_for_trigger
+1
View File
@@ -133,6 +133,7 @@ def test_poll_single_m365_source_persists_import_state():
assert db_source.m365_ingested_ids == '["message-1"]'
assert db_source.m365_access_token == "new-tok"
assert db_source.m365_refresh_token == "new-ref"
mock_client.fetch_reports.assert_called_once_with(days=7)
db.commit.assert_called_once()
db.close.assert_called_once()
@@ -192,6 +192,8 @@ class TestMicrosoftGraphFetchReports:
def test_fetch_reports_uses_selected_folder_id_and_records_context(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
if url.endswith("/users/shared%40example.com/mailFolders/folder-id/messages"):
assert "$filter" in params
assert params["$filter"].startswith("receivedDateTime ge ")
return httpx.Response(
200,
json={
@@ -226,6 +228,55 @@ class TestMicrosoftGraphFetchReports:
assert result["processed"] == 1
assert result["target_mailbox"] == "shared@example.com"
assert result["target_folder"] == "DMARC Reports"
assert result["search_window_days"] == 7
def test_fetch_reports_applies_requested_search_window(self, monkeypatch):
def fake_request(method, url, headers=None, params=None, timeout=None):
assert url.endswith("/me/mailFolders/inbox/messages")
assert params["$top"] == 100
assert params["$select"] == "id,subject,from,hasAttachments,receivedDateTime"
assert params["$orderby"] == "receivedDateTime desc"
assert params["$filter"].startswith("receivedDateTime ge ")
return httpx.Response(200, json={"value": []})
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = _make_client()
result = client.fetch_reports(days=30)
assert result["success"] is True
assert result["search_window_days"] == 30
assert result["processed"] == 0
def test_request_retries_graph_throttling(self, monkeypatch):
responses = [
httpx.Response(
429,
headers={"Retry-After": "0"},
json={"error": {"code": "TooManyRequests", "message": "slow down"}},
),
httpx.Response(200, json={"value": []}),
]
sleeps = []
def fake_request(method, url, headers=None, params=None, timeout=None):
return responses.pop(0)
monkeypatch.setattr("app.services.microsoft_graph_client.httpx.request", fake_request)
client = MicrosoftGraphClient(
tenant_id="organizations",
client_id="client-id",
client_secret="client-secret",
access_token="access-token",
refresh_token="refresh-token",
sleep=sleeps.append,
)
result = client.fetch_reports()
assert result["success"] is True
assert sleeps == [0.0]
assert responses == []
def test_fetch_reports_imports_zip_attachment(self, monkeypatch, db_session):
attachment_bytes = _zip_xml()