gitea-admin/gh-christianlouis-dmarq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: import selected fork operational fixes

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-05-22 23:48:00 +02:00
parent 9f5cbdb851
commit 302159aeb6
16 changed files with 507 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/api_v1/api.py
+2
View File
@@ -10,6 +10,7 @@ from app.api.api_v1.endpoints import (
settings,
setup,
stats,
webhook,
)
api_router = APIRouter()
@@ -24,3 +25,4 @@ api_router.include_router(imap.router, prefix="/imap", tags=["imap"])
api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["mail-sources"])
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
api_router.include_router(webhook.router, prefix="/webhook", tags=["webhook"])
backend/app/api/api_v1/endpoints/imap.py
+13 -11
View File
@@ -4,9 +4,9 @@ from typing import Any, Dict, Optional
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
from app.core.database import SessionLocal, get_db
from app.core.database import SessionLocal
from app.core.security import require_admin_auth
from app.services.imap_client import IMAPClient
@@ -24,13 +24,14 @@ class IMAPTestRequest(BaseModel):
ssl: bool = True
def _fetch_imap_reports_background(days: int, delete_emails: bool) -> None:
"""Fetch IMAP reports with a standalone DB session for background imports."""
def _fetch_imap_reports_sync(days: int, delete_emails: Optional[bool]) -> Dict[str, Any]:
"""Fetch IMAP reports with a standalone DB session."""
db = SessionLocal()
try:
imap_client = IMAPClient(delete_emails=delete_emails, db=db)
imap_client.fetch_reports(days=days)
results = imap_client.fetch_reports(days=days)
db.commit()
return results
except Exception:
db.rollback()
raise
@@ -38,6 +39,11 @@ def _fetch_imap_reports_background(days: int, delete_emails: bool) -> None:
db.close()
def _fetch_imap_reports_background(days: int, delete_emails: Optional[bool]) -> None:
"""Fetch IMAP reports from a FastAPI background task."""
_fetch_imap_reports_sync(days, delete_emails)
@router.post("/test-connection")
async def test_imap_connection(
request: IMAPTestRequest,
@@ -73,9 +79,8 @@ async def test_imap_connection(
async def fetch_imap_reports(
background_tasks: BackgroundTasks,
_auth: dict = Depends(require_admin_auth),
db: Session = Depends(get_db),
days: int = 7,
delete_emails: bool = False,
delete_emails: Optional[bool] = None,
) -> Dict[str, Any]:
"""
Fetch DMARC reports from the configured IMAP mailbox
@@ -86,8 +91,6 @@ async def fetch_imap_reports(
if days < 1 or days > 365:
raise HTTPException(status_code=400, detail="Days parameter must be between 1 and 365")
imap_client = IMAPClient(delete_emails=delete_emails, db=db)
# Run in background if it might take a while
if days > 14:
background_tasks.add_task(_fetch_imap_reports_background, days, delete_emails)
@@ -99,8 +102,7 @@ async def fetch_imap_reports(
# Otherwise run immediately
try:
results = imap_client.fetch_reports(days=days)
db.commit()
results = await run_in_threadpool(_fetch_imap_reports_sync, days, delete_emails)
return {
"success": results["success"],
backend/app/api/api_v1/endpoints/mail_sources.py
-1
View File
@@ -287,7 +287,6 @@ def _fetch_imap_source(source: MailSource, db: Session, days: int) -> Dict[str,
port=source.port or 993,
username=source.username,
password=source.password,
delete_emails=False,
folder=source.folder,
db=db,
)
backend/app/api/api_v1/endpoints/webhook.py
+173
View File
@@ -0,0 +1,173 @@
"""Webhook ingestion endpoints for inbound DMARC report emails."""
import base64
import email
import hmac
import logging
from email.header import decode_header
from typing import Any, Dict, Optional
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.database import get_db
from app.core.redaction import sanitize_for_log
from app.services.dmarc_parser import DMARCParser
from app.services.report_persistence import report_exists, save_parsed_report
from app.services.report_store import ReportStore
logger = logging.getLogger(__name__)
router = APIRouter()
class EmailWebhookPayload(BaseModel):
"""Payload for JSON webhook delivery from an email worker."""
raw_email: str
from_address: Optional[str] = None
to_address: Optional[str] = None
subject: Optional[str] = None
def _decode_email_header(header: Optional[str]) -> str:
"""Decode an RFC 2047 email header to display text."""
if not header:
return ""
decoded_parts = []
for text, encoding in decode_header(header):
if isinstance(text, bytes):
decoded_parts.append(text.decode(encoding or "utf-8", errors="replace"))
else:
decoded_parts.append(text)
return " ".join(decoded_parts)
def _is_dmarc_filename(filename: str) -> bool:
lower = filename.lower()
return lower.endswith((".xml", ".zip", ".gz", ".gzip"))
def _require_webhook_secret(x_webhook_secret: Optional[str]) -> None:
settings = get_settings()
if not settings.WEBHOOK_SECRET:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Webhook ingestion is not configured.",
)
if not x_webhook_secret or not hmac.compare_digest(x_webhook_secret, settings.WEBHOOK_SECRET):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid webhook secret.",
)
def _store_report(db: Session, store: ReportStore, report: Dict[str, Any]) -> str:
domain = report.get("domain") or "unknown"
report_id = report.get("report_id") or ""
if report_id and (store.has_report(domain, report_id) or report_exists(db, domain, report_id)):
return "duplicate"
save_parsed_report(db, report)
store.add_report(report)
return "imported"
def _process_email_attachments(msg: email.message.Message, db: Session) -> Dict[str, Any]:
store = ReportStore.get_instance()
results: Dict[str, Any] = {
"reports_found": 0,
"imported": 0,
"duplicates": 0,
"errors": [],
}
for part in msg.walk():
if part.get_content_disposition() != "attachment":
continue
filename = _decode_email_header(part.get_filename())
if not filename or not _is_dmarc_filename(filename):
continue
try:
content = part.get_payload(decode=True)
if not content:
continue
report = DMARCParser.parse_file(content, filename)
outcome = _store_report(db, store, report)
results["reports_found"] += 1
if outcome == "duplicate":
results["duplicates"] += 1
else:
results["imported"] += 1
except Exception as exc: # pylint: disable=broad-exception-caught
logger.warning(
"Webhook failed to process DMARC attachment %s: %s",
sanitize_for_log(filename),
sanitize_for_log(exc),
)
results["errors"].append(filename)
return results
def _subject_from_message(msg: email.message.Message, fallback: Optional[str] = None) -> str:
return fallback or _decode_email_header(msg.get("Subject"))
@router.post("/email")
async def receive_email(
payload: EmailWebhookPayload,
x_webhook_secret: Optional[str] = Header(None),
db: Session = Depends(get_db),
) -> Dict[str, Any]:
"""Receive a base64 encoded raw email from an email worker webhook."""
_require_webhook_secret(x_webhook_secret)
try:
raw_email = base64.b64decode(payload.raw_email, validate=True)
except (ValueError, TypeError) as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="raw_email must be valid base64.",
) from exc
return _handle_raw_email(raw_email, db, subject=payload.subject)
@router.post("/email/raw")
async def receive_raw_email(
request: Request,
x_webhook_secret: Optional[str] = Header(None),
db: Session = Depends(get_db),
) -> Dict[str, Any]:
"""Receive raw RFC 822 email bytes from an email worker webhook."""
_require_webhook_secret(x_webhook_secret)
return _handle_raw_email(await request.body(), db)
def _handle_raw_email(
raw_email: bytes,
db: Session,
*,
subject: Optional[str] = None,
) -> Dict[str, Any]:
try:
msg = email.message_from_bytes(raw_email)
attachment_results = _process_email_attachments(msg, db)
db.commit()
except HTTPException:
raise
except Exception as exc: # pylint: disable=broad-exception-caught
db.rollback()
logger.warning("Webhook failed to process email: %s", sanitize_for_log(exc))
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Error processing email.",
) from exc
return {
"success": True,
"subject": _subject_from_message(msg, subject),
**attachment_results,
}
backend/app/core/config.py
+2
View File
@@ -42,6 +42,7 @@ class Settings(BaseSettings):
IMAP_USERNAME: Optional[str] = None
IMAP_PASSWORD: Optional[str] = None
IMAP_FOLDER: str = "INBOX"
DELETE_IMPORTED_EMAILS: bool = False
# Admin User
FIRST_SUPERUSER: Optional[EmailStr] = None
@@ -50,6 +51,7 @@ class Settings(BaseSettings):
# Optional Cloudflare Integration
CLOUDFLARE_API_TOKEN: Optional[str] = None
CLOUDFLARE_ZONE_ID: Optional[str] = None
WEBHOOK_SECRET: Optional[str] = None
# Admin API Key (optional)
# If set, this key is used directly instead of generating a random one at startup.
backend/app/main.py
+34 -24
View File
@@ -3,11 +3,12 @@ import logging
import os
from datetime import datetime
from fastapi import Depends, FastAPI, Request
from fastapi import Depends, FastAPI, Query, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.concurrency import run_in_threadpool
import app.models.alert # noqa: F401 – ensure AlertHistory table is registered
import app.models.dns_cache # noqa: F401 – ensure DNSCache table is registered
@@ -54,7 +55,6 @@ def _poll_single_imap_source(source: MailSource) -> None:
port=poll_source.port or 993,
username=poll_source.username,
password=poll_source.password,
delete_emails=False,
folder=poll_source.folder,
db=db,
)
@@ -547,7 +547,7 @@ async def health():
# ---------------------------------------------------------------------------
def _trigger_poll_imap_source(source: MailSource, db) -> dict:
def _trigger_poll_imap_source(source: MailSource, db, days: int = 7) -> dict:
"""Poll a single IMAP source and return a result dict for the API response."""
global last_check_time # pylint: disable=global-statement
@@ -556,12 +556,11 @@ def _trigger_poll_imap_source(source: MailSource, db) -> dict:
port=source.port or 993,
username=source.username,
password=source.password,
delete_emails=False,
folder=source.folder,
db=db,
)
started_at = datetime.utcnow()
results = imap_client.fetch_reports(days=7)
results = imap_client.fetch_reports(days=days)
last_check_time = datetime.now()
source.last_checked = datetime.utcnow()
record_import_attempt(db, source, results, started_at=started_at, trigger="manual")
@@ -614,7 +613,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict:
}
def _poll_source_for_trigger(source: MailSource, db) -> dict:
def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict:
"""Dispatch a single mail source for the manual trigger-poll endpoint.
Returns a result/summary dict that is included in the API response.
@@ -639,7 +638,7 @@ def _poll_source_for_trigger(source: MailSource, db) -> dict:
}
if source.method == "IMAP":
try:
return _trigger_poll_imap_source(source, db)
return _trigger_poll_imap_source(source, db, days=days)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error polling mail source id=%d: %s", source.id, str(e))
return {
@@ -656,14 +655,8 @@ def _poll_source_for_trigger(source: MailSource, db) -> dict:
}
# API endpoint to manually trigger IMAP polling
@app.post("/api/v1/admin/trigger-poll")
async def trigger_imap_poll(auth: dict = Depends(require_admin_auth)):
"""
Manually trigger IMAP polling for all enabled mail sources (admin only).
Security: Requires either X-API-Key header or Bearer token
"""
def _poll_enabled_sources_for_trigger(days: int) -> list[dict]:
"""Poll all enabled mail sources for the manual trigger-poll endpoint."""
results_summary = []
db = SessionLocal()
try:
@@ -671,22 +664,39 @@ async def trigger_imap_poll(auth: dict = Depends(require_admin_auth)):
db.query(MailSource).filter(MailSource.enabled == True).all() # noqa: E712
)
if not enabled_sources:
return {
"success": True,
"message": "No enabled mail sources configured.",
"sources_polled": 0,
"authenticated_by": auth.get("auth_type"),
}
for source in enabled_sources:
results_summary.append(_poll_source_for_trigger(source, db))
results_summary.append(_poll_source_for_trigger(source, db, days=days))
finally:
db.close()
return results_summary
# API endpoint to manually trigger IMAP polling
@app.post("/api/v1/admin/trigger-poll")
async def trigger_imap_poll(
auth: dict = Depends(require_admin_auth),
days: int = Query(7, ge=1, le=365, title="Number of days to fetch for IMAP sources"),
):
"""
Manually trigger IMAP polling for all enabled mail sources (admin only).
Security: Requires either X-API-Key header or Bearer token
"""
results_summary = await run_in_threadpool(_poll_enabled_sources_for_trigger, days)
if not results_summary:
return {
"success": True,
"message": "No enabled mail sources configured.",
"sources_polled": 0,
"days": days,
"authenticated_by": auth.get("auth_type"),
}
return {
"success": all(r.get("success", True) for r in results_summary),
"timestamp": last_check_time.isoformat() if last_check_time else None,
"days": days,
"sources": results_summary,
"authenticated_by": auth.get("auth_type"),
}
backend/app/services/imap_client.py
+13 -7
View File
@@ -3,7 +3,7 @@ import imaplib
import logging
from datetime import datetime, timedelta
from email.header import decode_header
from typing import Any, Dict, Tuple
from typing import Any, Dict, Optional, Tuple
from app.core.config import get_settings
from app.services.dmarc_parser import DMARCParser
@@ -25,7 +25,7 @@ class IMAPClient:
port: int = None,
username: str = None,
password: str = None,
delete_emails: bool = False,
delete_emails: Optional[bool] = None,
folder: str = None,
db: Any = None,
):
@@ -37,7 +37,8 @@ class IMAPClient:
port: IMAP server port (if None, uses settings)
username: IMAP username (if None, uses settings)
password: IMAP password (if None, uses settings)
delete_emails: Whether to delete emails after processing (default: False)
delete_emails: Whether to delete emails after successful report imports.
If omitted, uses DELETE_IMPORTED_EMAILS from settings.
folder: IMAP mailbox folder to read (if None, uses settings or INBOX)
db: Optional SQLAlchemy session used to persist imported reports
"""
@@ -50,7 +51,10 @@ class IMAPClient:
self.port = port or settings.IMAP_PORT
self.username = username or settings.IMAP_USERNAME
self.password = password or settings.IMAP_PASSWORD
self.delete_emails = delete_emails
configured_delete = getattr(settings, "DELETE_IMPORTED_EMAILS", False)
if not isinstance(configured_delete, bool):
configured_delete = False
self.delete_emails = configured_delete if delete_emails is None else delete_emails
self.folder = folder or settings_folder or "INBOX"
self.db = db
@@ -170,10 +174,11 @@ class IMAPClient:
reports_found = self._process_attachments(msg, stats, message_id=message_id)
stats["reports_found"] += reports_found
# Mark email as read (and optionally delete)
# Mark DMARC-looking email as read, and delete only after a successful import.
mail.store(email_id, "+FLAGS", "\\Seen")
if self.delete_emails:
if self.delete_emails and reports_found > 0:
mail.store(email_id, "+FLAGS", "\\Deleted")
stats["deleted"] = stats.get("deleted", 0) + 1
stats["processed"] += 1
except Exception as e: # pylint: disable=broad-exception-caught
@@ -206,6 +211,7 @@ class IMAPClient:
"success": True,
"processed": 0,
"reports_found": 0,
"deleted": 0,
"duplicate_reports": 0,
"new_domains": [],
"errors": [],
@@ -243,7 +249,7 @@ class IMAPClient:
self._process_single_email(mail, email_id, stats)
# Actually remove emails marked for deletion
if self.delete_emails:
if self.delete_emails and stats["deleted"] > 0:
mail.expunge()
# Logout
backend/app/tests/test_config.py
+14
View File
@@ -188,6 +188,20 @@ class TestLogtoSettings:
assert settings.LOGTO_SKIP_SSL_VERIFY is True
class TestImapSettings:
def test_delete_imported_emails_defaults_false(self, monkeypatch):
monkeypatch.delenv("DELETE_IMPORTED_EMAILS", raising=False)
settings = Settings()
assert settings.DELETE_IMPORTED_EMAILS is False
def test_delete_imported_emails_reads_env(self, monkeypatch):
monkeypatch.setenv("DELETE_IMPORTED_EMAILS", "true")
settings = Settings()
assert settings.DELETE_IMPORTED_EMAILS is True
class TestProductionStartupSettings:
"""Tests for production-critical settings and startup validation."""
backend/app/tests/test_imap_client.py
+69 -2
View File
@@ -139,6 +139,19 @@ class TestIMAPClientInit:
assert client.password == "secret"
assert client.delete_emails is True
def test_delete_emails_defaults_to_settings(self):
settings = SimpleNamespace(
IMAP_SERVER="imap.example.com",
IMAP_PORT=993,
IMAP_USERNAME="u",
IMAP_PASSWORD="p",
DELETE_IMPORTED_EMAILS=True,
)
with patch("app.services.imap_client.get_settings", return_value=settings):
client = IMAPClient()
assert client.delete_emails is True
def test_folder_uses_explicit_value_or_settings_default(self):
"""Folder defaults to settings and can be overridden explicitly."""
settings = SimpleNamespace(
@@ -223,6 +236,7 @@ class TestTestConnection:
IMAP_PORT=993,
IMAP_USERNAME=username,
IMAP_PASSWORD=password,
DELETE_IMPORTED_EMAILS=False,
)
return IMAPClient()
@@ -233,6 +247,7 @@ class TestTestConnection:
IMAP_PORT=993,
IMAP_USERNAME=None,
IMAP_PASSWORD=None,
DELETE_IMPORTED_EMAILS=False,
)
client = IMAPClient()
success, message, stats = client.test_connection()
@@ -634,11 +649,31 @@ class TestProcessSingleEmail:
mock_mail.fetch.return_value = ("OK", [(b"1", raw)])
mock_mail.store.return_value = ("OK", None)
stats = {"processed": 0, "reports_found": 0, "errors": []}
stats = {"processed": 0, "reports_found": 0, "deleted": 0, "errors": []}
client._process_single_email(mock_mail, b"1", stats)
# store should have been called twice: once for \\Seen, once for \\Deleted
assert mock_mail.store.call_count >= 2
assert stats["deleted"] == 1
def test_does_not_delete_when_no_report_imported(self):
client = self._make_client()
client.delete_emails = True
raw = _make_email_with_attachment(
"not-a-report.txt",
b"not a report",
"text/plain",
subject="DMARC Report",
)
mock_mail = MagicMock()
mock_mail.fetch.return_value = ("OK", [(b"1", raw)])
mock_mail.store.return_value = ("OK", None)
stats = {"processed": 0, "reports_found": 0, "deleted": 0, "errors": []}
client._process_single_email(mock_mail, b"1", stats)
mock_mail.store.assert_called_once_with(b"1", "+FLAGS", "\\Seen")
assert stats["deleted"] == 0
# ---------------------------------------------------------------------------
@@ -736,7 +771,22 @@ class TestFetchReports:
mock_mail = MagicMock()
mock_mail.login.return_value = None
mock_mail.select.return_value = ("OK", [b"0"])
mock_mail.search.return_value = ("OK", [b""])
mock_mail.search.return_value = ("OK", [b"1"])
mock_mail.fetch.return_value = (
"OK",
[
(
b"1",
_make_email_with_attachment(
"report.xml",
MINIMAL_DMARC_XML,
"application/xml",
subject="DMARC Report",
),
)
],
)
mock_mail.store.return_value = ("OK", None)
mock_mail.logout.return_value = None
with patch("imaplib.IMAP4_SSL", return_value=mock_mail):
@@ -744,3 +794,20 @@ class TestFetchReports:
mock_mail.expunge.assert_called_once()
assert result["success"] is True
assert result["deleted"] == 1
def test_delete_emails_skips_expunge_when_nothing_deleted(self):
client = self._make_client()
client.delete_emails = True
mock_mail = MagicMock()
mock_mail.login.return_value = None
mock_mail.select.return_value = ("OK", [b"0"])
mock_mail.search.return_value = ("OK", [b""])
mock_mail.logout.return_value = None
with patch("imaplib.IMAP4_SSL", return_value=mock_mail):
result = client.fetch_reports(days=3)
mock_mail.expunge.assert_not_called()
assert result["success"] is True
assert result["deleted"] == 0
backend/app/tests/test_imap_endpoints.py
+16 -4
View File
@@ -83,7 +83,10 @@ class TestImapFetchReports:
"errors": [],
}
with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client):
with (
patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client),
patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()),
):
response = authed_client.post("/api/v1/imap/fetch-reports?days=7")
assert response.status_code == 200
@@ -103,7 +106,10 @@ class TestImapFetchReports:
def test_fetch_background_for_long_range(self, authed_client: TestClient):
"""Days > 14 should queue a background task."""
mock_client = MagicMock()
with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client):
with (
patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client),
patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()),
):
response = authed_client.post("/api/v1/imap/fetch-reports?days=30")
assert response.status_code == 200
@@ -121,7 +127,10 @@ class TestImapFetchReports:
"errors": ["Could not parse email 1"],
}
with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client):
with (
patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client),
patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()),
):
response = authed_client.post("/api/v1/imap/fetch-reports?days=3")
assert response.status_code == 200
@@ -132,7 +141,10 @@ class TestImapFetchReports:
mock_client = MagicMock()
mock_client.fetch_reports.side_effect = RuntimeError("unexpected")
with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client):
with (
patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client),
patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()),
):
response = authed_client.post("/api/v1/imap/fetch-reports?days=5")
assert response.status_code == 500
backend/app/tests/test_mail_sources.py
+34 -6
View File
@@ -875,9 +875,7 @@ class TestManualSourceFetchEndpoint:
assert data["error_count"] == 1
assert "bad attachment with newline" in caplog.text
history_resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/imports")
assert history_resp.json()[0]["details"] == [
{"status": "error", "filename": "bad.xml"}
]
assert history_resp.json()[0]["details"] == [{"status": "error", "filename": "bad.xml"}]
mock_imap.fetch_reports.assert_called_once_with(days=30)
def test_fetch_gmail_source(self, authed_client: TestClient, db_session: Session):
@@ -1806,8 +1804,33 @@ class TestTriggerPollImapSource:
assert result["processed"] == 3
assert result["reports_found"] == 2
assert result["new_domains"] == ["dom.example"]
mock_imap.fetch_reports.assert_called_once_with(days=7)
mock_db.commit.assert_called_once()
def test_uses_requested_days(self):
from app.main import _trigger_poll_imap_source
src = MagicMock()
src.id = 5
src.name = "My IMAP"
src.server = "imap.example.com"
src.port = 993
src.username = "u"
src.password = "p"
mock_imap = MagicMock()
mock_imap.fetch_reports.return_value = {
"success": True,
"processed": 0,
"reports_found": 0,
"new_domains": [],
}
with patch("app.main.IMAPClient", return_value=mock_imap):
_trigger_poll_imap_source(src, MagicMock(), days=30)
mock_imap.fetch_reports.assert_called_once_with(days=30)
class TestTriggerPollGmailSource:
"""Unit tests for app.main._trigger_poll_gmail_source."""
@@ -1938,7 +1961,7 @@ class TestPollSourceForTrigger:
result = _poll_source_for_trigger(src, MagicMock())
assert result is expected
mock_fn.assert_called_once()
assert mock_fn.call_args.kwargs["days"] == 7
def test_imap_exception_returns_failure_dict(self):
from app.main import _poll_source_for_trigger
@@ -2092,15 +2115,20 @@ class TestTriggerPollEndpoint:
with TestClient(main_app) as tc:
with (
patch("app.main.SessionLocal", return_value=mock_db),
patch("app.main._poll_source_for_trigger", return_value=mock_result),
patch(
"app.main._poll_source_for_trigger", return_value=mock_result
) as mock_poll,
):
resp = tc.post("/api/v1/admin/trigger-poll")
resp = tc.post("/api/v1/admin/trigger-poll?days=30")
assert resp.status_code == 200
data = resp.json()
assert data["days"] == 30
assert "sources" in data
assert len(data["sources"]) == 1
assert data["sources"][0]["success"] is True
assert mock_result == data["sources"][0]
assert mock_poll.call_args.kwargs["days"] == 30
finally:
main_app.dependency_overrides.clear()
backend/app/tests/test_main_polling.py
-1
View File
@@ -79,7 +79,6 @@ def test_poll_single_imap_source_passes_configured_folder():
port=993,
username="u",
password="p",
delete_emails=False,
folder="Junk Mail",
db=db,
)
backend/app/tests/test_webhook.py
+128
View File
@@ -0,0 +1,128 @@
import base64
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
import pytest
from fastapi.testclient import TestClient
from app.core.config import get_settings
from app.models.report import DMARCReport
MINIMAL_DMARC_XML = b"""\
<?xml version="1.0"?>
<feedback>
<report_metadata>
<org_name>Webhook Test</org_name>
<email>dmarc@example.com</email>
<report_id>webhook-001</report_id>
<date_range>
<begin>1609459200</begin>
<end>1609545600</end>
</date_range>
</report_metadata>
<policy_published>
<domain>webhook.example</domain>
<adkim>r</adkim>
<aspf>r</aspf>
<p>none</p>
<sp>none</sp>
<pct>100</pct>
</policy_published>
<record>
<row>
<source_ip>1.2.3.4</source_ip>
<count>1</count>
<policy_evaluated>
<disposition>none</disposition>
<dkim>pass</dkim>
<spf>pass</spf>
</policy_evaluated>
</row>
<identifiers>
<header_from>webhook.example</header_from>
</identifiers>
</record>
</feedback>
"""
def _raw_email_with_report() -> bytes:
msg = MIMEMultipart()
msg["Subject"] = "DMARC report"
part = MIMEApplication(MINIMAL_DMARC_XML, _subtype="xml")
part.add_header("Content-Disposition", "attachment", filename="report.xml")
msg.attach(part)
return msg.as_bytes()
def _set_webhook_secret(monkeypatch, value="test-webhook-secret"):
monkeypatch.setenv("WEBHOOK_SECRET", value)
get_settings.cache_clear()
return value
@pytest.fixture(autouse=True)
def _clear_settings_cache():
yield
get_settings.cache_clear()
def test_webhook_requires_configured_secret(client: TestClient, monkeypatch):
monkeypatch.delenv("WEBHOOK_SECRET", raising=False)
get_settings.cache_clear()
response = client.post(
"/api/v1/webhook/email",
json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")},
)
assert response.status_code == 503
def test_webhook_rejects_invalid_secret(client: TestClient, monkeypatch):
_set_webhook_secret(monkeypatch)
response = client.post(
"/api/v1/webhook/email",
headers={"X-Webhook-Secret": "wrong"},
json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")},
)
assert response.status_code == 401
def test_webhook_imports_base64_email(client: TestClient, db_session, monkeypatch):
secret = _set_webhook_secret(monkeypatch)
response = client.post(
"/api/v1/webhook/email",
headers={"X-Webhook-Secret": secret},
json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")},
)
assert response.status_code == 200
data = response.json()
assert data["success"] is True
assert data["reports_found"] == 1
assert data["imported"] == 1
assert db_session.query(DMARCReport).count() == 1
def test_webhook_raw_email_marks_duplicate(client: TestClient, monkeypatch):
secret = _set_webhook_secret(monkeypatch)
raw_email = _raw_email_with_report()
first = client.post(
"/api/v1/webhook/email/raw",
headers={"X-Webhook-Secret": secret},
content=raw_email,
)
second = client.post(
"/api/v1/webhook/email/raw",
headers={"X-Webhook-Secret": secret},
content=raw_email,
)
assert first.status_code == 200
assert second.status_code == 200
assert second.json()["duplicates"] == 1
docker-compose.yml
+2 -3
View File
@@ -26,22 +26,21 @@ services:
context: ./backend
dockerfile: Dockerfile
volumes:
- ./backend:/app # development: mount source code for live reload
- app_data:/app/data # persist SQLite database and other application data
depends_on:
db:
condition: service_healthy
environment:
- SERVICE_FQDN_APP_8080
- DATABASE_URL=postgresql://dmarq_user:dmarq_secure_password@db:5432/dmarq_db
- SECRET_KEY=your_secret_key_change_in_production
- DEBUG=True
- ENVIRONMENT=development
- DELETE_IMPORTED_EMAILS=false
# Add NODE_ENV for Tailwind
- NODE_ENV=production
networks:
- dmarq-network
ports:
- "80:8080" # Map directly to port 80 for web access
# Define networks
networks:
docs/deployment/configuration.md
+2
View File
@@ -62,6 +62,7 @@ For database operations, use the [Database Backup and Restore](backups.md) guide
| `IMAP_FOLDER` | IMAP folder to check | `INBOX` | `DMARC`, `reports` |
| `IMAP_MARK_AS_READ` | Mark processed emails as read | `true` | `true`, `false` |
| `IMAP_ARCHIVE_FOLDER` | Folder to move processed emails to | - | `Processed`, `Archive` |
| `DELETE_IMPORTED_EMAILS` | Delete IMAP emails after a DMARC report is successfully imported | `false` | `true`, `false` |
### Application Settings
@@ -108,6 +109,7 @@ policy if long-term storage size matters.
| `CF_ENABLED` | Enable Cloudflare integration | `false` | `true`, `false` |
| `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` |
| `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` |
| `WEBHOOK_SECRET` | Required secret for inbound email worker webhooks | - | `openssl rand -hex 32` |
### DNS Result Cache
docs/deployment/docker.md
+4
View File
@@ -45,9 +45,11 @@ The fastest way to get DMARQ running is to use Docker Compose:
# IMAP_PASSWORD=your_secure_password
# IMAP_USE_SSL=true
# IMAP_POLLING_INTERVAL=60
# DELETE_IMPORTED_EMAILS=false
# Security Settings
SECRET_KEY=generate_a_secure_random_key
# WEBHOOK_SECRET=generate_a_separate_webhook_secret
ALLOWED_HOSTS=localhost,127.0.0.1
```
@@ -104,7 +106,9 @@ services:
- IMAP_PASSWORD=${IMAP_PASSWORD:-}
- IMAP_USE_SSL=${IMAP_USE_SSL:-true}
- IMAP_POLLING_INTERVAL=${IMAP_POLLING_INTERVAL:-60}
- DELETE_IMPORTED_EMAILS=${DELETE_IMPORTED_EMAILS:-false}
- SECRET_KEY=${SECRET_KEY:-insecure_key_change_me_in_production}
- WEBHOOK_SECRET=${WEBHOOK_SECRET:-}
- ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1}
depends_on:
- db