diff --git a/backend/app/api/api_v1/api.py b/backend/app/api/api_v1/api.py index 51512d3..6c8a518 100644 --- a/backend/app/api/api_v1/api.py +++ b/backend/app/api/api_v1/api.py @@ -10,6 +10,7 @@ from app.api.api_v1.endpoints import ( settings, setup, stats, + webhook, ) api_router = APIRouter() @@ -24,3 +25,4 @@ api_router.include_router(imap.router, prefix="/imap", tags=["imap"]) api_router.include_router(stats.router, prefix="/stats", tags=["stats"]) api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["mail-sources"]) api_router.include_router(settings.router, prefix="/settings", tags=["settings"]) +api_router.include_router(webhook.router, prefix="/webhook", tags=["webhook"]) diff --git a/backend/app/api/api_v1/endpoints/imap.py b/backend/app/api/api_v1/endpoints/imap.py index 6686d0e..920581f 100644 --- a/backend/app/api/api_v1/endpoints/imap.py +++ b/backend/app/api/api_v1/endpoints/imap.py @@ -4,9 +4,9 @@ from typing import Any, Dict, Optional from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from pydantic import BaseModel -from sqlalchemy.orm import Session +from starlette.concurrency import run_in_threadpool -from app.core.database import SessionLocal, get_db +from app.core.database import SessionLocal from app.core.security import require_admin_auth from app.services.imap_client import IMAPClient @@ -24,13 +24,14 @@ class IMAPTestRequest(BaseModel): ssl: bool = True -def _fetch_imap_reports_background(days: int, delete_emails: bool) -> None: - """Fetch IMAP reports with a standalone DB session for background imports.""" +def _fetch_imap_reports_sync(days: int, delete_emails: Optional[bool]) -> Dict[str, Any]: + """Fetch IMAP reports with a standalone DB session.""" db = SessionLocal() try: imap_client = IMAPClient(delete_emails=delete_emails, db=db) - imap_client.fetch_reports(days=days) + results = imap_client.fetch_reports(days=days) db.commit() + return results except Exception: db.rollback() raise @@ -38,6 +39,11 @@ def _fetch_imap_reports_background(days: int, delete_emails: bool) -> None: db.close() +def _fetch_imap_reports_background(days: int, delete_emails: Optional[bool]) -> None: + """Fetch IMAP reports from a FastAPI background task.""" + _fetch_imap_reports_sync(days, delete_emails) + + @router.post("/test-connection") async def test_imap_connection( request: IMAPTestRequest, @@ -73,9 +79,8 @@ async def test_imap_connection( async def fetch_imap_reports( background_tasks: BackgroundTasks, _auth: dict = Depends(require_admin_auth), - db: Session = Depends(get_db), days: int = 7, - delete_emails: bool = False, + delete_emails: Optional[bool] = None, ) -> Dict[str, Any]: """ Fetch DMARC reports from the configured IMAP mailbox @@ -86,8 +91,6 @@ async def fetch_imap_reports( if days < 1 or days > 365: raise HTTPException(status_code=400, detail="Days parameter must be between 1 and 365") - imap_client = IMAPClient(delete_emails=delete_emails, db=db) - # Run in background if it might take a while if days > 14: background_tasks.add_task(_fetch_imap_reports_background, days, delete_emails) @@ -99,8 +102,7 @@ async def fetch_imap_reports( # Otherwise run immediately try: - results = imap_client.fetch_reports(days=days) - db.commit() + results = await run_in_threadpool(_fetch_imap_reports_sync, days, delete_emails) return { "success": results["success"], diff --git a/backend/app/api/api_v1/endpoints/mail_sources.py b/backend/app/api/api_v1/endpoints/mail_sources.py index 5b352ce..088017b 100644 --- a/backend/app/api/api_v1/endpoints/mail_sources.py +++ b/backend/app/api/api_v1/endpoints/mail_sources.py @@ -287,7 +287,6 @@ def _fetch_imap_source(source: MailSource, db: Session, days: int) -> Dict[str, port=source.port or 993, username=source.username, password=source.password, - delete_emails=False, folder=source.folder, db=db, ) diff --git a/backend/app/api/api_v1/endpoints/webhook.py b/backend/app/api/api_v1/endpoints/webhook.py new file mode 100644 index 0000000..ae45451 --- /dev/null +++ b/backend/app/api/api_v1/endpoints/webhook.py @@ -0,0 +1,173 @@ +"""Webhook ingestion endpoints for inbound DMARC report emails.""" + +import base64 +import email +import hmac +import logging +from email.header import decode_header +from typing import Any, Dict, Optional + +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.core.config import get_settings +from app.core.database import get_db +from app.core.redaction import sanitize_for_log +from app.services.dmarc_parser import DMARCParser +from app.services.report_persistence import report_exists, save_parsed_report +from app.services.report_store import ReportStore + +logger = logging.getLogger(__name__) +router = APIRouter() + + +class EmailWebhookPayload(BaseModel): + """Payload for JSON webhook delivery from an email worker.""" + + raw_email: str + from_address: Optional[str] = None + to_address: Optional[str] = None + subject: Optional[str] = None + + +def _decode_email_header(header: Optional[str]) -> str: + """Decode an RFC 2047 email header to display text.""" + if not header: + return "" + decoded_parts = [] + for text, encoding in decode_header(header): + if isinstance(text, bytes): + decoded_parts.append(text.decode(encoding or "utf-8", errors="replace")) + else: + decoded_parts.append(text) + return " ".join(decoded_parts) + + +def _is_dmarc_filename(filename: str) -> bool: + lower = filename.lower() + return lower.endswith((".xml", ".zip", ".gz", ".gzip")) + + +def _require_webhook_secret(x_webhook_secret: Optional[str]) -> None: + settings = get_settings() + if not settings.WEBHOOK_SECRET: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Webhook ingestion is not configured.", + ) + if not x_webhook_secret or not hmac.compare_digest(x_webhook_secret, settings.WEBHOOK_SECRET): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid webhook secret.", + ) + + +def _store_report(db: Session, store: ReportStore, report: Dict[str, Any]) -> str: + domain = report.get("domain") or "unknown" + report_id = report.get("report_id") or "" + if report_id and (store.has_report(domain, report_id) or report_exists(db, domain, report_id)): + return "duplicate" + save_parsed_report(db, report) + store.add_report(report) + return "imported" + + +def _process_email_attachments(msg: email.message.Message, db: Session) -> Dict[str, Any]: + store = ReportStore.get_instance() + results: Dict[str, Any] = { + "reports_found": 0, + "imported": 0, + "duplicates": 0, + "errors": [], + } + + for part in msg.walk(): + if part.get_content_disposition() != "attachment": + continue + + filename = _decode_email_header(part.get_filename()) + if not filename or not _is_dmarc_filename(filename): + continue + + try: + content = part.get_payload(decode=True) + if not content: + continue + report = DMARCParser.parse_file(content, filename) + outcome = _store_report(db, store, report) + results["reports_found"] += 1 + if outcome == "duplicate": + results["duplicates"] += 1 + else: + results["imported"] += 1 + except Exception as exc: # pylint: disable=broad-exception-caught + logger.warning( + "Webhook failed to process DMARC attachment %s: %s", + sanitize_for_log(filename), + sanitize_for_log(exc), + ) + results["errors"].append(filename) + + return results + + +def _subject_from_message(msg: email.message.Message, fallback: Optional[str] = None) -> str: + return fallback or _decode_email_header(msg.get("Subject")) + + +@router.post("/email") +async def receive_email( + payload: EmailWebhookPayload, + x_webhook_secret: Optional[str] = Header(None), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Receive a base64 encoded raw email from an email worker webhook.""" + _require_webhook_secret(x_webhook_secret) + try: + raw_email = base64.b64decode(payload.raw_email, validate=True) + except (ValueError, TypeError) as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="raw_email must be valid base64.", + ) from exc + + return _handle_raw_email(raw_email, db, subject=payload.subject) + + +@router.post("/email/raw") +async def receive_raw_email( + request: Request, + x_webhook_secret: Optional[str] = Header(None), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Receive raw RFC 822 email bytes from an email worker webhook.""" + _require_webhook_secret(x_webhook_secret) + return _handle_raw_email(await request.body(), db) + + +def _handle_raw_email( + raw_email: bytes, + db: Session, + *, + subject: Optional[str] = None, +) -> Dict[str, Any]: + try: + msg = email.message_from_bytes(raw_email) + attachment_results = _process_email_attachments(msg, db) + db.commit() + except HTTPException: + raise + except Exception as exc: # pylint: disable=broad-exception-caught + db.rollback() + logger.warning("Webhook failed to process email: %s", sanitize_for_log(exc)) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Error processing email.", + ) from exc + + return { + "success": True, + "subject": _subject_from_message(msg, subject), + **attachment_results, + } diff --git a/backend/app/core/config.py b/backend/app/core/config.py index fe31ae0..30a2a28 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -42,6 +42,7 @@ class Settings(BaseSettings): IMAP_USERNAME: Optional[str] = None IMAP_PASSWORD: Optional[str] = None IMAP_FOLDER: str = "INBOX" + DELETE_IMPORTED_EMAILS: bool = False # Admin User FIRST_SUPERUSER: Optional[EmailStr] = None @@ -50,6 +51,7 @@ class Settings(BaseSettings): # Optional Cloudflare Integration CLOUDFLARE_API_TOKEN: Optional[str] = None CLOUDFLARE_ZONE_ID: Optional[str] = None + WEBHOOK_SECRET: Optional[str] = None # Admin API Key (optional) # If set, this key is used directly instead of generating a random one at startup. diff --git a/backend/app/main.py b/backend/app/main.py index 20ff084..eb1c543 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,11 +3,12 @@ import logging import os from datetime import datetime -from fastapi import Depends, FastAPI, Request +from fastapi import Depends, FastAPI, Query, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +from starlette.concurrency import run_in_threadpool import app.models.alert # noqa: F401 – ensure AlertHistory table is registered import app.models.dns_cache # noqa: F401 – ensure DNSCache table is registered @@ -54,7 +55,6 @@ def _poll_single_imap_source(source: MailSource) -> None: port=poll_source.port or 993, username=poll_source.username, password=poll_source.password, - delete_emails=False, folder=poll_source.folder, db=db, ) @@ -547,7 +547,7 @@ async def health(): # --------------------------------------------------------------------------- -def _trigger_poll_imap_source(source: MailSource, db) -> dict: +def _trigger_poll_imap_source(source: MailSource, db, days: int = 7) -> dict: """Poll a single IMAP source and return a result dict for the API response.""" global last_check_time # pylint: disable=global-statement @@ -556,12 +556,11 @@ def _trigger_poll_imap_source(source: MailSource, db) -> dict: port=source.port or 993, username=source.username, password=source.password, - delete_emails=False, folder=source.folder, db=db, ) started_at = datetime.utcnow() - results = imap_client.fetch_reports(days=7) + results = imap_client.fetch_reports(days=days) last_check_time = datetime.now() source.last_checked = datetime.utcnow() record_import_attempt(db, source, results, started_at=started_at, trigger="manual") @@ -614,7 +613,7 @@ def _trigger_poll_gmail_source(source: MailSource, db) -> dict: } -def _poll_source_for_trigger(source: MailSource, db) -> dict: +def _poll_source_for_trigger(source: MailSource, db, days: int = 7) -> dict: """Dispatch a single mail source for the manual trigger-poll endpoint. Returns a result/summary dict that is included in the API response. @@ -639,7 +638,7 @@ def _poll_source_for_trigger(source: MailSource, db) -> dict: } if source.method == "IMAP": try: - return _trigger_poll_imap_source(source, db) + return _trigger_poll_imap_source(source, db, days=days) except Exception as e: # pylint: disable=broad-exception-caught logger.error("Error polling mail source id=%d: %s", source.id, str(e)) return { @@ -656,14 +655,8 @@ def _poll_source_for_trigger(source: MailSource, db) -> dict: } -# API endpoint to manually trigger IMAP polling -@app.post("/api/v1/admin/trigger-poll") -async def trigger_imap_poll(auth: dict = Depends(require_admin_auth)): - """ - Manually trigger IMAP polling for all enabled mail sources (admin only). - - Security: Requires either X-API-Key header or Bearer token - """ +def _poll_enabled_sources_for_trigger(days: int) -> list[dict]: + """Poll all enabled mail sources for the manual trigger-poll endpoint.""" results_summary = [] db = SessionLocal() try: @@ -671,22 +664,39 @@ async def trigger_imap_poll(auth: dict = Depends(require_admin_auth)): db.query(MailSource).filter(MailSource.enabled == True).all() # noqa: E712 ) - if not enabled_sources: - return { - "success": True, - "message": "No enabled mail sources configured.", - "sources_polled": 0, - "authenticated_by": auth.get("auth_type"), - } - for source in enabled_sources: - results_summary.append(_poll_source_for_trigger(source, db)) + results_summary.append(_poll_source_for_trigger(source, db, days=days)) finally: db.close() + return results_summary + + +# API endpoint to manually trigger IMAP polling +@app.post("/api/v1/admin/trigger-poll") +async def trigger_imap_poll( + auth: dict = Depends(require_admin_auth), + days: int = Query(7, ge=1, le=365, title="Number of days to fetch for IMAP sources"), +): + """ + Manually trigger IMAP polling for all enabled mail sources (admin only). + + Security: Requires either X-API-Key header or Bearer token + """ + results_summary = await run_in_threadpool(_poll_enabled_sources_for_trigger, days) + if not results_summary: + return { + "success": True, + "message": "No enabled mail sources configured.", + "sources_polled": 0, + "days": days, + "authenticated_by": auth.get("auth_type"), + } + return { "success": all(r.get("success", True) for r in results_summary), "timestamp": last_check_time.isoformat() if last_check_time else None, + "days": days, "sources": results_summary, "authenticated_by": auth.get("auth_type"), } diff --git a/backend/app/services/imap_client.py b/backend/app/services/imap_client.py index ea6aa3c..99e31da 100644 --- a/backend/app/services/imap_client.py +++ b/backend/app/services/imap_client.py @@ -3,7 +3,7 @@ import imaplib import logging from datetime import datetime, timedelta from email.header import decode_header -from typing import Any, Dict, Tuple +from typing import Any, Dict, Optional, Tuple from app.core.config import get_settings from app.services.dmarc_parser import DMARCParser @@ -25,7 +25,7 @@ class IMAPClient: port: int = None, username: str = None, password: str = None, - delete_emails: bool = False, + delete_emails: Optional[bool] = None, folder: str = None, db: Any = None, ): @@ -37,7 +37,8 @@ class IMAPClient: port: IMAP server port (if None, uses settings) username: IMAP username (if None, uses settings) password: IMAP password (if None, uses settings) - delete_emails: Whether to delete emails after processing (default: False) + delete_emails: Whether to delete emails after successful report imports. + If omitted, uses DELETE_IMPORTED_EMAILS from settings. folder: IMAP mailbox folder to read (if None, uses settings or INBOX) db: Optional SQLAlchemy session used to persist imported reports """ @@ -50,7 +51,10 @@ class IMAPClient: self.port = port or settings.IMAP_PORT self.username = username or settings.IMAP_USERNAME self.password = password or settings.IMAP_PASSWORD - self.delete_emails = delete_emails + configured_delete = getattr(settings, "DELETE_IMPORTED_EMAILS", False) + if not isinstance(configured_delete, bool): + configured_delete = False + self.delete_emails = configured_delete if delete_emails is None else delete_emails self.folder = folder or settings_folder or "INBOX" self.db = db @@ -170,10 +174,11 @@ class IMAPClient: reports_found = self._process_attachments(msg, stats, message_id=message_id) stats["reports_found"] += reports_found - # Mark email as read (and optionally delete) + # Mark DMARC-looking email as read, and delete only after a successful import. mail.store(email_id, "+FLAGS", "\\Seen") - if self.delete_emails: + if self.delete_emails and reports_found > 0: mail.store(email_id, "+FLAGS", "\\Deleted") + stats["deleted"] = stats.get("deleted", 0) + 1 stats["processed"] += 1 except Exception as e: # pylint: disable=broad-exception-caught @@ -206,6 +211,7 @@ class IMAPClient: "success": True, "processed": 0, "reports_found": 0, + "deleted": 0, "duplicate_reports": 0, "new_domains": [], "errors": [], @@ -243,7 +249,7 @@ class IMAPClient: self._process_single_email(mail, email_id, stats) # Actually remove emails marked for deletion - if self.delete_emails: + if self.delete_emails and stats["deleted"] > 0: mail.expunge() # Logout diff --git a/backend/app/tests/test_config.py b/backend/app/tests/test_config.py index 66aa8b7..1f38f21 100644 --- a/backend/app/tests/test_config.py +++ b/backend/app/tests/test_config.py @@ -188,6 +188,20 @@ class TestLogtoSettings: assert settings.LOGTO_SKIP_SSL_VERIFY is True +class TestImapSettings: + def test_delete_imported_emails_defaults_false(self, monkeypatch): + monkeypatch.delenv("DELETE_IMPORTED_EMAILS", raising=False) + settings = Settings() + + assert settings.DELETE_IMPORTED_EMAILS is False + + def test_delete_imported_emails_reads_env(self, monkeypatch): + monkeypatch.setenv("DELETE_IMPORTED_EMAILS", "true") + settings = Settings() + + assert settings.DELETE_IMPORTED_EMAILS is True + + class TestProductionStartupSettings: """Tests for production-critical settings and startup validation.""" diff --git a/backend/app/tests/test_imap_client.py b/backend/app/tests/test_imap_client.py index f1928c8..3068b83 100644 --- a/backend/app/tests/test_imap_client.py +++ b/backend/app/tests/test_imap_client.py @@ -139,6 +139,19 @@ class TestIMAPClientInit: assert client.password == "secret" assert client.delete_emails is True + def test_delete_emails_defaults_to_settings(self): + settings = SimpleNamespace( + IMAP_SERVER="imap.example.com", + IMAP_PORT=993, + IMAP_USERNAME="u", + IMAP_PASSWORD="p", + DELETE_IMPORTED_EMAILS=True, + ) + with patch("app.services.imap_client.get_settings", return_value=settings): + client = IMAPClient() + + assert client.delete_emails is True + def test_folder_uses_explicit_value_or_settings_default(self): """Folder defaults to settings and can be overridden explicitly.""" settings = SimpleNamespace( @@ -223,6 +236,7 @@ class TestTestConnection: IMAP_PORT=993, IMAP_USERNAME=username, IMAP_PASSWORD=password, + DELETE_IMPORTED_EMAILS=False, ) return IMAPClient() @@ -233,6 +247,7 @@ class TestTestConnection: IMAP_PORT=993, IMAP_USERNAME=None, IMAP_PASSWORD=None, + DELETE_IMPORTED_EMAILS=False, ) client = IMAPClient() success, message, stats = client.test_connection() @@ -634,11 +649,31 @@ class TestProcessSingleEmail: mock_mail.fetch.return_value = ("OK", [(b"1", raw)]) mock_mail.store.return_value = ("OK", None) - stats = {"processed": 0, "reports_found": 0, "errors": []} + stats = {"processed": 0, "reports_found": 0, "deleted": 0, "errors": []} client._process_single_email(mock_mail, b"1", stats) # store should have been called twice: once for \\Seen, once for \\Deleted assert mock_mail.store.call_count >= 2 + assert stats["deleted"] == 1 + + def test_does_not_delete_when_no_report_imported(self): + client = self._make_client() + client.delete_emails = True + raw = _make_email_with_attachment( + "not-a-report.txt", + b"not a report", + "text/plain", + subject="DMARC Report", + ) + mock_mail = MagicMock() + mock_mail.fetch.return_value = ("OK", [(b"1", raw)]) + mock_mail.store.return_value = ("OK", None) + + stats = {"processed": 0, "reports_found": 0, "deleted": 0, "errors": []} + client._process_single_email(mock_mail, b"1", stats) + + mock_mail.store.assert_called_once_with(b"1", "+FLAGS", "\\Seen") + assert stats["deleted"] == 0 # --------------------------------------------------------------------------- @@ -736,7 +771,22 @@ class TestFetchReports: mock_mail = MagicMock() mock_mail.login.return_value = None mock_mail.select.return_value = ("OK", [b"0"]) - mock_mail.search.return_value = ("OK", [b""]) + mock_mail.search.return_value = ("OK", [b"1"]) + mock_mail.fetch.return_value = ( + "OK", + [ + ( + b"1", + _make_email_with_attachment( + "report.xml", + MINIMAL_DMARC_XML, + "application/xml", + subject="DMARC Report", + ), + ) + ], + ) + mock_mail.store.return_value = ("OK", None) mock_mail.logout.return_value = None with patch("imaplib.IMAP4_SSL", return_value=mock_mail): @@ -744,3 +794,20 @@ class TestFetchReports: mock_mail.expunge.assert_called_once() assert result["success"] is True + assert result["deleted"] == 1 + + def test_delete_emails_skips_expunge_when_nothing_deleted(self): + client = self._make_client() + client.delete_emails = True + mock_mail = MagicMock() + mock_mail.login.return_value = None + mock_mail.select.return_value = ("OK", [b"0"]) + mock_mail.search.return_value = ("OK", [b""]) + mock_mail.logout.return_value = None + + with patch("imaplib.IMAP4_SSL", return_value=mock_mail): + result = client.fetch_reports(days=3) + + mock_mail.expunge.assert_not_called() + assert result["success"] is True + assert result["deleted"] == 0 diff --git a/backend/app/tests/test_imap_endpoints.py b/backend/app/tests/test_imap_endpoints.py index d868917..ff15b87 100644 --- a/backend/app/tests/test_imap_endpoints.py +++ b/backend/app/tests/test_imap_endpoints.py @@ -83,7 +83,10 @@ class TestImapFetchReports: "errors": [], } - with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client): + with ( + patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client), + patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()), + ): response = authed_client.post("/api/v1/imap/fetch-reports?days=7") assert response.status_code == 200 @@ -103,7 +106,10 @@ class TestImapFetchReports: def test_fetch_background_for_long_range(self, authed_client: TestClient): """Days > 14 should queue a background task.""" mock_client = MagicMock() - with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client): + with ( + patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client), + patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()), + ): response = authed_client.post("/api/v1/imap/fetch-reports?days=30") assert response.status_code == 200 @@ -121,7 +127,10 @@ class TestImapFetchReports: "errors": ["Could not parse email 1"], } - with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client): + with ( + patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client), + patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()), + ): response = authed_client.post("/api/v1/imap/fetch-reports?days=3") assert response.status_code == 200 @@ -132,7 +141,10 @@ class TestImapFetchReports: mock_client = MagicMock() mock_client.fetch_reports.side_effect = RuntimeError("unexpected") - with patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client): + with ( + patch("app.api.api_v1.endpoints.imap.IMAPClient", return_value=mock_client), + patch("app.api.api_v1.endpoints.imap.SessionLocal", return_value=MagicMock()), + ): response = authed_client.post("/api/v1/imap/fetch-reports?days=5") assert response.status_code == 500 diff --git a/backend/app/tests/test_mail_sources.py b/backend/app/tests/test_mail_sources.py index 17a880d..3a168c8 100644 --- a/backend/app/tests/test_mail_sources.py +++ b/backend/app/tests/test_mail_sources.py @@ -875,9 +875,7 @@ class TestManualSourceFetchEndpoint: assert data["error_count"] == 1 assert "bad attachment with newline" in caplog.text history_resp = authed_client.get(f"/api/v1/mail-sources/{source_id}/imports") - assert history_resp.json()[0]["details"] == [ - {"status": "error", "filename": "bad.xml"} - ] + assert history_resp.json()[0]["details"] == [{"status": "error", "filename": "bad.xml"}] mock_imap.fetch_reports.assert_called_once_with(days=30) def test_fetch_gmail_source(self, authed_client: TestClient, db_session: Session): @@ -1806,8 +1804,33 @@ class TestTriggerPollImapSource: assert result["processed"] == 3 assert result["reports_found"] == 2 assert result["new_domains"] == ["dom.example"] + mock_imap.fetch_reports.assert_called_once_with(days=7) mock_db.commit.assert_called_once() + def test_uses_requested_days(self): + from app.main import _trigger_poll_imap_source + + src = MagicMock() + src.id = 5 + src.name = "My IMAP" + src.server = "imap.example.com" + src.port = 993 + src.username = "u" + src.password = "p" + + mock_imap = MagicMock() + mock_imap.fetch_reports.return_value = { + "success": True, + "processed": 0, + "reports_found": 0, + "new_domains": [], + } + + with patch("app.main.IMAPClient", return_value=mock_imap): + _trigger_poll_imap_source(src, MagicMock(), days=30) + + mock_imap.fetch_reports.assert_called_once_with(days=30) + class TestTriggerPollGmailSource: """Unit tests for app.main._trigger_poll_gmail_source.""" @@ -1938,7 +1961,7 @@ class TestPollSourceForTrigger: result = _poll_source_for_trigger(src, MagicMock()) assert result is expected - mock_fn.assert_called_once() + assert mock_fn.call_args.kwargs["days"] == 7 def test_imap_exception_returns_failure_dict(self): from app.main import _poll_source_for_trigger @@ -2092,15 +2115,20 @@ class TestTriggerPollEndpoint: with TestClient(main_app) as tc: with ( patch("app.main.SessionLocal", return_value=mock_db), - patch("app.main._poll_source_for_trigger", return_value=mock_result), + patch( + "app.main._poll_source_for_trigger", return_value=mock_result + ) as mock_poll, ): - resp = tc.post("/api/v1/admin/trigger-poll") + resp = tc.post("/api/v1/admin/trigger-poll?days=30") assert resp.status_code == 200 data = resp.json() + assert data["days"] == 30 assert "sources" in data assert len(data["sources"]) == 1 assert data["sources"][0]["success"] is True + assert mock_result == data["sources"][0] + assert mock_poll.call_args.kwargs["days"] == 30 finally: main_app.dependency_overrides.clear() diff --git a/backend/app/tests/test_main_polling.py b/backend/app/tests/test_main_polling.py index a46c26f..2134ba3 100644 --- a/backend/app/tests/test_main_polling.py +++ b/backend/app/tests/test_main_polling.py @@ -79,7 +79,6 @@ def test_poll_single_imap_source_passes_configured_folder(): port=993, username="u", password="p", - delete_emails=False, folder="Junk Mail", db=db, ) diff --git a/backend/app/tests/test_webhook.py b/backend/app/tests/test_webhook.py new file mode 100644 index 0000000..9960041 --- /dev/null +++ b/backend/app/tests/test_webhook.py @@ -0,0 +1,128 @@ +import base64 +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart + +import pytest +from fastapi.testclient import TestClient + +from app.core.config import get_settings +from app.models.report import DMARCReport + +MINIMAL_DMARC_XML = b"""\ + + + + Webhook Test + dmarc@example.com + webhook-001 + + 1609459200 + 1609545600 + + + + webhook.example + r + r +

none

+ none + 100 + + + + 1.2.3.4 + 1 + + none + pass + pass + + + + webhook.example + + + +""" + + +def _raw_email_with_report() -> bytes: + msg = MIMEMultipart() + msg["Subject"] = "DMARC report" + part = MIMEApplication(MINIMAL_DMARC_XML, _subtype="xml") + part.add_header("Content-Disposition", "attachment", filename="report.xml") + msg.attach(part) + return msg.as_bytes() + + +def _set_webhook_secret(monkeypatch, value="test-webhook-secret"): + monkeypatch.setenv("WEBHOOK_SECRET", value) + get_settings.cache_clear() + return value + + +@pytest.fixture(autouse=True) +def _clear_settings_cache(): + yield + get_settings.cache_clear() + + +def test_webhook_requires_configured_secret(client: TestClient, monkeypatch): + monkeypatch.delenv("WEBHOOK_SECRET", raising=False) + get_settings.cache_clear() + + response = client.post( + "/api/v1/webhook/email", + json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")}, + ) + + assert response.status_code == 503 + + +def test_webhook_rejects_invalid_secret(client: TestClient, monkeypatch): + _set_webhook_secret(monkeypatch) + + response = client.post( + "/api/v1/webhook/email", + headers={"X-Webhook-Secret": "wrong"}, + json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")}, + ) + + assert response.status_code == 401 + + +def test_webhook_imports_base64_email(client: TestClient, db_session, monkeypatch): + secret = _set_webhook_secret(monkeypatch) + + response = client.post( + "/api/v1/webhook/email", + headers={"X-Webhook-Secret": secret}, + json={"raw_email": base64.b64encode(_raw_email_with_report()).decode("ascii")}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["reports_found"] == 1 + assert data["imported"] == 1 + assert db_session.query(DMARCReport).count() == 1 + + +def test_webhook_raw_email_marks_duplicate(client: TestClient, monkeypatch): + secret = _set_webhook_secret(monkeypatch) + raw_email = _raw_email_with_report() + + first = client.post( + "/api/v1/webhook/email/raw", + headers={"X-Webhook-Secret": secret}, + content=raw_email, + ) + second = client.post( + "/api/v1/webhook/email/raw", + headers={"X-Webhook-Secret": secret}, + content=raw_email, + ) + + assert first.status_code == 200 + assert second.status_code == 200 + assert second.json()["duplicates"] == 1 diff --git a/docker-compose.yml b/docker-compose.yml index d7c6f39..8bca348 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,22 +26,21 @@ services: context: ./backend dockerfile: Dockerfile volumes: - - ./backend:/app # development: mount source code for live reload - app_data:/app/data # persist SQLite database and other application data depends_on: db: condition: service_healthy environment: + - SERVICE_FQDN_APP_8080 - DATABASE_URL=postgresql://dmarq_user:dmarq_secure_password@db:5432/dmarq_db - SECRET_KEY=your_secret_key_change_in_production - DEBUG=True - ENVIRONMENT=development + - DELETE_IMPORTED_EMAILS=false # Add NODE_ENV for Tailwind - NODE_ENV=production networks: - dmarq-network - ports: - - "80:8080" # Map directly to port 80 for web access # Define networks networks: @@ -53,4 +52,4 @@ volumes: postgres_data: driver: local app_data: - driver: local \ No newline at end of file + driver: local diff --git a/docs/deployment/configuration.md b/docs/deployment/configuration.md index 3659703..31e016e 100644 --- a/docs/deployment/configuration.md +++ b/docs/deployment/configuration.md @@ -62,6 +62,7 @@ For database operations, use the [Database Backup and Restore](backups.md) guide | `IMAP_FOLDER` | IMAP folder to check | `INBOX` | `DMARC`, `reports` | | `IMAP_MARK_AS_READ` | Mark processed emails as read | `true` | `true`, `false` | | `IMAP_ARCHIVE_FOLDER` | Folder to move processed emails to | - | `Processed`, `Archive` | +| `DELETE_IMPORTED_EMAILS` | Delete IMAP emails after a DMARC report is successfully imported | `false` | `true`, `false` | ### Application Settings @@ -108,6 +109,7 @@ policy if long-term storage size matters. | `CF_ENABLED` | Enable Cloudflare integration | `false` | `true`, `false` | | `CF_API_TOKEN` | Cloudflare API token | - | `your_cloudflare_api_token` | | `CF_ZONE_ID` | Cloudflare Zone ID | - | `your_cloudflare_zone_id` | +| `WEBHOOK_SECRET` | Required secret for inbound email worker webhooks | - | `openssl rand -hex 32` | ### DNS Result Cache diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md index 6656c73..8bda247 100644 --- a/docs/deployment/docker.md +++ b/docs/deployment/docker.md @@ -45,9 +45,11 @@ The fastest way to get DMARQ running is to use Docker Compose: # IMAP_PASSWORD=your_secure_password # IMAP_USE_SSL=true # IMAP_POLLING_INTERVAL=60 + # DELETE_IMPORTED_EMAILS=false # Security Settings SECRET_KEY=generate_a_secure_random_key + # WEBHOOK_SECRET=generate_a_separate_webhook_secret ALLOWED_HOSTS=localhost,127.0.0.1 ``` @@ -104,7 +106,9 @@ services: - IMAP_PASSWORD=${IMAP_PASSWORD:-} - IMAP_USE_SSL=${IMAP_USE_SSL:-true} - IMAP_POLLING_INTERVAL=${IMAP_POLLING_INTERVAL:-60} + - DELETE_IMPORTED_EMAILS=${DELETE_IMPORTED_EMAILS:-false} - SECRET_KEY=${SECRET_KEY:-insecure_key_change_me_in_production} + - WEBHOOK_SECRET=${WEBHOOK_SECRET:-} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1} depends_on: - db