feat: add outbound webhook event framework
This commit is contained in:
@@ -29,6 +29,7 @@ import app.models.mail_source_import # noqa: E402, F401
|
||||
import app.models.report # noqa: E402, F401
|
||||
import app.models.setting # noqa: E402, F401
|
||||
import app.models.user # noqa: E402, F401
|
||||
import app.models.webhook # noqa: E402, F401
|
||||
|
||||
# Import all models so that autogenerate can detect them
|
||||
from app.core.database import Base # noqa: E402
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""add webhook event framework
|
||||
|
||||
Revision ID: 1b2c3d4e5f6a
|
||||
Revises: 0a1b2c3d4e5f
|
||||
Create Date: 2026-05-23 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "1b2c3d4e5f6a"
|
||||
down_revision: Union[str, Sequence[str], None] = "0a1b2c3d4e5f"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create outbound webhook endpoint and delivery tables."""
|
||||
op.create_table(
|
||||
"webhook_endpoints",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=120), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("secret", sa.Text(), nullable=False),
|
||||
sa.Column("event_types", sa.Text(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("timeout_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_success_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_failure_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("failure_count", sa.Integer(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(op.f("ix_webhook_endpoints_id"), "webhook_endpoints", ["id"])
|
||||
op.create_index(op.f("ix_webhook_endpoints_enabled"), "webhook_endpoints", ["enabled"])
|
||||
op.create_index(op.f("ix_webhook_endpoints_created_at"), "webhook_endpoints", ["created_at"])
|
||||
op.create_index(
|
||||
op.f("ix_webhook_endpoints_last_success_at"),
|
||||
"webhook_endpoints",
|
||||
["last_success_at"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_webhook_endpoints_last_failure_at"),
|
||||
"webhook_endpoints",
|
||||
["last_failure_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_webhook_endpoints_enabled_events",
|
||||
"webhook_endpoints",
|
||||
["enabled", "event_types"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"webhook_deliveries",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("endpoint_id", sa.Integer(), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=80), nullable=False),
|
||||
sa.Column("payload", sa.Text(), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=160), nullable=False),
|
||||
sa.Column("status", sa.String(length=24), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("delivered_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_status_code", sa.Integer(), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("response_excerpt", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(["endpoint_id"], ["webhook_endpoints.id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(op.f("ix_webhook_deliveries_id"), "webhook_deliveries", ["id"])
|
||||
op.create_index(
|
||||
op.f("ix_webhook_deliveries_endpoint_id"), "webhook_deliveries", ["endpoint_id"]
|
||||
)
|
||||
op.create_index(op.f("ix_webhook_deliveries_event_type"), "webhook_deliveries", ["event_type"])
|
||||
op.create_index(
|
||||
op.f("ix_webhook_deliveries_idempotency_key"), "webhook_deliveries", ["idempotency_key"]
|
||||
)
|
||||
op.create_index(op.f("ix_webhook_deliveries_status"), "webhook_deliveries", ["status"])
|
||||
op.create_index(
|
||||
op.f("ix_webhook_deliveries_next_attempt_at"),
|
||||
"webhook_deliveries",
|
||||
["next_attempt_at"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_webhook_deliveries_last_attempt_at"),
|
||||
"webhook_deliveries",
|
||||
["last_attempt_at"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_webhook_deliveries_delivered_at"), "webhook_deliveries", ["delivered_at"]
|
||||
)
|
||||
op.create_index(op.f("ix_webhook_deliveries_created_at"), "webhook_deliveries", ["created_at"])
|
||||
op.create_index(
|
||||
"ix_webhook_delivery_endpoint_idempotency",
|
||||
"webhook_deliveries",
|
||||
["endpoint_id", "idempotency_key"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_webhook_delivery_due",
|
||||
"webhook_deliveries",
|
||||
["status", "next_attempt_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop outbound webhook endpoint and delivery tables."""
|
||||
op.drop_index("ix_webhook_delivery_due", table_name="webhook_deliveries")
|
||||
op.drop_index("ix_webhook_delivery_endpoint_idempotency", table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_created_at"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_delivered_at"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_last_attempt_at"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_next_attempt_at"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_status"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_idempotency_key"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_event_type"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_endpoint_id"), table_name="webhook_deliveries")
|
||||
op.drop_index(op.f("ix_webhook_deliveries_id"), table_name="webhook_deliveries")
|
||||
op.drop_table("webhook_deliveries")
|
||||
|
||||
op.drop_index("ix_webhook_endpoints_enabled_events", table_name="webhook_endpoints")
|
||||
op.drop_index(op.f("ix_webhook_endpoints_last_failure_at"), table_name="webhook_endpoints")
|
||||
op.drop_index(op.f("ix_webhook_endpoints_last_success_at"), table_name="webhook_endpoints")
|
||||
op.drop_index(op.f("ix_webhook_endpoints_created_at"), table_name="webhook_endpoints")
|
||||
op.drop_index(op.f("ix_webhook_endpoints_enabled"), table_name="webhook_endpoints")
|
||||
op.drop_index(op.f("ix_webhook_endpoints_id"), table_name="webhook_endpoints")
|
||||
op.drop_table("webhook_endpoints")
|
||||
@@ -15,6 +15,7 @@ from app.api.api_v1.endpoints import (
|
||||
stats,
|
||||
tls_reports,
|
||||
webhook,
|
||||
webhooks,
|
||||
)
|
||||
|
||||
api_router = APIRouter()
|
||||
@@ -34,3 +35,4 @@ api_router.include_router(mail_sources.router, prefix="/mail-sources", tags=["ma
|
||||
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
|
||||
api_router.include_router(tls_reports.router, prefix="/tls-reports", tags=["tls-reports"])
|
||||
api_router.include_router(webhook.router, prefix="/webhook", tags=["webhook"])
|
||||
api_router.include_router(webhooks.router, prefix="/webhooks", tags=["webhooks"])
|
||||
|
||||
@@ -29,7 +29,11 @@ from app.services.alert_history import (
|
||||
record_alert_config_change,
|
||||
record_alert_evaluation,
|
||||
)
|
||||
from app.services.alert_rules import evaluate_alert_rules, send_current_alerts
|
||||
from app.services.alert_rules import (
|
||||
enqueue_alert_webhook_events,
|
||||
evaluate_alert_rules,
|
||||
send_current_alerts,
|
||||
)
|
||||
from app.services.notifications import send_notification
|
||||
from app.services.summary_notifications import build_summary, send_summary_notification
|
||||
|
||||
@@ -496,6 +500,7 @@ async def evaluate_notification_alerts(
|
||||
_seed_defaults(db)
|
||||
alerts = evaluate_alert_rules(db)
|
||||
record_alert_evaluation(db, alerts)
|
||||
enqueue_alert_webhook_events(db, alerts)
|
||||
return {"alerts": alerts}
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.core.redaction import sanitize_for_log
|
||||
from app.services.dmarc_parser import DMARCParser
|
||||
from app.services.report_persistence import report_exists, save_parsed_report
|
||||
from app.services.report_store import ReportStore
|
||||
from app.services.webhook_events import EVENT_REPORT_IMPORTED, enqueue_webhook_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -69,6 +70,22 @@ def _store_report(db: Session, store: ReportStore, report: Dict[str, Any]) -> st
|
||||
if report_id and (store.has_report(domain, report_id) or report_exists(db, domain, report_id)):
|
||||
return "duplicate"
|
||||
save_parsed_report(db, report)
|
||||
try:
|
||||
enqueue_webhook_event(
|
||||
db,
|
||||
event_type=EVENT_REPORT_IMPORTED,
|
||||
payload={
|
||||
"domain": domain,
|
||||
"report_id": report_id,
|
||||
"org_name": report.get("org_name"),
|
||||
"begin_date": report.get("begin_date"),
|
||||
"end_date": report.get("end_date"),
|
||||
"records": len(report.get("records") or []),
|
||||
},
|
||||
idempotency_key=f"{EVENT_REPORT_IMPORTED}:{domain}:{report_id or 'unknown'}",
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to queue report-import webhook event: %s", sanitize_for_log(exc))
|
||||
store.add_report(report)
|
||||
return "imported"
|
||||
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Admin endpoints for outbound webhook event delivery."""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.core.security import require_admin_auth
|
||||
from app.models.webhook import WebhookDelivery, WebhookEndpoint
|
||||
from app.services.webhook_events import (
|
||||
SUPPORTED_EVENT_TYPES,
|
||||
create_webhook_endpoint,
|
||||
deliver_due_webhooks,
|
||||
delivery_to_dict,
|
||||
endpoint_to_dict,
|
||||
queue_test_webhook,
|
||||
update_webhook_endpoint,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class WebhookEndpointCreate(BaseModel):
|
||||
"""Create payload for outbound webhook endpoints."""
|
||||
|
||||
name: str
|
||||
url: str
|
||||
secret: Optional[str] = None
|
||||
event_types: List[str] = ["*"]
|
||||
enabled: bool = True
|
||||
max_attempts: int = 5
|
||||
timeout_seconds: int = 10
|
||||
|
||||
|
||||
class WebhookEndpointUpdate(BaseModel):
|
||||
"""Update payload for outbound webhook endpoints."""
|
||||
|
||||
name: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
secret: Optional[str] = None
|
||||
event_types: Optional[List[str]] = None
|
||||
enabled: Optional[bool] = None
|
||||
max_attempts: Optional[int] = None
|
||||
timeout_seconds: Optional[int] = None
|
||||
|
||||
|
||||
class WebhookEndpointResponse(BaseModel):
|
||||
"""API-safe webhook endpoint metadata."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
url: str
|
||||
event_types: List[str]
|
||||
enabled: bool
|
||||
max_attempts: int
|
||||
timeout_seconds: int
|
||||
created_at: Optional[str]
|
||||
updated_at: Optional[str]
|
||||
last_success_at: Optional[str]
|
||||
last_failure_at: Optional[str]
|
||||
failure_count: int
|
||||
secret_configured: bool
|
||||
url_encrypted: bool
|
||||
secret: Optional[str] = None
|
||||
|
||||
|
||||
class WebhookEndpointListResponse(BaseModel):
|
||||
"""List response for outbound webhook endpoints."""
|
||||
|
||||
endpoints: List[WebhookEndpointResponse]
|
||||
supported_event_types: List[str]
|
||||
|
||||
|
||||
class WebhookDeliveryResponse(BaseModel):
|
||||
"""API-safe webhook delivery metadata."""
|
||||
|
||||
id: int
|
||||
endpoint_id: int
|
||||
event_type: str
|
||||
idempotency_key: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
max_attempts: int
|
||||
next_attempt_at: Optional[str]
|
||||
last_attempt_at: Optional[str]
|
||||
delivered_at: Optional[str]
|
||||
last_status_code: Optional[int]
|
||||
last_error: Optional[str]
|
||||
response_excerpt: Optional[str]
|
||||
created_at: Optional[str]
|
||||
updated_at: Optional[str]
|
||||
|
||||
|
||||
class WebhookDeliveryListResponse(BaseModel):
|
||||
"""List response for outbound webhook deliveries."""
|
||||
|
||||
deliveries: List[WebhookDeliveryResponse]
|
||||
|
||||
|
||||
class WebhookTestResponse(BaseModel):
|
||||
"""Response for a webhook test delivery."""
|
||||
|
||||
delivery: WebhookDeliveryResponse
|
||||
|
||||
|
||||
@router.get("", response_model=WebhookEndpointListResponse)
|
||||
async def list_webhook_endpoints(
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return configured outbound webhook endpoints."""
|
||||
endpoints = db.query(WebhookEndpoint).order_by(WebhookEndpoint.created_at.desc()).all()
|
||||
return {
|
||||
"endpoints": [endpoint_to_dict(endpoint) for endpoint in endpoints],
|
||||
"supported_event_types": SUPPORTED_EVENT_TYPES,
|
||||
}
|
||||
|
||||
|
||||
@router.post("", response_model=WebhookEndpointResponse)
|
||||
async def create_webhook(
|
||||
payload: WebhookEndpointCreate,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Create an outbound webhook endpoint."""
|
||||
try:
|
||||
endpoint, raw_secret = create_webhook_endpoint(db, **payload.model_dump())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
body = endpoint_to_dict(endpoint)
|
||||
body["secret"] = raw_secret
|
||||
return body
|
||||
|
||||
|
||||
@router.put("/{endpoint_id}", response_model=WebhookEndpointResponse)
|
||||
async def update_webhook(
|
||||
endpoint_id: int,
|
||||
payload: WebhookEndpointUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Update an outbound webhook endpoint."""
|
||||
endpoint = db.query(WebhookEndpoint).filter(WebhookEndpoint.id == endpoint_id).first()
|
||||
if endpoint is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Webhook endpoint not found"
|
||||
)
|
||||
try:
|
||||
endpoint, raw_secret = update_webhook_endpoint(
|
||||
db,
|
||||
endpoint,
|
||||
**payload.model_dump(exclude_unset=True),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
body = endpoint_to_dict(endpoint)
|
||||
body["secret"] = raw_secret
|
||||
return body
|
||||
|
||||
|
||||
@router.delete("/{endpoint_id}", response_model=WebhookEndpointResponse)
|
||||
async def disable_webhook(
|
||||
endpoint_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Disable a webhook endpoint without deleting delivery history."""
|
||||
endpoint = db.query(WebhookEndpoint).filter(WebhookEndpoint.id == endpoint_id).first()
|
||||
if endpoint is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Webhook endpoint not found"
|
||||
)
|
||||
endpoint.enabled = False
|
||||
db.commit()
|
||||
db.refresh(endpoint)
|
||||
body = endpoint_to_dict(endpoint)
|
||||
body["secret"] = None
|
||||
return body
|
||||
|
||||
|
||||
@router.get("/deliveries", response_model=WebhookDeliveryListResponse)
|
||||
async def list_webhook_deliveries(
|
||||
endpoint_id: Optional[int] = None,
|
||||
delivery_status: Optional[str] = Query(None, alias="status"),
|
||||
limit: int = 50,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return recent outbound webhook deliveries."""
|
||||
query = db.query(WebhookDelivery)
|
||||
if endpoint_id is not None:
|
||||
query = query.filter(WebhookDelivery.endpoint_id == endpoint_id)
|
||||
if delivery_status:
|
||||
query = query.filter(WebhookDelivery.status == delivery_status)
|
||||
deliveries = (
|
||||
query.order_by(WebhookDelivery.created_at.desc(), WebhookDelivery.id.desc())
|
||||
.limit(max(1, min(limit, 200)))
|
||||
.all()
|
||||
)
|
||||
return {"deliveries": [delivery_to_dict(delivery) for delivery in deliveries]}
|
||||
|
||||
|
||||
@router.post("/{endpoint_id}/test", response_model=WebhookTestResponse)
|
||||
async def test_webhook(
|
||||
endpoint_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Queue and immediately attempt a test delivery for an endpoint."""
|
||||
try:
|
||||
delivery = queue_test_webhook(db, endpoint_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
|
||||
delivered = deliver_due_webhooks(db, endpoint_id=endpoint_id, limit=1)
|
||||
return {"delivery": delivery_to_dict(delivered[0] if delivered else delivery)}
|
||||
|
||||
|
||||
@router.post("/deliveries/process", response_model=WebhookDeliveryListResponse)
|
||||
async def process_due_webhooks(
|
||||
limit: int = 25,
|
||||
db: Session = Depends(get_db),
|
||||
_auth: dict = Depends(require_admin_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Attempt due pending webhook deliveries."""
|
||||
deliveries = deliver_due_webhooks(db, limit=max(1, min(limit, 100)))
|
||||
return {"deliveries": [delivery_to_dict(delivery) for delivery in deliveries]}
|
||||
@@ -19,6 +19,7 @@ import app.models.mail_source_import # noqa: F401 – ensure import history tab
|
||||
import app.models.report # noqa: F401 – ensure DMARCReport/ReportRecord tables are registered
|
||||
import app.models.setting # noqa: F401 – ensure Setting table is registered
|
||||
import app.models.user # noqa: F401 – ensure User table is registered
|
||||
import app.models.webhook # noqa: F401 – ensure webhook tables are registered
|
||||
from app.api.api_v1.api import api_router
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import Base, SessionLocal, engine
|
||||
@@ -42,6 +43,7 @@ from app.services.runtime_status import (
|
||||
mark_scheduler_success,
|
||||
)
|
||||
from app.services.summary_notifications import send_due_scheduled_summaries
|
||||
from app.services.webhook_events import deliver_due_webhooks
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -290,6 +292,20 @@ def _send_due_summary_notifications() -> None:
|
||||
db.close()
|
||||
|
||||
|
||||
def _deliver_due_webhook_events() -> None:
|
||||
"""Attempt due outbound webhook deliveries."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deliveries = deliver_due_webhooks(db)
|
||||
if deliveries:
|
||||
delivered = sum(1 for item in deliveries if item.status == "delivered")
|
||||
logger.info(
|
||||
"Processed %d webhook deliveries (%d delivered)", len(deliveries), delivered
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _next_sleep_seconds(
|
||||
min_sleep: int = 60, enabled_sources: Optional[List[MailSource]] = None
|
||||
) -> int:
|
||||
@@ -318,6 +334,7 @@ async def scheduled_imap_polling():
|
||||
try:
|
||||
enabled_sources = _poll_all_enabled_sources()
|
||||
_send_due_summary_notifications()
|
||||
_deliver_due_webhook_events()
|
||||
mark_scheduler_success()
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error in IMAP polling task: %s", str(e))
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text
|
||||
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class WebhookEndpoint(Base):
|
||||
"""Outbound webhook endpoint configured by an operator."""
|
||||
|
||||
__tablename__ = "webhook_endpoints"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(120), nullable=False)
|
||||
url = Column(Text, nullable=False)
|
||||
secret = Column(Text, nullable=False)
|
||||
event_types = Column(Text, nullable=False, default="*")
|
||||
enabled = Column(Boolean, default=True, nullable=False, index=True)
|
||||
max_attempts = Column(Integer, default=5, nullable=False)
|
||||
timeout_seconds = Column(Integer, default=10, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
last_success_at = Column(DateTime, nullable=True, index=True)
|
||||
last_failure_at = Column(DateTime, nullable=True, index=True)
|
||||
failure_count = Column(Integer, default=0, nullable=False)
|
||||
|
||||
__table_args__ = (Index("ix_webhook_endpoints_enabled_events", "enabled", "event_types"),)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<WebhookEndpoint {self.name} enabled={self.enabled}>"
|
||||
|
||||
|
||||
class WebhookDelivery(Base):
|
||||
"""Single outbound webhook delivery attempt state."""
|
||||
|
||||
__tablename__ = "webhook_deliveries"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
endpoint_id = Column(Integer, ForeignKey("webhook_endpoints.id"), nullable=False, index=True)
|
||||
event_type = Column(String(80), nullable=False, index=True)
|
||||
payload = Column(Text, nullable=False)
|
||||
idempotency_key = Column(String(160), nullable=False, index=True)
|
||||
status = Column(String(24), nullable=False, default="pending", index=True)
|
||||
attempt_count = Column(Integer, default=0, nullable=False)
|
||||
max_attempts = Column(Integer, default=5, nullable=False)
|
||||
next_attempt_at = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
|
||||
last_attempt_at = Column(DateTime, nullable=True, index=True)
|
||||
delivered_at = Column(DateTime, nullable=True, index=True)
|
||||
last_status_code = Column(Integer, nullable=True)
|
||||
last_error = Column(Text, nullable=True)
|
||||
response_excerpt = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_webhook_delivery_endpoint_idempotency",
|
||||
"endpoint_id",
|
||||
"idempotency_key",
|
||||
unique=True,
|
||||
),
|
||||
Index("ix_webhook_delivery_due", "status", "next_attempt_at"),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<WebhookDelivery endpoint={self.endpoint_id} event={self.event_type} status={self.status}>"
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -13,6 +14,15 @@ from app.models.report import DMARCReport, ReportRecord
|
||||
from app.models.setting import Setting
|
||||
from app.services.alert_history import record_alert_evaluation
|
||||
from app.services.notifications import NotificationResult, send_notification
|
||||
from app.services.webhook_events import (
|
||||
EVENT_ALERT_CREATED,
|
||||
EVENT_COMPLIANCE_DROP,
|
||||
EVENT_REPORTS_MISSING,
|
||||
EVENT_SENDER_NEW,
|
||||
enqueue_webhook_event,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _truthy(value: Optional[str], default: bool = True) -> bool:
|
||||
@@ -235,10 +245,34 @@ def evaluate_alert_rules(db: Session) -> List[Dict[str, Any]]:
|
||||
return alerts
|
||||
|
||||
|
||||
def enqueue_alert_webhook_events(db: Session, alerts: List[Dict[str, Any]]) -> None:
|
||||
"""Queue webhook events for alert-rule results without failing alert evaluation."""
|
||||
event_by_rule = {
|
||||
"new_sender_source": EVENT_SENDER_NEW,
|
||||
"missing_reports": EVENT_REPORTS_MISSING,
|
||||
"compliance_drop": EVENT_COMPLIANCE_DROP,
|
||||
}
|
||||
for alert in alerts:
|
||||
rule = alert.get("rule", "alert")
|
||||
event_type = event_by_rule.get(rule, EVENT_ALERT_CREATED)
|
||||
domain = alert.get("domain", "global")
|
||||
idempotency_key = f"{event_type}:{domain}:{rule}:{alert.get('detail', '')}"
|
||||
try:
|
||||
enqueue_webhook_event(
|
||||
db,
|
||||
event_type=event_type,
|
||||
payload=alert,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to queue alert webhook event: %s", exc)
|
||||
|
||||
|
||||
def send_current_alerts(db: Session) -> Dict[str, Any]:
|
||||
"""Evaluate current alert rules and send one summary notification when needed."""
|
||||
alerts = evaluate_alert_rules(db)
|
||||
record_alert_evaluation(db, alerts)
|
||||
enqueue_alert_webhook_events(db, alerts)
|
||||
if not alerts:
|
||||
return {
|
||||
"alerts": [],
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
"""Outbound webhook event creation, signing, and delivery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.credential_encryption import decrypt_secret, encrypt_secret, is_encrypted_secret
|
||||
from app.models.webhook import WebhookDelivery, WebhookEndpoint
|
||||
|
||||
EVENT_REPORT_IMPORTED = "dmarq.report.imported"
|
||||
EVENT_SENDER_NEW = "dmarq.sender.new"
|
||||
EVENT_COMPLIANCE_DROP = "dmarq.compliance.drop"
|
||||
EVENT_REPORTS_MISSING = "dmarq.reports.missing"
|
||||
EVENT_ALERT_CREATED = "dmarq.alert.created"
|
||||
EVENT_ALERT_RESOLVED = "dmarq.alert.resolved"
|
||||
EVENT_WEBHOOK_TEST = "dmarq.webhook.test"
|
||||
|
||||
SUPPORTED_EVENT_TYPES = [
|
||||
EVENT_REPORT_IMPORTED,
|
||||
EVENT_SENDER_NEW,
|
||||
EVENT_COMPLIANCE_DROP,
|
||||
EVENT_REPORTS_MISSING,
|
||||
EVENT_ALERT_CREATED,
|
||||
EVENT_ALERT_RESOLVED,
|
||||
EVENT_WEBHOOK_TEST,
|
||||
]
|
||||
|
||||
DELIVERY_PENDING = "pending"
|
||||
DELIVERY_DELIVERED = "delivered"
|
||||
DELIVERY_FAILED = "failed"
|
||||
DELIVERY_ABANDONED = "abandoned"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeliveryAttemptResult:
|
||||
"""Result returned by a webhook HTTP sender."""
|
||||
|
||||
status_code: int
|
||||
body: str = ""
|
||||
|
||||
|
||||
WebhookSender = Callable[[str, bytes, Dict[str, str], int], DeliveryAttemptResult]
|
||||
|
||||
|
||||
def normalize_event_types(event_types: Iterable[str]) -> List[str]:
|
||||
"""Return validated event types for endpoint storage."""
|
||||
cleaned = sorted({item.strip() for item in event_types if item and item.strip()})
|
||||
if not cleaned:
|
||||
return ["*"]
|
||||
if "*" in cleaned:
|
||||
return ["*"]
|
||||
invalid = [item for item in cleaned if item not in SUPPORTED_EVENT_TYPES]
|
||||
if invalid:
|
||||
raise ValueError(f"Unsupported webhook event type: {', '.join(invalid)}")
|
||||
return cleaned
|
||||
|
||||
|
||||
def event_types_to_string(event_types: Iterable[str]) -> str:
|
||||
"""Serialize event types for storage."""
|
||||
return ",".join(normalize_event_types(event_types))
|
||||
|
||||
|
||||
def parse_event_types(value: str) -> List[str]:
|
||||
"""Parse stored event types."""
|
||||
return normalize_event_types((value or "*").split(","))
|
||||
|
||||
|
||||
def endpoint_matches_event(endpoint: WebhookEndpoint, event_type: str) -> bool:
|
||||
"""Return True when an endpoint should receive an event."""
|
||||
event_types = parse_event_types(endpoint.event_types)
|
||||
return "*" in event_types or event_type in event_types
|
||||
|
||||
|
||||
def generate_webhook_secret() -> str:
|
||||
"""Generate a signing secret for outbound webhook deliveries."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def _encrypt(value: str) -> str:
|
||||
return encrypt_secret(value) or ""
|
||||
|
||||
|
||||
def _decrypt(value: str) -> str:
|
||||
return decrypt_secret(value) or ""
|
||||
|
||||
|
||||
def _redact_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return "[invalid url]"
|
||||
host = parsed.hostname or parsed.netloc
|
||||
port = f":{parsed.port}" if parsed.port else ""
|
||||
path = parsed.path or "/"
|
||||
return f"{parsed.scheme}://{host}{port}{path}"
|
||||
|
||||
|
||||
def validate_webhook_url(url: str) -> str:
|
||||
"""Validate and normalize an outbound webhook URL."""
|
||||
clean_url = url.strip()
|
||||
parsed = urlparse(clean_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("Webhook URL must be an absolute http or https URL")
|
||||
return clean_url
|
||||
|
||||
|
||||
def create_webhook_endpoint(
|
||||
db: Session,
|
||||
*,
|
||||
name: str,
|
||||
url: str,
|
||||
secret: Optional[str] = None,
|
||||
event_types: Iterable[str] = ("*",),
|
||||
enabled: bool = True,
|
||||
max_attempts: int = 5,
|
||||
timeout_seconds: int = 10,
|
||||
) -> Tuple[WebhookEndpoint, str]:
|
||||
"""Create a webhook endpoint and return the endpoint plus raw signing secret."""
|
||||
clean_name = name.strip()
|
||||
if not clean_name:
|
||||
raise ValueError("Webhook name is required")
|
||||
clean_url = validate_webhook_url(url)
|
||||
raw_secret = secret.strip() if secret else generate_webhook_secret()
|
||||
if len(raw_secret) < 16:
|
||||
raise ValueError("Webhook signing secret must be at least 16 characters")
|
||||
|
||||
endpoint = WebhookEndpoint(
|
||||
name=clean_name,
|
||||
url=_encrypt(clean_url),
|
||||
secret=_encrypt(raw_secret),
|
||||
event_types=event_types_to_string(event_types),
|
||||
enabled=enabled,
|
||||
max_attempts=max(1, min(int(max_attempts or 5), 10)),
|
||||
timeout_seconds=max(1, min(int(timeout_seconds or 10), 30)),
|
||||
)
|
||||
db.add(endpoint)
|
||||
db.commit()
|
||||
db.refresh(endpoint)
|
||||
return endpoint, raw_secret
|
||||
|
||||
|
||||
def update_webhook_endpoint(
|
||||
db: Session,
|
||||
endpoint: WebhookEndpoint,
|
||||
*,
|
||||
name: Optional[str] = None,
|
||||
url: Optional[str] = None,
|
||||
secret: Optional[str] = None,
|
||||
event_types: Optional[Iterable[str]] = None,
|
||||
enabled: Optional[bool] = None,
|
||||
max_attempts: Optional[int] = None,
|
||||
timeout_seconds: Optional[int] = None,
|
||||
) -> Tuple[WebhookEndpoint, Optional[str]]:
|
||||
"""Update a webhook endpoint. Return the endpoint and newly supplied/generated secret."""
|
||||
returned_secret = None
|
||||
if name is not None:
|
||||
clean_name = name.strip()
|
||||
if not clean_name:
|
||||
raise ValueError("Webhook name is required")
|
||||
endpoint.name = clean_name
|
||||
if url is not None and url.strip() and url != "**redacted**":
|
||||
endpoint.url = _encrypt(validate_webhook_url(url))
|
||||
if secret is not None and secret.strip() and secret != "**redacted**":
|
||||
returned_secret = secret.strip()
|
||||
if len(returned_secret) < 16:
|
||||
raise ValueError("Webhook signing secret must be at least 16 characters")
|
||||
endpoint.secret = _encrypt(returned_secret)
|
||||
if event_types is not None:
|
||||
endpoint.event_types = event_types_to_string(event_types)
|
||||
if enabled is not None:
|
||||
endpoint.enabled = enabled
|
||||
if max_attempts is not None:
|
||||
endpoint.max_attempts = max(1, min(int(max_attempts), 10))
|
||||
if timeout_seconds is not None:
|
||||
endpoint.timeout_seconds = max(1, min(int(timeout_seconds), 30))
|
||||
db.commit()
|
||||
db.refresh(endpoint)
|
||||
return endpoint, returned_secret
|
||||
|
||||
|
||||
def _stable_json(value: Dict[str, Any]) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def build_event_payload(event_type: str, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Wrap event data in a stable, documented envelope."""
|
||||
now = datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
|
||||
return {
|
||||
"event_type": event_type,
|
||||
"created_at": now,
|
||||
"data": payload,
|
||||
}
|
||||
|
||||
|
||||
def default_idempotency_key(event_type: str, payload: Dict[str, Any]) -> str:
|
||||
"""Build a deterministic idempotency key for an event payload."""
|
||||
digest = hashlib.sha256(_stable_json(payload).encode("utf-8")).hexdigest()[:32]
|
||||
return f"{event_type}:{digest}"
|
||||
|
||||
|
||||
def enqueue_webhook_event(
|
||||
db: Session,
|
||||
*,
|
||||
event_type: str,
|
||||
payload: Dict[str, Any],
|
||||
idempotency_key: Optional[str] = None,
|
||||
) -> List[WebhookDelivery]:
|
||||
"""Create pending deliveries for all enabled endpoints matching an event."""
|
||||
if event_type not in SUPPORTED_EVENT_TYPES:
|
||||
raise ValueError(f"Unsupported webhook event type: {event_type}")
|
||||
endpoints = db.query(WebhookEndpoint).filter(WebhookEndpoint.enabled.is_(True)).all()
|
||||
event_payload = build_event_payload(event_type, payload)
|
||||
key = idempotency_key or default_idempotency_key(event_type, event_payload)
|
||||
deliveries: List[WebhookDelivery] = []
|
||||
|
||||
for endpoint in endpoints:
|
||||
if not endpoint_matches_event(endpoint, event_type):
|
||||
continue
|
||||
delivery = WebhookDelivery(
|
||||
endpoint_id=endpoint.id,
|
||||
event_type=event_type,
|
||||
payload=_stable_json(event_payload),
|
||||
idempotency_key=key,
|
||||
status=DELIVERY_PENDING,
|
||||
max_attempts=endpoint.max_attempts,
|
||||
)
|
||||
db.add(delivery)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
existing = (
|
||||
db.query(WebhookDelivery)
|
||||
.filter(
|
||||
WebhookDelivery.endpoint_id == endpoint.id,
|
||||
WebhookDelivery.idempotency_key == key,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
deliveries.append(existing)
|
||||
continue
|
||||
db.refresh(delivery)
|
||||
deliveries.append(delivery)
|
||||
return deliveries
|
||||
|
||||
|
||||
def _delivery_body(delivery: WebhookDelivery) -> bytes:
|
||||
return delivery.payload.encode("utf-8")
|
||||
|
||||
|
||||
def sign_delivery(secret: str, delivery: WebhookDelivery, timestamp: int, body: bytes) -> str:
|
||||
"""Return the v1 HMAC signature for a delivery."""
|
||||
signed = f"{timestamp}.{delivery.id}.".encode("utf-8") + body
|
||||
digest = hmac.new(secret.encode("utf-8"), signed, hashlib.sha256).hexdigest()
|
||||
return f"v1={digest}"
|
||||
|
||||
|
||||
def build_delivery_headers(endpoint: WebhookEndpoint, delivery: WebhookDelivery) -> Dict[str, str]:
|
||||
"""Build outbound webhook headers with event metadata and HMAC signature."""
|
||||
body = _delivery_body(delivery)
|
||||
timestamp = int(datetime.utcnow().timestamp())
|
||||
secret = _decrypt(endpoint.secret)
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "DMARQ-Webhooks/1.0",
|
||||
"X-DMARQ-Event": delivery.event_type,
|
||||
"X-DMARQ-Delivery": str(delivery.id),
|
||||
"X-DMARQ-Idempotency-Key": delivery.idempotency_key,
|
||||
"X-DMARQ-Timestamp": str(timestamp),
|
||||
"X-DMARQ-Signature": sign_delivery(secret, delivery, timestamp, body),
|
||||
}
|
||||
|
||||
|
||||
def default_webhook_sender(
|
||||
url: str, body: bytes, headers: Dict[str, str], timeout_seconds: int
|
||||
) -> DeliveryAttemptResult:
|
||||
"""Send a webhook delivery using the Python standard library."""
|
||||
request = urllib.request.Request(url=url, data=body, headers=headers, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response: # nosec B310
|
||||
response_body = response.read(4096).decode("utf-8", errors="replace")
|
||||
return DeliveryAttemptResult(status_code=response.status, body=response_body)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response_body = exc.read(4096).decode("utf-8", errors="replace")
|
||||
return DeliveryAttemptResult(status_code=exc.code, body=response_body)
|
||||
except urllib.error.URLError as exc:
|
||||
raise ConnectionError(str(exc.reason)) from exc
|
||||
|
||||
|
||||
def _backoff_for_attempt(attempt_count: int) -> timedelta:
|
||||
seconds = min(3600, 60 * (2 ** max(0, attempt_count - 1)))
|
||||
return timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def _response_excerpt(value: str) -> str:
|
||||
return (value or "")[:500]
|
||||
|
||||
|
||||
def deliver_webhook_delivery(
|
||||
db: Session,
|
||||
delivery: WebhookDelivery,
|
||||
*,
|
||||
sender: WebhookSender = default_webhook_sender,
|
||||
) -> WebhookDelivery:
|
||||
"""Attempt one webhook delivery and persist retry state."""
|
||||
endpoint = db.query(WebhookEndpoint).filter(WebhookEndpoint.id == delivery.endpoint_id).first()
|
||||
now = datetime.utcnow()
|
||||
delivery.attempt_count = int(delivery.attempt_count or 0) + 1
|
||||
delivery.last_attempt_at = now
|
||||
|
||||
if endpoint is None or not endpoint.enabled:
|
||||
delivery.status = DELIVERY_ABANDONED
|
||||
delivery.last_error = "Webhook endpoint is disabled or missing."
|
||||
delivery.next_attempt_at = now
|
||||
db.commit()
|
||||
db.refresh(delivery)
|
||||
return delivery
|
||||
|
||||
try:
|
||||
body = _delivery_body(delivery)
|
||||
result = sender(
|
||||
_decrypt(endpoint.url),
|
||||
body,
|
||||
build_delivery_headers(endpoint, delivery),
|
||||
endpoint.timeout_seconds,
|
||||
)
|
||||
delivery.last_status_code = result.status_code
|
||||
delivery.response_excerpt = _response_excerpt(result.body)
|
||||
if 200 <= result.status_code < 300:
|
||||
delivery.status = DELIVERY_DELIVERED
|
||||
delivery.delivered_at = now
|
||||
delivery.last_error = None
|
||||
delivery.next_attempt_at = now
|
||||
endpoint.last_success_at = now
|
||||
endpoint.failure_count = 0
|
||||
else:
|
||||
delivery.last_error = f"HTTP {result.status_code}"
|
||||
_mark_delivery_failure(delivery, endpoint, now)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
delivery.last_error = str(exc)[:500]
|
||||
_mark_delivery_failure(delivery, endpoint, now)
|
||||
|
||||
db.commit()
|
||||
db.refresh(delivery)
|
||||
return delivery
|
||||
|
||||
|
||||
def _mark_delivery_failure(
|
||||
delivery: WebhookDelivery, endpoint: WebhookEndpoint, now: datetime
|
||||
) -> None:
|
||||
endpoint.last_failure_at = now
|
||||
endpoint.failure_count = int(endpoint.failure_count or 0) + 1
|
||||
if delivery.attempt_count >= delivery.max_attempts:
|
||||
delivery.status = DELIVERY_FAILED
|
||||
delivery.next_attempt_at = now
|
||||
return
|
||||
delivery.status = DELIVERY_PENDING
|
||||
delivery.next_attempt_at = now + _backoff_for_attempt(delivery.attempt_count)
|
||||
|
||||
|
||||
def deliver_due_webhooks(
|
||||
db: Session,
|
||||
*,
|
||||
limit: int = 25,
|
||||
endpoint_id: Optional[int] = None,
|
||||
sender: WebhookSender = default_webhook_sender,
|
||||
) -> List[WebhookDelivery]:
|
||||
"""Deliver pending webhook deliveries whose retry time has arrived."""
|
||||
now = datetime.utcnow()
|
||||
query = db.query(WebhookDelivery).filter(
|
||||
WebhookDelivery.status == DELIVERY_PENDING,
|
||||
WebhookDelivery.next_attempt_at <= now,
|
||||
)
|
||||
if endpoint_id is not None:
|
||||
query = query.filter(WebhookDelivery.endpoint_id == endpoint_id)
|
||||
deliveries = (
|
||||
query.order_by(WebhookDelivery.next_attempt_at, WebhookDelivery.id).limit(limit).all()
|
||||
)
|
||||
return [deliver_webhook_delivery(db, delivery, sender=sender) for delivery in deliveries]
|
||||
|
||||
|
||||
def queue_test_webhook(db: Session, endpoint_id: int) -> WebhookDelivery:
|
||||
"""Queue a one-off test delivery for a specific webhook endpoint."""
|
||||
endpoint = db.query(WebhookEndpoint).filter(WebhookEndpoint.id == endpoint_id).first()
|
||||
if endpoint is None:
|
||||
raise ValueError("Webhook endpoint not found")
|
||||
payload = {
|
||||
"endpoint_id": endpoint.id,
|
||||
"endpoint_name": endpoint.name,
|
||||
"message": "DMARQ webhook test delivery",
|
||||
}
|
||||
delivery = WebhookDelivery(
|
||||
endpoint_id=endpoint.id,
|
||||
event_type=EVENT_WEBHOOK_TEST,
|
||||
payload=_stable_json(build_event_payload(EVENT_WEBHOOK_TEST, payload)),
|
||||
idempotency_key=f"{EVENT_WEBHOOK_TEST}:{secrets.token_hex(16)}",
|
||||
status=DELIVERY_PENDING,
|
||||
max_attempts=endpoint.max_attempts,
|
||||
)
|
||||
db.add(delivery)
|
||||
db.commit()
|
||||
db.refresh(delivery)
|
||||
return delivery
|
||||
|
||||
|
||||
def endpoint_to_dict(endpoint: WebhookEndpoint) -> Dict[str, Any]:
|
||||
"""Return an API-safe webhook endpoint representation."""
|
||||
raw_url = _decrypt(endpoint.url)
|
||||
return {
|
||||
"id": endpoint.id,
|
||||
"name": endpoint.name,
|
||||
"url": _redact_url(raw_url),
|
||||
"event_types": parse_event_types(endpoint.event_types),
|
||||
"enabled": endpoint.enabled,
|
||||
"max_attempts": endpoint.max_attempts,
|
||||
"timeout_seconds": endpoint.timeout_seconds,
|
||||
"created_at": endpoint.created_at.isoformat() if endpoint.created_at else None,
|
||||
"updated_at": endpoint.updated_at.isoformat() if endpoint.updated_at else None,
|
||||
"last_success_at": (
|
||||
endpoint.last_success_at.isoformat() if endpoint.last_success_at else None
|
||||
),
|
||||
"last_failure_at": (
|
||||
endpoint.last_failure_at.isoformat() if endpoint.last_failure_at else None
|
||||
),
|
||||
"failure_count": endpoint.failure_count,
|
||||
"secret_configured": bool(endpoint.secret),
|
||||
"url_encrypted": is_encrypted_secret(endpoint.url),
|
||||
}
|
||||
|
||||
|
||||
def delivery_to_dict(delivery: WebhookDelivery) -> Dict[str, Any]:
|
||||
"""Return an API-safe delivery representation."""
|
||||
return {
|
||||
"id": delivery.id,
|
||||
"endpoint_id": delivery.endpoint_id,
|
||||
"event_type": delivery.event_type,
|
||||
"idempotency_key": delivery.idempotency_key,
|
||||
"status": delivery.status,
|
||||
"attempt_count": delivery.attempt_count,
|
||||
"max_attempts": delivery.max_attempts,
|
||||
"next_attempt_at": (
|
||||
delivery.next_attempt_at.isoformat() if delivery.next_attempt_at else None
|
||||
),
|
||||
"last_attempt_at": (
|
||||
delivery.last_attempt_at.isoformat() if delivery.last_attempt_at else None
|
||||
),
|
||||
"delivered_at": delivery.delivered_at.isoformat() if delivery.delivered_at else None,
|
||||
"last_status_code": delivery.last_status_code,
|
||||
"last_error": delivery.last_error,
|
||||
"response_excerpt": delivery.response_excerpt,
|
||||
"created_at": delivery.created_at.isoformat() if delivery.created_at else None,
|
||||
"updated_at": delivery.updated_at.isoformat() if delivery.updated_at else None,
|
||||
}
|
||||
@@ -633,6 +633,151 @@
|
||||
{% endcall %}
|
||||
{% endcall %}
|
||||
|
||||
<!-- ── Webhooks ──────────────────────────────────────────────────────── -->
|
||||
{% call card() %}
|
||||
{% call card_header() %}
|
||||
{% call card_title() %}Webhooks{% endcall %}
|
||||
{% call card_description() %}Send signed operational events to downstream systems{% endcall %}
|
||||
{% endcall %}
|
||||
{% call card_content() %}
|
||||
<div class="space-y-5">
|
||||
<form @submit.prevent="createWebhook()" class="space-y-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Name</span></label>
|
||||
<input type="text" x-model="newWebhook.name" class="input input-bordered w-full" placeholder="Security automation" />
|
||||
</div>
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Endpoint URL</span></label>
|
||||
<input type="url" x-model="newWebhook.url" class="input input-bordered w-full" placeholder="https://example.com/dmarq/webhook" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Events</span></label>
|
||||
<select x-model="newWebhook.eventType" class="input input-bordered w-full">
|
||||
<option value="*">All events</option>
|
||||
<template x-for="eventType in webhookEventTypes" :key="eventType">
|
||||
<option :value="eventType" x-text="eventType"></option>
|
||||
</template>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Max Attempts</span></label>
|
||||
<input type="number" x-model.number="newWebhook.max_attempts" class="input input-bordered w-full" min="1" max="10" />
|
||||
</div>
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text font-medium">Timeout Seconds</span></label>
|
||||
<input type="number" x-model.number="newWebhook.timeout_seconds" class="input input-bordered w-full" min="1" max="30" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col sm:flex-row justify-end gap-2">
|
||||
<button type="button" class="btn btn-outline btn-md" :disabled="loadingWebhooks" @click="loadWebhooks()">
|
||||
<template x-if="!loadingWebhooks">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none"
|
||||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="mr-2">
|
||||
<path d="M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8"></path>
|
||||
<path d="M3 3v5h5"></path>
|
||||
</svg>
|
||||
</template>
|
||||
<template x-if="loadingWebhooks"><span class="loading loading-spinner loading-xs mr-2"></span></template>
|
||||
Refresh
|
||||
</button>
|
||||
<button type="submit" class="btn btn-default btn-md" :disabled="savingWebhook">
|
||||
<template x-if="!savingWebhook">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none"
|
||||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="mr-2">
|
||||
<path d="M12 5v14"></path>
|
||||
<path d="M5 12h14"></path>
|
||||
</svg>
|
||||
</template>
|
||||
<template x-if="savingWebhook"><span class="loading loading-spinner loading-xs mr-2"></span></template>
|
||||
Add Webhook
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<template x-if="webhooks.length === 0">
|
||||
<div class="alert alert-info">
|
||||
<span>No outbound webhooks configured yet.</span>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<div class="overflow-x-auto rounded-md border border-border" x-show="webhooks.length > 0">
|
||||
<table class="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>URL</th>
|
||||
<th>Events</th>
|
||||
<th>Status</th>
|
||||
<th>Last Result</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<template x-for="hook in webhooks" :key="hook.id">
|
||||
<tr>
|
||||
<td class="font-medium" x-text="hook.name"></td>
|
||||
<td class="font-mono text-xs" x-text="hook.url"></td>
|
||||
<td class="text-xs" x-text="hook.event_types.join(', ')"></td>
|
||||
<td><span class="badge" :class="hook.enabled ? 'badge-success' : 'badge-ghost'" x-text="hook.enabled ? 'Enabled' : 'Disabled'"></span></td>
|
||||
<td class="text-xs text-muted-foreground" x-text="hook.last_success_at ? 'Success ' + new Date(hook.last_success_at).toLocaleString() : (hook.last_failure_at ? 'Failed ' + new Date(hook.last_failure_at).toLocaleString() : 'No deliveries')"></td>
|
||||
<td class="text-right">
|
||||
<button type="button" class="btn btn-outline btn-xs" :disabled="testingWebhookId === hook.id" @click="testWebhook(hook.id)">
|
||||
<span x-show="testingWebhookId !== hook.id">Test</span>
|
||||
<span x-show="testingWebhookId === hook.id" class="loading loading-spinner loading-xs"></span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-ghost btn-xs" :disabled="disablingWebhookId === hook.id || !hook.enabled" @click="disableWebhook(hook.id)">
|
||||
Disable
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="border-t border-border pt-4 space-y-3">
|
||||
<div class="flex items-center justify-between gap-3">
|
||||
<h3 class="text-sm font-semibold">Recent Deliveries</h3>
|
||||
<div class="flex gap-2">
|
||||
<button type="button" class="btn btn-outline btn-sm" :disabled="processingWebhooks" @click="processWebhooks()">
|
||||
Process Due
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline btn-sm" :disabled="loadingWebhookDeliveries" @click="loadWebhookDeliveries()">
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<template x-if="webhookDeliveries.length === 0">
|
||||
<div class="alert alert-info">
|
||||
<span>No webhook deliveries recorded yet.</span>
|
||||
</div>
|
||||
</template>
|
||||
<div class="space-y-2" x-show="webhookDeliveries.length > 0">
|
||||
<template x-for="delivery in webhookDeliveries" :key="delivery.id">
|
||||
<div class="rounded-md border border-border p-3">
|
||||
<div class="flex flex-col gap-1 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
<div class="text-sm font-semibold" x-text="delivery.event_type"></div>
|
||||
<div class="text-sm text-muted-foreground" x-text="delivery.last_error || delivery.response_excerpt || delivery.idempotency_key"></div>
|
||||
</div>
|
||||
<div class="flex items-center gap-2 text-xs">
|
||||
<span class="badge" :class="delivery.status === 'delivered' ? 'badge-success' : (delivery.status === 'failed' ? 'badge-error' : 'badge-warning')" x-text="delivery.status"></span>
|
||||
<span class="badge badge-outline" x-text="delivery.attempt_count + '/' + delivery.max_attempts"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endcall %}
|
||||
{% endcall %}
|
||||
|
||||
<!-- ── Mail Sources shortcut ──────────────────────────────────────────── -->
|
||||
{% call card() %}
|
||||
{% call card_header() %}
|
||||
@@ -681,6 +826,22 @@ function settingsApp() {
|
||||
importingCfZones: false,
|
||||
cfZones: [],
|
||||
showCfToken: false,
|
||||
loadingWebhooks: false,
|
||||
savingWebhook: false,
|
||||
testingWebhookId: null,
|
||||
disablingWebhookId: null,
|
||||
processingWebhooks: false,
|
||||
loadingWebhookDeliveries: false,
|
||||
webhooks: [],
|
||||
webhookDeliveries: [],
|
||||
webhookEventTypes: [],
|
||||
newWebhook: {
|
||||
name: '',
|
||||
url: '',
|
||||
eventType: '*',
|
||||
max_attempts: 5,
|
||||
timeout_seconds: 10,
|
||||
},
|
||||
|
||||
// Session cookie is sent automatically by the browser (httpOnly, same-origin).
|
||||
// No manual auth header needed for API calls from the UI.
|
||||
@@ -705,6 +866,8 @@ function settingsApp() {
|
||||
this.s = map;
|
||||
await this.loadAlertHistory(false);
|
||||
await this.loadConfigAudit(false);
|
||||
await this.loadWebhooks(false);
|
||||
await this.loadWebhookDeliveries(false);
|
||||
} catch (err) {
|
||||
this.showFlash('Error loading settings: ' + err.message, false);
|
||||
}
|
||||
@@ -951,6 +1114,142 @@ function settingsApp() {
|
||||
}
|
||||
},
|
||||
|
||||
async loadWebhooks(showMessage = true) {
|
||||
this.loadingWebhooks = true;
|
||||
try {
|
||||
const res = await fetch('/api/v1/webhooks', { headers: this.apiHeaders() });
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
if (showMessage) this.showFlash('Webhook load failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
this.webhooks = data.endpoints || [];
|
||||
this.webhookEventTypes = data.supported_event_types || [];
|
||||
if (showMessage) this.showFlash('Webhooks refreshed.', true);
|
||||
}
|
||||
} catch (err) {
|
||||
if (showMessage) this.showFlash('Error loading webhooks: ' + err.message, false);
|
||||
} finally {
|
||||
this.loadingWebhooks = false;
|
||||
}
|
||||
},
|
||||
|
||||
async createWebhook() {
|
||||
this.savingWebhook = true;
|
||||
try {
|
||||
const payload = {
|
||||
name: this.newWebhook.name,
|
||||
url: this.newWebhook.url,
|
||||
event_types: [this.newWebhook.eventType || '*'],
|
||||
max_attempts: Number(this.newWebhook.max_attempts || 5),
|
||||
timeout_seconds: Number(this.newWebhook.timeout_seconds || 10),
|
||||
enabled: true,
|
||||
};
|
||||
const res = await fetch('/api/v1/webhooks', {
|
||||
method: 'POST',
|
||||
headers: this.apiHeaders(),
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
this.showFlash('Webhook create failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
this.newWebhook = { name: '', url: '', eventType: '*', max_attempts: 5, timeout_seconds: 10 };
|
||||
await this.loadWebhooks(false);
|
||||
this.showFlash('Webhook created. Signing secret was generated and stored securely.', true);
|
||||
}
|
||||
} catch (err) {
|
||||
this.showFlash('Error creating webhook: ' + err.message, false);
|
||||
} finally {
|
||||
this.savingWebhook = false;
|
||||
}
|
||||
},
|
||||
|
||||
async testWebhook(endpointId) {
|
||||
this.testingWebhookId = endpointId;
|
||||
try {
|
||||
const res = await fetch(`/api/v1/webhooks/${endpointId}/test`, {
|
||||
method: 'POST',
|
||||
headers: this.apiHeaders(),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
this.showFlash('Webhook test failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
await this.loadWebhooks(false);
|
||||
await this.loadWebhookDeliveries(false);
|
||||
const status = data.delivery ? data.delivery.status : 'queued';
|
||||
this.showFlash(`Webhook test ${status}.`, status === 'delivered');
|
||||
}
|
||||
} catch (err) {
|
||||
this.showFlash('Error testing webhook: ' + err.message, false);
|
||||
} finally {
|
||||
this.testingWebhookId = null;
|
||||
}
|
||||
},
|
||||
|
||||
async disableWebhook(endpointId) {
|
||||
this.disablingWebhookId = endpointId;
|
||||
try {
|
||||
const res = await fetch(`/api/v1/webhooks/${endpointId}`, {
|
||||
method: 'DELETE',
|
||||
headers: this.apiHeaders(),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
this.showFlash('Webhook disable failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
await this.loadWebhooks(false);
|
||||
this.showFlash('Webhook disabled.', true);
|
||||
}
|
||||
} catch (err) {
|
||||
this.showFlash('Error disabling webhook: ' + err.message, false);
|
||||
} finally {
|
||||
this.disablingWebhookId = null;
|
||||
}
|
||||
},
|
||||
|
||||
async loadWebhookDeliveries(showMessage = true) {
|
||||
this.loadingWebhookDeliveries = true;
|
||||
try {
|
||||
const res = await fetch('/api/v1/webhooks/deliveries?limit=10', {
|
||||
headers: this.apiHeaders(),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
if (showMessage) this.showFlash('Delivery history failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
this.webhookDeliveries = data.deliveries || [];
|
||||
if (showMessage) this.showFlash('Webhook deliveries refreshed.', true);
|
||||
}
|
||||
} catch (err) {
|
||||
if (showMessage) this.showFlash('Error loading webhook deliveries: ' + err.message, false);
|
||||
} finally {
|
||||
this.loadingWebhookDeliveries = false;
|
||||
}
|
||||
},
|
||||
|
||||
async processWebhooks() {
|
||||
this.processingWebhooks = true;
|
||||
try {
|
||||
const res = await fetch('/api/v1/webhooks/deliveries/process', {
|
||||
method: 'POST',
|
||||
headers: this.apiHeaders(),
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
this.showFlash('Webhook processing failed: ' + (data.detail || res.statusText), false);
|
||||
} else {
|
||||
await this.loadWebhooks(false);
|
||||
await this.loadWebhookDeliveries(false);
|
||||
this.showFlash(`${(data.deliveries || []).length} due webhook deliver${(data.deliveries || []).length === 1 ? 'y' : 'ies'} processed.`, true);
|
||||
}
|
||||
} catch (err) {
|
||||
this.showFlash('Error processing webhooks: ' + err.message, false);
|
||||
} finally {
|
||||
this.processingWebhooks = false;
|
||||
}
|
||||
},
|
||||
|
||||
showFlash(msg, ok) {
|
||||
this.flashMsg = msg;
|
||||
this.flashOk = ok;
|
||||
|
||||
@@ -15,6 +15,7 @@ import app.models.mail_source_import # noqa: F401 # pylint: disable=unused-imp
|
||||
import app.models.report # noqa: F401 # pylint: disable=unused-import
|
||||
import app.models.setting # noqa: F401 # pylint: disable=unused-import
|
||||
import app.models.user # noqa: F401 # pylint: disable=unused-import
|
||||
import app.models.webhook # noqa: F401 # pylint: disable=unused-import
|
||||
from app.core.database import Base, get_db
|
||||
from app.core.security import require_admin_auth
|
||||
from app.main import create_app
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
import hmac
|
||||
import json
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models.webhook import WebhookDelivery
|
||||
from app.services.webhook_events import (
|
||||
DELIVERY_ABANDONED,
|
||||
DELIVERY_DELIVERED,
|
||||
DELIVERY_FAILED,
|
||||
DELIVERY_PENDING,
|
||||
EVENT_ALERT_CREATED,
|
||||
EVENT_REPORTS_MISSING,
|
||||
EVENT_WEBHOOK_TEST,
|
||||
DeliveryAttemptResult,
|
||||
create_webhook_endpoint,
|
||||
deliver_due_webhooks,
|
||||
deliver_webhook_delivery,
|
||||
endpoint_to_dict,
|
||||
enqueue_webhook_event,
|
||||
normalize_event_types,
|
||||
queue_test_webhook,
|
||||
sign_delivery,
|
||||
update_webhook_endpoint,
|
||||
validate_webhook_url,
|
||||
)
|
||||
|
||||
|
||||
def test_webhook_event_delivery_signs_and_records_success(db_session):
|
||||
"""Webhook deliveries include replay-resistant metadata and mark success."""
|
||||
endpoint, secret = create_webhook_endpoint(
|
||||
db_session,
|
||||
name="receiver",
|
||||
url="https://receiver.example/webhook?token=hidden",
|
||||
event_types=[EVENT_ALERT_CREATED],
|
||||
)
|
||||
deliveries = enqueue_webhook_event(
|
||||
db_session,
|
||||
event_type=EVENT_ALERT_CREATED,
|
||||
payload={"domain": "example.com", "detail": "alert"},
|
||||
idempotency_key="alert-example",
|
||||
)
|
||||
assert len(deliveries) == 1
|
||||
|
||||
seen = {}
|
||||
|
||||
def sender(url, body, headers, timeout_seconds):
|
||||
seen["url"] = url
|
||||
seen["body"] = body
|
||||
seen["headers"] = headers
|
||||
seen["timeout_seconds"] = timeout_seconds
|
||||
expected = sign_delivery(
|
||||
secret,
|
||||
deliveries[0],
|
||||
int(headers["X-DMARQ-Timestamp"]),
|
||||
body,
|
||||
)
|
||||
assert hmac.compare_digest(headers["X-DMARQ-Signature"], expected)
|
||||
assert headers["X-DMARQ-Event"] == EVENT_ALERT_CREATED
|
||||
assert headers["X-DMARQ-Idempotency-Key"] == "alert-example"
|
||||
return DeliveryAttemptResult(status_code=204, body="")
|
||||
|
||||
delivered = deliver_due_webhooks(db_session, sender=sender)
|
||||
|
||||
assert delivered[0].status == DELIVERY_DELIVERED
|
||||
assert json.loads(seen["body"])["data"]["domain"] == "example.com"
|
||||
assert seen["url"] == "https://receiver.example/webhook?token=hidden"
|
||||
db_session.refresh(endpoint)
|
||||
assert endpoint.last_success_at is not None
|
||||
assert endpoint.failure_count == 0
|
||||
|
||||
|
||||
def test_webhook_delivery_retries_then_fails(db_session):
|
||||
"""Transient failures remain pending with backoff, then fail at max attempts."""
|
||||
create_webhook_endpoint(
|
||||
db_session,
|
||||
name="receiver",
|
||||
url="https://receiver.example/webhook",
|
||||
event_types=[EVENT_ALERT_CREATED],
|
||||
max_attempts=2,
|
||||
)
|
||||
enqueue_webhook_event(
|
||||
db_session,
|
||||
event_type=EVENT_ALERT_CREATED,
|
||||
payload={"domain": "example.com"},
|
||||
idempotency_key="retry-example",
|
||||
)[0]
|
||||
|
||||
def failing_sender(url, body, headers, timeout_seconds):
|
||||
return DeliveryAttemptResult(status_code=503, body="try later")
|
||||
|
||||
first = deliver_due_webhooks(db_session, sender=failing_sender)[0]
|
||||
assert first.status == DELIVERY_PENDING
|
||||
assert first.attempt_count == 1
|
||||
assert first.next_attempt_at > first.last_attempt_at
|
||||
|
||||
first.next_attempt_at = first.last_attempt_at
|
||||
db_session.commit()
|
||||
second = deliver_due_webhooks(db_session, sender=failing_sender)[0]
|
||||
assert second.status == DELIVERY_FAILED
|
||||
assert second.attempt_count == 2
|
||||
assert second.last_status_code == 503
|
||||
|
||||
|
||||
def test_webhook_idempotency_skips_duplicate_deliveries(db_session):
|
||||
"""Repeated events with the same idempotency key reuse the existing delivery."""
|
||||
create_webhook_endpoint(
|
||||
db_session,
|
||||
name="receiver",
|
||||
url="https://receiver.example/webhook",
|
||||
event_types=[EVENT_ALERT_CREATED],
|
||||
)
|
||||
first = enqueue_webhook_event(
|
||||
db_session,
|
||||
event_type=EVENT_ALERT_CREATED,
|
||||
payload={"domain": "example.com"},
|
||||
idempotency_key="same-key",
|
||||
)
|
||||
second = enqueue_webhook_event(
|
||||
db_session,
|
||||
event_type=EVENT_ALERT_CREATED,
|
||||
payload={"domain": "example.com"},
|
||||
idempotency_key="same-key",
|
||||
)
|
||||
|
||||
assert first[0].id == second[0].id
|
||||
assert db_session.query(WebhookDelivery).count() == 1
|
||||
|
||||
|
||||
def test_webhook_validation_update_and_abandoned_delivery(db_session):
|
||||
"""Endpoint helpers validate input, update secrets, and abandon disabled endpoints."""
|
||||
assert normalize_event_types([]) == ["*"]
|
||||
assert normalize_event_types(["*", EVENT_ALERT_CREATED]) == ["*"]
|
||||
assert (
|
||||
validate_webhook_url(" https://receiver.example/hook ") == "https://receiver.example/hook"
|
||||
)
|
||||
|
||||
for bad_events in [["bad.event"]]:
|
||||
try:
|
||||
normalize_event_types(bad_events)
|
||||
except ValueError as exc:
|
||||
assert "Unsupported webhook event type" in str(exc)
|
||||
else: # pragma: no cover - defensive assertion shape
|
||||
raise AssertionError("invalid event type was accepted")
|
||||
|
||||
for bad_url in ["ftp://receiver.example/hook", "not-a-url"]:
|
||||
try:
|
||||
validate_webhook_url(bad_url)
|
||||
except ValueError as exc:
|
||||
assert "absolute http or https URL" in str(exc)
|
||||
else: # pragma: no cover - defensive assertion shape
|
||||
raise AssertionError("invalid webhook URL was accepted")
|
||||
|
||||
endpoint, _secret = create_webhook_endpoint(
|
||||
db_session,
|
||||
name="receiver",
|
||||
url="https://receiver.example/webhook",
|
||||
event_types=[EVENT_ALERT_CREATED],
|
||||
max_attempts=50,
|
||||
timeout_seconds=50,
|
||||
)
|
||||
updated, returned_secret = update_webhook_endpoint(
|
||||
db_session,
|
||||
endpoint,
|
||||
name="receiver two",
|
||||
url="https://receiver.example/updated?secret=hidden",
|
||||
secret="a-new-secret-value",
|
||||
event_types=[EVENT_REPORTS_MISSING],
|
||||
enabled=False,
|
||||
max_attempts=20,
|
||||
timeout_seconds=40,
|
||||
)
|
||||
|
||||
assert updated.name == "receiver two"
|
||||
assert returned_secret == "a-new-secret-value"
|
||||
assert updated.event_types == EVENT_REPORTS_MISSING
|
||||
assert updated.max_attempts == 10
|
||||
assert updated.timeout_seconds == 30
|
||||
assert endpoint_to_dict(updated)["url"] == "https://receiver.example/updated"
|
||||
|
||||
delivery = queue_test_webhook(db_session, updated.id)
|
||||
result = deliver_webhook_delivery(db_session, delivery)
|
||||
assert result.status == DELIVERY_ABANDONED
|
||||
assert "disabled or missing" in result.last_error
|
||||
|
||||
|
||||
def test_admin_webhook_endpoints_hide_secrets_and_show_delivery_status(
|
||||
authed_client: TestClient,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Operators can create, inspect, and test webhooks without secret leakage."""
|
||||
created = authed_client.post(
|
||||
"/api/v1/webhooks",
|
||||
json={
|
||||
"name": "ops",
|
||||
"url": "https://ops.example/hooks/dmarq?secret=hidden",
|
||||
"event_types": [EVENT_WEBHOOK_TEST],
|
||||
},
|
||||
)
|
||||
assert created.status_code == 200
|
||||
body = created.json()
|
||||
assert body["secret"]
|
||||
assert "hidden" not in body["url"]
|
||||
|
||||
listed = authed_client.get("/api/v1/webhooks")
|
||||
assert listed.status_code == 200
|
||||
assert listed.json()["endpoints"][0]["url"] == "https://ops.example/hooks/dmarq"
|
||||
assert body["secret"] not in listed.text
|
||||
|
||||
def fake_deliver_due_webhooks(db, endpoint_id=None, limit=25):
|
||||
delivery = (
|
||||
db.query(WebhookDelivery)
|
||||
.filter(WebhookDelivery.endpoint_id == endpoint_id)
|
||||
.order_by(WebhookDelivery.id.desc())
|
||||
.first()
|
||||
)
|
||||
delivery.status = DELIVERY_DELIVERED
|
||||
delivery.attempt_count = 1
|
||||
db.commit()
|
||||
db.refresh(delivery)
|
||||
return [delivery]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.api.api_v1.endpoints.webhooks.deliver_due_webhooks",
|
||||
fake_deliver_due_webhooks,
|
||||
)
|
||||
|
||||
tested = authed_client.post(f"/api/v1/webhooks/{body['id']}/test")
|
||||
assert tested.status_code == 200
|
||||
delivery = tested.json()["delivery"]
|
||||
assert delivery["event_type"] == EVENT_WEBHOOK_TEST
|
||||
assert delivery["status"] == DELIVERY_DELIVERED
|
||||
|
||||
history = authed_client.get("/api/v1/webhooks/deliveries")
|
||||
assert history.status_code == 200
|
||||
assert history.json()["deliveries"][0]["event_type"] == EVENT_WEBHOOK_TEST
|
||||
|
||||
|
||||
def test_admin_webhook_update_disable_filter_and_process(
|
||||
authed_client: TestClient,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Admin API covers update, filtering, due processing, and not-found paths."""
|
||||
created = authed_client.post(
|
||||
"/api/v1/webhooks",
|
||||
json={"name": "ops", "url": "https://ops.example/hooks/dmarq"},
|
||||
).json()
|
||||
|
||||
bad_create = authed_client.post(
|
||||
"/api/v1/webhooks",
|
||||
json={"name": "bad", "url": "ftp://ops.example/hooks/dmarq"},
|
||||
)
|
||||
assert bad_create.status_code == 400
|
||||
|
||||
missing_update = authed_client.put("/api/v1/webhooks/9999", json={"name": "missing"})
|
||||
assert missing_update.status_code == 404
|
||||
|
||||
bad_update = authed_client.put(
|
||||
f"/api/v1/webhooks/{created['id']}",
|
||||
json={"event_types": ["not.supported"]},
|
||||
)
|
||||
assert bad_update.status_code == 400
|
||||
|
||||
updated = authed_client.put(
|
||||
f"/api/v1/webhooks/{created['id']}",
|
||||
json={
|
||||
"name": "ops updated",
|
||||
"event_types": [EVENT_REPORTS_MISSING],
|
||||
"max_attempts": 2,
|
||||
"timeout_seconds": 3,
|
||||
},
|
||||
)
|
||||
assert updated.status_code == 200
|
||||
assert updated.json()["name"] == "ops updated"
|
||||
assert updated.json()["event_types"] == [EVENT_REPORTS_MISSING]
|
||||
|
||||
missing_test = authed_client.post("/api/v1/webhooks/9999/test")
|
||||
assert missing_test.status_code == 404
|
||||
|
||||
def fake_process(db, endpoint_id=None, limit=25):
|
||||
delivery = queue_test_webhook(db, created["id"])
|
||||
delivery.status = DELIVERY_DELIVERED
|
||||
delivery.attempt_count = 1
|
||||
db.commit()
|
||||
db.refresh(delivery)
|
||||
return [delivery]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.api.api_v1.endpoints.webhooks.deliver_due_webhooks",
|
||||
fake_process,
|
||||
)
|
||||
|
||||
processed = authed_client.post("/api/v1/webhooks/deliveries/process?limit=500")
|
||||
assert processed.status_code == 200
|
||||
assert processed.json()["deliveries"][0]["status"] == DELIVERY_DELIVERED
|
||||
|
||||
endpoint_filtered = authed_client.get(
|
||||
f"/api/v1/webhooks/deliveries?endpoint_id={created['id']}&status={DELIVERY_DELIVERED}"
|
||||
)
|
||||
assert endpoint_filtered.status_code == 200
|
||||
assert len(endpoint_filtered.json()["deliveries"]) == 1
|
||||
|
||||
disabled = authed_client.delete(f"/api/v1/webhooks/{created['id']}")
|
||||
assert disabled.status_code == 200
|
||||
assert disabled.json()["enabled"] is False
|
||||
|
||||
missing_delete = authed_client.delete("/api/v1/webhooks/9999")
|
||||
assert missing_delete.status_code == 404
|
||||
+1
-1
@@ -236,7 +236,7 @@ Goal: let DMARQ integrate cleanly into existing security and operations workflow
|
||||
|
||||
Planned:
|
||||
- A stable, documented read-only API surface for posture and reporting queries. Delivered with scoped `reports:read`, `posture:read`, and `tls-reports:read` API tokens, public read-only endpoints, and per-token usage audit fields.
|
||||
- Webhook event delivery for key events (new sender source, compliance drop, missing reports, alert lifecycle).
|
||||
- Webhook event delivery for key events (new sender source, compliance drop, missing reports, alert lifecycle). Delivered with encrypted webhook endpoints, signed delivery headers, idempotency keys, retry/backoff state, test sends, and delivery inspection.
|
||||
- Integration templates for SIEM and ticketing workflows (export formats, payload schemas, examples).
|
||||
- Token/scoping model for API access that matches governance needs (service accounts, least privilege).
|
||||
|
||||
|
||||
+34
-12
@@ -510,17 +510,39 @@ The API uses versioning in the URL path (/api/v1/) to ensure backward compatibil
|
||||
|
||||
## Webhooks
|
||||
|
||||
DMARQ can notify your systems about events via webhooks:
|
||||
DMARQ can notify downstream systems about operational events from
|
||||
**Settings > Webhooks** or the admin API.
|
||||
|
||||
1. Navigate to **Settings** > **API Access** > **Webhooks**
|
||||
2. Click **Add Webhook**
|
||||
3. Configure:
|
||||
- Destination URL
|
||||
- Secret token (for verification)
|
||||
- Events to subscribe to
|
||||
| Endpoint | Purpose |
|
||||
| --- | --- |
|
||||
| `GET /api/v1/webhooks` | List endpoints and supported event types |
|
||||
| `POST /api/v1/webhooks` | Create an endpoint |
|
||||
| `PUT /api/v1/webhooks/{id}` | Update an endpoint |
|
||||
| `DELETE /api/v1/webhooks/{id}` | Disable an endpoint while keeping delivery history |
|
||||
| `POST /api/v1/webhooks/{id}/test` | Queue and attempt a test delivery |
|
||||
| `GET /api/v1/webhooks/deliveries` | Inspect recent delivery attempts |
|
||||
| `POST /api/v1/webhooks/deliveries/process` | Attempt due retries |
|
||||
|
||||
Supported events:
|
||||
- `report.processed` - When a new report is processed
|
||||
- `compliance.threshold` - When compliance falls below threshold
|
||||
- `domain.added` - When a domain is added
|
||||
- `domain.removed` - When a domain is removed
|
||||
Supported event types:
|
||||
- `dmarq.report.imported`
|
||||
- `dmarq.sender.new`
|
||||
- `dmarq.compliance.drop`
|
||||
- `dmarq.reports.missing`
|
||||
- `dmarq.alert.created`
|
||||
- `dmarq.alert.resolved`
|
||||
- `dmarq.webhook.test`
|
||||
|
||||
Deliveries are signed with HMAC-SHA256 using the endpoint signing secret.
|
||||
Receivers should verify these headers:
|
||||
|
||||
| Header | Description |
|
||||
| --- | --- |
|
||||
| `X-DMARQ-Event` | Event type |
|
||||
| `X-DMARQ-Delivery` | Delivery id |
|
||||
| `X-DMARQ-Idempotency-Key` | Stable deduplication key |
|
||||
| `X-DMARQ-Timestamp` | Unix timestamp used in the signature |
|
||||
| `X-DMARQ-Signature` | `v1=<hex hmac>` over `timestamp.delivery_id.body` |
|
||||
|
||||
Non-2xx responses are retried with exponential backoff until the endpoint's
|
||||
maximum attempt count is reached. Operators can inspect the delivery status,
|
||||
last response code, error text, and response excerpt without reading logs.
|
||||
|
||||
@@ -135,6 +135,46 @@ and are never stored.
|
||||
| last_used_ip | VARCHAR(64) | Source IP from the last successful API use |
|
||||
| usage_count | INTEGER | Successful API use count |
|
||||
|
||||
### Webhook_Endpoints
|
||||
|
||||
The `webhook_endpoints` table stores outbound webhook destinations. Target
|
||||
URLs and signing secrets are encrypted at rest.
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| id | INTEGER | Primary key |
|
||||
| name | VARCHAR(120) | Operator-facing endpoint name |
|
||||
| url | TEXT | Encrypted destination URL |
|
||||
| secret | TEXT | Encrypted signing secret |
|
||||
| event_types | TEXT | Comma-separated event subscriptions, or `*` |
|
||||
| enabled | BOOLEAN | Whether deliveries can be sent |
|
||||
| max_attempts | INTEGER | Maximum attempts before a delivery fails |
|
||||
| timeout_seconds | INTEGER | Per-request timeout |
|
||||
| last_success_at | TIMESTAMP | Last successful delivery |
|
||||
| last_failure_at | TIMESTAMP | Last failed delivery attempt |
|
||||
| failure_count | INTEGER | Consecutive endpoint-level failures |
|
||||
|
||||
### Webhook_Deliveries
|
||||
|
||||
The `webhook_deliveries` table records delivery attempts and retry state.
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| id | INTEGER | Primary key |
|
||||
| endpoint_id | INTEGER | Foreign key to webhook_endpoints.id |
|
||||
| event_type | VARCHAR(80) | Delivered event type |
|
||||
| payload | TEXT | Event envelope JSON |
|
||||
| idempotency_key | VARCHAR(160) | Stable deduplication key per endpoint |
|
||||
| status | VARCHAR(24) | pending, delivered, failed, or abandoned |
|
||||
| attempt_count | INTEGER | Attempts already made |
|
||||
| max_attempts | INTEGER | Maximum attempts for this delivery |
|
||||
| next_attempt_at | TIMESTAMP | Next retry time |
|
||||
| last_attempt_at | TIMESTAMP | Last attempt time |
|
||||
| delivered_at | TIMESTAMP | Successful delivery time |
|
||||
| last_status_code | INTEGER | Last HTTP status code |
|
||||
| last_error | TEXT | Last sanitized error |
|
||||
| response_excerpt | TEXT | Truncated downstream response |
|
||||
|
||||
## DNS and Configuration Tables
|
||||
|
||||
### DNS_Records
|
||||
@@ -243,6 +283,9 @@ The schema includes several indexes to optimize query performance:
|
||||
- `ix_api_tokens_key_hash`: On api_tokens.key_hash
|
||||
- `ix_api_tokens_key_prefix`: On api_tokens.key_prefix
|
||||
- `ix_api_tokens_active_scope`: On api_tokens.active and api_tokens.scopes
|
||||
- `ix_webhook_endpoints_enabled_events`: On webhook_endpoints.enabled and webhook_endpoints.event_types
|
||||
- `ix_webhook_delivery_endpoint_idempotency`: Unique on webhook_deliveries.endpoint_id and idempotency_key
|
||||
- `ix_webhook_delivery_due`: On webhook_deliveries.status and webhook_deliveries.next_attempt_at
|
||||
- `idx_activity_logs_timestamp`: On activity_logs.timestamp
|
||||
- `idx_activity_logs_user_id`: On activity_logs.user_id
|
||||
- `idx_system_logs_timestamp`: On system_logs.timestamp
|
||||
|
||||
@@ -86,6 +86,21 @@ Apprise supports email, Slack, Teams, Discord, generic webhooks, and many other
|
||||
targets through the same notification field. Add each destination on a separate
|
||||
line.
|
||||
|
||||
## Webhooks
|
||||
|
||||
Use **Settings** > **Webhooks** when another system needs structured DMARQ
|
||||
events instead of human-readable notifications.
|
||||
|
||||
1. Add a name and HTTPS endpoint URL.
|
||||
2. Choose all events or one event type.
|
||||
3. Save the endpoint.
|
||||
4. Use **Test** to send a signed test event.
|
||||
5. Inspect **Recent Deliveries** to see status, attempts, response codes, and errors.
|
||||
|
||||
DMARQ signs each delivery with `X-DMARQ-Signature` and includes
|
||||
`X-DMARQ-Idempotency-Key` so receivers can reject replays and deduplicate
|
||||
retries. Endpoint URLs and signing secrets are encrypted at rest.
|
||||
|
||||
## API Access
|
||||
|
||||
DMARQ provides an API for integration with other systems:
|
||||
|
||||
Reference in New Issue
Block a user