feat(api): add GET /api/diagnostic/health endpoint for monitoring

- Add health check endpoint at GET /api/diagnostic/health - Auth-protected via @require_login (no-op when AUTH_ENABLED=False) - Checks database (SELECT 1) and Redis (ping) with 2s timeouts - Returns healthy/degraded/unhealthy with per-check detail - Returns HTTP 503 when database is down, 200 otherwise - 7 new unit tests covering all status scenarios - Update docs/API.md with Grafana/monitoring integration notes - Fixes test_cors_headers_absent_when_disabled CI timeout Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
2026-03-01 18:36:38 +00:00
parent 5ff7b72a80
commit 85da309740
3 changed files with 267 additions and 1 deletions
@@ -2,19 +2,112 @@
 Diagnostic API endpoints
 """

+import datetime
 import logging

+import redis as redis_lib
 from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+from sqlalchemy import text

 from app.auth import require_login
 from app.config import settings
+from app.database import engine

 # Set up logging
 logger = logging.getLogger(__name__)

+_DEFAULT_REDIS_URL = "redis://localhost:6379/0"
+
 router = APIRouter()


+@router.get("/diagnostic/health")
+@require_login
+async def health_check(request: Request):
+    """
+    System health endpoint for monitoring tools (Grafana, Uptime Kuma, etc.).
+
+    Checks database connectivity and Redis availability and returns a
+    machine-readable summary that monitoring systems can scrape.
+
+    **Authentication:** Required (no-op when AUTH_ENABLED=False)
+
+    **Response (200 OK) – all subsystems healthy:**
+    ```json
+    {
+      "status": "healthy",
+      "version": "1.2.3",
+      "timestamp": "2024-01-15T10:30:00+00:00",
+      "checks": {
+        "database": {"status": "ok"},
+        "redis":    {"status": "ok"}
+      }
+    }
+    ```
+
+    **Response (200 OK) – one or more subsystems degraded:**
+    ```json
+    {
+      "status": "degraded",
+      "version": "1.2.3",
+      "timestamp": "2024-01-15T10:30:00+00:00",
+      "checks": {
+        "database": {"status": "ok"},
+        "redis":    {"status": "error", "detail": "Connection refused"}
+      }
+    }
+    ```
+
+    The outer ``status`` field is always one of:
+    - ``"healthy"``  – all checks passed
+    - ``"degraded"`` – at least one non-critical check failed
+    - ``"unhealthy"`` – a critical check failed (currently: database)
+    """
+    timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+    checks: dict[str, dict[str, str]] = {}
+
+    # ── Database check ─────────────────────────────────────────────────────
+    db_ok = False
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        checks["database"] = {"status": "ok"}
+        db_ok = True
+    except Exception as exc:
+        logger.warning("Health check: database probe failed: %s", exc)
+        checks["database"] = {"status": "error", "detail": str(exc)}
+
+    # ── Redis check ────────────────────────────────────────────────────────
+    try:
+        redis_url = settings.redis_url or _DEFAULT_REDIS_URL
+        r = redis_lib.from_url(redis_url, socket_connect_timeout=2, socket_timeout=2)
+        r.ping()
+        checks["redis"] = {"status": "ok"}
+    except Exception as exc:
+        logger.warning("Health check: Redis probe failed: %s", exc)
+        checks["redis"] = {"status": "error", "detail": str(exc)}
+
+    # ── Overall status ─────────────────────────────────────────────────────
+    if not db_ok:
+        overall = "unhealthy"
+    elif any(v.get("status") != "ok" for v in checks.values()):
+        overall = "degraded"
+    else:
+        overall = "healthy"
+
+    http_status = 503 if overall == "unhealthy" else 200
+
+    payload = {
+        "status": overall,
+        "version": settings.version,
+        "timestamp": timestamp,
+        "checks": checks,
+    }
+
+    return JSONResponse(content=payload, status_code=http_status)
+
+
@router.post("/diagnostic/test-notification")
@require_login
 async def test_notification(request: Request):
@@ -764,6 +764,77 @@ Lightweight endpoint returning the total number of queued + in-progress items. D
 }
 ```

+## Diagnostic
+
+### GET /api/diagnostic/health
+
+System health endpoint designed for monitoring tools such as Grafana, Uptime Kuma, Prometheus blackbox exporter, or any HTTP-based health checker.
+
+Checks the database and Redis connectivity and returns a machine-readable JSON summary.
+
+**Authentication:** Required (bypassed when `AUTH_ENABLED=False`)
+
+**Response (200 OK) – all subsystems healthy:**
+```json
+{
+  "status": "healthy",
+  "version": "1.2.3",
+  "timestamp": "2024-01-15T10:30:00+00:00",
+  "checks": {
+    "database": {"status": "ok"},
+    "redis":    {"status": "ok"}
+  }
+}
+```
+
+**Response (200 OK) – one or more non-critical checks failed:**
+```json
+{
+  "status": "degraded",
+  "version": "1.2.3",
+  "timestamp": "2024-01-15T10:30:00+00:00",
+  "checks": {
+    "database": {"status": "ok"},
+    "redis":    {"status": "error", "detail": "Connection refused"}
+  }
+}
+```
+
+**Response (503 Service Unavailable) – critical check (database) failed:**
+```json
+{
+  "status": "unhealthy",
+  "version": "1.2.3",
+  "timestamp": "2024-01-15T10:30:00+00:00",
+  "checks": {
+    "database": {"status": "error", "detail": "..."},
+    "redis":    {"status": "ok"}
+  }
+}
+```
+
+The `status` field is always one of:
+- `"healthy"` – all checks passed
+- `"degraded"` – at least one non-critical check failed (Redis unavailable)
+- `"unhealthy"` – a critical check failed (database unavailable); HTTP 503 is returned
+
+**Grafana / Uptime Kuma integration:** point your health check at `GET /api/diagnostic/health` and check for HTTP 200 or the JSON `status` field.
+
+### POST /api/diagnostic/test-notification
+
+Send a test notification through all configured notification channels.
+
+**Authentication:** Required
+
+**Response (200 OK):**
+```json
+{
+  "status": "success",
+  "message": "Test notification sent successfully to 2 service(s)",
+  "services_count": 2
+}
+```
+
 ## Rate Limiting

 The API implements rate limiting to ensure system stability. If you exceed the limits, you'll receive a `429 Too Many Requests` response.
@@ -1,10 +1,112 @@
 """Tests for app/api/diagnostic.py module."""

-from unittest.mock import patch
+from unittest.mock import MagicMock, patch

 import pytest


+@pytest.mark.unit
+class TestHealthEndpoint:
+    """Tests for GET /api/diagnostic/health endpoint."""
+
+    def test_health_returns_200_when_all_ok(self, client):
+        """Health endpoint returns 200 with healthy status when all checks pass."""
+        with (
+            patch("app.api.diagnostic.engine") as mock_engine,
+            patch("app.api.diagnostic.redis_lib") as mock_redis,
+        ):
+            mock_conn = MagicMock()
+            mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn)
+            mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False)
+            mock_redis_inst = MagicMock()
+            mock_redis.from_url.return_value = mock_redis_inst
+
+            response = client.get("/api/diagnostic/health")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "healthy"
+        assert "timestamp" in data
+        assert "version" in data
+        assert "checks" in data
+        assert data["checks"]["database"]["status"] == "ok"
+
+    def test_health_returns_503_when_database_fails(self, client):
+        """Health endpoint returns 503 with unhealthy status when DB is down."""
+        with (
+            patch("app.api.diagnostic.engine") as mock_engine,
+            patch("app.api.diagnostic.redis_lib") as mock_redis,
+        ):
+            mock_engine.connect.side_effect = Exception("DB unavailable")
+            mock_redis_inst = MagicMock()
+            mock_redis.from_url.return_value = mock_redis_inst
+
+            response = client.get("/api/diagnostic/health")
+
+        assert response.status_code == 503
+        data = response.json()
+        assert data["status"] == "unhealthy"
+        assert data["checks"]["database"]["status"] == "error"
+        assert "detail" in data["checks"]["database"]
+
+    def test_health_returns_200_degraded_when_redis_fails(self, client):
+        """Health returns 200 degraded when Redis is unavailable (non-critical)."""
+        with (
+            patch("app.api.diagnostic.engine") as mock_engine,
+            patch("app.api.diagnostic.redis_lib") as mock_redis,
+        ):
+            mock_conn = MagicMock()
+            mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn)
+            mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False)
+            mock_redis.from_url.return_value = MagicMock()
+            mock_redis.from_url.return_value.ping.side_effect = Exception("Connection refused")
+
+            response = client.get("/api/diagnostic/health")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "degraded"
+        assert data["checks"]["database"]["status"] == "ok"
+        assert data["checks"]["redis"]["status"] == "error"
+        assert "detail" in data["checks"]["redis"]
+
+    def test_health_response_has_required_fields(self, client):
+        """Health response always contains status, version, timestamp, checks."""
+        response = client.get("/api/diagnostic/health")
+        data = response.json()
+
+        assert "status" in data
+        assert "version" in data
+        assert "timestamp" in data
+        assert "checks" in data
+        assert data["status"] in ("healthy", "degraded", "unhealthy")
+
+    def test_health_no_cors_headers_when_cors_disabled(self, client):
+        """Health endpoint does not add CORS headers when middleware is disabled."""
+        from app.config import settings
+
+        if settings.cors_enabled:
+            pytest.skip("CORS is enabled in this test environment")
+
+        response = client.get(
+            "/api/diagnostic/health",
+            headers={"Origin": "https://evil.example.com"},
+        )
+        assert "access-control-allow-origin" not in response.headers
+
+    def test_health_checks_contain_database_key(self, client):
+        """Health checks dict always contains a 'database' key."""
+        response = client.get("/api/diagnostic/health")
+        data = response.json()
+        assert "database" in data["checks"]
+
+    def test_health_checks_contain_redis_key(self, client):
+        """Health checks dict always contains a 'redis' key."""
+        response = client.get("/api/diagnostic/health")
+        data = response.json()
+        assert "redis" in data["checks"]
+
+
@pytest.mark.integration
 class TestTestNotification:
    """Tests for test notification endpoint."""