gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #753 from christianlouis/copilot/scale-worker-and-api-pods

Browse Source 
feat(scaling): enable horizontal scaling for API and worker pods
This commit is contained in:
Christian Krakau-Louis
2026-03-20 13:55:31 +01:00
committed by GitHub
parent 74ed8b9bd9 4d019d53d9
commit 91eecd9396
10 changed files with 384 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/diagnostic.py
+61
View File
@@ -21,6 +21,67 @@ _DEFAULT_REDIS_URL = "redis://localhost:6379/0"
router = APIRouter()
# ---------------------------------------------------------------------------
# Unauthenticated probe endpoints for Kubernetes liveness / readiness checks.
# These intentionally skip authentication so that kubelet can reach them
# without credentials. They live under /diagnostic/healthz/* so that the
# existing authenticated /diagnostic/health endpoint is unaffected.
# ---------------------------------------------------------------------------
@router.get("/diagnostic/healthz/live")
async def liveness_probe() -> JSONResponse:
"""Lightweight liveness probe for Kubernetes.
Returns **200 OK** as long as the process is running. Kubernetes uses
this to decide whether to *restart* the container — it should therefore
be as cheap as possible and **never** check external dependencies.
**Authentication:** None (designed for kubelet probes).
"""
return JSONResponse(content={"status": "ok"}, status_code=200)
@router.get("/diagnostic/healthz/ready")
async def readiness_probe() -> JSONResponse:
"""Readiness probe for Kubernetes.
Verifies that the application can serve traffic by checking the database
and Redis. Kubernetes uses this to decide whether to *route traffic* to
the pod.
Returns **200 OK** when all critical subsystems are reachable, or
**503 Service Unavailable** when the database is down.
**Authentication:** None (designed for kubelet probes).
"""
checks: dict[str, dict[str, str]] = {}
db_ok = False
# ── Database check ─────────────────────────────────────────────────
try:
with engine.connect() as conn:
conn.execute(text("SELECT 1"))
checks["database"] = {"status": "ok"}
db_ok = True
except Exception as exc:
logger.warning("Readiness probe: database check failed: %s", exc)
checks["database"] = {"status": "error", "detail": str(exc)}
# ── Redis check ────────────────────────────────────────────────────
try:
redis_url = settings.redis_url or _DEFAULT_REDIS_URL
r = redis_lib.from_url(redis_url, socket_connect_timeout=2, socket_timeout=2)
r.ping()
checks["redis"] = {"status": "ok"}
except Exception as exc:
logger.warning("Readiness probe: Redis check failed: %s", exc)
checks["redis"] = {"status": "error", "detail": str(exc)}
http_status = 503 if not db_ok else 200
overall = "ready" if db_ok else "not_ready"
return JSONResponse(content={"status": overall, "checks": checks}, status_code=http_status)
@router.get("/diagnostic/health")
@require_login
docker-compose.yaml
+25 -4
View File
@@ -3,7 +3,7 @@ services:
build:
context: .
dockerfile: Dockerfile
container_name: document_api
# No container_name — allows `docker compose up --scale api=N`
restart: always
# We'll keep the code in /app, but set working_dir to the shared data directory
@@ -24,7 +24,7 @@ services:
depends_on:
- redis
- worker
- beat
# Mount the shared working directory for data
volumes:
@@ -34,13 +34,14 @@ services:
build:
context: .
dockerfile: Dockerfile
container_name: document_worker
# No container_name — allows `docker compose up --scale worker=N`
restart: always
# same shared working directory
working_dir: /workdir
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
# Workers process tasks only — no -B flag (Beat runs in the dedicated beat service)
command: ["celery", "-A", "app.celery_worker", "worker", "--loglevel=info", "-Q", "document_processor,default,celery"]
env_file:
- .env
environment:
@@ -54,6 +55,26 @@ services:
volumes:
- /var/docparse/workdir:/workdir
# Dedicated Celery Beat scheduler — exactly one instance must run at all times.
# Beat publishes periodic tasks to the Redis broker; workers pick them up.
# Do NOT scale this service (replicas must stay at 1).
beat:
build:
context: .
dockerfile: Dockerfile
container_name: document_beat
restart: always
working_dir: /workdir
command: ["celery", "-A", "app.celery_worker", "beat", "--loglevel=info"]
env_file:
- .env
environment:
- PYTHONPATH=/app
depends_on:
- redis
volumes:
- /var/docparse/workdir:/workdir
gotenberg:
image: gotenberg/gotenberg:latest
container_name: gotenberg
docs/API.md
+41
View File
@@ -1594,6 +1594,47 @@ Lightweight endpoint returning the total number of queued + in-progress items. D
## Diagnostic
### GET /api/diagnostic/healthz/live
Lightweight liveness probe for Kubernetes. Returns **200 OK** as long as the process is running. This endpoint does **not** check external dependencies and is intentionally cheap.
**Authentication:** None (designed for kubelet probes)
**Response (200 OK):**
```json
{
"status": "ok"
}
```
### GET /api/diagnostic/healthz/ready
Readiness probe for Kubernetes. Verifies that the application can serve traffic by checking database and Redis connectivity.
**Authentication:** None (designed for kubelet probes)
**Response (200 OK) – ready to serve traffic:**
```json
{
"status": "ready",
"checks": {
"database": {"status": "ok"},
"redis": {"status": "ok"}
}
}
```
**Response (503 Service Unavailable) – database unreachable:**
```json
{
"status": "not_ready",
"checks": {
"database": {"status": "error", "detail": "..."},
"redis": {"status": "ok"}
}
}
```
### GET /api/diagnostic/health
System health endpoint designed for monitoring tools such as Grafana, Uptime Kuma, Prometheus blackbox exporter, or any HTTP-based health checker.
docs/DeploymentGuide.md
+10 -6
View File
@@ -349,16 +349,18 @@ workdir:
## Scaling
DocuElevate is designed for horizontal scaling. Both API and worker pods are stateless and can be scaled independently.
### Docker Compose
Add more worker containers:
Scale workers (task processing) and API pods (request handling) independently:
```yaml
worker:
deploy:
replicas: 3
```bash
docker compose up -d --scale worker=3 --scale api=2
```
> **Note:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. Do not scale it. It publishes periodic tasks to the Redis broker; workers pick them up.
### Kubernetes / Helm
Enable HPA:
@@ -377,13 +379,15 @@ worker:
maxReplicas: 10
```
The Helm chart deploys a separate **beat** pod (always 1 replica, `Recreate` strategy) so that scheduled tasks are never duplicated when workers scale.
---
## Monitoring
- **Docker Compose**: `docker-compose logs -f`, `docker stats`
- **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f`
- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed.
- **Prometheus / Grafana**: Scrape the `/api/diagnostic/healthz/ready` endpoint for readiness; add custom metrics as needed.
- **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring.
---
docs/KubernetesDeployment.md
+13 -5
View File
@@ -373,6 +373,8 @@ worker:
replicaCount: 4
```
> **Beat scheduler:** The Helm chart deploys a dedicated `beat` pod (always exactly 1 replica with `Recreate` strategy) that publishes periodic tasks to the Redis broker. Workers consume these tasks — scaling workers does **not** duplicate scheduled jobs.
### Horizontal Pod Autoscaler
```yaml
@@ -433,24 +435,30 @@ externalRedis:
### Kubernetes Probes
The Helm chart configures liveness and readiness probes on the API pods via `/api/health`. Default settings:
The Helm chart configures **unauthenticated** liveness and readiness probes on the API pods so kubelet can reach them without credentials. Default settings:
```yaml
api:
livenessProbe:
httpGet:
path: /api/health
path: /api/diagnostic/healthz/live
port: 8000
initialDelaySeconds: 30
periodSeconds: 30
periodSeconds: 20
readinessProbe:
httpGet:
path: /api/health
path: /api/diagnostic/healthz/ready
port: 8000
initialDelaySeconds: 10
initialDelaySeconds: 15
periodSeconds: 10
```
| Endpoint | Auth | Purpose |
|----------|------|---------|
| `/api/diagnostic/healthz/live` | None | Lightweight liveness check — returns 200 if the process is running |
| `/api/diagnostic/healthz/ready` | None | Readiness check — verifies database and Redis connectivity (503 when DB is down) |
| `/api/diagnostic/health` | Required | Full health status for monitoring dashboards (Grafana, Uptime Kuma) |
### Prometheus Scraping
Add annotations to expose metrics (if using a Prometheus-compatible exporter):
docs/ProductionReadiness.md
+38 -15
View File
@@ -34,7 +34,7 @@ Use this checklist to track readiness before going live.
- [ ] **Redis** — Running and accessible only from internal network
- [ ] **Meilisearch** — Running and accessible only from internal network
- [ ] **Worker replicas** — At least 2 workers configured for redundancy
- [ ] **Monitoring** — `/api/health` polled by uptime checker
- [ ] **Monitoring** — `/api/diagnostic/health` polled by uptime checker
- [ ] **Backups** — Automated backup of database, workdir, and Meilisearch data
- [ ] **Log retention** — Logs shipped to a persistent store or aggregator
- [ ] **Secrets management** — API keys not committed to source control
@@ -285,22 +285,24 @@ For SSO/OIDC (Authentik, Keycloak, Auth0, etc.) see the [Authentication Setup Gu
### Docker Compose
Use the `deploy.replicas` setting (requires Docker Swarm mode) or simply run multiple workers:
```yaml
worker:
deploy:
replicas: 3
```
Or scale after deployment:
Scale workers independently:
```bash
docker-compose up -d --scale worker=3
docker compose up -d --scale worker=3
```
Each worker processes tasks from the Celery queue independently. Ensure the shared `workdir` volume is accessible from all worker containers.
> **Important:** The `beat` service (Celery Beat scheduler) must always run as exactly **one** instance. It is defined as a dedicated service in `docker-compose.yaml` with a fixed `container_name`. Do not scale it.
### Scaling the API
API pods are fully stateless (sessions use encrypted cookies, not server-side state) and can be scaled behind a load balancer:
```bash
docker compose up -d --scale api=3
```
### Kubernetes (Helm)
```yaml
@@ -339,11 +341,32 @@ celery -A app.celery_worker worker -Q default,celery --concurrency=2
### Health Check Endpoint
DocuElevate exposes `/api/health` for readiness probing. Configure your uptime monitor to poll this endpoint:
DocuElevate exposes three health-related endpoints:
| Endpoint | Auth | Purpose |
|----------|------|---------|
| `GET /api/diagnostic/healthz/live` | None | Lightweight liveness probe — returns 200 if the process is running |
| `GET /api/diagnostic/healthz/ready` | None | Readiness probe — checks database and Redis (503 when DB is down) |
| `GET /api/diagnostic/health` | Required | Full status for monitoring dashboards (Grafana, Uptime Kuma) |
For **Kubernetes probes**, use the unauthenticated endpoints:
```yaml
livenessProbe:
httpGet:
path: /api/diagnostic/healthz/live
port: 8000
readinessProbe:
httpGet:
path: /api/diagnostic/healthz/ready
port: 8000
```
For **uptime monitors** (Uptime Kuma, Grafana, etc.), use the authenticated endpoint:
```bash
curl http://docuelevate.example.com/api/health
# Expected: {"status": "ok", ...}
curl http://docuelevate.example.com/api/diagnostic/health
# Expected: {"status": "healthy", ...}
```
Set `UPTIME_KUMA_URL` to your Uptime Kuma push URL for heartbeat monitoring:
@@ -502,4 +525,4 @@ For a dedicated Kubernetes deployment guide, including architecture diagrams, PV
- **Image Pull Policy**: Use `IfNotPresent` in production with pinned image tags (not `latest`) for reproducible deployments.
- **Liveness & Readiness Probes**: Already configured in the Helm chart via `/api/health`. Verify they are tuned to your startup time.
- **Liveness & Readiness Probes**: Already configured in the Helm chart via unauthenticated endpoints (`/api/diagnostic/healthz/live` and `/api/diagnostic/healthz/ready`). Verify they are tuned to your startup time.
helm/docuelevate/templates/beat-deployment.yaml
+83
View File
@@ -0,0 +1,83 @@
{{- /*
Celery Beat scheduler — publishes periodic tasks to the broker.
Exactly ONE replica must run; never scale this deployment.
*/ -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "docuelevate.fullname" . }}-beat
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: beat
spec:
replicas: 1
strategy:
type: Recreate # Prevent two Beat instances from running simultaneously
selector:
matchLabels:
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: beat
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: beat
{{- with .Values.beat.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.beat.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: beat
image: {{ include "docuelevate.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
command:
- celery
- -A
- app.celery_worker
- beat
- --loglevel=info
envFrom:
- configMapRef:
name: {{ include "docuelevate.fullname" . }}-config
- secretRef:
name: {{ include "docuelevate.fullname" . }}-secret
{{- with .Values.beat.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.beat.resources | nindent 12 }}
volumeMounts:
- name: workdir
mountPath: /workdir
volumes:
- name: workdir
{{- if .Values.workdir.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.beat.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.beat.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.beat.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
helm/docuelevate/templates/worker-deployment.yaml
-1
View File
@@ -42,7 +42,6 @@ spec:
- -A
- app.celery_worker
- worker
- -B
- --loglevel=info
- -Q
- document_processor,default,celery
helm/docuelevate/values.yaml
+33 -4
View File
@@ -121,10 +121,10 @@ api:
type: ClusterIP
port: 8000
# Liveness / readiness probes
# Liveness / readiness probes (unauthenticated endpoints for kubelet)
livenessProbe:
httpGet:
path: /api/health
path: /api/diagnostic/healthz/live
port: 8000
initialDelaySeconds: 30
periodSeconds: 20
@@ -132,7 +132,7 @@ api:
readinessProbe:
httpGet:
path: /api/health
path: /api/diagnostic/healthz/ready
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
@@ -191,7 +191,36 @@ worker:
drop: ["ALL"]
# ---------------------------------------------------------------------------
# Shared workdir volume (api + worker mount the same PVC)
# Celery Beat scheduler (singleton — always exactly 1 replica)
# Beat publishes periodic tasks; workers consume them from the broker.
# ---------------------------------------------------------------------------
beat:
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
podSecurityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroup: 1000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop: ["ALL"]
# ---------------------------------------------------------------------------
# Shared workdir volume (api + worker + beat mount the same PVC)
# ---------------------------------------------------------------------------
workdir:
persistence:
tests/test_diagnostic.py
+80
View File
@@ -5,6 +5,86 @@ from unittest.mock import MagicMock, patch
import pytest
@pytest.mark.unit
class TestLivenessProbe:
"""Tests for GET /api/diagnostic/healthz/live (unauthenticated)."""
def test_liveness_returns_200(self, client):
"""Liveness probe always returns 200 OK."""
response = client.get("/api/diagnostic/healthz/live")
assert response.status_code == 200
data = response.json()
assert data["status"] == "ok"
@pytest.mark.unit
class TestReadinessProbe:
"""Tests for GET /api/diagnostic/healthz/ready (unauthenticated)."""
def test_readiness_returns_200_when_all_ok(self, client):
"""Readiness probe returns 200 when database and Redis are reachable."""
with (
patch("app.api.diagnostic.engine") as mock_engine,
patch("app.api.diagnostic.redis_lib") as mock_redis,
):
mock_conn = MagicMock()
mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn)
mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False)
mock_redis_inst = MagicMock()
mock_redis.from_url.return_value = mock_redis_inst
response = client.get("/api/diagnostic/healthz/ready")
assert response.status_code == 200
data = response.json()
assert data["status"] == "ready"
assert data["checks"]["database"]["status"] == "ok"
def test_readiness_returns_503_when_database_fails(self, client):
"""Readiness probe returns 503 when database is unreachable."""
with (
patch("app.api.diagnostic.engine") as mock_engine,
patch("app.api.diagnostic.redis_lib") as mock_redis,
):
mock_engine.connect.side_effect = Exception("DB unavailable")
mock_redis_inst = MagicMock()
mock_redis.from_url.return_value = mock_redis_inst
response = client.get("/api/diagnostic/healthz/ready")
assert response.status_code == 503
data = response.json()
assert data["status"] == "not_ready"
assert data["checks"]["database"]["status"] == "error"
def test_readiness_returns_200_when_redis_fails(self, client):
"""Readiness remains 200 when only Redis is down (non-critical)."""
with (
patch("app.api.diagnostic.engine") as mock_engine,
patch("app.api.diagnostic.redis_lib") as mock_redis,
):
mock_conn = MagicMock()
mock_engine.connect.return_value.__enter__ = MagicMock(return_value=mock_conn)
mock_engine.connect.return_value.__exit__ = MagicMock(return_value=False)
mock_redis.from_url.return_value = MagicMock()
mock_redis.from_url.return_value.ping.side_effect = Exception("Connection refused")
response = client.get("/api/diagnostic/healthz/ready")
assert response.status_code == 200
data = response.json()
assert data["status"] == "ready"
assert data["checks"]["redis"]["status"] == "error"
def test_readiness_contains_checks_keys(self, client):
"""Readiness response always contains database and redis checks."""
response = client.get("/api/diagnostic/healthz/ready")
data = response.json()
assert "checks" in data
assert "database" in data["checks"]
assert "redis" in data["checks"]
@pytest.mark.unit
class TestHealthEndpoint:
"""Tests for GET /api/diagnostic/health endpoint."""