diff --git a/CHANGELOG.md b/CHANGELOG.md index 308879c..a19da68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Prometheus metrics** (`/metrics` endpoint on the FastAPI backend, scraped every 15 s): + - **HTTP layer** — `http_requests_total` (counter, labelled `method`/`endpoint`/`status_code`) and `http_request_duration_seconds` (histogram). Path segments that are numeric IDs are normalised to `{id}` to avoid label-set explosion. + - **Mail processing** — `mail_processing_runs_total` (counter, by `status`: `completed` / `partial_failure` / `failed`), `mail_processing_emails_total` (counter, by `operation`: `fetched` / `forwarded` / `failed`), `mail_processing_duration_seconds` (histogram), `active_mail_accounts_total` (gauge — set each scheduler cycle). + - **Gmail API** — `gmail_api_requests_total` (counter, by `operation` and `status`), `gmail_api_duration_seconds` (histogram, by `operation`), `gmail_token_refreshes_total` (counter), `gmail_credentials_invalidated_total` (counter). + - **Authentication / OAuth** — `auth_logins_total` (counter, `method` × `status`), `auth_registrations_total` (counter, `method` × `status`), `oauth_callbacks_total` (counter, `provider` × `status`). + - **Celery tasks** — `celery_tasks_total` (counter, `task_name` × `status`) and `celery_task_duration_seconds` (histogram, by `task_name`). +- **All metrics** defined as module-level singletons in `backend/app/core/metrics.py` (imported by HTTP middleware, task workers, GmailService, and auth endpoints). +- **Prometheus service** added to `docker-compose.new.yml` (port 9090, 30-day retention, config from `monitoring/prometheus.yml`). +- **Grafana service** added to `docker-compose.new.yml` (port 3001, auto-provisioned datasource + pre-built dashboard). Default credentials: `admin` / `admin`. +- **Pre-built Grafana dashboard** (`monitoring/grafana/dashboards/inboxrescue.json`) with five sections: Mail Processing, Gmail API, Authentication & OAuth, HTTP API, and Celery Workers. Dashboard auto-refreshes every 30 s. + + - **Admin interface**: Superusers now have access to a dedicated Admin section in the sidebar with three pages: - **Admin Overview** (`/admin`): System-wide stats (total users, mail accounts, processing runs). - **Manage Users** (`/admin/users`): Table of all registered users with their subscription tier, status, mail account count, and last login. Admins can edit any user's name, email, plan, active status, and promote/demote admin (superuser) privileges. Users can be deleted (with confirmation). diff --git a/backend/app/api/v1/endpoints/auth.py b/backend/app/api/v1/endpoints/auth.py index 1c99855..bf91b83 100644 --- a/backend/app/api/v1/endpoints/auth.py +++ b/backend/app/api/v1/endpoints/auth.py @@ -13,6 +13,11 @@ import logging from app.core.config import settings from app.core.database import get_db from app.core.security import verify_password, get_password_hash, encrypt_credential +from app.core.metrics import ( + AUTH_LOGINS_TOTAL, + AUTH_REGISTRATIONS_TOTAL, + OAUTH_CALLBACKS_TOTAL, +) from app.models.database_models import User, SubscriptionTier, GmailCredential from app.models.schemas import Token, UserCreate, UserResponse, GoogleAuthRequest from app.services.auth_service import oauth_service @@ -109,6 +114,7 @@ async def register(user_in: UserCreate, db: AsyncSession = Depends(get_db)): await db.commit() await db.refresh(user) + AUTH_REGISTRATIONS_TOTAL.labels(method="password", status="success").inc() logger.info(f"New user registered: {user.email}") return user @@ -125,6 +131,7 @@ async def login( user = result.scalar_one_or_none() if not user or not user.hashed_password: + AUTH_LOGINS_TOTAL.labels(method="password", status="failure").inc() raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect email or password", @@ -133,6 +140,7 @@ async def login( # Verify password if not verify_password(form_data.password, user.hashed_password): # type: ignore[arg-type] + AUTH_LOGINS_TOTAL.labels(method="password", status="failure").inc() raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect email or password", @@ -141,6 +149,7 @@ async def login( # Check if user is active if not user.is_active: + AUTH_LOGINS_TOTAL.labels(method="password", status="failure").inc() raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="User account is inactive" ) @@ -162,6 +171,7 @@ async def login( # Create tokens tokens = oauth_service.create_tokens_for_user(user) + AUTH_LOGINS_TOTAL.labels(method="password", status="success").inc() logger.info(f"User logged in: {user.email}") return tokens @@ -182,6 +192,7 @@ async def google_oauth( ) if not user_info.get("verified_email"): + OAUTH_CALLBACKS_TOTAL.labels(provider="google", status="error").inc() raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Email not verified with Google", @@ -215,6 +226,7 @@ async def google_oauth( logger.info(f"Auto-promoted admin user via Google OAuth: {user.email}") logger.info(f"Existing user logged in with Google: {user.email}") + AUTH_LOGINS_TOTAL.labels(method="google", status="success").inc() else: # Domain restriction check before creating the account _check_domain_allowed(email) @@ -233,6 +245,7 @@ async def google_oauth( db.add(user) logger.info(f"New user registered with Google: {user.email}") + AUTH_REGISTRATIONS_TOTAL.labels(method="google", status="success").inc() await db.commit() await db.refresh(user) @@ -300,6 +313,7 @@ async def google_oauth( # Create tokens tokens = oauth_service.create_tokens_for_user(user) + OAUTH_CALLBACKS_TOTAL.labels(provider="google", status="success").inc() return tokens diff --git a/backend/app/core/metrics.py b/backend/app/core/metrics.py new file mode 100644 index 0000000..2918b34 --- /dev/null +++ b/backend/app/core/metrics.py @@ -0,0 +1,122 @@ +""" +Prometheus metrics definitions for InboxRescue. + +All application metrics are defined here as module-level singletons so that +every subsystem (HTTP layer, Celery workers, Gmail service, auth) imports +the same registry objects. +""" + +from prometheus_client import Counter, Histogram, Gauge + +# --------------------------------------------------------------------------- +# HTTP layer +# --------------------------------------------------------------------------- + +HTTP_REQUESTS_TOTAL = Counter( + "http_requests_total", + "Total HTTP requests received", + ["method", "endpoint", "status_code"], +) + +HTTP_REQUEST_DURATION_SECONDS = Histogram( + "http_request_duration_seconds", + "HTTP request duration in seconds", + ["method", "endpoint"], + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0), +) + +# --------------------------------------------------------------------------- +# Mail processing +# --------------------------------------------------------------------------- + +MAIL_PROCESSING_RUNS_TOTAL = Counter( + "mail_processing_runs_total", + "Total mail-account processing runs by final status", + ["status"], # completed | partial_failure | failed +) + +MAIL_PROCESSING_EMAILS_TOTAL = Counter( + "mail_processing_emails_total", + "Total emails encountered during processing runs", + ["operation"], # fetched | forwarded | failed +) + +MAIL_PROCESSING_DURATION_SECONDS = Histogram( + "mail_processing_duration_seconds", + "Duration of a single mail-account processing run in seconds", + buckets=(1, 5, 10, 30, 60, 120, 300, 600), +) + +ACTIVE_MAIL_ACCOUNTS = Gauge( + "active_mail_accounts_total", + "Number of enabled mail accounts queued for this scheduler cycle", +) + +# --------------------------------------------------------------------------- +# Gmail API +# --------------------------------------------------------------------------- + +GMAIL_API_REQUESTS_TOTAL = Counter( + "gmail_api_requests_total", + "Total Gmail API requests by operation and outcome", + [ + "operation", + "status", + ], # operation: inject|verify|get_label|get_profile status: success|error +) + +GMAIL_API_DURATION_SECONDS = Histogram( + "gmail_api_duration_seconds", + "Gmail API call duration in seconds", + ["operation"], + buckets=(0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0), +) + +GMAIL_TOKEN_REFRESHES_TOTAL = Counter( + "gmail_token_refreshes_total", + "Total number of OAuth access-token refreshes performed by GmailService", +) + +GMAIL_CREDENTIALS_INVALIDATED_TOTAL = Counter( + "gmail_credentials_invalidated_total", + "Total times a user's Gmail credentials were marked invalid (revoked token)", +) + +# --------------------------------------------------------------------------- +# Authentication / OAuth +# --------------------------------------------------------------------------- + +AUTH_LOGINS_TOTAL = Counter( + "auth_logins_total", + "Total login attempts by method and outcome", + ["method", "status"], # method: password|google status: success|failure +) + +AUTH_REGISTRATIONS_TOTAL = Counter( + "auth_registrations_total", + "Total registration attempts by method and outcome", + ["method", "status"], # method: password|google status: success|failure +) + +OAUTH_CALLBACKS_TOTAL = Counter( + "oauth_callbacks_total", + "Total OAuth2 callback events by provider and outcome", + ["provider", "status"], # provider: google status: success|error +) + +# --------------------------------------------------------------------------- +# Celery tasks +# --------------------------------------------------------------------------- + +CELERY_TASKS_TOTAL = Counter( + "celery_tasks_total", + "Total Celery task executions by task name and status", + ["task_name", "status"], # status: success|failure +) + +CELERY_TASK_DURATION_SECONDS = Histogram( + "celery_task_duration_seconds", + "Celery task execution duration in seconds", + ["task_name"], + buckets=(1, 5, 10, 30, 60, 120, 300, 600, 1800), +) diff --git a/backend/app/main.py b/backend/app/main.py index 31b2c53..ed40431 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,14 +2,18 @@ Main FastAPI application. """ +import re +import time from contextlib import asynccontextmanager from collections.abc import AsyncIterator -from fastapi import FastAPI +from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware +from prometheus_client import generate_latest, CONTENT_TYPE_LATEST import logging from app.core.config import settings from app.core.middleware import SecurityHeadersMiddleware, CSRFProtectionMiddleware +from app.core.metrics import HTTP_REQUESTS_TOTAL, HTTP_REQUEST_DURATION_SECONDS from app.api.v1.api import api_router # Configure logging @@ -88,6 +92,30 @@ def create_application() -> FastAPI: # Include API router app.include_router(api_router, prefix=settings.API_V1_PREFIX) + @app.middleware("http") + async def prometheus_middleware(request: Request, call_next): + """Record per-request Prometheus metrics.""" + # Normalize the path so high-cardinality IDs don't explode label sets. + path = request.url.path + # Strip numeric path segments (e.g. /api/v1/accounts/42 → /api/v1/accounts/{id}) + normalized = re.sub(r"/\d+", "/{id}", path) + + start = time.perf_counter() + response = await call_next(request) + duration = time.perf_counter() - start + + HTTP_REQUESTS_TOTAL.labels( + method=request.method, + endpoint=normalized, + status_code=str(response.status_code), + ).inc() + HTTP_REQUEST_DURATION_SECONDS.labels( + method=request.method, + endpoint=normalized, + ).observe(duration) + + return response + @app.get("/") async def root(): """Root endpoint""" @@ -102,6 +130,12 @@ def create_application() -> FastAPI: """Health check endpoint for container orchestration""" return {"status": "healthy"} + @app.get("/metrics", include_in_schema=False) + async def metrics(): + """Prometheus metrics endpoint.""" + data = generate_latest() + return Response(content=data, media_type=CONTENT_TYPE_LATEST) + return app diff --git a/backend/app/services/gmail_service.py b/backend/app/services/gmail_service.py index 505d976..ea5b93e 100644 --- a/backend/app/services/gmail_service.py +++ b/backend/app/services/gmail_service.py @@ -10,6 +10,7 @@ import asyncio import base64 import logging import textwrap +import time from datetime import datetime, timezone from email.mime.text import MIMEText from email.utils import format_datetime @@ -19,6 +20,12 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError +from app.core.metrics import ( + GMAIL_API_REQUESTS_TOTAL, + GMAIL_API_DURATION_SECONDS, + GMAIL_TOKEN_REFRESHES_TOTAL, +) + logger = logging.getLogger(__name__) # Gmail API scopes needed for email injection @@ -116,6 +123,7 @@ class GmailService: loop = asyncio.get_event_loop() + _start = time.perf_counter() try: result = await loop.run_in_executor( None, @@ -125,6 +133,10 @@ class GmailService: .execute(), ) + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="success").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur) + logger.info( f"Injected email into Gmail: id={result.get('id')}" f"{f' from {source_account_name}' if source_account_name else ''}" @@ -137,6 +149,9 @@ class GmailService: } except HttpError as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="error").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur) error_msg = ( f"Gmail API error: {e.reason if hasattr(e, 'reason') else str(e)}" ) @@ -144,6 +159,9 @@ class GmailService: # Surface 401 so callers can mark credentials as invalid raise GmailInjectionError(error_msg) except Exception as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="error").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur) error_msg = f"Failed to inject email into Gmail: {str(e)}" logger.error(error_msg) raise GmailInjectionError(error_msg) @@ -157,15 +175,22 @@ class GmailService: """ loop = asyncio.get_event_loop() + _start = time.perf_counter() try: result = await loop.run_in_executor( None, lambda: self.service.users().getProfile(userId="me").execute(), ) + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="verify", status="success").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="verify").observe(_dur) email = result.get("emailAddress", "unknown") logger.info(f"Gmail API access verified for: {email}") return True except Exception as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="verify", status="error").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="verify").observe(_dur) logger.error(f"Gmail API access verification failed: {e}") return False @@ -178,13 +203,24 @@ class GmailService: """ loop = asyncio.get_event_loop() + _start = time.perf_counter() try: result = await loop.run_in_executor( None, lambda: self.service.users().getProfile(userId="me").execute(), ) + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels( + operation="get_profile", status="success" + ).inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_profile").observe(_dur) return result.get("emailAddress") except Exception as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels( + operation="get_profile", status="error" + ).inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_profile").observe(_dur) logger.error(f"Failed to get Gmail email address: {e}") return None @@ -207,6 +243,7 @@ class GmailService: """ loop = asyncio.get_event_loop() + _start = time.perf_counter() try: labels_resp = await loop.run_in_executor( None, @@ -214,6 +251,13 @@ class GmailService: ) for label in labels_resp.get("labels", []): if label.get("name", "").lower() == name.lower(): + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels( + operation="get_label", status="success" + ).inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe( + _dur + ) return label["id"] # Label not found – create it @@ -224,14 +268,25 @@ class GmailService: .create(userId="me", body={"name": name}) .execute(), ) + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels( + operation="get_label", status="success" + ).inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur) logger.info(f"Created Gmail label '{name}' with id={created['id']}") return created["id"] except HttpError as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="get_label", status="error").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur) error_msg = f"Gmail API error while managing label '{name}': {e.reason if hasattr(e, 'reason') else str(e)}" logger.error(error_msg) raise GmailInjectionError(error_msg) except Exception as e: + _dur = time.perf_counter() - _start + GMAIL_API_REQUESTS_TOTAL.labels(operation="get_label", status="error").inc() + GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur) error_msg = f"Failed to get/create Gmail label '{name}': {str(e)}" logger.error(error_msg) raise GmailInjectionError(error_msg) @@ -316,6 +371,7 @@ class GmailService: """ current_token = self.credentials.token if current_token and current_token != self._initial_access_token: + GMAIL_TOKEN_REFRESHES_TOTAL.inc() return { "access_token": current_token, "expiry": self.credentials.expiry, diff --git a/backend/app/workers/tasks.py b/backend/app/workers/tasks.py index 72e6a24..30287a1 100644 --- a/backend/app/workers/tasks.py +++ b/backend/app/workers/tasks.py @@ -3,6 +3,7 @@ Celery tasks for background email processing. """ import asyncio +import time from datetime import datetime, timedelta, timezone from celery import Task import logging @@ -10,6 +11,15 @@ import logging from app.workers.celery_app import celery_app from app.core.database import async_session_maker, engine from app.core.security import decrypt_credential, encrypt_credential +from app.core.metrics import ( + MAIL_PROCESSING_RUNS_TOTAL, + MAIL_PROCESSING_EMAILS_TOTAL, + MAIL_PROCESSING_DURATION_SECONDS, + ACTIVE_MAIL_ACCOUNTS, + GMAIL_CREDENTIALS_INVALIDATED_TOTAL, + CELERY_TASKS_TOTAL, + CELERY_TASK_DURATION_SECONDS, +) from app.models.database_models import ( MailAccount, ProcessingRun, @@ -65,6 +75,7 @@ async def process_mail_account(account_id: int): Args: account_id: ID of mail account to process """ + _task_start = time.monotonic() async with async_session_maker() as db: try: # Get account @@ -108,6 +119,7 @@ async def process_mail_account(account_id: int): ) run.emails_fetched = len(emails) # type: ignore[assignment] + MAIL_PROCESSING_EMAILS_TOTAL.labels(operation="fetched").inc(len(emails)) # Forward emails emails_forwarded = 0 @@ -222,6 +234,7 @@ async def process_mail_account(account_id: int): ) ): gmail_cred.is_valid = False # type: ignore[assignment] + GMAIL_CREDENTIALS_INVALIDATED_TOTAL.inc() logger.warning( f"Gmail credentials revoked for user {account.user_id}. " "User must re-authorise." @@ -276,6 +289,22 @@ async def process_mail_account(account_id: int): await db.commit() + # Record Prometheus metrics for this completed run + _run_status = "completed" if emails_failed == 0 else "partial_failure" + MAIL_PROCESSING_RUNS_TOTAL.labels(status=_run_status).inc() + MAIL_PROCESSING_EMAILS_TOTAL.labels(operation="forwarded").inc( + emails_forwarded + ) + MAIL_PROCESSING_EMAILS_TOTAL.labels(operation="failed").inc(emails_failed) + _task_duration = time.monotonic() - _task_start + MAIL_PROCESSING_DURATION_SECONDS.observe(_task_duration) + CELERY_TASKS_TOTAL.labels( + task_name="process_mail_account", status="success" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels( + task_name="process_mail_account" + ).observe(_task_duration) + logger.info( f"Processed account {account.id}: " f"{emails_forwarded} forwarded, {emails_failed} failed" @@ -284,6 +313,16 @@ async def process_mail_account(account_id: int): except Exception as e: logger.error(f"Error processing account {account_id}: {e}") + # Record failure metric + _task_duration = time.monotonic() - _task_start + MAIL_PROCESSING_RUNS_TOTAL.labels(status="failed").inc() + CELERY_TASKS_TOTAL.labels( + task_name="process_mail_account", status="failure" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels( + task_name="process_mail_account" + ).observe(_task_duration) + # Mark run as failed if "run" in locals(): run.status = "failed" # type: ignore[assignment] @@ -308,6 +347,7 @@ async def process_all_enabled_accounts(): Process all enabled mail accounts. This task is scheduled to run periodically. """ + _task_start = time.monotonic() async with async_session_maker() as db: try: # Fetch all enabled accounts regardless of operational status so @@ -320,6 +360,7 @@ async def process_all_enabled_accounts(): accounts = result.scalars().all() logger.info(f"Processing {len(accounts)} enabled mail accounts") + ACTIVE_MAIL_ACCOUNTS.set(len(accounts)) # Process each account for account in accounts: @@ -337,8 +378,23 @@ async def process_all_enabled_accounts(): # Queue processing task process_mail_account.delay(account.id) + _task_duration = time.monotonic() - _task_start + CELERY_TASKS_TOTAL.labels( + task_name="process_all_enabled_accounts", status="success" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels( + task_name="process_all_enabled_accounts" + ).observe(_task_duration) + except Exception as e: logger.error(f"Error processing accounts: {e}") + _task_duration = time.monotonic() - _task_start + CELERY_TASKS_TOTAL.labels( + task_name="process_all_enabled_accounts", status="failure" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels( + task_name="process_all_enabled_accounts" + ).observe(_task_duration) @celery_app.task(base=AsyncTask, name="app.workers.tasks.cleanup_old_logs") @@ -349,6 +405,7 @@ async def cleanup_old_logs(days_to_keep: int = 30): Args: days_to_keep: Number of days of data to retain """ + _task_start = time.monotonic() async with async_session_maker() as db: try: cutoff_date = datetime.now(timezone.utc) - timedelta(days=days_to_keep) @@ -386,5 +443,20 @@ async def cleanup_old_logs(days_to_keep: int = 30): f"{len(old_logs)} old logs" ) + _task_duration = time.monotonic() - _task_start + CELERY_TASKS_TOTAL.labels( + task_name="cleanup_old_logs", status="success" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels(task_name="cleanup_old_logs").observe( + _task_duration + ) + except Exception as e: logger.error(f"Error cleaning up logs: {e}") + _task_duration = time.monotonic() - _task_start + CELERY_TASKS_TOTAL.labels( + task_name="cleanup_old_logs", status="failure" + ).inc() + CELERY_TASK_DURATION_SECONDS.labels(task_name="cleanup_old_logs").observe( + _task_duration + ) diff --git a/docker-compose.new.yml b/docker-compose.new.yml index baf6605..297ba11 100644 --- a/docker-compose.new.yml +++ b/docker-compose.new.yml @@ -101,6 +101,44 @@ services: - backend restart: unless-stopped + # Prometheus metrics collection + prometheus: + image: prom/prometheus:v2.51.2 + container_name: pop3-prometheus + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.enable-lifecycle" + depends_on: + - backend + restart: unless-stopped + + # Grafana dashboards + grafana: + image: grafana/grafana:10.4.3 + container_name: pop3-grafana + ports: + - "3001:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro + depends_on: + - prometheus + restart: unless-stopped + volumes: postgres_data: redis_data: + prometheus_data: + grafana_data: diff --git a/docs/TODO.md b/docs/TODO.md index eb04299..87e711b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -164,12 +164,21 @@ Comprehensive task breakdown for repository improvements and production readines ## 📊 Medium Priority - Observability +### Completed ✅ +- [x] Add Prometheus metrics endpoint (`/metrics`) to FastAPI backend +- [x] Instrument HTTP layer (request count + latency histograms per method/endpoint/status) +- [x] Instrument mail-processing tasks (runs, emails fetched/forwarded/failed, duration) +- [x] Instrument Gmail API operations (inject, verify, get_profile, get_label — count + latency) +- [x] Track OAuth token refreshes and credential invalidation events +- [x] Instrument auth endpoints (logins, registrations, OAuth callbacks — by method/status) +- [x] Instrument Celery tasks (count + duration per task name) +- [x] Add Prometheus scrape config (`monitoring/prometheus.yml`) +- [x] Add Grafana auto-provisioned datasource and pre-built dashboard (`monitoring/grafana/`) +- [x] Add Prometheus + Grafana services to `docker-compose.new.yml` (Grafana on port 3001) + ### Not Started 📋 -- [ ] Add Prometheus metrics endpoints - [ ] Integrate Sentry for error tracking - [ ] Add structured logging with correlation IDs -- [ ] Create Grafana dashboard templates -- [ ] Document monitoring setup - [ ] Add APM (Application Performance Monitoring) - [ ] Set up uptime monitoring - [ ] Create runbook for common issues diff --git a/monitoring/grafana/dashboards/inboxrescue.json b/monitoring/grafana/dashboards/inboxrescue.json new file mode 100644 index 0000000..018d5d2 --- /dev/null +++ b/monitoring/grafana/dashboards/inboxrescue.json @@ -0,0 +1,1128 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "InboxRescue system health: mail processing, Gmail API, auth, and HTTP metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "📬 Mail Processing", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "active_mail_accounts_total", + "legendFormat": "Active accounts", + "refId": "A" + } + ], + "title": "Active Mail Accounts", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(mail_processing_emails_total{operation=\"fetched\"}[$__rate_interval])", + "legendFormat": "Fetched", + "refId": "A" + } + ], + "title": "Emails Fetched (rate)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(mail_processing_emails_total{operation=\"forwarded\"}[$__rate_interval])", + "legendFormat": "Forwarded", + "refId": "A" + } + ], + "title": "Emails Forwarded (rate)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(mail_processing_emails_total{operation=\"failed\"}[$__rate_interval])", + "legendFormat": "Failed", + "refId": "A" + } + ], + "title": "Emails Failed (rate)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 3 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(mail_processing_runs_total{status=\"failed\"}[$__rate_interval])", + "legendFormat": "Failed runs", + "refId": "A" + } + ], + "title": "Failed Processing Runs (rate)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mail_processing_emails_total{operation=\"fetched\"}[5m])", + "legendFormat": "Fetched/s", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mail_processing_emails_total{operation=\"forwarded\"}[5m])", + "legendFormat": "Forwarded/s", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mail_processing_emails_total{operation=\"failed\"}[5m])", + "legendFormat": "Failed/s", + "refId": "C" + } + ], + "title": "Email Processing Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "p95"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.5, rate(mail_processing_duration_seconds_bucket[5m]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, rate(mail_processing_duration_seconds_bucket[5m]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, rate(mail_processing_duration_seconds_bucket[5m]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Mail Processing Duration (percentiles)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "id": 101, + "title": "📧 Gmail API", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 10, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(gmail_api_requests_total{status=\"success\"}[5m])", + "legendFormat": "{{operation}} success", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(gmail_api_requests_total{status=\"error\"}[5m])", + "legendFormat": "{{operation}} error", + "refId": "B" + } + ], + "title": "Gmail API Request Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 11, + "options": { + "legend": { + "calcs": ["mean", "p95"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.5, rate(gmail_api_duration_seconds_bucket[5m]))", + "legendFormat": "{{operation}} p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, rate(gmail_api_duration_seconds_bucket[5m]))", + "legendFormat": "{{operation}} p95", + "refId": "B" + } + ], + "title": "Gmail API Latency (percentiles)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 22 }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(gmail_token_refreshes_total[$__rate_interval])", + "legendFormat": "Token refreshes", + "refId": "A" + } + ], + "title": "Token Refreshes", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 22 }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(gmail_credentials_invalidated_total[$__rate_interval])", + "legendFormat": "Revoked credentials", + "refId": "A" + } + ], + "title": "Credentials Invalidated (revoked tokens)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "id": 102, + "title": "🔐 Authentication & OAuth", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, + "id": 20, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(auth_logins_total{status=\"success\"}[5m])", + "legendFormat": "{{method}} success", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(auth_logins_total{status=\"failure\"}[5m])", + "legendFormat": "{{method}} failure", + "refId": "B" + } + ], + "title": "Login Rate (success vs failure)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, + "id": 21, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(oauth_callbacks_total[5m])", + "legendFormat": "{{provider}} {{status}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(auth_registrations_total[5m])", + "legendFormat": "Registration {{method}} {{status}}", + "refId": "B" + } + ], + "title": "OAuth Callbacks & Registrations", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, + "id": 103, + "title": "🌐 HTTP API", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 30, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_requests_total[5m])) by (method, status_code)", + "legendFormat": "{{method}} {{status_code}}", + "refId": "A" + } + ], + "title": "HTTP Request Rate by Method & Status", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean", "p95"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, endpoint))", + "legendFormat": "{{endpoint}} p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, endpoint))", + "legendFormat": "{{endpoint}} p95", + "refId": "B" + } + ], + "title": "HTTP Request Latency (percentiles)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "status_code" }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 400 }, + { "color": "red", "value": 500 } + ] + } + } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 44 }, + "id": 32, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Value" }] + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sort_desc(sum(rate(http_requests_total[5m])) by (method, endpoint, status_code))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top Endpoints (request rate)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "indexByName": {}, + "renameByName": { + "Value": "req/s", + "endpoint": "Endpoint", + "method": "Method", + "status_code": "Status" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 52 }, + "id": 104, + "title": "⚙️ Celery Workers", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 }, + "id": 40, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(celery_tasks_total{status=\"success\"}[5m])", + "legendFormat": "{{task_name}} success", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(celery_tasks_total{status=\"failure\"}[5m])", + "legendFormat": "{{task_name}} failure", + "refId": "B" + } + ], + "title": "Celery Task Throughput", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 53 }, + "id": 41, + "options": { + "legend": { + "calcs": ["mean", "p95"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.5, rate(celery_task_duration_seconds_bucket[5m]))", + "legendFormat": "{{task_name}} p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, rate(celery_task_duration_seconds_bucket[5m]))", + "legendFormat": "{{task_name}} p95", + "refId": "B" + } + ], + "title": "Celery Task Duration (percentiles)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["inboxrescue", "monitoring", "email"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "InboxRescue - System Health", + "uid": "inboxrescue-health", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboard.yml b/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..6b620d1 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: InboxRescue + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..bb009bb --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 0000000..f3a8447 --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "inboxrescue-backend" + static_configs: + - targets: ["backend:8000"] + metrics_path: /metrics