feat: add Prometheus metrics and Grafana dashboard for system health monitoring

Agent-Logs-Url: https://github.com/christianlouis/pop_puller_to_gmail/sessions/11ac4536-eaad-44fe-920c-e8306918f06f

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-26 16:31:23 +00:00
parent d85bd6cfd1
commit ca87abce0a
12 changed files with 1517 additions and 4 deletions
+56
View File
@@ -10,6 +10,7 @@ import asyncio
import base64
import logging
import textwrap
import time
from datetime import datetime, timezone
from email.mime.text import MIMEText
from email.utils import format_datetime
@@ -19,6 +20,12 @@ from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from app.core.metrics import (
GMAIL_API_REQUESTS_TOTAL,
GMAIL_API_DURATION_SECONDS,
GMAIL_TOKEN_REFRESHES_TOTAL,
)
logger = logging.getLogger(__name__)
# Gmail API scopes needed for email injection
@@ -116,6 +123,7 @@ class GmailService:
loop = asyncio.get_event_loop()
_start = time.perf_counter()
try:
result = await loop.run_in_executor(
None,
@@ -125,6 +133,10 @@ class GmailService:
.execute(),
)
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="success").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur)
logger.info(
f"Injected email into Gmail: id={result.get('id')}"
f"{f' from {source_account_name}' if source_account_name else ''}"
@@ -137,6 +149,9 @@ class GmailService:
}
except HttpError as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="error").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur)
error_msg = (
f"Gmail API error: {e.reason if hasattr(e, 'reason') else str(e)}"
)
@@ -144,6 +159,9 @@ class GmailService:
# Surface 401 so callers can mark credentials as invalid
raise GmailInjectionError(error_msg)
except Exception as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="inject", status="error").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="inject").observe(_dur)
error_msg = f"Failed to inject email into Gmail: {str(e)}"
logger.error(error_msg)
raise GmailInjectionError(error_msg)
@@ -157,15 +175,22 @@ class GmailService:
"""
loop = asyncio.get_event_loop()
_start = time.perf_counter()
try:
result = await loop.run_in_executor(
None,
lambda: self.service.users().getProfile(userId="me").execute(),
)
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="verify", status="success").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="verify").observe(_dur)
email = result.get("emailAddress", "unknown")
logger.info(f"Gmail API access verified for: {email}")
return True
except Exception as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="verify", status="error").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="verify").observe(_dur)
logger.error(f"Gmail API access verification failed: {e}")
return False
@@ -178,13 +203,24 @@ class GmailService:
"""
loop = asyncio.get_event_loop()
_start = time.perf_counter()
try:
result = await loop.run_in_executor(
None,
lambda: self.service.users().getProfile(userId="me").execute(),
)
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(
operation="get_profile", status="success"
).inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_profile").observe(_dur)
return result.get("emailAddress")
except Exception as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(
operation="get_profile", status="error"
).inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_profile").observe(_dur)
logger.error(f"Failed to get Gmail email address: {e}")
return None
@@ -207,6 +243,7 @@ class GmailService:
"""
loop = asyncio.get_event_loop()
_start = time.perf_counter()
try:
labels_resp = await loop.run_in_executor(
None,
@@ -214,6 +251,13 @@ class GmailService:
)
for label in labels_resp.get("labels", []):
if label.get("name", "").lower() == name.lower():
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(
operation="get_label", status="success"
).inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(
_dur
)
return label["id"]
# Label not found create it
@@ -224,14 +268,25 @@ class GmailService:
.create(userId="me", body={"name": name})
.execute(),
)
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(
operation="get_label", status="success"
).inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur)
logger.info(f"Created Gmail label '{name}' with id={created['id']}")
return created["id"]
except HttpError as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="get_label", status="error").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur)
error_msg = f"Gmail API error while managing label '{name}': {e.reason if hasattr(e, 'reason') else str(e)}"
logger.error(error_msg)
raise GmailInjectionError(error_msg)
except Exception as e:
_dur = time.perf_counter() - _start
GMAIL_API_REQUESTS_TOTAL.labels(operation="get_label", status="error").inc()
GMAIL_API_DURATION_SECONDS.labels(operation="get_label").observe(_dur)
error_msg = f"Failed to get/create Gmail label '{name}': {str(e)}"
logger.error(error_msg)
raise GmailInjectionError(error_msg)
@@ -316,6 +371,7 @@ class GmailService:
"""
current_token = self.credentials.token
if current_token and current_token != self._initial_access_token:
GMAIL_TOKEN_REFRESHES_TOTAL.inc()
return {
"access_token": current_token,
"expiry": self.credentials.expiry,