Add database-backed configuration (AppSetting model, ConfigService, settings API)

- Add AppSetting model for key-value settings storage in PostgreSQL
- Create ConfigService with DB-first, env-var-fallback resolution
- Add admin-only /api/v1/settings CRUD endpoints
- Update tasks.py to use ConfigService for SMTP config
- Seed default settings on application startup
- Add 24 unit tests for ConfigService (all passing)

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/pop_puller_to_gmail/sessions/ce88d4a8-d8c2-49b3-95a1-30592105a769
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 13:35:52 +00:00
parent 91c545ffd4
commit 702650376e
8 changed files with 840 additions and 10 deletions
+2
View File
@@ -12,6 +12,7 @@ from app.api.v1.endpoints import (
subscriptions,
admin,
providers,
app_settings,
)
api_router = APIRouter()
@@ -32,3 +33,4 @@ api_router.include_router(
subscriptions.router, prefix="/subscriptions", tags=["Subscriptions"]
)
api_router.include_router(admin.router, prefix="/admin", tags=["Admin"])
api_router.include_router(app_settings.router, prefix="/settings", tags=["Settings"])
@@ -0,0 +1,148 @@
"""Admin API endpoints for managing database-backed application settings."""
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import BaseModel, ConfigDict
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import get_db
from app.core.deps import get_current_superuser
from app.models.database_models import User
from app.services.config_service import BOOTSTRAP_KEYS, ConfigService
router = APIRouter()
# ── Schemas ────────────────────────────────────────────────────────
class AppSettingResponse(BaseModel):
"""Public representation of a stored setting."""
id: int
key: str
value: Optional[str] = None
value_type: str
description: Optional[str] = None
is_secret: bool
category: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
class AppSettingCreate(BaseModel):
"""Payload for creating / updating a setting."""
key: str
value: str
value_type: str = "string"
description: Optional[str] = None
is_secret: bool = False
category: Optional[str] = None
# ── Endpoints ──────────────────────────────────────────────────────
@router.get("", response_model=List[AppSettingResponse])
async def list_settings(
category: Optional[str] = Query(None, description="Filter by category"),
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""List all application settings (admin only).
Secrets are returned with their values masked.
"""
settings = await ConfigService.list_all(db, category=category)
results: List[AppSettingResponse] = []
for s in settings:
val = s.value if not s.is_secret else "********" # type: ignore[arg-type]
results.append(
AppSettingResponse(
id=s.id, # type: ignore[arg-type]
key=s.key, # type: ignore[arg-type]
value=val, # type: ignore[arg-type]
value_type=s.value_type or "string", # type: ignore[arg-type]
description=s.description, # type: ignore[arg-type]
is_secret=s.is_secret, # type: ignore[arg-type]
category=s.category, # type: ignore[arg-type]
)
)
return results
@router.put("/{key}", response_model=AppSettingResponse)
async def upsert_setting(
key: str,
payload: AppSettingCreate,
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""Create or update a setting (admin only).
Bootstrap settings (DATABASE_URL, SECRET_KEY, ENCRYPTION_KEY) cannot
be stored in the database — they must be set via environment variables.
"""
if key in BOOTSTRAP_KEYS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=(
f"'{key}' is a bootstrap setting and cannot be stored in the "
"database. Set it via environment variables."
),
)
setting = await ConfigService.set(
key=key,
value=payload.value,
db=db,
value_type=payload.value_type,
description=payload.description,
is_secret=payload.is_secret,
category=payload.category,
)
return AppSettingResponse(
id=setting.id, # type: ignore[arg-type]
key=setting.key, # type: ignore[arg-type]
value="********" if setting.is_secret else setting.value, # type: ignore[arg-type]
value_type=setting.value_type or "string", # type: ignore[arg-type]
description=setting.description, # type: ignore[arg-type]
is_secret=setting.is_secret, # type: ignore[arg-type]
category=setting.category, # type: ignore[arg-type]
)
@router.delete("/{key}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_setting(
key: str,
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""Delete a setting from the database (admin only)."""
if key in BOOTSTRAP_KEYS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=(f"'{key}' is a bootstrap setting and cannot be deleted."),
)
deleted = await ConfigService.delete(key, db)
if not deleted:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Setting '{key}' not found",
)
@router.post("/seed-defaults", status_code=status.HTTP_200_OK)
async def seed_default_settings(
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""Seed default settings into the database (admin only).
Only creates settings that do not already exist.
"""
count = await ConfigService.seed_defaults(db)
return {"message": f"Seeded {count} default settings", "created": count}
+12 -1
View File
@@ -1,6 +1,17 @@
"""
Application configuration using Pydantic settings.
Supports environment variables and .env files.
Supports a hybrid configuration model:
• **Bootstrap settings** (DATABASE_URL, SECRET_KEY, ENCRYPTION_KEY)
are always loaded from environment variables or ``.env`` files.
• **Application settings** (SMTP, processing, Gmail API, etc.)
can be managed in the database via the ``AppSetting`` model and
the ``/api/v1/settings`` admin endpoints. When a setting exists
in the database it takes precedence over environment variables.
See ``app.services.config_service.ConfigService`` for the runtime
lookup logic and ``app.models.database_models.AppSetting`` for the
database model.
"""
from typing import Optional, List
+11
View File
@@ -28,6 +28,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
logger.info(f"Starting {settings.APP_NAME} v{settings.APP_VERSION}")
logger.info(f"Debug mode: {settings.DEBUG}")
logger.info("API documentation: /api/docs")
# Seed default database-backed settings (no-op if they already exist)
try:
from app.core.database import async_session_maker
from app.services.config_service import ConfigService
async with async_session_maker() as db:
await ConfigService.seed_defaults(db)
except Exception as exc:
logger.warning("Could not seed default settings: %s", exc)
yield
# Shutdown
logger.info("Shutting down application")
+27
View File
@@ -430,3 +430,30 @@ class GmailCredential(Base):
# Relationships
user = relationship("User", backref="gmail_credential")
class AppSetting(Base):
"""
Application settings stored in the database.
Provides database-backed configuration that supplements or overrides
environment variable settings. Bootstrap settings (DATABASE_URL,
SECRET_KEY, ENCRYPTION_KEY) must still come from environment variables,
but all other settings can be managed via the database.
"""
__tablename__ = "app_settings"
id = Column(Integer, primary_key=True, index=True)
key = Column(String(255), unique=True, nullable=False, index=True)
value = Column(Text, nullable=True)
value_type = Column(String(50), default="string") # string, int, float, bool, json
description = Column(Text, nullable=True)
is_secret = Column(Boolean, default=False)
category = Column(String(100), nullable=True, index=True)
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
+338
View File
@@ -0,0 +1,338 @@
"""
Configuration service for hybrid env + database settings.
Provides a unified interface for reading application configuration.
Settings are resolved in this order:
1. Database (app_settings table) — highest priority
2. Environment variables / .env file — fallback
3. Hard-coded defaults — last resort
Bootstrap settings (DATABASE_URL, SECRET_KEY, ENCRYPTION_KEY) always
come from environment variables because the database connection itself
depends on them.
"""
import json
import logging
import os
from typing import Any, Dict, List, Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.database_models import AppSetting
logger = logging.getLogger(__name__)
# Settings that MUST come from env vars (bootstrap / chicken-and-egg)
BOOTSTRAP_KEYS = frozenset(
{
"DATABASE_URL",
"DATABASE_POOL_SIZE",
"DATABASE_MAX_OVERFLOW",
"SECRET_KEY",
"ENCRYPTION_KEY",
}
)
# Default settings to seed into the database on first run
DEFAULT_SETTINGS: List[Dict[str, Any]] = [
# SMTP / Forwarding
{
"key": "SMTP_HOST",
"value": "smtp.gmail.com",
"value_type": "string",
"description": "SMTP server hostname for email forwarding",
"is_secret": False,
"category": "smtp",
},
{
"key": "SMTP_PORT",
"value": "587",
"value_type": "int",
"description": "SMTP server port",
"is_secret": False,
"category": "smtp",
},
{
"key": "SMTP_USER",
"value": "",
"value_type": "string",
"description": "SMTP username for authentication",
"is_secret": False,
"category": "smtp",
},
{
"key": "SMTP_PASSWORD",
"value": "",
"value_type": "string",
"description": "SMTP password for authentication",
"is_secret": True,
"category": "smtp",
},
{
"key": "SMTP_USE_TLS",
"value": "true",
"value_type": "bool",
"description": "Whether to use STARTTLS for SMTP connections",
"is_secret": False,
"category": "smtp",
},
# Email Processing
{
"key": "MAX_EMAILS_PER_RUN",
"value": "50",
"value_type": "int",
"description": "Maximum emails to process per account per run",
"is_secret": False,
"category": "processing",
},
{
"key": "CHECK_INTERVAL_MINUTES",
"value": "5",
"value_type": "int",
"description": "Default interval between mail checks (minutes)",
"is_secret": False,
"category": "processing",
},
{
"key": "THROTTLE_EMAILS_PER_MINUTE",
"value": "10",
"value_type": "int",
"description": "Rate limit for email forwarding",
"is_secret": False,
"category": "processing",
},
# Gmail API
{
"key": "GMAIL_API_ENABLED",
"value": "true",
"value_type": "bool",
"description": "Enable Gmail API for direct email injection (preferred over SMTP)",
"is_secret": False,
"category": "gmail",
},
{
"key": "GMAIL_INJECT_LABEL_IDS",
"value": '["INBOX"]',
"value_type": "json",
"description": "Gmail label IDs to apply when injecting emails via API",
"is_secret": False,
"category": "gmail",
},
# Notifications
{
"key": "APPRISE_ENABLED",
"value": "true",
"value_type": "bool",
"description": "Enable Apprise multi-channel notifications",
"is_secret": False,
"category": "notifications",
},
# Logging
{
"key": "LOG_LEVEL",
"value": "INFO",
"value_type": "string",
"description": "Application log level (DEBUG, INFO, WARNING, ERROR)",
"is_secret": False,
"category": "general",
},
]
def _cast_value(raw: str, value_type: str) -> Any:
"""Cast a raw string value to its declared type."""
if raw is None:
return None
if value_type == "int":
return int(raw)
if value_type == "float":
return float(raw)
if value_type == "bool":
return raw.lower() in ("true", "1", "yes")
if value_type == "json":
return json.loads(raw)
return raw # string
class ConfigService:
"""
Hybrid configuration service: database-first with env-var fallback.
Usage::
value = await ConfigService.get("SMTP_HOST", db=session)
smtp_config = await ConfigService.get_smtp_config(db=session)
"""
@staticmethod
async def get(
key: str,
db: Optional[AsyncSession] = None,
default: Any = None,
) -> Any:
"""
Retrieve a single setting value.
Resolution order: database → environment variable → *default*.
Bootstrap keys (DATABASE_URL, SECRET_KEY, ENCRYPTION_KEY) skip the
database lookup entirely.
"""
if key not in BOOTSTRAP_KEYS and db is not None:
try:
result = await db.execute(
select(AppSetting).where(AppSetting.key == key)
)
setting = result.scalar_one_or_none()
if setting is not None and setting.value is not None:
return _cast_value(
setting.value, # type: ignore[arg-type]
setting.value_type or "string", # type: ignore[arg-type]
)
except Exception:
logger.debug("DB lookup failed for key=%s, falling back to env", key)
env_val = os.getenv(key)
if env_val is not None:
return env_val
return default
@staticmethod
async def get_many(
keys: List[str],
db: Optional[AsyncSession] = None,
) -> Dict[str, Any]:
"""Retrieve multiple settings at once."""
result: Dict[str, Any] = {}
if db is not None:
try:
db_result = await db.execute(
select(AppSetting).where(AppSetting.key.in_(keys))
)
for setting in db_result.scalars().all():
result[setting.key] = _cast_value( # type: ignore[index]
setting.value, # type: ignore[arg-type]
setting.value_type or "string", # type: ignore[arg-type]
)
except Exception:
logger.debug("DB bulk lookup failed, falling back to env")
for key in keys:
if key not in result:
env_val = os.getenv(key)
if env_val is not None:
result[key] = env_val
return result
@staticmethod
async def set(
key: str,
value: str,
db: AsyncSession,
value_type: str = "string",
description: Optional[str] = None,
is_secret: bool = False,
category: Optional[str] = None,
) -> AppSetting:
"""Create or update a setting in the database."""
if key in BOOTSTRAP_KEYS:
raise ValueError(
f"'{key}' is a bootstrap setting and cannot be stored in the database. "
"Set it via environment variables instead."
)
result = await db.execute(select(AppSetting).where(AppSetting.key == key))
existing = result.scalar_one_or_none()
if existing:
existing.value = value # type: ignore[assignment]
if value_type:
existing.value_type = value_type # type: ignore[assignment]
if description is not None:
existing.description = description # type: ignore[assignment]
if is_secret is not None:
existing.is_secret = is_secret # type: ignore[assignment]
if category is not None:
existing.category = category # type: ignore[assignment]
await db.commit()
await db.refresh(existing)
return existing
setting = AppSetting(
key=key,
value=value,
value_type=value_type,
description=description,
is_secret=is_secret,
category=category,
)
db.add(setting)
await db.commit()
await db.refresh(setting)
return setting
@staticmethod
async def delete(key: str, db: AsyncSession) -> bool:
"""Delete a setting from the database."""
if key in BOOTSTRAP_KEYS:
raise ValueError(
f"'{key}' is a bootstrap setting and cannot be deleted from the database."
)
result = await db.execute(select(AppSetting).where(AppSetting.key == key))
existing = result.scalar_one_or_none()
if existing:
await db.delete(existing)
await db.commit()
return True
return False
@staticmethod
async def list_all(
db: AsyncSession,
category: Optional[str] = None,
) -> List[AppSetting]:
"""List all settings, optionally filtered by category."""
query = select(AppSetting).order_by(AppSetting.category, AppSetting.key)
if category:
query = query.where(AppSetting.category == category)
result = await db.execute(query)
return list(result.scalars().all())
# ── convenience helpers ─────────────────────────────────────────
@staticmethod
async def get_smtp_config(db: Optional[AsyncSession] = None) -> Dict[str, Any]:
"""Return a ready-to-use SMTP configuration dict."""
keys = ["SMTP_HOST", "SMTP_PORT", "SMTP_USER", "SMTP_PASSWORD", "SMTP_USE_TLS"]
values = await ConfigService.get_many(keys, db=db)
return {
"host": values.get("SMTP_HOST", "smtp.gmail.com"),
"port": int(values.get("SMTP_PORT", 587)),
"username": values.get("SMTP_USER", ""),
"password": values.get("SMTP_PASSWORD", ""),
"use_tls": str(values.get("SMTP_USE_TLS", "true")).lower()
in ("true", "1", "yes"),
}
@staticmethod
async def seed_defaults(db: AsyncSession) -> int:
"""
Populate the database with default settings (skip existing keys).
Returns the number of settings created.
"""
created = 0
for item in DEFAULT_SETTINGS:
result = await db.execute(
select(AppSetting).where(AppSetting.key == item["key"])
)
if result.scalar_one_or_none() is None:
db.add(AppSetting(**item))
created += 1
if created:
await db.commit()
logger.info("Seeded %d default settings into the database", created)
return created
+3 -9
View File
@@ -3,7 +3,6 @@ Celery tasks for background email processing.
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from celery import Task
import logging
@@ -21,6 +20,7 @@ from app.models.database_models import (
)
from app.services.mail_processor import MailProcessor
from app.services.gmail_service import GmailService
from app.services.config_service import ConfigService
from app.core.config import settings
from sqlalchemy import select, and_
@@ -118,14 +118,8 @@ async def process_mail_account(account_id: int):
use_gmail_api = False # type: ignore[assignment]
if not use_gmail_api:
# Fall back to SMTP
smtp_config = {
"host": os.getenv("SMTP_HOST", "smtp.gmail.com"),
"port": int(os.getenv("SMTP_PORT", "587")),
"username": os.getenv("SMTP_USER", ""),
"password": os.getenv("SMTP_PASSWORD", ""),
"use_tls": os.getenv("SMTP_USE_TLS", "true").lower() == "true",
}
# Fall back to SMTP read config from DB with env fallback
smtp_config = await ConfigService.get_smtp_config(db=db)
if not smtp_config["username"] or not smtp_config["password"]:
logger.error(