Merge branch 'main' into copilot/add-apprise-alerting-capabilities

This commit is contained in:
Christian Krakau-Louis
2026-03-26 19:39:27 +01:00
committed by GitHub
63 changed files with 1601 additions and 223 deletions
+5 -3
View File
@@ -1,5 +1,5 @@
# Database Configuration
DATABASE_URL=postgresql+asyncpg://postgres:password@localhost:5432/pop3_forwarder
DATABASE_URL=postgresql+asyncpg://postgres:password@localhost:5432/inbox_converge
# Security
SECRET_KEY=change-this-to-a-secure-random-secret-key-minimum-32-characters
@@ -41,12 +41,14 @@ CELERY_RESULT_BACKEND=redis://localhost:6379/0
LOG_LEVEL=INFO
# Admin Account (created on first startup)
ADMIN_EMAIL=admin@example.com
ADMIN_EMAIL=christian@inboxconverge.com
ADMIN_PASSWORD=change-this-secure-password
# Application
APP_NAME=POP3 Forwarder SaaS
APP_NAME=InboxConverge
APP_VERSION=2.0.0
APP_URL=https://inboxconverge.com
CONTACT_EMAIL=christian@inboxconverge.com
DEBUG=false
HOST=0.0.0.0
PORT=8000
+4
View File
@@ -13,6 +13,7 @@ from app.api.v1.endpoints import (
admin,
providers,
app_settings,
logs,
)
api_router = APIRouter()
@@ -34,3 +35,6 @@ api_router.include_router(
)
api_router.include_router(admin.router, prefix="/admin", tags=["Admin"])
api_router.include_router(app_settings.router, prefix="/settings", tags=["Settings"])
api_router.include_router(
logs.router, prefix="/processing-runs", tags=["Processing Logs"]
)
+169 -2
View File
@@ -1,15 +1,18 @@
"""Admin endpoints"""
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
import math
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from app.core.database import get_db
from app.core.deps import get_current_superuser
from app.core.gdpr import mask_email, mask_from_header
from app.models.database_models import (
User,
MailAccount,
ProcessingLog,
ProcessingRun,
SubscriptionPlan,
SubscriptionTier,
@@ -27,6 +30,10 @@ from app.models.schemas import (
AdminNotificationConfigResponse,
NotificationTestRequest,
NotificationTestResponse,
AdminProcessingRunResponse,
AdminProcessingLogResponse,
PaginatedAdminRunsResponse,
PaginatedAdminLogsResponse,
)
from app.services.notification_service import test_notification
@@ -400,3 +407,163 @@ async def test_admin_notification_config(
"""Test an admin notification channel by sending a test message (admin only)"""
success, message = await test_notification(request.apprise_url)
return NotificationTestResponse(success=success, message=message)
# ── Admin Logs ─────────────────────────────────────────────────────────────────
def _admin_paginate(total: int, page: int, page_size: int) -> dict:
pages = max(1, math.ceil(total / page_size)) if total else 1
return {"total": total, "page": page, "page_size": page_size, "pages": pages}
@router.get(
"/processing-runs",
response_model=PaginatedAdminRunsResponse,
summary="List all processing runs across all users (admin only)",
)
async def admin_list_processing_runs(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
user_id: Optional[int] = Query(None, description="Filter by user ID"),
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
status_filter: Optional[str] = Query(
None,
alias="status",
description="Filter by run status",
),
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""
Return a paginated list of all processing runs in the system with
GDPR-masked user / account email addresses.
"""
base = (
select(
ProcessingRun,
MailAccount.name.label("account_name"),
MailAccount.email_address.label("account_email"),
MailAccount.user_id.label("uid"),
User.email.label("user_email"),
)
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
.join(User, MailAccount.user_id == User.id)
)
if user_id is not None:
base = base.where(MailAccount.user_id == user_id)
if account_id is not None:
base = base.where(ProcessingRun.mail_account_id == account_id)
if status_filter:
base = base.where(ProcessingRun.status == status_filter)
total = (
await db.execute(select(func.count()).select_from(base.subquery()))
).scalar_one()
offset = (page - 1) * page_size
rows = (
await db.execute(
base.order_by(ProcessingRun.started_at.desc())
.offset(offset)
.limit(page_size)
)
).all()
items = [
AdminProcessingRunResponse(
id=row.ProcessingRun.id, # type: ignore[arg-type]
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
status=row.ProcessingRun.status, # type: ignore[arg-type]
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
account_name=row.account_name,
account_email=mask_email(row.account_email) if row.account_email else None,
user_id=row.uid,
user_email=mask_email(row.user_email) if row.user_email else None,
)
for row in rows
]
return PaginatedAdminRunsResponse(
items=items, **_admin_paginate(total, page, page_size) # type: ignore[arg-type]
)
@router.get(
"/processing-logs",
response_model=PaginatedAdminLogsResponse,
summary="List all per-email processing logs across all users (admin only)",
)
async def admin_list_processing_logs(
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
user_id: Optional[int] = Query(None, description="Filter by user ID"),
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
run_id: Optional[int] = Query(None, description="Filter by processing run ID"),
level: Optional[str] = Query(
None, description="Filter by level (INFO, WARNING, ERROR)"
),
current_user: User = Depends(get_current_superuser),
db: AsyncSession = Depends(get_db),
):
"""
Return paginated per-email log entries with GDPR-masked sender addresses.
Subject lines are shown as-is (the user owns their own mail content);
sender addresses are pseudonymised for operator privacy.
"""
base = select(
ProcessingLog,
User.email.label("user_email"),
).join(User, ProcessingLog.user_id == User.id)
if user_id is not None:
base = base.where(ProcessingLog.user_id == user_id)
if account_id is not None:
base = base.where(ProcessingLog.mail_account_id == account_id)
if run_id is not None:
base = base.where(ProcessingLog.processing_run_id == run_id)
if level:
base = base.where(ProcessingLog.level == level.upper())
total = (
await db.execute(select(func.count()).select_from(base.subquery()))
).scalar_one()
offset = (page - 1) * page_size
rows = (
await db.execute(
base.order_by(ProcessingLog.timestamp.desc())
.offset(offset)
.limit(page_size)
)
).all()
items = [
AdminProcessingLogResponse(
id=row.ProcessingLog.id, # type: ignore[arg-type]
timestamp=row.ProcessingLog.timestamp, # type: ignore[arg-type]
level=row.ProcessingLog.level, # type: ignore[arg-type]
message=row.ProcessingLog.message, # type: ignore[arg-type]
email_subject=row.ProcessingLog.email_subject, # type: ignore[arg-type]
email_from=(
mask_from_header(row.ProcessingLog.email_from)
if row.ProcessingLog.email_from
else None
),
success=row.ProcessingLog.success, # type: ignore[arg-type]
mail_account_id=row.ProcessingLog.mail_account_id, # type: ignore[arg-type]
processing_run_id=row.ProcessingLog.processing_run_id, # type: ignore[arg-type]
email_size_bytes=row.ProcessingLog.email_size_bytes, # type: ignore[arg-type]
error_details=row.ProcessingLog.error_details, # type: ignore[arg-type]
user_id=row.ProcessingLog.user_id, # type: ignore[arg-type]
user_email=mask_email(row.user_email) if row.user_email else None,
)
for row in rows
]
return PaginatedAdminLogsResponse(
items=items, **_admin_paginate(total, page, page_size) # type: ignore[arg-type]
)
+6 -2
View File
@@ -22,6 +22,7 @@ from app.models.database_models import User, SubscriptionTier, GmailCredential
from app.models.schemas import Token, UserCreate, UserResponse, GoogleAuthRequest
from app.services.auth_service import oauth_service
from app.services.gmail_service import GmailService, GMAIL_SCOPES
from app.utils.gmail_labels import build_gmail_credential_scopes
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -286,7 +287,10 @@ async def google_oauth(
if encrypted_refresh:
existing_cred.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
existing_cred.token_expiry = token_expiry # type: ignore[assignment]
existing_cred.scopes = scope_list # type: ignore[assignment]
existing_cred.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
scope_list,
existing_cred.import_label_templates,
)
existing_cred.is_valid = True # type: ignore[assignment]
existing_cred.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
else:
@@ -296,7 +300,7 @@ async def google_oauth(
encrypted_access_token=encrypted_access,
encrypted_refresh_token=encrypted_refresh,
token_expiry=token_expiry,
scopes=scope_list,
scopes=build_gmail_credential_scopes(scope_list),
is_valid=True,
last_verified_at=datetime.now(timezone.utc),
)
+234
View File
@@ -0,0 +1,234 @@
"""
Processing logs and run history endpoints for users.
Users can view the full history of processing runs and per-email logs
for their own mailboxes. Admin equivalents live in admin.py.
"""
from __future__ import annotations
import math
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import get_db
from app.core.deps import get_current_active_user
from app.models.database_models import (
MailAccount,
ProcessingLog,
ProcessingRun,
User,
)
from app.models.schemas import (
PaginatedProcessingLogsResponse,
PaginatedProcessingRunsResponse,
ProcessingLogDetailResponse,
ProcessingRunDetailResponse,
)
router = APIRouter()
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def _paginate(total: int, page: int, page_size: int) -> dict:
pages = max(1, math.ceil(total / page_size)) if total else 1
return {"total": total, "page": page, "page_size": page_size, "pages": pages}
# ---------------------------------------------------------------------------
# Processing Runs (all accounts belonging to the current user)
# ---------------------------------------------------------------------------
@router.get(
"/processing-runs",
response_model=PaginatedProcessingRunsResponse,
summary="List processing runs for the current user",
)
async def list_processing_runs(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
status_filter: Optional[str] = Query(
None,
alias="status",
description="Filter by run status (completed, failed, partial_failure, running)",
),
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""
Return a paginated list of processing runs for all mail accounts owned by
the authenticated user, optionally filtered by account or status.
"""
# Base query: join with MailAccount to enforce ownership
base = (
select(ProcessingRun, MailAccount.name, MailAccount.email_address)
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
.where(MailAccount.user_id == current_user.id) # type: ignore[arg-type]
)
if account_id is not None:
base = base.where(ProcessingRun.mail_account_id == account_id)
if status_filter:
base = base.where(ProcessingRun.status == status_filter)
# Total count
count_q = select(func.count()).select_from(base.subquery())
total = (await db.execute(count_q)).scalar_one()
offset = (page - 1) * page_size
rows = (
await db.execute(
base.order_by(ProcessingRun.started_at.desc())
.offset(offset)
.limit(page_size)
)
).all()
items = [
ProcessingRunDetailResponse(
id=row.ProcessingRun.id, # type: ignore[arg-type]
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
status=row.ProcessingRun.status, # type: ignore[arg-type]
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
account_name=row.name,
account_email=row.email_address,
)
for row in rows
]
return PaginatedProcessingRunsResponse(
items=items, **_paginate(total, page, page_size) # type: ignore[arg-type]
)
@router.get(
"/processing-runs/{run_id}",
response_model=ProcessingRunDetailResponse,
summary="Get a single processing run",
)
async def get_processing_run(
run_id: int,
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Return details for a single processing run owned by the current user."""
row = (
await db.execute(
select(ProcessingRun, MailAccount.name, MailAccount.email_address)
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
.where(
ProcessingRun.id == run_id,
MailAccount.user_id == current_user.id, # type: ignore[arg-type]
)
)
).one_or_none()
if not row:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Processing run not found"
)
return ProcessingRunDetailResponse(
id=row.ProcessingRun.id, # type: ignore[arg-type]
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
status=row.ProcessingRun.status, # type: ignore[arg-type]
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
account_name=row.name,
account_email=row.email_address,
)
@router.get(
"/processing-runs/{run_id}/logs",
response_model=PaginatedProcessingLogsResponse,
summary="Get per-email logs for a processing run",
)
async def get_run_logs(
run_id: int,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""
Return the detailed per-email log entries recorded during a specific
processing run. Ownership is verified by joining with MailAccount.
"""
# Verify the run belongs to this user
run_row = (
await db.execute(
select(ProcessingRun)
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
.where(
ProcessingRun.id == run_id,
MailAccount.user_id == current_user.id, # type: ignore[arg-type]
)
)
).scalar_one_or_none()
if not run_row:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Processing run not found"
)
count_q = select(func.count(ProcessingLog.id)).where(
ProcessingLog.processing_run_id == run_id
)
total = (await db.execute(count_q)).scalar_one()
offset = (page - 1) * page_size
logs = (
(
await db.execute(
select(ProcessingLog)
.where(ProcessingLog.processing_run_id == run_id)
.order_by(ProcessingLog.timestamp.asc())
.offset(offset)
.limit(page_size)
)
)
.scalars()
.all()
)
items = [
ProcessingLogDetailResponse(
id=log.id, # type: ignore[arg-type]
timestamp=log.timestamp, # type: ignore[arg-type]
level=log.level, # type: ignore[arg-type]
message=log.message, # type: ignore[arg-type]
email_subject=log.email_subject, # type: ignore[arg-type]
email_from=log.email_from, # type: ignore[arg-type]
success=log.success, # type: ignore[arg-type]
mail_account_id=log.mail_account_id, # type: ignore[arg-type]
processing_run_id=log.processing_run_id, # type: ignore[arg-type]
email_size_bytes=log.email_size_bytes, # type: ignore[arg-type]
error_details=log.error_details, # type: ignore[arg-type]
)
for log in logs
]
return PaginatedProcessingLogsResponse(
items=items, **_paginate(total, page, page_size) # type: ignore[arg-type]
)
+155 -3
View File
@@ -1,9 +1,10 @@
"""Mail account management endpoints"""
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
import math
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, desc
from sqlalchemy import select, desc, func
from app.core.database import get_db
from app.core.deps import get_current_active_user
@@ -11,6 +12,8 @@ from app.core.security import encrypt_credential
from app.models.database_models import (
User,
MailAccount,
ProcessingLog,
ProcessingRun,
AccountStatus,
SubscriptionPlan,
)
@@ -22,6 +25,10 @@ from app.models.schemas import (
MailAccountTestResponse,
MailAccountAutoDetectRequest,
MailAccountAutoDetectResponse,
PaginatedProcessingRunsResponse,
PaginatedProcessingLogsResponse,
ProcessingRunDetailResponse,
ProcessingLogDetailResponse,
)
from app.services.mail_processor import MailProcessor, MailServerAutoDetect
from app.core.config import settings
@@ -274,3 +281,148 @@ async def auto_detect_mail_settings(
return MailAccountAutoDetectResponse(
success=len(suggestions) > 0, suggestions=suggestions
)
# ---------------------------------------------------------------------------
# Per-account processing runs & logs
# ---------------------------------------------------------------------------
@router.get(
"/{account_id}/processing-runs",
response_model=PaginatedProcessingRunsResponse,
summary="List processing runs for a specific mail account",
)
async def list_account_runs(
account_id: int,
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Return paginated processing runs for a mail account owned by the user."""
result = await db.execute(
select(MailAccount).where(
MailAccount.id == account_id,
MailAccount.user_id == current_user.id,
)
)
account = result.scalar_one_or_none()
if not account:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Mail account not found"
)
base = select(ProcessingRun).where(ProcessingRun.mail_account_id == account_id)
total = (
await db.execute(select(func.count()).select_from(base.subquery()))
).scalar_one()
offset = (page - 1) * page_size
runs = (
(
await db.execute(
base.order_by(desc(ProcessingRun.started_at))
.offset(offset)
.limit(page_size)
)
)
.scalars()
.all()
)
pages = max(1, math.ceil(total / page_size)) if total else 1
items = [
ProcessingRunDetailResponse(
id=r.id, # type: ignore[arg-type]
mail_account_id=r.mail_account_id, # type: ignore[arg-type]
started_at=r.started_at, # type: ignore[arg-type]
completed_at=r.completed_at, # type: ignore[arg-type]
duration_seconds=r.duration_seconds, # type: ignore[arg-type]
emails_fetched=r.emails_fetched, # type: ignore[arg-type]
emails_forwarded=r.emails_forwarded, # type: ignore[arg-type]
emails_failed=r.emails_failed, # type: ignore[arg-type]
status=r.status, # type: ignore[arg-type]
error_message=r.error_message, # type: ignore[arg-type]
account_name=account.name, # type: ignore[arg-type]
account_email=account.email_address, # type: ignore[arg-type]
)
for r in runs
]
return PaginatedProcessingRunsResponse(
items=items, total=total, page=page, page_size=page_size, pages=pages
)
@router.get(
"/{account_id}/logs",
response_model=PaginatedProcessingLogsResponse,
summary="List processing logs for a specific mail account",
)
async def list_account_logs(
account_id: int,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
level: Optional[str] = Query(
None, description="Filter by log level (INFO, WARNING, ERROR)"
),
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Return paginated per-email log entries for a mail account owned by the user."""
result = await db.execute(
select(MailAccount).where(
MailAccount.id == account_id,
MailAccount.user_id == current_user.id,
)
)
account = result.scalar_one_or_none()
if not account:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Mail account not found"
)
base = select(ProcessingLog).where(
ProcessingLog.mail_account_id == account_id,
ProcessingLog.user_id == current_user.id, # type: ignore[arg-type]
)
if level:
base = base.where(ProcessingLog.level == level.upper())
total = (
await db.execute(select(func.count()).select_from(base.subquery()))
).scalar_one()
offset = (page - 1) * page_size
logs = (
(
await db.execute(
base.order_by(ProcessingLog.timestamp.desc())
.offset(offset)
.limit(page_size)
)
)
.scalars()
.all()
)
pages = max(1, math.ceil(total / page_size)) if total else 1
items = [
ProcessingLogDetailResponse(
id=log.id, # type: ignore[arg-type]
timestamp=log.timestamp, # type: ignore[arg-type]
level=log.level, # type: ignore[arg-type]
message=log.message, # type: ignore[arg-type]
email_subject=log.email_subject, # type: ignore[arg-type]
email_from=log.email_from, # type: ignore[arg-type]
success=log.success, # type: ignore[arg-type]
mail_account_id=log.mail_account_id, # type: ignore[arg-type]
processing_run_id=log.processing_run_id, # type: ignore[arg-type]
email_size_bytes=log.email_size_bytes, # type: ignore[arg-type]
error_details=log.error_details, # type: ignore[arg-type]
)
for log in logs
]
return PaginatedProcessingLogsResponse(
items=items, total=total, page=page, page_size=page_size, pages=pages
)
+56 -2
View File
@@ -21,12 +21,30 @@ from app.models.schemas import (
GmailCredentialResponse,
GmailAuthorizeResponse,
GmailCallbackRequest,
GmailImportLabelsUpdate,
)
from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES
from app.utils.gmail_labels import (
MAX_IMPORT_LABELS,
build_gmail_credential_scopes,
extract_granted_scopes,
normalize_import_label_templates,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _validated_import_label_templates(label_templates: List[str]) -> List[str]:
normalized = normalize_import_label_templates(label_templates)
if len(normalized) > MAX_IMPORT_LABELS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"You can configure up to {MAX_IMPORT_LABELS} Gmail import labels.",
)
return normalized
# Gmail API scopes requested during the "Connect Gmail" OAuth flow.
# GMAIL_SCOPES (gmail.insert, gmail.labels, gmail.readonly) are imported from
# gmail_service so the scope list stays in sync with what GmailService uses.
@@ -223,6 +241,10 @@ async def save_gmail_credential(
existing.gmail_email = credential_in.gmail_email # type: ignore[assignment]
existing.encrypted_access_token = encrypted_access # type: ignore[assignment]
existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
existing.scopes = build_gmail_credential_scopes(
existing.granted_scopes,
existing.import_label_templates,
) # type: ignore[assignment]
existing.is_valid = True # type: ignore[assignment]
existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
await db.commit()
@@ -235,6 +257,7 @@ async def save_gmail_credential(
gmail_email=credential_in.gmail_email,
encrypted_access_token=encrypted_access,
encrypted_refresh_token=encrypted_refresh,
scopes=build_gmail_credential_scopes(),
is_valid=True,
last_verified_at=datetime.now(timezone.utc),
)
@@ -285,6 +308,33 @@ async def delete_gmail_credential(
await db.commit()
@router.put("/gmail-credential/labels", response_model=GmailCredentialResponse)
async def update_gmail_import_labels(
labels_in: GmailImportLabelsUpdate,
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Update the Gmail labels applied to imported messages."""
result = await db.execute(
select(GmailCredential).where(GmailCredential.user_id == current_user.id)
)
credential = result.scalar_one_or_none()
if not credential:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No Gmail credentials found. Connect Gmail first.",
)
credential.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
extract_granted_scopes(credential.scopes),
_validated_import_label_templates(labels_in.import_label_templates),
)
await db.commit()
await db.refresh(credential)
return credential
@router.get("/gmail/authorize-url", response_model=GmailAuthorizeResponse)
async def get_gmail_authorize_url(
redirect_uri: str,
@@ -365,6 +415,7 @@ async def send_gmail_debug_email(
try:
inject_result = await gmail_service.inject_debug_email(
recipient_email=credential.gmail_email, # type: ignore[arg-type]
import_label_templates=credential.import_label_templates,
)
except GmailInjectionError as exc:
raise HTTPException(
@@ -498,7 +549,10 @@ async def gmail_oauth_callback(
if encrypted_refresh:
existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
existing.token_expiry = token_expiry # type: ignore[assignment]
existing.scopes = token_data.get("scope", "").split() # type: ignore[assignment]
existing.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
token_data.get("scope", "").split(),
existing.import_label_templates,
)
existing.is_valid = True # type: ignore[assignment]
existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
await db.commit()
@@ -511,7 +565,7 @@ async def gmail_oauth_callback(
encrypted_access_token=encrypted_access,
encrypted_refresh_token=encrypted_refresh,
token_expiry=token_expiry,
scopes=token_data.get("scope", "").split(),
scopes=build_gmail_credential_scopes(token_data.get("scope", "").split()),
is_valid=True,
last_verified_at=datetime.now(timezone.utc),
)
+5 -3
View File
@@ -27,8 +27,10 @@ class Settings(BaseSettings):
)
# Application
APP_NAME: str = "InboxRescue"
APP_NAME: str = "InboxConverge"
APP_VERSION: str = "2.0.0"
APP_URL: str = "https://inboxconverge.com"
CONTACT_EMAIL: str = "christian@inboxconverge.com"
DEBUG: bool = False
API_V1_PREFIX: str = "/api/v1"
@@ -38,7 +40,7 @@ class Settings(BaseSettings):
# Database
DATABASE_URL: str = (
"postgresql+asyncpg://user:password@localhost:5432/pop3_forwarder"
"postgresql+asyncpg://user:password@localhost:5432/inbox_converge"
)
DATABASE_POOL_SIZE: int = 20
DATABASE_MAX_OVERFLOW: int = 10
@@ -94,7 +96,7 @@ class Settings(BaseSettings):
LOG_LEVEL: str = "INFO"
# Admin
ADMIN_EMAIL: Optional[str] = "christianlouis@gmail.com"
ADMIN_EMAIL: Optional[str] = "christian@inboxconverge.com"
ADMIN_PASSWORD: Optional[str] = None
# User defaults & access control
+106
View File
@@ -0,0 +1,106 @@
"""
GDPR-compliant data masking utilities.
These helpers are used in admin-facing API responses to pseudonymise
personal data (email addresses, names) so that operators can audit
system behaviour without seeing full end-user PII.
"""
from __future__ import annotations
import re
def mask_email(email: str) -> str:
"""
Partially mask an email address for GDPR-compliant display.
Examples
--------
>>> mask_email("john.doe@example.com")
'jo***@e***.com'
>>> mask_email("ab@x.io")
'ab***@x***.io'
>>> mask_email("a@b.de")
'a***@b***.de'
"""
if not email or "@" not in email:
return "***"
local, _, domain = email.partition("@")
# Local part: keep first 2 chars (or all if shorter), then "***"
visible_local = local[:2] if len(local) >= 2 else local
masked_local = f"{visible_local}***"
# Domain part: keep first char of SLD and the TLD unchanged
domain_parts = domain.rsplit(".", 1)
if len(domain_parts) == 2:
sld, tld = domain_parts
visible_sld = sld[:1] if sld else ""
masked_domain = f"{visible_sld}***.{tld}"
else:
masked_domain = "***"
return f"{masked_local}@{masked_domain}"
def mask_name(name: str) -> str:
"""
Partially mask a display name.
Examples
--------
>>> mask_name("John Doe")
'Jo*** D***'
>>> mask_name("Alice")
'Al***'
"""
if not name:
return "***"
words = name.split()
masked_words = []
for word in words:
visible = word[:2] if len(word) >= 2 else word
masked_words.append(f"{visible}***")
return " ".join(masked_words)
# RFC 5322 address pattern extracts the bare email from strings like
# "John Doe <john@example.com>" or just "john@example.com".
_ADDR_RE = re.compile(r"<([^>]+)>|(\S+@\S+\.\S+)")
def mask_from_header(from_header: str) -> str:
"""
Mask a raw RFC 5322 From header value for GDPR-compliant display.
Examples
--------
>>> mask_from_header("John Doe <john.doe@example.com>")
'Jo*** D*** <jo***@e***.com>'
>>> mask_from_header("john.doe@example.com")
'jo***@e***.com'
"""
if not from_header:
return "***"
# Try to parse "Display Name <email>" form
angle_match = re.search(r"^(.*?)<([^>]+)>", from_header.strip())
if angle_match:
display_name = angle_match.group(1).strip().strip('"')
email_part = angle_match.group(2).strip()
masked_email = mask_email(email_part)
if display_name:
masked_display = mask_name(display_name)
return f"{masked_display} <{masked_email}>"
return masked_email
# Plain email address
bare_match = _ADDR_RE.search(from_header)
if bare_match:
email_part = bare_match.group(1) or bare_match.group(2)
return mask_email(email_part)
# Fallback: mask the whole string
return mask_name(from_header)
+1 -1
View File
@@ -1,5 +1,5 @@
"""
Prometheus metrics definitions for InboxRescue.
Prometheus metrics definitions for InboxConverge.
All application metrics are defined here as module-level singletons so that
every subsystem (HTTP layer, Celery workers, Gmail service, auth) imports
+1 -1
View File
@@ -148,7 +148,7 @@ def create_application() -> FastAPI:
async def root():
"""Root endpoint"""
return {
"message": "InboxRescue API",
"message": "InboxConverge API",
"version": settings.APP_VERSION,
"docs": "/api/docs",
}
+18
View File
@@ -17,6 +17,12 @@ from sqlalchemy import (
Index,
)
from sqlalchemy.orm import relationship
from app.utils.gmail_labels import (
DEFAULT_IMPORT_LABEL_TEMPLATES,
extract_granted_scopes,
extract_import_label_templates,
)
import enum
from app.core.database import Base
@@ -562,6 +568,18 @@ class GmailCredential(Base):
# Relationships
user = relationship("User", backref="gmail_credential")
@property
def granted_scopes(self) -> list[str]:
return extract_granted_scopes(self.scopes)
@property
def import_label_templates(self) -> list[str]:
return extract_import_label_templates(self.scopes)
@property
def default_import_label_templates(self) -> list[str]:
return DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
class AppSetting(Base):
"""
+76
View File
@@ -204,6 +204,15 @@ class ProcessingRunResponse(BaseModel):
model_config = ConfigDict(from_attributes=True)
class ProcessingRunDetailResponse(ProcessingRunResponse):
"""ProcessingRunResponse with optional account metadata."""
account_name: Optional[str] = None
account_email: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
# Processing Log Schemas
class ProcessingLogResponse(BaseModel):
id: int
@@ -217,6 +226,67 @@ class ProcessingLogResponse(BaseModel):
model_config = ConfigDict(from_attributes=True)
class ProcessingLogDetailResponse(ProcessingLogResponse):
"""ProcessingLogResponse with additional fields."""
mail_account_id: int
processing_run_id: Optional[int] = None
email_size_bytes: Optional[int] = None
error_details: Optional[Dict[str, Any]] = None
model_config = ConfigDict(from_attributes=True)
class PaginatedProcessingRunsResponse(BaseModel):
items: List[ProcessingRunDetailResponse]
total: int
page: int
page_size: int
pages: int
class PaginatedProcessingLogsResponse(BaseModel):
items: List[ProcessingLogDetailResponse]
total: int
page: int
page_size: int
pages: int
class AdminProcessingRunResponse(ProcessingRunDetailResponse):
"""ProcessingRunDetailResponse with user info for admin views."""
user_id: Optional[int] = None
user_email: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
class PaginatedAdminRunsResponse(BaseModel):
items: List[AdminProcessingRunResponse]
total: int
page: int
page_size: int
pages: int
class AdminProcessingLogResponse(ProcessingLogDetailResponse):
"""ProcessingLogDetailResponse with user info for admin views."""
user_id: int
user_email: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
class PaginatedAdminLogsResponse(BaseModel):
items: List[AdminProcessingLogResponse]
total: int
page: int
page_size: int
pages: int
# Notification Config Schemas
class NotificationConfigBase(BaseModel):
name: str = Field(
@@ -376,6 +446,8 @@ class GmailCredentialResponse(BaseModel):
user_id: int
gmail_email: str
is_valid: bool
import_label_templates: List[str] = Field(default_factory=list)
default_import_label_templates: List[str] = Field(default_factory=list)
last_verified_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
@@ -430,6 +502,10 @@ class GmailCallbackRequest(BaseModel):
redirect_uri: str
class GmailImportLabelsUpdate(BaseModel):
import_label_templates: List[str] = Field(default_factory=list)
# Admin Schemas
+22 -4
View File
@@ -25,6 +25,7 @@ from app.core.metrics import (
GMAIL_API_DURATION_SECONDS,
GMAIL_TOKEN_REFRESHES_TOTAL,
)
from app.utils.gmail_labels import render_import_labels
logger = logging.getLogger(__name__)
@@ -294,6 +295,7 @@ class GmailService:
async def inject_debug_email(
self,
recipient_email: str,
import_label_templates: Optional[list[str]] = None,
) -> Dict[str, Any]:
"""
Inject a debug/test email into the user's Gmail inbox.
@@ -344,11 +346,10 @@ class GmailService:
raw_bytes = msg.as_bytes()
# Resolve label IDs (create labels if they don't exist yet)
label_ids = await self.build_import_label_ids(import_label_templates)
test_label_id = await self.get_or_create_label("test")
imported_label_id = await self.get_or_create_label("imported")
label_ids = ["INBOX", test_label_id, imported_label_id]
if test_label_id not in label_ids:
label_ids.append(test_label_id)
return await self.inject_email(
raw_email=raw_bytes,
@@ -356,6 +357,23 @@ class GmailService:
source_account_name="debug",
)
async def build_import_label_ids(
self,
import_label_templates: Optional[list[str]] = None,
source_email: Optional[str] = None,
) -> list[str]:
"""Resolve configured import labels into Gmail label IDs."""
label_ids = ["INBOX"]
for label_name in render_import_labels(import_label_templates, source_email):
if label_name.upper() == "INBOX":
continue
label_id = await self.get_or_create_label(label_name)
if label_id not in label_ids:
label_ids.append(label_id)
return label_ids
def get_refreshed_token(self) -> Optional[Dict[str, Any]]:
"""
Return the current access token and expiry if the token was refreshed
+97
View File
@@ -0,0 +1,97 @@
"""Helpers for Gmail import label configuration and rendering."""
from typing import Any, Iterable, Optional
SOURCE_EMAIL_LABEL_TEMPLATE = "{{source_email}}"
DEFAULT_IMPORT_LABEL_TEMPLATES = [SOURCE_EMAIL_LABEL_TEMPLATE, "imported"]
MAX_IMPORT_LABELS = 10
def _normalize_string_list(values: Optional[Iterable[str]]) -> list[str]:
normalized: list[str] = []
seen: set[str] = set()
for value in values or []:
cleaned = value.strip()
if not cleaned:
continue
lowered = cleaned.casefold()
if lowered in seen:
continue
seen.add(lowered)
normalized.append(cleaned)
return normalized
def normalize_import_label_templates(
label_templates: Optional[Iterable[str]],
) -> list[str]:
"""Return a cleaned, de-duplicated label template list."""
normalized = _normalize_string_list(label_templates)
return normalized or DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
def extract_granted_scopes(scopes_data: Any) -> list[str]:
"""Read granted scopes from legacy list or new JSON object storage."""
if isinstance(scopes_data, list):
return _normalize_string_list(
value for value in scopes_data if isinstance(value, str)
)
if isinstance(scopes_data, dict):
granted_scopes = scopes_data.get("granted_scopes", [])
if isinstance(granted_scopes, list):
return _normalize_string_list(
value for value in granted_scopes if isinstance(value, str)
)
return []
def extract_import_label_templates(scopes_data: Any) -> list[str]:
"""Read import label templates from stored Gmail credential metadata."""
if isinstance(scopes_data, dict):
stored_templates = scopes_data.get("import_label_templates", [])
if isinstance(stored_templates, list):
return normalize_import_label_templates(
value for value in stored_templates if isinstance(value, str)
)
return DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
def build_gmail_credential_scopes(
granted_scopes: Optional[Iterable[str]],
import_label_templates: Optional[Iterable[str]] = None,
) -> dict[str, list[str]]:
"""Persist Gmail metadata in the existing JSON column."""
return {
"granted_scopes": _normalize_string_list(granted_scopes),
"import_label_templates": normalize_import_label_templates(
import_label_templates
),
}
def render_import_labels(
import_label_templates: Optional[Iterable[str]],
source_email: Optional[str],
) -> list[str]:
"""Render label templates into actual Gmail label names."""
rendered_labels: list[str] = []
seen: set[str] = set()
resolved_source_email = source_email.strip() if source_email else ""
for template in normalize_import_label_templates(import_label_templates):
rendered = template.replace(SOURCE_EMAIL_LABEL_TEMPLATE, resolved_source_email)
rendered = rendered.strip()
if not rendered:
continue
lowered = rendered.casefold()
if lowered in seen:
continue
seen.add(lowered)
rendered_labels.append(rendered)
return rendered_labels
+1 -1
View File
@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
# Create Celery app
celery_app = Celery(
"pop3_forwarder",
"inboxconverge",
broker=settings.CELERY_BROKER_URL,
backend=settings.CELERY_RESULT_BACKEND,
include=["app.workers.tasks"],
+55 -1
View File
@@ -3,6 +3,7 @@ Celery tasks for background email processing.
"""
import asyncio
import email as email_lib
import time
from datetime import datetime, timedelta, timezone
from celery import Task
@@ -199,15 +200,46 @@ async def process_mail_account(account_id: int):
f"for account {account.id}; truncating to shorter list"
)
# Field length limits matching the DB column definitions
_MAX_SUBJECT_LEN = 500
_MAX_FROM_LEN = 255
for email_data, uid in zip(emails, new_uids):
# ── Parse email metadata for logging ───────────────────────
email_subject: str | None = None
email_from: str | None = None
try:
msg = email_lib.message_from_bytes(email_data)
raw_subject = msg.get("Subject", "") or ""
email_subject = (
raw_subject[:_MAX_SUBJECT_LEN] if raw_subject else None
)
raw_from = msg.get("From", "") or ""
email_from = raw_from[:_MAX_FROM_LEN] if raw_from else None
except (ValueError, TypeError, UnicodeDecodeError) as exc:
logger.debug(
"Could not parse email headers for account %s: %s",
account.id,
exc,
)
email_size_bytes = len(email_data)
forwarded_ok = False
error_msg: str | None = None
try:
if use_gmail_api and gmail_service:
# Inject via Gmail API (preferred)
label_ids = await gmail_service.build_import_label_ids(
import_label_templates=gmail_cred.import_label_templates,
source_email=account.email_address, # type: ignore[arg-type]
)
await gmail_service.inject_email(
raw_email=email_data,
label_ids=["INBOX"],
label_ids=label_ids,
source_account_name=account.name, # type: ignore[arg-type]
)
forwarded_ok = True
emails_forwarded += 1
successfully_forwarded_uids.append(uid)
else:
@@ -216,6 +248,7 @@ async def process_mail_account(account_id: int):
email_data, account.name, account.forward_to, smtp_config # type: ignore[arg-type]
)
if success:
forwarded_ok = True
emails_forwarded += 1
successfully_forwarded_uids.append(uid)
else:
@@ -253,8 +286,29 @@ async def process_mail_account(account_id: int):
f"Failed to send revocation notification: {notify_exc}"
)
logger.error(f"Error delivering email: {e}")
error_msg = str(e)
emails_failed += 1
# ── Write per-email ProcessingLog entry ─────────────────────
db.add(
ProcessingLog(
user_id=account.user_id,
mail_account_id=account.id,
processing_run_id=run.id,
level="INFO" if forwarded_ok else "ERROR",
message=(
f"Forwarded: {email_subject or '(no subject)'}"
if forwarded_ok
else f"Failed: {error_msg or 'delivery error'}"
),
email_subject=email_subject,
email_from=email_from,
email_size_bytes=email_size_bytes,
success=forwarded_ok,
error_details={"error": error_msg} if error_msg else None,
)
)
# Persist new message UIDs so they are not processed again
for uid in successfully_forwarded_uids:
if uid not in already_seen_uids:
+1 -1
View File
@@ -16,7 +16,7 @@ from app.core.security import get_password_hash, create_access_token
# Test database URL (use different database for tests)
TEST_DATABASE_URL = settings.DATABASE_URL.replace(
"/pop3_forwarder", "/pop3_forwarder_test"
"/inbox_converge", "/inbox_converge_test"
)
+1 -1
View File
@@ -89,7 +89,7 @@ class TestApplicationFactory:
async def test_app_title(self, app):
"""Test that app has correct title"""
assert app.title == "InboxRescue"
assert app.title == "InboxConverge"
async def test_app_version(self, app):
"""Test that app has a version"""
+50 -1
View File
@@ -3,8 +3,16 @@ Unit tests for Gmail service module.
"""
import pytest
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES
from app.utils.gmail_labels import (
DEFAULT_IMPORT_LABEL_TEMPLATES,
SOURCE_EMAIL_LABEL_TEMPLATE,
build_gmail_credential_scopes,
extract_granted_scopes,
extract_import_label_templates,
render_import_labels,
)
class TestGmailService:
@@ -153,3 +161,44 @@ class TestGmailService:
email = await service.get_email_address()
assert email is None
def test_gmail_label_metadata_helpers(self):
"""Test Gmail metadata extraction remains backward compatible."""
scopes = build_gmail_credential_scopes(
["scope-a", "scope-b"],
[SOURCE_EMAIL_LABEL_TEMPLATE, "Imported", " imported "],
)
assert extract_granted_scopes(scopes) == ["scope-a", "scope-b"]
assert extract_import_label_templates(scopes) == [
SOURCE_EMAIL_LABEL_TEMPLATE,
"Imported",
]
assert (
extract_import_label_templates(["legacy-scope"])
== DEFAULT_IMPORT_LABEL_TEMPLATES
)
def test_render_import_labels_uses_source_email_template(self):
"""Test that source email templates render to the source mailbox address."""
rendered = render_import_labels(
[SOURCE_EMAIL_LABEL_TEMPLATE, "Imported", ""],
"source@example.com",
)
assert rendered == ["source@example.com", "Imported"]
@pytest.mark.asyncio
async def test_build_import_label_ids_creates_configured_labels(self):
"""Test that configured import labels are created and added alongside INBOX."""
service = GmailService(access_token="test-access-token")
service.get_or_create_label = AsyncMock(
side_effect=["Label-source", "Label-imported"]
) # type: ignore[method-assign]
label_ids = await service.build_import_label_ids(
import_label_templates=[SOURCE_EMAIL_LABEL_TEMPLATE, "imported"],
source_email="source@example.com",
)
assert label_ids == ["INBOX", "Label-source", "Label-imported"]