Merge branch 'main' into copilot/add-apprise-alerting-capabilities
This commit is contained in:
Christian Krakau-Louis
@@ -13,6 +13,7 @@ from app.api.v1.endpoints import (
|
||||
admin,
|
||||
providers,
|
||||
app_settings,
|
||||
logs,
|
||||
)
|
||||
|
||||
api_router = APIRouter()
|
||||
@@ -34,3 +35,6 @@ api_router.include_router(
|
||||
)
|
||||
api_router.include_router(admin.router, prefix="/admin", tags=["Admin"])
|
||||
api_router.include_router(app_settings.router, prefix="/settings", tags=["Settings"])
|
||||
api_router.include_router(
|
||||
logs.router, prefix="/processing-runs", tags=["Processing Logs"]
|
||||
)
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
"""Admin endpoints"""
|
||||
|
||||
from typing import List
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
import math
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.core.deps import get_current_superuser
|
||||
from app.core.gdpr import mask_email, mask_from_header
|
||||
from app.models.database_models import (
|
||||
User,
|
||||
MailAccount,
|
||||
ProcessingLog,
|
||||
ProcessingRun,
|
||||
SubscriptionPlan,
|
||||
SubscriptionTier,
|
||||
@@ -27,6 +30,10 @@ from app.models.schemas import (
|
||||
AdminNotificationConfigResponse,
|
||||
NotificationTestRequest,
|
||||
NotificationTestResponse,
|
||||
AdminProcessingRunResponse,
|
||||
AdminProcessingLogResponse,
|
||||
PaginatedAdminRunsResponse,
|
||||
PaginatedAdminLogsResponse,
|
||||
)
|
||||
from app.services.notification_service import test_notification
|
||||
|
||||
@@ -400,3 +407,163 @@ async def test_admin_notification_config(
|
||||
"""Test an admin notification channel by sending a test message (admin only)"""
|
||||
success, message = await test_notification(request.apprise_url)
|
||||
return NotificationTestResponse(success=success, message=message)
|
||||
# ── Admin Logs ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _admin_paginate(total: int, page: int, page_size: int) -> dict:
|
||||
pages = max(1, math.ceil(total / page_size)) if total else 1
|
||||
return {"total": total, "page": page, "page_size": page_size, "pages": pages}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/processing-runs",
|
||||
response_model=PaginatedAdminRunsResponse,
|
||||
summary="List all processing runs across all users (admin only)",
|
||||
)
|
||||
async def admin_list_processing_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
user_id: Optional[int] = Query(None, description="Filter by user ID"),
|
||||
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
|
||||
status_filter: Optional[str] = Query(
|
||||
None,
|
||||
alias="status",
|
||||
description="Filter by run status",
|
||||
),
|
||||
current_user: User = Depends(get_current_superuser),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Return a paginated list of all processing runs in the system with
|
||||
GDPR-masked user / account email addresses.
|
||||
"""
|
||||
base = (
|
||||
select(
|
||||
ProcessingRun,
|
||||
MailAccount.name.label("account_name"),
|
||||
MailAccount.email_address.label("account_email"),
|
||||
MailAccount.user_id.label("uid"),
|
||||
User.email.label("user_email"),
|
||||
)
|
||||
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
|
||||
.join(User, MailAccount.user_id == User.id)
|
||||
)
|
||||
|
||||
if user_id is not None:
|
||||
base = base.where(MailAccount.user_id == user_id)
|
||||
if account_id is not None:
|
||||
base = base.where(ProcessingRun.mail_account_id == account_id)
|
||||
if status_filter:
|
||||
base = base.where(ProcessingRun.status == status_filter)
|
||||
|
||||
total = (
|
||||
await db.execute(select(func.count()).select_from(base.subquery()))
|
||||
).scalar_one()
|
||||
offset = (page - 1) * page_size
|
||||
rows = (
|
||||
await db.execute(
|
||||
base.order_by(ProcessingRun.started_at.desc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
).all()
|
||||
|
||||
items = [
|
||||
AdminProcessingRunResponse(
|
||||
id=row.ProcessingRun.id, # type: ignore[arg-type]
|
||||
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
|
||||
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
|
||||
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
|
||||
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
|
||||
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
|
||||
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
|
||||
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
|
||||
status=row.ProcessingRun.status, # type: ignore[arg-type]
|
||||
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
|
||||
account_name=row.account_name,
|
||||
account_email=mask_email(row.account_email) if row.account_email else None,
|
||||
user_id=row.uid,
|
||||
user_email=mask_email(row.user_email) if row.user_email else None,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
return PaginatedAdminRunsResponse(
|
||||
items=items, **_admin_paginate(total, page, page_size) # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/processing-logs",
|
||||
response_model=PaginatedAdminLogsResponse,
|
||||
summary="List all per-email processing logs across all users (admin only)",
|
||||
)
|
||||
async def admin_list_processing_logs(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=200),
|
||||
user_id: Optional[int] = Query(None, description="Filter by user ID"),
|
||||
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
|
||||
run_id: Optional[int] = Query(None, description="Filter by processing run ID"),
|
||||
level: Optional[str] = Query(
|
||||
None, description="Filter by level (INFO, WARNING, ERROR)"
|
||||
),
|
||||
current_user: User = Depends(get_current_superuser),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Return paginated per-email log entries with GDPR-masked sender addresses.
|
||||
Subject lines are shown as-is (the user owns their own mail content);
|
||||
sender addresses are pseudonymised for operator privacy.
|
||||
"""
|
||||
base = select(
|
||||
ProcessingLog,
|
||||
User.email.label("user_email"),
|
||||
).join(User, ProcessingLog.user_id == User.id)
|
||||
|
||||
if user_id is not None:
|
||||
base = base.where(ProcessingLog.user_id == user_id)
|
||||
if account_id is not None:
|
||||
base = base.where(ProcessingLog.mail_account_id == account_id)
|
||||
if run_id is not None:
|
||||
base = base.where(ProcessingLog.processing_run_id == run_id)
|
||||
if level:
|
||||
base = base.where(ProcessingLog.level == level.upper())
|
||||
|
||||
total = (
|
||||
await db.execute(select(func.count()).select_from(base.subquery()))
|
||||
).scalar_one()
|
||||
offset = (page - 1) * page_size
|
||||
rows = (
|
||||
await db.execute(
|
||||
base.order_by(ProcessingLog.timestamp.desc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
).all()
|
||||
|
||||
items = [
|
||||
AdminProcessingLogResponse(
|
||||
id=row.ProcessingLog.id, # type: ignore[arg-type]
|
||||
timestamp=row.ProcessingLog.timestamp, # type: ignore[arg-type]
|
||||
level=row.ProcessingLog.level, # type: ignore[arg-type]
|
||||
message=row.ProcessingLog.message, # type: ignore[arg-type]
|
||||
email_subject=row.ProcessingLog.email_subject, # type: ignore[arg-type]
|
||||
email_from=(
|
||||
mask_from_header(row.ProcessingLog.email_from)
|
||||
if row.ProcessingLog.email_from
|
||||
else None
|
||||
),
|
||||
success=row.ProcessingLog.success, # type: ignore[arg-type]
|
||||
mail_account_id=row.ProcessingLog.mail_account_id, # type: ignore[arg-type]
|
||||
processing_run_id=row.ProcessingLog.processing_run_id, # type: ignore[arg-type]
|
||||
email_size_bytes=row.ProcessingLog.email_size_bytes, # type: ignore[arg-type]
|
||||
error_details=row.ProcessingLog.error_details, # type: ignore[arg-type]
|
||||
user_id=row.ProcessingLog.user_id, # type: ignore[arg-type]
|
||||
user_email=mask_email(row.user_email) if row.user_email else None,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
return PaginatedAdminLogsResponse(
|
||||
items=items, **_admin_paginate(total, page, page_size) # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.models.database_models import User, SubscriptionTier, GmailCredential
|
||||
from app.models.schemas import Token, UserCreate, UserResponse, GoogleAuthRequest
|
||||
from app.services.auth_service import oauth_service
|
||||
from app.services.gmail_service import GmailService, GMAIL_SCOPES
|
||||
from app.utils.gmail_labels import build_gmail_credential_scopes
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -286,7 +287,10 @@ async def google_oauth(
|
||||
if encrypted_refresh:
|
||||
existing_cred.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
|
||||
existing_cred.token_expiry = token_expiry # type: ignore[assignment]
|
||||
existing_cred.scopes = scope_list # type: ignore[assignment]
|
||||
existing_cred.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
|
||||
scope_list,
|
||||
existing_cred.import_label_templates,
|
||||
)
|
||||
existing_cred.is_valid = True # type: ignore[assignment]
|
||||
existing_cred.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
|
||||
else:
|
||||
@@ -296,7 +300,7 @@ async def google_oauth(
|
||||
encrypted_access_token=encrypted_access,
|
||||
encrypted_refresh_token=encrypted_refresh,
|
||||
token_expiry=token_expiry,
|
||||
scopes=scope_list,
|
||||
scopes=build_gmail_credential_scopes(scope_list),
|
||||
is_valid=True,
|
||||
last_verified_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
"""
|
||||
Processing logs and run history endpoints for users.
|
||||
|
||||
Users can view the full history of processing runs and per-email logs
|
||||
for their own mailboxes. Admin equivalents live in admin.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.core.deps import get_current_active_user
|
||||
from app.models.database_models import (
|
||||
MailAccount,
|
||||
ProcessingLog,
|
||||
ProcessingRun,
|
||||
User,
|
||||
)
|
||||
from app.models.schemas import (
|
||||
PaginatedProcessingLogsResponse,
|
||||
PaginatedProcessingRunsResponse,
|
||||
ProcessingLogDetailResponse,
|
||||
ProcessingRunDetailResponse,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _paginate(total: int, page: int, page_size: int) -> dict:
|
||||
pages = max(1, math.ceil(total / page_size)) if total else 1
|
||||
return {"total": total, "page": page, "page_size": page_size, "pages": pages}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Processing Runs (all accounts belonging to the current user)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get(
|
||||
"/processing-runs",
|
||||
response_model=PaginatedProcessingRunsResponse,
|
||||
summary="List processing runs for the current user",
|
||||
)
|
||||
async def list_processing_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
account_id: Optional[int] = Query(None, description="Filter by mail account ID"),
|
||||
status_filter: Optional[str] = Query(
|
||||
None,
|
||||
alias="status",
|
||||
description="Filter by run status (completed, failed, partial_failure, running)",
|
||||
),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Return a paginated list of processing runs for all mail accounts owned by
|
||||
the authenticated user, optionally filtered by account or status.
|
||||
"""
|
||||
# Base query: join with MailAccount to enforce ownership
|
||||
base = (
|
||||
select(ProcessingRun, MailAccount.name, MailAccount.email_address)
|
||||
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
|
||||
.where(MailAccount.user_id == current_user.id) # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
if account_id is not None:
|
||||
base = base.where(ProcessingRun.mail_account_id == account_id)
|
||||
if status_filter:
|
||||
base = base.where(ProcessingRun.status == status_filter)
|
||||
|
||||
# Total count
|
||||
count_q = select(func.count()).select_from(base.subquery())
|
||||
total = (await db.execute(count_q)).scalar_one()
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
rows = (
|
||||
await db.execute(
|
||||
base.order_by(ProcessingRun.started_at.desc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
).all()
|
||||
|
||||
items = [
|
||||
ProcessingRunDetailResponse(
|
||||
id=row.ProcessingRun.id, # type: ignore[arg-type]
|
||||
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
|
||||
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
|
||||
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
|
||||
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
|
||||
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
|
||||
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
|
||||
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
|
||||
status=row.ProcessingRun.status, # type: ignore[arg-type]
|
||||
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
|
||||
account_name=row.name,
|
||||
account_email=row.email_address,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
return PaginatedProcessingRunsResponse(
|
||||
items=items, **_paginate(total, page, page_size) # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/processing-runs/{run_id}",
|
||||
response_model=ProcessingRunDetailResponse,
|
||||
summary="Get a single processing run",
|
||||
)
|
||||
async def get_processing_run(
|
||||
run_id: int,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return details for a single processing run owned by the current user."""
|
||||
row = (
|
||||
await db.execute(
|
||||
select(ProcessingRun, MailAccount.name, MailAccount.email_address)
|
||||
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
|
||||
.where(
|
||||
ProcessingRun.id == run_id,
|
||||
MailAccount.user_id == current_user.id, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
).one_or_none()
|
||||
|
||||
if not row:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Processing run not found"
|
||||
)
|
||||
|
||||
return ProcessingRunDetailResponse(
|
||||
id=row.ProcessingRun.id, # type: ignore[arg-type]
|
||||
mail_account_id=row.ProcessingRun.mail_account_id, # type: ignore[arg-type]
|
||||
started_at=row.ProcessingRun.started_at, # type: ignore[arg-type]
|
||||
completed_at=row.ProcessingRun.completed_at, # type: ignore[arg-type]
|
||||
duration_seconds=row.ProcessingRun.duration_seconds, # type: ignore[arg-type]
|
||||
emails_fetched=row.ProcessingRun.emails_fetched, # type: ignore[arg-type]
|
||||
emails_forwarded=row.ProcessingRun.emails_forwarded, # type: ignore[arg-type]
|
||||
emails_failed=row.ProcessingRun.emails_failed, # type: ignore[arg-type]
|
||||
status=row.ProcessingRun.status, # type: ignore[arg-type]
|
||||
error_message=row.ProcessingRun.error_message, # type: ignore[arg-type]
|
||||
account_name=row.name,
|
||||
account_email=row.email_address,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/processing-runs/{run_id}/logs",
|
||||
response_model=PaginatedProcessingLogsResponse,
|
||||
summary="Get per-email logs for a processing run",
|
||||
)
|
||||
async def get_run_logs(
|
||||
run_id: int,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=200),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Return the detailed per-email log entries recorded during a specific
|
||||
processing run. Ownership is verified by joining with MailAccount.
|
||||
"""
|
||||
# Verify the run belongs to this user
|
||||
run_row = (
|
||||
await db.execute(
|
||||
select(ProcessingRun)
|
||||
.join(MailAccount, ProcessingRun.mail_account_id == MailAccount.id)
|
||||
.where(
|
||||
ProcessingRun.id == run_id,
|
||||
MailAccount.user_id == current_user.id, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not run_row:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Processing run not found"
|
||||
)
|
||||
|
||||
count_q = select(func.count(ProcessingLog.id)).where(
|
||||
ProcessingLog.processing_run_id == run_id
|
||||
)
|
||||
total = (await db.execute(count_q)).scalar_one()
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
logs = (
|
||||
(
|
||||
await db.execute(
|
||||
select(ProcessingLog)
|
||||
.where(ProcessingLog.processing_run_id == run_id)
|
||||
.order_by(ProcessingLog.timestamp.asc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
items = [
|
||||
ProcessingLogDetailResponse(
|
||||
id=log.id, # type: ignore[arg-type]
|
||||
timestamp=log.timestamp, # type: ignore[arg-type]
|
||||
level=log.level, # type: ignore[arg-type]
|
||||
message=log.message, # type: ignore[arg-type]
|
||||
email_subject=log.email_subject, # type: ignore[arg-type]
|
||||
email_from=log.email_from, # type: ignore[arg-type]
|
||||
success=log.success, # type: ignore[arg-type]
|
||||
mail_account_id=log.mail_account_id, # type: ignore[arg-type]
|
||||
processing_run_id=log.processing_run_id, # type: ignore[arg-type]
|
||||
email_size_bytes=log.email_size_bytes, # type: ignore[arg-type]
|
||||
error_details=log.error_details, # type: ignore[arg-type]
|
||||
)
|
||||
for log in logs
|
||||
]
|
||||
|
||||
return PaginatedProcessingLogsResponse(
|
||||
items=items, **_paginate(total, page, page_size) # type: ignore[arg-type]
|
||||
)
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Mail account management endpoints"""
|
||||
|
||||
from typing import List
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
import math
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, desc
|
||||
from sqlalchemy import select, desc, func
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.core.deps import get_current_active_user
|
||||
@@ -11,6 +12,8 @@ from app.core.security import encrypt_credential
|
||||
from app.models.database_models import (
|
||||
User,
|
||||
MailAccount,
|
||||
ProcessingLog,
|
||||
ProcessingRun,
|
||||
AccountStatus,
|
||||
SubscriptionPlan,
|
||||
)
|
||||
@@ -22,6 +25,10 @@ from app.models.schemas import (
|
||||
MailAccountTestResponse,
|
||||
MailAccountAutoDetectRequest,
|
||||
MailAccountAutoDetectResponse,
|
||||
PaginatedProcessingRunsResponse,
|
||||
PaginatedProcessingLogsResponse,
|
||||
ProcessingRunDetailResponse,
|
||||
ProcessingLogDetailResponse,
|
||||
)
|
||||
from app.services.mail_processor import MailProcessor, MailServerAutoDetect
|
||||
from app.core.config import settings
|
||||
@@ -274,3 +281,148 @@ async def auto_detect_mail_settings(
|
||||
return MailAccountAutoDetectResponse(
|
||||
success=len(suggestions) > 0, suggestions=suggestions
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-account processing runs & logs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{account_id}/processing-runs",
|
||||
response_model=PaginatedProcessingRunsResponse,
|
||||
summary="List processing runs for a specific mail account",
|
||||
)
|
||||
async def list_account_runs(
|
||||
account_id: int,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return paginated processing runs for a mail account owned by the user."""
|
||||
result = await db.execute(
|
||||
select(MailAccount).where(
|
||||
MailAccount.id == account_id,
|
||||
MailAccount.user_id == current_user.id,
|
||||
)
|
||||
)
|
||||
account = result.scalar_one_or_none()
|
||||
if not account:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Mail account not found"
|
||||
)
|
||||
|
||||
base = select(ProcessingRun).where(ProcessingRun.mail_account_id == account_id)
|
||||
total = (
|
||||
await db.execute(select(func.count()).select_from(base.subquery()))
|
||||
).scalar_one()
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
runs = (
|
||||
(
|
||||
await db.execute(
|
||||
base.order_by(desc(ProcessingRun.started_at))
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
pages = max(1, math.ceil(total / page_size)) if total else 1
|
||||
items = [
|
||||
ProcessingRunDetailResponse(
|
||||
id=r.id, # type: ignore[arg-type]
|
||||
mail_account_id=r.mail_account_id, # type: ignore[arg-type]
|
||||
started_at=r.started_at, # type: ignore[arg-type]
|
||||
completed_at=r.completed_at, # type: ignore[arg-type]
|
||||
duration_seconds=r.duration_seconds, # type: ignore[arg-type]
|
||||
emails_fetched=r.emails_fetched, # type: ignore[arg-type]
|
||||
emails_forwarded=r.emails_forwarded, # type: ignore[arg-type]
|
||||
emails_failed=r.emails_failed, # type: ignore[arg-type]
|
||||
status=r.status, # type: ignore[arg-type]
|
||||
error_message=r.error_message, # type: ignore[arg-type]
|
||||
account_name=account.name, # type: ignore[arg-type]
|
||||
account_email=account.email_address, # type: ignore[arg-type]
|
||||
)
|
||||
for r in runs
|
||||
]
|
||||
return PaginatedProcessingRunsResponse(
|
||||
items=items, total=total, page=page, page_size=page_size, pages=pages
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{account_id}/logs",
|
||||
response_model=PaginatedProcessingLogsResponse,
|
||||
summary="List processing logs for a specific mail account",
|
||||
)
|
||||
async def list_account_logs(
|
||||
account_id: int,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=200),
|
||||
level: Optional[str] = Query(
|
||||
None, description="Filter by log level (INFO, WARNING, ERROR)"
|
||||
),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return paginated per-email log entries for a mail account owned by the user."""
|
||||
result = await db.execute(
|
||||
select(MailAccount).where(
|
||||
MailAccount.id == account_id,
|
||||
MailAccount.user_id == current_user.id,
|
||||
)
|
||||
)
|
||||
account = result.scalar_one_or_none()
|
||||
if not account:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Mail account not found"
|
||||
)
|
||||
|
||||
base = select(ProcessingLog).where(
|
||||
ProcessingLog.mail_account_id == account_id,
|
||||
ProcessingLog.user_id == current_user.id, # type: ignore[arg-type]
|
||||
)
|
||||
if level:
|
||||
base = base.where(ProcessingLog.level == level.upper())
|
||||
|
||||
total = (
|
||||
await db.execute(select(func.count()).select_from(base.subquery()))
|
||||
).scalar_one()
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
logs = (
|
||||
(
|
||||
await db.execute(
|
||||
base.order_by(ProcessingLog.timestamp.desc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
pages = max(1, math.ceil(total / page_size)) if total else 1
|
||||
items = [
|
||||
ProcessingLogDetailResponse(
|
||||
id=log.id, # type: ignore[arg-type]
|
||||
timestamp=log.timestamp, # type: ignore[arg-type]
|
||||
level=log.level, # type: ignore[arg-type]
|
||||
message=log.message, # type: ignore[arg-type]
|
||||
email_subject=log.email_subject, # type: ignore[arg-type]
|
||||
email_from=log.email_from, # type: ignore[arg-type]
|
||||
success=log.success, # type: ignore[arg-type]
|
||||
mail_account_id=log.mail_account_id, # type: ignore[arg-type]
|
||||
processing_run_id=log.processing_run_id, # type: ignore[arg-type]
|
||||
email_size_bytes=log.email_size_bytes, # type: ignore[arg-type]
|
||||
error_details=log.error_details, # type: ignore[arg-type]
|
||||
)
|
||||
for log in logs
|
||||
]
|
||||
return PaginatedProcessingLogsResponse(
|
||||
items=items, total=total, page=page, page_size=page_size, pages=pages
|
||||
)
|
||||
|
||||
@@ -21,12 +21,30 @@ from app.models.schemas import (
|
||||
GmailCredentialResponse,
|
||||
GmailAuthorizeResponse,
|
||||
GmailCallbackRequest,
|
||||
GmailImportLabelsUpdate,
|
||||
)
|
||||
from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES
|
||||
from app.utils.gmail_labels import (
|
||||
MAX_IMPORT_LABELS,
|
||||
build_gmail_credential_scopes,
|
||||
extract_granted_scopes,
|
||||
normalize_import_label_templates,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _validated_import_label_templates(label_templates: List[str]) -> List[str]:
|
||||
normalized = normalize_import_label_templates(label_templates)
|
||||
if len(normalized) > MAX_IMPORT_LABELS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"You can configure up to {MAX_IMPORT_LABELS} Gmail import labels.",
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
# Gmail API scopes requested during the "Connect Gmail" OAuth flow.
|
||||
# GMAIL_SCOPES (gmail.insert, gmail.labels, gmail.readonly) are imported from
|
||||
# gmail_service so the scope list stays in sync with what GmailService uses.
|
||||
@@ -223,6 +241,10 @@ async def save_gmail_credential(
|
||||
existing.gmail_email = credential_in.gmail_email # type: ignore[assignment]
|
||||
existing.encrypted_access_token = encrypted_access # type: ignore[assignment]
|
||||
existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
|
||||
existing.scopes = build_gmail_credential_scopes(
|
||||
existing.granted_scopes,
|
||||
existing.import_label_templates,
|
||||
) # type: ignore[assignment]
|
||||
existing.is_valid = True # type: ignore[assignment]
|
||||
existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
|
||||
await db.commit()
|
||||
@@ -235,6 +257,7 @@ async def save_gmail_credential(
|
||||
gmail_email=credential_in.gmail_email,
|
||||
encrypted_access_token=encrypted_access,
|
||||
encrypted_refresh_token=encrypted_refresh,
|
||||
scopes=build_gmail_credential_scopes(),
|
||||
is_valid=True,
|
||||
last_verified_at=datetime.now(timezone.utc),
|
||||
)
|
||||
@@ -285,6 +308,33 @@ async def delete_gmail_credential(
|
||||
await db.commit()
|
||||
|
||||
|
||||
@router.put("/gmail-credential/labels", response_model=GmailCredentialResponse)
|
||||
async def update_gmail_import_labels(
|
||||
labels_in: GmailImportLabelsUpdate,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update the Gmail labels applied to imported messages."""
|
||||
result = await db.execute(
|
||||
select(GmailCredential).where(GmailCredential.user_id == current_user.id)
|
||||
)
|
||||
credential = result.scalar_one_or_none()
|
||||
|
||||
if not credential:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="No Gmail credentials found. Connect Gmail first.",
|
||||
)
|
||||
|
||||
credential.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
|
||||
extract_granted_scopes(credential.scopes),
|
||||
_validated_import_label_templates(labels_in.import_label_templates),
|
||||
)
|
||||
await db.commit()
|
||||
await db.refresh(credential)
|
||||
return credential
|
||||
|
||||
|
||||
@router.get("/gmail/authorize-url", response_model=GmailAuthorizeResponse)
|
||||
async def get_gmail_authorize_url(
|
||||
redirect_uri: str,
|
||||
@@ -365,6 +415,7 @@ async def send_gmail_debug_email(
|
||||
try:
|
||||
inject_result = await gmail_service.inject_debug_email(
|
||||
recipient_email=credential.gmail_email, # type: ignore[arg-type]
|
||||
import_label_templates=credential.import_label_templates,
|
||||
)
|
||||
except GmailInjectionError as exc:
|
||||
raise HTTPException(
|
||||
@@ -498,7 +549,10 @@ async def gmail_oauth_callback(
|
||||
if encrypted_refresh:
|
||||
existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment]
|
||||
existing.token_expiry = token_expiry # type: ignore[assignment]
|
||||
existing.scopes = token_data.get("scope", "").split() # type: ignore[assignment]
|
||||
existing.scopes = build_gmail_credential_scopes( # type: ignore[assignment]
|
||||
token_data.get("scope", "").split(),
|
||||
existing.import_label_templates,
|
||||
)
|
||||
existing.is_valid = True # type: ignore[assignment]
|
||||
existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment]
|
||||
await db.commit()
|
||||
@@ -511,7 +565,7 @@ async def gmail_oauth_callback(
|
||||
encrypted_access_token=encrypted_access,
|
||||
encrypted_refresh_token=encrypted_refresh,
|
||||
token_expiry=token_expiry,
|
||||
scopes=token_data.get("scope", "").split(),
|
||||
scopes=build_gmail_credential_scopes(token_data.get("scope", "").split()),
|
||||
is_valid=True,
|
||||
last_verified_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@@ -27,8 +27,10 @@ class Settings(BaseSettings):
|
||||
)
|
||||
|
||||
# Application
|
||||
APP_NAME: str = "InboxRescue"
|
||||
APP_NAME: str = "InboxConverge"
|
||||
APP_VERSION: str = "2.0.0"
|
||||
APP_URL: str = "https://inboxconverge.com"
|
||||
CONTACT_EMAIL: str = "christian@inboxconverge.com"
|
||||
DEBUG: bool = False
|
||||
API_V1_PREFIX: str = "/api/v1"
|
||||
|
||||
@@ -38,7 +40,7 @@ class Settings(BaseSettings):
|
||||
|
||||
# Database
|
||||
DATABASE_URL: str = (
|
||||
"postgresql+asyncpg://user:password@localhost:5432/pop3_forwarder"
|
||||
"postgresql+asyncpg://user:password@localhost:5432/inbox_converge"
|
||||
)
|
||||
DATABASE_POOL_SIZE: int = 20
|
||||
DATABASE_MAX_OVERFLOW: int = 10
|
||||
@@ -94,7 +96,7 @@ class Settings(BaseSettings):
|
||||
LOG_LEVEL: str = "INFO"
|
||||
|
||||
# Admin
|
||||
ADMIN_EMAIL: Optional[str] = "christianlouis@gmail.com"
|
||||
ADMIN_EMAIL: Optional[str] = "christian@inboxconverge.com"
|
||||
ADMIN_PASSWORD: Optional[str] = None
|
||||
|
||||
# User defaults & access control
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
GDPR-compliant data masking utilities.
|
||||
|
||||
These helpers are used in admin-facing API responses to pseudonymise
|
||||
personal data (email addresses, names) so that operators can audit
|
||||
system behaviour without seeing full end-user PII.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def mask_email(email: str) -> str:
|
||||
"""
|
||||
Partially mask an email address for GDPR-compliant display.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> mask_email("john.doe@example.com")
|
||||
'jo***@e***.com'
|
||||
>>> mask_email("ab@x.io")
|
||||
'ab***@x***.io'
|
||||
>>> mask_email("a@b.de")
|
||||
'a***@b***.de'
|
||||
"""
|
||||
if not email or "@" not in email:
|
||||
return "***"
|
||||
|
||||
local, _, domain = email.partition("@")
|
||||
|
||||
# Local part: keep first 2 chars (or all if shorter), then "***"
|
||||
visible_local = local[:2] if len(local) >= 2 else local
|
||||
masked_local = f"{visible_local}***"
|
||||
|
||||
# Domain part: keep first char of SLD and the TLD unchanged
|
||||
domain_parts = domain.rsplit(".", 1)
|
||||
if len(domain_parts) == 2:
|
||||
sld, tld = domain_parts
|
||||
visible_sld = sld[:1] if sld else ""
|
||||
masked_domain = f"{visible_sld}***.{tld}"
|
||||
else:
|
||||
masked_domain = "***"
|
||||
|
||||
return f"{masked_local}@{masked_domain}"
|
||||
|
||||
|
||||
def mask_name(name: str) -> str:
|
||||
"""
|
||||
Partially mask a display name.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> mask_name("John Doe")
|
||||
'Jo*** D***'
|
||||
>>> mask_name("Alice")
|
||||
'Al***'
|
||||
"""
|
||||
if not name:
|
||||
return "***"
|
||||
words = name.split()
|
||||
masked_words = []
|
||||
for word in words:
|
||||
visible = word[:2] if len(word) >= 2 else word
|
||||
masked_words.append(f"{visible}***")
|
||||
return " ".join(masked_words)
|
||||
|
||||
|
||||
# RFC 5322 address pattern – extracts the bare email from strings like
|
||||
# "John Doe <john@example.com>" or just "john@example.com".
|
||||
_ADDR_RE = re.compile(r"<([^>]+)>|(\S+@\S+\.\S+)")
|
||||
|
||||
|
||||
def mask_from_header(from_header: str) -> str:
|
||||
"""
|
||||
Mask a raw RFC 5322 From header value for GDPR-compliant display.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> mask_from_header("John Doe <john.doe@example.com>")
|
||||
'Jo*** D*** <jo***@e***.com>'
|
||||
>>> mask_from_header("john.doe@example.com")
|
||||
'jo***@e***.com'
|
||||
"""
|
||||
if not from_header:
|
||||
return "***"
|
||||
|
||||
# Try to parse "Display Name <email>" form
|
||||
angle_match = re.search(r"^(.*?)<([^>]+)>", from_header.strip())
|
||||
if angle_match:
|
||||
display_name = angle_match.group(1).strip().strip('"')
|
||||
email_part = angle_match.group(2).strip()
|
||||
masked_email = mask_email(email_part)
|
||||
if display_name:
|
||||
masked_display = mask_name(display_name)
|
||||
return f"{masked_display} <{masked_email}>"
|
||||
return masked_email
|
||||
|
||||
# Plain email address
|
||||
bare_match = _ADDR_RE.search(from_header)
|
||||
if bare_match:
|
||||
email_part = bare_match.group(1) or bare_match.group(2)
|
||||
return mask_email(email_part)
|
||||
|
||||
# Fallback: mask the whole string
|
||||
return mask_name(from_header)
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Prometheus metrics definitions for InboxRescue.
|
||||
Prometheus metrics definitions for InboxConverge.
|
||||
|
||||
All application metrics are defined here as module-level singletons so that
|
||||
every subsystem (HTTP layer, Celery workers, Gmail service, auth) imports
|
||||
|
||||
+1
-1
@@ -148,7 +148,7 @@ def create_application() -> FastAPI:
|
||||
async def root():
|
||||
"""Root endpoint"""
|
||||
return {
|
||||
"message": "InboxRescue API",
|
||||
"message": "InboxConverge API",
|
||||
"version": settings.APP_VERSION,
|
||||
"docs": "/api/docs",
|
||||
}
|
||||
|
||||
@@ -17,6 +17,12 @@ from sqlalchemy import (
|
||||
Index,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.utils.gmail_labels import (
|
||||
DEFAULT_IMPORT_LABEL_TEMPLATES,
|
||||
extract_granted_scopes,
|
||||
extract_import_label_templates,
|
||||
)
|
||||
import enum
|
||||
|
||||
from app.core.database import Base
|
||||
@@ -562,6 +568,18 @@ class GmailCredential(Base):
|
||||
# Relationships
|
||||
user = relationship("User", backref="gmail_credential")
|
||||
|
||||
@property
|
||||
def granted_scopes(self) -> list[str]:
|
||||
return extract_granted_scopes(self.scopes)
|
||||
|
||||
@property
|
||||
def import_label_templates(self) -> list[str]:
|
||||
return extract_import_label_templates(self.scopes)
|
||||
|
||||
@property
|
||||
def default_import_label_templates(self) -> list[str]:
|
||||
return DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
|
||||
|
||||
|
||||
class AppSetting(Base):
|
||||
"""
|
||||
|
||||
@@ -204,6 +204,15 @@ class ProcessingRunResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class ProcessingRunDetailResponse(ProcessingRunResponse):
|
||||
"""ProcessingRunResponse with optional account metadata."""
|
||||
|
||||
account_name: Optional[str] = None
|
||||
account_email: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
# Processing Log Schemas
|
||||
class ProcessingLogResponse(BaseModel):
|
||||
id: int
|
||||
@@ -217,6 +226,67 @@ class ProcessingLogResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class ProcessingLogDetailResponse(ProcessingLogResponse):
|
||||
"""ProcessingLogResponse with additional fields."""
|
||||
|
||||
mail_account_id: int
|
||||
processing_run_id: Optional[int] = None
|
||||
email_size_bytes: Optional[int] = None
|
||||
error_details: Optional[Dict[str, Any]] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class PaginatedProcessingRunsResponse(BaseModel):
|
||||
items: List[ProcessingRunDetailResponse]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
pages: int
|
||||
|
||||
|
||||
class PaginatedProcessingLogsResponse(BaseModel):
|
||||
items: List[ProcessingLogDetailResponse]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
pages: int
|
||||
|
||||
|
||||
class AdminProcessingRunResponse(ProcessingRunDetailResponse):
|
||||
"""ProcessingRunDetailResponse with user info for admin views."""
|
||||
|
||||
user_id: Optional[int] = None
|
||||
user_email: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class PaginatedAdminRunsResponse(BaseModel):
|
||||
items: List[AdminProcessingRunResponse]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
pages: int
|
||||
|
||||
|
||||
class AdminProcessingLogResponse(ProcessingLogDetailResponse):
|
||||
"""ProcessingLogDetailResponse with user info for admin views."""
|
||||
|
||||
user_id: int
|
||||
user_email: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class PaginatedAdminLogsResponse(BaseModel):
|
||||
items: List[AdminProcessingLogResponse]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
pages: int
|
||||
|
||||
|
||||
# Notification Config Schemas
|
||||
class NotificationConfigBase(BaseModel):
|
||||
name: str = Field(
|
||||
@@ -376,6 +446,8 @@ class GmailCredentialResponse(BaseModel):
|
||||
user_id: int
|
||||
gmail_email: str
|
||||
is_valid: bool
|
||||
import_label_templates: List[str] = Field(default_factory=list)
|
||||
default_import_label_templates: List[str] = Field(default_factory=list)
|
||||
last_verified_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -430,6 +502,10 @@ class GmailCallbackRequest(BaseModel):
|
||||
redirect_uri: str
|
||||
|
||||
|
||||
class GmailImportLabelsUpdate(BaseModel):
|
||||
import_label_templates: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
# Admin Schemas
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from app.core.metrics import (
|
||||
GMAIL_API_DURATION_SECONDS,
|
||||
GMAIL_TOKEN_REFRESHES_TOTAL,
|
||||
)
|
||||
from app.utils.gmail_labels import render_import_labels
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -294,6 +295,7 @@ class GmailService:
|
||||
async def inject_debug_email(
|
||||
self,
|
||||
recipient_email: str,
|
||||
import_label_templates: Optional[list[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Inject a debug/test email into the user's Gmail inbox.
|
||||
@@ -344,11 +346,10 @@ class GmailService:
|
||||
|
||||
raw_bytes = msg.as_bytes()
|
||||
|
||||
# Resolve label IDs (create labels if they don't exist yet)
|
||||
label_ids = await self.build_import_label_ids(import_label_templates)
|
||||
test_label_id = await self.get_or_create_label("test")
|
||||
imported_label_id = await self.get_or_create_label("imported")
|
||||
|
||||
label_ids = ["INBOX", test_label_id, imported_label_id]
|
||||
if test_label_id not in label_ids:
|
||||
label_ids.append(test_label_id)
|
||||
|
||||
return await self.inject_email(
|
||||
raw_email=raw_bytes,
|
||||
@@ -356,6 +357,23 @@ class GmailService:
|
||||
source_account_name="debug",
|
||||
)
|
||||
|
||||
async def build_import_label_ids(
|
||||
self,
|
||||
import_label_templates: Optional[list[str]] = None,
|
||||
source_email: Optional[str] = None,
|
||||
) -> list[str]:
|
||||
"""Resolve configured import labels into Gmail label IDs."""
|
||||
label_ids = ["INBOX"]
|
||||
|
||||
for label_name in render_import_labels(import_label_templates, source_email):
|
||||
if label_name.upper() == "INBOX":
|
||||
continue
|
||||
label_id = await self.get_or_create_label(label_name)
|
||||
if label_id not in label_ids:
|
||||
label_ids.append(label_id)
|
||||
|
||||
return label_ids
|
||||
|
||||
def get_refreshed_token(self) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Return the current access token and expiry if the token was refreshed
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Helpers for Gmail import label configuration and rendering."""
|
||||
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
SOURCE_EMAIL_LABEL_TEMPLATE = "{{source_email}}"
|
||||
DEFAULT_IMPORT_LABEL_TEMPLATES = [SOURCE_EMAIL_LABEL_TEMPLATE, "imported"]
|
||||
MAX_IMPORT_LABELS = 10
|
||||
|
||||
|
||||
def _normalize_string_list(values: Optional[Iterable[str]]) -> list[str]:
|
||||
normalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for value in values or []:
|
||||
cleaned = value.strip()
|
||||
if not cleaned:
|
||||
continue
|
||||
lowered = cleaned.casefold()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
normalized.append(cleaned)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_import_label_templates(
|
||||
label_templates: Optional[Iterable[str]],
|
||||
) -> list[str]:
|
||||
"""Return a cleaned, de-duplicated label template list."""
|
||||
normalized = _normalize_string_list(label_templates)
|
||||
return normalized or DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
|
||||
|
||||
|
||||
def extract_granted_scopes(scopes_data: Any) -> list[str]:
|
||||
"""Read granted scopes from legacy list or new JSON object storage."""
|
||||
if isinstance(scopes_data, list):
|
||||
return _normalize_string_list(
|
||||
value for value in scopes_data if isinstance(value, str)
|
||||
)
|
||||
|
||||
if isinstance(scopes_data, dict):
|
||||
granted_scopes = scopes_data.get("granted_scopes", [])
|
||||
if isinstance(granted_scopes, list):
|
||||
return _normalize_string_list(
|
||||
value for value in granted_scopes if isinstance(value, str)
|
||||
)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def extract_import_label_templates(scopes_data: Any) -> list[str]:
|
||||
"""Read import label templates from stored Gmail credential metadata."""
|
||||
if isinstance(scopes_data, dict):
|
||||
stored_templates = scopes_data.get("import_label_templates", [])
|
||||
if isinstance(stored_templates, list):
|
||||
return normalize_import_label_templates(
|
||||
value for value in stored_templates if isinstance(value, str)
|
||||
)
|
||||
|
||||
return DEFAULT_IMPORT_LABEL_TEMPLATES.copy()
|
||||
|
||||
|
||||
def build_gmail_credential_scopes(
|
||||
granted_scopes: Optional[Iterable[str]],
|
||||
import_label_templates: Optional[Iterable[str]] = None,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Persist Gmail metadata in the existing JSON column."""
|
||||
return {
|
||||
"granted_scopes": _normalize_string_list(granted_scopes),
|
||||
"import_label_templates": normalize_import_label_templates(
|
||||
import_label_templates
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def render_import_labels(
|
||||
import_label_templates: Optional[Iterable[str]],
|
||||
source_email: Optional[str],
|
||||
) -> list[str]:
|
||||
"""Render label templates into actual Gmail label names."""
|
||||
rendered_labels: list[str] = []
|
||||
seen: set[str] = set()
|
||||
resolved_source_email = source_email.strip() if source_email else ""
|
||||
|
||||
for template in normalize_import_label_templates(import_label_templates):
|
||||
rendered = template.replace(SOURCE_EMAIL_LABEL_TEMPLATE, resolved_source_email)
|
||||
rendered = rendered.strip()
|
||||
if not rendered:
|
||||
continue
|
||||
lowered = rendered.casefold()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
rendered_labels.append(rendered)
|
||||
|
||||
return rendered_labels
|
||||
@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# Create Celery app
|
||||
celery_app = Celery(
|
||||
"pop3_forwarder",
|
||||
"inboxconverge",
|
||||
broker=settings.CELERY_BROKER_URL,
|
||||
backend=settings.CELERY_RESULT_BACKEND,
|
||||
include=["app.workers.tasks"],
|
||||
|
||||
@@ -3,6 +3,7 @@ Celery tasks for background email processing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import email as email_lib
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from celery import Task
|
||||
@@ -199,15 +200,46 @@ async def process_mail_account(account_id: int):
|
||||
f"for account {account.id}; truncating to shorter list"
|
||||
)
|
||||
|
||||
# Field length limits matching the DB column definitions
|
||||
_MAX_SUBJECT_LEN = 500
|
||||
_MAX_FROM_LEN = 255
|
||||
|
||||
for email_data, uid in zip(emails, new_uids):
|
||||
# ── Parse email metadata for logging ───────────────────────
|
||||
email_subject: str | None = None
|
||||
email_from: str | None = None
|
||||
try:
|
||||
msg = email_lib.message_from_bytes(email_data)
|
||||
raw_subject = msg.get("Subject", "") or ""
|
||||
email_subject = (
|
||||
raw_subject[:_MAX_SUBJECT_LEN] if raw_subject else None
|
||||
)
|
||||
raw_from = msg.get("From", "") or ""
|
||||
email_from = raw_from[:_MAX_FROM_LEN] if raw_from else None
|
||||
except (ValueError, TypeError, UnicodeDecodeError) as exc:
|
||||
logger.debug(
|
||||
"Could not parse email headers for account %s: %s",
|
||||
account.id,
|
||||
exc,
|
||||
)
|
||||
|
||||
email_size_bytes = len(email_data)
|
||||
forwarded_ok = False
|
||||
error_msg: str | None = None
|
||||
|
||||
try:
|
||||
if use_gmail_api and gmail_service:
|
||||
# Inject via Gmail API (preferred)
|
||||
label_ids = await gmail_service.build_import_label_ids(
|
||||
import_label_templates=gmail_cred.import_label_templates,
|
||||
source_email=account.email_address, # type: ignore[arg-type]
|
||||
)
|
||||
await gmail_service.inject_email(
|
||||
raw_email=email_data,
|
||||
label_ids=["INBOX"],
|
||||
label_ids=label_ids,
|
||||
source_account_name=account.name, # type: ignore[arg-type]
|
||||
)
|
||||
forwarded_ok = True
|
||||
emails_forwarded += 1
|
||||
successfully_forwarded_uids.append(uid)
|
||||
else:
|
||||
@@ -216,6 +248,7 @@ async def process_mail_account(account_id: int):
|
||||
email_data, account.name, account.forward_to, smtp_config # type: ignore[arg-type]
|
||||
)
|
||||
if success:
|
||||
forwarded_ok = True
|
||||
emails_forwarded += 1
|
||||
successfully_forwarded_uids.append(uid)
|
||||
else:
|
||||
@@ -253,8 +286,29 @@ async def process_mail_account(account_id: int):
|
||||
f"Failed to send revocation notification: {notify_exc}"
|
||||
)
|
||||
logger.error(f"Error delivering email: {e}")
|
||||
error_msg = str(e)
|
||||
emails_failed += 1
|
||||
|
||||
# ── Write per-email ProcessingLog entry ─────────────────────
|
||||
db.add(
|
||||
ProcessingLog(
|
||||
user_id=account.user_id,
|
||||
mail_account_id=account.id,
|
||||
processing_run_id=run.id,
|
||||
level="INFO" if forwarded_ok else "ERROR",
|
||||
message=(
|
||||
f"Forwarded: {email_subject or '(no subject)'}"
|
||||
if forwarded_ok
|
||||
else f"Failed: {error_msg or 'delivery error'}"
|
||||
),
|
||||
email_subject=email_subject,
|
||||
email_from=email_from,
|
||||
email_size_bytes=email_size_bytes,
|
||||
success=forwarded_ok,
|
||||
error_details={"error": error_msg} if error_msg else None,
|
||||
)
|
||||
)
|
||||
|
||||
# Persist new message UIDs so they are not processed again
|
||||
for uid in successfully_forwarded_uids:
|
||||
if uid not in already_seen_uids:
|
||||
|
||||
Reference in New Issue
Block a user