Add Gmail API injection, provider presets, delivery method, and frontend wizard

- Add gmail_service.py with Gmail API users.messages.insert() for direct email injection
- Add DeliveryMethod enum and GmailCredential model to database models
- Add delivery_method field to MailAccount schema and model
- Create providers.py endpoint with 12 provider presets (Gmail, GMX, WEB.DE, Outlook, Yahoo, AOL, T-Online, 1&1/IONOS, Freenet, Posteo, mail.de, iCloud)
- Expand MailServerAutoDetect with 30+ domain mappings
- Update Celery tasks to prefer Gmail API injection, with SMTP fallback
- Add ProviderWizard.tsx frontend component for quick provider setup
- Update AddMailAccountModal.tsx with wizard integration
- Add Google API Python client libraries to requirements.txt
- Add Gmail API config settings
- Add unit tests for Gmail service and provider presets

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/pop_puller_to_gmail/sessions/de3ef930-a980-4958-8a9d-a2c802918e81
This commit is contained in:
copilot-swe-agent[bot]
2026-03-22 19:00:30 +00:00
parent 6435af3251
commit 40150da3a9
14 changed files with 1580 additions and 191 deletions
+2 -1
View File
@@ -3,7 +3,7 @@ API v1 router aggregation.
"""
from fastapi import APIRouter
from app.api.v1.endpoints import auth, users, mail_accounts, notifications, subscriptions, admin
from app.api.v1.endpoints import auth, users, mail_accounts, notifications, subscriptions, admin, providers
api_router = APIRouter()
@@ -11,6 +11,7 @@ api_router = APIRouter()
api_router.include_router(auth.router, prefix="/auth", tags=["Authentication"])
api_router.include_router(users.router, prefix="/users", tags=["Users"])
api_router.include_router(mail_accounts.router, prefix="/mail-accounts", tags=["Mail Accounts"])
api_router.include_router(providers.router, prefix="/providers", tags=["Providers & Gmail"])
api_router.include_router(notifications.router, prefix="/notifications", tags=["Notifications"])
api_router.include_router(subscriptions.router, prefix="/subscriptions", tags=["Subscriptions"])
api_router.include_router(admin.router, prefix="/admin", tags=["Admin"])
@@ -64,6 +64,7 @@ async def create_mail_account(
username=account_in.username,
encrypted_password=encrypted_password,
forward_to=account_in.forward_to,
delivery_method=account_in.delivery_method,
is_enabled=account_in.is_enabled,
check_interval_minutes=account_in.check_interval_minutes,
max_emails_per_check=account_in.max_emails_per_check,
+260
View File
@@ -0,0 +1,260 @@
"""Provider presets and Gmail credential management endpoints"""
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.core.database import get_db
from app.core.deps import get_current_active_user
from app.core.security import encrypt_credential, decrypt_credential
from app.core.config import settings
from app.models.database_models import User, GmailCredential
from app.models.schemas import (
ProviderPreset,
ProviderListResponse,
GmailCredentialCreate,
GmailCredentialResponse,
)
from app.services.gmail_service import GmailService
router = APIRouter()
# Provider presets with server configurations
PROVIDER_PRESETS: List[ProviderPreset] = [
ProviderPreset(
id="gmail",
name="Gmail",
icon="gmail",
domains=["gmail.com", "googlemail.com"],
imap_ssl={"host": "imap.gmail.com", "port": 993},
pop3_ssl={"host": "pop.gmail.com", "port": 995},
notes="Enable IMAP/POP3 in Gmail settings. Use an App Password if 2FA is enabled.",
),
ProviderPreset(
id="gmx",
name="GMX",
icon="gmx",
domains=["gmx.de", "gmx.net", "gmx.at", "gmx.ch", "gmx.com"],
imap_ssl={"host": "imap.gmx.net", "port": 993},
pop3_ssl={"host": "pop.gmx.net", "port": 995},
notes="Enable POP3/IMAP in GMX settings under E-Mail > POP3/IMAP Abruf.",
),
ProviderPreset(
id="webde",
name="WEB.DE",
icon="webde",
domains=["web.de"],
imap_ssl={"host": "imap.web.de", "port": 993},
pop3_ssl={"host": "pop3.web.de", "port": 995},
notes="Enable POP3/IMAP in WEB.DE settings under E-Mail > POP3/IMAP Abruf.",
),
ProviderPreset(
id="outlook",
name="Outlook / Hotmail",
icon="outlook",
domains=["outlook.com", "hotmail.com", "live.com", "msn.com", "outlook.de"],
imap_ssl={"host": "outlook.office365.com", "port": 993},
pop3_ssl={"host": "outlook.office365.com", "port": 995},
notes="Use your Microsoft account credentials.",
),
ProviderPreset(
id="yahoo",
name="Yahoo Mail",
icon="yahoo",
domains=["yahoo.com", "yahoo.de", "yahoo.co.uk", "ymail.com"],
imap_ssl={"host": "imap.mail.yahoo.com", "port": 993},
pop3_ssl={"host": "pop.mail.yahoo.com", "port": 995},
notes="Generate an App Password in Yahoo account security settings.",
),
ProviderPreset(
id="aol",
name="AOL Mail",
icon="aol",
domains=["aol.com", "aim.com"],
imap_ssl={"host": "imap.aol.com", "port": 993},
pop3_ssl={"host": "pop.aol.com", "port": 995},
notes="Generate an App Password in AOL account security settings.",
),
ProviderPreset(
id="tonline",
name="T-Online",
icon="tonline",
domains=["t-online.de"],
imap_ssl={"host": "secureimap.t-online.de", "port": 993},
pop3_ssl={"host": "securepop.t-online.de", "port": 995},
notes="Use your T-Online E-Mail-Passwort (not your Telekom login password).",
),
ProviderPreset(
id="ionos",
name="1&1 / IONOS",
icon="ionos",
domains=["online.de", "onlinehome.de", "1und1.de"],
imap_ssl={"host": "imap.ionos.de", "port": 993},
pop3_ssl={"host": "pop.ionos.de", "port": 995},
notes="Use your IONOS email credentials.",
),
ProviderPreset(
id="freenet",
name="Freenet",
icon="freenet",
domains=["freenet.de"],
imap_ssl={"host": "mx.freenet.de", "port": 993},
pop3_ssl={"host": "mx.freenet.de", "port": 995},
notes="Use your Freenet email credentials.",
),
ProviderPreset(
id="posteo",
name="Posteo",
icon="posteo",
domains=["posteo.de", "posteo.net"],
imap_ssl={"host": "posteo.de", "port": 993},
pop3_ssl=None,
notes="Posteo supports IMAP only. Use your Posteo credentials.",
),
ProviderPreset(
id="mailde",
name="mail.de",
icon="mailde",
domains=["mail.de"],
imap_ssl={"host": "imap.mail.de", "port": 993},
pop3_ssl={"host": "pop.mail.de", "port": 995},
notes="Use your mail.de email credentials.",
),
ProviderPreset(
id="icloud",
name="iCloud Mail",
icon="icloud",
domains=["icloud.com", "me.com", "mac.com"],
imap_ssl={"host": "imap.mail.me.com", "port": 993},
pop3_ssl=None,
notes="Generate an app-specific password at appleid.apple.com.",
),
]
@router.get("/presets", response_model=ProviderListResponse)
async def list_provider_presets(
current_user: User = Depends(get_current_active_user),
):
"""List all available mail provider presets for quick setup wizard"""
return ProviderListResponse(providers=PROVIDER_PRESETS)
@router.get("/presets/{provider_id}", response_model=ProviderPreset)
async def get_provider_preset(
provider_id: str,
current_user: User = Depends(get_current_active_user),
):
"""Get a specific provider preset by ID"""
for preset in PROVIDER_PRESETS:
if preset.id == provider_id:
return preset
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Provider '{provider_id}' not found",
)
@router.post("/gmail-credential", response_model=GmailCredentialResponse, status_code=status.HTTP_201_CREATED)
async def save_gmail_credential(
credential_in: GmailCredentialCreate,
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""
Save Gmail API OAuth2 credentials for the current user.
These are used to inject emails directly into Gmail via the API.
"""
# Verify the credentials work
gmail_service = GmailService(
access_token=credential_in.access_token,
refresh_token=credential_in.refresh_token,
client_id=settings.GOOGLE_CLIENT_ID,
client_secret=settings.GOOGLE_CLIENT_SECRET,
)
is_valid = await gmail_service.verify_access()
if not is_valid:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Gmail API credentials are invalid or expired",
)
# Check for existing credential
result = await db.execute(
select(GmailCredential).where(GmailCredential.user_id == current_user.id)
)
existing = result.scalar_one_or_none()
encrypted_access = encrypt_credential(credential_in.access_token)
encrypted_refresh = (
encrypt_credential(credential_in.refresh_token) if credential_in.refresh_token else None
)
if existing:
# Update existing
existing.gmail_email = credential_in.gmail_email
existing.encrypted_access_token = encrypted_access
existing.encrypted_refresh_token = encrypted_refresh
existing.is_valid = True
from datetime import datetime
existing.last_verified_at = datetime.utcnow()
await db.commit()
await db.refresh(existing)
return existing
else:
# Create new
from datetime import datetime
credential = GmailCredential(
user_id=current_user.id,
gmail_email=credential_in.gmail_email,
encrypted_access_token=encrypted_access,
encrypted_refresh_token=encrypted_refresh,
is_valid=True,
last_verified_at=datetime.utcnow(),
)
db.add(credential)
await db.commit()
await db.refresh(credential)
return credential
@router.get("/gmail-credential", response_model=GmailCredentialResponse)
async def get_gmail_credential(
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Get the current user's Gmail API credential status"""
result = await db.execute(
select(GmailCredential).where(GmailCredential.user_id == current_user.id)
)
credential = result.scalar_one_or_none()
if not credential:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No Gmail credentials configured. Set up Gmail API access first.",
)
return credential
@router.delete("/gmail-credential", status_code=status.HTTP_204_NO_CONTENT)
async def delete_gmail_credential(
current_user: User = Depends(get_current_active_user),
db: AsyncSession = Depends(get_db),
):
"""Delete the current user's Gmail API credentials"""
result = await db.execute(
select(GmailCredential).where(GmailCredential.user_id == current_user.id)
)
credential = result.scalar_one_or_none()
if not credential:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No Gmail credentials found",
)
await db.delete(credential)
await db.commit()
+4
View File
@@ -46,6 +46,10 @@ class Settings(BaseSettings):
GOOGLE_CLIENT_SECRET: Optional[str] = None
GOOGLE_REDIRECT_URI: str = "http://localhost:3000/auth/callback/google"
# Gmail API (for direct email injection)
GMAIL_API_ENABLED: bool = True
GMAIL_INJECT_LABEL_IDS: List[str] = ["INBOX"]
# CORS
CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"]
+39
View File
@@ -29,6 +29,12 @@ class MailProtocol(str, enum.Enum):
IMAP_SSL = "imap_ssl"
class DeliveryMethod(str, enum.Enum):
"""How emails are delivered to Gmail"""
SMTP = "smtp" # Forward via SMTP (legacy)
GMAIL_API = "gmail_api" # Inject via Gmail API (preferred)
class AccountStatus(str, enum.Enum):
"""Mail account status"""
ACTIVE = "active"
@@ -104,6 +110,9 @@ class MailAccount(Base):
# Forwarding destination
forward_to = Column(String(255), nullable=False)
# Delivery method
delivery_method = Column(SQLEnum(DeliveryMethod), default=DeliveryMethod.GMAIL_API)
# Status and settings
status = Column(SQLEnum(AccountStatus), default=AccountStatus.ACTIVE)
is_enabled = Column(Boolean, default=True)
@@ -326,3 +335,33 @@ class AuditLog(Base):
Index('idx_user_action', 'user_id', 'action'),
Index('idx_timestamp_action', 'timestamp', 'action'),
)
class GmailCredential(Base):
"""Stores OAuth2 credentials for Gmail API access (per-user)"""
__tablename__ = "gmail_credentials"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, unique=True)
# Gmail account email
gmail_email = Column(String(255), nullable=False)
# OAuth2 tokens (encrypted)
encrypted_access_token = Column(Text, nullable=False)
encrypted_refresh_token = Column(Text, nullable=True)
# Token metadata
token_expiry = Column(DateTime, nullable=True)
scopes = Column(JSON, nullable=True)
# Status
is_valid = Column(Boolean, default=True)
last_verified_at = Column(DateTime, nullable=True)
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
# Relationships
user = relationship("User", backref="gmail_credential")
+43
View File
@@ -29,6 +29,11 @@ class AccountStatus(str, Enum):
TESTING = "testing"
class DeliveryMethod(str, Enum):
SMTP = "smtp"
GMAIL_API = "gmail_api"
class NotificationChannel(str, Enum):
EMAIL = "email"
TELEGRAM = "telegram"
@@ -103,6 +108,7 @@ class MailAccountBase(BaseModel):
use_tls: bool = False
username: str = Field(..., max_length=255)
forward_to: EmailStr
delivery_method: DeliveryMethod = DeliveryMethod.GMAIL_API
is_enabled: bool = True
check_interval_minutes: int = Field(default=5, gt=0, le=1440)
max_emails_per_check: int = Field(default=50, gt=0, le=1000)
@@ -117,6 +123,7 @@ class MailAccountUpdate(BaseModel):
name: Optional[str] = Field(None, max_length=255)
password: Optional[str] = None
forward_to: Optional[EmailStr] = None
delivery_method: Optional[DeliveryMethod] = None
is_enabled: Optional[bool] = None
check_interval_minutes: Optional[int] = Field(None, gt=0, le=1440)
max_emails_per_check: Optional[int] = Field(None, gt=0, le=1000)
@@ -127,6 +134,7 @@ class MailAccountResponse(MailAccountBase):
id: int
user_id: int
status: AccountStatus
delivery_method: DeliveryMethod
provider_name: Optional[str] = None
auto_detected: bool
total_emails_processed: int
@@ -302,3 +310,38 @@ class MailServerPresetResponse(BaseModel):
class Config:
from_attributes = True
# Gmail Credential Schemas
class GmailCredentialCreate(BaseModel):
access_token: str
refresh_token: Optional[str] = None
gmail_email: EmailStr
class GmailCredentialResponse(BaseModel):
id: int
user_id: int
gmail_email: str
is_valid: bool
last_verified_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
class Config:
from_attributes = True
# Provider Wizard Schemas
class ProviderPreset(BaseModel):
id: str
name: str
icon: Optional[str] = None
domains: List[str]
imap_ssl: Optional[Dict[str, Any]] = None
pop3_ssl: Optional[Dict[str, Any]] = None
notes: Optional[str] = None
class ProviderListResponse(BaseModel):
providers: List[ProviderPreset]
+187
View File
@@ -0,0 +1,187 @@
"""
Gmail API service for injecting emails directly into Gmail.
Uses the Gmail API's users.messages.insert() method to inject emails
into a user's Gmail account, preserving original headers and metadata.
This is preferred over SMTP forwarding as it doesn't modify the email.
"""
import base64
import logging
from typing import Optional, Dict, Any
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
logger = logging.getLogger(__name__)
# Gmail API scopes needed for email injection
GMAIL_SCOPES = [
"https://www.googleapis.com/auth/gmail.insert",
"https://www.googleapis.com/auth/gmail.labels",
]
class GmailInjectionError(Exception):
"""Raised when Gmail API injection fails"""
pass
class GmailService:
"""
Service for injecting emails into Gmail via the Gmail API.
Uses users.messages.insert() which places emails directly into
the user's mailbox without sending them through SMTP.
"""
def __init__(
self,
access_token: str,
refresh_token: Optional[str] = None,
token_uri: str = "https://oauth2.googleapis.com/token",
client_id: Optional[str] = None,
client_secret: Optional[str] = None,
):
"""
Initialize Gmail service with OAuth2 credentials.
Args:
access_token: Valid OAuth2 access token
refresh_token: OAuth2 refresh token for automatic renewal
token_uri: OAuth2 token endpoint
client_id: Google OAuth2 client ID
client_secret: Google OAuth2 client secret
"""
self.credentials = Credentials(
token=access_token,
refresh_token=refresh_token,
token_uri=token_uri,
client_id=client_id,
client_secret=client_secret,
scopes=GMAIL_SCOPES,
)
self._service = None
@property
def service(self):
"""Lazy-initialize the Gmail API service."""
if self._service is None:
self._service = build("gmail", "v1", credentials=self.credentials)
return self._service
async def inject_email(
self,
raw_email: bytes,
label_ids: Optional[list] = None,
source_account_name: Optional[str] = None,
) -> Dict[str, Any]:
"""
Inject a raw email into the user's Gmail account.
Uses users.messages.insert() to place the email directly
into the mailbox. The email appears as if it was received
normally, preserving all original headers.
Args:
raw_email: Raw email bytes (RFC 2822 format)
label_ids: Gmail label IDs to apply (defaults to ["INBOX"])
source_account_name: Optional name for logging
Returns:
Dict with message id and thread id
Raises:
GmailInjectionError: If injection fails
"""
import asyncio
if label_ids is None:
label_ids = ["INBOX"]
# Base64url encode the raw email
encoded_message = base64.urlsafe_b64encode(raw_email).decode("utf-8")
message_body = {
"raw": encoded_message,
"labelIds": label_ids,
}
loop = asyncio.get_event_loop()
try:
result = await loop.run_in_executor(
None,
lambda: self.service.users()
.messages()
.insert(userId="me", body=message_body)
.execute(),
)
logger.info(
f"Injected email into Gmail: id={result.get('id')}"
f"{f' from {source_account_name}' if source_account_name else ''}"
)
return {
"message_id": result.get("id"),
"thread_id": result.get("threadId"),
"label_ids": result.get("labelIds", []),
}
except HttpError as e:
error_msg = f"Gmail API error: {e.reason if hasattr(e, 'reason') else str(e)}"
logger.error(error_msg)
raise GmailInjectionError(error_msg)
except Exception as e:
error_msg = f"Failed to inject email into Gmail: {str(e)}"
logger.error(error_msg)
raise GmailInjectionError(error_msg)
async def verify_access(self) -> bool:
"""
Verify that the Gmail API credentials are valid.
Returns:
True if credentials are valid and can access Gmail
"""
import asyncio
loop = asyncio.get_event_loop()
try:
result = await loop.run_in_executor(
None,
lambda: self.service.users()
.getProfile(userId="me")
.execute(),
)
email = result.get("emailAddress", "unknown")
logger.info(f"Gmail API access verified for: {email}")
return True
except Exception as e:
logger.error(f"Gmail API access verification failed: {e}")
return False
async def get_email_address(self) -> Optional[str]:
"""
Get the email address associated with the Gmail credentials.
Returns:
Email address string or None if unavailable
"""
import asyncio
loop = asyncio.get_event_loop()
try:
result = await loop.run_in_executor(
None,
lambda: self.service.users()
.getProfile(userId="me")
.execute(),
)
return result.get("emailAddress")
except Exception as e:
logger.error(f"Failed to get Gmail email address: {e}")
return None
+107 -2
View File
@@ -395,6 +395,11 @@ class MailServerAutoDetect:
"pop3_ssl": {"host": "pop.gmail.com", "port": 995},
"imap_ssl": {"host": "imap.gmail.com", "port": 993},
},
"googlemail.com": {
"name": "Gmail",
"pop3_ssl": {"host": "pop.gmail.com", "port": 995},
"imap_ssl": {"host": "imap.gmail.com", "port": 993},
},
"outlook.com": {
"name": "Outlook.com",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
@@ -405,6 +410,21 @@ class MailServerAutoDetect:
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"live.com": {
"name": "Live",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"msn.com": {
"name": "MSN",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"outlook.de": {
"name": "Outlook.de",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"gmx.com": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.com", "port": 995},
@@ -415,6 +435,21 @@ class MailServerAutoDetect:
"pop3_ssl": {"host": "pop.gmx.net", "port": 995},
"imap_ssl": {"host": "imap.gmx.net", "port": 993},
},
"gmx.net": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.net", "port": 995},
"imap_ssl": {"host": "imap.gmx.net", "port": 993},
},
"gmx.at": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.net", "port": 995},
"imap_ssl": {"host": "imap.gmx.net", "port": 993},
},
"gmx.ch": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.net", "port": 995},
"imap_ssl": {"host": "imap.gmx.net", "port": 993},
},
"web.de": {
"name": "WEB.DE",
"pop3_ssl": {"host": "pop3.web.de", "port": 995},
@@ -422,14 +457,84 @@ class MailServerAutoDetect:
},
"t-online.de": {
"name": "T-Online",
"pop3_ssl": {"host": "pop.t-online.de", "port": 995},
"imap_ssl": {"host": "imap.t-online.de", "port": 993},
"pop3_ssl": {"host": "securepop.t-online.de", "port": 995},
"imap_ssl": {"host": "secureimap.t-online.de", "port": 993},
},
"yahoo.com": {
"name": "Yahoo",
"pop3_ssl": {"host": "pop.mail.yahoo.com", "port": 995},
"imap_ssl": {"host": "imap.mail.yahoo.com", "port": 993},
},
"yahoo.de": {
"name": "Yahoo",
"pop3_ssl": {"host": "pop.mail.yahoo.com", "port": 995},
"imap_ssl": {"host": "imap.mail.yahoo.com", "port": 993},
},
"yahoo.co.uk": {
"name": "Yahoo",
"pop3_ssl": {"host": "pop.mail.yahoo.com", "port": 995},
"imap_ssl": {"host": "imap.mail.yahoo.com", "port": 993},
},
"ymail.com": {
"name": "Yahoo",
"pop3_ssl": {"host": "pop.mail.yahoo.com", "port": 995},
"imap_ssl": {"host": "imap.mail.yahoo.com", "port": 993},
},
"aol.com": {
"name": "AOL",
"pop3_ssl": {"host": "pop.aol.com", "port": 995},
"imap_ssl": {"host": "imap.aol.com", "port": 993},
},
"aim.com": {
"name": "AOL",
"pop3_ssl": {"host": "pop.aol.com", "port": 995},
"imap_ssl": {"host": "imap.aol.com", "port": 993},
},
"online.de": {
"name": "1&1 / IONOS",
"pop3_ssl": {"host": "pop.ionos.de", "port": 995},
"imap_ssl": {"host": "imap.ionos.de", "port": 993},
},
"onlinehome.de": {
"name": "1&1 / IONOS",
"pop3_ssl": {"host": "pop.ionos.de", "port": 995},
"imap_ssl": {"host": "imap.ionos.de", "port": 993},
},
"1und1.de": {
"name": "1&1 / IONOS",
"pop3_ssl": {"host": "pop.ionos.de", "port": 995},
"imap_ssl": {"host": "imap.ionos.de", "port": 993},
},
"freenet.de": {
"name": "Freenet",
"pop3_ssl": {"host": "mx.freenet.de", "port": 995},
"imap_ssl": {"host": "mx.freenet.de", "port": 993},
},
"posteo.de": {
"name": "Posteo",
"imap_ssl": {"host": "posteo.de", "port": 993},
},
"posteo.net": {
"name": "Posteo",
"imap_ssl": {"host": "posteo.de", "port": 993},
},
"icloud.com": {
"name": "iCloud",
"imap_ssl": {"host": "imap.mail.me.com", "port": 993},
},
"me.com": {
"name": "iCloud",
"imap_ssl": {"host": "imap.mail.me.com", "port": 993},
},
"mac.com": {
"name": "iCloud",
"imap_ssl": {"host": "imap.mail.me.com", "port": 993},
},
"mail.de": {
"name": "mail.de",
"pop3_ssl": {"host": "pop.mail.de", "port": 995},
"imap_ssl": {"host": "imap.mail.de", "port": 993},
},
}
@classmethod
+79 -27
View File
@@ -11,8 +11,13 @@ import logging
from app.workers.celery_app import celery_app
from app.core.database import async_session_maker
from app.core.security import decrypt_credential
from app.models.database_models import MailAccount, ProcessingRun, ProcessingLog, AccountStatus
from app.models.database_models import (
MailAccount, ProcessingRun, ProcessingLog, AccountStatus,
DeliveryMethod, GmailCredential,
)
from app.services.mail_processor import MailProcessor
from app.services.gmail_service import GmailService, GmailInjectionError
from app.core.config import settings
from sqlalchemy import select, and_
from sqlalchemy.ext.asyncio import AsyncSession
@@ -73,39 +78,86 @@ async def process_mail_account(account_id: int):
emails_forwarded = 0
emails_failed = 0
# Get SMTP config from environment or user settings
# TODO: Make this configurable per user in the database
smtp_config = {
"host": os.getenv("SMTP_HOST", "smtp.gmail.com"),
"port": int(os.getenv("SMTP_PORT", "587")),
"username": os.getenv("SMTP_USER", ""),
"password": os.getenv("SMTP_PASSWORD", ""),
"use_tls": os.getenv("SMTP_USE_TLS", "true").lower() == "true"
}
# Determine delivery method
use_gmail_api = (
account.delivery_method == DeliveryMethod.GMAIL_API
)
if not smtp_config["username"] or not smtp_config["password"]:
logger.error(f"SMTP credentials not configured for account {account.id}")
run.status = "failed"
run.error_message = "SMTP credentials not configured"
await db.commit()
return
gmail_service = None
smtp_config = None
if use_gmail_api:
# Get user's Gmail credentials
gmail_cred_result = await db.execute(
select(GmailCredential).where(
GmailCredential.user_id == account.user_id,
GmailCredential.is_valid == True,
)
)
gmail_cred = gmail_cred_result.scalar_one_or_none()
if gmail_cred:
access_token = decrypt_credential(gmail_cred.encrypted_access_token)
refresh_token = (
decrypt_credential(gmail_cred.encrypted_refresh_token)
if gmail_cred.encrypted_refresh_token
else None
)
gmail_service = GmailService(
access_token=access_token,
refresh_token=refresh_token,
client_id=settings.GOOGLE_CLIENT_ID,
client_secret=settings.GOOGLE_CLIENT_SECRET,
)
else:
logger.warning(
f"Gmail API credentials not found for user {account.user_id}, "
f"falling back to SMTP for account {account.id}"
)
use_gmail_api = False
if not use_gmail_api:
# Fall back to SMTP
smtp_config = {
"host": os.getenv("SMTP_HOST", "smtp.gmail.com"),
"port": int(os.getenv("SMTP_PORT", "587")),
"username": os.getenv("SMTP_USER", ""),
"password": os.getenv("SMTP_PASSWORD", ""),
"use_tls": os.getenv("SMTP_USE_TLS", "true").lower() == "true"
}
if not smtp_config["username"] or not smtp_config["password"]:
logger.error(f"SMTP credentials not configured for account {account.id}")
run.status = "failed"
run.error_message = "No delivery method configured (SMTP credentials missing and Gmail API not set up)"
await db.commit()
return
for email_data in emails:
try:
success = await MailProcessor.forward_email(
email_data,
account.name,
account.forward_to,
smtp_config
)
if success:
if use_gmail_api and gmail_service:
# Inject via Gmail API (preferred)
await gmail_service.inject_email(
raw_email=email_data,
label_ids=["INBOX"],
source_account_name=account.name,
)
emails_forwarded += 1
else:
emails_failed += 1
# Forward via SMTP (fallback)
success = await MailProcessor.forward_email(
email_data,
account.name,
account.forward_to,
smtp_config
)
if success:
emails_forwarded += 1
else:
emails_failed += 1
except Exception as e:
logger.error(f"Error forwarding email: {e}")
except (GmailInjectionError, Exception) as e:
logger.error(f"Error delivering email: {e}")
emails_failed += 1
# Update run
+6
View File
@@ -26,6 +26,12 @@ aiohttp==3.13.3 # Updated: Fixed zip bomb, DoS, and directory traversal vulnera
aioimaplib==1.0.1
email-validator==2.1.0.post1
# Gmail API (for direct email injection)
google-api-python-client==2.193.0
google-auth==2.49.1
google-auth-oauthlib==1.2.0
google-auth-httplib2==0.2.0
# Job Queue & Cache
celery==5.3.6
redis==5.0.1
+152
View File
@@ -0,0 +1,152 @@
"""
Unit tests for Gmail service module.
"""
import pytest
from unittest.mock import MagicMock, patch, AsyncMock
from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES
class TestGmailService:
"""Test Gmail API service"""
def test_gmail_scopes(self):
"""Test that required Gmail scopes are defined"""
assert "https://www.googleapis.com/auth/gmail.insert" in GMAIL_SCOPES
assert "https://www.googleapis.com/auth/gmail.labels" in GMAIL_SCOPES
def test_init_creates_credentials(self):
"""Test that GmailService initializes with credentials"""
service = GmailService(
access_token="test-access-token",
refresh_token="test-refresh-token",
client_id="test-client-id",
client_secret="test-client-secret",
)
assert service.credentials is not None
assert service.credentials.token == "test-access-token"
assert service.credentials.refresh_token == "test-refresh-token"
assert service.credentials.client_id == "test-client-id"
assert service.credentials.client_secret == "test-client-secret"
def test_init_without_refresh_token(self):
"""Test initialization without refresh token"""
service = GmailService(access_token="test-access-token")
assert service.credentials is not None
assert service.credentials.token == "test-access-token"
assert service.credentials.refresh_token is None
def test_service_lazy_initialization(self):
"""Test that the API service is not created until accessed"""
service = GmailService(access_token="test-access-token")
assert service._service is None
@pytest.mark.asyncio
async def test_inject_email_success(self):
"""Test successful email injection"""
service = GmailService(access_token="test-access-token")
mock_api = MagicMock()
mock_api.users().messages().insert().execute.return_value = {
"id": "msg123",
"threadId": "thread456",
"labelIds": ["INBOX"],
}
service._service = mock_api
result = await service.inject_email(
raw_email=b"From: test@example.com\r\nSubject: Test\r\n\r\nHello",
label_ids=["INBOX"],
source_account_name="Test Account",
)
assert result["message_id"] == "msg123"
assert result["thread_id"] == "thread456"
assert "INBOX" in result["label_ids"]
@pytest.mark.asyncio
async def test_inject_email_default_labels(self):
"""Test that INBOX is used as default label"""
service = GmailService(access_token="test-access-token")
mock_api = MagicMock()
mock_api.users().messages().insert().execute.return_value = {
"id": "msg123",
"threadId": "thread456",
"labelIds": ["INBOX"],
}
service._service = mock_api
# No label_ids specified - should default to INBOX
result = await service.inject_email(
raw_email=b"From: test@example.com\r\nSubject: Test\r\n\r\nHello",
)
assert result["message_id"] == "msg123"
@pytest.mark.asyncio
async def test_inject_email_api_error(self):
"""Test that GmailInjectionError is raised on API error"""
service = GmailService(access_token="test-access-token")
mock_api = MagicMock()
mock_api.users().messages().insert().execute.side_effect = Exception("API Error")
service._service = mock_api
with pytest.raises(GmailInjectionError, match="Failed to inject email"):
await service.inject_email(
raw_email=b"From: test@example.com\r\nSubject: Test\r\n\r\nHello",
)
@pytest.mark.asyncio
async def test_verify_access_success(self):
"""Test successful access verification"""
service = GmailService(access_token="test-access-token")
mock_api = MagicMock()
mock_api.users().getProfile().execute.return_value = {
"emailAddress": "test@gmail.com",
}
service._service = mock_api
result = await service.verify_access()
assert result is True
@pytest.mark.asyncio
async def test_verify_access_failure(self):
"""Test failed access verification"""
service = GmailService(access_token="bad-token")
mock_api = MagicMock()
mock_api.users().getProfile().execute.side_effect = Exception("Invalid token")
service._service = mock_api
result = await service.verify_access()
assert result is False
@pytest.mark.asyncio
async def test_get_email_address_success(self):
"""Test getting email address"""
service = GmailService(access_token="test-access-token")
mock_api = MagicMock()
mock_api.users().getProfile().execute.return_value = {
"emailAddress": "user@gmail.com",
}
service._service = mock_api
email = await service.get_email_address()
assert email == "user@gmail.com"
@pytest.mark.asyncio
async def test_get_email_address_failure(self):
"""Test getting email address when API fails"""
service = GmailService(access_token="bad-token")
mock_api = MagicMock()
mock_api.users().getProfile().execute.side_effect = Exception("Error")
service._service = mock_api
email = await service.get_email_address()
assert email is None
+216
View File
@@ -0,0 +1,216 @@
"""
Unit tests for provider presets and mail server auto-detection.
"""
import pytest
from app.services.mail_processor import MailServerAutoDetect
class TestMailServerAutoDetect:
"""Test mail server auto-detection with expanded provider list"""
def test_detect_gmail(self):
"""Test Gmail auto-detection"""
suggestions = MailServerAutoDetect.detect("user@gmail.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "pop.gmail.com" in hosts or "imap.gmail.com" in hosts
def test_detect_googlemail(self):
"""Test googlemail.com auto-detection"""
suggestions = MailServerAutoDetect.detect("user@googlemail.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.gmail.com" in hosts
def test_detect_gmx_de(self):
"""Test GMX.de auto-detection"""
suggestions = MailServerAutoDetect.detect("user@gmx.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.gmx.net" in hosts
def test_detect_gmx_net(self):
"""Test GMX.net auto-detection"""
suggestions = MailServerAutoDetect.detect("user@gmx.net")
assert len(suggestions) > 0
def test_detect_webde(self):
"""Test WEB.DE auto-detection"""
suggestions = MailServerAutoDetect.detect("user@web.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.web.de" in hosts
def test_detect_outlook(self):
"""Test Outlook.com auto-detection"""
suggestions = MailServerAutoDetect.detect("user@outlook.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "outlook.office365.com" in hosts
def test_detect_hotmail(self):
"""Test Hotmail auto-detection"""
suggestions = MailServerAutoDetect.detect("user@hotmail.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "outlook.office365.com" in hosts
def test_detect_yahoo(self):
"""Test Yahoo auto-detection"""
suggestions = MailServerAutoDetect.detect("user@yahoo.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.mail.yahoo.com" in hosts
def test_detect_aol(self):
"""Test AOL auto-detection"""
suggestions = MailServerAutoDetect.detect("user@aol.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.aol.com" in hosts
def test_detect_tonline(self):
"""Test T-Online auto-detection"""
suggestions = MailServerAutoDetect.detect("user@t-online.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "secureimap.t-online.de" in hosts
def test_detect_ionos(self):
"""Test 1&1/IONOS auto-detection"""
suggestions = MailServerAutoDetect.detect("user@online.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.ionos.de" in hosts
def test_detect_freenet(self):
"""Test Freenet auto-detection"""
suggestions = MailServerAutoDetect.detect("user@freenet.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "mx.freenet.de" in hosts
def test_detect_posteo(self):
"""Test Posteo auto-detection (IMAP only)"""
suggestions = MailServerAutoDetect.detect("user@posteo.de")
assert len(suggestions) > 0
# Posteo only has IMAP
protocols = [s["protocol"] for s in suggestions]
assert "imap_ssl" in protocols
def test_detect_icloud(self):
"""Test iCloud auto-detection"""
suggestions = MailServerAutoDetect.detect("user@icloud.com")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.mail.me.com" in hosts
def test_detect_unknown_domain(self):
"""Test auto-detection for unknown domain"""
suggestions = MailServerAutoDetect.detect("user@unknowndomain123.com")
assert len(suggestions) > 0
# Should return generic suggestions
providers = set(s["provider_name"] for s in suggestions)
assert "Generic" in providers
def test_detect_case_insensitive(self):
"""Test that domain detection is case-insensitive"""
suggestions_lower = MailServerAutoDetect.detect("user@Gmail.com")
suggestions_upper = MailServerAutoDetect.detect("user@GMAIL.COM")
# Both should detect as Gmail
assert len(suggestions_lower) > 0
assert len(suggestions_upper) > 0
def test_all_suggestions_have_required_fields(self):
"""Test that all suggestions have the required fields"""
for domain in ["gmail.com", "gmx.de", "web.de", "yahoo.com", "aol.com"]:
suggestions = MailServerAutoDetect.detect(f"user@{domain}")
for suggestion in suggestions:
assert "protocol" in suggestion
assert "host" in suggestion
assert "port" in suggestion
assert "provider_name" in suggestion
assert "use_ssl" in suggestion
def test_detect_live_com(self):
"""Test Live.com auto-detection (Microsoft)"""
suggestions = MailServerAutoDetect.detect("user@live.com")
assert len(suggestions) > 0
def test_detect_ymail(self):
"""Test ymail.com auto-detection (Yahoo)"""
suggestions = MailServerAutoDetect.detect("user@ymail.com")
assert len(suggestions) > 0
def test_detect_mailde(self):
"""Test mail.de auto-detection"""
suggestions = MailServerAutoDetect.detect("user@mail.de")
assert len(suggestions) > 0
hosts = [s["host"] for s in suggestions]
assert "imap.mail.de" in hosts
class TestProviderPresets:
"""Test that provider presets module defines correct values"""
def test_provider_presets_import(self):
"""Test that provider presets can be imported"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
assert len(PROVIDER_PRESETS) > 0
def test_all_presets_have_required_fields(self):
"""Test that all presets have required fields"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
for preset in PROVIDER_PRESETS:
assert preset.id
assert preset.name
assert len(preset.domains) > 0
# Must have at least one protocol
assert preset.imap_ssl is not None or preset.pop3_ssl is not None
def test_gmail_preset_exists(self):
"""Test that Gmail preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
gmail = next((p for p in PROVIDER_PRESETS if p.id == "gmail"), None)
assert gmail is not None
assert gmail.imap_ssl is not None
assert gmail.imap_ssl["host"] == "imap.gmail.com"
def test_gmx_preset_exists(self):
"""Test that GMX preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
gmx = next((p for p in PROVIDER_PRESETS if p.id == "gmx"), None)
assert gmx is not None
assert "gmx.de" in gmx.domains
def test_webde_preset_exists(self):
"""Test that WEB.DE preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
webde = next((p for p in PROVIDER_PRESETS if p.id == "webde"), None)
assert webde is not None
assert "web.de" in webde.domains
def test_outlook_preset_exists(self):
"""Test that Outlook preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
outlook = next((p for p in PROVIDER_PRESETS if p.id == "outlook"), None)
assert outlook is not None
assert "hotmail.com" in outlook.domains
def test_yahoo_preset_exists(self):
"""Test that Yahoo preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
yahoo = next((p for p in PROVIDER_PRESETS if p.id == "yahoo"), None)
assert yahoo is not None
def test_aol_preset_exists(self):
"""Test that AOL preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
aol = next((p for p in PROVIDER_PRESETS if p.id == "aol"), None)
assert aol is not None
def test_tonline_preset_exists(self):
"""Test that T-Online preset is included"""
from app.api.v1.endpoints.providers import PROVIDER_PRESETS
tonline = next((p for p in PROVIDER_PRESETS if p.id == "tonline"), None)
assert tonline is not None