Add backend foundation: database models, security, and mail processing service

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-01 12:44:22 +00:00
parent d128e34aa7
commit 130b53d37f
15 changed files with 1597 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""App package"""
+1
View File
@@ -0,0 +1 @@
"""API package"""
+1
View File
@@ -0,0 +1 @@
"""Core package"""
+98
View File
@@ -0,0 +1,98 @@
"""
Application configuration using Pydantic settings.
Supports environment variables and .env files.
"""
from typing import Optional, List
from pydantic_settings import BaseSettings, SettingsConfigDict
from pydantic import PostgresDsn, field_validator, ValidationInfo
class Settings(BaseSettings):
"""Application settings loaded from environment variables"""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore"
)
# Application
APP_NAME: str = "POP3 Forwarder SaaS"
APP_VERSION: str = "2.0.0"
DEBUG: bool = False
API_V1_PREFIX: str = "/api/v1"
# Server
HOST: str = "0.0.0.0"
PORT: int = 8000
# Database
DATABASE_URL: str = "postgresql+asyncpg://user:password@localhost:5432/pop3_forwarder"
DATABASE_POOL_SIZE: int = 20
DATABASE_MAX_OVERFLOW: int = 10
# Security
SECRET_KEY: str = "change-this-to-a-secure-random-secret-key-in-production"
ALGORITHM: str = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
REFRESH_TOKEN_EXPIRE_DAYS: int = 7
# Encryption (for storing POP3/IMAP credentials)
ENCRYPTION_KEY: str = "change-this-to-a-secure-encryption-key"
# OAuth2 - Google
GOOGLE_CLIENT_ID: Optional[str] = None
GOOGLE_CLIENT_SECRET: Optional[str] = None
GOOGLE_REDIRECT_URI: str = "http://localhost:3000/auth/callback/google"
# CORS
CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"]
# Stripe Payment
STRIPE_API_KEY: Optional[str] = None
STRIPE_WEBHOOK_SECRET: Optional[str] = None
STRIPE_PUBLISHABLE_KEY: Optional[str] = None
# Subscription Tiers
TIER_FREE_MAX_ACCOUNTS: int = 1
TIER_BASIC_MAX_ACCOUNTS: int = 5
TIER_PRO_MAX_ACCOUNTS: int = 20
TIER_ENTERPRISE_MAX_ACCOUNTS: int = 100
# Email Processing
MAX_EMAILS_PER_RUN: int = 50
CHECK_INTERVAL_MINUTES: int = 5
THROTTLE_EMAILS_PER_MINUTE: int = 10
# Redis (for Celery and caching)
REDIS_URL: str = "redis://localhost:6379/0"
# Celery
CELERY_BROKER_URL: str = "redis://localhost:6379/0"
CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
# Apprise (notifications)
APPRISE_ENABLED: bool = True
# Logging
LOG_LEVEL: str = "INFO"
# Admin
ADMIN_EMAIL: Optional[str] = None
ADMIN_PASSWORD: Optional[str] = None
# Mail Server Presets
MAIL_SERVER_PRESETS_FILE: str = "app/data/mail_server_presets.json"
@field_validator("CORS_ORIGINS", mode="before")
@classmethod
def assemble_cors_origins(cls, v: str | List[str]) -> List[str]:
"""Parse CORS origins from environment variable"""
if isinstance(v, str):
return [i.strip() for i in v.split(",")]
return v
# Global settings instance
settings = Settings()
+40
View File
@@ -0,0 +1,40 @@
"""
Database configuration and session management.
"""
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import declarative_base
from app.core.config import settings
# Create async engine
engine = create_async_engine(
settings.DATABASE_URL,
echo=settings.DEBUG,
pool_size=settings.DATABASE_POOL_SIZE,
max_overflow=settings.DATABASE_MAX_OVERFLOW,
pool_pre_ping=True,
)
# Create async session factory
async_session_maker = async_sessionmaker(
engine,
class_=AsyncSession,
expire_on_commit=False,
autocommit=False,
autoflush=False,
)
# Base class for models
Base = declarative_base()
async def get_db() -> AsyncSession:
"""Dependency for getting async database session"""
async with async_session_maker() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
+133
View File
@@ -0,0 +1,133 @@
"""
Authentication dependencies for FastAPI.
"""
from typing import Optional
from fastapi import Depends, HTTPException, status
from fastapi.security import OAuth2PasswordBearer, HTTPBearer, HTTPAuthorizationCredentials
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.core.database import get_db
from app.core.security import decode_token
from app.models.database_models import User
# OAuth2 scheme for token authentication
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f"/api/v1/auth/login", auto_error=False)
http_bearer = HTTPBearer(auto_error=False)
async def get_current_user(
token: Optional[str] = Depends(oauth2_scheme),
credentials: Optional[HTTPAuthorizationCredentials] = Depends(http_bearer),
db: AsyncSession = Depends(get_db)
) -> User:
"""
Get current authenticated user from JWT token.
Supports both OAuth2 password bearer and HTTP Bearer authentication.
"""
# Get token from either source
auth_token = token or (credentials.credentials if credentials else None)
if not auth_token:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
# Decode token
payload = decode_token(auth_token)
if not payload:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authentication credentials",
headers={"WWW-Authenticate": "Bearer"},
)
# Verify token type
token_type = payload.get("type")
if token_type != "access":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token type",
headers={"WWW-Authenticate": "Bearer"},
)
# Get user ID from token
user_id: Optional[int] = payload.get("sub")
if user_id is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token payload",
headers={"WWW-Authenticate": "Bearer"},
)
# Fetch user from database
result = await db.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if user is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found",
headers={"WWW-Authenticate": "Bearer"},
)
if not user.is_active:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="User account is inactive"
)
return user
async def get_current_active_user(
current_user: User = Depends(get_current_user),
) -> User:
"""Get current active user"""
if not current_user.is_active:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Inactive user"
)
return current_user
async def get_current_superuser(
current_user: User = Depends(get_current_user),
) -> User:
"""Get current superuser"""
if not current_user.is_superuser:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not enough permissions"
)
return current_user
def check_subscription_tier(required_tier: str):
"""
Dependency factory to check if user has required subscription tier.
Returns a dependency function.
"""
tier_hierarchy = {
"free": 0,
"basic": 1,
"pro": 2,
"enterprise": 3
}
async def check_tier(current_user: User = Depends(get_current_active_user)) -> User:
user_tier_level = tier_hierarchy.get(current_user.subscription_tier.value, 0)
required_tier_level = tier_hierarchy.get(required_tier, 0)
if user_tier_level < required_tier_level:
raise HTTPException(
status_code=status.HTTP_402_PAYMENT_REQUIRED,
detail=f"This feature requires {required_tier} subscription or higher"
)
return current_user
return check_tier
+113
View File
@@ -0,0 +1,113 @@
"""
Security utilities for encryption, hashing, and token generation.
"""
import secrets
from datetime import datetime, timedelta
from typing import Optional, Dict, Any
from jose import JWTError, jwt
from passlib.context import CryptContext
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2
import base64
from app.core.config import settings
# Password hashing context
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
def verify_password(plain_password: str, hashed_password: str) -> bool:
"""Verify a password against its hash"""
return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password: str) -> str:
"""Generate password hash"""
return pwd_context.hash(password)
def create_access_token(data: Dict[str, Any], expires_delta: Optional[timedelta] = None) -> str:
"""Create JWT access token"""
to_encode = data.copy()
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
to_encode.update({"exp": expire, "type": "access"})
encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
return encoded_jwt
def create_refresh_token(data: Dict[str, Any]) -> str:
"""Create JWT refresh token"""
to_encode = data.copy()
expire = datetime.utcnow() + timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
to_encode.update({"exp": expire, "type": "refresh"})
encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
return encoded_jwt
def decode_token(token: str) -> Optional[Dict[str, Any]]:
"""Decode and validate JWT token"""
try:
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
return payload
except JWTError:
return None
def generate_random_token(length: int = 32) -> str:
"""Generate a secure random token"""
return secrets.token_urlsafe(length)
class CredentialEncryption:
"""Handles encryption/decryption of sensitive credentials (POP3/IMAP passwords)"""
def __init__(self, key: Optional[str] = None):
"""
Initialize encryption with a key.
If no key provided, uses the one from settings.
"""
if key is None:
key = settings.ENCRYPTION_KEY
# Derive a proper Fernet key from the provided key
kdf = PBKDF2(
algorithm=hashes.SHA256(),
length=32,
salt=b'pop3_forwarder_salt', # In production, use unique salt per user
iterations=100000,
)
key_bytes = key.encode('utf-8')
derived_key = base64.urlsafe_b64encode(kdf.derive(key_bytes))
self.fernet = Fernet(derived_key)
def encrypt(self, plain_text: str) -> str:
"""Encrypt a string and return base64-encoded ciphertext"""
encrypted = self.fernet.encrypt(plain_text.encode('utf-8'))
return base64.b64encode(encrypted).decode('utf-8')
def decrypt(self, encrypted_text: str) -> str:
"""Decrypt a base64-encoded ciphertext"""
encrypted_bytes = base64.b64decode(encrypted_text.encode('utf-8'))
decrypted = self.fernet.decrypt(encrypted_bytes)
return decrypted.decode('utf-8')
# Global encryption instance
credential_encryptor = CredentialEncryption()
def encrypt_credential(credential: str) -> str:
"""Convenience function to encrypt a credential"""
return credential_encryptor.encrypt(credential)
def decrypt_credential(encrypted_credential: str) -> str:
"""Convenience function to decrypt a credential"""
return credential_encryptor.decrypt(encrypted_credential)
+12
View File
@@ -0,0 +1,12 @@
"""Models package"""
from app.models.database_models import (
User, MailAccount, ProcessingRun, ProcessingLog,
NotificationConfig, MailServerPreset, SubscriptionPlan, AuditLog,
SubscriptionTier, MailProtocol, AccountStatus, NotificationChannel
)
__all__ = [
"User", "MailAccount", "ProcessingRun", "ProcessingLog",
"NotificationConfig", "MailServerPreset", "SubscriptionPlan", "AuditLog",
"SubscriptionTier", "MailProtocol", "AccountStatus", "NotificationChannel"
]
+328
View File
@@ -0,0 +1,328 @@
"""
Database models for the multi-tenant POP3 forwarder application.
"""
from datetime import datetime
from typing import Optional
from sqlalchemy import (
Column, Integer, String, Boolean, DateTime, ForeignKey,
Text, Enum as SQLEnum, JSON, Float, Index
)
from sqlalchemy.orm import relationship
import enum
from app.core.database import Base
class SubscriptionTier(str, enum.Enum):
"""Subscription tier levels"""
FREE = "free"
BASIC = "basic"
PRO = "pro"
ENTERPRISE = "enterprise"
class MailProtocol(str, enum.Enum):
"""Supported mail protocols"""
POP3 = "pop3"
POP3_SSL = "pop3_ssl"
IMAP = "imap"
IMAP_SSL = "imap_ssl"
class AccountStatus(str, enum.Enum):
"""Mail account status"""
ACTIVE = "active"
INACTIVE = "inactive"
ERROR = "error"
TESTING = "testing"
class NotificationChannel(str, enum.Enum):
"""Notification channel types"""
EMAIL = "email"
TELEGRAM = "telegram"
WEBHOOK = "webhook"
SLACK = "slack"
DISCORD = "discord"
class User(Base):
"""User model - represents a user account"""
__tablename__ = "users"
id = Column(Integer, primary_key=True, index=True)
email = Column(String(255), unique=True, index=True, nullable=False)
hashed_password = Column(String(255), nullable=True) # Nullable for OAuth-only users
full_name = Column(String(255))
is_active = Column(Boolean, default=True)
is_superuser = Column(Boolean, default=False)
# OAuth
google_id = Column(String(255), unique=True, index=True, nullable=True)
oauth_provider = Column(String(50), nullable=True)
# Subscription
subscription_tier = Column(SQLEnum(SubscriptionTier), default=SubscriptionTier.FREE)
subscription_status = Column(String(50), default="active") # active, canceled, past_due
stripe_customer_id = Column(String(255), unique=True, nullable=True)
stripe_subscription_id = Column(String(255), unique=True, nullable=True)
subscription_expires_at = Column(DateTime, nullable=True)
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
last_login_at = Column(DateTime, nullable=True)
# Relationships
mail_accounts = relationship("MailAccount", back_populates="user", cascade="all, delete-orphan")
notifications = relationship("NotificationConfig", back_populates="user", cascade="all, delete-orphan")
logs = relationship("ProcessingLog", back_populates="user", cascade="all, delete-orphan")
class MailAccount(Base):
"""Mail account configuration (POP3/IMAP)"""
__tablename__ = "mail_accounts"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
# Account details
name = Column(String(255), nullable=False) # User-friendly name
email_address = Column(String(255), nullable=False)
# Server configuration
protocol = Column(SQLEnum(MailProtocol), default=MailProtocol.POP3_SSL)
host = Column(String(255), nullable=False)
port = Column(Integer, nullable=False)
use_ssl = Column(Boolean, default=True)
use_tls = Column(Boolean, default=False)
# Credentials (encrypted)
username = Column(String(255), nullable=False)
encrypted_password = Column(Text, nullable=False)
# Forwarding destination
forward_to = Column(String(255), nullable=False)
# Status and settings
status = Column(SQLEnum(AccountStatus), default=AccountStatus.ACTIVE)
is_enabled = Column(Boolean, default=True)
check_interval_minutes = Column(Integer, default=5)
max_emails_per_check = Column(Integer, default=50)
delete_after_forward = Column(Boolean, default=True)
# Auto-detection metadata
provider_name = Column(String(100), nullable=True) # e.g., "Gmail", "GMX"
auto_detected = Column(Boolean, default=False)
# Statistics
total_emails_processed = Column(Integer, default=0)
total_emails_failed = Column(Integer, default=0)
last_check_at = Column(DateTime, nullable=True)
last_successful_check_at = Column(DateTime, nullable=True)
last_error_at = Column(DateTime, nullable=True)
last_error_message = Column(Text, nullable=True)
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
# Relationships
user = relationship("User", back_populates="mail_accounts")
processing_runs = relationship("ProcessingRun", back_populates="mail_account", cascade="all, delete-orphan")
# Indexes
__table_args__ = (
Index('idx_user_email', 'user_id', 'email_address'),
Index('idx_status_enabled', 'status', 'is_enabled'),
)
class ProcessingRun(Base):
"""Records of email processing runs for each mail account"""
__tablename__ = "processing_runs"
id = Column(Integer, primary_key=True, index=True)
mail_account_id = Column(Integer, ForeignKey("mail_accounts.id", ondelete="CASCADE"), nullable=False)
# Run details
started_at = Column(DateTime, default=datetime.utcnow, nullable=False)
completed_at = Column(DateTime, nullable=True)
duration_seconds = Column(Float, nullable=True)
# Results
emails_fetched = Column(Integer, default=0)
emails_forwarded = Column(Integer, default=0)
emails_failed = Column(Integer, default=0)
# Status
status = Column(String(50), default="running") # running, completed, failed
error_message = Column(Text, nullable=True)
# Relationships
mail_account = relationship("MailAccount", back_populates="processing_runs")
# Indexes
__table_args__ = (
Index('idx_account_started', 'mail_account_id', 'started_at'),
)
class ProcessingLog(Base):
"""Detailed logs of individual email processing attempts"""
__tablename__ = "processing_logs"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
mail_account_id = Column(Integer, ForeignKey("mail_accounts.id", ondelete="CASCADE"), nullable=False)
processing_run_id = Column(Integer, ForeignKey("processing_runs.id", ondelete="CASCADE"), nullable=True)
# Log details
timestamp = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
level = Column(String(20), nullable=False) # INFO, WARNING, ERROR
message = Column(Text, nullable=False)
# Email metadata (if applicable)
email_subject = Column(String(500), nullable=True)
email_from = Column(String(255), nullable=True)
email_size_bytes = Column(Integer, nullable=True)
# Status
success = Column(Boolean, default=True)
error_details = Column(JSON, nullable=True)
# Relationships
user = relationship("User", back_populates="logs")
# Indexes
__table_args__ = (
Index('idx_user_timestamp', 'user_id', 'timestamp'),
Index('idx_account_timestamp', 'mail_account_id', 'timestamp'),
)
class NotificationConfig(Base):
"""User notification channel configurations"""
__tablename__ = "notification_configs"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
# Channel details
channel = Column(SQLEnum(NotificationChannel), nullable=False)
is_enabled = Column(Boolean, default=True)
# Channel-specific configuration (stored as JSON)
config = Column(JSON, nullable=False)
# Examples:
# EMAIL: {"address": "user@example.com"}
# TELEGRAM: {"bot_token": "xxx", "chat_id": "yyy"}
# WEBHOOK: {"url": "https://example.com/webhook", "headers": {...}}
# Notification preferences
notify_on_errors = Column(Boolean, default=True)
notify_on_success = Column(Boolean, default=False)
notify_threshold = Column(Integer, default=3) # Notify after N consecutive errors
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
# Relationships
user = relationship("User", back_populates="notifications")
# Indexes
__table_args__ = (
Index('idx_user_channel', 'user_id', 'channel'),
)
class MailServerPreset(Base):
"""Predefined mail server configurations for common providers"""
__tablename__ = "mail_server_presets"
id = Column(Integer, primary_key=True, index=True)
# Provider info
provider_name = Column(String(100), unique=True, nullable=False, index=True)
provider_domain = Column(String(255), nullable=False) # e.g., "gmail.com"
# Server configurations (can have multiple protocols)
configs = Column(JSON, nullable=False)
# Example:
# {
# "pop3_ssl": {"host": "pop.gmail.com", "port": 995, "ssl": true},
# "imap_ssl": {"host": "imap.gmail.com", "port": 993, "ssl": true}
# }
# Metadata
is_verified = Column(Boolean, default=False)
popularity_score = Column(Integer, default=0) # For sorting recommendations
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
class SubscriptionPlan(Base):
"""Available subscription plans and their features"""
__tablename__ = "subscription_plans"
id = Column(Integer, primary_key=True, index=True)
# Plan details
tier = Column(SQLEnum(SubscriptionTier), unique=True, nullable=False)
name = Column(String(100), nullable=False)
description = Column(Text, nullable=True)
# Pricing
price_monthly = Column(Float, nullable=False)
price_yearly = Column(Float, nullable=True)
# Stripe integration
stripe_price_id_monthly = Column(String(255), nullable=True)
stripe_price_id_yearly = Column(String(255), nullable=True)
# Features/Limits
max_mail_accounts = Column(Integer, nullable=False)
max_emails_per_day = Column(Integer, nullable=False)
check_interval_minutes = Column(Integer, nullable=False)
support_level = Column(String(50), default="community") # community, email, priority
features = Column(JSON, nullable=True) # Additional features as JSON
# Status
is_active = Column(Boolean, default=True)
# Timestamps
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
class AuditLog(Base):
"""Audit trail for security and compliance"""
__tablename__ = "audit_logs"
id = Column(Integer, primary_key=True, index=True)
# Who
user_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
user_email = Column(String(255), nullable=True) # Cached for deleted users
ip_address = Column(String(45), nullable=True) # IPv4 or IPv6
# What
action = Column(String(100), nullable=False, index=True)
resource_type = Column(String(50), nullable=True)
resource_id = Column(Integer, nullable=True)
# Details
details = Column(JSON, nullable=True)
status = Column(String(20), default="success") # success, failure
# When
timestamp = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
# Indexes
__table_args__ = (
Index('idx_user_action', 'user_id', 'action'),
Index('idx_timestamp_action', 'timestamp', 'action'),
)
+304
View File
@@ -0,0 +1,304 @@
"""
Pydantic schemas for API request/response validation.
"""
from datetime import datetime
from typing import Optional, Dict, Any, List
from pydantic import BaseModel, EmailStr, Field, validator
from enum import Enum
# Enums matching database models
class SubscriptionTier(str, Enum):
FREE = "free"
BASIC = "basic"
PRO = "pro"
ENTERPRISE = "enterprise"
class MailProtocol(str, Enum):
POP3 = "pop3"
POP3_SSL = "pop3_ssl"
IMAP = "imap"
IMAP_SSL = "imap_ssl"
class AccountStatus(str, Enum):
ACTIVE = "active"
INACTIVE = "inactive"
ERROR = "error"
TESTING = "testing"
class NotificationChannel(str, Enum):
EMAIL = "email"
TELEGRAM = "telegram"
WEBHOOK = "webhook"
SLACK = "slack"
DISCORD = "discord"
# User Schemas
class UserBase(BaseModel):
email: EmailStr
full_name: Optional[str] = None
class UserCreate(UserBase):
password: Optional[str] = None
class UserUpdate(BaseModel):
full_name: Optional[str] = None
email: Optional[EmailStr] = None
class UserResponse(UserBase):
id: int
is_active: bool
subscription_tier: SubscriptionTier
subscription_status: str
created_at: datetime
class Config:
from_attributes = True
class UserDetailResponse(UserResponse):
google_id: Optional[str] = None
oauth_provider: Optional[str] = None
stripe_customer_id: Optional[str] = None
subscription_expires_at: Optional[datetime] = None
last_login_at: Optional[datetime] = None
class Config:
from_attributes = True
# Authentication Schemas
class Token(BaseModel):
access_token: str
refresh_token: str
token_type: str = "bearer"
class TokenPayload(BaseModel):
sub: Optional[int] = None
exp: Optional[int] = None
type: Optional[str] = None
class GoogleAuthRequest(BaseModel):
code: str
redirect_uri: str
# Mail Account Schemas
class MailAccountBase(BaseModel):
name: str = Field(..., max_length=255)
email_address: EmailStr
protocol: MailProtocol = MailProtocol.POP3_SSL
host: str = Field(..., max_length=255)
port: int = Field(..., gt=0, lt=65536)
use_ssl: bool = True
use_tls: bool = False
username: str = Field(..., max_length=255)
forward_to: EmailStr
is_enabled: bool = True
check_interval_minutes: int = Field(default=5, gt=0, le=1440)
max_emails_per_check: int = Field(default=50, gt=0, le=1000)
delete_after_forward: bool = True
class MailAccountCreate(MailAccountBase):
password: str # Will be encrypted before storage
class MailAccountUpdate(BaseModel):
name: Optional[str] = Field(None, max_length=255)
password: Optional[str] = None
forward_to: Optional[EmailStr] = None
is_enabled: Optional[bool] = None
check_interval_minutes: Optional[int] = Field(None, gt=0, le=1440)
max_emails_per_check: Optional[int] = Field(None, gt=0, le=1000)
delete_after_forward: Optional[bool] = None
class MailAccountResponse(MailAccountBase):
id: int
user_id: int
status: AccountStatus
provider_name: Optional[str] = None
auto_detected: bool
total_emails_processed: int
total_emails_failed: int
last_check_at: Optional[datetime] = None
last_successful_check_at: Optional[datetime] = None
last_error_at: Optional[datetime] = None
last_error_message: Optional[str] = None
created_at: datetime
updated_at: datetime
# Don't expose password or username in responses
password: str = Field(exclude=True, default="")
username: str = Field(exclude=True, default="")
class Config:
from_attributes = True
class MailAccountTestRequest(BaseModel):
"""Test connection to mail server"""
host: str
port: int
protocol: MailProtocol
username: str
password: str
use_ssl: bool = True
use_tls: bool = False
class MailAccountTestResponse(BaseModel):
success: bool
message: str
details: Optional[Dict[str, Any]] = None
class MailAccountAutoDetectRequest(BaseModel):
"""Auto-detect mail server settings"""
email_address: EmailStr
class MailAccountAutoDetectResponse(BaseModel):
success: bool
suggestions: List[Dict[str, Any]]
# Processing Run Schemas
class ProcessingRunResponse(BaseModel):
id: int
mail_account_id: int
started_at: datetime
completed_at: Optional[datetime] = None
duration_seconds: Optional[float] = None
emails_fetched: int
emails_forwarded: int
emails_failed: int
status: str
error_message: Optional[str] = None
class Config:
from_attributes = True
# Processing Log Schemas
class ProcessingLogResponse(BaseModel):
id: int
timestamp: datetime
level: str
message: str
email_subject: Optional[str] = None
email_from: Optional[str] = None
success: bool
class Config:
from_attributes = True
# Notification Config Schemas
class NotificationConfigBase(BaseModel):
channel: NotificationChannel
is_enabled: bool = True
config: Dict[str, Any]
notify_on_errors: bool = True
notify_on_success: bool = False
notify_threshold: int = Field(default=3, gt=0, le=100)
class NotificationConfigCreate(NotificationConfigBase):
pass
class NotificationConfigUpdate(BaseModel):
is_enabled: Optional[bool] = None
config: Optional[Dict[str, Any]] = None
notify_on_errors: Optional[bool] = None
notify_on_success: Optional[bool] = None
notify_threshold: Optional[int] = Field(None, gt=0, le=100)
class NotificationConfigResponse(NotificationConfigBase):
id: int
user_id: int
created_at: datetime
updated_at: datetime
class Config:
from_attributes = True
# Subscription Schemas
class SubscriptionPlanResponse(BaseModel):
id: int
tier: SubscriptionTier
name: str
description: Optional[str] = None
price_monthly: float
price_yearly: Optional[float] = None
max_mail_accounts: int
max_emails_per_day: int
check_interval_minutes: int
support_level: str
features: Optional[Dict[str, Any]] = None
is_active: bool
class Config:
from_attributes = True
class SubscriptionCheckoutRequest(BaseModel):
tier: SubscriptionTier
billing_period: str = Field(..., pattern="^(monthly|yearly)$")
success_url: str
cancel_url: str
class SubscriptionCheckoutResponse(BaseModel):
checkout_url: str
session_id: str
# Statistics Schemas
class AccountStatistics(BaseModel):
total_accounts: int
active_accounts: int
inactive_accounts: int
error_accounts: int
total_emails_processed: int
total_emails_failed: int
class ProcessingStatistics(BaseModel):
last_24h_processed: int
last_24h_failed: int
last_7d_processed: int
last_7d_failed: int
success_rate: float
class DashboardStatistics(BaseModel):
account_stats: AccountStatistics
processing_stats: ProcessingStatistics
recent_runs: List[ProcessingRunResponse]
recent_errors: List[ProcessingLogResponse]
# Mail Server Preset Schemas
class MailServerPresetResponse(BaseModel):
id: int
provider_name: str
provider_domain: str
configs: Dict[str, Any]
is_verified: bool
class Config:
from_attributes = True
+1
View File
@@ -0,0 +1 @@
"""Services package"""
+507
View File
@@ -0,0 +1,507 @@
"""
Mail processing service for fetching and forwarding emails.
Supports both POP3 and IMAP protocols with secure connections.
"""
import asyncio
import poplib
import smtplib
import ssl
from email import parser
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.utils import formatdate, make_msgid
from typing import List, Dict, Any, Optional, Tuple
from datetime import datetime
import logging
from aioimaplib import aioimaplib
from app.models.database_models import MailAccount, MailProtocol
from app.core.config import settings
logger = logging.getLogger(__name__)
class MailConnectionError(Exception):
"""Raised when unable to connect to mail server"""
pass
class MailAuthenticationError(Exception):
"""Raised when authentication fails"""
pass
class MailFetchError(Exception):
"""Raised when fetching emails fails"""
pass
class MailForwardError(Exception):
"""Raised when forwarding email fails"""
pass
class MailProcessor:
"""Handles mail fetching and forwarding operations"""
def __init__(self, account: MailAccount, decrypted_password: str):
self.account = account
self.password = decrypted_password
async def test_connection(self) -> Tuple[bool, str]:
"""
Test connection to mail server.
Returns (success, message)
"""
try:
if self.account.protocol in [MailProtocol.POP3, MailProtocol.POP3_SSL]:
return await self._test_pop3_connection()
else:
return await self._test_imap_connection()
except Exception as e:
logger.error(f"Connection test failed: {e}")
return False, str(e)
async def _test_pop3_connection(self) -> Tuple[bool, str]:
"""Test POP3 connection"""
try:
loop = asyncio.get_event_loop()
# Run blocking POP3 operations in thread pool
def connect_pop3():
if self.account.protocol == MailProtocol.POP3_SSL:
context = ssl.create_default_context()
pop_conn = poplib.POP3_SSL(
self.account.host,
self.account.port,
context=context,
timeout=10
)
else:
pop_conn = poplib.POP3(
self.account.host,
self.account.port,
timeout=10
)
# Try authentication
pop_conn.user(self.account.username)
pop_conn.pass_(self.password)
# Get mailbox stats
message_count, mailbox_size = pop_conn.stat()
pop_conn.quit()
return message_count, mailbox_size
message_count, mailbox_size = await loop.run_in_executor(None, connect_pop3)
return True, f"Connection successful. {message_count} messages in mailbox."
except poplib.error_proto as e:
error_msg = str(e)
if "authentication" in error_msg.lower() or "auth" in error_msg.lower():
return False, f"Authentication failed: {error_msg}"
return False, f"POP3 protocol error: {error_msg}"
except Exception as e:
return False, f"Connection failed: {str(e)}"
async def _test_imap_connection(self) -> Tuple[bool, str]:
"""Test IMAP connection"""
try:
# Create IMAP client
if self.account.protocol == MailProtocol.IMAP_SSL:
imap_client = aioimaplib.IMAP4_SSL(
host=self.account.host,
port=self.account.port,
timeout=10
)
else:
imap_client = aioimaplib.IMAP4(
host=self.account.host,
port=self.account.port,
timeout=10
)
await imap_client.wait_hello_from_server()
# Authenticate
response = await imap_client.login(self.account.username, self.password)
if response.result != 'OK':
return False, f"Authentication failed: {response.lines}"
# Select inbox
await imap_client.select('INBOX')
# Get message count
response = await imap_client.search('ALL')
message_ids = response.lines[0].split()
message_count = len(message_ids)
await imap_client.logout()
return True, f"Connection successful. {message_count} messages in mailbox."
except Exception as e:
return False, f"IMAP connection failed: {str(e)}"
async def fetch_emails(self, max_count: Optional[int] = None) -> List[bytes]:
"""
Fetch emails from the mail server.
Returns list of raw email data.
"""
max_count = max_count or self.account.max_emails_per_check
if self.account.protocol in [MailProtocol.POP3, MailProtocol.POP3_SSL]:
return await self._fetch_pop3_emails(max_count)
else:
return await self._fetch_imap_emails(max_count)
async def _fetch_pop3_emails(self, max_count: int) -> List[bytes]:
"""Fetch emails via POP3"""
emails = []
try:
loop = asyncio.get_event_loop()
def fetch_pop3():
# Connect
if self.account.protocol == MailProtocol.POP3_SSL:
context = ssl.create_default_context()
pop_conn = poplib.POP3_SSL(
self.account.host,
self.account.port,
context=context,
timeout=30
)
else:
pop_conn = poplib.POP3(
self.account.host,
self.account.port,
timeout=30
)
# Authenticate
pop_conn.user(self.account.username)
pop_conn.pass_(self.password)
# Get message count
num_messages = len(pop_conn.list()[1])
logger.info(f"Found {num_messages} messages for account {self.account.id}")
fetched_emails = []
messages_to_delete = []
# Fetch emails (limited by max_count)
for i in range(1, min(num_messages + 1, max_count + 1)):
try:
response, lines, octets = pop_conn.retr(i)
email_data = b'\r\n'.join(lines)
fetched_emails.append(email_data)
messages_to_delete.append(i)
logger.info(f"Retrieved message {i} from account {self.account.id}")
except Exception as e:
logger.error(f"Error retrieving message {i}: {e}")
# Delete messages if configured
if self.account.delete_after_forward:
for msg_id in messages_to_delete:
try:
pop_conn.dele(msg_id)
except Exception as e:
logger.error(f"Error deleting message {msg_id}: {e}")
pop_conn.quit()
return fetched_emails
emails = await loop.run_in_executor(None, fetch_pop3)
except Exception as e:
logger.error(f"Error fetching POP3 emails: {e}")
raise MailFetchError(f"POP3 fetch error: {str(e)}")
return emails
async def _fetch_imap_emails(self, max_count: int) -> List[bytes]:
"""Fetch emails via IMAP"""
emails = []
try:
# Create IMAP client
if self.account.protocol == MailProtocol.IMAP_SSL:
imap_client = aioimaplib.IMAP4_SSL(
host=self.account.host,
port=self.account.port,
timeout=30
)
else:
imap_client = aioimaplib.IMAP4(
host=self.account.host,
port=self.account.port,
timeout=30
)
await imap_client.wait_hello_from_server()
await imap_client.login(self.account.username, self.password)
await imap_client.select('INBOX')
# Search for all messages
response = await imap_client.search('UNSEEN') # Only fetch unread
message_ids = response.lines[0].split()
# Limit to max_count
message_ids = message_ids[:max_count]
logger.info(f"Found {len(message_ids)} unread messages for account {self.account.id}")
# Fetch each message
for msg_id in message_ids:
try:
response = await imap_client.fetch(msg_id, '(RFC822)')
# Extract email data from response
email_data = None
for line in response.lines:
if isinstance(line, bytes) and b'RFC822' in line:
# Find the email content
start_idx = line.find(b'{')
if start_idx != -1:
# Email data is in the next parts
continue
elif isinstance(line, bytes) and not line.startswith(b'*'):
email_data = line
break
if email_data:
emails.append(email_data)
# Mark as seen if deleting after forward
if self.account.delete_after_forward:
await imap_client.store(msg_id, '+FLAGS', '\\Deleted')
except Exception as e:
logger.error(f"Error fetching message {msg_id}: {e}")
# Expunge deleted messages
if self.account.delete_after_forward:
await imap_client.expunge()
await imap_client.logout()
except Exception as e:
logger.error(f"Error fetching IMAP emails: {e}")
raise MailFetchError(f"IMAP fetch error: {str(e)}")
return emails
@staticmethod
async def forward_email(
email_data: bytes,
source_account_name: str,
destination: str,
smtp_config: Dict[str, Any]
) -> bool:
"""
Forward an email to the destination address.
Args:
email_data: Raw email bytes
source_account_name: Name of source account for labeling
destination: Destination email address
smtp_config: SMTP configuration dict with keys:
- host: SMTP host
- port: SMTP port
- username: SMTP username
- password: SMTP password
- use_tls: Whether to use STARTTLS
Returns:
True if successful, False otherwise
"""
try:
loop = asyncio.get_event_loop()
def send_email():
# Parse the email
msg = parser.BytesParser().parsebytes(email_data)
# Create forwarding message
forward_msg = MIMEMultipart('mixed')
forward_msg['From'] = smtp_config['username']
forward_msg['To'] = destination
forward_msg['Date'] = formatdate(localtime=True)
forward_msg['Message-ID'] = make_msgid()
# Preserve original subject with prefix
original_subject = msg.get('Subject', 'No Subject')
forward_msg['Subject'] = f"[Fwd from {source_account_name}] {original_subject}"
# Add original headers
header_info = f"Originally from: {msg.get('From', 'Unknown')}\n"
header_info += f"Original Date: {msg.get('Date', 'Unknown')}\n"
header_info += f"Original Subject: {original_subject}\n"
header_info += f"Source Account: {source_account_name}\n"
header_info += "-" * 50 + "\n\n"
# Get email body
body = ""
if msg.is_multipart():
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True).decode('utf-8', errors='ignore')
break
else:
payload = msg.get_payload(decode=True)
if payload:
body = payload.decode('utf-8', errors='ignore')
# Combine header and body
full_body = header_info + body
forward_msg.attach(MIMEText(full_body, 'plain', 'utf-8'))
# Send via SMTP
if smtp_config.get('use_tls', True):
server = smtplib.SMTP(smtp_config['host'], smtp_config['port'], timeout=30)
server.starttls()
else:
server = smtplib.SMTP_SSL(smtp_config['host'], smtp_config['port'], timeout=30)
try:
server.login(smtp_config['username'], smtp_config['password'])
server.send_message(forward_msg)
logger.info(f"Successfully forwarded email to {destination}")
return True
finally:
try:
server.quit()
except:
pass
return await loop.run_in_executor(None, send_email)
except Exception as e:
logger.error(f"Error forwarding email: {e}")
raise MailForwardError(f"Forward error: {str(e)}")
class MailServerAutoDetect:
"""Auto-detect mail server settings based on email domain"""
# Common mail server configurations
KNOWN_PROVIDERS = {
"gmail.com": {
"name": "Gmail",
"pop3_ssl": {"host": "pop.gmail.com", "port": 995},
"imap_ssl": {"host": "imap.gmail.com", "port": 993},
},
"outlook.com": {
"name": "Outlook.com",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"hotmail.com": {
"name": "Hotmail",
"pop3_ssl": {"host": "outlook.office365.com", "port": 995},
"imap_ssl": {"host": "outlook.office365.com", "port": 993},
},
"gmx.com": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.com", "port": 995},
"imap_ssl": {"host": "imap.gmx.com", "port": 993},
},
"gmx.de": {
"name": "GMX",
"pop3_ssl": {"host": "pop.gmx.net", "port": 995},
"imap_ssl": {"host": "imap.gmx.net", "port": 993},
},
"web.de": {
"name": "WEB.DE",
"pop3_ssl": {"host": "pop3.web.de", "port": 995},
"imap_ssl": {"host": "imap.web.de", "port": 993},
},
"t-online.de": {
"name": "T-Online",
"pop3_ssl": {"host": "pop.t-online.de", "port": 995},
"imap_ssl": {"host": "imap.t-online.de", "port": 993},
},
"yahoo.com": {
"name": "Yahoo",
"pop3_ssl": {"host": "pop.mail.yahoo.com", "port": 995},
"imap_ssl": {"host": "imap.mail.yahoo.com", "port": 993},
},
}
@classmethod
def detect(cls, email_address: str) -> List[Dict[str, Any]]:
"""
Detect mail server settings for an email address.
Returns list of possible configurations.
"""
domain = email_address.split('@')[-1].lower()
suggestions = []
# Check if we have a known provider
if domain in cls.KNOWN_PROVIDERS:
provider = cls.KNOWN_PROVIDERS[domain]
# Add POP3 SSL suggestion
if "pop3_ssl" in provider:
suggestions.append({
"protocol": "pop3_ssl",
"provider_name": provider["name"],
"host": provider["pop3_ssl"]["host"],
"port": provider["pop3_ssl"]["port"],
"use_ssl": True,
"use_tls": False,
})
# Add IMAP SSL suggestion
if "imap_ssl" in provider:
suggestions.append({
"protocol": "imap_ssl",
"provider_name": provider["name"],
"host": provider["imap_ssl"]["host"],
"port": provider["imap_ssl"]["port"],
"use_ssl": True,
"use_tls": False,
})
else:
# Generic suggestions based on common patterns
suggestions.extend([
{
"protocol": "pop3_ssl",
"provider_name": "Generic",
"host": f"pop.{domain}",
"port": 995,
"use_ssl": True,
"use_tls": False,
},
{
"protocol": "pop3_ssl",
"provider_name": "Generic",
"host": f"pop3.{domain}",
"port": 995,
"use_ssl": True,
"use_tls": False,
},
{
"protocol": "imap_ssl",
"provider_name": "Generic",
"host": f"imap.{domain}",
"port": 993,
"use_ssl": True,
"use_tls": False,
},
{
"protocol": "imap_ssl",
"provider_name": "Generic",
"host": f"mail.{domain}",
"port": 993,
"use_ssl": True,
"use_tls": False,
},
])
return suggestions
+1
View File
@@ -0,0 +1 @@
"""Utils package"""
+1
View File
@@ -0,0 +1 @@
"""Workers package"""
+56
View File
@@ -0,0 +1,56 @@
# Core Framework
fastapi==0.109.0
uvicorn[standard]==0.27.0
pydantic==2.5.3
pydantic-settings==2.1.0
# Database
sqlalchemy==2.0.25
alembic==1.13.1
psycopg2-binary==2.9.9
asyncpg==0.29.0
# Authentication
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
python-multipart==0.0.6
authlib==1.3.0
httpx==0.26.0
# Payment Processing
stripe==7.11.0
# Email & Mail Processing
aiosmtplib==3.0.1
aiohttp==3.9.1
aioimaplib==1.0.1
email-validator==2.1.0.post1
# Job Queue & Cache
celery==5.3.6
redis==5.0.1
# Security & Encryption
cryptography==42.0.0
# Notifications
apprise==1.7.1
# Monitoring & Logging
prometheus-client==0.19.0
python-json-logger==2.0.7
# Development & Testing
pytest==7.4.4
pytest-asyncio==0.23.3
pytest-cov==4.1.0
httpx==0.26.0
faker==22.6.0
# Utilities
python-dotenv==1.0.0
schedule==1.2.0
tenacity==8.2.3
# Legacy support (for migration)
poplib3==0.0.4