diff --git a/CHANGELOG.md b/CHANGELOG.md index 0921fce..baf61b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Architecture Decision Records ADR-003 through ADR-010**: Added eight new ADRs covering FastAPI web framework (ADR-003), PostgreSQL database (ADR-004), Celery task retry strategy (ADR-005), key management in production (ADR-006), JWT authentication (ADR-007), Next.js frontend (ADR-008), Gmail API email delivery (ADR-009), and hybrid configuration model (ADR-010) - `userApi.updateProfile()` method in `frontend/src/lib/api.ts` for updating user profile via `PUT /users/me` +- **Account enable/disable toggle**: `PATCH /mail-accounts/{id}/toggle` backend endpoint and a Power-icon toggle button on each account card in the UI. Disabled accounts are visually dimmed. Re-enabling an account that was in ERROR state resets its status to ACTIVE so the scheduler picks it up again. +- **`is_enabled` checkbox in edit modal**: The Add/Edit mail account form now includes an "Enabled" checkbox so the flag can be set when creating or editing an account. +- **Message deduplication tracking** (`DownloadedMessageId` table): Both POP3 and IMAP fetch paths now track downloaded message UIDs so the same message is never delivered twice, even when `delete_after_forward=False`. + - IMAP: messages are marked `\Seen` after fetching so they don't appear in future `UNSEEN` searches. DB UIDs provide a secondary guard. + - POP3: UIDL-based deduplication; messages are skipped if their UID is already in the DB. + - Old UID records are pruned by `cleanup_old_logs` after `days_to_keep` days. +- **Gmail API "one-click" OAuth grant flow**: New `GET /providers/gmail/authorize-url` and `POST /providers/gmail/callback` endpoints. The flow requests `gmail.insert + gmail.labels` scopes with `access_type=offline` so a long-lived refresh token is issued. A new `/auth/gmail-callback` frontend page handles the redirect from Google, exchanges the code, and redirects the user back to Settings. +- **Gmail token auto-refresh and persistence**: `GmailService` now records whether the `google-auth` library refreshed the access token during a Celery run. If it did, the Celery task writes the new access token and expiry back to `GmailCredential`, eliminating an unnecessary extra refresh call on the next run. A `401/403` or `invalid_grant` error during delivery marks `GmailCredential.is_valid = False` so the user is prompted to re-authorise. +- **Per-user SMTP relay configuration** (`UserSmtpConfig` table): New `GET/PUT/DELETE /users/smtp-config` endpoints let each user store their own SMTP relay (host, port, username, password, TLS flag). The Celery task checks for per-user SMTP first; falls back to the global `AppSetting` SMTP config if none is set. +- **Settings page – Gmail & SMTP sections**: The Settings page now shows a "Gmail API Delivery" card with connection status, "Connect Gmail" / "Re-authorise" / "Disconnect" buttons, and a token-lifetime explanation. An "SMTP Fallback" card lets users save their own SMTP relay credentials. +- **Celery scheduling fix**: `process_all_enabled_accounts` previously only polled accounts with `status IN [ACTIVE, TESTING]`, causing ERROR-status accounts to be silently skipped forever. It now polls all `is_enabled = True` accounts regardless of status, so transient errors are retried automatically. - **Backend URL logged at startup**: The Next.js server now logs the resolved `BACKEND_URL` (e.g. `[proxy] BACKEND_URL = http://backend:8000`) via `src/instrumentation.ts` when the server starts, making it easy to diagnose `ECONNREFUSED` proxy errors. The per-request error log now also includes the full target URL. - **Dual-registry Docker deployment**: CI now builds separate backend and frontend images and pushes to both GHCR (`ghcr.io`) and private registry (`registry.cklnet.com`) using a matrix strategy - **Database-backed configuration**: `AppSetting` model and `ConfigService` for hybrid config (DB-first, env-var fallback) diff --git a/backend/app/api/v1/endpoints/mail_accounts.py b/backend/app/api/v1/endpoints/mail_accounts.py index a61a94c..f5f28b2 100644 --- a/backend/app/api/v1/endpoints/mail_accounts.py +++ b/backend/app/api/v1/endpoints/mail_accounts.py @@ -8,7 +8,7 @@ from sqlalchemy import select, desc from app.core.database import get_db from app.core.deps import get_current_active_user from app.core.security import encrypt_credential -from app.models.database_models import User, MailAccount +from app.models.database_models import User, MailAccount, AccountStatus from app.models.schemas import ( MailAccountCreate, MailAccountResponse, @@ -182,6 +182,38 @@ async def delete_mail_account( await db.commit() +@router.patch("/{account_id}/toggle", response_model=MailAccountResponse) +async def toggle_mail_account( + account_id: int, + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """Toggle the enabled/disabled state of a mail account""" + result = await db.execute( + select(MailAccount).where( + MailAccount.id == account_id, MailAccount.user_id == current_user.id + ) + ) + account = result.scalar_one_or_none() + + if not account: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Mail account not found" + ) + + account.is_enabled = not account.is_enabled # type: ignore[assignment] + + # When re-enabling a previously errored account, reset status to ACTIVE + # so the scheduler picks it up on the next run. + if account.is_enabled and account.status == AccountStatus.ERROR: + account.status = AccountStatus.ACTIVE # type: ignore[assignment] + + await db.commit() + await db.refresh(account) + + return account + + @router.post("/test", response_model=MailAccountTestResponse) async def test_mail_connection( test_request: MailAccountTestRequest, diff --git a/backend/app/api/v1/endpoints/providers.py b/backend/app/api/v1/endpoints/providers.py index d6603fd..6ef5ea5 100644 --- a/backend/app/api/v1/endpoints/providers.py +++ b/backend/app/api/v1/endpoints/providers.py @@ -1,10 +1,12 @@ """Provider presets and Gmail credential management endpoints""" -from datetime import datetime, timezone -from typing import List +from datetime import datetime, timedelta, timezone +from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select +import httpx +import logging from app.core.database import get_db from app.core.deps import get_current_active_user @@ -16,10 +18,21 @@ from app.models.schemas import ( ProviderListResponse, GmailCredentialCreate, GmailCredentialResponse, + GmailAuthorizeResponse, + GmailCallbackRequest, ) from app.services.gmail_service import GmailService router = APIRouter() +logger = logging.getLogger(__name__) + +# Gmail API scopes needed for email injection +GMAIL_API_SCOPES = [ + "openid", + "email", + "https://www.googleapis.com/auth/gmail.insert", + "https://www.googleapis.com/auth/gmail.labels", +] # Provider presets with server configurations PROVIDER_PRESETS: List[ProviderPreset] = [ @@ -268,3 +281,166 @@ async def delete_gmail_credential( await db.delete(credential) await db.commit() + + +@router.get("/gmail/authorize-url", response_model=GmailAuthorizeResponse) +async def get_gmail_authorize_url( + redirect_uri: str, + current_user: User = Depends(get_current_active_user), +): + """ + Return a Google OAuth2 URL that grants this app Gmail API write permissions. + + The URL requests the gmail.insert + gmail.labels scopes with + access_type=offline so a long-lived refresh token is issued. + prompt=consent forces Google to always issue a new refresh token even if + the user has authorised the app before. + """ + if not settings.GOOGLE_CLIENT_ID: + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail="Google OAuth2 is not configured on this server.", + ) + + scope = " ".join(GMAIL_API_SCOPES) + url = ( + "https://accounts.google.com/o/oauth2/v2/auth" + f"?client_id={settings.GOOGLE_CLIENT_ID}" + "&response_type=code" + f"&scope={scope}" + f"&redirect_uri={redirect_uri}" + "&access_type=offline" + "&prompt=consent" + ) + return GmailAuthorizeResponse(authorization_url=url) + + +@router.post( + "/gmail/callback", + response_model=GmailCredentialResponse, + status_code=status.HTTP_201_CREATED, +) +async def gmail_oauth_callback( + callback_in: GmailCallbackRequest, + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """ + Exchange a Google authorization code for Gmail API tokens and persist them. + + This is the backend half of the "Connect Gmail" one-click flow. The + frontend redirects the user to Google, Google sends a code back to the + frontend callback page, and the frontend posts that code here. + + Token lifetime + -------------- + * Access token : 1 hour. The google-auth library auto-refreshes it + during Celery tasks; the new value is written back to this row so the + next run doesn't need an extra refresh round-trip. + * Refresh token : does not expire unless the user revokes access or the + app credentials change. If revoked, ``is_valid`` is set to False by + the next Celery run that hits a 401, and the user must re-authorise. + """ + if not settings.GOOGLE_CLIENT_ID or not settings.GOOGLE_CLIENT_SECRET: + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail="Google OAuth2 is not configured on this server.", + ) + + # Exchange code for tokens + async with httpx.AsyncClient() as client: + token_resp = await client.post( + "https://oauth2.googleapis.com/token", + data={ + "code": callback_in.code, + "client_id": settings.GOOGLE_CLIENT_ID, + "client_secret": settings.GOOGLE_CLIENT_SECRET, + "redirect_uri": callback_in.redirect_uri, + "grant_type": "authorization_code", + }, + ) + + if token_resp.status_code != 200: + logger.error(f"Gmail token exchange failed: {token_resp.text}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Failed to exchange authorization code with Google.", + ) + + token_data = token_resp.json() + access_token: Optional[str] = token_data.get("access_token") + refresh_token: Optional[str] = token_data.get("refresh_token") + + if not access_token: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Google did not return an access token.", + ) + + # Fetch the Gmail email address to associate with this credential + async with httpx.AsyncClient() as client: + profile_resp = await client.get( + "https://www.googleapis.com/oauth2/v2/userinfo", + headers={"Authorization": f"Bearer {access_token}"}, + ) + + gmail_email = current_user.email # fallback + if profile_resp.status_code == 200: + gmail_email = profile_resp.json().get("email", current_user.email) + + # Calculate token expiry (Google access tokens last 1 hour) + token_expiry = datetime.now(timezone.utc) + timedelta( + seconds=int(token_data.get("expires_in", 3600)) + ) + + # Verify the credentials actually work with the Gmail API + gmail_service = GmailService( + access_token=access_token, + refresh_token=refresh_token, + client_id=settings.GOOGLE_CLIENT_ID, + client_secret=settings.GOOGLE_CLIENT_SECRET, + ) + is_valid = await gmail_service.verify_access() + if not is_valid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Obtained tokens but could not verify Gmail API access. " + "Ensure the gmail.insert scope was granted.", + ) + + # Persist tokens + result = await db.execute( + select(GmailCredential).where(GmailCredential.user_id == current_user.id) + ) + existing = result.scalar_one_or_none() + + encrypted_access = encrypt_credential(access_token) + encrypted_refresh = encrypt_credential(refresh_token) if refresh_token else None + + if existing: + existing.gmail_email = gmail_email # type: ignore[assignment] + existing.encrypted_access_token = encrypted_access # type: ignore[assignment] + if encrypted_refresh: + existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment] + existing.token_expiry = token_expiry # type: ignore[assignment] + existing.scopes = token_data.get("scope", "").split() # type: ignore[assignment] + existing.is_valid = True # type: ignore[assignment] + existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment] + await db.commit() + await db.refresh(existing) + return existing + else: + credential = GmailCredential( + user_id=current_user.id, + gmail_email=gmail_email, + encrypted_access_token=encrypted_access, + encrypted_refresh_token=encrypted_refresh, + token_expiry=token_expiry, + scopes=token_data.get("scope", "").split(), + is_valid=True, + last_verified_at=datetime.now(timezone.utc), + ) + db.add(credential) + await db.commit() + await db.refresh(credential) + return credential diff --git a/backend/app/api/v1/endpoints/users.py b/backend/app/api/v1/endpoints/users.py index a609652..60c05be 100644 --- a/backend/app/api/v1/endpoints/users.py +++ b/backend/app/api/v1/endpoints/users.py @@ -1,12 +1,19 @@ """User management endpoints""" -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select from app.core.database import get_db from app.core.deps import get_current_active_user -from app.models.database_models import User -from app.models.schemas import UserDetailResponse, UserUpdate +from app.core.security import encrypt_credential +from app.models.database_models import User, UserSmtpConfig +from app.models.schemas import ( + UserDetailResponse, + UserUpdate, + UserSmtpConfigUpdate, + UserSmtpConfigResponse, +) router = APIRouter() @@ -34,3 +41,96 @@ async def update_current_user_profile( await db.commit() await db.refresh(current_user) return current_user + + +@router.get("/smtp-config", response_model=UserSmtpConfigResponse) +async def get_smtp_config( + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """Get the current user's SMTP relay configuration""" + result = await db.execute( + select(UserSmtpConfig).where(UserSmtpConfig.user_id == current_user.id) + ) + config = result.scalar_one_or_none() + + if not config: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No SMTP configuration found. Save one first.", + ) + + return UserSmtpConfigResponse( + id=config.id, # type: ignore[arg-type] + user_id=config.user_id, # type: ignore[arg-type] + host=config.host, # type: ignore[arg-type] + port=config.port, # type: ignore[arg-type] + username=config.username, # type: ignore[arg-type] + use_tls=config.use_tls, # type: ignore[arg-type] + has_password=bool(config.encrypted_password), + created_at=config.created_at, # type: ignore[arg-type] + updated_at=config.updated_at, # type: ignore[arg-type] + ) + + +@router.put("/smtp-config", response_model=UserSmtpConfigResponse) +async def upsert_smtp_config( + config_in: UserSmtpConfigUpdate, + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """Create or update the current user's SMTP relay configuration""" + result = await db.execute( + select(UserSmtpConfig).where(UserSmtpConfig.user_id == current_user.id) + ) + config = result.scalar_one_or_none() + + if config: + config.host = config_in.host # type: ignore[assignment] + config.port = config_in.port # type: ignore[assignment] + config.username = config_in.username # type: ignore[assignment] + config.use_tls = config_in.use_tls # type: ignore[assignment] + if config_in.password is not None: + config.encrypted_password = encrypt_credential(config_in.password) # type: ignore[assignment] + else: + config = UserSmtpConfig( + user_id=current_user.id, + host=config_in.host, + port=config_in.port, + username=config_in.username, + encrypted_password=( + encrypt_credential(config_in.password) if config_in.password else "" + ), + use_tls=config_in.use_tls, + ) + db.add(config) + + await db.commit() + await db.refresh(config) + + return UserSmtpConfigResponse( + id=config.id, # type: ignore[arg-type] + user_id=config.user_id, # type: ignore[arg-type] + host=config.host, # type: ignore[arg-type] + port=config.port, # type: ignore[arg-type] + username=config.username, # type: ignore[arg-type] + use_tls=config.use_tls, # type: ignore[arg-type] + has_password=bool(config.encrypted_password), + created_at=config.created_at, # type: ignore[arg-type] + updated_at=config.updated_at, # type: ignore[arg-type] + ) + + +@router.delete("/smtp-config", status_code=status.HTTP_204_NO_CONTENT) +async def delete_smtp_config( + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """Delete the current user's SMTP relay configuration""" + result = await db.execute( + select(UserSmtpConfig).where(UserSmtpConfig.user_id == current_user.id) + ) + config = result.scalar_one_or_none() + if config: + await db.delete(config) + await db.commit() diff --git a/backend/app/models/database_models.py b/backend/app/models/database_models.py index 3bf2126..7932792 100644 --- a/backend/app/models/database_models.py +++ b/backend/app/models/database_models.py @@ -446,6 +446,76 @@ class AuditLog(Base): ) +class UserSmtpConfig(Base): + """Per-user SMTP relay configuration for email forwarding fallback.""" + + __tablename__ = "user_smtp_configs" + + id = Column(Integer, primary_key=True, index=True) + user_id = Column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, unique=True + ) + + host = Column(String(255), nullable=False, default="smtp.gmail.com") + port = Column(Integer, nullable=False, default=587) + username = Column(String(255), nullable=False, default="") + encrypted_password = Column(Text, nullable=False, default="") + use_tls = Column(Boolean, default=True) + + created_at = Column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + updated_at = Column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + # Relationships + user = relationship("User", backref="smtp_config") + + +class DownloadedMessageId(Base): + """ + Tracks unique message IDs that have already been downloaded and forwarded. + + - For POP3: stores the UIDL string returned by the server. + - For IMAP: stores the IMAP UID (numeric string) of the message. + + This prevents re-processing the same message when delete_after_forward=False. + """ + + __tablename__ = "downloaded_message_ids" + + id = Column(Integer, primary_key=True, index=True) + mail_account_id = Column( + Integer, ForeignKey("mail_accounts.id", ondelete="CASCADE"), nullable=False + ) + + # Unique message identifier (UIDL for POP3, UID for IMAP) + message_uid = Column(String(512), nullable=False) + + downloaded_at = Column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + # Indexes — unique constraint prevents duplicates + __table_args__ = ( + Index( + "idx_account_message_uid", + "mail_account_id", + "message_uid", + unique=True, + ), + Index("idx_downloaded_at", "downloaded_at"), + ) + + class GmailCredential(Base): """Stores OAuth2 credentials for Gmail API access (per-user)""" diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 8261291..6321eb0 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -339,3 +339,35 @@ class ProviderPreset(BaseModel): class ProviderListResponse(BaseModel): providers: List[ProviderPreset] + + +# User SMTP Config Schemas +class UserSmtpConfigBase(BaseModel): + host: str = "smtp.gmail.com" + port: int = Field(587, gt=0, lt=65536) + username: str = "" + use_tls: bool = True + + +class UserSmtpConfigUpdate(UserSmtpConfigBase): + password: Optional[str] = None # Only provided when changing the password + + +class UserSmtpConfigResponse(UserSmtpConfigBase): + id: int + user_id: int + has_password: bool # True if a password is stored (value is never returned) + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Gmail OAuth Schemas +class GmailAuthorizeResponse(BaseModel): + authorization_url: str + + +class GmailCallbackRequest(BaseModel): + code: str + redirect_uri: str diff --git a/backend/app/services/gmail_service.py b/backend/app/services/gmail_service.py index 403bf46..717fba1 100644 --- a/backend/app/services/gmail_service.py +++ b/backend/app/services/gmail_service.py @@ -56,6 +56,7 @@ class GmailService: client_id: Google OAuth2 client ID client_secret: Google OAuth2 client secret """ + self._initial_access_token = access_token self.credentials = Credentials( token=access_token, refresh_token=refresh_token, @@ -135,6 +136,7 @@ class GmailService: f"Gmail API error: {e.reason if hasattr(e, 'reason') else str(e)}" ) logger.error(error_msg) + # Surface 401 so callers can mark credentials as invalid raise GmailInjectionError(error_msg) except Exception as e: error_msg = f"Failed to inject email into Gmail: {str(e)}" @@ -180,3 +182,24 @@ class GmailService: except Exception as e: logger.error(f"Failed to get Gmail email address: {e}") return None + + def get_refreshed_token(self) -> Optional[Dict[str, Any]]: + """ + Return the current access token and expiry if the token was refreshed + since this service instance was created. + + The google-auth library auto-refreshes the access token when an API + call is made with an expired token. Call this after inject_email() to + check whether a refresh happened and persist the new token. + + Returns: + Dict with ``access_token`` and ``expiry`` (datetime | None), or + None if the token has not changed from the one passed to __init__. + """ + current_token = self.credentials.token + if current_token and current_token != self._initial_access_token: + return { + "access_token": current_token, + "expiry": self.credentials.expiry, + } + return None diff --git a/backend/app/services/mail_processor.py b/backend/app/services/mail_processor.py index 9829bf8..144e6a5 100644 --- a/backend/app/services/mail_processor.py +++ b/backend/app/services/mail_processor.py @@ -11,7 +11,7 @@ from email import parser from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from email.utils import formatdate, make_msgid -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Dict, Any, Optional, Set, Tuple import logging from aioimaplib import aioimaplib @@ -143,26 +143,42 @@ class MailProcessor: except Exception as e: return False, f"IMAP connection failed: {str(e)}" - async def fetch_emails(self, max_count: Optional[int] = None) -> List[bytes]: + async def fetch_emails( + self, + max_count: Optional[int] = None, + already_seen_uids: Optional[Set[str]] = None, + ) -> Tuple[List[bytes], List[str]]: """ Fetch emails from the mail server. - Returns list of raw email data. + + Args: + max_count: Maximum number of messages to fetch. + already_seen_uids: Set of message UIDs that have already been + processed and should be skipped. + + Returns: + A tuple of (raw_email_bytes_list, new_uid_strings_list). + The caller should persist the new UIDs to prevent re-processing. """ effective_max: int = max_count if max_count is not None else self.account.max_emails_per_check # type: ignore[assignment] + seen: Set[str] = already_seen_uids or set() if self.account.protocol in [MailProtocol.POP3, MailProtocol.POP3_SSL]: - return await self._fetch_pop3_emails(effective_max) + return await self._fetch_pop3_emails(effective_max, seen) else: - return await self._fetch_imap_emails(effective_max) + return await self._fetch_imap_emails(effective_max, seen) - async def _fetch_pop3_emails(self, max_count: int) -> List[bytes]: - """Fetch emails via POP3""" - emails = [] + async def _fetch_pop3_emails( + self, max_count: int, already_seen_uids: Set[str] + ) -> Tuple[List[bytes], List[str]]: + """Fetch emails via POP3, skipping already-downloaded UIDs.""" + emails: List[bytes] = [] + new_uids: List[str] = [] try: loop = asyncio.get_event_loop() - def fetch_pop3(): + def fetch_pop3() -> Tuple[List[bytes], List[str]]: # Connect if self.account.protocol == MailProtocol.POP3_SSL: context = ssl.create_default_context() @@ -181,27 +197,49 @@ class MailProcessor: pop_conn.user(self.account.username) pop_conn.pass_(self.password) - # Get message count - num_messages = len(pop_conn.list()[1]) + # Retrieve UIDL map: {msg_number: uid_string} + uidl_response = pop_conn.uidl() + uid_map: Dict[int, str] = {} + for entry in uidl_response[1]: + parts = entry.decode().split(" ", 1) + if len(parts) == 2: + uid_map[int(parts[0])] = parts[1].strip() + + num_messages = len(uid_map) logger.info( f"Found {num_messages} messages for account {self.account.id}" ) - fetched_emails = [] - messages_to_delete = [] + fetched: List[bytes] = [] + fetched_uids: List[str] = [] + messages_to_delete: List[int] = [] + fetched_count = 0 + + for msg_num, uid in uid_map.items(): + if fetched_count >= max_count: + break + + # Skip messages we already processed + if uid in already_seen_uids: + logger.debug( + f"Skipping already-downloaded message {uid} " + f"for account {self.account.id}" + ) + continue - # Fetch emails (limited by max_count) - for i in range(1, min(num_messages + 1, max_count + 1)): try: - response, lines, octets = pop_conn.retr(i) + response, lines, octets = pop_conn.retr(msg_num) email_data = b"\r\n".join(lines) - fetched_emails.append(email_data) - messages_to_delete.append(i) + fetched.append(email_data) + fetched_uids.append(uid) + messages_to_delete.append(msg_num) + fetched_count += 1 logger.info( - f"Retrieved message {i} from account {self.account.id}" + f"Retrieved message {msg_num} (uid={uid}) " + f"from account {self.account.id}" ) except Exception as e: - logger.error(f"Error retrieving message {i}: {e}") + logger.error(f"Error retrieving message {msg_num}: {e}") # Delete messages if configured if self.account.delete_after_forward: @@ -212,19 +250,22 @@ class MailProcessor: logger.error(f"Error deleting message {msg_id}: {e}") pop_conn.quit() - return fetched_emails + return fetched, fetched_uids - emails = await loop.run_in_executor(None, fetch_pop3) + emails, new_uids = await loop.run_in_executor(None, fetch_pop3) except Exception as e: logger.error(f"Error fetching POP3 emails: {e}") raise MailFetchError(f"POP3 fetch error: {str(e)}") - return emails + return emails, new_uids - async def _fetch_imap_emails(self, max_count: int) -> List[bytes]: - """Fetch emails via IMAP""" - emails = [] + async def _fetch_imap_emails( + self, max_count: int, already_seen_uids: Set[str] + ) -> Tuple[List[bytes], List[str]]: + """Fetch emails via IMAP, marking each message \Seen to prevent re-fetch.""" + emails: List[bytes] = [] + new_uids: List[str] = [] try: # Create IMAP client @@ -241,8 +282,8 @@ class MailProcessor: await imap_client.login(self.account.username, self.password) await imap_client.select("INBOX") - # Search for all messages - response = await imap_client.search("UNSEEN") # Only fetch unread + # Search for unseen messages only + response = await imap_client.search("UNSEEN") message_ids = response.lines[0].split() # Limit to max_count @@ -254,6 +295,18 @@ class MailProcessor: # Fetch each message for msg_id in message_ids: + uid_str = msg_id.decode() if isinstance(msg_id, bytes) else str(msg_id) + + # Skip messages already tracked in our DB + if uid_str in already_seen_uids: + logger.debug( + f"Skipping already-processed IMAP message {uid_str} " + f"for account {self.account.id}" + ) + # Still mark as Seen so it doesn't show up in UNSEEN searches + await imap_client.store(msg_id, "+FLAGS", "\\Seen") + continue + try: response = await imap_client.fetch(msg_id, "(RFC822)") @@ -272,8 +325,12 @@ class MailProcessor: if email_data: emails.append(email_data) + new_uids.append(uid_str) + + # Always mark as Seen after fetching so the message is + # not picked up again on the next UNSEEN search. + await imap_client.store(msg_id, "+FLAGS", "\\Seen") - # Mark as seen if deleting after forward if self.account.delete_after_forward: await imap_client.store(msg_id, "+FLAGS", "\\Deleted") @@ -290,7 +347,7 @@ class MailProcessor: logger.error(f"Error fetching IMAP emails: {e}") raise MailFetchError(f"IMAP fetch error: {str(e)}") - return emails + return emails, new_uids @staticmethod async def forward_email( diff --git a/backend/app/workers/tasks.py b/backend/app/workers/tasks.py index 4038659..da4891a 100644 --- a/backend/app/workers/tasks.py +++ b/backend/app/workers/tasks.py @@ -9,7 +9,7 @@ import logging from app.workers.celery_app import celery_app from app.core.database import async_session_maker -from app.core.security import decrypt_credential +from app.core.security import decrypt_credential, encrypt_credential from app.models.database_models import ( MailAccount, ProcessingRun, @@ -17,12 +17,14 @@ from app.models.database_models import ( AccountStatus, DeliveryMethod, GmailCredential, + DownloadedMessageId, + UserSmtpConfig, ) from app.services.mail_processor import MailProcessor from app.services.gmail_service import GmailService from app.services.config_service import ConfigService from app.core.config import settings -from sqlalchemy import select, and_ +from sqlalchemy import select, delete logger = logging.getLogger(__name__) @@ -69,11 +71,22 @@ async def process_mail_account(account_id: int): # Decrypt password password = decrypt_credential(account.encrypted_password) # type: ignore[arg-type] + # Load already-downloaded UIDs to prevent re-processing + seen_result = await db.execute( + select(DownloadedMessageId.message_uid).where( + DownloadedMessageId.mail_account_id == account.id + ) + ) + already_seen_uids = set(seen_result.scalars().all()) + # Create processor processor = MailProcessor(account, password) - # Fetch emails - emails = await processor.fetch_emails(account.max_emails_per_check) # type: ignore[arg-type] + # Fetch emails (returns raw bytes + new UIDs) + emails, new_uids = await processor.fetch_emails( + account.max_emails_per_check, # type: ignore[arg-type] + already_seen_uids=already_seen_uids, + ) run.emails_fetched = len(emails) # type: ignore[assignment] @@ -118,8 +131,24 @@ async def process_mail_account(account_id: int): use_gmail_api = False # type: ignore[assignment] if not use_gmail_api: - # Fall back to SMTP – read config from DB with env fallback - smtp_config = await ConfigService.get_smtp_config(db=db) + # Fall back to SMTP – check per-user config first, then global + user_smtp_result = await db.execute( + select(UserSmtpConfig).where( + UserSmtpConfig.user_id == account.user_id + ) + ) + user_smtp = user_smtp_result.scalar_one_or_none() + + if user_smtp and user_smtp.username and user_smtp.encrypted_password: + smtp_config = { + "host": user_smtp.host, + "port": user_smtp.port, + "username": user_smtp.username, + "password": decrypt_credential(user_smtp.encrypted_password), # type: ignore[arg-type] + "use_tls": user_smtp.use_tls, + } + else: + smtp_config = await ConfigService.get_smtp_config(db=db) if not smtp_config["username"] or not smtp_config["password"]: logger.error( @@ -130,7 +159,15 @@ async def process_mail_account(account_id: int): await db.commit() return - for email_data in emails: + successfully_forwarded_uids: list[str] = [] + + if len(emails) != len(new_uids): + logger.error( + f"emails/uids length mismatch ({len(emails)} vs {len(new_uids)}) " + f"for account {account.id}; truncating to shorter list" + ) + + for email_data, uid in zip(emails, new_uids): try: if use_gmail_api and gmail_service: # Inject via Gmail API (preferred) @@ -140,6 +177,7 @@ async def process_mail_account(account_id: int): source_account_name=account.name, # type: ignore[arg-type] ) emails_forwarded += 1 + successfully_forwarded_uids.append(uid) else: # Forward via SMTP (fallback) success = await MailProcessor.forward_email( @@ -147,13 +185,54 @@ async def process_mail_account(account_id: int): ) if success: emails_forwarded += 1 + successfully_forwarded_uids.append(uid) else: emails_failed += 1 except Exception as e: + error_str = str(e).lower() + # If Gmail returns 401/403 the refresh token was revoked – + # mark credentials invalid so the user gets notified. + if ( + use_gmail_api + and gmail_cred + and ( + "401" in error_str + or "403" in error_str + or "invalid_grant" in error_str + ) + ): + gmail_cred.is_valid = False # type: ignore[assignment] + logger.warning( + f"Gmail credentials revoked for user {account.user_id}. " + "User must re-authorise." + ) logger.error(f"Error delivering email: {e}") emails_failed += 1 + # Persist new message UIDs so they are not processed again + for uid in successfully_forwarded_uids: + if uid not in already_seen_uids: + db.add( + DownloadedMessageId( + mail_account_id=account.id, + message_uid=uid, + ) + ) + + # If Gmail API was used, persist any refreshed access token back to + # the DB so the next run doesn't need an extra token-refresh call. + if use_gmail_api and gmail_service and gmail_cred: + refreshed = gmail_service.get_refreshed_token() + if refreshed and refreshed["access_token"] != access_token: + gmail_cred.encrypted_access_token = encrypt_credential(refreshed["access_token"]) # type: ignore[assignment] + if refreshed.get("expiry"): + gmail_cred.token_expiry = refreshed["expiry"] # type: ignore[assignment] + gmail_cred.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment] + logger.info( + f"Persisted refreshed Gmail access token for user {account.user_id}" + ) + # Update run run.emails_forwarded = emails_forwarded # type: ignore[assignment] run.emails_failed = emails_failed # type: ignore[assignment] @@ -210,15 +289,11 @@ async def process_all_enabled_accounts(): """ async with async_session_maker() as db: try: - # Get all enabled accounts + # Fetch all enabled accounts regardless of operational status so + # that accounts in ERROR state are retried automatically. result = await db.execute( select(MailAccount).where( - and_( - MailAccount.is_enabled == True, # noqa: E712 - MailAccount.status.in_( - [AccountStatus.ACTIVE, AccountStatus.TESTING] - ), - ) + MailAccount.is_enabled == True, # noqa: E712 ) ) accounts = result.scalars().all() @@ -248,10 +323,10 @@ async def process_all_enabled_accounts(): @celery_app.task(base=AsyncTask, name="app.workers.tasks.cleanup_old_logs") async def cleanup_old_logs(days_to_keep: int = 30): """ - Clean up old processing logs and runs. + Clean up old processing logs, runs, and downloaded message ID records. Args: - days_to_keep: Number of days of logs to retain + days_to_keep: Number of days of data to retain """ async with async_session_maker() as db: try: @@ -275,6 +350,14 @@ async def cleanup_old_logs(days_to_keep: int = 30): for log in old_logs: await db.delete(log) + # Delete old downloaded message ID records so the table doesn't + # grow unboundedly for accounts that never delete messages. + await db.execute( + delete(DownloadedMessageId).where( + DownloadedMessageId.downloaded_at < cutoff_date + ) + ) + await db.commit() logger.info( diff --git a/docs/TODO.md b/docs/TODO.md index 23e2604..a0679b8 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -179,6 +179,10 @@ Comprehensive task breakdown for repository improvements and production readines ### Not Started 📋 - [ ] Implement Stripe webhook handling - [ ] Add scheduled Celery tasks for email processing +- [x] Account enable/disable toggle (UX + backend) +- [x] Per-user SMTP configuration (UX + backend) +- [x] Gmail API one-click OAuth grant flow with token refresh and revocation handling +- [x] Message deduplication (POP3 UIDL + IMAP \Seen flag + DB tracking) - [ ] Implement GDPR data export endpoint - [ ] Complete notification service integration (Apprise) - [ ] Add advanced email filtering @@ -207,12 +211,13 @@ because the API client layer is missing. - [x] Registration page - [x] OAuth callback handler - [x] Dashboard with stats cards and processing runs table -- [x] Mail accounts list with CRUD operations -- [x] Settings page -- [x] `AddMailAccountModal` component (auto-detect, test connection, all required fields) +- [x] Mail accounts list with CRUD operations + enable/disable toggle +- [x] Settings page — Profile, Gmail API connection, SMTP relay, Account info, Security +- [x] `AddMailAccountModal` component (auto-detect, test connection, all required fields, is_enabled checkbox) - [x] `DashboardLayout` with responsive sidebar - [x] `AuthGuard` for protected routes - [x] Fix wizard grey screen (Tailwind v4 `bg-opacity` → `/75` syntax, modal restructure) +- [x] `/auth/gmail-callback` page for Gmail OAuth one-click flow ### Not Started 📋 - [ ] End-to-end testing of frontend against backend API diff --git a/frontend/src/app/accounts/page.tsx b/frontend/src/app/accounts/page.tsx index 0a69fe8..35e08e5 100644 --- a/frontend/src/app/accounts/page.tsx +++ b/frontend/src/app/accounts/page.tsx @@ -4,7 +4,7 @@ import { AuthGuard } from '@/components/AuthGuard'; import { DashboardLayout } from '@/components/DashboardLayout'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { mailAccountsApi, MailAccount } from '@/lib/api'; -import { Plus, Edit2, Trash2, CheckCircle, XCircle, AlertTriangle } from 'lucide-react'; +import { Plus, Edit2, Trash2, CheckCircle, XCircle, AlertTriangle, Power } from 'lucide-react'; import { useState } from 'react'; import { AddMailAccountModal } from '@/components/AddMailAccountModal'; @@ -25,6 +25,13 @@ export default function AccountsPage() { }, }); + const toggleMutation = useMutation({ + mutationFn: mailAccountsApi.toggle, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['mail-accounts'] }); + }, + }); + const handleEdit = (account: MailAccount) => { setEditingAccount(account); setIsModalOpen(true); @@ -40,6 +47,14 @@ export default function AccountsPage() { } }; + const handleToggle = async (id: number) => { + try { + await toggleMutation.mutateAsync(id); + } catch { + alert('Failed to update account'); + } + }; + const handleCloseModal = () => { setIsModalOpen(false); setEditingAccount(null); @@ -69,7 +84,9 @@ export default function AccountsPage() { {accounts.map((account) => (
+ Grant this app permission to inject emails directly into your Gmail inbox. + This is the preferred delivery method — emails arrive with original headers + intact, bypassing SMTP entirely. +
++ Token lifetime: Access tokens expire after 1 hour and are + refreshed automatically. Refresh tokens do not expire unless you revoke + access via your{' '} + + Google Account permissions + + . If revoked, click “Connect Gmail” again to re-authorise. +
+ + {gmailLoading && ( ++ Connected as {gmailCredential.gmail_email} +
+ {gmailCredential.last_verified_at && ( ++ Last verified: {new Date(gmailCredential.last_verified_at).toLocaleString()} +
+ )} ++ Used when Gmail API is not connected or when a mail account is configured + to use SMTP delivery. Your credentials are stored encrypted. +
+ + {smtpLoading ? ( +