diff --git a/app/api/__init__.py b/app/api/__init__.py index 5b158592..8817768f 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -16,6 +16,7 @@ from app.api.dropbox import router as dropbox_router from app.api.duplicates import router as duplicates_router from app.api.files import router as files_router from app.api.google_drive import router as google_drive_router +from app.api.imap_accounts import router as imap_accounts_router from app.api.logs import router as logs_router from app.api.onboarding import router as onboarding_router from app.api.onedrive import router as onedrive_router @@ -68,3 +69,4 @@ router.include_router(plans_router) router.include_router(onboarding_router) router.include_router(billing_router) router.include_router(pipelines_router) +router.include_router(imap_accounts_router) diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py new file mode 100644 index 00000000..ace727b3 --- /dev/null +++ b/app/api/imap_accounts.py @@ -0,0 +1,350 @@ +"""API endpoints for managing per-user IMAP ingestion accounts. + +Provides CRUD operations for a user's IMAP accounts, quota enforcement +against their subscription plan's ``max_mailboxes`` limit, and a +test-connection endpoint so users can verify credentials before saving. +""" + +import imaplib +import logging +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models import UserImapAccount +from app.utils.subscription import get_tier, get_user_tier_id +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/imap-accounts", tags=["imap-accounts"]) + +DbSession = Annotated[Session, Depends(get_db)] + +# --------------------------------------------------------------------------- +# Auth helpers +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if owner_id is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + + +# --------------------------------------------------------------------------- +# Quota helpers +# --------------------------------------------------------------------------- + +_FREE_TIER_ID = "free" + + +def _get_max_mailboxes(tier: dict[str, Any]) -> int | None: + """Return the maximum number of IMAP accounts allowed by *tier*. + + Returns: + ``None`` — unlimited (paid tiers with ``max_mailboxes == 0``) + ``0`` — no mailboxes allowed (free tier) + positive — the configured limit + """ + tier_id: str = tier.get("id", _FREE_TIER_ID) + max_mb: int = tier.get("max_mailboxes", 0) + + # Free tier: 0 means "no access" (not "unlimited") + if tier_id == _FREE_TIER_ID: + return 0 + + # Paid tiers: 0 means unlimited + if max_mb == 0: + return None + + return max_mb + + +def _check_quota(db: Session, owner_id: str) -> None: + """Raise 403 if the user has reached their IMAP account quota.""" + tier_id = get_user_tier_id(db, owner_id) + tier = get_tier(tier_id, db) + max_mb = _get_max_mailboxes(tier) + + if max_mb == 0: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=("Your current plan does not include email ingestion. Upgrade to a paid plan to add IMAP accounts."), + ) + + if max_mb is not None: + current_count = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).count() + if current_count >= max_mb: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=( + f"You have reached your plan limit of {max_mb} IMAP account(s). " + "Please delete an existing account or upgrade your plan." + ), + ) + + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + + +class ImapAccountCreate(BaseModel): + """Schema for creating a new IMAP account.""" + + name: str = Field(..., min_length=1, max_length=255, description="Human-readable label") + host: str = Field(..., min_length=1, max_length=255, description="IMAP server hostname") + port: int = Field(default=993, ge=1, le=65535, description="IMAP server port") + username: str = Field(..., min_length=1, max_length=255, description="IMAP login username") + password: str = Field(..., min_length=1, max_length=1024, description="IMAP login password") + use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") + delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") + is_active: bool = Field(default=True, description="Whether to poll this mailbox") + + +class ImapAccountUpdate(BaseModel): + """Schema for updating an existing IMAP account (all fields optional).""" + + name: str | None = Field(default=None, min_length=1, max_length=255) + host: str | None = Field(default=None, min_length=1, max_length=255) + port: int | None = Field(default=None, ge=1, le=65535) + username: str | None = Field(default=None, min_length=1, max_length=255) + password: str | None = Field(default=None, min_length=1, max_length=1024) + use_ssl: bool | None = None + delete_after_process: bool | None = None + is_active: bool | None = None + + +class ImapTestRequest(BaseModel): + """Schema for testing an IMAP connection without saving it.""" + + host: str = Field(..., min_length=1, max_length=255) + port: int = Field(default=993, ge=1, le=65535) + username: str = Field(..., min_length=1, max_length=255) + password: str = Field(..., min_length=1, max_length=1024) + use_ssl: bool = Field(default=True) + + +# --------------------------------------------------------------------------- +# Serialisation helpers +# --------------------------------------------------------------------------- + + +def _to_response(acct: UserImapAccount) -> dict[str, Any]: + """Serialize a ``UserImapAccount`` row to a response dict. + + Passwords are never included in responses. + """ + return { + "id": acct.id, + "owner_id": acct.owner_id, + "name": acct.name, + "host": acct.host, + "port": acct.port, + "username": acct.username, + "use_ssl": acct.use_ssl, + "delete_after_process": acct.delete_after_process, + "is_active": acct.is_active, + "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, + "last_error": acct.last_error, + "created_at": acct.created_at.isoformat() if acct.created_at else None, + "updated_at": acct.updated_at.isoformat() if acct.updated_at else None, + } + + +# --------------------------------------------------------------------------- +# Connection test helper +# --------------------------------------------------------------------------- + + +def _test_imap_connection(host: str, port: int, username: str, password: str, use_ssl: bool) -> dict[str, Any]: + """Attempt to connect and log in to the IMAP server. + + Returns a dict with ``{"success": bool, "message": str}``. + """ + try: + if use_ssl: + mail = imaplib.IMAP4_SSL(host, port) + else: + mail = imaplib.IMAP4(host, port) + + mail.login(username, password) + mail.logout() + return {"success": True, "message": "Connection successful"} + except OSError as exc: + logger.warning("IMAP network error for %s@%s: %s", username, host, exc) + return {"success": False, "message": f"Connection error: {exc}"} + except Exception as exc: # noqa: BLE001 + logger.warning("IMAP error for %s@%s: %s", username, host, exc) + return {"success": False, "message": f"IMAP error: {exc}"} + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/", summary="List IMAP accounts for the current user") +def list_imap_accounts(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return all IMAP accounts belonging to the authenticated user.""" + accounts = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).order_by(UserImapAccount.id).all() + return [_to_response(a) for a in accounts] + + +@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new IMAP account") +def create_imap_account( + request: Request, body: ImapAccountCreate, db: DbSession, owner_id: CurrentOwner +) -> dict[str, Any]: + """Create a new IMAP ingestion account for the current user. + + Quota is enforced against the user's subscription plan's ``max_mailboxes`` + limit before the account is persisted. + """ + _check_quota(db, owner_id) + + acct = UserImapAccount( + owner_id=owner_id, + name=body.name, + host=body.host, + port=body.port, + username=body.username, + password=body.password, + use_ssl=body.use_ssl, + delete_after_process=body.delete_after_process, + is_active=body.is_active, + ) + try: + db.add(acct) + db.commit() + db.refresh(acct) + except Exception: + db.rollback() + raise + + logger.info("User %s created IMAP account %d (%s)", owner_id, acct.id, body.host) + return _to_response(acct) + + +@router.get("/{account_id}", summary="Get a single IMAP account") +def get_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Return a single IMAP account by ID (must belong to the current user).""" + acct = ( + db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first() + ) + if not acct: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found") + return _to_response(acct) + + +@router.put("/{account_id}", summary="Update an IMAP account") +def update_imap_account( + account_id: int, + request: Request, + body: ImapAccountUpdate, + db: DbSession, + owner_id: CurrentOwner, +) -> dict[str, Any]: + """Update an existing IMAP account. Only provided fields are changed.""" + acct = ( + db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first() + ) + if not acct: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found") + + if body.name is not None: + acct.name = body.name + if body.host is not None: + acct.host = body.host + if body.port is not None: + acct.port = body.port + if body.username is not None: + acct.username = body.username + if body.password is not None: + acct.password = body.password + if body.use_ssl is not None: + acct.use_ssl = body.use_ssl + if body.delete_after_process is not None: + acct.delete_after_process = body.delete_after_process + if body.is_active is not None: + acct.is_active = body.is_active + + # Reset last_error so the next poll gives a fresh result + acct.last_error = None + acct.updated_at = datetime.now(timezone.utc) + + try: + db.commit() + db.refresh(acct) + except Exception: + db.rollback() + raise + + logger.info("User %s updated IMAP account %d", owner_id, account_id) + return _to_response(acct) + + +@router.delete("/{account_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an IMAP account") +def delete_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None: + """Delete an IMAP account permanently.""" + acct = ( + db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first() + ) + if not acct: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found") + + try: + db.delete(acct) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("User %s deleted IMAP account %d", owner_id, account_id) + + +@router.post("/{account_id}/test", summary="Test an existing IMAP account's connection") +def test_saved_imap_account(account_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Test the connection for an already-saved IMAP account.""" + acct = ( + db.query(UserImapAccount).filter(UserImapAccount.id == account_id, UserImapAccount.owner_id == owner_id).first() + ) + if not acct: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found") + + return _test_imap_connection(acct.host, acct.port, acct.username, acct.password, acct.use_ssl) + + +@router.post("/test", summary="Test an IMAP connection without saving") +def test_imap_connection(request: Request, body: ImapTestRequest, owner_id: CurrentOwner) -> dict[str, Any]: + """Test IMAP credentials without persisting anything. + + Useful for the "Test connection" button in the UI before the user saves + a new account. + """ + return _test_imap_connection(body.host, body.port, body.username, body.password, body.use_ssl) + + +@router.get("/quota/", summary="Get IMAP account quota information for the current user") +def get_imap_quota(request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Return the user's current IMAP account usage vs. their plan quota.""" + tier_id = get_user_tier_id(db, owner_id) + tier = get_tier(tier_id, db) + max_mb = _get_max_mailboxes(tier) + current_count = db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).count() + + return { + "current_count": current_count, + "max_mailboxes": max_mb, # None = unlimited, 0 = not allowed + "can_add": max_mb is None or (max_mb > 0 and current_count < max_mb), + "tier_id": tier_id, + "tier_name": tier.get("name", tier_id), + } diff --git a/app/models.py b/app/models.py index 811c6fb4..a1bb99d5 100644 --- a/app/models.py +++ b/app/models.py @@ -376,6 +376,53 @@ class PipelineStep(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class UserImapAccount(Base): + """Per-user IMAP ingestion account. + + Each row represents one IMAP mailbox that a user wants DocuElevate to + poll for document attachments. The periodic ``pull_all_inboxes`` Celery + task iterates over all active accounts and processes any new emails. + + Quota enforcement: the user's subscription plan's ``max_mailboxes`` field + controls how many accounts a user may configure (0 = unlimited for paid + plans; free tier is not permitted any accounts). + """ + + __tablename__ = "user_imap_accounts" + + id = Column(Integer, primary_key=True, index=True) + + # Stable owner identifier — matches FileRecord.owner_id + owner_id = Column(String, nullable=False, index=True) + + # Human-readable label chosen by the user (e.g. "Work Gmail", "Scanner mailbox") + name = Column(String(255), nullable=False) + + # IMAP connection settings + host = Column(String(255), nullable=False) + port = Column(Integer, nullable=False, default=993) + username = Column(String(255), nullable=False) + # Password stored in plain text — the admin is responsible for access control + password = Column(String(1024), nullable=False) + use_ssl = Column(Boolean, nullable=False, default=True) + + # Processing options + # When True, emails are deleted from the mailbox after their attachments are processed + delete_after_process = Column(Boolean, nullable=False, default=False) + + # When False the account is not polled by the periodic task (but not deleted) + is_active = Column(Boolean, nullable=False, default=True) + + # Last time this mailbox was successfully polled + last_checked_at = Column(DateTime(timezone=True), nullable=True) + + # Last error message if the most recent poll failed (NULL = last poll succeeded) + last_error = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class BackupRecord(Base): """Tracks database backup files and their retention metadata. diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index 981fdbcd..4677fa41 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -15,6 +15,20 @@ from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES +# Database session for per-user IMAP accounts (imported lazily to avoid circular imports) +_db_session_factory = None + + +def _get_db_session(): + """Return a new SQLAlchemy session (lazy import to avoid startup issues).""" + global _db_session_factory # noqa: PLW0603 + if _db_session_factory is None: + from app.database import SessionLocal + + _db_session_factory = SessionLocal + return _db_session_factory() + + logger = logging.getLogger(__name__) # Initialize Redis connection using Celery's Redis settings @@ -82,6 +96,10 @@ def pull_all_inboxes(): Periodic Celery task that checks all configured IMAP mailboxes and fetches attachments from new emails. Ensures only one instance runs at a time using Redis-based locking. + + Processes: + 1. System-level mailboxes configured via environment variables (IMAP1, IMAP2). + 2. Per-user IMAP accounts stored in the ``user_imap_accounts`` database table. """ if not acquire_lock(): logger.info("Skipping execution: Another instance is running.") @@ -112,12 +130,61 @@ def pull_all_inboxes(): delete_after_process=settings.imap2_delete_after_process, ) + # Per-user IMAP accounts from the database + _pull_user_imap_accounts() + logger.info("Finished pull_all_inboxes") finally: release_lock() +def _pull_user_imap_accounts() -> None: + """Iterate over all active per-user IMAP accounts and pull their inboxes.""" + try: + from app.models import UserImapAccount + + db = _get_db_session() + try: + accounts = db.query(UserImapAccount).filter(UserImapAccount.is_active.is_(True)).all() + logger.info("Processing %d per-user IMAP account(s)", len(accounts)) + for acct in accounts: + mailbox_key = f"user_{acct.owner_id}_{acct.id}" + try: + pull_inbox( + mailbox_key=mailbox_key, + host=acct.host, + port=acct.port, + username=acct.username, + password=acct.password, + use_ssl=acct.use_ssl, + delete_after_process=acct.delete_after_process, + ) + # Record successful poll + acct.last_checked_at = datetime.now(timezone.utc) + acct.last_error = None + db.commit() + except Exception as exc: # noqa: BLE001 + error_msg = str(exc)[:500] + logger.error( + "Error pulling user IMAP account %d (%s@%s): %s", + acct.id, + acct.username, + acct.host, + error_msg, + ) + try: + acct.last_checked_at = datetime.now(timezone.utc) + acct.last_error = error_msg + db.commit() + except Exception: # noqa: BLE001 + db.rollback() + finally: + db.close() + except Exception as exc: # noqa: BLE001 + logger.error("Failed to process per-user IMAP accounts: %s", exc) + + def check_and_pull_mailbox( mailbox_key: str, host: str | None, diff --git a/app/views/__init__.py b/app/views/__init__.py index e499da1e..c57c027a 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -14,6 +14,7 @@ from app.views.filemanager import router as filemanager_router from app.views.general import router as general_router from app.views.google_drive import router as google_drive_router from app.views.help import router as help_router # Built-in help / How-To docs +from app.views.imap_accounts import router as imap_accounts_router from app.views.license_routes import router as license_router # Add the license router from app.views.onboarding import router as onboarding_router from app.views.onedrive import router as onedrive_router @@ -46,4 +47,5 @@ router.include_router(subscriptions_router) # Pricing + subscription pages router.include_router(plans_router) # Admin Plan Designer router.include_router(onboarding_router) # User onboarding wizard router.include_router(pipelines_router) # Processing pipelines +router.include_router(imap_accounts_router) # Per-user IMAP ingestion accounts router.include_router(help_router) # Built-in help / How-To docs diff --git a/app/views/imap_accounts.py b/app/views/imap_accounts.py new file mode 100644 index 00000000..460c83ba --- /dev/null +++ b/app/views/imap_accounts.py @@ -0,0 +1,63 @@ +"""User-facing view for the per-user IMAP ingestion dashboard.""" + +import logging + +from fastapi import Request +from sqlalchemy.orm import Session + +from app.models import UserImapAccount +from app.utils.subscription import get_tier, get_user_tier_id +from app.utils.user_scope import get_current_owner_id +from app.views.base import APIRouter, Depends, get_db, require_login, templates + +logger = logging.getLogger(__name__) +router = APIRouter() + + +def _get_max_mailboxes(tier: dict) -> int | None: + """Mirror of the quota helper from the API module — avoids a circular import.""" + tier_id: str = tier.get("id", "free") + max_mb: int = tier.get("max_mailboxes", 0) + if tier_id == "free": + return 0 + if max_mb == 0: + return None + return max_mb + + +@router.get("/imap-accounts") +@require_login +async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): + """IMAP ingestion account management page for the current user.""" + owner_id = get_current_owner_id(request) + + accounts: list[UserImapAccount] = [] + current_count = 0 + max_mailboxes: int | None = 0 + can_add = False + tier_name = "Free" + tier_id = "free" + + if owner_id: + accounts = ( + db.query(UserImapAccount).filter(UserImapAccount.owner_id == owner_id).order_by(UserImapAccount.id).all() + ) + current_count = len(accounts) + tier_id = get_user_tier_id(db, owner_id) + tier = get_tier(tier_id, db) + tier_name = tier.get("name", tier_id) + max_mailboxes = _get_max_mailboxes(tier) + can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes) + + return templates.TemplateResponse( + "imap_accounts.html", + { + "request": request, + "accounts": accounts, + "current_count": current_count, + "max_mailboxes": max_mailboxes, + "can_add": can_add, + "tier_id": tier_id, + "tier_name": tier_name, + }, + ) diff --git a/docs/UserGuide.md b/docs/UserGuide.md index a6332bc9..d06943a4 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -107,6 +107,39 @@ DocuElevate acts as an email *client* that automatically retrieves document atta > **HP Scanners and MFPs (Scan to Email)**: Configure your scanner's "Scan to Email" feature to send scanned documents to a dedicated email account. Point DocuElevate at that mailbox using the IMAP settings. DocuElevate will retrieve the scanned PDFs automatically — no manual forwarding required. +#### Per-User IMAP Accounts (Email Ingestion Dashboard) + +In addition to the system-wide IMAP mailboxes configured by the administrator via environment variables, each user can configure their own personal IMAP accounts directly from the **Email Ingestion** page (`/imap-accounts`). + +**To add a personal IMAP account:** + +1. Navigate to **Email Ingestion** in the top navigation bar. +2. Click **Add Account**. +3. Fill in: + - **Label** — a friendly name for this account (e.g. "Work Gmail", "Scanner inbox") + - **IMAP Host** — your mail server hostname (e.g. `imap.gmail.com`) + - **Port** — typically `993` for SSL or `143` for plain/STARTTLS + - **Username** — usually your full email address + - **Password** — your email password or app-specific password +4. Select **Use SSL/TLS** (recommended). +5. Click **Test Connection** to verify the credentials before saving. +6. Click **Add Account** to save. + +**Account options:** +- **Active** — when checked, the mailbox is polled on each cycle; uncheck to pause without deleting. +- **Delete emails after processing** — when checked, processed emails are permanently deleted from the mailbox instead of being marked as read/labelled. + +**Quota limits** are determined by your subscription plan: + +| Plan | IMAP accounts | +|--------------|---------------| +| Free | None (not available) | +| Starter | 1 | +| Professional | 3 | +| Power | Unlimited | + +The quota bar on the Email Ingestion page shows your current usage against your plan limit. If you have reached the limit, delete an existing account or upgrade your plan. + ### Watch Folders (Automatic Folder Ingestion) Watch folders allow DocuElevate to automatically monitor directories for new files and ingest them without any manual action. diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 6fcbfa37..37a7414b 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -106,6 +106,11 @@ {% if request and request.url.path == '/pipelines' %}aria-current="page"{% endif %}> Pipelines + + Email Ingestion +