From af44af98b894eadd8e3513ce8d25214066f837ef Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:17:39 +0000 Subject: [PATCH 01/16] Initial plan From b03ea416386fac4d550d320df62ba4a38d33d0dd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:30:08 +0000 Subject: [PATCH 02/16] Initial plan From 5a77e36ac688444f2dae939a4d0df1cca14f6204 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:34:13 +0000 Subject: [PATCH 03/16] Initial plan From fcefd0978f36fc66ec2a34c9ab26021e38b1fcfb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:34:25 +0000 Subject: [PATCH 04/16] feat(notifications): add per-user notification system with inbox, email, and webhook targets - Add UserNotificationTarget, UserNotificationPreference, InAppNotification models - Add migration 025_add_user_notifications (tables + indexes) - Add app/utils/user_notification.py dispatch service - Add app/api/notifications.py REST endpoints (inbox, targets, preferences) - Add app/views/notifications.py view route - Add frontend/templates/notifications_dashboard.html Alpine.js dashboard - Add bell icon with unread badge in base.html nav (desktop + mobile) - Register routers in app/api/__init__.py and app/views/__init__.py - Add 32 unit tests in tests/test_notifications_api.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/api/__init__.py | 2 + app/api/notifications.py | 484 +++++++++ app/models.py | 47 + app/utils/user_notification.py | 227 ++++ app/views/__init__.py | 2 + app/views/notifications.py | 20 + frontend/templates/base.html | 43 + .../templates/notifications_dashboard.html | 979 ++++++++++++++++++ .../versions/025_add_user_notifications.py | 83 ++ tests/test_notifications_api.py | 832 +++++++++++++++ 10 files changed, 2719 insertions(+) create mode 100644 app/api/notifications.py create mode 100644 app/utils/user_notification.py create mode 100644 app/views/notifications.py create mode 100644 frontend/templates/notifications_dashboard.html create mode 100644 migrations/versions/025_add_user_notifications.py create mode 100644 tests/test_notifications_api.py diff --git a/app/api/__init__.py b/app/api/__init__.py index dcee1703..311c572e 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -20,6 +20,7 @@ from app.api.google_drive import router as google_drive_router from app.api.imap_accounts import router as imap_accounts_router from app.api.integrations import router as integrations_router from app.api.logs import router as logs_router +from app.api.notifications import router as notifications_router from app.api.onboarding import router as onboarding_router from app.api.onedrive import router as onedrive_router from app.api.openai import router as openai_router @@ -74,3 +75,4 @@ router.include_router(billing_router) router.include_router(pipelines_router) router.include_router(imap_accounts_router) router.include_router(integrations_router) +router.include_router(notifications_router) diff --git a/app/api/notifications.py b/app/api/notifications.py new file mode 100644 index 00000000..7927b255 --- /dev/null +++ b/app/api/notifications.py @@ -0,0 +1,484 @@ +"""API endpoints for per-user notification targets, preferences, and in-app inbox. + +Users can define notification targets (email via SMTP, webhook via HTTP POST) +and configure which document events trigger which targets. In-app notifications +are always created and surfaced via the bell icon / inbox endpoints. +""" + +import json +import logging +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models import InAppNotification, UserNotificationPreference, UserNotificationTarget +from app.utils.user_notification import USER_EVENT_LABELS +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/user-notifications", tags=["user-notifications"]) + +DbSession = Annotated[Session, Depends(get_db)] + +# --------------------------------------------------------------------------- +# Auth helper (mirrors api_tokens.py pattern) +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if not owner_id: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + +VALID_CHANNEL_TYPES = {"email", "webhook"} +VALID_EVENT_TYPES = set(USER_EVENT_LABELS.keys()) + + +class NotificationTargetCreate(BaseModel): + """Schema for creating a new notification target.""" + + channel_type: str = Field(..., pattern="^(email|webhook)$") + name: str = Field(..., min_length=1, max_length=255) + config: dict[str, Any] = Field(default_factory=dict) + is_active: bool = True + + +class NotificationTargetUpdate(BaseModel): + """Schema for updating an existing notification target.""" + + name: str | None = Field(None, min_length=1, max_length=255) + config: dict[str, Any] | None = None + is_active: bool | None = None + + +class PreferenceItem(BaseModel): + """A single preference toggle for one event+channel combination.""" + + is_enabled: bool + target_id: int | None = None + + +class PreferenceItemFull(BaseModel): + """Full preference item including event and channel type (used in bulk update).""" + + event_type: str + channel_type: str + is_enabled: bool + target_id: int | None = None + + +class PreferencesUpdate(BaseModel): + """Bulk preferences update payload — a flat list of preference items.""" + + preferences: list[PreferenceItemFull] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _mask_email_config(config: dict[str, Any]) -> dict[str, Any]: + """Return a copy of an email config dict with the password masked.""" + masked = dict(config) + if masked.get("smtp_password"): + masked["smtp_password"] = "****" + return masked + + +def _target_to_dict(target: UserNotificationTarget) -> dict[str, Any]: + """Serialize a UserNotificationTarget to a response dict, masking secrets.""" + config: dict[str, Any] = {} + if target.config: + try: + config = json.loads(target.config) + except (json.JSONDecodeError, ValueError): + config = {} + + if target.channel_type == "email": + config = _mask_email_config(config) + + return { + "id": target.id, + "channel_type": target.channel_type, + "name": target.name, + "config": config, + "is_active": target.is_active, + "created_at": target.created_at, + "updated_at": target.updated_at, + } + + +# --------------------------------------------------------------------------- +# Inbox endpoints +# --------------------------------------------------------------------------- + + +@router.get("/inbox") +async def list_inbox( + owner_id: CurrentOwner, + db: DbSession, + skip: int = 0, + limit: int = 50, +) -> list[dict[str, Any]]: + """List in-app notifications for the authenticated user, newest first.""" + notifications = ( + db.query(InAppNotification) + .filter(InAppNotification.owner_id == owner_id) + .order_by(InAppNotification.created_at.desc()) + .offset(skip) + .limit(limit) + .all() + ) + return [ + { + "id": n.id, + "event_type": n.event_type, + "title": n.title, + "message": n.message, + "is_read": n.is_read, + "file_id": n.file_id, + "created_at": n.created_at, + } + for n in notifications + ] + + +@router.get("/inbox/unread-count") +async def unread_count( + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, int]: + """Return the number of unread in-app notifications.""" + count = ( + db.query(InAppNotification) + .filter(InAppNotification.owner_id == owner_id, InAppNotification.is_read == False) # noqa: E712 + .count() + ) + return {"count": count} + + +@router.post("/inbox/{notification_id}/read", status_code=status.HTTP_200_OK) +async def mark_read( + notification_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, str]: + """Mark a single in-app notification as read.""" + notif = ( + db.query(InAppNotification) + .filter(InAppNotification.id == notification_id, InAppNotification.owner_id == owner_id) + .first() + ) + if not notif: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Notification not found") + try: + notif.is_read = True + db.commit() + except Exception: + db.rollback() + raise + return {"detail": "Marked as read"} + + +@router.post("/inbox/read-all", status_code=status.HTTP_200_OK) +async def mark_all_read( + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, str]: + """Mark all in-app notifications as read for the authenticated user.""" + try: + db.query(InAppNotification).filter( + InAppNotification.owner_id == owner_id, + InAppNotification.is_read == False, # noqa: E712 + ).update({"is_read": True}) + db.commit() + except Exception: + db.rollback() + raise + return {"detail": "All notifications marked as read"} + + +# --------------------------------------------------------------------------- +# Notification target endpoints +# --------------------------------------------------------------------------- + + +@router.get("/targets") +async def list_targets( + owner_id: CurrentOwner, + db: DbSession, +) -> list[dict[str, Any]]: + """List all notification targets for the authenticated user.""" + targets = ( + db.query(UserNotificationTarget) + .filter(UserNotificationTarget.owner_id == owner_id) + .order_by(UserNotificationTarget.created_at.desc()) + .all() + ) + return [_target_to_dict(t) for t in targets] + + +@router.post("/targets", status_code=status.HTTP_201_CREATED) +async def create_target( + body: NotificationTargetCreate, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Create a new notification target (email or webhook).""" + target = UserNotificationTarget( + owner_id=owner_id, + channel_type=body.channel_type, + name=body.name, + config=json.dumps(body.config), + is_active=body.is_active, + ) + try: + db.add(target) + db.commit() + db.refresh(target) + except Exception: + db.rollback() + raise + + logger.info("Notification target created: id=%s owner=%s type=%s", target.id, owner_id, body.channel_type) + return _target_to_dict(target) + + +@router.put("/targets/{target_id}", status_code=status.HTTP_200_OK) +async def update_target( + target_id: int, + body: NotificationTargetUpdate, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Update an existing notification target.""" + target = ( + db.query(UserNotificationTarget) + .filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id) + .first() + ) + if not target: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found") + + try: + if body.name is not None: + target.name = body.name + if body.config is not None: + # Merge new config over existing, preserving masked password field if unchanged + existing_config: dict[str, Any] = {} + if target.config: + try: + existing_config = json.loads(target.config) + except (json.JSONDecodeError, ValueError): + existing_config = {} + merged = dict(existing_config) + for k, v in body.config.items(): + # Skip writing back a masked password placeholder + if k == "smtp_password" and v == "****": + continue + merged[k] = v + target.config = json.dumps(merged) + if body.is_active is not None: + target.is_active = body.is_active + db.commit() + db.refresh(target) + except Exception: + db.rollback() + raise + + logger.info("Notification target updated: id=%s owner=%s", target_id, owner_id) + return _target_to_dict(target) + + +@router.delete("/targets/{target_id}", status_code=status.HTTP_200_OK) +async def delete_target( + target_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, str]: + """Delete a notification target and its associated preferences.""" + target = ( + db.query(UserNotificationTarget) + .filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id) + .first() + ) + if not target: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found") + + try: + # Remove any preferences that reference this target + db.query(UserNotificationPreference).filter( + UserNotificationPreference.owner_id == owner_id, + UserNotificationPreference.target_id == target_id, + ).delete() + db.delete(target) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("Notification target deleted: id=%s owner=%s", target_id, owner_id) + return {"detail": "Target deleted"} + + +@router.post("/targets/{target_id}/test", status_code=status.HTTP_200_OK) +async def test_target( + target_id: int, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, str]: + """Send a test notification to the specified target.""" + target = ( + db.query(UserNotificationTarget) + .filter(UserNotificationTarget.id == target_id, UserNotificationTarget.owner_id == owner_id) + .first() + ) + if not target: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Target not found") + + config: dict[str, Any] = {} + if target.config: + try: + config = json.loads(target.config) + except (json.JSONDecodeError, ValueError): + config = {} + + title = "DocuElevate Test Notification" + message = f"This is a test notification from DocuElevate for target '{target.name}'." + + if target.channel_type == "email": + from app.utils.user_notification import _send_email_notification + + ok = _send_email_notification(config, title, message) + elif target.channel_type == "webhook": + from app.utils.user_notification import _send_webhook_notification + + ok = _send_webhook_notification(config, "test", title, message) + else: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown channel type") + + if not ok: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="Failed to send test notification") + + return {"detail": "Test notification sent"} + + +# --------------------------------------------------------------------------- +# Preferences endpoints +# --------------------------------------------------------------------------- + + +@router.get("/preferences") +async def get_preferences( + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, Any]: + """Return all notification preferences for the authenticated user. + + Response structure: + { + "event_types": ["document.processed", "document.failed"], + "event_labels": {"document.processed": "Document Processed", ...}, + "preferences": { + "document.processed": { + "in_app": {"is_enabled": true, "target_id": null}, + "email": {"is_enabled": false, "target_id": 1}, + ... + } + } + } + """ + prefs = db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all() + + # Build nested dict: event_type -> channel_type -> {is_enabled, target_id} + result: dict[str, dict[str, dict[str, Any]]] = {} + for pref in prefs: + result.setdefault(pref.event_type, {})[pref.channel_type] = { + "is_enabled": pref.is_enabled, + "target_id": pref.target_id, + } + + return { + "event_types": list(USER_EVENT_LABELS.keys()), + "event_labels": USER_EVENT_LABELS, + "preferences": result, + } + + +@router.put("/preferences", status_code=status.HTTP_200_OK) +async def update_preferences( + body: PreferencesUpdate, + owner_id: CurrentOwner, + db: DbSession, +) -> dict[str, str]: + """Bulk upsert notification preferences for the authenticated user. + + Validates that any referenced target_id belongs to the requesting user. + """ + # Collect all target IDs referenced in the payload for ownership validation + referenced_target_ids: set[int] = set() + for item in body.preferences: + if item.target_id is not None: + referenced_target_ids.add(item.target_id) + + if referenced_target_ids: + owned_ids = { + row.id + for row in db.query(UserNotificationTarget.id) + .filter( + UserNotificationTarget.owner_id == owner_id, + UserNotificationTarget.id.in_(referenced_target_ids), + ) + .all() + } + invalid = referenced_target_ids - owned_ids + if invalid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid or inaccessible target_id(s): {sorted(invalid)}", + ) + + try: + for item in body.preferences: + existing = ( + db.query(UserNotificationPreference) + .filter( + UserNotificationPreference.owner_id == owner_id, + UserNotificationPreference.event_type == item.event_type, + UserNotificationPreference.channel_type == item.channel_type, + UserNotificationPreference.target_id == item.target_id, + ) + .first() + ) + if existing: + existing.is_enabled = item.is_enabled + else: + db.add( + UserNotificationPreference( + owner_id=owner_id, + event_type=item.event_type, + channel_type=item.channel_type, + target_id=item.target_id, + is_enabled=item.is_enabled, + ) + ) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("Notification preferences updated for owner=%s", owner_id) + return {"detail": "Preferences updated"} diff --git a/app/models.py b/app/models.py index 687bf3da..be05e2b5 100644 --- a/app/models.py +++ b/app/models.py @@ -637,3 +637,50 @@ class ApiToken(Base): is_active = Column(Boolean, nullable=False, default=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) revoked_at = Column(DateTime(timezone=True), nullable=True) + + +class UserNotificationTarget(Base): + """Per-user notification target (email or webhook channel).""" + + __tablename__ = "user_notification_targets" + + id = Column(Integer, primary_key=True, index=True) + owner_id = Column(String, nullable=False, index=True) + channel_type = Column(String(20), nullable=False) # "email" or "webhook" + name = Column(String(255), nullable=False) # Human-readable label + config = Column(Text, nullable=True) # JSON: smtp config or webhook url + is_active = Column(Boolean, nullable=False, default=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class UserNotificationPreference(Base): + """Mapping: which user events trigger which notification channel.""" + + __tablename__ = "user_notification_preferences" + + id = Column(Integer, primary_key=True, index=True) + owner_id = Column(String, nullable=False, index=True) + event_type = Column(String(50), nullable=False) # "document.processed", "document.failed" + channel_type = Column(String(20), nullable=False) # "in_app", "email", "webhook" + target_id = Column(Integer, nullable=True) # NULL = in_app, else UserNotificationTarget.id + is_enabled = Column(Boolean, nullable=False, default=True) + + __table_args__ = (UniqueConstraint("owner_id", "event_type", "channel_type", "target_id"),) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class InAppNotification(Base): + """In-app notification record for the bell icon / inbox.""" + + __tablename__ = "in_app_notifications" + + id = Column(Integer, primary_key=True, index=True) + owner_id = Column(String, nullable=False, index=True) + event_type = Column(String(50), nullable=False) # "document.processed", "document.failed" + title = Column(String(255), nullable=False) + message = Column(Text, nullable=True) + is_read = Column(Boolean, nullable=False, default=False, index=True) + file_id = Column(Integer, nullable=True) # Optional link to FileRecord + created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) diff --git a/app/utils/user_notification.py b/app/utils/user_notification.py new file mode 100644 index 00000000..b7b79fd1 --- /dev/null +++ b/app/utils/user_notification.py @@ -0,0 +1,227 @@ +"""Per-user notification dispatch service. + +Handles user-centric events (document.processed, document.failed) by: +1. Always creating an InAppNotification record +2. Sending via configured email/webhook targets (UserNotificationTarget) + if the user has enabled that channel/event combination. +""" + +import json +import logging +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Any + +import httpx + +from app.database import SessionLocal +from app.models import InAppNotification, UserNotificationPreference, UserNotificationTarget + +logger = logging.getLogger(__name__) + +# Supported user-centric event types +EVENT_DOCUMENT_PROCESSED = "document.processed" +EVENT_DOCUMENT_FAILED = "document.failed" + +USER_EVENT_LABELS: dict[str, str] = { + EVENT_DOCUMENT_PROCESSED: "Document Processed", + EVENT_DOCUMENT_FAILED: "Document Processing Failed", +} + + +def create_in_app_notification( + owner_id: str, + event_type: str, + title: str, + message: str, + file_id: int | None = None, +) -> InAppNotification | None: + """Persist an InAppNotification record for the given user. + + Returns: + The created InAppNotification, or None on error. + """ + db = SessionLocal() + try: + notif = InAppNotification( + owner_id=owner_id, + event_type=event_type, + title=title, + message=message, + file_id=file_id, + ) + db.add(notif) + db.commit() + db.refresh(notif) + return notif + except Exception: + db.rollback() + logger.exception("Failed to create in-app notification for owner_id=%s", owner_id) + return None + finally: + db.close() + + +def _send_email_notification(target_config: dict[str, Any], title: str, message: str) -> bool: + """Send an email notification via the configured SMTP target. + + Args: + target_config: dict with keys: smtp_host, smtp_port, smtp_username, + smtp_password, smtp_use_tls, recipient_email + title: Email subject + message: Email body text + """ + try: + smtp_host = target_config.get("smtp_host", "") + smtp_port = int(target_config.get("smtp_port", 587)) + smtp_username = target_config.get("smtp_username", "") + smtp_password = target_config.get("smtp_password", "") + smtp_use_tls = bool(target_config.get("smtp_use_tls", True)) + recipient_email = target_config.get("recipient_email", "") + sender_email = target_config.get("sender_email") or smtp_username or "noreply@docuelevate.local" + + if not smtp_host or not recipient_email: + logger.warning("Email notification target missing smtp_host or recipient_email") + return False + + msg = MIMEMultipart("alternative") + msg["Subject"] = title + msg["From"] = sender_email + msg["To"] = recipient_email + msg.attach(MIMEText(message, "plain")) + + with smtplib.SMTP(smtp_host, smtp_port, timeout=30) as server: + if smtp_use_tls: + server.starttls() + if smtp_username and smtp_password: + server.login(smtp_username, smtp_password) + server.send_message(msg) + + logger.info("Email notification sent to %s", recipient_email) + return True + except Exception: + logger.exception("Failed to send email notification") + return False + + +def _send_webhook_notification(target_config: dict[str, Any], event_type: str, title: str, message: str) -> bool: + """Send a webhook POST notification to the configured URL. + + Args: + target_config: dict with keys: url, secret (optional HMAC header value) + event_type: The event type string + title: Notification title + message: Notification body + """ + try: + url = target_config.get("url", "") + secret = target_config.get("secret", "") + + if not url: + logger.warning("Webhook notification target missing url") + return False + + payload = { + "event": event_type, + "title": title, + "message": message, + } + headers = {"Content-Type": "application/json"} + if secret: + headers["X-DocuElevate-Secret"] = secret + + response = httpx.post(url, json=payload, headers=headers, timeout=10) + response.raise_for_status() + logger.info("Webhook notification sent to %s (status %s)", url, response.status_code) + return True + except Exception: + logger.exception("Failed to send webhook notification to %s", target_config.get("url", "")) + return False + + +def dispatch_user_notification( + owner_id: str, + event_type: str, + title: str, + message: str, + file_id: int | None = None, +) -> None: + """Dispatch a user notification for the given event. + + Always creates an in-app notification. Also sends via email/webhook + targets if the user has configured and enabled them for this event. + + Args: + owner_id: The user's stable identifier. + event_type: e.g. "document.processed" or "document.failed" + title: Short notification title. + message: Longer notification body. + file_id: Optional FileRecord.id to link. + """ + # 1. Always create an in-app notification + create_in_app_notification( + owner_id=owner_id, + event_type=event_type, + title=title, + message=message, + file_id=file_id, + ) + + # 2. Check for configured email/webhook preferences + db = SessionLocal() + try: + prefs = ( + db.query(UserNotificationPreference) + .filter( + UserNotificationPreference.owner_id == owner_id, + UserNotificationPreference.event_type == event_type, + UserNotificationPreference.is_enabled == True, # noqa: E712 + UserNotificationPreference.channel_type.in_(["email", "webhook"]), + ) + .all() + ) + + for pref in prefs: + if not pref.target_id: + continue + target = db.get(UserNotificationTarget, pref.target_id) + if not target or not target.is_active: + continue + config: dict[str, Any] = {} + if target.config: + try: + config = json.loads(target.config) + except (json.JSONDecodeError, ValueError): + config = {} + + if pref.channel_type == "email": + _send_email_notification(config, title, message) + elif pref.channel_type == "webhook": + _send_webhook_notification(config, event_type, title, message) + except Exception: + logger.exception("Error dispatching user notification for owner_id=%s event=%s", owner_id, event_type) + finally: + db.close() + + +def notify_user_document_processed(owner_id: str, filename: str, file_id: int | None = None) -> None: + """Notify a user that their document was successfully processed.""" + dispatch_user_notification( + owner_id=owner_id, + event_type=EVENT_DOCUMENT_PROCESSED, + title=f"Document processed: {filename}", + message=f"Your document '{filename}' has been successfully processed and uploaded.", + file_id=file_id, + ) + + +def notify_user_document_failed(owner_id: str, filename: str, error: str, file_id: int | None = None) -> None: + """Notify a user that their document processing failed.""" + dispatch_user_notification( + owner_id=owner_id, + event_type=EVENT_DOCUMENT_FAILED, + title=f"Document processing failed: {filename}", + message=f"Processing of '{filename}' failed: {error}", + file_id=file_id, + ) diff --git a/app/views/__init__.py b/app/views/__init__.py index b8fb9b0d..6e50a63e 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -18,6 +18,7 @@ from app.views.help import router as help_router # Built-in help / How-To docs from app.views.imap_accounts import router as imap_accounts_router from app.views.integrations import router as integrations_router # Unified integrations dashboard from app.views.license_routes import router as license_router # Add the license router +from app.views.notifications import router as notifications_router from app.views.onboarding import router as onboarding_router from app.views.onedrive import router as onedrive_router from app.views.pipelines import router as pipelines_router # Processing pipelines @@ -52,4 +53,5 @@ router.include_router(onboarding_router) # User onboarding wizard router.include_router(pipelines_router) # Processing pipelines router.include_router(imap_accounts_router) # Per-user IMAP ingestion accounts router.include_router(integrations_router) # Unified integrations dashboard +router.include_router(notifications_router) # User notification dashboard router.include_router(help_router) # Built-in help / How-To docs diff --git a/app/views/notifications.py b/app/views/notifications.py new file mode 100644 index 00000000..7e4e26a3 --- /dev/null +++ b/app/views/notifications.py @@ -0,0 +1,20 @@ +"""View route for the notifications dashboard.""" + +import logging + +from fastapi import Request + +from app.views.base import APIRouter, require_login, templates + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/notifications") +@require_login +async def notifications_dashboard(request: Request): + """Render the notifications dashboard.""" + return templates.TemplateResponse( + "notifications_dashboard.html", + {"request": request, "page_title": "Notifications"}, + ) diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 4ef16130..ba25d36c 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -194,6 +194,20 @@ Help + + + + + + + + + + + +
+ +
+
+ + +
+ +
+ + + + + + + + + +
+ + +
+ + +
+
+

+ Notification Targets +

+ +
+ + + + + + +
+ + +
+
+

+ Event Preferences +

+ +
+ +
+
+ + + + + + + + + + + + +
+ Event + + In-App + + Email + + Webhook +
+
+
+
+
+ + + + + +
+ + +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/migrations/versions/025_add_user_notifications.py b/migrations/versions/025_add_user_notifications.py new file mode 100644 index 00000000..4bbdd9d0 --- /dev/null +++ b/migrations/versions/025_add_user_notifications.py @@ -0,0 +1,83 @@ +"""Add user notification tables (targets, preferences, in-app inbox) + +Revision ID: 025_add_user_notifications +Revises: 024_add_api_tokens +Create Date: 2026-03-09 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "025_add_user_notifications" +down_revision: Union[str, None] = "024_add_api_tokens" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Create user_notification_targets, user_notification_preferences, and in_app_notifications tables.""" + op.create_table( + "user_notification_targets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("owner_id", sa.String(), nullable=False), + sa.Column("channel_type", sa.String(20), nullable=False), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("config", sa.Text(), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default="1"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_user_notification_targets_id", "user_notification_targets", ["id"]) + op.create_index("ix_user_notification_targets_owner_id", "user_notification_targets", ["owner_id"]) + + op.create_table( + "user_notification_preferences", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("owner_id", sa.String(), nullable=False), + sa.Column("event_type", sa.String(50), nullable=False), + sa.Column("channel_type", sa.String(20), nullable=False), + sa.Column("target_id", sa.Integer(), nullable=True), + sa.Column("is_enabled", sa.Boolean(), nullable=False, server_default="1"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("owner_id", "event_type", "channel_type", "target_id"), + ) + op.create_index("ix_user_notification_preferences_id", "user_notification_preferences", ["id"]) + op.create_index("ix_user_notification_preferences_owner_id", "user_notification_preferences", ["owner_id"]) + + op.create_table( + "in_app_notifications", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("owner_id", sa.String(), nullable=False), + sa.Column("event_type", sa.String(50), nullable=False), + sa.Column("title", sa.String(255), nullable=False), + sa.Column("message", sa.Text(), nullable=True), + sa.Column("is_read", sa.Boolean(), nullable=False, server_default="0"), + sa.Column("file_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_in_app_notifications_id", "in_app_notifications", ["id"]) + op.create_index("ix_in_app_notifications_owner_id", "in_app_notifications", ["owner_id"]) + op.create_index("ix_in_app_notifications_is_read", "in_app_notifications", ["is_read"]) + op.create_index("ix_in_app_notifications_created_at", "in_app_notifications", ["created_at"]) + + +def downgrade() -> None: + """Drop user notification tables.""" + op.drop_index("ix_in_app_notifications_created_at", "in_app_notifications") + op.drop_index("ix_in_app_notifications_is_read", "in_app_notifications") + op.drop_index("ix_in_app_notifications_owner_id", "in_app_notifications") + op.drop_index("ix_in_app_notifications_id", "in_app_notifications") + op.drop_table("in_app_notifications") + + op.drop_index("ix_user_notification_preferences_owner_id", "user_notification_preferences") + op.drop_index("ix_user_notification_preferences_id", "user_notification_preferences") + op.drop_table("user_notification_preferences") + + op.drop_index("ix_user_notification_targets_owner_id", "user_notification_targets") + op.drop_index("ix_user_notification_targets_id", "user_notification_targets") + op.drop_table("user_notification_targets") diff --git a/tests/test_notifications_api.py b/tests/test_notifications_api.py new file mode 100644 index 00000000..32bf94f4 --- /dev/null +++ b/tests/test_notifications_api.py @@ -0,0 +1,832 @@ +"""Tests for the per-user notification system (app/api/notifications.py).""" + +import json + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base, get_db +from app.models import InAppNotification, UserNotificationPreference, UserNotificationTarget + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +_OWNER = "notifuser@example.com" +_OTHER_OWNER = "other@example.com" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def notif_engine(): + """In-memory SQLite engine for notification tests.""" + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + yield engine + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def notif_session(notif_engine): + """DB session scoped to one test.""" + Session = sessionmaker(bind=notif_engine) + session = Session() + yield session + session.close() + + +def _make_client(notif_engine, owner_id: str = _OWNER) -> TestClient: + """Return a TestClient with *owner_id* injected as the authenticated user.""" + from app.api.notifications import _get_owner_id + from app.main import app + + Session = sessionmaker(bind=notif_engine) + + def _override_get_db(): + session = Session() + try: + yield session + finally: + session.close() + + def _override_owner(): + return owner_id + + app.dependency_overrides[get_db] = _override_get_db + app.dependency_overrides[_get_owner_id] = _override_owner + + return TestClient(app, base_url="http://localhost", raise_server_exceptions=False) + + +def _cleanup(app): + """Remove dependency overrides after test.""" + app.dependency_overrides.clear() + + +# --------------------------------------------------------------------------- +# Tests – Auth / 401 guard +# --------------------------------------------------------------------------- + + +class TestAuthGuard: + """Verify that unauthenticated requests are rejected.""" + + @pytest.mark.unit + def test_inbox_requires_auth(self): + """GET /api/user-notifications/inbox should return 401 when not authenticated.""" + from app.main import app + + client = TestClient(app, base_url="http://localhost", raise_server_exceptions=False) + resp = client.get("/api/user-notifications/inbox") + assert resp.status_code == 401 + + @pytest.mark.unit + def test_unread_count_requires_auth(self): + """GET /api/user-notifications/inbox/unread-count should return 401 when not authenticated.""" + from app.main import app + + client = TestClient(app, base_url="http://localhost", raise_server_exceptions=False) + resp = client.get("/api/user-notifications/inbox/unread-count") + assert resp.status_code == 401 + + +# --------------------------------------------------------------------------- +# Tests – Inbox +# --------------------------------------------------------------------------- + + +class TestInbox: + """Tests for the in-app notification inbox.""" + + @pytest.mark.unit + def test_unread_count_empty(self, notif_engine): + """Unread count should be 0 when no notifications exist.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/inbox/unread-count") + assert resp.status_code == 200 + assert resp.json() == {"count": 0} + finally: + _cleanup(app) + + @pytest.mark.unit + def test_inbox_empty(self, notif_engine): + """Listing inbox when empty should return an empty list.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/inbox") + assert resp.status_code == 200 + assert resp.json() == [] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_inbox_shows_notifications(self, notif_engine, notif_session): + """Inbox should return notifications for the authenticated user.""" + from app.main import app + + notif_session.add( + InAppNotification( + owner_id=_OWNER, + event_type="document.processed", + title="Test", + message="Done", + ) + ) + notif_session.commit() + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/inbox") + assert resp.status_code == 200 + items = resp.json() + assert len(items) == 1 + assert items[0]["title"] == "Test" + assert items[0]["is_read"] is False + finally: + _cleanup(app) + + @pytest.mark.unit + def test_inbox_isolation(self, notif_engine, notif_session): + """Users should only see their own notifications.""" + from app.main import app + + notif_session.add( + InAppNotification( + owner_id=_OTHER_OWNER, + event_type="document.processed", + title="Other user notif", + message="Not yours", + ) + ) + notif_session.commit() + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.get("/api/user-notifications/inbox") + assert resp.status_code == 200 + assert resp.json() == [] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_unread_count_reflects_notifications(self, notif_engine, notif_session): + """Unread count should reflect actual unread notifications.""" + from app.main import app + + for i in range(3): + notif_session.add( + InAppNotification( + owner_id=_OWNER, + event_type="document.processed", + title=f"Notif {i}", + message="", + is_read=False, + ) + ) + notif_session.commit() + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/inbox/unread-count") + assert resp.status_code == 200 + assert resp.json()["count"] == 3 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_mark_read(self, notif_engine, notif_session): + """Marking a notification as read should update is_read.""" + from app.main import app + + notif = InAppNotification( + owner_id=_OWNER, + event_type="document.processed", + title="Unread", + message="", + ) + notif_session.add(notif) + notif_session.commit() + notif_session.refresh(notif) + notif_id = notif.id + + client = _make_client(notif_engine) + try: + resp = client.post(f"/api/user-notifications/inbox/{notif_id}/read") + assert resp.status_code == 200 + + # Verify in DB + notif_session.refresh(notif) + assert notif.is_read is True + finally: + _cleanup(app) + + @pytest.mark.unit + def test_mark_read_wrong_user(self, notif_engine, notif_session): + """Marking another user's notification should return 404.""" + from app.main import app + + notif = InAppNotification( + owner_id=_OTHER_OWNER, + event_type="document.processed", + title="Other", + message="", + ) + notif_session.add(notif) + notif_session.commit() + notif_session.refresh(notif) + notif_id = notif.id + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.post(f"/api/user-notifications/inbox/{notif_id}/read") + assert resp.status_code == 404 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_mark_all_read(self, notif_engine, notif_session): + """Mark all read should set all user's notifications to read.""" + from app.main import app + + for i in range(4): + notif_session.add( + InAppNotification( + owner_id=_OWNER, + event_type="document.processed", + title=f"N{i}", + message="", + is_read=False, + ) + ) + notif_session.commit() + + client = _make_client(notif_engine) + try: + resp = client.post("/api/user-notifications/inbox/read-all") + assert resp.status_code == 200 + + count_resp = client.get("/api/user-notifications/inbox/unread-count") + assert count_resp.json()["count"] == 0 + finally: + _cleanup(app) + + +# --------------------------------------------------------------------------- +# Tests – Notification Targets +# --------------------------------------------------------------------------- + + +class TestTargets: + """Tests for notification target CRUD.""" + + @pytest.mark.unit + def test_list_targets_empty(self, notif_engine): + """Listing targets when none exist should return empty list.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/targets") + assert resp.status_code == 200 + assert resp.json() == [] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_create_email_target(self, notif_engine): + """Creating an email target should persist and mask the password in response.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.post( + "/api/user-notifications/targets", + json={ + "channel_type": "email", + "name": "My Gmail", + "config": { + "smtp_host": "smtp.gmail.com", + "smtp_port": 587, + "smtp_username": "me@gmail.com", + "smtp_password": "s3cr3t", + "recipient_email": "me@gmail.com", + "smtp_use_tls": True, + }, + "is_active": True, + }, + ) + assert resp.status_code == 201, resp.text + data = resp.json() + assert data["channel_type"] == "email" + assert data["name"] == "My Gmail" + assert data["is_active"] is True + # Password must be masked + assert data["config"]["smtp_password"] == "****" + finally: + _cleanup(app) + + @pytest.mark.unit + def test_create_webhook_target(self, notif_engine): + """Creating a webhook target should persist correctly.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.post( + "/api/user-notifications/targets", + json={ + "channel_type": "webhook", + "name": "Slack Webhook", + "config": {"url": "https://hooks.slack.com/abc", "secret": ""}, + "is_active": True, + }, + ) + assert resp.status_code == 201, resp.text + data = resp.json() + assert data["channel_type"] == "webhook" + assert data["name"] == "Slack Webhook" + finally: + _cleanup(app) + + @pytest.mark.unit + def test_create_target_invalid_channel_type(self, notif_engine): + """Creating a target with an invalid channel_type should return 422.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.post( + "/api/user-notifications/targets", + json={"channel_type": "sms", "name": "Bad", "config": {}}, + ) + assert resp.status_code == 422 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_list_targets_returns_created(self, notif_engine): + """Listing targets should include newly created ones.""" + from app.main import app + + client = _make_client(notif_engine) + try: + client.post( + "/api/user-notifications/targets", + json={"channel_type": "webhook", "name": "W1", "config": {"url": "https://example.com"}}, + ) + client.post( + "/api/user-notifications/targets", + json={"channel_type": "email", "name": "E1", "config": {"smtp_host": "smtp.example.com"}}, + ) + resp = client.get("/api/user-notifications/targets") + assert resp.status_code == 200 + assert len(resp.json()) == 2 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_targets_isolation(self, notif_engine): + """Users should only see their own targets.""" + from app.main import app + + client_a = _make_client(notif_engine, _OWNER) + try: + client_a.post( + "/api/user-notifications/targets", + json={"channel_type": "webhook", "name": "Owner A target", "config": {"url": "https://a.example.com"}}, + ) + finally: + _cleanup(app) + + client_b = _make_client(notif_engine, _OTHER_OWNER) + try: + resp = client_b.get("/api/user-notifications/targets") + assert resp.status_code == 200 + assert resp.json() == [] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_update_target(self, notif_engine): + """Updating a target should change its name and active status.""" + from app.main import app + + client = _make_client(notif_engine) + try: + create_resp = client.post( + "/api/user-notifications/targets", + json={"channel_type": "webhook", "name": "Old Name", "config": {"url": "https://x.com"}}, + ) + target_id = create_resp.json()["id"] + + resp = client.put( + f"/api/user-notifications/targets/{target_id}", + json={"name": "New Name", "is_active": False}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == "New Name" + assert data["is_active"] is False + finally: + _cleanup(app) + + @pytest.mark.unit + def test_update_target_wrong_user(self, notif_engine, notif_session): + """Updating another user's target should return 404.""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OTHER_OWNER, + channel_type="webhook", + name="Other target", + config=json.dumps({"url": "https://other.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.put( + f"/api/user-notifications/targets/{target.id}", + json={"name": "Hacked"}, + ) + assert resp.status_code == 404 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_delete_target(self, notif_engine): + """Deleting a target should remove it from the list.""" + from app.main import app + + client = _make_client(notif_engine) + try: + create_resp = client.post( + "/api/user-notifications/targets", + json={"channel_type": "webhook", "name": "To Delete", "config": {"url": "https://x.com"}}, + ) + target_id = create_resp.json()["id"] + + del_resp = client.delete(f"/api/user-notifications/targets/{target_id}") + assert del_resp.status_code == 200 + + list_resp = client.get("/api/user-notifications/targets") + assert list_resp.json() == [] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_delete_target_wrong_user(self, notif_engine, notif_session): + """Deleting another user's target should return 404.""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OTHER_OWNER, + channel_type="webhook", + name="Not yours", + config=json.dumps({"url": "https://other.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.delete(f"/api/user-notifications/targets/{target.id}") + assert resp.status_code == 404 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_delete_target_also_removes_preferences(self, notif_engine, notif_session): + """Deleting a target should also remove associated preferences.""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OWNER, + channel_type="webhook", + name="With prefs", + config=json.dumps({"url": "https://x.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + pref = UserNotificationPreference( + owner_id=_OWNER, + event_type="document.processed", + channel_type="webhook", + target_id=target.id, + is_enabled=True, + ) + notif_session.add(pref) + notif_session.commit() + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.delete(f"/api/user-notifications/targets/{target.id}") + assert resp.status_code == 200 + + remaining = ( + notif_session.query(UserNotificationPreference) + .filter(UserNotificationPreference.owner_id == _OWNER) + .all() + ) + assert remaining == [] + finally: + _cleanup(app) + + +# --------------------------------------------------------------------------- +# Tests – Preferences +# --------------------------------------------------------------------------- + + +class TestPreferences: + """Tests for notification preferences CRUD.""" + + @pytest.mark.unit + def test_get_preferences_empty(self, notif_engine): + """Getting preferences returns event_types and event_labels even with no prefs set.""" + from app.main import app + + client = _make_client(notif_engine) + try: + resp = client.get("/api/user-notifications/preferences") + assert resp.status_code == 200 + data = resp.json() + assert "event_types" in data + assert "event_labels" in data + assert "preferences" in data + assert "document.processed" in data["event_types"] + assert "document.failed" in data["event_types"] + finally: + _cleanup(app) + + @pytest.mark.unit + def test_update_preferences(self, notif_engine, notif_session): + """Updating preferences should persist the changes.""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OWNER, + channel_type="webhook", + name="My Webhook", + config=json.dumps({"url": "https://x.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.put( + "/api/user-notifications/preferences", + json={ + "preferences": [ + { + "event_type": "document.processed", + "channel_type": "webhook", + "is_enabled": True, + "target_id": target.id, + } + ] + }, + ) + assert resp.status_code == 200 + + # Verify stored + pref = ( + notif_session.query(UserNotificationPreference) + .filter( + UserNotificationPreference.owner_id == _OWNER, + UserNotificationPreference.event_type == "document.processed", + UserNotificationPreference.channel_type == "webhook", + ) + .first() + ) + assert pref is not None + assert pref.is_enabled is True + assert pref.target_id == target.id + finally: + _cleanup(app) + + @pytest.mark.unit + def test_update_preferences_upsert(self, notif_engine, notif_session): + """Updating preferences twice should upsert (not duplicate).""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OWNER, + channel_type="webhook", + name="W", + config=json.dumps({"url": "https://x.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + client = _make_client(notif_engine, _OWNER) + try: + pref_item = { + "event_type": "document.processed", + "channel_type": "webhook", + "is_enabled": True, + "target_id": target.id, + } + client.put("/api/user-notifications/preferences", json={"preferences": [pref_item]}) + # Disable it + pref_item["is_enabled"] = False + resp = client.put("/api/user-notifications/preferences", json={"preferences": [pref_item]}) + assert resp.status_code == 200 + + prefs = ( + notif_session.query(UserNotificationPreference) + .filter(UserNotificationPreference.owner_id == _OWNER) + .all() + ) + assert len(prefs) == 1 + assert prefs[0].is_enabled is False + finally: + _cleanup(app) + + @pytest.mark.unit + def test_update_preferences_rejects_foreign_target(self, notif_engine, notif_session): + """Preferences referencing another user's target_id should be rejected.""" + from app.main import app + + other_target = UserNotificationTarget( + owner_id=_OTHER_OWNER, + channel_type="webhook", + name="Other webhook", + config=json.dumps({"url": "https://other.com"}), + ) + notif_session.add(other_target) + notif_session.commit() + notif_session.refresh(other_target) + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.put( + "/api/user-notifications/preferences", + json={ + "preferences": [ + { + "event_type": "document.processed", + "channel_type": "webhook", + "is_enabled": True, + "target_id": other_target.id, + } + ] + }, + ) + assert resp.status_code == 400 + finally: + _cleanup(app) + + @pytest.mark.unit + def test_get_preferences_reflects_saved(self, notif_engine, notif_session): + """GET preferences should reflect previously saved preferences.""" + from app.main import app + + target = UserNotificationTarget( + owner_id=_OWNER, + channel_type="email", + name="Email target", + config=json.dumps({"smtp_host": "smtp.example.com", "recipient_email": "me@example.com"}), + ) + notif_session.add(target) + notif_session.commit() + notif_session.refresh(target) + + notif_session.add( + UserNotificationPreference( + owner_id=_OWNER, + event_type="document.failed", + channel_type="email", + target_id=target.id, + is_enabled=True, + ) + ) + notif_session.commit() + + client = _make_client(notif_engine, _OWNER) + try: + resp = client.get("/api/user-notifications/preferences") + assert resp.status_code == 200 + data = resp.json() + assert "document.failed" in data["preferences"] + assert "email" in data["preferences"]["document.failed"] + assert data["preferences"]["document.failed"]["email"]["is_enabled"] is True + finally: + _cleanup(app) + + +# --------------------------------------------------------------------------- +# Tests – user_notification service +# --------------------------------------------------------------------------- + + +class TestUserNotificationService: + """Unit tests for the user notification dispatch service.""" + + @pytest.mark.unit + def test_create_in_app_notification(self, notif_engine, notif_session): + """create_in_app_notification should persist a record.""" + from unittest.mock import patch + + from app.utils.user_notification import create_in_app_notification + + Session = sessionmaker(bind=notif_engine) + + with patch("app.utils.user_notification.SessionLocal", Session): + result = create_in_app_notification( + owner_id=_OWNER, + event_type="document.processed", + title="Test", + message="Done", + file_id=42, + ) + + assert result is not None + assert result.owner_id == _OWNER + assert result.title == "Test" + assert result.file_id == 42 + + @pytest.mark.unit + def test_notify_user_document_processed(self, notif_engine): + """notify_user_document_processed should create an in-app notification.""" + from unittest.mock import patch + + from app.utils.user_notification import notify_user_document_processed + + Session = sessionmaker(bind=notif_engine) + + with patch("app.utils.user_notification.SessionLocal", Session): + notify_user_document_processed(owner_id=_OWNER, filename="test.pdf", file_id=1) + + s = Session() + notifs = s.query(InAppNotification).filter(InAppNotification.owner_id == _OWNER).all() + s.close() + assert len(notifs) == 1 + assert "test.pdf" in notifs[0].title + + @pytest.mark.unit + def test_notify_user_document_failed(self, notif_engine): + """notify_user_document_failed should create an in-app notification.""" + from unittest.mock import patch + + from app.utils.user_notification import notify_user_document_failed + + Session = sessionmaker(bind=notif_engine) + + with patch("app.utils.user_notification.SessionLocal", Session): + notify_user_document_failed(owner_id=_OWNER, filename="doc.pdf", error="OCR timeout") + + s = Session() + notifs = s.query(InAppNotification).filter(InAppNotification.owner_id == _OWNER).all() + s.close() + assert len(notifs) == 1 + assert notifs[0].event_type == "document.failed" + assert "OCR timeout" in notifs[0].message + + @pytest.mark.unit + def test_send_webhook_notification_missing_url(self): + """_send_webhook_notification should return False when url is missing.""" + from app.utils.user_notification import _send_webhook_notification + + result = _send_webhook_notification({}, "document.processed", "Title", "Body") + assert result is False + + @pytest.mark.unit + def test_send_email_notification_missing_host(self): + """_send_email_notification should return False when smtp_host is missing.""" + from app.utils.user_notification import _send_email_notification + + result = _send_email_notification({"recipient_email": "me@example.com"}, "Title", "Body") + assert result is False + + @pytest.mark.unit + def test_send_email_notification_missing_recipient(self): + """_send_email_notification should return False when recipient_email is missing.""" + from app.utils.user_notification import _send_email_notification + + result = _send_email_notification({"smtp_host": "smtp.example.com"}, "Title", "Body") + assert result is False From d48e36813e37d3db220a04044f319698ecb48056 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:44:01 +0000 Subject: [PATCH 05/16] feat(notifications): build per-user notification system (email, webhook, in-app) Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/utils/user_notification.py | 6 ++++++ frontend/templates/base.html | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/utils/user_notification.py b/app/utils/user_notification.py index b7b79fd1..f6a277cd 100644 --- a/app/utils/user_notification.py +++ b/app/utils/user_notification.py @@ -71,6 +71,9 @@ def _send_email_notification(target_config: dict[str, Any], title: str, message: smtp_password, smtp_use_tls, recipient_email title: Email subject message: Email body text + + Returns: + True if the email was sent successfully, False otherwise. """ try: smtp_host = target_config.get("smtp_host", "") @@ -113,6 +116,9 @@ def _send_webhook_notification(target_config: dict[str, Any], event_type: str, t event_type: The event type string title: Notification title message: Notification body + + Returns: + True if the webhook was delivered successfully, False otherwise. """ try: url = target_config.get("url", "") diff --git a/frontend/templates/base.html b/frontend/templates/base.html index ba25d36c..c1fd21dc 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -204,8 +204,7 @@ + aria-live="polite"> From ae075908d1256ad728f4b992c020de2ef978fb65 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:44:54 +0000 Subject: [PATCH 06/16] docs: add per-user notification system documentation to ConfigurationGuide.md Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- docs/ConfigurationGuide.md | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index fcb22f84..b812a3d8 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -1070,6 +1070,55 @@ payment processors. For detailed setup instructions, see the [Notifications Setup Guide](NotificationsSetup.md). +#### Per-User Notification System + +In addition to the system-level Apprise notifications, DocuElevate includes a **per-user notification system** that gives each user full control over how they are notified about their own document events. + +**Notification Dashboard** — available at `/notifications` for every logged-in user. It has three tabs: + +| Tab | Description | +|-----|-------------| +| **Inbox** | In-app bell-icon notification feed. Persisted in the database; shows unread count badge in the navigation bar. Users can mark individual items or all items as read. | +| **Targets** | User-defined notification channels: **Email (SMTP)** and **Webhook (HTTP POST)**. Each target can be tested independently from the UI. | +| **Preferences** | Event/channel matrix. Users choose which channels are triggered for each event type. In-app notifications are always enabled. | + +**User-centric event types:** + +| Event | Description | +|-------|-------------| +| `document.processed` | A document uploaded by the user was successfully processed and uploaded to destinations | +| `document.failed` | A document uploaded by the user failed during processing | + +**Email target configuration fields:** + +| Field | Description | +|-------|-------------| +| `smtp_host` | SMTP server hostname | +| `smtp_port` | SMTP port (default `587`) | +| `smtp_username` | SMTP login username | +| `smtp_password` | SMTP login password (stored in database, masked in UI) | +| `smtp_use_tls` | Enable STARTTLS (`true`/`false`, default `true`) | +| `sender_email` | From address (defaults to `smtp_username` if omitted) | +| `recipient_email` | Destination address for this target | + +**Webhook target configuration fields:** + +| Field | Description | +|-------|-------------| +| `url` | HTTP(S) URL to POST the notification payload to | +| `secret` | Optional secret string sent as `X-DocuElevate-Secret` header | + +**Webhook payload format:** +```json +{ + "event": "document.processed", + "title": "Document processed: invoice.pdf", + "message": "Your document 'invoice.pdf' has been successfully processed and uploaded." +} +``` + +> **Note:** There are no additional environment variables for the per-user notification system — all settings are stored in the database and managed through the user-facing `/notifications` dashboard. + ### Webhooks Webhooks notify external systems via HTTP POST when document events occur. From a2a4c6fc9a077a80d1877a441f61f72c5e611e58 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:47:12 +0000 Subject: [PATCH 07/16] feat(ocr): add multi-language OCR support with per-pipeline language override - Add OCR_LANGUAGES constant (28 languages, EN/DE/FR/ES/IT/PT/RU/ZH/JA/KO/AR/etc.) - Add TESSERACT_TO_EASYOCR mapping for automatic code translation - Add optional language constructor arg to TesseractOCRProvider/EasyOCRProvider - Update get_ocr_providers() to accept and pass per-call language override - Add language parameter to process_with_ocr Celery task - Add _get_pipeline_ocr_language() helper to resolve OCR language from pipeline step config - Update process_document to look up and pass pipeline OCR language to process_with_ocr - Add ocr_language select config field (28 options) to pipeline OCR step schema - Add language dropdown to pipeline UI (pipelines.html) - Update docs/UserGuide.md and docs/API.md with language override documentation - Add 27 new tests covering language constants, provider overrides, and pipeline lookup Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/pipelines.py | 43 ++++- app/tasks/process_document.py | 81 +++++++++- app/tasks/process_with_ocr.py | 23 ++- app/utils/ocr_provider.py | 146 ++++++++++++++++- docs/API.md | 32 +++- docs/UserGuide.md | 39 ++++- frontend/templates/pipelines.html | 47 ++++++ tests/test_ocr_provider_coverage.py | 238 ++++++++++++++++++++++++++++ tests/test_process_document.py | 177 +++++++++++++++++++++ 9 files changed, 808 insertions(+), 18 deletions(-) diff --git a/app/api/pipelines.py b/app/api/pipelines.py index 3980d71d..8175832f 100644 --- a/app/api/pipelines.py +++ b/app/api/pipelines.py @@ -51,7 +51,48 @@ PIPELINE_STEP_TYPES: dict[str, dict[str, Any]] = { "type": "boolean", "default": False, "description": "Always use cloud OCR even if the PDF already has embedded text.", - } + }, + "ocr_language": { + "type": "select", + "default": "auto", + "description": ( + "Language(s) used for OCR text extraction. Applies to Tesseract and EasyOCR " + "providers; Azure and Mistral perform auto-detection by default. " + "Use Tesseract codes such as 'eng', 'deu', or 'eng+deu' for multi-language " + "documents. 'auto' falls back to the global system setting." + ), + "options": [ + {"value": "auto", "label": "Auto (use system default)"}, + {"value": "ara", "label": "Arabic"}, + {"value": "chi_sim", "label": "Chinese (Simplified)"}, + {"value": "chi_tra", "label": "Chinese (Traditional)"}, + {"value": "ces", "label": "Czech"}, + {"value": "dan", "label": "Danish"}, + {"value": "nld", "label": "Dutch"}, + {"value": "eng", "label": "English"}, + {"value": "fin", "label": "Finnish"}, + {"value": "fra", "label": "French"}, + {"value": "deu", "label": "German"}, + {"value": "ell", "label": "Greek"}, + {"value": "heb", "label": "Hebrew"}, + {"value": "hin", "label": "Hindi"}, + {"value": "hun", "label": "Hungarian"}, + {"value": "ita", "label": "Italian"}, + {"value": "jpn", "label": "Japanese"}, + {"value": "kor", "label": "Korean"}, + {"value": "nor", "label": "Norwegian"}, + {"value": "pol", "label": "Polish"}, + {"value": "por", "label": "Portuguese"}, + {"value": "ron", "label": "Romanian"}, + {"value": "rus", "label": "Russian"}, + {"value": "spa", "label": "Spanish"}, + {"value": "swe", "label": "Swedish"}, + {"value": "tha", "label": "Thai"}, + {"value": "tur", "label": "Turkish"}, + {"value": "ukr", "label": "Ukrainian"}, + {"value": "vie", "label": "Vietnamese"}, + ], + }, }, }, "extract_metadata": { diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 6eb2da13..3d51a400 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import logging import mimetypes import os @@ -12,7 +13,7 @@ from pypdf.errors import PdfReadError from app.celery_app import celery from app.config import settings from app.database import SessionLocal -from app.models import FileRecord +from app.models import FileRecord, Pipeline, PipelineStep from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt from app.tasks.process_with_ocr import process_with_ocr from app.tasks.retry_config import BaseTaskWithRetry @@ -23,6 +24,69 @@ from app.utils.text_quality import check_text_quality, detect_pdf_text_source logger = logging.getLogger(__name__) +def _get_pipeline_ocr_language(db, file_record: FileRecord, owner_id: str | None) -> str | None: + """Look up the OCR language override from the file's pipeline OCR step config. + + Resolution order: + 1. Explicit pipeline assigned to the file (``file_record.pipeline_id``). + 2. User's own default pipeline (``owner_id``, ``is_default=True``). + 3. System default pipeline (``owner_id=NULL``, ``is_default=True``). + + Returns the ``ocr_language`` value from the pipeline's OCR step config, or + ``None`` when no override is configured. + """ + pipeline = None + + if file_record.pipeline_id: + pipeline = db.query(Pipeline).filter(Pipeline.id == file_record.pipeline_id).first() + + if pipeline is None and owner_id: + pipeline = ( + db.query(Pipeline) + .filter( + Pipeline.owner_id == owner_id, + Pipeline.is_default.is_(True), + Pipeline.is_active.is_(True), + ) + .first() + ) + + if pipeline is None: + pipeline = ( + db.query(Pipeline) + .filter( + Pipeline.owner_id.is_(None), + Pipeline.is_default.is_(True), + Pipeline.is_active.is_(True), + ) + .first() + ) + + if pipeline is None: + return None + + ocr_step = ( + db.query(PipelineStep) + .filter( + PipelineStep.pipeline_id == pipeline.id, + PipelineStep.step_type == "ocr", + PipelineStep.enabled.is_(True), + ) + .first() + ) + + if ocr_step is None or not ocr_step.config: + return None + + try: + step_config = json.loads(ocr_step.config) + lang = step_config.get("ocr_language") or None + # "auto" is treated as no override + return lang if lang and lang != "auto" else None + except Exception: + return None + + @celery.task(base=BaseTaskWithRetry, bind=True) def process_document( self, @@ -109,6 +173,7 @@ def process_document( ) # Acquire DB session in the task + ocr_language: str | None = None # Pipeline OCR language override resolved inside DB session with SessionLocal() as db: # When file_id is provided, we are reprocessing an existing file. # Skip the duplicate check and reuse the existing record. @@ -305,6 +370,14 @@ def process_document( new_record.local_filename = new_local_path db.commit() + # Look up pipeline OCR language override before the session closes. + # This reads the OCR step config from the file's assigned pipeline (or + # the user/system default pipeline) so the language is available when + # dispatching process_with_ocr below. + ocr_language = _get_pipeline_ocr_language(db, new_record, owner_id) + if ocr_language: + logger.info(f"[{task_id}] Pipeline OCR language override: {ocr_language!r}") + # Store file_id before session closes to avoid DetachedInstanceError file_id = new_record.id @@ -334,7 +407,7 @@ def process_document( "Queued for forced OCR processing", file_id=file_id, ) - process_with_ocr.delay(new_filename, file_id) + process_with_ocr.delay(new_filename, file_id, language=ocr_language) return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id} # If the file is not a PDF, skip embedded text check and convert to PDF first @@ -491,7 +564,7 @@ def process_document( "Queued for OCR (text quality too low)", file_id=file_id, ) - process_with_ocr.delay(new_filename, file_id, extracted_text) + process_with_ocr.delay(new_filename, file_id, extracted_text, language=ocr_language) return { "file": new_local_path, "status": "Queued for OCR (poor embedded text quality)", @@ -564,5 +637,5 @@ def process_document( "Queued for OCR processing", file_id=file_id, ) - process_with_ocr.delay(new_filename, file_id) + process_with_ocr.delay(new_filename, file_id, language=ocr_language) return {"file": new_local_path, "status": "Queued for OCR", "file_id": file_id} diff --git a/app/tasks/process_with_ocr.py b/app/tasks/process_with_ocr.py index 8d4817ce..8447ec1f 100644 --- a/app/tasks/process_with_ocr.py +++ b/app/tasks/process_with_ocr.py @@ -33,7 +33,13 @@ logger = logging.getLogger(__name__) @celery.task(base=OcrTaskWithRetry, bind=True) -def process_with_ocr(self, filename: str, file_id: Optional[int] = None, original_text: Optional[str] = None): +def process_with_ocr( + self, + filename: str, + file_id: Optional[int] = None, + original_text: Optional[str] = None, + language: Optional[str] = None, +): """Run the configured OCR providers on *filename* and continue the pipeline. When multiple OCR providers are configured the results are merged using the @@ -47,6 +53,10 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina filename: Base name of the file inside ``/tmp/``. file_id: Optional database record ID passed through to downstream tasks. original_text: Optional original embedded text for head-to-head comparison. + language: Optional Tesseract-style language code(s) (e.g. ``"eng+deu"``) + to override the global OCR language settings for this specific run. + Pass ``None`` or ``"auto"`` to use the global settings. This + enables per-pipeline language configuration. """ task_id = self.request.id log_task_progress( @@ -62,7 +72,7 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina if not os.path.exists(tmp_file_path): raise FileNotFoundError(f"Local file not found: {tmp_file_path}") - providers = get_ocr_providers() + providers = get_ocr_providers(language=language) provider_names = [p.name for p in providers] logger.info(f"[{task_id}] Running {len(providers)} OCR provider(s): {provider_names}") @@ -122,7 +132,12 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina # PDF with ocrmypdf to embed an invisible text layer so the output is # selectable/searchable in PDF viewers. if searchable_pdf_path is None: - lang = getattr(settings, "tesseract_language", None) or "eng" + # Use the per-call language override; fall back to global setting + embed_lang = ( + language + if language and language != "auto" + else (getattr(settings, "tesseract_language", None) or "eng") + ) log_task_progress( task_id, "embed_text_layer", @@ -130,7 +145,7 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina "Embedding searchable text layer into PDF", file_id=file_id, ) - embedded = embed_text_layer(tmp_file_path, tmp_file_path, language=lang) + embedded = embed_text_layer(tmp_file_path, tmp_file_path, language=embed_lang) if embedded: searchable_pdf_path = tmp_file_path log_task_progress( diff --git a/app/utils/ocr_provider.py b/app/utils/ocr_provider.py index 4cf1ef0b..41bdbee7 100644 --- a/app/utils/ocr_provider.py +++ b/app/utils/ocr_provider.py @@ -192,6 +192,95 @@ class OCRResult: ) +# --------------------------------------------------------------------------- +# Multi-language support +# --------------------------------------------------------------------------- + +#: Canonical list of supported OCR languages for pipeline configuration. +#: Keys are display names; values are Tesseract language code(s). +#: Tesseract codes are used as the canonical format because they are the most +#: widely applicable across self-hosted providers (Tesseract + ocrmypdf). +#: "auto" falls back to the global ``tesseract_language`` / ``easyocr_languages`` +#: settings (i.e. no per-call override). +OCR_LANGUAGES: Dict[str, str] = { + "Auto (use system default)": "auto", + "Arabic": "ara", + "Chinese (Simplified)": "chi_sim", + "Chinese (Traditional)": "chi_tra", + "Czech": "ces", + "Danish": "dan", + "Dutch": "nld", + "English": "eng", + "Finnish": "fin", + "French": "fra", + "German": "deu", + "Greek": "ell", + "Hebrew": "heb", + "Hindi": "hin", + "Hungarian": "hun", + "Italian": "ita", + "Japanese": "jpn", + "Korean": "kor", + "Norwegian": "nor", + "Polish": "pol", + "Portuguese": "por", + "Romanian": "ron", + "Russian": "rus", + "Spanish": "spa", + "Swedish": "swe", + "Thai": "tha", + "Turkish": "tur", + "Ukrainian": "ukr", + "Vietnamese": "vie", +} + +#: Mapping from Tesseract language codes to EasyOCR language codes. +#: Used when ``TesseractOCRProvider``-style codes are specified but EasyOCR is +#: the active provider. Codes not present in this map are passed through as-is +#: (EasyOCR accepts its own ISO 639-1 codes such as ``"en"`` or ``"de"``). +TESSERACT_TO_EASYOCR: Dict[str, str] = { + "ara": "ar", + "ces": "cs", + "chi_sim": "ch_sim", + "chi_tra": "ch_tra", + "dan": "da", + "deu": "de", + "ell": "el", + "eng": "en", + "fin": "fi", + "fra": "fr", + "heb": "he", + "hin": "hi", + "hun": "hu", + "ita": "it", + "jpn": "ja", + "kor": "ko", + "nld": "nl", + "nor": "no", + "pol": "pl", + "por": "pt", + "ron": "ro", + "rus": "ru", + "spa": "es", + "swe": "sv", + "tha": "th", + "tur": "tr", + "ukr": "uk", + "vie": "vi", +} + + +def _tesseract_codes_to_easyocr(tesseract_lang: str) -> List[str]: + """Convert a Tesseract language string (e.g. ``"eng+deu"``) to a list of + EasyOCR language codes (e.g. ``["en", "de"]``). + + Unknown codes are passed through unchanged, so native EasyOCR codes such + as ``"en"`` also work transparently. + """ + codes = [part.strip() for part in tesseract_lang.split("+") if part.strip()] + return [TESSERACT_TO_EASYOCR.get(code, code) for code in codes] + + class OCRProvider(ABC): """Abstract base class for OCR providers. @@ -290,10 +379,24 @@ class TesseractOCRProvider(OCRProvider): - ``tesseract_cmd`` – path to the ``tesseract`` binary (optional). - ``tesseract_language`` – Tesseract language code(s), e.g. ``"eng"`` or ``"eng+deu"`` (default: ``"eng"``). + + The optional *language* constructor argument overrides the global + ``tesseract_language`` setting for this specific provider instance, enabling + per-pipeline language configuration. """ name = "tesseract" + def __init__(self, language: Optional[str] = None) -> None: + """Initialise the Tesseract provider. + + Args: + language: Optional Tesseract language code(s) to use instead of the + global ``tesseract_language`` setting (e.g. ``"eng+deu"``). + Pass ``None`` or ``"auto"`` to use the global setting. + """ + self._language_override: Optional[str] = language if language and language != "auto" else None + def process(self, file_path: str) -> OCRResult: try: import pytesseract @@ -308,7 +411,7 @@ class TesseractOCRProvider(OCRProvider): if tesseract_cmd: pytesseract.pytesseract.tesseract_cmd = tesseract_cmd - lang = getattr(settings, "tesseract_language", None) or "eng" + lang = self._language_override or getattr(settings, "tesseract_language", None) or "eng" # Ensure language data files are present; attempt download if missing. from app.utils.ocr_language_manager import ensure_tesseract_languages # noqa: PLC0415 @@ -349,10 +452,26 @@ class EasyOCRProvider(OCRProvider): - ``easyocr_languages`` – comma-separated list of language codes (default: ``"en"``). - ``easyocr_gpu`` – whether to use GPU acceleration (default: ``False``). + + The optional *language* constructor argument accepts a Tesseract-style + language string (e.g. ``"eng+deu"``) which is automatically translated to + EasyOCR codes (e.g. ``["en", "de"]``), overriding the global + ``easyocr_languages`` setting for this provider instance. """ name = "easyocr" + def __init__(self, language: Optional[str] = None) -> None: + """Initialise the EasyOCR provider. + + Args: + language: Optional Tesseract-style language code(s) (e.g. ``"eng+deu"``) + or a comma-separated EasyOCR language list (e.g. ``"en,de"``). + Pass ``None`` or ``"auto"`` to use the global ``easyocr_languages`` + setting. + """ + self._language_override: Optional[str] = language if language and language != "auto" else None + def process(self, file_path: str) -> OCRResult: try: import easyocr @@ -363,8 +482,12 @@ class EasyOCRProvider(OCRProvider): "Install them with: pip install easyocr pdf2image" ) from exc - lang_str = getattr(settings, "easyocr_languages", None) or "en" - langs = [lang.strip() for lang in lang_str.split(",") if lang.strip()] + if self._language_override: + # Convert Tesseract-style codes to EasyOCR codes + langs = _tesseract_codes_to_easyocr(self._language_override) + else: + lang_str = getattr(settings, "easyocr_languages", None) or "en" + langs = [lang.strip() for lang in lang_str.split(",") if lang.strip()] gpu = getattr(settings, "easyocr_gpu", False) logger.info(f"[EasyOCR] Processing {os.path.basename(file_path)} (langs={langs}, gpu={gpu})") @@ -679,23 +802,36 @@ KNOWN_OCR_PROVIDERS: List[str] = sorted(_PROVIDER_MAP.keys()) MAX_OCR_TEXT_FOR_AI_MERGE = 4000 -def get_ocr_providers() -> List[OCRProvider]: +def get_ocr_providers(language: Optional[str] = None) -> List[OCRProvider]: """Return a list of configured OCR provider instances. Reads ``settings.ocr_providers`` (comma-separated provider names) and returns one instantiated provider per entry. Falls back to ``["azure"]`` when the setting is absent. + + Args: + language: Optional Tesseract-style language code(s) (e.g. ``"eng+deu"``) + to override the global language settings for providers that support + per-call language configuration (Tesseract and EasyOCR). Pass + ``None`` or ``"auto"`` to use the global settings. """ raw = getattr(settings, "ocr_providers", None) or "azure" provider_names = [name.strip().lower() for name in raw.split(",") if name.strip()] + # Normalise "auto" to None so providers fall back to global settings + effective_language = language if language and language != "auto" else None + providers: List[OCRProvider] = [] for name in provider_names: cls = _PROVIDER_MAP.get(name) if cls is None: logger.warning(f"Unknown OCR provider '{name}' in OCR_PROVIDERS – skipping.") continue - providers.append(cls()) + # Pass language override to providers that support per-call language config + if effective_language is not None and name in ("tesseract", "easyocr"): + providers.append(cls(language=effective_language)) + else: + providers.append(cls()) logger.debug(f"Registered OCR provider: {name}") if not providers: diff --git a/docs/API.md b/docs/API.md index 7766e8d9..05e168cc 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1785,12 +1785,27 @@ Returns the catalogue of built-in step types. "label": "OCR Processing", "description": "Extract text using Azure Document Intelligence or local Tesseract.", "config_schema": { - "force_cloud_ocr": { "type": "boolean", "default": false } + "force_cloud_ocr": { "type": "boolean", "default": false }, + "ocr_language": { + "type": "select", + "default": "auto", + "description": "Language(s) for OCR. Overrides the global setting for Tesseract/EasyOCR. Azure/Mistral auto-detect.", + "options": [ + { "value": "auto", "label": "Auto (use system default)" }, + { "value": "eng", "label": "English" }, + { "value": "deu", "label": "German" }, + { "value": "fra", "label": "French" }, + { "value": "spa", "label": "Spanish" }, + "..." + ] + } } } } ``` +The `ocr_language` field accepts Tesseract language codes (e.g. `"eng"`, `"deu"`, `"eng+deu"` for multi-language) or `"auto"` to fall back to the global system setting. The full list of 28 supported language codes is returned by the step-types endpoint. + ### List pipelines ```bash @@ -1883,12 +1898,23 @@ Content-Type: application/json { "step_type": "ocr", - "label": "Cloud OCR", - "config": { "force_cloud_ocr": true }, + "label": "German OCR", + "config": { "force_cloud_ocr": false, "ocr_language": "deu" }, "enabled": true } ``` +Multi-language (Tesseract `+`-separated codes): + +```bash +{ + "step_type": "ocr", + "config": { "ocr_language": "eng+deu" } +} +``` + +Use `"ocr_language": "auto"` (or omit the field) to fall back to the global system language setting. + ### Update step ```bash diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 7ad1776d..15da09b3 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -519,13 +519,50 @@ Processing pipelines let you define exactly what happens to your documents when |-----------|-------------| | `convert_to_pdf` | Convert non-PDF files to PDF using Gotenberg | | `check_duplicates` | Detect duplicate files by content hash | -| `ocr` | Extract text with Azure Document Intelligence or local Tesseract | +| `ocr` | Extract text with OCR (supports multi-language configuration, see below) | | `extract_metadata` | Extract structured metadata (type, sender, tags) with AI | | `embed_metadata` | Write extracted metadata into the PDF document properties | | `compute_embedding` | Compute semantic embeddings for similarity search | | `send_to_destinations` | Upload the processed document to all configured storage destinations | | `classify` | Classify the document type with AI | +#### OCR step options + +The `ocr` step supports two optional configuration fields: + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `force_cloud_ocr` | boolean | `false` | Always run cloud OCR even if the PDF already has embedded text | +| `ocr_language` | string | `"auto"` | Language(s) to use for OCR text extraction (see below) | + +**`ocr_language` — per-pipeline language override** + +This option enables manual language control per pipeline, overriding the global Tesseract/EasyOCR language settings for all documents processed by that pipeline. The following values are supported (28 languages total): + +| Value | Language | Value | Language | +|-------|----------|-------|----------| +| `auto` | Auto (use system default) | `jpn` | Japanese | +| `ara` | Arabic | `kor` | Korean | +| `chi_sim` | Chinese (Simplified) | `nor` | Norwegian | +| `chi_tra` | Chinese (Traditional) | `pol` | Polish | +| `ces` | Czech | `por` | Portuguese | +| `dan` | Danish | `ron` | Romanian | +| `nld` | Dutch | `rus` | Russian | +| `eng` | English | `spa` | Spanish | +| `fin` | Finnish | `swe` | Swedish | +| `fra` | French | `tha` | Thai | +| `deu` | German | `tur` | Turkish | +| `ell` | Greek | `ukr` | Ukrainian | +| `heb` | Hebrew | `vie` | Vietnamese | +| `hin` | Hindi | | | +| `hun` | Hungarian | | | +| `ita` | Italian | | | + +> **Notes:** +> - The language override applies to **Tesseract** and **EasyOCR** providers. **Azure Document Intelligence** and **Mistral OCR** perform automatic language detection regardless of this setting. +> - For multi-language documents with Tesseract, combine codes with `+`, e.g. `eng+deu`. +> - Setting `ocr_language` to `auto` or leaving it unset uses the global `TESSERACT_LANGUAGE` / `EASYOCR_LANGUAGES` environment variables. + ### Assigning a pipeline to a file You can assign (or change) the pipeline for an individual document via the file detail page or the API: diff --git a/frontend/templates/pipelines.html b/frontend/templates/pipelines.html index 92d04b07..9bd718a8 100644 --- a/frontend/templates/pipelines.html +++ b/frontend/templates/pipelines.html @@ -404,6 +404,53 @@ + + +