diff --git a/CHANGELOG.md b/CHANGELOG.md index 8308888..ce1837b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Dashboard** — "Recent Processing Runs" table now reads from the new `/processing-runs` endpoint; shows account name and a *View all logs* link. ### Added +- **Configurable Gmail import labels**: Users can now define which Gmail labels are applied to imported messages from the Settings page. The default setup is opinionated: `{{source_email}}` (rendered to the mailbox address each message came from) plus `imported`, and a reset button restores those defaults instantly. - **Prometheus metrics** (`/metrics` endpoint on the FastAPI backend, scraped every 15 s): - **HTTP layer** — `http_requests_total` (counter, labelled `method`/`endpoint`/`status_code`) and `http_request_duration_seconds` (histogram). Path segments that are numeric IDs are normalised to `{id}` to avoid label-set explosion. - **Mail processing** — `mail_processing_runs_total` (counter, by `status`: `completed` / `partial_failure` / `failed`), `mail_processing_emails_total` (counter, by `operation`: `fetched` / `forwarded` / `failed`), `mail_processing_duration_seconds` (histogram), `active_mail_accounts_total` (gauge — set each scheduler cycle). diff --git a/backend/app/api/v1/endpoints/auth.py b/backend/app/api/v1/endpoints/auth.py index bf91b83..e17f427 100644 --- a/backend/app/api/v1/endpoints/auth.py +++ b/backend/app/api/v1/endpoints/auth.py @@ -22,6 +22,7 @@ from app.models.database_models import User, SubscriptionTier, GmailCredential from app.models.schemas import Token, UserCreate, UserResponse, GoogleAuthRequest from app.services.auth_service import oauth_service from app.services.gmail_service import GmailService, GMAIL_SCOPES +from app.utils.gmail_labels import build_gmail_credential_scopes router = APIRouter() logger = logging.getLogger(__name__) @@ -286,7 +287,10 @@ async def google_oauth( if encrypted_refresh: existing_cred.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment] existing_cred.token_expiry = token_expiry # type: ignore[assignment] - existing_cred.scopes = scope_list # type: ignore[assignment] + existing_cred.scopes = build_gmail_credential_scopes( # type: ignore[assignment] + scope_list, + existing_cred.import_label_templates, + ) existing_cred.is_valid = True # type: ignore[assignment] existing_cred.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment] else: @@ -296,7 +300,7 @@ async def google_oauth( encrypted_access_token=encrypted_access, encrypted_refresh_token=encrypted_refresh, token_expiry=token_expiry, - scopes=scope_list, + scopes=build_gmail_credential_scopes(scope_list), is_valid=True, last_verified_at=datetime.now(timezone.utc), ) diff --git a/backend/app/api/v1/endpoints/providers.py b/backend/app/api/v1/endpoints/providers.py index 59db266..7a9a72a 100644 --- a/backend/app/api/v1/endpoints/providers.py +++ b/backend/app/api/v1/endpoints/providers.py @@ -21,12 +21,30 @@ from app.models.schemas import ( GmailCredentialResponse, GmailAuthorizeResponse, GmailCallbackRequest, + GmailImportLabelsUpdate, ) from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES +from app.utils.gmail_labels import ( + MAX_IMPORT_LABELS, + build_gmail_credential_scopes, + extract_granted_scopes, + normalize_import_label_templates, +) router = APIRouter() logger = logging.getLogger(__name__) + +def _validated_import_label_templates(label_templates: List[str]) -> List[str]: + normalized = normalize_import_label_templates(label_templates) + if len(normalized) > MAX_IMPORT_LABELS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"You can configure up to {MAX_IMPORT_LABELS} Gmail import labels.", + ) + return normalized + + # Gmail API scopes requested during the "Connect Gmail" OAuth flow. # GMAIL_SCOPES (gmail.insert, gmail.labels, gmail.readonly) are imported from # gmail_service so the scope list stays in sync with what GmailService uses. @@ -223,6 +241,10 @@ async def save_gmail_credential( existing.gmail_email = credential_in.gmail_email # type: ignore[assignment] existing.encrypted_access_token = encrypted_access # type: ignore[assignment] existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment] + existing.scopes = build_gmail_credential_scopes( + existing.granted_scopes, + existing.import_label_templates, + ) # type: ignore[assignment] existing.is_valid = True # type: ignore[assignment] existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment] await db.commit() @@ -235,6 +257,7 @@ async def save_gmail_credential( gmail_email=credential_in.gmail_email, encrypted_access_token=encrypted_access, encrypted_refresh_token=encrypted_refresh, + scopes=build_gmail_credential_scopes(), is_valid=True, last_verified_at=datetime.now(timezone.utc), ) @@ -285,6 +308,33 @@ async def delete_gmail_credential( await db.commit() +@router.put("/gmail-credential/labels", response_model=GmailCredentialResponse) +async def update_gmail_import_labels( + labels_in: GmailImportLabelsUpdate, + current_user: User = Depends(get_current_active_user), + db: AsyncSession = Depends(get_db), +): + """Update the Gmail labels applied to imported messages.""" + result = await db.execute( + select(GmailCredential).where(GmailCredential.user_id == current_user.id) + ) + credential = result.scalar_one_or_none() + + if not credential: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No Gmail credentials found. Connect Gmail first.", + ) + + credential.scopes = build_gmail_credential_scopes( # type: ignore[assignment] + extract_granted_scopes(credential.scopes), + _validated_import_label_templates(labels_in.import_label_templates), + ) + await db.commit() + await db.refresh(credential) + return credential + + @router.get("/gmail/authorize-url", response_model=GmailAuthorizeResponse) async def get_gmail_authorize_url( redirect_uri: str, @@ -365,6 +415,7 @@ async def send_gmail_debug_email( try: inject_result = await gmail_service.inject_debug_email( recipient_email=credential.gmail_email, # type: ignore[arg-type] + import_label_templates=credential.import_label_templates, ) except GmailInjectionError as exc: raise HTTPException( @@ -498,7 +549,10 @@ async def gmail_oauth_callback( if encrypted_refresh: existing.encrypted_refresh_token = encrypted_refresh # type: ignore[assignment] existing.token_expiry = token_expiry # type: ignore[assignment] - existing.scopes = token_data.get("scope", "").split() # type: ignore[assignment] + existing.scopes = build_gmail_credential_scopes( # type: ignore[assignment] + token_data.get("scope", "").split(), + existing.import_label_templates, + ) existing.is_valid = True # type: ignore[assignment] existing.last_verified_at = datetime.now(timezone.utc) # type: ignore[assignment] await db.commit() @@ -511,7 +565,7 @@ async def gmail_oauth_callback( encrypted_access_token=encrypted_access, encrypted_refresh_token=encrypted_refresh, token_expiry=token_expiry, - scopes=token_data.get("scope", "").split(), + scopes=build_gmail_credential_scopes(token_data.get("scope", "").split()), is_valid=True, last_verified_at=datetime.now(timezone.utc), ) diff --git a/backend/app/models/database_models.py b/backend/app/models/database_models.py index 7932792..c4d50fd 100644 --- a/backend/app/models/database_models.py +++ b/backend/app/models/database_models.py @@ -17,6 +17,12 @@ from sqlalchemy import ( Index, ) from sqlalchemy.orm import relationship + +from app.utils.gmail_labels import ( + DEFAULT_IMPORT_LABEL_TEMPLATES, + extract_granted_scopes, + extract_import_label_templates, +) import enum from app.core.database import Base @@ -557,6 +563,18 @@ class GmailCredential(Base): # Relationships user = relationship("User", backref="gmail_credential") + @property + def granted_scopes(self) -> list[str]: + return extract_granted_scopes(self.scopes) + + @property + def import_label_templates(self) -> list[str]: + return extract_import_label_templates(self.scopes) + + @property + def default_import_label_templates(self) -> list[str]: + return DEFAULT_IMPORT_LABEL_TEMPLATES.copy() + class AppSetting(Base): """ diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 9765f3e..20f7704 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -396,6 +396,8 @@ class GmailCredentialResponse(BaseModel): user_id: int gmail_email: str is_valid: bool + import_label_templates: List[str] = Field(default_factory=list) + default_import_label_templates: List[str] = Field(default_factory=list) last_verified_at: Optional[datetime] = None created_at: datetime updated_at: datetime @@ -450,6 +452,10 @@ class GmailCallbackRequest(BaseModel): redirect_uri: str +class GmailImportLabelsUpdate(BaseModel): + import_label_templates: List[str] = Field(default_factory=list) + + # Admin Schemas diff --git a/backend/app/services/gmail_service.py b/backend/app/services/gmail_service.py index ea5b93e..48a4149 100644 --- a/backend/app/services/gmail_service.py +++ b/backend/app/services/gmail_service.py @@ -25,6 +25,7 @@ from app.core.metrics import ( GMAIL_API_DURATION_SECONDS, GMAIL_TOKEN_REFRESHES_TOTAL, ) +from app.utils.gmail_labels import render_import_labels logger = logging.getLogger(__name__) @@ -294,6 +295,7 @@ class GmailService: async def inject_debug_email( self, recipient_email: str, + import_label_templates: Optional[list[str]] = None, ) -> Dict[str, Any]: """ Inject a debug/test email into the user's Gmail inbox. @@ -344,11 +346,10 @@ class GmailService: raw_bytes = msg.as_bytes() - # Resolve label IDs (create labels if they don't exist yet) + label_ids = await self.build_import_label_ids(import_label_templates) test_label_id = await self.get_or_create_label("test") - imported_label_id = await self.get_or_create_label("imported") - - label_ids = ["INBOX", test_label_id, imported_label_id] + if test_label_id not in label_ids: + label_ids.append(test_label_id) return await self.inject_email( raw_email=raw_bytes, @@ -356,6 +357,23 @@ class GmailService: source_account_name="debug", ) + async def build_import_label_ids( + self, + import_label_templates: Optional[list[str]] = None, + source_email: Optional[str] = None, + ) -> list[str]: + """Resolve configured import labels into Gmail label IDs.""" + label_ids = ["INBOX"] + + for label_name in render_import_labels(import_label_templates, source_email): + if label_name.upper() == "INBOX": + continue + label_id = await self.get_or_create_label(label_name) + if label_id not in label_ids: + label_ids.append(label_id) + + return label_ids + def get_refreshed_token(self) -> Optional[Dict[str, Any]]: """ Return the current access token and expiry if the token was refreshed diff --git a/backend/app/utils/gmail_labels.py b/backend/app/utils/gmail_labels.py new file mode 100644 index 0000000..25d8848 --- /dev/null +++ b/backend/app/utils/gmail_labels.py @@ -0,0 +1,97 @@ +"""Helpers for Gmail import label configuration and rendering.""" + +from typing import Any, Iterable, Optional + +SOURCE_EMAIL_LABEL_TEMPLATE = "{{source_email}}" +DEFAULT_IMPORT_LABEL_TEMPLATES = [SOURCE_EMAIL_LABEL_TEMPLATE, "imported"] +MAX_IMPORT_LABELS = 10 + + +def _normalize_string_list(values: Optional[Iterable[str]]) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + + for value in values or []: + cleaned = value.strip() + if not cleaned: + continue + lowered = cleaned.casefold() + if lowered in seen: + continue + seen.add(lowered) + normalized.append(cleaned) + + return normalized + + +def normalize_import_label_templates( + label_templates: Optional[Iterable[str]], +) -> list[str]: + """Return a cleaned, de-duplicated label template list.""" + normalized = _normalize_string_list(label_templates) + return normalized or DEFAULT_IMPORT_LABEL_TEMPLATES.copy() + + +def extract_granted_scopes(scopes_data: Any) -> list[str]: + """Read granted scopes from legacy list or new JSON object storage.""" + if isinstance(scopes_data, list): + return _normalize_string_list( + value for value in scopes_data if isinstance(value, str) + ) + + if isinstance(scopes_data, dict): + granted_scopes = scopes_data.get("granted_scopes", []) + if isinstance(granted_scopes, list): + return _normalize_string_list( + value for value in granted_scopes if isinstance(value, str) + ) + + return [] + + +def extract_import_label_templates(scopes_data: Any) -> list[str]: + """Read import label templates from stored Gmail credential metadata.""" + if isinstance(scopes_data, dict): + stored_templates = scopes_data.get("import_label_templates", []) + if isinstance(stored_templates, list): + return normalize_import_label_templates( + value for value in stored_templates if isinstance(value, str) + ) + + return DEFAULT_IMPORT_LABEL_TEMPLATES.copy() + + +def build_gmail_credential_scopes( + granted_scopes: Optional[Iterable[str]], + import_label_templates: Optional[Iterable[str]] = None, +) -> dict[str, list[str]]: + """Persist Gmail metadata in the existing JSON column.""" + return { + "granted_scopes": _normalize_string_list(granted_scopes), + "import_label_templates": normalize_import_label_templates( + import_label_templates + ), + } + + +def render_import_labels( + import_label_templates: Optional[Iterable[str]], + source_email: Optional[str], +) -> list[str]: + """Render label templates into actual Gmail label names.""" + rendered_labels: list[str] = [] + seen: set[str] = set() + resolved_source_email = source_email.strip() if source_email else "" + + for template in normalize_import_label_templates(import_label_templates): + rendered = template.replace(SOURCE_EMAIL_LABEL_TEMPLATE, resolved_source_email) + rendered = rendered.strip() + if not rendered: + continue + lowered = rendered.casefold() + if lowered in seen: + continue + seen.add(lowered) + rendered_labels.append(rendered) + + return rendered_labels diff --git a/backend/app/workers/tasks.py b/backend/app/workers/tasks.py index 5276593..417129a 100644 --- a/backend/app/workers/tasks.py +++ b/backend/app/workers/tasks.py @@ -229,9 +229,13 @@ async def process_mail_account(account_id: int): try: if use_gmail_api and gmail_service: # Inject via Gmail API (preferred) + label_ids = await gmail_service.build_import_label_ids( + import_label_templates=gmail_cred.import_label_templates, + source_email=account.email_address, # type: ignore[arg-type] + ) await gmail_service.inject_email( raw_email=email_data, - label_ids=["INBOX"], + label_ids=label_ids, source_account_name=account.name, # type: ignore[arg-type] ) forwarded_ok = True diff --git a/backend/tests/unit/test_gmail_service.py b/backend/tests/unit/test_gmail_service.py index 09810e9..4289104 100644 --- a/backend/tests/unit/test_gmail_service.py +++ b/backend/tests/unit/test_gmail_service.py @@ -3,8 +3,16 @@ Unit tests for Gmail service module. """ import pytest -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock from app.services.gmail_service import GmailService, GmailInjectionError, GMAIL_SCOPES +from app.utils.gmail_labels import ( + DEFAULT_IMPORT_LABEL_TEMPLATES, + SOURCE_EMAIL_LABEL_TEMPLATE, + build_gmail_credential_scopes, + extract_granted_scopes, + extract_import_label_templates, + render_import_labels, +) class TestGmailService: @@ -153,3 +161,44 @@ class TestGmailService: email = await service.get_email_address() assert email is None + + def test_gmail_label_metadata_helpers(self): + """Test Gmail metadata extraction remains backward compatible.""" + scopes = build_gmail_credential_scopes( + ["scope-a", "scope-b"], + [SOURCE_EMAIL_LABEL_TEMPLATE, "Imported", " imported "], + ) + + assert extract_granted_scopes(scopes) == ["scope-a", "scope-b"] + assert extract_import_label_templates(scopes) == [ + SOURCE_EMAIL_LABEL_TEMPLATE, + "Imported", + ] + assert ( + extract_import_label_templates(["legacy-scope"]) + == DEFAULT_IMPORT_LABEL_TEMPLATES + ) + + def test_render_import_labels_uses_source_email_template(self): + """Test that source email templates render to the source mailbox address.""" + rendered = render_import_labels( + [SOURCE_EMAIL_LABEL_TEMPLATE, "Imported", ""], + "source@example.com", + ) + + assert rendered == ["source@example.com", "Imported"] + + @pytest.mark.asyncio + async def test_build_import_label_ids_creates_configured_labels(self): + """Test that configured import labels are created and added alongside INBOX.""" + service = GmailService(access_token="test-access-token") + service.get_or_create_label = AsyncMock( + side_effect=["Label-source", "Label-imported"] + ) # type: ignore[method-assign] + + label_ids = await service.build_import_label_ids( + import_label_templates=[SOURCE_EMAIL_LABEL_TEMPLATE, "imported"], + source_email="source@example.com", + ) + + assert label_ids == ["INBOX", "Label-source", "Label-imported"] diff --git a/docs/TODO.md b/docs/TODO.md index 35f5f0e..f8c50e7 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -199,6 +199,7 @@ Comprehensive task breakdown for repository improvements and production readines - [x] Account enable/disable toggle (UX + backend) - [x] Per-user SMTP configuration (UX + backend) - [x] Gmail API one-click OAuth grant flow with token refresh and revocation handling +- [x] Configurable Gmail import labels (default `{{source_email}}` + `imported`, editable in Settings with reset-to-default action) - [x] Unified Google OAuth flow: sign-in requests all Gmail scopes; single `/auth/callback` redirect URI needed in Google Console - [x] Message deduplication (POP3 UIDL + IMAP \Seen flag + DB tracking) - [x] **Debug email**: "Send Debug Email" button in Settings injects a test message (from christian@docuelevate.org, dated today, labelled `test` + `imported`, placed in inbox) to verify end-to-end Gmail API delivery diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index a12f6a0..6253e9f 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -16,8 +16,112 @@ import { AlertTriangle, XCircle, Bug, + RotateCcw, + Tags, } from 'lucide-react'; +const DEFAULT_GMAIL_IMPORT_LABEL_TEMPLATES = ['{{source_email}}', 'imported']; + +function parseImportLabelTemplates(input: string): string[] { + return input + .split('\n') + .map((value) => value.trim()) + .filter((value, index, values) => value.length > 0 && values.indexOf(value) === index); +} + +function GmailImportLabelsForm({ + gmailCredential, + onSave, + isSaving, +}: { + gmailCredential: { + gmail_email: string; + import_label_templates: string[]; + default_import_label_templates: string[]; + }; + onSave: (labels: string[]) => void; + isSaving: boolean; +}) { + const [labelsInput, setLabelsInput] = useState( + gmailCredential.import_label_templates.join('\n') + ); + + const defaultTemplates = + gmailCredential.default_import_label_templates.length > 0 + ? gmailCredential.default_import_label_templates + : DEFAULT_GMAIL_IMPORT_LABEL_TEMPLATES; + const parsedLabels = parseImportLabelTemplates(labelsInput); + const isDefaultSelection = + parsedLabels.length === defaultTemplates.length && + parsedLabels.every((value, index) => value === defaultTemplates[index]); + + return ( +
+
+ +

Import labels

+
+

+ One label is created per line. We recommend keeping{' '} + + {'{{source_email}}'} + {' '} + so each imported message is tagged with the mailbox it came from, plus a + catch-all label like imported. +

+

+ Example: a mail pulled from billing@example.com will be + labeled as billing@example.com when{' '} + + {'{{source_email}}'} + {' '} + is present. +

+