diff --git a/.env.demo b/.env.demo index 9412e754..5bf0a14d 100644 --- a/.env.demo +++ b/.env.demo @@ -258,6 +258,13 @@ OPENAI_MODEL=gpt-4o-mini # AZURE_OPENAI_API_VERSION=2024-02-01 # AI_MODEL=gpt-4o # deployment name in Azure +# **Document Translation** +# After processing, documents whose detected language differs from the default +# target language are automatically translated. Only the original and this +# default-language version are persisted; other translations are on-the-fly. +# Users can override this in their profile settings. +# DEFAULT_DOCUMENT_LANGUAGE=en + # Azure Document Intelligence (OCR – separate from AI provider above) # **Email Settings (shared SMTP – password reset, verification, and system notifications)** EMAIL_HOST=smtp.example.com diff --git a/app/api/__init__.py b/app/api/__init__.py index 25344332..246e47f3 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router from app.api.shared_links import router as shared_links_router from app.api.similarity import router as similarity_router from app.api.subscriptions import router as subscriptions_router +from app.api.translation import router as translation_router from app.api.url_upload import router as url_upload_router # Import all the individual routers @@ -96,3 +97,4 @@ router.include_router(audit_logs_router) router.include_router(i18n_router) router.include_router(mobile_router) router.include_router(compliance_router) +router.include_router(translation_router) diff --git a/app/api/profile.py b/app/api/profile.py index 90369f73..b3482e3c 100644 --- a/app/api/profile.py +++ b/app/api/profile.py @@ -99,6 +99,8 @@ class ProfileResponse(BaseModel): contact_email: str | None preferred_language: str | None preferred_theme: str | None + default_document_language: str | None + """ISO 639-1 code for the user's preferred document translation target language.""" avatar_url: str """Gravatar URL or ``data:`` URI for a custom uploaded avatar.""" is_local_user: bool @@ -112,6 +114,10 @@ class ProfileUpdateRequest(BaseModel): contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail") preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'") preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'") + default_document_language: str | None = Field( + default=None, + description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'", + ) class ChangePasswordRequest(BaseModel): @@ -149,6 +155,7 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse: contact_email=profile.contact_email, # type: ignore[arg-type] preferred_language=profile.preferred_language, # type: ignore[arg-type] preferred_theme=profile.preferred_theme, # type: ignore[arg-type] + default_document_language=profile.default_document_language, # type: ignore[arg-type] avatar_url=avatar_url, is_local_user=is_local, ) @@ -201,6 +208,16 @@ async def update_profile( ) profile.preferred_theme = theme or None # type: ignore[assignment] + # Validate default document language + if body.default_document_language is not None: + doc_lang = body.default_document_language.lower().strip() + if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unsupported language code: {doc_lang}", + ) + profile.default_document_language = doc_lang or None # type: ignore[assignment] + if body.display_name is not None: profile.display_name = body.display_name.strip() or None # type: ignore[assignment] @@ -225,6 +242,7 @@ async def update_profile( contact_email=profile.contact_email, # type: ignore[arg-type] preferred_language=profile.preferred_language, # type: ignore[arg-type] preferred_theme=profile.preferred_theme, # type: ignore[arg-type] + default_document_language=profile.default_document_language, # type: ignore[arg-type] avatar_url=avatar_url, is_local_user=is_local, ) diff --git a/app/api/translation.py b/app/api/translation.py new file mode 100644 index 00000000..20c1af46 --- /dev/null +++ b/app/api/translation.py @@ -0,0 +1,158 @@ +""" +API endpoints for document translation. + +Provides on-the-fly translation via the AI provider and access to the +persisted default-language translation. +""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.responses import JSONResponse +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.config import settings +from app.database import get_db +from app.models import FileRecord +from app.utils.ai_provider import get_ai_provider +from app.utils.user_scope import apply_owner_filter, get_current_owner_id + +logger = logging.getLogger(__name__) + +router = APIRouter() + +DbSession = Annotated[Session, Depends(get_db)] + +# Maximum characters sent to the AI provider for a single translation request. +_MAX_TRANSLATION_INPUT = 50_000 + + +def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord: + """Fetch a FileRecord visible to the current user or raise 404.""" + query = db.query(FileRecord).filter(FileRecord.id == file_id) + owner_id = get_current_owner_id(request) + if owner_id: + query = apply_owner_filter(query, owner_id, FileRecord) + record = query.first() + if not record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + return record + + +@router.get("/files/{file_id}/translation/default") +@require_login +def get_default_translation( + request: Request, + file_id: int, + db: DbSession, +) -> JSONResponse: + """Return the persisted default-language translation for a document. + + Returns 404 if no default-language translation has been generated yet + (e.g. because the document is already in the default language). + """ + record = _get_file_or_404(db, file_id, request) + + if not record.default_language_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No default-language translation available for this file", + ) + + return JSONResponse( + content={ + "file_id": record.id, + "detected_language": record.detected_language, + "default_language_code": record.default_language_code, + "text": record.default_language_text, + } + ) + + +@router.get("/files/{file_id}/translate") +@require_login +def translate_on_the_fly( + request: Request, + file_id: int, + db: DbSession, + lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"), +) -> JSONResponse: + """Translate a document's extracted text into an arbitrary language on the fly. + + The translation is generated via the configured AI provider and is **not** + persisted. For the default-language translation, use the + ``/files/{file_id}/translation/default`` endpoint instead. + """ + record = _get_file_or_404(db, file_id, request) + + source_text = record.ocr_text + if not source_text: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No extracted text available for this file — translation requires OCR text", + ) + + # If the requested language matches what is already stored, return it directly. + if record.default_language_code and lang == record.default_language_code and record.default_language_text: + return JSONResponse( + content={ + "file_id": record.id, + "source_language": record.detected_language, + "target_language": lang, + "text": record.default_language_text, + "cached": True, + } + ) + + # If the detected language already matches, return the original text. + detected = record.detected_language + if detected and detected == lang: + return JSONResponse( + content={ + "file_id": record.id, + "source_language": detected, + "target_language": lang, + "text": source_text, + "cached": True, + } + ) + + # Truncate to keep AI costs bounded. + text_to_translate = source_text[:_MAX_TRANSLATION_INPUT] + + try: + provider = get_ai_provider() + model = settings.ai_model or settings.openai_model + translated = provider.chat_completion( + messages=[ + { + "role": "system", + "content": ( + f"You are a professional translator. Translate the following text " + f"into {lang}. Preserve the original formatting, paragraph structure, " + f"and meaning. Do not add any commentary — output ONLY the translated text." + ), + }, + {"role": "user", "content": text_to_translate}, + ], + model=model, + temperature=0.3, + ) + except Exception as exc: + logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Translation failed — the AI provider returned an error", + ) + + return JSONResponse( + content={ + "file_id": record.id, + "source_language": detected or "unknown", + "target_language": lang, + "text": translated, + "cached": False, + } + ) diff --git a/app/celery_worker.py b/app/celery_worker.py index fd68c332..17110a66 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -39,6 +39,7 @@ from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401 from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401 from app.tasks.send_to_all import send_to_all_destinations # noqa: F401 from app.tasks.subscription_tasks import apply_pending_subscription_changes_all # noqa: F401 +from app.tasks.translate_to_default_language import translate_to_default_language # noqa: F401 # Import new send tasks from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401 diff --git a/app/config.py b/app/config.py index 30b434d1..ac9ab7bb 100644 --- a/app/config.py +++ b/app/config.py @@ -166,6 +166,25 @@ class Settings(BaseSettings): google_docai_location: str = "us" # Processor location, e.g. "us" or "eu" external_hostname: str = "localhost" # Default to localhost + # --------------------------------------------------------------------------- + # Document Translation Settings + # --------------------------------------------------------------------------- + # Default target language for automatic document translation (ISO 639-1 code). + # After OCR / metadata extraction, if the detected document language differs + # from this value the system translates the extracted text into this language + # and stores it alongside the original. Other language translations are + # generated on the fly via the AI provider and are NOT persisted. + # Per-user overrides are stored in UserProfile.default_document_language. + default_document_language: str = Field( + default="en", + description=( + "ISO 639-1 language code for the default translation target " + "(e.g. 'en', 'de', 'fr'). Documents whose detected language " + "differs are automatically translated into this language after " + "processing. Default: 'en' (English)." + ), + ) + # Authentication settings auth_enabled: bool = True # Default to enabled admin_username: Optional[str] = None diff --git a/app/models.py b/app/models.py index b752fdb6..54cef37d 100644 --- a/app/models.py +++ b/app/models.py @@ -84,6 +84,19 @@ class FileRecord(Base): # Processing pipeline assigned to this file (NULL = use system default) pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True) + # Detected document language (ISO 639-1 code, e.g. "de", "en", "fr") + # Extracted from AI metadata during processing; cached here for fast access. + detected_language = Column(String(10), nullable=True) + + # Default-language translation of the extracted text. + # Stored when the detected language differs from the user's/system default + # document language. Only the original text and this translation are persisted; + # other languages are translated on the fly via the AI provider. + default_language_text = Column(Text, nullable=True) + + # ISO 639-1 code of the default-language translation stored above (e.g. "en"). + default_language_code = Column(String(10), nullable=True) + # Timestamp when we inserted this record created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) @@ -282,6 +295,12 @@ class UserProfile(Base): # NULL means "auto-detect from browser Accept-Language header" preferred_language = Column(String(10), nullable=True) + # Default document language for translated versions (ISO 639-1 code). + # When a document's detected language differs from this value, the system + # automatically generates and stores a translation into this language. + # NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting". + default_document_language = Column(String(10), nullable=True) + # UI colour scheme preference: "light" | "dark" | "system" (NULL = "system") preferred_theme = Column(String(10), nullable=True) diff --git a/app/tasks/embed_metadata_into_pdf.py b/app/tasks/embed_metadata_into_pdf.py index 3bc78664..e0f40716 100644 --- a/app/tasks/embed_metadata_into_pdf.py +++ b/app/tasks/embed_metadata_into_pdf.py @@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met except Exception as search_exc: logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}") + # Cache the detected language on the FileRecord and trigger + # default-language translation when the document is in a + # different language. + detected_lang = metadata.get("language") if metadata else None + if detected_lang and extracted_text: + try: + file_record.detected_language = detected_lang + db.commit() + + from app.tasks.translate_to_default_language import translate_to_default_language + + translate_to_default_language.delay( + file_id, + extracted_text, + detected_lang, + owner_id=file_record.owner_id, + ) + logger.info( + f"[{task_id}] Queued default-language translation for file {file_id} " + f"(detected: {detected_lang})" + ) + except Exception as trans_exc: + logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}") + # Persist the metadata into a JSON file with the same base name. # Include file path references for traceability logger.info(f"[{task_id}] Persisting metadata to JSON") diff --git a/app/tasks/translate_to_default_language.py b/app/tasks/translate_to_default_language.py new file mode 100644 index 00000000..966fc4f7 --- /dev/null +++ b/app/tasks/translate_to_default_language.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Celery task to translate extracted document text into the default target language. + +This task is triggered after metadata extraction when the detected document +language differs from the user's (or system) default document language. The +translated text is persisted in ``FileRecord.default_language_text`` so that +users can always read a reference copy in their preferred language. + +Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate`` +endpoint and are NOT persisted. +""" + +import logging + +from app.celery_app import celery +from app.config import settings +from app.database import SessionLocal +from app.models import FileRecord, UserProfile +from app.tasks.retry_config import BaseTaskWithRetry +from app.utils import log_task_progress +from app.utils.ai_provider import get_ai_provider + +logger = logging.getLogger(__name__) + + +def _resolve_default_language(owner_id: str | None) -> str: + """Return the default document language for the given owner. + + Resolution order: + 1. ``UserProfile.default_document_language`` (per-user override) + 2. ``settings.default_document_language`` (global setting) + """ + if owner_id: + with SessionLocal() as db: + profile = db.query(UserProfile).filter_by(user_id=owner_id).first() + if profile and profile.default_document_language: + return profile.default_document_language + return settings.default_document_language + + +@celery.task(base=BaseTaskWithRetry, bind=True) +def translate_to_default_language( + self, + file_id: int, + extracted_text: str, + detected_language: str, + owner_id: str | None = None, +) -> dict: + """Translate *extracted_text* into the default document language and persist the result. + + Args: + file_id: Primary key of the :class:`FileRecord`. + extracted_text: The OCR / refined text in the document's original language. + detected_language: ISO 639-1 code of the document's detected language. + owner_id: Owner identifier used to resolve per-user language preference. + + Returns: + A dict with ``status``, ``target_language``, and the translated text length. + """ + task_id = self.request.id + target_language = _resolve_default_language(owner_id) + + # Nothing to do when the document is already in the target language. + if detected_language == target_language: + logger.info( + f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation" + ) + log_task_progress( + task_id, + "translate_to_default_language", + "skipped", + f"Document already in {target_language}", + file_id=file_id, + ) + return {"status": "skipped", "reason": "already_in_target_language"} + + logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'") + log_task_progress( + task_id, + "translate_to_default_language", + "in_progress", + f"Translating from {detected_language} to {target_language}", + file_id=file_id, + ) + + try: + provider = get_ai_provider() + model = settings.ai_model or settings.openai_model + translated_text = provider.chat_completion( + messages=[ + { + "role": "system", + "content": ( + f"You are a professional translator. Translate the following text " + f"from {detected_language} to {target_language}. " + f"Preserve the original formatting, paragraph structure, and meaning. " + f"Do not add any commentary or explanation — output ONLY the translated text." + ), + }, + {"role": "user", "content": extracted_text}, + ], + model=model, + temperature=0.3, + ) + + # Persist the translation. + with SessionLocal() as db: + record = db.query(FileRecord).filter_by(id=file_id).first() + if record: + record.default_language_text = translated_text + record.default_language_code = target_language + record.detected_language = detected_language + db.commit() + logger.info( + f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}" + ) + + log_task_progress( + task_id, + "translate_to_default_language", + "success", + f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})", + file_id=file_id, + ) + + return { + "status": "success", + "target_language": target_language, + "translated_length": len(translated_text), + } + + except Exception as exc: + logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}") + log_task_progress( + task_id, + "translate_to_default_language", + "failure", + f"Exception: {exc}", + file_id=file_id, + ) + raise diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index a0120670..e130fbd9 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -522,6 +522,19 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Document Translation + "default_document_language": { + "category": "AI Services", + "description": ( + "ISO 639-1 language code for the default document translation target " + "(e.g. 'en', 'de', 'fr'). Documents whose detected language differs " + "are automatically translated into this language after processing." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # OCR Engine Configuration "ocr_providers": { "category": "OCR Engines", diff --git a/app/views/files.py b/app/views/files.py index 58efa132..aa1ee577 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get ) +@router.get("/files/{file_id}/text/default-language") +@require_login +def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)): + """Return the persisted default-language translation for the file view.""" + from fastapi import status + from fastapi.responses import JSONResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) + + if not file_record.default_language_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No default-language translation available", + ) + + return JSONResponse( + content={ + "text": file_record.default_language_text, + "language_code": file_record.default_language_code, + "detected_language": file_record.detected_language, + } + ) + + @router.get("/duplicates") @require_login def duplicates_page( diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 37acf9c2..45f4ba13 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -934,6 +934,48 @@ OPENAI_API_KEY=sk-ant-... # passed as the api_key to LiteLLM --- +### Document Translation + +After processing, DocuElevate can automatically translate a document's extracted text into a configurable *default language* (e.g. English). This reference translation is stored alongside the original text so users always have a version in a language they understand. + +Other languages are translated **on the fly** via the AI provider and are not persisted. + +#### Settings + +| **Variable** | **Description** | **Default** | +|------------------------------|-----------------------------------------------------------------------------------------------------------|-------------| +| `DEFAULT_DOCUMENT_LANGUAGE` | ISO 639-1 code for the default translation target (e.g. `en`, `de`, `fr`). Documents whose detected language differs are automatically translated into this language after processing. | `en` | + +Each user can override this global default in their profile (`UserProfile.default_document_language`). + +#### How It Works + +1. During metadata extraction the AI detects the document language (stored as `detected_language` on the file record). +2. If the detected language differs from the default target language, a background Celery task (`translate_to_default_language`) translates the extracted text. +3. The translated text is persisted in `default_language_text` and the target code in `default_language_code`. +4. The file detail view shows both the original text and the default-language version. +5. Users can also request on-the-fly translations to any language via the **Translate** dropdown. + +#### API Endpoints + +| **Endpoint** | **Method** | **Description** | +|-----------------------------------------------|------------|------------------------------------------------------------------------| +| `/api/files/{id}/translation/default` | GET | Returns the persisted default-language translation (404 if unavailable)| +| `/api/files/{id}/translate?lang=xx` | GET | On-the-fly translation to any ISO 639-1 language code | +| `/files/{id}/text/default-language` | GET | View endpoint returning the default-language text as JSON | + +#### Example + +```bash +# Get the stored English translation of a German document +curl http://localhost:8000/api/files/42/translation/default + +# Translate on the fly to French +curl "http://localhost:8000/api/files/42/translate?lang=fr" +``` + +--- + ### OCR Providers DocuElevate supports multiple OCR engines that can be used individually or in combination. Configure the list of active providers with `OCR_PROVIDERS` and tune each provider with the settings below. diff --git a/frontend/templates/file_view.html b/frontend/templates/file_view.html index 3489ab2a..01083b78 100644 --- a/frontend/templates/file_view.html +++ b/frontend/templates/file_view.html @@ -439,10 +439,105 @@ + {% if file.detected_language %} +
+ + Detected language: {{ file.detected_language }} +
+ {% endif %} + + + {% if file.default_language_text %} +
+
+
+ + Default Language Version + {% if file.default_language_code %} + ({{ file.default_language_code }}) + {% endif %} +
+
+ + +
+
+ +
+ {% elif file.detected_language %} + +
+
+
+ + Default Language Version +
+ +
+
+
+ {% endif %} + + + {% if file.ocr_text %} +
+
+
+ + Translate to Another Language +
+
+
+ + + + +
+
+
+ {% endif %} + {% elif processed_file_exists or original_file_exists %}
@@ -688,6 +783,149 @@ }); } + // ── Default-language text toggle ── + function toggleDefaultLangText() { + var block = document.getElementById('default-lang-text-block'); + var icon = document.getElementById('default-lang-toggle-icon'); + var label = document.getElementById('default-lang-toggle-label'); + var btn = icon ? icon.closest('button') : null; + if (!block) return; + if (block.style.display === 'none') { + block.style.display = 'block'; + if (icon) icon.className = 'fas fa-chevron-up'; + if (label) label.textContent = 'Hide text'; + if (btn) btn.setAttribute('aria-expanded', 'true'); + } else { + block.style.display = 'none'; + if (icon) icon.className = 'fas fa-chevron-down'; + if (label) label.textContent = 'Show text'; + if (btn) btn.setAttribute('aria-expanded', 'false'); + } + } + + // ── Copy default-language text ── + function copyDefaultLangText() { + var content = document.getElementById('default-lang-text-content'); + if (!content) return; + var text = content.textContent; + var btn = document.getElementById('default-lang-copy-btn'); + if (!btn) return; + var orig = btn.innerHTML; + navigator.clipboard.writeText(text).then(function() { + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + }).catch(function() { + try { + var ta = document.createElement('textarea'); + ta.value = text; + ta.style.cssText = 'position:fixed;opacity:0;'; + document.body.appendChild(ta); + ta.select(); + document.execCommand('copy'); + document.body.removeChild(ta); + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + } catch(e) {} + }); + } + + // ── Load default-language text on demand ── + function loadDefaultLangText(fileId) { + var area = document.getElementById('default-lang-load-area'); + if (!area) return; + area.innerHTML = '

Loading translation…

'; + fetch('/files/' + fileId + '/text/default-language') + .then(function(r) { + if (r.status === 404) throw new Error('No translation available yet — it may still be processing'); + if (!r.ok) throw new Error('HTTP ' + r.status); + return r.json(); + }) + .then(function(data) { + area.innerHTML = ''; + var info = document.createElement('div'); + info.style.cssText = 'font-size:0.75rem;color:#6b7280;margin-bottom:0.5rem;'; + info.textContent = 'Translated to: ' + data.language_code + (data.detected_language ? ' (from ' + data.detected_language + ')' : ''); + area.appendChild(info); + var pre = document.createElement('pre'); + pre.className = 'ocr-text'; + pre.textContent = data.text; + area.appendChild(pre); + }) + .catch(function(err) { + area.innerHTML = '
' + err.message + '
'; + }); + } + + // ── On-the-fly translation ── + var _translateCache = {}; + function translateOnTheFly(fileId) { + var select = document.getElementById('translate-lang-select'); + var area = document.getElementById('translate-result-area'); + var copyBtn = document.getElementById('translate-copy-btn'); + if (!select || !area) return; + var lang = select.value; + if (!lang) { area.innerHTML = '
Please select a target language.
'; return; } + + // Check cache + if (_translateCache[lang]) { + renderTranslation(area, _translateCache[lang], lang); + if (copyBtn) copyBtn.style.display = ''; + return; + } + + area.innerHTML = '

Translating…

'; + if (copyBtn) copyBtn.style.display = 'none'; + + fetch('/api/files/' + fileId + '/translate?lang=' + encodeURIComponent(lang)) + .then(function(r) { if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); }) + .then(function(data) { + _translateCache[lang] = data.text; + renderTranslation(area, data.text, lang); + if (copyBtn) copyBtn.style.display = ''; + }) + .catch(function(err) { + area.innerHTML = '
Translation failed: ' + err.message + '
'; + }); + } + + function renderTranslation(container, text, lang) { + container.innerHTML = ''; + var info = document.createElement('div'); + info.style.cssText = 'font-size:0.75rem;color:#6b7280;margin-bottom:0.5rem;'; + info.textContent = 'Translated to: ' + lang; + container.appendChild(info); + var pre = document.createElement('pre'); + pre.className = 'ocr-text'; + pre.id = 'translated-text-content'; + pre.textContent = text; + container.appendChild(pre); + } + + function copyTranslatedText() { + var content = document.getElementById('translated-text-content'); + if (!content) return; + var text = content.textContent; + var btn = document.getElementById('translate-copy-btn'); + if (!btn) return; + var orig = btn.innerHTML; + navigator.clipboard.writeText(text).then(function() { + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + }).catch(function() { + try { + var ta = document.createElement('textarea'); + ta.value = text; + ta.style.cssText = 'position:fixed;opacity:0;'; + document.body.appendChild(ta); + ta.select(); + document.execCommand('copy'); + document.body.removeChild(ta); + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + } catch(e) {} + }); + } + // ── Initialise on load ── document.addEventListener('DOMContentLoaded', function() { {% if (processed_file_exists or original_file_exists) and file %} diff --git a/frontend/templates/profile.html b/frontend/templates/profile.html index 50b2371e..62973360 100644 --- a/frontend/templates/profile.html +++ b/frontend/templates/profile.html @@ -231,6 +231,28 @@ {{ _("profile.theme_hint") }}

+ + +
+ + +

+ {{ _("profile.default_document_language_hint") }} +

+
@@ -340,6 +362,7 @@ function profileSettings() { contact_email: '', preferred_language: '', preferred_theme: 'system', + default_document_language: '', }, pwForm: { @@ -363,6 +386,7 @@ function profileSettings() { this.form.contact_email = data.contact_email || ''; this.form.preferred_language = data.preferred_language || ''; this.form.preferred_theme = data.preferred_theme || 'system'; + this.form.default_document_language = data.default_document_language || ''; this._initialLanguage = this.form.preferred_language; } catch (_e) { // Silently ignore — user might not be logged in (rare for this page) diff --git a/frontend/translations/en.json b/frontend/translations/en.json index 08eaf2de..2d868c47 100644 --- a/frontend/translations/en.json +++ b/frontend/translations/en.json @@ -1377,6 +1377,9 @@ "profile.contact_email_label": "Contact / Notification E-mail", "profile.contact_email_placeholder": "you@example.com", "profile.current_password": "Current Password", + "profile.default_document_language_auto": "Use system default", + "profile.default_document_language_hint": "Documents in other languages are automatically translated into this language. Leave blank to use the system default (English).", + "profile.default_document_language_label": "Default Document Language", "profile.dismiss": "Dismiss", "profile.display_name_hint": "Leave blank to use your account username or email.", "profile.display_name_label": "Display Name", @@ -1731,5 +1734,21 @@ "upload.uploading": "Uploading...", "upload.url_description": "Enter a direct link to a file (PDF, Office documents, or images)", "upload.url_label": "File URL", - "upload.url_placeholder": "https://example.com/document.pdf" + "upload.url_placeholder": "https://example.com/document.pdf", + + "translation.default_language_version": "Default Language Version", + "translation.detected_language": "Detected language", + "translation.show_text": "Show text", + "translation.hide_text": "Hide text", + "translation.copy": "Copy", + "translation.load_translation": "Load translation", + "translation.translate_to": "Translate to Another Language", + "translation.select_language": "Select language…", + "translation.translate_btn": "Translate", + "translation.translating": "Translating…", + "translation.no_translation": "No translation available yet — it may still be processing", + "translation.translated_to": "Translated to", + "translation.translation_failed": "Translation failed", + "translation.select_target": "Please select a target language.", + "translation.copied": "Copied!" } diff --git a/migrations/versions/036_add_document_translation_fields.py b/migrations/versions/036_add_document_translation_fields.py new file mode 100644 index 00000000..f73eeb43 --- /dev/null +++ b/migrations/versions/036_add_document_translation_fields.py @@ -0,0 +1,44 @@ +"""Add document translation fields to files and user_profiles tables. + +Adds detected_language, default_language_text, and default_language_code to +the files table so that a translated version of the document text can be +stored alongside the original. + +Adds default_document_language to user_profiles so each user can override +the system-wide default translation target language. + +Revision ID: 036_add_document_translation_fields +Revises: 035_add_routing_rules +Create Date: 2026-03-16 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "036_add_document_translation_fields" +down_revision: Union[str, None] = "035_add_routing_rules" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Add translation columns to files and user_profiles.""" + with op.batch_alter_table("files") as batch_op: + batch_op.add_column(sa.Column("detected_language", sa.String(10), nullable=True)) + batch_op.add_column(sa.Column("default_language_text", sa.Text(), nullable=True)) + batch_op.add_column(sa.Column("default_language_code", sa.String(10), nullable=True)) + + with op.batch_alter_table("user_profiles") as batch_op: + batch_op.add_column(sa.Column("default_document_language", sa.String(10), nullable=True)) + + +def downgrade() -> None: + """Remove translation columns.""" + with op.batch_alter_table("user_profiles") as batch_op: + batch_op.drop_column("default_document_language") + + with op.batch_alter_table("files") as batch_op: + batch_op.drop_column("default_language_code") + batch_op.drop_column("default_language_text") + batch_op.drop_column("detected_language") diff --git a/tests/test_translation.py b/tests/test_translation.py new file mode 100644 index 00000000..49ce52bd --- /dev/null +++ b/tests/test_translation.py @@ -0,0 +1,478 @@ +"""Tests for document translation feature. + +Covers: +- translate_to_default_language Celery task +- /api/files/{id}/translate on-the-fly translation endpoint +- /api/files/{id}/translation/default stored translation endpoint +- /files/{id}/text/default-language view endpoint +- _resolve_default_language helper +""" + +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.models import FileRecord, UserProfile + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def file_with_ocr(db_session): + """Create a FileRecord with OCR text and detected language.""" + record = FileRecord( + filehash="abc123translationtest", + local_filename="/tmp/test_translate.pdf", + file_size=1024, + mime_type="application/pdf", + original_filename="test_translate.pdf", + ocr_text="Dies ist ein Testdokument in deutscher Sprache.", + detected_language="de", + ) + db_session.add(record) + db_session.commit() + db_session.refresh(record) + return record + + +@pytest.fixture +def file_with_translation(db_session): + """Create a FileRecord with a persisted default-language translation.""" + record = FileRecord( + filehash="def456translationtest", + local_filename="/tmp/test_translated.pdf", + file_size=2048, + mime_type="application/pdf", + original_filename="test_translated.pdf", + ocr_text="Ceci est un document de test en français.", + detected_language="fr", + default_language_text="This is a test document in French.", + default_language_code="en", + ) + db_session.add(record) + db_session.commit() + db_session.refresh(record) + return record + + +@pytest.fixture +def file_without_ocr(db_session): + """Create a FileRecord without OCR text.""" + record = FileRecord( + filehash="ghi789translationtest", + local_filename="/tmp/test_no_ocr.pdf", + file_size=512, + mime_type="application/pdf", + original_filename="test_no_ocr.pdf", + ) + db_session.add(record) + db_session.commit() + db_session.refresh(record) + return record + + +@pytest.fixture +def user_profile_with_language(db_session): + """Create a UserProfile with a custom default_document_language.""" + profile = UserProfile( + user_id="test-user-lang", + default_document_language="de", + ) + db_session.add(profile) + db_session.commit() + db_session.refresh(profile) + return profile + + +# --------------------------------------------------------------------------- +# Model tests +# --------------------------------------------------------------------------- + + +class TestFileRecordTranslationFields: + """Verify that the new translation columns exist on FileRecord.""" + + @pytest.mark.unit + def test_detected_language_column(self, file_with_ocr): + assert file_with_ocr.detected_language == "de" + + @pytest.mark.unit + def test_default_language_text_column(self, file_with_translation): + assert file_with_translation.default_language_text == "This is a test document in French." + + @pytest.mark.unit + def test_default_language_code_column(self, file_with_translation): + assert file_with_translation.default_language_code == "en" + + @pytest.mark.unit + def test_translation_columns_nullable(self, file_with_ocr): + """Translation columns should be NULL when no translation exists.""" + assert file_with_ocr.default_language_text is None + assert file_with_ocr.default_language_code is None + + +class TestUserProfileDefaultLanguage: + """Verify UserProfile.default_document_language column.""" + + @pytest.mark.unit + def test_default_document_language_set(self, user_profile_with_language): + assert user_profile_with_language.default_document_language == "de" + + @pytest.mark.unit + def test_default_document_language_nullable(self, db_session): + profile = UserProfile(user_id="test-user-no-lang") + db_session.add(profile) + db_session.commit() + db_session.refresh(profile) + assert profile.default_document_language is None + + +# --------------------------------------------------------------------------- +# Celery task tests +# --------------------------------------------------------------------------- + + +class TestTranslateToDefaultLanguageTask: + """Tests for the translate_to_default_language Celery task.""" + + @pytest.mark.unit + @patch("app.tasks.translate_to_default_language.get_ai_provider") + def test_translate_stores_result(self, mock_provider_fn, db_session, file_with_ocr): + """Successful translation is persisted to the FileRecord.""" + mock_provider = MagicMock() + mock_provider.chat_completion.return_value = "This is a test document in German." + mock_provider_fn.return_value = mock_provider + + from app.tasks.translate_to_default_language import translate_to_default_language + + # Patch SessionLocal to use our test session + with patch("app.tasks.translate_to_default_language.SessionLocal") as mock_session_cls: + mock_ctx = MagicMock() + mock_ctx.__enter__ = MagicMock(return_value=db_session) + mock_ctx.__exit__ = MagicMock(return_value=False) + mock_session_cls.return_value = mock_ctx + + task = translate_to_default_language + # Call the underlying function (not .delay) for synchronous testing + result = task.apply( + args=[file_with_ocr.id, file_with_ocr.ocr_text, "de"], + kwargs={"owner_id": None}, + ).get() + + assert result["status"] == "success" + assert result["target_language"] == "en" + + # Verify it was stored + db_session.refresh(file_with_ocr) + assert file_with_ocr.default_language_text == "This is a test document in German." + assert file_with_ocr.default_language_code == "en" + assert file_with_ocr.detected_language == "de" + + @pytest.mark.unit + @patch("app.tasks.translate_to_default_language.get_ai_provider") + def test_skip_when_already_in_target_language(self, mock_provider_fn, db_session, file_with_ocr): + """No translation when document language matches default target.""" + file_with_ocr.detected_language = "en" + db_session.commit() + + from app.tasks.translate_to_default_language import translate_to_default_language + + with patch("app.tasks.translate_to_default_language.SessionLocal") as mock_session_cls: + mock_ctx = MagicMock() + mock_ctx.__enter__ = MagicMock(return_value=db_session) + mock_ctx.__exit__ = MagicMock(return_value=False) + mock_session_cls.return_value = mock_ctx + + result = translate_to_default_language.apply( + args=[file_with_ocr.id, file_with_ocr.ocr_text, "en"], + ).get() + + assert result["status"] == "skipped" + mock_provider_fn.assert_not_called() + + @pytest.mark.unit + def test_resolve_default_language_global(self): + """Falls back to the global setting when no user profile override.""" + from app.tasks.translate_to_default_language import _resolve_default_language + + with patch("app.tasks.translate_to_default_language.settings") as mock_settings: + mock_settings.default_document_language = "en" + assert _resolve_default_language(None) == "en" + + @pytest.mark.unit + def test_resolve_default_language_user_override(self, db_session, user_profile_with_language): + """Per-user override is used when available.""" + from app.tasks.translate_to_default_language import _resolve_default_language + + with patch("app.tasks.translate_to_default_language.SessionLocal") as mock_session_cls: + mock_ctx = MagicMock() + mock_ctx.__enter__ = MagicMock(return_value=db_session) + mock_ctx.__exit__ = MagicMock(return_value=False) + mock_session_cls.return_value = mock_ctx + + result = _resolve_default_language("test-user-lang") + + assert result == "de" + + +# --------------------------------------------------------------------------- +# API endpoint tests +# --------------------------------------------------------------------------- + + +class TestDefaultTranslationEndpoint: + """Tests for GET /api/files/{id}/translation/default.""" + + @pytest.mark.integration + def test_returns_default_translation(self, client: TestClient, file_with_translation): + response = client.get(f"/api/files/{file_with_translation.id}/translation/default") + assert response.status_code == 200 + data = response.json() + assert data["text"] == "This is a test document in French." + assert data["default_language_code"] == "en" + assert data["detected_language"] == "fr" + assert data["file_id"] == file_with_translation.id + + @pytest.mark.integration + def test_404_when_no_translation(self, client: TestClient, file_with_ocr): + response = client.get(f"/api/files/{file_with_ocr.id}/translation/default") + assert response.status_code == 404 + + @pytest.mark.integration + def test_404_for_nonexistent_file(self, client: TestClient): + response = client.get("/api/files/999999/translation/default") + assert response.status_code == 404 + + +class TestOnTheFlyTranslateEndpoint: + """Tests for GET /api/files/{id}/translate?lang=xx.""" + + @pytest.mark.integration + @patch("app.api.translation.get_ai_provider") + def test_translate_on_the_fly(self, mock_provider_fn, client: TestClient, file_with_ocr): + mock_provider = MagicMock() + mock_provider.chat_completion.return_value = "This is a test document in German language." + mock_provider_fn.return_value = mock_provider + + response = client.get(f"/api/files/{file_with_ocr.id}/translate?lang=en") + assert response.status_code == 200 + data = response.json() + assert data["text"] == "This is a test document in German language." + assert data["target_language"] == "en" + assert data["cached"] is False + + @pytest.mark.integration + def test_returns_cached_default_language(self, client: TestClient, file_with_translation): + """If the requested language matches the stored default, return cached text.""" + response = client.get(f"/api/files/{file_with_translation.id}/translate?lang=en") + assert response.status_code == 200 + data = response.json() + assert data["text"] == "This is a test document in French." + assert data["cached"] is True + + @pytest.mark.integration + def test_returns_original_when_same_language(self, client: TestClient, file_with_ocr): + """Return the original text when target matches detected language.""" + response = client.get(f"/api/files/{file_with_ocr.id}/translate?lang=de") + assert response.status_code == 200 + data = response.json() + assert data["text"] == file_with_ocr.ocr_text + assert data["cached"] is True + + @pytest.mark.integration + def test_400_when_no_ocr_text(self, client: TestClient, file_without_ocr): + response = client.get(f"/api/files/{file_without_ocr.id}/translate?lang=en") + assert response.status_code == 400 + + @pytest.mark.integration + def test_missing_lang_param(self, client: TestClient, file_with_ocr): + response = client.get(f"/api/files/{file_with_ocr.id}/translate") + assert response.status_code == 422 # validation error + + @pytest.mark.integration + def test_404_for_nonexistent_file(self, client: TestClient): + response = client.get("/api/files/999999/translate?lang=en") + assert response.status_code == 404 + + @pytest.mark.integration + @patch("app.api.translation.get_ai_provider") + def test_502_on_provider_error(self, mock_provider_fn, client: TestClient, file_with_ocr): + mock_provider = MagicMock() + mock_provider.chat_completion.side_effect = RuntimeError("AI error") + mock_provider_fn.return_value = mock_provider + + response = client.get(f"/api/files/{file_with_ocr.id}/translate?lang=fr") + assert response.status_code == 502 + + +# --------------------------------------------------------------------------- +# View endpoint tests +# --------------------------------------------------------------------------- + + +class TestDefaultLanguageTextView: + """Tests for GET /files/{id}/text/default-language.""" + + @pytest.mark.integration + def test_returns_default_language_text(self, client: TestClient, file_with_translation): + response = client.get(f"/files/{file_with_translation.id}/text/default-language") + assert response.status_code == 200 + data = response.json() + assert data["text"] == "This is a test document in French." + assert data["language_code"] == "en" + assert data["detected_language"] == "fr" + + @pytest.mark.integration + def test_404_when_no_default_text(self, client: TestClient, file_with_ocr): + response = client.get(f"/files/{file_with_ocr.id}/text/default-language") + assert response.status_code == 404 + + @pytest.mark.integration + def test_404_for_nonexistent_file(self, client: TestClient): + response = client.get("/files/999999/text/default-language") + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# Config tests +# --------------------------------------------------------------------------- + + +class TestDefaultDocumentLanguageConfig: + """Verify the DEFAULT_DOCUMENT_LANGUAGE setting.""" + + @pytest.mark.unit + def test_default_value_is_english(self): + from app.config import settings + + assert settings.default_document_language == "en" + + +# --------------------------------------------------------------------------- +# Profile API integration tests +# --------------------------------------------------------------------------- + + +class TestProfileDefaultDocumentLanguage: + """Tests for default_document_language in the profile API.""" + + @pytest.fixture + def prof_engine(self): + """In-memory SQLite engine for profile tests.""" + from sqlalchemy import create_engine + from sqlalchemy.pool import StaticPool + + from app.database import Base + + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + yield engine + + @pytest.fixture + def prof_session(self, prof_engine): + """DB session for profile tests.""" + from sqlalchemy.orm import sessionmaker + + Session = sessionmaker(bind=self.prof_engine if hasattr(self, "prof_engine") else prof_engine) + session = Session() + yield session + session.close() + + @pytest.mark.asyncio + @pytest.mark.unit + async def test_get_profile_includes_default_document_language(self, prof_engine): + """GET handler returns default_document_language in response.""" + from unittest.mock import MagicMock + + from sqlalchemy.orm import sessionmaker + + from app.api.profile import get_profile + + Session = sessionmaker(bind=prof_engine) + session = Session() + + req = MagicMock() + req.session = {"user": {"preferred_username": "languser", "email": "lang@test.com"}} + + result = await get_profile(req, session) + assert hasattr(result, "default_document_language") + session.close() + + @pytest.mark.asyncio + @pytest.mark.unit + async def test_update_default_document_language(self, prof_engine): + """PATCH handler updates default_document_language.""" + from unittest.mock import MagicMock + + from sqlalchemy.orm import sessionmaker + + from app.api.profile import ProfileUpdateRequest, update_profile + + Session = sessionmaker(bind=prof_engine) + session = Session() + + req = MagicMock() + req.session = {"user": {"preferred_username": "languser2", "email": "lang2@test.com"}} + resp = MagicMock() + + body = ProfileUpdateRequest(default_document_language="de") + result = await update_profile(body, req, resp, session) + assert result.default_document_language == "de" + session.close() + + @pytest.mark.asyncio + @pytest.mark.unit + async def test_clear_default_document_language(self, prof_engine): + """Setting default_document_language to empty string clears it.""" + from unittest.mock import MagicMock + + from sqlalchemy.orm import sessionmaker + + from app.api.profile import ProfileUpdateRequest, update_profile + + Session = sessionmaker(bind=prof_engine) + session = Session() + + req = MagicMock() + req.session = {"user": {"preferred_username": "languser3", "email": "lang3@test.com"}} + resp = MagicMock() + + # Set + body = ProfileUpdateRequest(default_document_language="fr") + await update_profile(body, req, resp, session) + # Clear + body = ProfileUpdateRequest(default_document_language="") + result = await update_profile(body, req, resp, session) + assert result.default_document_language is None + session.close() + + @pytest.mark.asyncio + @pytest.mark.unit + async def test_reject_invalid_default_document_language(self, prof_engine): + """Invalid language codes are rejected with 422.""" + from unittest.mock import MagicMock + + from fastapi import HTTPException + from sqlalchemy.orm import sessionmaker + + from app.api.profile import ProfileUpdateRequest, update_profile + + Session = sessionmaker(bind=prof_engine) + session = Session() + + req = MagicMock() + req.session = {"user": {"preferred_username": "languser4", "email": "lang4@test.com"}} + resp = MagicMock() + + body = ProfileUpdateRequest(default_document_language="xx_invalid") + with pytest.raises(HTTPException) as exc_info: + await update_profile(body, req, resp, session) + assert exc_info.value.status_code == 422 + session.close()