diff --git a/.env.demo b/.env.demo index 9412e754..5bf0a14d 100644 --- a/.env.demo +++ b/.env.demo @@ -258,6 +258,13 @@ OPENAI_MODEL=gpt-4o-mini # AZURE_OPENAI_API_VERSION=2024-02-01 # AI_MODEL=gpt-4o # deployment name in Azure +# **Document Translation** +# After processing, documents whose detected language differs from the default +# target language are automatically translated. Only the original and this +# default-language version are persisted; other translations are on-the-fly. +# Users can override this in their profile settings. +# DEFAULT_DOCUMENT_LANGUAGE=en + # Azure Document Intelligence (OCR – separate from AI provider above) # **Email Settings (shared SMTP – password reset, verification, and system notifications)** EMAIL_HOST=smtp.example.com diff --git a/app/api/__init__.py b/app/api/__init__.py index 25344332..246e47f3 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router from app.api.shared_links import router as shared_links_router from app.api.similarity import router as similarity_router from app.api.subscriptions import router as subscriptions_router +from app.api.translation import router as translation_router from app.api.url_upload import router as url_upload_router # Import all the individual routers @@ -96,3 +97,4 @@ router.include_router(audit_logs_router) router.include_router(i18n_router) router.include_router(mobile_router) router.include_router(compliance_router) +router.include_router(translation_router) diff --git a/app/api/translation.py b/app/api/translation.py new file mode 100644 index 00000000..26e65fe6 --- /dev/null +++ b/app/api/translation.py @@ -0,0 +1,158 @@ +""" +API endpoints for document translation. + +Provides on-the-fly translation via the AI provider and access to the +persisted default-language translation. +""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.responses import JSONResponse +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.config import settings +from app.database import get_db +from app.models import FileRecord +from app.utils.ai_provider import get_ai_provider +from app.utils.user_scope import apply_owner_filter, get_current_owner_id + +logger = logging.getLogger(__name__) + +router = APIRouter() + +DbSession = Annotated[Session, Depends(get_db)] + +# Maximum characters sent to the AI provider for a single translation request. +_MAX_TRANSLATION_INPUT = 50_000 + + +def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord: + """Fetch a FileRecord visible to the current user or raise 404.""" + query = db.query(FileRecord).filter(FileRecord.id == file_id) + owner_id = get_current_owner_id(request, db) + if owner_id: + query = apply_owner_filter(query, owner_id, FileRecord) + record = query.first() + if not record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + return record + + +@router.get("/files/{file_id}/translation/default") +@require_login +def get_default_translation( + request: Request, + file_id: int, + db: DbSession, +) -> JSONResponse: + """Return the persisted default-language translation for a document. + + Returns 404 if no default-language translation has been generated yet + (e.g. because the document is already in the default language). + """ + record = _get_file_or_404(db, file_id, request) + + if not record.default_language_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No default-language translation available for this file", + ) + + return JSONResponse( + content={ + "file_id": record.id, + "detected_language": record.detected_language, + "default_language_code": record.default_language_code, + "text": record.default_language_text, + } + ) + + +@router.get("/files/{file_id}/translate") +@require_login +def translate_on_the_fly( + request: Request, + file_id: int, + db: DbSession, + lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"), +) -> JSONResponse: + """Translate a document's extracted text into an arbitrary language on the fly. + + The translation is generated via the configured AI provider and is **not** + persisted. For the default-language translation, use the + ``/files/{file_id}/translation/default`` endpoint instead. + """ + record = _get_file_or_404(db, file_id, request) + + source_text = record.ocr_text + if not source_text: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No extracted text available for this file — translation requires OCR text", + ) + + # If the requested language matches what is already stored, return it directly. + if record.default_language_code and lang == record.default_language_code and record.default_language_text: + return JSONResponse( + content={ + "file_id": record.id, + "source_language": record.detected_language, + "target_language": lang, + "text": record.default_language_text, + "cached": True, + } + ) + + # If the detected language already matches, return the original text. + detected = record.detected_language + if detected and detected == lang: + return JSONResponse( + content={ + "file_id": record.id, + "source_language": detected, + "target_language": lang, + "text": source_text, + "cached": True, + } + ) + + # Truncate to keep AI costs bounded. + text_to_translate = source_text[:_MAX_TRANSLATION_INPUT] + + try: + provider = get_ai_provider() + model = settings.ai_model or settings.openai_model + translated = provider.chat_completion( + messages=[ + { + "role": "system", + "content": ( + f"You are a professional translator. Translate the following text " + f"into {lang}. Preserve the original formatting, paragraph structure, " + f"and meaning. Do not add any commentary — output ONLY the translated text." + ), + }, + {"role": "user", "content": text_to_translate}, + ], + model=model, + temperature=0.3, + ) + except Exception as exc: + logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Translation failed — the AI provider returned an error", + ) + + return JSONResponse( + content={ + "file_id": record.id, + "source_language": detected or "unknown", + "target_language": lang, + "text": translated, + "cached": False, + } + ) diff --git a/app/config.py b/app/config.py index 30b434d1..ac9ab7bb 100644 --- a/app/config.py +++ b/app/config.py @@ -166,6 +166,25 @@ class Settings(BaseSettings): google_docai_location: str = "us" # Processor location, e.g. "us" or "eu" external_hostname: str = "localhost" # Default to localhost + # --------------------------------------------------------------------------- + # Document Translation Settings + # --------------------------------------------------------------------------- + # Default target language for automatic document translation (ISO 639-1 code). + # After OCR / metadata extraction, if the detected document language differs + # from this value the system translates the extracted text into this language + # and stores it alongside the original. Other language translations are + # generated on the fly via the AI provider and are NOT persisted. + # Per-user overrides are stored in UserProfile.default_document_language. + default_document_language: str = Field( + default="en", + description=( + "ISO 639-1 language code for the default translation target " + "(e.g. 'en', 'de', 'fr'). Documents whose detected language " + "differs are automatically translated into this language after " + "processing. Default: 'en' (English)." + ), + ) + # Authentication settings auth_enabled: bool = True # Default to enabled admin_username: Optional[str] = None diff --git a/app/models.py b/app/models.py index b752fdb6..54cef37d 100644 --- a/app/models.py +++ b/app/models.py @@ -84,6 +84,19 @@ class FileRecord(Base): # Processing pipeline assigned to this file (NULL = use system default) pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True) + # Detected document language (ISO 639-1 code, e.g. "de", "en", "fr") + # Extracted from AI metadata during processing; cached here for fast access. + detected_language = Column(String(10), nullable=True) + + # Default-language translation of the extracted text. + # Stored when the detected language differs from the user's/system default + # document language. Only the original text and this translation are persisted; + # other languages are translated on the fly via the AI provider. + default_language_text = Column(Text, nullable=True) + + # ISO 639-1 code of the default-language translation stored above (e.g. "en"). + default_language_code = Column(String(10), nullable=True) + # Timestamp when we inserted this record created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) @@ -282,6 +295,12 @@ class UserProfile(Base): # NULL means "auto-detect from browser Accept-Language header" preferred_language = Column(String(10), nullable=True) + # Default document language for translated versions (ISO 639-1 code). + # When a document's detected language differs from this value, the system + # automatically generates and stores a translation into this language. + # NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting". + default_document_language = Column(String(10), nullable=True) + # UI colour scheme preference: "light" | "dark" | "system" (NULL = "system") preferred_theme = Column(String(10), nullable=True) diff --git a/app/tasks/embed_metadata_into_pdf.py b/app/tasks/embed_metadata_into_pdf.py index 3bc78664..e0f40716 100644 --- a/app/tasks/embed_metadata_into_pdf.py +++ b/app/tasks/embed_metadata_into_pdf.py @@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met except Exception as search_exc: logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}") + # Cache the detected language on the FileRecord and trigger + # default-language translation when the document is in a + # different language. + detected_lang = metadata.get("language") if metadata else None + if detected_lang and extracted_text: + try: + file_record.detected_language = detected_lang + db.commit() + + from app.tasks.translate_to_default_language import translate_to_default_language + + translate_to_default_language.delay( + file_id, + extracted_text, + detected_lang, + owner_id=file_record.owner_id, + ) + logger.info( + f"[{task_id}] Queued default-language translation for file {file_id} " + f"(detected: {detected_lang})" + ) + except Exception as trans_exc: + logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}") + # Persist the metadata into a JSON file with the same base name. # Include file path references for traceability logger.info(f"[{task_id}] Persisting metadata to JSON") diff --git a/app/tasks/translate_to_default_language.py b/app/tasks/translate_to_default_language.py new file mode 100644 index 00000000..966fc4f7 --- /dev/null +++ b/app/tasks/translate_to_default_language.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Celery task to translate extracted document text into the default target language. + +This task is triggered after metadata extraction when the detected document +language differs from the user's (or system) default document language. The +translated text is persisted in ``FileRecord.default_language_text`` so that +users can always read a reference copy in their preferred language. + +Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate`` +endpoint and are NOT persisted. +""" + +import logging + +from app.celery_app import celery +from app.config import settings +from app.database import SessionLocal +from app.models import FileRecord, UserProfile +from app.tasks.retry_config import BaseTaskWithRetry +from app.utils import log_task_progress +from app.utils.ai_provider import get_ai_provider + +logger = logging.getLogger(__name__) + + +def _resolve_default_language(owner_id: str | None) -> str: + """Return the default document language for the given owner. + + Resolution order: + 1. ``UserProfile.default_document_language`` (per-user override) + 2. ``settings.default_document_language`` (global setting) + """ + if owner_id: + with SessionLocal() as db: + profile = db.query(UserProfile).filter_by(user_id=owner_id).first() + if profile and profile.default_document_language: + return profile.default_document_language + return settings.default_document_language + + +@celery.task(base=BaseTaskWithRetry, bind=True) +def translate_to_default_language( + self, + file_id: int, + extracted_text: str, + detected_language: str, + owner_id: str | None = None, +) -> dict: + """Translate *extracted_text* into the default document language and persist the result. + + Args: + file_id: Primary key of the :class:`FileRecord`. + extracted_text: The OCR / refined text in the document's original language. + detected_language: ISO 639-1 code of the document's detected language. + owner_id: Owner identifier used to resolve per-user language preference. + + Returns: + A dict with ``status``, ``target_language``, and the translated text length. + """ + task_id = self.request.id + target_language = _resolve_default_language(owner_id) + + # Nothing to do when the document is already in the target language. + if detected_language == target_language: + logger.info( + f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation" + ) + log_task_progress( + task_id, + "translate_to_default_language", + "skipped", + f"Document already in {target_language}", + file_id=file_id, + ) + return {"status": "skipped", "reason": "already_in_target_language"} + + logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'") + log_task_progress( + task_id, + "translate_to_default_language", + "in_progress", + f"Translating from {detected_language} to {target_language}", + file_id=file_id, + ) + + try: + provider = get_ai_provider() + model = settings.ai_model or settings.openai_model + translated_text = provider.chat_completion( + messages=[ + { + "role": "system", + "content": ( + f"You are a professional translator. Translate the following text " + f"from {detected_language} to {target_language}. " + f"Preserve the original formatting, paragraph structure, and meaning. " + f"Do not add any commentary or explanation — output ONLY the translated text." + ), + }, + {"role": "user", "content": extracted_text}, + ], + model=model, + temperature=0.3, + ) + + # Persist the translation. + with SessionLocal() as db: + record = db.query(FileRecord).filter_by(id=file_id).first() + if record: + record.default_language_text = translated_text + record.default_language_code = target_language + record.detected_language = detected_language + db.commit() + logger.info( + f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}" + ) + + log_task_progress( + task_id, + "translate_to_default_language", + "success", + f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})", + file_id=file_id, + ) + + return { + "status": "success", + "target_language": target_language, + "translated_length": len(translated_text), + } + + except Exception as exc: + logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}") + log_task_progress( + task_id, + "translate_to_default_language", + "failure", + f"Exception: {exc}", + file_id=file_id, + ) + raise diff --git a/app/views/files.py b/app/views/files.py index 58efa132..aa1ee577 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get ) +@router.get("/files/{file_id}/text/default-language") +@require_login +def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)): + """Return the persisted default-language translation for the file view.""" + from fastapi import status + from fastapi.responses import JSONResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) + + if not file_record.default_language_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No default-language translation available", + ) + + return JSONResponse( + content={ + "text": file_record.default_language_text, + "language_code": file_record.default_language_code, + "detected_language": file_record.detected_language, + } + ) + + @router.get("/duplicates") @require_login def duplicates_page( diff --git a/frontend/templates/file_view.html b/frontend/templates/file_view.html index 3489ab2a..01083b78 100644 --- a/frontend/templates/file_view.html +++ b/frontend/templates/file_view.html @@ -439,10 +439,105 @@ + {% if file.detected_language %} +
+ + Detected language: {{ file.detected_language }} +
+ {% endif %} + + + {% if file.default_language_text %} +
+
+
+ + Default Language Version + {% if file.default_language_code %} + ({{ file.default_language_code }}) + {% endif %} +
+
+ + +
+
+ +
+ {% elif file.detected_language %} + +
+
+
+ + Default Language Version +
+ +
+
+
+ {% endif %} + + + {% if file.ocr_text %} +
+
+
+ + Translate to Another Language +
+
+
+ + + + +
+
+
+ {% endif %} + {% elif processed_file_exists or original_file_exists %}
@@ -688,6 +783,149 @@ }); } + // ── Default-language text toggle ── + function toggleDefaultLangText() { + var block = document.getElementById('default-lang-text-block'); + var icon = document.getElementById('default-lang-toggle-icon'); + var label = document.getElementById('default-lang-toggle-label'); + var btn = icon ? icon.closest('button') : null; + if (!block) return; + if (block.style.display === 'none') { + block.style.display = 'block'; + if (icon) icon.className = 'fas fa-chevron-up'; + if (label) label.textContent = 'Hide text'; + if (btn) btn.setAttribute('aria-expanded', 'true'); + } else { + block.style.display = 'none'; + if (icon) icon.className = 'fas fa-chevron-down'; + if (label) label.textContent = 'Show text'; + if (btn) btn.setAttribute('aria-expanded', 'false'); + } + } + + // ── Copy default-language text ── + function copyDefaultLangText() { + var content = document.getElementById('default-lang-text-content'); + if (!content) return; + var text = content.textContent; + var btn = document.getElementById('default-lang-copy-btn'); + if (!btn) return; + var orig = btn.innerHTML; + navigator.clipboard.writeText(text).then(function() { + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + }).catch(function() { + try { + var ta = document.createElement('textarea'); + ta.value = text; + ta.style.cssText = 'position:fixed;opacity:0;'; + document.body.appendChild(ta); + ta.select(); + document.execCommand('copy'); + document.body.removeChild(ta); + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + } catch(e) {} + }); + } + + // ── Load default-language text on demand ── + function loadDefaultLangText(fileId) { + var area = document.getElementById('default-lang-load-area'); + if (!area) return; + area.innerHTML = '

Loading translation…

'; + fetch('/files/' + fileId + '/text/default-language') + .then(function(r) { + if (r.status === 404) throw new Error('No translation available yet — it may still be processing'); + if (!r.ok) throw new Error('HTTP ' + r.status); + return r.json(); + }) + .then(function(data) { + area.innerHTML = ''; + var info = document.createElement('div'); + info.style.cssText = 'font-size:0.75rem;color:#6b7280;margin-bottom:0.5rem;'; + info.textContent = 'Translated to: ' + data.language_code + (data.detected_language ? ' (from ' + data.detected_language + ')' : ''); + area.appendChild(info); + var pre = document.createElement('pre'); + pre.className = 'ocr-text'; + pre.textContent = data.text; + area.appendChild(pre); + }) + .catch(function(err) { + area.innerHTML = '
' + err.message + '
'; + }); + } + + // ── On-the-fly translation ── + var _translateCache = {}; + function translateOnTheFly(fileId) { + var select = document.getElementById('translate-lang-select'); + var area = document.getElementById('translate-result-area'); + var copyBtn = document.getElementById('translate-copy-btn'); + if (!select || !area) return; + var lang = select.value; + if (!lang) { area.innerHTML = '
Please select a target language.
'; return; } + + // Check cache + if (_translateCache[lang]) { + renderTranslation(area, _translateCache[lang], lang); + if (copyBtn) copyBtn.style.display = ''; + return; + } + + area.innerHTML = '

Translating…

'; + if (copyBtn) copyBtn.style.display = 'none'; + + fetch('/api/files/' + fileId + '/translate?lang=' + encodeURIComponent(lang)) + .then(function(r) { if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); }) + .then(function(data) { + _translateCache[lang] = data.text; + renderTranslation(area, data.text, lang); + if (copyBtn) copyBtn.style.display = ''; + }) + .catch(function(err) { + area.innerHTML = '
Translation failed: ' + err.message + '
'; + }); + } + + function renderTranslation(container, text, lang) { + container.innerHTML = ''; + var info = document.createElement('div'); + info.style.cssText = 'font-size:0.75rem;color:#6b7280;margin-bottom:0.5rem;'; + info.textContent = 'Translated to: ' + lang; + container.appendChild(info); + var pre = document.createElement('pre'); + pre.className = 'ocr-text'; + pre.id = 'translated-text-content'; + pre.textContent = text; + container.appendChild(pre); + } + + function copyTranslatedText() { + var content = document.getElementById('translated-text-content'); + if (!content) return; + var text = content.textContent; + var btn = document.getElementById('translate-copy-btn'); + if (!btn) return; + var orig = btn.innerHTML; + navigator.clipboard.writeText(text).then(function() { + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + }).catch(function() { + try { + var ta = document.createElement('textarea'); + ta.value = text; + ta.style.cssText = 'position:fixed;opacity:0;'; + document.body.appendChild(ta); + ta.select(); + document.execCommand('copy'); + document.body.removeChild(ta); + btn.innerHTML = ' Copied!'; + setTimeout(function() { btn.innerHTML = orig; }, 2000); + } catch(e) {} + }); + } + // ── Initialise on load ── document.addEventListener('DOMContentLoaded', function() { {% if (processed_file_exists or original_file_exists) and file %} diff --git a/frontend/translations/en.json b/frontend/translations/en.json index 08eaf2de..88fea2a5 100644 --- a/frontend/translations/en.json +++ b/frontend/translations/en.json @@ -1731,5 +1731,21 @@ "upload.uploading": "Uploading...", "upload.url_description": "Enter a direct link to a file (PDF, Office documents, or images)", "upload.url_label": "File URL", - "upload.url_placeholder": "https://example.com/document.pdf" + "upload.url_placeholder": "https://example.com/document.pdf", + + "translation.default_language_version": "Default Language Version", + "translation.detected_language": "Detected language", + "translation.show_text": "Show text", + "translation.hide_text": "Hide text", + "translation.copy": "Copy", + "translation.load_translation": "Load translation", + "translation.translate_to": "Translate to Another Language", + "translation.select_language": "Select language…", + "translation.translate_btn": "Translate", + "translation.translating": "Translating…", + "translation.no_translation": "No translation available yet — it may still be processing", + "translation.translated_to": "Translated to", + "translation.translation_failed": "Translation failed", + "translation.select_target": "Please select a target language.", + "translation.copied": "Copied!" } diff --git a/migrations/versions/036_add_document_translation_fields.py b/migrations/versions/036_add_document_translation_fields.py new file mode 100644 index 00000000..f73eeb43 --- /dev/null +++ b/migrations/versions/036_add_document_translation_fields.py @@ -0,0 +1,44 @@ +"""Add document translation fields to files and user_profiles tables. + +Adds detected_language, default_language_text, and default_language_code to +the files table so that a translated version of the document text can be +stored alongside the original. + +Adds default_document_language to user_profiles so each user can override +the system-wide default translation target language. + +Revision ID: 036_add_document_translation_fields +Revises: 035_add_routing_rules +Create Date: 2026-03-16 +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "036_add_document_translation_fields" +down_revision: Union[str, None] = "035_add_routing_rules" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Add translation columns to files and user_profiles.""" + with op.batch_alter_table("files") as batch_op: + batch_op.add_column(sa.Column("detected_language", sa.String(10), nullable=True)) + batch_op.add_column(sa.Column("default_language_text", sa.Text(), nullable=True)) + batch_op.add_column(sa.Column("default_language_code", sa.String(10), nullable=True)) + + with op.batch_alter_table("user_profiles") as batch_op: + batch_op.add_column(sa.Column("default_document_language", sa.String(10), nullable=True)) + + +def downgrade() -> None: + """Remove translation columns.""" + with op.batch_alter_table("user_profiles") as batch_op: + batch_op.drop_column("default_document_language") + + with op.batch_alter_table("files") as batch_op: + batch_op.drop_column("default_language_code") + batch_op.drop_column("default_language_text") + batch_op.drop_column("detected_language")