gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(translation): add model, config, task, API, and UI for default document language translation

Browse Source 
- Add detected_language, default_language_text, default_language_code columns to FileRecord
- Add default_document_language column to UserProfile
- Add DEFAULT_DOCUMENT_LANGUAGE config setting (defaults to "en")
- Create translate_to_default_language Celery task
- Integrate translation trigger into embed_metadata_into_pdf pipeline
- Add /api/files/{id}/translate and /api/files/{id}/translation/default API endpoints
- Add /files/{id}/text/default-language view endpoint
- Update file_view.html with translation sections (default language, on-the-fly)
- Add translation keys to en.json
- Create Alembic migration 036
- Update .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-16 11:54:13 +00:00
parent ff1fde5e48
commit 3a221a62cd
11 changed files with 697 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+2
View File
@@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router
from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router
# Import all the individual routers
@@ -96,3 +97,4 @@ router.include_router(audit_logs_router)
router.include_router(i18n_router)
router.include_router(mobile_router)
router.include_router(compliance_router)
router.include_router(translation_router)
app/api/translation.py
+158
View File
@@ -0,0 +1,158 @@
"""
API endpoints for document translation.
Provides on-the-fly translation via the AI provider and access to the
persisted default-language translation.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord
from app.utils.ai_provider import get_ai_provider
from app.utils.user_scope import apply_owner_filter, get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
# Maximum characters sent to the AI provider for a single translation request.
_MAX_TRANSLATION_INPUT = 50_000
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
"""Fetch a FileRecord visible to the current user or raise 404."""
query = db.query(FileRecord).filter(FileRecord.id == file_id)
owner_id = get_current_owner_id(request, db)
if owner_id:
query = apply_owner_filter(query, owner_id, FileRecord)
record = query.first()
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
return record
@router.get("/files/{file_id}/translation/default")
@require_login
def get_default_translation(
request: Request,
file_id: int,
db: DbSession,
) -> JSONResponse:
"""Return the persisted default-language translation for a document.
Returns 404 if no default-language translation has been generated yet
(e.g. because the document is already in the default language).
"""
record = _get_file_or_404(db, file_id, request)
if not record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available for this file",
)
return JSONResponse(
content={
"file_id": record.id,
"detected_language": record.detected_language,
"default_language_code": record.default_language_code,
"text": record.default_language_text,
}
)
@router.get("/files/{file_id}/translate")
@require_login
def translate_on_the_fly(
request: Request,
file_id: int,
db: DbSession,
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
) -> JSONResponse:
"""Translate a document's extracted text into an arbitrary language on the fly.
The translation is generated via the configured AI provider and is **not**
persisted. For the default-language translation, use the
``/files/{file_id}/translation/default`` endpoint instead.
"""
record = _get_file_or_404(db, file_id, request)
source_text = record.ocr_text
if not source_text:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No extracted text available for this file — translation requires OCR text",
)
# If the requested language matches what is already stored, return it directly.
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
return JSONResponse(
content={
"file_id": record.id,
"source_language": record.detected_language,
"target_language": lang,
"text": record.default_language_text,
"cached": True,
}
)
# If the detected language already matches, return the original text.
detected = record.detected_language
if detected and detected == lang:
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected,
"target_language": lang,
"text": source_text,
"cached": True,
}
)
# Truncate to keep AI costs bounded.
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"into {lang}. Preserve the original formatting, paragraph structure, "
f"and meaning. Do not add any commentary — output ONLY the translated text."
),
},
{"role": "user", "content": text_to_translate},
],
model=model,
temperature=0.3,
)
except Exception as exc:
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Translation failed — the AI provider returned an error",
)
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected or "unknown",
"target_language": lang,
"text": translated,
"cached": False,
}
)
app/config.py
+19
View File
@@ -166,6 +166,25 @@ class Settings(BaseSettings):
google_docai_location: str = "us" # Processor location, e.g. "us" or "eu"
external_hostname: str = "localhost" # Default to localhost
# ---------------------------------------------------------------------------
# Document Translation Settings
# ---------------------------------------------------------------------------
# Default target language for automatic document translation (ISO 639-1 code).
# After OCR / metadata extraction, if the detected document language differs
# from this value the system translates the extracted text into this language
# and stores it alongside the original. Other language translations are
# generated on the fly via the AI provider and are NOT persisted.
# Per-user overrides are stored in UserProfile.default_document_language.
default_document_language: str = Field(
default="en",
description=(
"ISO 639-1 language code for the default translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language "
"differs are automatically translated into this language after "
"processing. Default: 'en' (English)."
),
)
# Authentication settings
auth_enabled: bool = True # Default to enabled
admin_username: Optional[str] = None
app/models.py
+19
View File
@@ -84,6 +84,19 @@ class FileRecord(Base):
# Processing pipeline assigned to this file (NULL = use system default)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True)
# Detected document language (ISO 639-1 code, e.g. "de", "en", "fr")
# Extracted from AI metadata during processing; cached here for fast access.
detected_language = Column(String(10), nullable=True)
# Default-language translation of the extracted text.
# Stored when the detected language differs from the user's/system default
# document language. Only the original text and this translation are persisted;
# other languages are translated on the fly via the AI provider.
default_language_text = Column(Text, nullable=True)
# ISO 639-1 code of the default-language translation stored above (e.g. "en").
default_language_code = Column(String(10), nullable=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
@@ -282,6 +295,12 @@ class UserProfile(Base):
# NULL means "auto-detect from browser Accept-Language header"
preferred_language = Column(String(10), nullable=True)
# Default document language for translated versions (ISO 639-1 code).
# When a document's detected language differs from this value, the system
# automatically generates and stores a translation into this language.
# NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting".
default_document_language = Column(String(10), nullable=True)
# UI colour scheme preference: "light" | "dark" | "system" (NULL = "system")
preferred_theme = Column(String(10), nullable=True)
app/tasks/embed_metadata_into_pdf.py
+24
View File
@@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as search_exc:
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
# Cache the detected language on the FileRecord and trigger
# default-language translation when the document is in a
# different language.
detected_lang = metadata.get("language") if metadata else None
if detected_lang and extracted_text:
try:
file_record.detected_language = detected_lang
db.commit()
from app.tasks.translate_to_default_language import translate_to_default_language
translate_to_default_language.delay(
file_id,
extracted_text,
detected_lang,
owner_id=file_record.owner_id,
)
logger.info(
f"[{task_id}] Queued default-language translation for file {file_id} "
f"(detected: {detected_lang})"
)
except Exception as trans_exc:
logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}")
# Persist the metadata into a JSON file with the same base name.
# Include file path references for traceability
logger.info(f"[{task_id}] Persisting metadata to JSON")
app/tasks/translate_to_default_language.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Celery task to translate extracted document text into the default target language.
This task is triggered after metadata extraction when the detected document
language differs from the user's (or system) default document language. The
translated text is persisted in ``FileRecord.default_language_text`` so that
users can always read a reference copy in their preferred language.
Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate``
endpoint and are NOT persisted.
"""
import logging
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord, UserProfile
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
logger = logging.getLogger(__name__)
def _resolve_default_language(owner_id: str | None) -> str:
"""Return the default document language for the given owner.
Resolution order:
1. ``UserProfile.default_document_language`` (per-user override)
2. ``settings.default_document_language`` (global setting)
"""
if owner_id:
with SessionLocal() as db:
profile = db.query(UserProfile).filter_by(user_id=owner_id).first()
if profile and profile.default_document_language:
return profile.default_document_language
return settings.default_document_language
@celery.task(base=BaseTaskWithRetry, bind=True)
def translate_to_default_language(
self,
file_id: int,
extracted_text: str,
detected_language: str,
owner_id: str | None = None,
) -> dict:
"""Translate *extracted_text* into the default document language and persist the result.
Args:
file_id: Primary key of the :class:`FileRecord`.
extracted_text: The OCR / refined text in the document's original language.
detected_language: ISO 639-1 code of the document's detected language.
owner_id: Owner identifier used to resolve per-user language preference.
Returns:
A dict with ``status``, ``target_language``, and the translated text length.
"""
task_id = self.request.id
target_language = _resolve_default_language(owner_id)
# Nothing to do when the document is already in the target language.
if detected_language == target_language:
logger.info(
f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation"
)
log_task_progress(
task_id,
"translate_to_default_language",
"skipped",
f"Document already in {target_language}",
file_id=file_id,
)
return {"status": "skipped", "reason": "already_in_target_language"}
logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'")
log_task_progress(
task_id,
"translate_to_default_language",
"in_progress",
f"Translating from {detected_language} to {target_language}",
file_id=file_id,
)
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated_text = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"from {detected_language} to {target_language}. "
f"Preserve the original formatting, paragraph structure, and meaning. "
f"Do not add any commentary or explanation — output ONLY the translated text."
),
},
{"role": "user", "content": extracted_text},
],
model=model,
temperature=0.3,
)
# Persist the translation.
with SessionLocal() as db:
record = db.query(FileRecord).filter_by(id=file_id).first()
if record:
record.default_language_text = translated_text
record.default_language_code = target_language
record.detected_language = detected_language
db.commit()
logger.info(
f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}"
)
log_task_progress(
task_id,
"translate_to_default_language",
"success",
f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})",
file_id=file_id,
)
return {
"status": "success",
"target_language": target_language,
"translated_length": len(translated_text),
}
except Exception as exc:
logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}")
log_task_progress(
task_id,
"translate_to_default_language",
"failure",
f"Exception: {exc}",
file_id=file_id,
)
raise
app/views/files.py
+28
View File
@@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
)
@router.get("/files/{file_id}/text/default-language")
@require_login
def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)):
"""Return the persisted default-language translation for the file view."""
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND)
if not file_record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available",
)
return JSONResponse(
content={
"text": file_record.default_language_text,
"language_code": file_record.default_language_code,
"detected_language": file_record.detected_language,
}
)
@router.get("/duplicates")
@require_login
def duplicates_page(