gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #729 from christianlouis/copilot/add-default-language-version-support

Browse Source 
feat(translation): automatic document translation to configurable default language
This commit is contained in:
Christian Krakau-Louis
2026-03-16 13:55:25 +01:00
committed by GitHub
parent 0e72a965d5 0c10fcbafc
commit c81914f78d
17 changed files with 1276 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+2
View File
@@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router
from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router
# Import all the individual routers
@@ -96,3 +97,4 @@ router.include_router(audit_logs_router)
router.include_router(i18n_router)
router.include_router(mobile_router)
router.include_router(compliance_router)
router.include_router(translation_router)
app/api/profile.py
+18
View File
@@ -99,6 +99,8 @@ class ProfileResponse(BaseModel):
contact_email: str | None
preferred_language: str | None
preferred_theme: str | None
default_document_language: str | None
"""ISO 639-1 code for the user's preferred document translation target language."""
avatar_url: str
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
is_local_user: bool
@@ -112,6 +114,10 @@ class ProfileUpdateRequest(BaseModel):
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
default_document_language: str | None = Field(
default=None,
description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'",
)
class ChangePasswordRequest(BaseModel):
@@ -149,6 +155,7 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
@@ -201,6 +208,16 @@ async def update_profile(
)
profile.preferred_theme = theme or None # type: ignore[assignment]
# Validate default document language
if body.default_document_language is not None:
doc_lang = body.default_document_language.lower().strip()
if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported language code: {doc_lang}",
)
profile.default_document_language = doc_lang or None # type: ignore[assignment]
if body.display_name is not None:
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
@@ -225,6 +242,7 @@ async def update_profile(
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
app/api/translation.py
+158
View File
@@ -0,0 +1,158 @@
"""
API endpoints for document translation.
Provides on-the-fly translation via the AI provider and access to the
persisted default-language translation.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord
from app.utils.ai_provider import get_ai_provider
from app.utils.user_scope import apply_owner_filter, get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
# Maximum characters sent to the AI provider for a single translation request.
_MAX_TRANSLATION_INPUT = 50_000
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
"""Fetch a FileRecord visible to the current user or raise 404."""
query = db.query(FileRecord).filter(FileRecord.id == file_id)
owner_id = get_current_owner_id(request)
if owner_id:
query = apply_owner_filter(query, owner_id, FileRecord)
record = query.first()
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
return record
@router.get("/files/{file_id}/translation/default")
@require_login
def get_default_translation(
request: Request,
file_id: int,
db: DbSession,
) -> JSONResponse:
"""Return the persisted default-language translation for a document.
Returns 404 if no default-language translation has been generated yet
(e.g. because the document is already in the default language).
"""
record = _get_file_or_404(db, file_id, request)
if not record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available for this file",
)
return JSONResponse(
content={
"file_id": record.id,
"detected_language": record.detected_language,
"default_language_code": record.default_language_code,
"text": record.default_language_text,
}
)
@router.get("/files/{file_id}/translate")
@require_login
def translate_on_the_fly(
request: Request,
file_id: int,
db: DbSession,
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
) -> JSONResponse:
"""Translate a document's extracted text into an arbitrary language on the fly.
The translation is generated via the configured AI provider and is **not**
persisted. For the default-language translation, use the
``/files/{file_id}/translation/default`` endpoint instead.
"""
record = _get_file_or_404(db, file_id, request)
source_text = record.ocr_text
if not source_text:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No extracted text available for this file — translation requires OCR text",
)
# If the requested language matches what is already stored, return it directly.
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
return JSONResponse(
content={
"file_id": record.id,
"source_language": record.detected_language,
"target_language": lang,
"text": record.default_language_text,
"cached": True,
}
)
# If the detected language already matches, return the original text.
detected = record.detected_language
if detected and detected == lang:
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected,
"target_language": lang,
"text": source_text,
"cached": True,
}
)
# Truncate to keep AI costs bounded.
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"into {lang}. Preserve the original formatting, paragraph structure, "
f"and meaning. Do not add any commentary — output ONLY the translated text."
),
},
{"role": "user", "content": text_to_translate},
],
model=model,
temperature=0.3,
)
except Exception as exc:
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Translation failed — the AI provider returned an error",
)
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected or "unknown",
"target_language": lang,
"text": translated,
"cached": False,
}
)
app/celery_worker.py
+1
View File
@@ -39,6 +39,7 @@ from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
from app.tasks.subscription_tasks import apply_pending_subscription_changes_all # noqa: F401
from app.tasks.translate_to_default_language import translate_to_default_language # noqa: F401
# Import new send tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
app/config.py
+19
View File
@@ -166,6 +166,25 @@ class Settings(BaseSettings):
google_docai_location: str = "us" # Processor location, e.g. "us" or "eu"
external_hostname: str = "localhost" # Default to localhost
# ---------------------------------------------------------------------------
# Document Translation Settings
# ---------------------------------------------------------------------------
# Default target language for automatic document translation (ISO 639-1 code).
# After OCR / metadata extraction, if the detected document language differs
# from this value the system translates the extracted text into this language
# and stores it alongside the original. Other language translations are
# generated on the fly via the AI provider and are NOT persisted.
# Per-user overrides are stored in UserProfile.default_document_language.
default_document_language: str = Field(
default="en",
description=(
"ISO 639-1 language code for the default translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language "
"differs are automatically translated into this language after "
"processing. Default: 'en' (English)."
),
)
# Authentication settings
auth_enabled: bool = True # Default to enabled
admin_username: Optional[str] = None
app/models.py
+19
View File
@@ -84,6 +84,19 @@ class FileRecord(Base):
# Processing pipeline assigned to this file (NULL = use system default)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True)
# Detected document language (ISO 639-1 code, e.g. "de", "en", "fr")
# Extracted from AI metadata during processing; cached here for fast access.
detected_language = Column(String(10), nullable=True)
# Default-language translation of the extracted text.
# Stored when the detected language differs from the user's/system default
# document language. Only the original text and this translation are persisted;
# other languages are translated on the fly via the AI provider.
default_language_text = Column(Text, nullable=True)
# ISO 639-1 code of the default-language translation stored above (e.g. "en").
default_language_code = Column(String(10), nullable=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
@@ -282,6 +295,12 @@ class UserProfile(Base):
# NULL means "auto-detect from browser Accept-Language header"
preferred_language = Column(String(10), nullable=True)
# Default document language for translated versions (ISO 639-1 code).
# When a document's detected language differs from this value, the system
# automatically generates and stores a translation into this language.
# NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting".
default_document_language = Column(String(10), nullable=True)
# UI colour scheme preference: "light" | "dark" | "system" (NULL = "system")
preferred_theme = Column(String(10), nullable=True)
app/tasks/embed_metadata_into_pdf.py
+24
View File
@@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as search_exc:
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
# Cache the detected language on the FileRecord and trigger
# default-language translation when the document is in a
# different language.
detected_lang = metadata.get("language") if metadata else None
if detected_lang and extracted_text:
try:
file_record.detected_language = detected_lang
db.commit()
from app.tasks.translate_to_default_language import translate_to_default_language
translate_to_default_language.delay(
file_id,
extracted_text,
detected_lang,
owner_id=file_record.owner_id,
)
logger.info(
f"[{task_id}] Queued default-language translation for file {file_id} "
f"(detected: {detected_lang})"
)
except Exception as trans_exc:
logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}")
# Persist the metadata into a JSON file with the same base name.
# Include file path references for traceability
logger.info(f"[{task_id}] Persisting metadata to JSON")
app/tasks/translate_to_default_language.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Celery task to translate extracted document text into the default target language.
This task is triggered after metadata extraction when the detected document
language differs from the user's (or system) default document language. The
translated text is persisted in ``FileRecord.default_language_text`` so that
users can always read a reference copy in their preferred language.
Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate``
endpoint and are NOT persisted.
"""
import logging
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord, UserProfile
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
logger = logging.getLogger(__name__)
def _resolve_default_language(owner_id: str | None) -> str:
"""Return the default document language for the given owner.
Resolution order:
1. ``UserProfile.default_document_language`` (per-user override)
2. ``settings.default_document_language`` (global setting)
"""
if owner_id:
with SessionLocal() as db:
profile = db.query(UserProfile).filter_by(user_id=owner_id).first()
if profile and profile.default_document_language:
return profile.default_document_language
return settings.default_document_language
@celery.task(base=BaseTaskWithRetry, bind=True)
def translate_to_default_language(
self,
file_id: int,
extracted_text: str,
detected_language: str,
owner_id: str | None = None,
) -> dict:
"""Translate *extracted_text* into the default document language and persist the result.
Args:
file_id: Primary key of the :class:`FileRecord`.
extracted_text: The OCR / refined text in the document's original language.
detected_language: ISO 639-1 code of the document's detected language.
owner_id: Owner identifier used to resolve per-user language preference.
Returns:
A dict with ``status``, ``target_language``, and the translated text length.
"""
task_id = self.request.id
target_language = _resolve_default_language(owner_id)
# Nothing to do when the document is already in the target language.
if detected_language == target_language:
logger.info(
f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation"
)
log_task_progress(
task_id,
"translate_to_default_language",
"skipped",
f"Document already in {target_language}",
file_id=file_id,
)
return {"status": "skipped", "reason": "already_in_target_language"}
logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'")
log_task_progress(
task_id,
"translate_to_default_language",
"in_progress",
f"Translating from {detected_language} to {target_language}",
file_id=file_id,
)
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated_text = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"from {detected_language} to {target_language}. "
f"Preserve the original formatting, paragraph structure, and meaning. "
f"Do not add any commentary or explanation — output ONLY the translated text."
),
},
{"role": "user", "content": extracted_text},
],
model=model,
temperature=0.3,
)
# Persist the translation.
with SessionLocal() as db:
record = db.query(FileRecord).filter_by(id=file_id).first()
if record:
record.default_language_text = translated_text
record.default_language_code = target_language
record.detected_language = detected_language
db.commit()
logger.info(
f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}"
)
log_task_progress(
task_id,
"translate_to_default_language",
"success",
f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})",
file_id=file_id,
)
return {
"status": "success",
"target_language": target_language,
"translated_length": len(translated_text),
}
except Exception as exc:
logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}")
log_task_progress(
task_id,
"translate_to_default_language",
"failure",
f"Exception: {exc}",
file_id=file_id,
)
raise
app/utils/settings_service.py
+13
View File
@@ -522,6 +522,19 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Document Translation
"default_document_language": {
"category": "AI Services",
"description": (
"ISO 639-1 language code for the default document translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language differs "
"are automatically translated into this language after processing."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# OCR Engine Configuration
"ocr_providers": {
"category": "OCR Engines",
app/views/files.py
+28
View File
@@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
)
@router.get("/files/{file_id}/text/default-language")
@require_login
def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)):
"""Return the persisted default-language translation for the file view."""
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND)
if not file_record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available",
)
return JSONResponse(
content={
"text": file_record.default_language_text,
"language_code": file_record.default_language_code,
"detected_language": file_record.detected_language,
}
)
@router.get("/duplicates")
@require_login
def duplicates_page(