gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ocr): add multi-language OCR support with per-pipeline language override

Browse Source 
- Add OCR_LANGUAGES constant (28 languages, EN/DE/FR/ES/IT/PT/RU/ZH/JA/KO/AR/etc.)
- Add TESSERACT_TO_EASYOCR mapping for automatic code translation
- Add optional language constructor arg to TesseractOCRProvider/EasyOCRProvider
- Update get_ocr_providers() to accept and pass per-call language override
- Add language parameter to process_with_ocr Celery task
- Add _get_pipeline_ocr_language() helper to resolve OCR language from pipeline step config
- Update process_document to look up and pass pipeline OCR language to process_with_ocr
- Add ocr_language select config field (28 options) to pipeline OCR step schema
- Add language dropdown to pipeline UI (pipelines.html)
- Update docs/UserGuide.md and docs/API.md with language override documentation
- Add 27 new tests covering language constants, provider overrides, and pipeline lookup

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-08 21:47:12 +00:00
parent b03ea41638
commit a2a4c6fc9a
9 changed files with 808 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/process_document.py
+77 -4
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import json
import logging
import mimetypes
import os
@@ -12,7 +13,7 @@ from pypdf.errors import PdfReadError
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.models import FileRecord, Pipeline, PipelineStep
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.tasks.process_with_ocr import process_with_ocr
from app.tasks.retry_config import BaseTaskWithRetry
@@ -23,6 +24,69 @@ from app.utils.text_quality import check_text_quality, detect_pdf_text_source
logger = logging.getLogger(__name__)
def _get_pipeline_ocr_language(db, file_record: FileRecord, owner_id: str | None) -> str | None:
"""Look up the OCR language override from the file's pipeline OCR step config.
Resolution order:
1. Explicit pipeline assigned to the file (``file_record.pipeline_id``).
2. User's own default pipeline (``owner_id``, ``is_default=True``).
3. System default pipeline (``owner_id=NULL``, ``is_default=True``).
Returns the ``ocr_language`` value from the pipeline's OCR step config, or
``None`` when no override is configured.
"""
pipeline = None
if file_record.pipeline_id:
pipeline = db.query(Pipeline).filter(Pipeline.id == file_record.pipeline_id).first()
if pipeline is None and owner_id:
pipeline = (
db.query(Pipeline)
.filter(
Pipeline.owner_id == owner_id,
Pipeline.is_default.is_(True),
Pipeline.is_active.is_(True),
)
.first()
)
if pipeline is None:
pipeline = (
db.query(Pipeline)
.filter(
Pipeline.owner_id.is_(None),
Pipeline.is_default.is_(True),
Pipeline.is_active.is_(True),
)
.first()
)
if pipeline is None:
return None
ocr_step = (
db.query(PipelineStep)
.filter(
PipelineStep.pipeline_id == pipeline.id,
PipelineStep.step_type == "ocr",
PipelineStep.enabled.is_(True),
)
.first()
)
if ocr_step is None or not ocr_step.config:
return None
try:
step_config = json.loads(ocr_step.config)
lang = step_config.get("ocr_language") or None
# "auto" is treated as no override
return lang if lang and lang != "auto" else None
except Exception:
return None
@celery.task(base=BaseTaskWithRetry, bind=True)
def process_document(
self,
@@ -109,6 +173,7 @@ def process_document(
)
# Acquire DB session in the task
ocr_language: str | None = None # Pipeline OCR language override resolved inside DB session
with SessionLocal() as db:
# When file_id is provided, we are reprocessing an existing file.
# Skip the duplicate check and reuse the existing record.
@@ -305,6 +370,14 @@ def process_document(
new_record.local_filename = new_local_path
db.commit()
# Look up pipeline OCR language override before the session closes.
# This reads the OCR step config from the file's assigned pipeline (or
# the user/system default pipeline) so the language is available when
# dispatching process_with_ocr below.
ocr_language = _get_pipeline_ocr_language(db, new_record, owner_id)
if ocr_language:
logger.info(f"[{task_id}] Pipeline OCR language override: {ocr_language!r}")
# Store file_id before session closes to avoid DetachedInstanceError
file_id = new_record.id
@@ -334,7 +407,7 @@ def process_document(
"Queued for forced OCR processing",
file_id=file_id,
)
process_with_ocr.delay(new_filename, file_id)
process_with_ocr.delay(new_filename, file_id, language=ocr_language)
return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id}
# If the file is not a PDF, skip embedded text check and convert to PDF first
@@ -491,7 +564,7 @@ def process_document(
"Queued for OCR (text quality too low)",
file_id=file_id,
)
process_with_ocr.delay(new_filename, file_id, extracted_text)
process_with_ocr.delay(new_filename, file_id, extracted_text, language=ocr_language)
return {
"file": new_local_path,
"status": "Queued for OCR (poor embedded text quality)",
@@ -564,5 +637,5 @@ def process_document(
"Queued for OCR processing",
file_id=file_id,
)
process_with_ocr.delay(new_filename, file_id)
process_with_ocr.delay(new_filename, file_id, language=ocr_language)
return {"file": new_local_path, "status": "Queued for OCR", "file_id": file_id}
app/tasks/process_with_ocr.py
+19 -4
View File
@@ -33,7 +33,13 @@ logger = logging.getLogger(__name__)
@celery.task(base=OcrTaskWithRetry, bind=True)
def process_with_ocr(self, filename: str, file_id: Optional[int] = None, original_text: Optional[str] = None):
def process_with_ocr(
self,
filename: str,
file_id: Optional[int] = None,
original_text: Optional[str] = None,
language: Optional[str] = None,
):
"""Run the configured OCR providers on *filename* and continue the pipeline.
When multiple OCR providers are configured the results are merged using the
@@ -47,6 +53,10 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
filename: Base name of the file inside ``<workdir>/tmp/``.
file_id: Optional database record ID passed through to downstream tasks.
original_text: Optional original embedded text for head-to-head comparison.
language: Optional Tesseract-style language code(s) (e.g. ``"eng+deu"``)
to override the global OCR language settings for this specific run.
Pass ``None`` or ``"auto"`` to use the global settings. This
enables per-pipeline language configuration.
"""
task_id = self.request.id
log_task_progress(
@@ -62,7 +72,7 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
if not os.path.exists(tmp_file_path):
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
providers = get_ocr_providers()
providers = get_ocr_providers(language=language)
provider_names = [p.name for p in providers]
logger.info(f"[{task_id}] Running {len(providers)} OCR provider(s): {provider_names}")
@@ -122,7 +132,12 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
# PDF with ocrmypdf to embed an invisible text layer so the output is
# selectable/searchable in PDF viewers.
if searchable_pdf_path is None:
lang = getattr(settings, "tesseract_language", None) or "eng"
# Use the per-call language override; fall back to global setting
embed_lang = (
language
if language and language != "auto"
else (getattr(settings, "tesseract_language", None) or "eng")
)
log_task_progress(
task_id,
"embed_text_layer",
@@ -130,7 +145,7 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
"Embedding searchable text layer into PDF",
file_id=file_id,
)
embedded = embed_text_layer(tmp_file_path, tmp_file_path, language=lang)
embedded = embed_text_layer(tmp_file_path, tmp_file_path, language=embed_lang)
if embedded:
searchable_pdf_path = tmp_file_path
log_task_progress(