feat(ocr): add multi-language OCR support with per-pipeline language override

- Add OCR_LANGUAGES constant (28 languages, EN/DE/FR/ES/IT/PT/RU/ZH/JA/KO/AR/etc.)
- Add TESSERACT_TO_EASYOCR mapping for automatic code translation
- Add optional language constructor arg to TesseractOCRProvider/EasyOCRProvider
- Update get_ocr_providers() to accept and pass per-call language override
- Add language parameter to process_with_ocr Celery task
- Add _get_pipeline_ocr_language() helper to resolve OCR language from pipeline step config
- Update process_document to look up and pass pipeline OCR language to process_with_ocr
- Add ocr_language select config field (28 options) to pipeline OCR step schema
- Add language dropdown to pipeline UI (pipelines.html)
- Update docs/UserGuide.md and docs/API.md with language override documentation
- Add 27 new tests covering language constants, provider overrides, and pipeline lookup

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-08 21:47:12 +00:00
parent b03ea41638
commit a2a4c6fc9a
9 changed files with 808 additions and 18 deletions
+177
View File
@@ -842,3 +842,180 @@ startxref
mock_init_steps.assert_called_once()
called_file_id = mock_init_steps.call_args[0][1]
assert called_file_id == result["file_id"]
# ---------------------------------------------------------------------------
# _get_pipeline_ocr_language helper
# ---------------------------------------------------------------------------
@pytest.mark.unit
@pytest.mark.requires_db
def test_get_pipeline_ocr_language_returns_none_when_no_pipeline(db_session):
"""Returns None when no pipeline exists in the database."""
from app.tasks.process_document import _get_pipeline_ocr_language
# FileRecord with no pipeline_id
file_record = FileRecord(
filehash="abc123",
original_filename="test.pdf",
local_filename="/tmp/test.pdf",
file_size=1024,
mime_type="application/pdf",
is_duplicate=False,
)
db_session.add(file_record)
db_session.commit()
result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
assert result is None
@pytest.mark.unit
@pytest.mark.requires_db
def test_get_pipeline_ocr_language_returns_language_from_system_default(db_session):
"""Returns ocr_language from the system default pipeline's OCR step config."""
import json
from app.models import Pipeline, PipelineStep
from app.tasks.process_document import _get_pipeline_ocr_language
# Create system default pipeline with OCR step configured to "deu"
pipeline = Pipeline(
owner_id=None,
name="System Default",
is_default=True,
is_active=True,
)
db_session.add(pipeline)
db_session.commit()
ocr_step = PipelineStep(
pipeline_id=pipeline.id,
position=0,
step_type="ocr",
config=json.dumps({"force_cloud_ocr": False, "ocr_language": "deu"}),
enabled=True,
)
db_session.add(ocr_step)
db_session.commit()
file_record = FileRecord(
filehash="def456",
original_filename="doc.pdf",
local_filename="/tmp/doc.pdf",
file_size=512,
mime_type="application/pdf",
is_duplicate=False,
)
db_session.add(file_record)
db_session.commit()
result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
assert result == "deu"
@pytest.mark.unit
@pytest.mark.requires_db
def test_get_pipeline_ocr_language_auto_returns_none(db_session):
"""Returns None when ocr_language is 'auto' (should use global settings)."""
import json
from app.models import Pipeline, PipelineStep
from app.tasks.process_document import _get_pipeline_ocr_language
pipeline = Pipeline(
owner_id=None,
name="Auto Lang Pipeline",
is_default=True,
is_active=True,
)
db_session.add(pipeline)
db_session.commit()
ocr_step = PipelineStep(
pipeline_id=pipeline.id,
position=0,
step_type="ocr",
config=json.dumps({"ocr_language": "auto"}),
enabled=True,
)
db_session.add(ocr_step)
db_session.commit()
file_record = FileRecord(
filehash="ghi789",
original_filename="auto.pdf",
local_filename="/tmp/auto.pdf",
file_size=128,
mime_type="application/pdf",
is_duplicate=False,
)
db_session.add(file_record)
db_session.commit()
result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
assert result is None
@pytest.mark.unit
@pytest.mark.requires_db
def test_get_pipeline_ocr_language_explicit_pipeline_takes_priority(db_session):
"""Explicit pipeline_id on file takes priority over system default pipeline."""
import json
from app.models import Pipeline, PipelineStep
from app.tasks.process_document import _get_pipeline_ocr_language
# System default pipeline with "eng"
sys_pipeline = Pipeline(
owner_id=None,
name="System Default",
is_default=True,
is_active=True,
)
db_session.add(sys_pipeline)
db_session.commit()
PipelineStep(
pipeline_id=sys_pipeline.id,
position=0,
step_type="ocr",
config=json.dumps({"ocr_language": "eng"}),
enabled=True,
)
# Explicit pipeline with "fra"
explicit_pipeline = Pipeline(
owner_id="user1",
name="French Pipeline",
is_default=False,
is_active=True,
)
db_session.add(explicit_pipeline)
db_session.commit()
explicit_step = PipelineStep(
pipeline_id=explicit_pipeline.id,
position=0,
step_type="ocr",
config=json.dumps({"ocr_language": "fra"}),
enabled=True,
)
db_session.add(explicit_step)
db_session.commit()
file_record = FileRecord(
filehash="jkl012",
original_filename="french.pdf",
local_filename="/tmp/french.pdf",
file_size=256,
mime_type="application/pdf",
is_duplicate=False,
pipeline_id=explicit_pipeline.id,
)
db_session.add(file_record)
db_session.commit()
result = _get_pipeline_ocr_language(db_session, file_record, owner_id="user1")
assert result == "fra"