Merge pull request #568 from christianlouis/copilot/add-multi-language-ocr-support

feat(ocr): per-pipeline language override for multi-language OCR
2026-03-08 23:11:41 +01:00
parent 2e241dd5f3 0b291995b9
commit 6376b6d73b
9 changed files with 816 additions and 19 deletions
@@ -1057,3 +1057,241 @@ class TestMergeOCRResults:
            ms.openai_model = "gpt-4"
            text, _, _ = merge_ocr_results([r1, r2], "doc.pdf")
        assert text == "this is the longer text from tesseract engine"
+
+
+# ---------------------------------------------------------------------------
+# Multi-language OCR support
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+class TestOCRLanguageConstants:
+    """Tests for the OCR_LANGUAGES constant and TESSERACT_TO_EASYOCR mapping."""
+
+    def test_ocr_languages_has_20_plus_entries(self):
+        """OCR_LANGUAGES contains at least 20 language options (excluding 'auto')."""
+        from app.utils.ocr_provider import OCR_LANGUAGES
+
+        language_entries = {k: v for k, v in OCR_LANGUAGES.items() if v != "auto"}
+        assert len(language_entries) >= 20, f"Expected ≥20 languages, got {len(language_entries)}"
+
+    def test_ocr_languages_includes_auto(self):
+        """OCR_LANGUAGES includes 'auto' as the first option."""
+        from app.utils.ocr_provider import OCR_LANGUAGES
+
+        assert "auto" in OCR_LANGUAGES.values()
+
+    def test_ocr_languages_common_languages(self):
+        """OCR_LANGUAGES includes the most common European and Asian languages."""
+        from app.utils.ocr_provider import OCR_LANGUAGES
+
+        expected_codes = {"eng", "deu", "fra", "spa", "ita", "por", "rus", "chi_sim", "jpn", "kor"}
+        all_codes = set(OCR_LANGUAGES.values())
+        missing = expected_codes - all_codes
+        assert not missing, f"Missing expected language codes: {missing}"
+
+    def test_tesseract_to_easyocr_mapping(self):
+        """TESSERACT_TO_EASYOCR maps common Tesseract codes to EasyOCR codes."""
+        from app.utils.ocr_provider import TESSERACT_TO_EASYOCR
+
+        assert TESSERACT_TO_EASYOCR["eng"] == "en"
+        assert TESSERACT_TO_EASYOCR["deu"] == "de"
+        assert TESSERACT_TO_EASYOCR["fra"] == "fr"
+        assert TESSERACT_TO_EASYOCR["chi_sim"] == "ch_sim"
+
+    def test_tesseract_codes_to_easyocr_single(self):
+        """_tesseract_codes_to_easyocr converts a single Tesseract code."""
+        from app.utils.ocr_provider import _tesseract_codes_to_easyocr
+
+        result = _tesseract_codes_to_easyocr("eng")
+        assert result == ["en"]
+
+    def test_tesseract_codes_to_easyocr_multi(self):
+        """_tesseract_codes_to_easyocr splits '+'-separated Tesseract codes."""
+        from app.utils.ocr_provider import _tesseract_codes_to_easyocr
+
+        result = _tesseract_codes_to_easyocr("eng+deu")
+        assert result == ["en", "de"]
+
+    def test_tesseract_codes_to_easyocr_passthrough_unknown(self):
+        """_tesseract_codes_to_easyocr passes through codes not in the mapping."""
+        from app.utils.ocr_provider import _tesseract_codes_to_easyocr
+
+        # EasyOCR-native codes are passed through unchanged
+        result = _tesseract_codes_to_easyocr("en")
+        assert result == ["en"]
+
+
+@pytest.mark.unit
+class TestTesseractLanguageOverride:
+    """Tests for per-call language override in TesseractOCRProvider."""
+
+    def test_language_override_used_in_process(self, tmp_path):
+        """Language override is used instead of global setting."""
+        pdf = _make_pdf(tmp_path)
+        provider = TesseractOCRProvider(language="deu")
+
+        mock_pytesseract = Mock()
+        mock_pytesseract.image_to_string.return_value = "Deutsches Text"
+        mock_pytesseract.pytesseract = Mock()
+        mock_pdf2image = Mock()
+        mock_pdf2image.convert_from_path.return_value = [Mock()]
+
+        with (
+            patch.dict(
+                sys.modules,
+                {"pytesseract": mock_pytesseract, "pdf2image": mock_pdf2image},
+            ),
+            patch("app.utils.ocr_provider.settings") as ms,
+            patch("app.utils.ocr_language_manager.ensure_tesseract_languages", return_value=[]),
+        ):
+            ms.tesseract_cmd = None
+            ms.tesseract_language = "eng"  # global setting; should be overridden
+            result = provider.process(pdf)
+
+        # Ensure image_to_string was called with the override language ("deu"), not global "eng"
+        mock_pytesseract.image_to_string.assert_called_once()
+        call_kwargs = mock_pytesseract.image_to_string.call_args
+        assert call_kwargs[1].get("lang") == "deu" or (call_kwargs[0] and call_kwargs[0][1] == "deu")
+        assert result.provider == "tesseract"
+
+    def test_auto_language_falls_back_to_global(self, tmp_path):
+        """'auto' language override falls back to global tesseract_language setting."""
+        pdf = _make_pdf(tmp_path)
+        provider = TesseractOCRProvider(language="auto")
+
+        mock_pytesseract = Mock()
+        mock_pytesseract.image_to_string.return_value = ""
+        mock_pytesseract.pytesseract = Mock()
+        mock_pdf2image = Mock()
+        mock_pdf2image.convert_from_path.return_value = [Mock()]
+
+        with (
+            patch.dict(
+                sys.modules,
+                {"pytesseract": mock_pytesseract, "pdf2image": mock_pdf2image},
+            ),
+            patch("app.utils.ocr_provider.settings") as ms,
+            patch("app.utils.ocr_language_manager.ensure_tesseract_languages", return_value=[]),
+        ):
+            ms.tesseract_cmd = None
+            ms.tesseract_language = "fra"
+            provider.process(pdf)
+
+        # Should use global setting "fra" since "auto" means no override
+        mock_pytesseract.image_to_string.assert_called_once()
+        call_kwargs = mock_pytesseract.image_to_string.call_args
+        lang_used = call_kwargs[1].get("lang") if call_kwargs[1] else call_kwargs[0][1]
+        assert lang_used == "fra"
+
+    def test_none_language_falls_back_to_global(self, tmp_path):
+        """None language override falls back to global setting."""
+        pdf = _make_pdf(tmp_path)
+        provider = TesseractOCRProvider(language=None)
+        assert provider._language_override is None
+
+
+@pytest.mark.unit
+class TestEasyOCRLanguageOverride:
+    """Tests for per-call language override in EasyOCRProvider."""
+
+    def test_language_override_converted_and_used(self, tmp_path):
+        """Tesseract-style language override is converted to EasyOCR codes."""
+        pdf = _make_pdf(tmp_path)
+        provider = EasyOCRProvider(language="deu")
+
+        mock_reader = Mock()
+        mock_reader.readtext.return_value = ["Deutsches Text"]
+        mock_easyocr = Mock()
+        mock_easyocr.Reader.return_value = mock_reader
+        mock_pdf2image = Mock()
+        mock_pdf2image.convert_from_path.return_value = [Mock()]
+
+        with (
+            patch.dict(
+                sys.modules,
+                {"easyocr": mock_easyocr, "pdf2image": mock_pdf2image},
+            ),
+            patch("app.utils.ocr_provider.settings") as ms,
+        ):
+            ms.easyocr_languages = "en"  # global; should be overridden
+            ms.easyocr_gpu = False
+            provider.process(pdf)
+
+        # Should call Reader with ["de"] (converted from "deu"), not global ["en"]
+        mock_easyocr.Reader.assert_called_once()
+        langs_arg = mock_easyocr.Reader.call_args[0][0]
+        assert langs_arg == ["de"]
+
+    def test_auto_language_uses_global_setting(self, tmp_path):
+        """'auto' language override falls back to global easyocr_languages setting."""
+        pdf = _make_pdf(tmp_path)
+        provider = EasyOCRProvider(language="auto")
+
+        mock_reader = Mock()
+        mock_reader.readtext.return_value = []
+        mock_easyocr = Mock()
+        mock_easyocr.Reader.return_value = mock_reader
+        mock_pdf2image = Mock()
+        mock_pdf2image.convert_from_path.return_value = [Mock()]
+
+        with (
+            patch.dict(
+                sys.modules,
+                {"easyocr": mock_easyocr, "pdf2image": mock_pdf2image},
+            ),
+            patch("app.utils.ocr_provider.settings") as ms,
+        ):
+            ms.easyocr_languages = "fr,es"
+            ms.easyocr_gpu = False
+            provider.process(pdf)
+
+        langs_arg = mock_easyocr.Reader.call_args[0][0]
+        assert langs_arg == ["fr", "es"]
+
+
+@pytest.mark.unit
+class TestGetOCRProvidersWithLanguage:
+    """Tests for get_ocr_providers(language=...) factory."""
+
+    def test_language_passed_to_tesseract_provider(self):
+        """Language override is passed to TesseractOCRProvider."""
+        with patch("app.utils.ocr_provider.settings") as ms:
+            ms.ocr_providers = "tesseract"
+            providers = get_ocr_providers(language="deu")
+        assert len(providers) == 1
+        assert isinstance(providers[0], TesseractOCRProvider)
+        assert providers[0]._language_override == "deu"
+
+    def test_language_passed_to_easyocr_provider(self):
+        """Language override is passed to EasyOCRProvider."""
+        with patch("app.utils.ocr_provider.settings") as ms:
+            ms.ocr_providers = "easyocr"
+            providers = get_ocr_providers(language="fra")
+        assert len(providers) == 1
+        assert isinstance(providers[0], EasyOCRProvider)
+        assert providers[0]._language_override == "fra"
+
+    def test_language_not_passed_to_azure(self):
+        """Language override is NOT passed to AzureOCRProvider (it auto-detects)."""
+        with patch("app.utils.ocr_provider.settings") as ms:
+            ms.ocr_providers = "azure"
+            providers = get_ocr_providers(language="deu")
+        assert len(providers) == 1
+        assert isinstance(providers[0], AzureOCRProvider)
+        # AzureOCRProvider has no _language_override attribute
+        assert not hasattr(providers[0], "_language_override")
+
+    def test_auto_language_not_passed_as_override(self):
+        """'auto' language is treated as no override for Tesseract."""
+        with patch("app.utils.ocr_provider.settings") as ms:
+            ms.ocr_providers = "tesseract"
+            providers = get_ocr_providers(language="auto")
+        assert providers[0]._language_override is None
+
+    def test_none_language_no_override(self):
+        """None language results in no override."""
+        with patch("app.utils.ocr_provider.settings") as ms:
+            ms.ocr_providers = "tesseract"
+            providers = get_ocr_providers(language=None)
+        assert providers[0]._language_override is None
@@ -842,3 +842,182 @@ startxref
        mock_init_steps.assert_called_once()
        called_file_id = mock_init_steps.call_args[0][1]
        assert called_file_id == result["file_id"]
+
+
+# ---------------------------------------------------------------------------
+# _get_pipeline_ocr_language helper
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+@pytest.mark.requires_db
+def test_get_pipeline_ocr_language_returns_none_when_no_pipeline(db_session):
+    """Returns None when no pipeline exists in the database."""
+    from app.tasks.process_document import _get_pipeline_ocr_language
+
+    # FileRecord with no pipeline_id
+    file_record = FileRecord(
+        filehash="abc123",
+        original_filename="test.pdf",
+        local_filename="/tmp/test.pdf",
+        file_size=1024,
+        mime_type="application/pdf",
+        is_duplicate=False,
+    )
+    db_session.add(file_record)
+    db_session.commit()
+
+    result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
+    assert result is None
+
+
+@pytest.mark.unit
+@pytest.mark.requires_db
+def test_get_pipeline_ocr_language_returns_language_from_system_default(db_session):
+    """Returns ocr_language from the system default pipeline's OCR step config."""
+    import json
+
+    from app.models import Pipeline, PipelineStep
+    from app.tasks.process_document import _get_pipeline_ocr_language
+
+    # Create system default pipeline with OCR step configured to "deu"
+    pipeline = Pipeline(
+        owner_id=None,
+        name="System Default",
+        is_default=True,
+        is_active=True,
+    )
+    db_session.add(pipeline)
+    db_session.commit()
+
+    ocr_step = PipelineStep(
+        pipeline_id=pipeline.id,
+        position=0,
+        step_type="ocr",
+        config=json.dumps({"force_cloud_ocr": False, "ocr_language": "deu"}),
+        enabled=True,
+    )
+    db_session.add(ocr_step)
+    db_session.commit()
+
+    file_record = FileRecord(
+        filehash="def456",
+        original_filename="doc.pdf",
+        local_filename="/tmp/doc.pdf",
+        file_size=512,
+        mime_type="application/pdf",
+        is_duplicate=False,
+    )
+    db_session.add(file_record)
+    db_session.commit()
+
+    result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
+    assert result == "deu"
+
+
+@pytest.mark.unit
+@pytest.mark.requires_db
+def test_get_pipeline_ocr_language_auto_returns_none(db_session):
+    """Returns None when ocr_language is 'auto' (should use global settings)."""
+    import json
+
+    from app.models import Pipeline, PipelineStep
+    from app.tasks.process_document import _get_pipeline_ocr_language
+
+    pipeline = Pipeline(
+        owner_id=None,
+        name="Auto Lang Pipeline",
+        is_default=True,
+        is_active=True,
+    )
+    db_session.add(pipeline)
+    db_session.commit()
+
+    ocr_step = PipelineStep(
+        pipeline_id=pipeline.id,
+        position=0,
+        step_type="ocr",
+        config=json.dumps({"ocr_language": "auto"}),
+        enabled=True,
+    )
+    db_session.add(ocr_step)
+    db_session.commit()
+
+    file_record = FileRecord(
+        filehash="ghi789",
+        original_filename="auto.pdf",
+        local_filename="/tmp/auto.pdf",
+        file_size=128,
+        mime_type="application/pdf",
+        is_duplicate=False,
+    )
+    db_session.add(file_record)
+    db_session.commit()
+
+    result = _get_pipeline_ocr_language(db_session, file_record, owner_id=None)
+    assert result is None
+
+
+@pytest.mark.unit
+@pytest.mark.requires_db
+def test_get_pipeline_ocr_language_explicit_pipeline_takes_priority(db_session):
+    """Explicit pipeline_id on file takes priority over system default pipeline."""
+    import json
+
+    from app.models import Pipeline, PipelineStep
+    from app.tasks.process_document import _get_pipeline_ocr_language
+
+    # System default pipeline with "eng"
+    sys_pipeline = Pipeline(
+        owner_id=None,
+        name="System Default",
+        is_default=True,
+        is_active=True,
+    )
+    db_session.add(sys_pipeline)
+    db_session.commit()
+
+    sys_step = PipelineStep(
+        pipeline_id=sys_pipeline.id,
+        position=0,
+        step_type="ocr",
+        config=json.dumps({"ocr_language": "eng"}),
+        enabled=True,
+    )
+    db_session.add(sys_step)
+    db_session.commit()
+
+    # Explicit pipeline with "fra"
+    explicit_pipeline = Pipeline(
+        owner_id="user1",
+        name="French Pipeline",
+        is_default=False,
+        is_active=True,
+    )
+    db_session.add(explicit_pipeline)
+    db_session.commit()
+
+    explicit_step = PipelineStep(
+        pipeline_id=explicit_pipeline.id,
+        position=0,
+        step_type="ocr",
+        config=json.dumps({"ocr_language": "fra"}),
+        enabled=True,
+    )
+    db_session.add(explicit_step)
+    db_session.commit()
+
+    file_record = FileRecord(
+        filehash="jkl012",
+        original_filename="french.pdf",
+        local_filename="/tmp/french.pdf",
+        file_size=256,
+        mime_type="application/pdf",
+        is_duplicate=False,
+        pipeline_id=explicit_pipeline.id,
+    )
+    db_session.add(file_record)
+    db_session.commit()
+
+    result = _get_pipeline_ocr_language(db_session, file_record, owner_id="user1")
+    assert result == "fra"