feat: persist ocr_quality_score and use it for numeric filtering

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/database.py

@@ -122,6 +122,12 @@ def _run_schema_migrations(engine: Any) -> None:
                 conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
             logger.info("Migration complete: 'document_title' column added to files")
 
+        if "ocr_quality_score" not in columns:
+            logger.info("Migrating files: adding 'ocr_quality_score' column")
+            with engine.begin() as conn:
+                conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
+            logger.info("Migration complete: 'ocr_quality_score' column added to files")
+
         # Migration: Drop unique index on filehash to allow duplicate records
         try:
             indexes = inspector.get_indexes("files")

app/models.py

@@ -58,6 +58,9 @@ class FileRecord(Base):
     # Full OCR/extracted text for full-text search and RAG
     ocr_text = Column(Text, nullable=True)
 
+    # AI-assessed quality score for the OCR/extracted text (0–100; NULL = not yet assessed)
+    ocr_quality_score = Column(Integer, nullable=True)
+
     # AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
     ai_metadata = Column(Text, nullable=True)
 

app/tasks/process_document.py

@@ -442,6 +442,14 @@ def process_document(
                 f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}"
             )
 
+            # Persist the quality score immediately so it's available for filtering
+            # even if the file is later sent to OCR for re-processing.
+            with SessionLocal() as _db:
+                _rec = _db.query(FileRecord).filter_by(id=file_id).first()
+                if _rec:
+                    _rec.ocr_quality_score = quality_result.quality_score
+                    _db.commit()
+
             if not quality_result.is_good_quality:
                 # Poor quality: discard embedded text and re-OCR instead.
                 # Pass the original embedded text so the OCR task can compare

app/tasks/process_with_ocr.py

@@ -21,11 +21,13 @@ from typing import Optional
 
 from app.celery_app import celery
 from app.config import settings
+from app.database import SessionLocal
+from app.models import FileRecord
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.rotate_pdf_pages import rotate_pdf_pages
 from app.utils import log_task_progress
 from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
-from app.utils.text_quality import compare_text_quality
+from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality
 
 logger = logging.getLogger(__name__)
 
@@ -241,6 +243,29 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
             f"final text length: {len(final_text)} chars",
         )
 
+        # Score the final embedded text — the text that will land in ocr_text.
+        # We always call check_text_quality() on final_text because:
+        #   - merge_ocr_results() may have AI-merged output from several engines
+        #   - compare_text_quality() scores are relative (not the same scale)
+        #   - The original may have been preferred, reversing the OCR output
+        # OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS
+        # so the quality AI call is always made.
+        if file_id is not None:
+            try:
+                quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS)
+                logger.info(
+                    f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, "
+                    f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}"
+                )
+                with SessionLocal() as _db:
+                    _rec = _db.query(FileRecord).filter_by(id=file_id).first()
+                    if _rec:
+                        _rec.ocr_quality_score = quality_result.quality_score
+                        _db.commit()
+                logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}")
+            except Exception as _score_exc:
+                logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}")
+
         # Continue pipeline: rotate pages (if needed), then extract metadata
         rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id)
 

app/views/files.py

@@ -96,13 +96,21 @@ def files_page(
                 query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
 
         # Apply OCR quality filter
-        if ocr_quality == "no_ocr":
+        if ocr_quality == "poor":
-            query = query.filter((FileRecord.ocr_text.is_(None)) | (FileRecord.ocr_text == ""))
+            # Files scored below the configured threshold
-        elif ocr_quality == "has_ocr":
+            threshold = settings.text_quality_threshold
             query = query.filter(
-                FileRecord.ocr_text.isnot(None),
+                FileRecord.ocr_quality_score.isnot(None),
-                FileRecord.ocr_text != "",
+                FileRecord.ocr_quality_score < threshold,
             )
+        elif ocr_quality == "good":
+            threshold = settings.text_quality_threshold
+            query = query.filter(
+                FileRecord.ocr_quality_score.isnot(None),
+                FileRecord.ocr_quality_score >= threshold,
+            )
+        elif ocr_quality == "unchecked":
+            query = query.filter(FileRecord.ocr_quality_score.is_(None))
 
         # Apply status filter (before pagination for correct counts)
         query = apply_status_filter(query, db, status)
@@ -169,6 +177,7 @@ def files_page(
                 "storage_provider": storage_provider or "",
                 "tags": tags or "",
                 "ocr_quality": ocr_quality or "",
+                "ocr_quality_threshold": settings.text_quality_threshold,
                 "mime_types": mime_types,
                 "upload_concurrency": settings.upload_concurrency,
                 "upload_queue_delay_ms": settings.upload_queue_delay_ms,

docs/API.md

@@ -295,7 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting
 - `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`)
 - `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`)
 - `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`)
-- `ocr_quality` (optional): Filter by OCR text availability (`no_ocr` = files without OCR text, `has_ocr` = files with OCR text)
+- `ocr_quality` (optional): Filter by AI-assessed OCR quality score (`poor` = score below threshold, `good` = score at or above threshold, `unchecked` = not yet assessed). The threshold is configured via `TEXT_QUALITY_THRESHOLD` (default: 85).
 
 All filters are combinable using AND logic.
 

frontend/templates/files.html

@@ -535,10 +535,11 @@
 
       <div class="filter-item">
         <label for="ocr_quality">OCR Quality</label>
-        <select id="ocr_quality" name="ocr_quality" aria-label="Filter by OCR quality">
+        <select id="ocr_quality" name="ocr_quality" aria-label="Filter by OCR quality score">
           <option value="">All Files</option>
-          <option value="no_ocr" {% if ocr_quality == "no_ocr" %}selected{% endif %}>No OCR Text</option>
+          <option value="poor" {% if ocr_quality == "poor" %}selected{% endif %} aria-label="Poor quality, score less than {{ ocr_quality_threshold }}">Poor (score &lt; {{ ocr_quality_threshold }})</option>
-          <option value="has_ocr" {% if ocr_quality == "has_ocr" %}selected{% endif %}>Has OCR Text</option>
+          <option value="good" {% if ocr_quality == "good" %}selected{% endif %} aria-label="Good quality, score at least {{ ocr_quality_threshold }}">Good (score &ge; {{ ocr_quality_threshold }})</option>
+          <option value="unchecked" {% if ocr_quality == "unchecked" %}selected{% endif %} aria-label="Not yet assessed">Not yet assessed</option>
         </select>
       </div>
 

tests/test_bulk_operations.py

@@ -464,57 +464,83 @@ class TestBulkReprocessCloudOcr:
 class TestOcrQualityFilter:
     """Tests for the ocr_quality filter on the /files view."""
 
-    def test_ocr_quality_no_ocr_filter(self, client: TestClient, db_session):
+    def test_ocr_quality_poor_filter(self, client: TestClient, db_session):
-        """Files without OCR text appear when filtering no_ocr."""
+        """Files with a low ocr_quality_score appear when filtering poor."""
-        rec_no_ocr = FileRecord(
+        rec_poor = FileRecord(
-            filehash="hash_noocr1",
+            filehash="hash_poor1",
-            original_filename="no_ocr.pdf",
+            original_filename="poor_quality.pdf",
-            local_filename="/tmp/no_ocr.pdf",
+            local_filename="/tmp/poor_quality.pdf",
             file_size=1024,
             mime_type="application/pdf",
-            ocr_text=None,
+            ocr_quality_score=40,
         )
-        rec_has_ocr = FileRecord(
+        rec_good = FileRecord(
-            filehash="hash_hasocr1",
+            filehash="hash_good1",
-            original_filename="has_ocr.pdf",
+            original_filename="good_quality.pdf",
-            local_filename="/tmp/has_ocr.pdf",
+            local_filename="/tmp/good_quality.pdf",
             file_size=1024,
             mime_type="application/pdf",
-            ocr_text="Some extracted text",
+            ocr_quality_score=95,
         )
-        db_session.add_all([rec_no_ocr, rec_has_ocr])
+        db_session.add_all([rec_poor, rec_good])
         db_session.commit()
 
-        response = client.get("/files?ocr_quality=no_ocr")
+        response = client.get("/files?ocr_quality=poor")
         assert response.status_code == 200
-        assert "no_ocr.pdf" in response.text
+        assert "poor_quality.pdf" in response.text
-        assert "has_ocr.pdf" not in response.text
+        assert "good_quality.pdf" not in response.text
 
-    def test_ocr_quality_has_ocr_filter(self, client: TestClient, db_session):
+    def test_ocr_quality_good_filter(self, client: TestClient, db_session):
-        """Files with OCR text appear when filtering has_ocr."""
+        """Files with a high ocr_quality_score appear when filtering good."""
-        rec_no_ocr = FileRecord(
+        rec_poor = FileRecord(
-            filehash="hash_noocr2",
+            filehash="hash_poor2",
-            original_filename="no_ocr2.pdf",
+            original_filename="poor_quality2.pdf",
-            local_filename="/tmp/no_ocr2.pdf",
+            local_filename="/tmp/poor_quality2.pdf",
             file_size=1024,
             mime_type="application/pdf",
-            ocr_text=None,
+            ocr_quality_score=40,
         )
-        rec_has_ocr = FileRecord(
+        rec_good = FileRecord(
-            filehash="hash_hasocr2",
+            filehash="hash_good2",
-            original_filename="has_ocr2.pdf",
+            original_filename="good_quality2.pdf",
-            local_filename="/tmp/has_ocr2.pdf",
+            local_filename="/tmp/good_quality2.pdf",
             file_size=1024,
             mime_type="application/pdf",
-            ocr_text="Meaningful extracted text",
+            ocr_quality_score=95,
         )
-        db_session.add_all([rec_no_ocr, rec_has_ocr])
+        db_session.add_all([rec_poor, rec_good])
         db_session.commit()
 
-        response = client.get("/files?ocr_quality=has_ocr")
+        response = client.get("/files?ocr_quality=good")
         assert response.status_code == 200
-        assert "has_ocr2.pdf" in response.text
+        assert "good_quality2.pdf" in response.text
-        assert "no_ocr2.pdf" not in response.text
+        assert "poor_quality2.pdf" not in response.text
+
+    def test_ocr_quality_unchecked_filter(self, client: TestClient, db_session):
+        """Files with no score appear when filtering unchecked."""
+        rec_unchecked = FileRecord(
+            filehash="hash_unch1",
+            original_filename="unchecked.pdf",
+            local_filename="/tmp/unchecked.pdf",
+            file_size=1024,
+            mime_type="application/pdf",
+            ocr_quality_score=None,
+        )
+        rec_scored = FileRecord(
+            filehash="hash_scored1",
+            original_filename="scored.pdf",
+            local_filename="/tmp/scored.pdf",
+            file_size=1024,
+            mime_type="application/pdf",
+            ocr_quality_score=90,
+        )
+        db_session.add_all([rec_unchecked, rec_scored])
+        db_session.commit()
+
+        response = client.get("/files?ocr_quality=unchecked")
+        assert response.status_code == 200
+        assert "unchecked.pdf" in response.text
+        assert "scored.pdf" not in response.text
 
     def test_ocr_quality_no_filter(self, client: TestClient, db_session):
         """All files appear when no ocr_quality filter is applied."""