feat: persist ocr_quality_score and use it for numeric filtering

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/database.py

@@ -122,6 +122,12 @@ def _run_schema_migrations(engine: Any) -> None:
                 conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
             logger.info("Migration complete: 'document_title' column added to files")
 
+        if "ocr_quality_score" not in columns:
+            logger.info("Migrating files: adding 'ocr_quality_score' column")
+            with engine.begin() as conn:
+                conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
+            logger.info("Migration complete: 'ocr_quality_score' column added to files")
+
         # Migration: Drop unique index on filehash to allow duplicate records
         try:
             indexes = inspector.get_indexes("files")

app/models.py

@@ -58,6 +58,9 @@ class FileRecord(Base):
     # Full OCR/extracted text for full-text search and RAG
     ocr_text = Column(Text, nullable=True)
 
+    # AI-assessed quality score for the OCR/extracted text (0–100; NULL = not yet assessed)
+    ocr_quality_score = Column(Integer, nullable=True)
+
     # AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
     ai_metadata = Column(Text, nullable=True)
 

app/tasks/process_document.py

@@ -442,6 +442,14 @@ def process_document(
                 f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}"
             )
 
+            # Persist the quality score immediately so it's available for filtering
+            # even if the file is later sent to OCR for re-processing.
+            with SessionLocal() as _db:
+                _rec = _db.query(FileRecord).filter_by(id=file_id).first()
+                if _rec:
+                    _rec.ocr_quality_score = quality_result.quality_score
+                    _db.commit()
+
             if not quality_result.is_good_quality:
                 # Poor quality: discard embedded text and re-OCR instead.
                 # Pass the original embedded text so the OCR task can compare

app/tasks/process_with_ocr.py

@@ -21,11 +21,13 @@ from typing import Optional
 
 from app.celery_app import celery
 from app.config import settings
+from app.database import SessionLocal
+from app.models import FileRecord
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.rotate_pdf_pages import rotate_pdf_pages
 from app.utils import log_task_progress
 from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
-from app.utils.text_quality import compare_text_quality
+from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality
 
 logger = logging.getLogger(__name__)
 
@@ -241,6 +243,29 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
             f"final text length: {len(final_text)} chars",
         )
 
+        # Score the final embedded text — the text that will land in ocr_text.
+        # We always call check_text_quality() on final_text because:
+        #   - merge_ocr_results() may have AI-merged output from several engines
+        #   - compare_text_quality() scores are relative (not the same scale)
+        #   - The original may have been preferred, reversing the OCR output
+        # OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS
+        # so the quality AI call is always made.
+        if file_id is not None:
+            try:
+                quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS)
+                logger.info(
+                    f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, "
+                    f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}"
+                )
+                with SessionLocal() as _db:
+                    _rec = _db.query(FileRecord).filter_by(id=file_id).first()
+                    if _rec:
+                        _rec.ocr_quality_score = quality_result.quality_score
+                        _db.commit()
+                logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}")
+            except Exception as _score_exc:
+                logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}")
+
         # Continue pipeline: rotate pages (if needed), then extract metadata
         rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id)
 

app/views/files.py

@@ -96,13 +96,21 @@ def files_page(
                 query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
 
         # Apply OCR quality filter
-        if ocr_quality == "no_ocr":
-            query = query.filter((FileRecord.ocr_text.is_(None)) | (FileRecord.ocr_text == ""))
-        elif ocr_quality == "has_ocr":
+        if ocr_quality == "poor":
+            # Files scored below the configured threshold
+            threshold = settings.text_quality_threshold
             query = query.filter(
-                FileRecord.ocr_text.isnot(None),
-                FileRecord.ocr_text != "",
+                FileRecord.ocr_quality_score.isnot(None),
+                FileRecord.ocr_quality_score < threshold,
             )
+        elif ocr_quality == "good":
+            threshold = settings.text_quality_threshold
+            query = query.filter(
+                FileRecord.ocr_quality_score.isnot(None),
+                FileRecord.ocr_quality_score >= threshold,
+            )
+        elif ocr_quality == "unchecked":
+            query = query.filter(FileRecord.ocr_quality_score.is_(None))
 
         # Apply status filter (before pagination for correct counts)
         query = apply_status_filter(query, db, status)
@@ -169,6 +177,7 @@ def files_page(
                 "storage_provider": storage_provider or "",
                 "tags": tags or "",
                 "ocr_quality": ocr_quality or "",
+                "ocr_quality_threshold": settings.text_quality_threshold,
                 "mime_types": mime_types,
                 "upload_concurrency": settings.upload_concurrency,
                 "upload_queue_delay_ms": settings.upload_queue_delay_ms,