From 83c3405c986700b079a1f8167ebbebfe23e7aa7f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 13:46:43 +0000 Subject: [PATCH] feat: persist ocr_quality_score and use it for numeric filtering Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/database.py | 6 +++ app/models.py | 3 ++ app/tasks/process_document.py | 8 ++++ app/tasks/process_with_ocr.py | 27 ++++++++++- app/views/files.py | 19 ++++++-- docs/API.md | 2 +- frontend/templates/files.html | 7 +-- tests/test_bulk_operations.py | 90 ++++++++++++++++++++++------------- 8 files changed, 120 insertions(+), 42 deletions(-) diff --git a/app/database.py b/app/database.py index 61c59263..f48b1ba5 100644 --- a/app/database.py +++ b/app/database.py @@ -122,6 +122,12 @@ def _run_schema_migrations(engine: Any) -> None: conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR")) logger.info("Migration complete: 'document_title' column added to files") + if "ocr_quality_score" not in columns: + logger.info("Migrating files: adding 'ocr_quality_score' column") + with engine.begin() as conn: + conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER")) + logger.info("Migration complete: 'ocr_quality_score' column added to files") + # Migration: Drop unique index on filehash to allow duplicate records try: indexes = inspector.get_indexes("files") diff --git a/app/models.py b/app/models.py index 7bd40499..81d0d5e5 100644 --- a/app/models.py +++ b/app/models.py @@ -58,6 +58,9 @@ class FileRecord(Base): # Full OCR/extracted text for full-text search and RAG ocr_text = Column(Text, nullable=True) + # AI-assessed quality score for the OCR/extracted text (0–100; NULL = not yet assessed) + ocr_quality_score = Column(Integer, nullable=True) + # AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.) ai_metadata = Column(Text, nullable=True) diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 17f3f8e7..dab68d49 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -442,6 +442,14 @@ def process_document( f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}" ) + # Persist the quality score immediately so it's available for filtering + # even if the file is later sent to OCR for re-processing. + with SessionLocal() as _db: + _rec = _db.query(FileRecord).filter_by(id=file_id).first() + if _rec: + _rec.ocr_quality_score = quality_result.quality_score + _db.commit() + if not quality_result.is_good_quality: # Poor quality: discard embedded text and re-OCR instead. # Pass the original embedded text so the OCR task can compare diff --git a/app/tasks/process_with_ocr.py b/app/tasks/process_with_ocr.py index 82f8545e..dace3862 100644 --- a/app/tasks/process_with_ocr.py +++ b/app/tasks/process_with_ocr.py @@ -21,11 +21,13 @@ from typing import Optional from app.celery_app import celery from app.config import settings +from app.database import SessionLocal +from app.models import FileRecord from app.tasks.retry_config import BaseTaskWithRetry from app.tasks.rotate_pdf_pages import rotate_pdf_pages from app.utils import log_task_progress from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results -from app.utils.text_quality import compare_text_quality +from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality logger = logging.getLogger(__name__) @@ -241,6 +243,29 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina f"final text length: {len(final_text)} chars", ) + # Score the final embedded text — the text that will land in ocr_text. + # We always call check_text_quality() on final_text because: + # - merge_ocr_results() may have AI-merged output from several engines + # - compare_text_quality() scores are relative (not the same scale) + # - The original may have been preferred, reversing the OCR output + # OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS + # so the quality AI call is always made. + if file_id is not None: + try: + quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS) + logger.info( + f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, " + f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}" + ) + with SessionLocal() as _db: + _rec = _db.query(FileRecord).filter_by(id=file_id).first() + if _rec: + _rec.ocr_quality_score = quality_result.quality_score + _db.commit() + logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}") + except Exception as _score_exc: + logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}") + # Continue pipeline: rotate pages (if needed), then extract metadata rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id) diff --git a/app/views/files.py b/app/views/files.py index b578b79a..64839983 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -96,13 +96,21 @@ def files_page( query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%")) # Apply OCR quality filter - if ocr_quality == "no_ocr": - query = query.filter((FileRecord.ocr_text.is_(None)) | (FileRecord.ocr_text == "")) - elif ocr_quality == "has_ocr": + if ocr_quality == "poor": + # Files scored below the configured threshold + threshold = settings.text_quality_threshold query = query.filter( - FileRecord.ocr_text.isnot(None), - FileRecord.ocr_text != "", + FileRecord.ocr_quality_score.isnot(None), + FileRecord.ocr_quality_score < threshold, ) + elif ocr_quality == "good": + threshold = settings.text_quality_threshold + query = query.filter( + FileRecord.ocr_quality_score.isnot(None), + FileRecord.ocr_quality_score >= threshold, + ) + elif ocr_quality == "unchecked": + query = query.filter(FileRecord.ocr_quality_score.is_(None)) # Apply status filter (before pagination for correct counts) query = apply_status_filter(query, db, status) @@ -169,6 +177,7 @@ def files_page( "storage_provider": storage_provider or "", "tags": tags or "", "ocr_quality": ocr_quality or "", + "ocr_quality_threshold": settings.text_quality_threshold, "mime_types": mime_types, "upload_concurrency": settings.upload_concurrency, "upload_queue_delay_ms": settings.upload_queue_delay_ms, diff --git a/docs/API.md b/docs/API.md index f76e9654..b045bc7c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -295,7 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting - `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`) - `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`) - `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`) -- `ocr_quality` (optional): Filter by OCR text availability (`no_ocr` = files without OCR text, `has_ocr` = files with OCR text) +- `ocr_quality` (optional): Filter by AI-assessed OCR quality score (`poor` = score below threshold, `good` = score at or above threshold, `unchecked` = not yet assessed). The threshold is configured via `TEXT_QUALITY_THRESHOLD` (default: 85). All filters are combinable using AND logic. diff --git a/frontend/templates/files.html b/frontend/templates/files.html index b706a354..7847bb34 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -535,10 +535,11 @@
- - - + + +
diff --git a/tests/test_bulk_operations.py b/tests/test_bulk_operations.py index f81a0f7b..dad25212 100644 --- a/tests/test_bulk_operations.py +++ b/tests/test_bulk_operations.py @@ -464,57 +464,83 @@ class TestBulkReprocessCloudOcr: class TestOcrQualityFilter: """Tests for the ocr_quality filter on the /files view.""" - def test_ocr_quality_no_ocr_filter(self, client: TestClient, db_session): - """Files without OCR text appear when filtering no_ocr.""" - rec_no_ocr = FileRecord( - filehash="hash_noocr1", - original_filename="no_ocr.pdf", - local_filename="/tmp/no_ocr.pdf", + def test_ocr_quality_poor_filter(self, client: TestClient, db_session): + """Files with a low ocr_quality_score appear when filtering poor.""" + rec_poor = FileRecord( + filehash="hash_poor1", + original_filename="poor_quality.pdf", + local_filename="/tmp/poor_quality.pdf", file_size=1024, mime_type="application/pdf", - ocr_text=None, + ocr_quality_score=40, ) - rec_has_ocr = FileRecord( - filehash="hash_hasocr1", - original_filename="has_ocr.pdf", - local_filename="/tmp/has_ocr.pdf", + rec_good = FileRecord( + filehash="hash_good1", + original_filename="good_quality.pdf", + local_filename="/tmp/good_quality.pdf", file_size=1024, mime_type="application/pdf", - ocr_text="Some extracted text", + ocr_quality_score=95, ) - db_session.add_all([rec_no_ocr, rec_has_ocr]) + db_session.add_all([rec_poor, rec_good]) db_session.commit() - response = client.get("/files?ocr_quality=no_ocr") + response = client.get("/files?ocr_quality=poor") assert response.status_code == 200 - assert "no_ocr.pdf" in response.text - assert "has_ocr.pdf" not in response.text + assert "poor_quality.pdf" in response.text + assert "good_quality.pdf" not in response.text - def test_ocr_quality_has_ocr_filter(self, client: TestClient, db_session): - """Files with OCR text appear when filtering has_ocr.""" - rec_no_ocr = FileRecord( - filehash="hash_noocr2", - original_filename="no_ocr2.pdf", - local_filename="/tmp/no_ocr2.pdf", + def test_ocr_quality_good_filter(self, client: TestClient, db_session): + """Files with a high ocr_quality_score appear when filtering good.""" + rec_poor = FileRecord( + filehash="hash_poor2", + original_filename="poor_quality2.pdf", + local_filename="/tmp/poor_quality2.pdf", file_size=1024, mime_type="application/pdf", - ocr_text=None, + ocr_quality_score=40, ) - rec_has_ocr = FileRecord( - filehash="hash_hasocr2", - original_filename="has_ocr2.pdf", - local_filename="/tmp/has_ocr2.pdf", + rec_good = FileRecord( + filehash="hash_good2", + original_filename="good_quality2.pdf", + local_filename="/tmp/good_quality2.pdf", file_size=1024, mime_type="application/pdf", - ocr_text="Meaningful extracted text", + ocr_quality_score=95, ) - db_session.add_all([rec_no_ocr, rec_has_ocr]) + db_session.add_all([rec_poor, rec_good]) db_session.commit() - response = client.get("/files?ocr_quality=has_ocr") + response = client.get("/files?ocr_quality=good") assert response.status_code == 200 - assert "has_ocr2.pdf" in response.text - assert "no_ocr2.pdf" not in response.text + assert "good_quality2.pdf" in response.text + assert "poor_quality2.pdf" not in response.text + + def test_ocr_quality_unchecked_filter(self, client: TestClient, db_session): + """Files with no score appear when filtering unchecked.""" + rec_unchecked = FileRecord( + filehash="hash_unch1", + original_filename="unchecked.pdf", + local_filename="/tmp/unchecked.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_quality_score=None, + ) + rec_scored = FileRecord( + filehash="hash_scored1", + original_filename="scored.pdf", + local_filename="/tmp/scored.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_quality_score=90, + ) + db_session.add_all([rec_unchecked, rec_scored]) + db_session.commit() + + response = client.get("/files?ocr_quality=unchecked") + assert response.status_code == 200 + assert "unchecked.pdf" in response.text + assert "scored.pdf" not in response.text def test_ocr_quality_no_filter(self, client: TestClient, db_session): """All files appear when no ocr_quality filter is applied."""