Merge pull request #439 from christianlouis/copilot/add-bulk-file-operations

feat: bulk operations (delete, reprocess, Cloud OCR, ZIP download) + persisted OCR quality score filter
This commit is contained in:
Christian Krakau-Louis
2026-03-01 14:58:01 +01:00
committed by GitHub
9 changed files with 674 additions and 1 deletions
+160
View File
@@ -2,14 +2,17 @@
File-related API endpoints
"""
import io
import logging
import mimetypes
import os
import uuid
import zipfile
from datetime import datetime, timezone
from typing import Annotated, List, Optional
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import StreamingResponse
from sqlalchemy import asc, desc
from sqlalchemy.orm import Session
@@ -429,6 +432,163 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
@router.post("/files/bulk-reprocess-cloud-ocr")
@require_login
def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: DbSession):
"""
Reprocess multiple files with forced Cloud OCR (Azure Document Intelligence).
Useful for re-running OCR on files with poor text quality or missing OCR text.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
task_ids = []
processed_files = []
errors = []
for file_record in file_records:
try:
source_file = None
for path in [file_record.original_file_path, file_record.local_filename]:
if path and os.path.exists(path):
source_file = path
break
if not source_file:
errors.append(
{
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": "File not found on disk",
}
)
continue
task = process_document.delay(
source_file,
original_filename=file_record.original_filename,
file_id=file_record.id,
force_cloud_ocr=True,
)
task_ids.append(task.id)
processed_files.append(
{
"file_id": file_record.id,
"filename": file_record.original_filename,
"task_id": task.id,
}
)
logger.info(
f"Bulk Cloud OCR reprocessing: ID={file_record.id}, "
f"Filename={file_record.original_filename}, TaskID={task.id}"
)
except Exception as e:
logger.exception(f"Error queuing Cloud OCR for file {file_record.id}: {str(e)}")
errors.append(
{
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": str(e),
}
)
return {
"status": "success" if processed_files else "error",
"message": f"Successfully queued {len(processed_files)} files for Cloud OCR reprocessing",
"processed_files": processed_files,
"errors": errors,
"task_ids": task_ids,
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
@router.post("/files/bulk-download")
@require_login
def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
"""
Download multiple files as a single ZIP archive.
For each file, the processed version is preferred; falls back to the original.
Files not found on disk are silently skipped.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
zip_buffer = io.BytesIO()
added = 0
seen_names: dict[str, int] = {}
with zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed")
for file_record in file_records:
# Resolve file path: processed first, then original/local
base_filename = os.path.splitext(file_record.original_filename or "file")[0]
candidate_paths = [
file_record.processed_file_path,
file_record.original_file_path,
file_record.local_filename,
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
]
file_path = None
for path in candidate_paths:
if path and os.path.exists(path):
file_path = path
break
if not file_path:
logger.warning(f"Skipping file {file_record.id}: no file found on disk")
continue
# Build a unique archive name to avoid collisions
archive_name = file_record.original_filename or os.path.basename(file_path)
if archive_name in seen_names:
seen_names[archive_name] += 1
stem, ext = os.path.splitext(archive_name)
archive_name = f"{stem}_{seen_names[archive_name]}{ext}"
else:
seen_names[archive_name] = 0
zf.write(file_path, archive_name)
added += 1
if added == 0:
raise HTTPException(status_code=404, detail="None of the selected files could be found on disk")
zip_buffer.seek(0)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
zip_filename = f"docuelevate_bulk_{timestamp}.zip"
logger.info(f"Bulk download: packed {added} file(s) into {zip_filename}")
return StreamingResponse(
zip_buffer,
media_type="application/zip",
headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'},
)
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error creating bulk download ZIP: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error creating bulk download ZIP: {str(e)}")
@router.post("/files/{file_id}/reprocess")
@require_login
def reprocess_single_file(request: Request, file_id: int, db: DbSession):
+6
View File
@@ -122,6 +122,12 @@ def _run_schema_migrations(engine: Any) -> None:
conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
logger.info("Migration complete: 'document_title' column added to files")
if "ocr_quality_score" not in columns:
logger.info("Migrating files: adding 'ocr_quality_score' column")
with engine.begin() as conn:
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
logger.info("Migration complete: 'ocr_quality_score' column added to files")
# Migration: Drop unique index on filehash to allow duplicate records
try:
indexes = inspector.get_indexes("files")
+3
View File
@@ -58,6 +58,9 @@ class FileRecord(Base):
# Full OCR/extracted text for full-text search and RAG
ocr_text = Column(Text, nullable=True)
# AI-assessed quality score for the OCR/extracted text (0100; NULL = not yet assessed)
ocr_quality_score = Column(Integer, nullable=True)
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
ai_metadata = Column(Text, nullable=True)
+8
View File
@@ -442,6 +442,14 @@ def process_document(
f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}"
)
# Persist the quality score immediately so it's available for filtering
# even if the file is later sent to OCR for re-processing.
with SessionLocal() as _db:
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
if _rec:
_rec.ocr_quality_score = quality_result.quality_score
_db.commit()
if not quality_result.is_good_quality:
# Poor quality: discard embedded text and re-OCR instead.
# Pass the original embedded text so the OCR task can compare
+26 -1
View File
@@ -21,11 +21,13 @@ from typing import Optional
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
from app.utils import log_task_progress
from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
from app.utils.text_quality import compare_text_quality
from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality
logger = logging.getLogger(__name__)
@@ -241,6 +243,29 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
f"final text length: {len(final_text)} chars",
)
# Score the final embedded text — the text that will land in ocr_text.
# We always call check_text_quality() on final_text because:
# - merge_ocr_results() may have AI-merged output from several engines
# - compare_text_quality() scores are relative (not the same scale)
# - The original may have been preferred, reversing the OCR output
# OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS
# so the quality AI call is always made.
if file_id is not None:
try:
quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS)
logger.info(
f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, "
f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}"
)
with SessionLocal() as _db:
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
if _rec:
_rec.ocr_quality_score = quality_result.quality_score
_db.commit()
logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}")
except Exception as _score_exc:
logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}")
# Continue pipeline: rotate pages (if needed), then extract metadata
rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id)
+20
View File
@@ -34,6 +34,7 @@ def files_page(
date_to: Optional[str] = Query(None),
storage_provider: Optional[str] = Query(None),
tags: Optional[str] = Query(None),
ocr_quality: Optional[str] = Query(None),
):
"""
Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -94,6 +95,23 @@ def files_page(
escaped_tag = tag.replace("%", r"\%").replace("_", r"\_")
query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
# Apply OCR quality filter
if ocr_quality == "poor":
# Files scored below the configured threshold
threshold = settings.text_quality_threshold
query = query.filter(
FileRecord.ocr_quality_score.isnot(None),
FileRecord.ocr_quality_score < threshold,
)
elif ocr_quality == "good":
threshold = settings.text_quality_threshold
query = query.filter(
FileRecord.ocr_quality_score.isnot(None),
FileRecord.ocr_quality_score >= threshold,
)
elif ocr_quality == "unchecked":
query = query.filter(FileRecord.ocr_quality_score.is_(None))
# Apply status filter (before pagination for correct counts)
query = apply_status_filter(query, db, status)
@@ -158,6 +176,8 @@ def files_page(
"date_to": date_to or "",
"storage_provider": storage_provider or "",
"tags": tags or "",
"ocr_quality": ocr_quality or "",
"ocr_quality_threshold": settings.text_quality_threshold,
"mime_types": mime_types,
"upload_concurrency": settings.upload_concurrency,
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
+91
View File
@@ -295,6 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting
- `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`)
- `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`)
- `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`)
- `ocr_quality` (optional): Filter by AI-assessed OCR quality score (`poor` = score below threshold, `good` = score at or above threshold, `unchecked` = not yet assessed). The threshold is configured via `TEXT_QUALITY_THRESHOLD` (default: 85).
All filters are combinable using AND logic.
@@ -461,6 +462,96 @@ Reprocess a specific file with forced Cloud OCR, regardless of embedded text qua
**Note**: This endpoint forces Azure Document Intelligence OCR processing even if the PDF contains embedded text. The original file (if available) is used for reprocessing to ensure the highest quality result.
### Bulk Operations
**POST** `/api/files/bulk-delete`
Delete multiple file records in a single request.
**Request body**: JSON array of file IDs
```bash
curl -X POST "http://<your-instance>/api/files/bulk-delete" \
-H "Content-Type: application/json" \
-d '[1, 2, 3]'
```
**Response**:
```json
{
"status": "success",
"message": "Successfully deleted 3 file records",
"deleted_ids": [1, 2, 3]
}
```
**Error Responses**:
- `403`: File deletion is disabled in configuration
- `404`: No files found with the provided IDs
---
**POST** `/api/files/bulk-reprocess`
Queue multiple files for full reprocessing.
**Request body**: JSON array of file IDs
**Response**:
```json
{
"status": "success",
"message": "Successfully queued 2 files for reprocessing",
"processed_files": [
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"},
{"file_id": 2, "filename": "b.pdf", "task_id": "def456"}
],
"errors": [],
"task_ids": ["abc123", "def456"]
}
```
---
**POST** `/api/files/bulk-reprocess-cloud-ocr`
Queue multiple files for reprocessing with forced Cloud OCR (Azure Document Intelligence). Useful for files that have missing or low-quality OCR text.
**Request body**: JSON array of file IDs
**Response**:
```json
{
"status": "success",
"message": "Successfully queued 2 files for Cloud OCR reprocessing",
"processed_files": [
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"}
],
"errors": [],
"task_ids": ["abc123"]
}
```
---
**POST** `/api/files/bulk-download`
Download multiple files as a single ZIP archive. For each file, the processed version is preferred; falls back to the original. Files not found on disk are silently skipped.
**Request body**: JSON array of file IDs
**Response**: `application/zip` stream with `Content-Disposition: attachment; filename="docuelevate_bulk_<timestamp>.zip"`
```bash
curl -X POST "http://<your-instance>/api/files/bulk-download" \
-H "Content-Type: application/json" \
-d '[1, 2, 3]' \
--output bulk_download.zip
```
**Error Responses**:
- `404`: No files found with the provided IDs, or none of the selected files exist on disk
### File Preview
**GET** `/api/files/{file_id}/preview`
+96
View File
@@ -533,6 +533,16 @@
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
</div>
<div class="filter-item">
<label for="ocr_quality">OCR Quality</label>
<select id="ocr_quality" name="ocr_quality" aria-label="Filter by OCR quality score">
<option value="">All Files</option>
<option value="poor" {% if ocr_quality == "poor" %}selected{% endif %} aria-label="Poor quality, score less than {{ ocr_quality_threshold }}">Poor (score &lt; {{ ocr_quality_threshold }})</option>
<option value="good" {% if ocr_quality == "good" %}selected{% endif %} aria-label="Good quality, score at least {{ ocr_quality_threshold }}">Good (score &ge; {{ ocr_quality_threshold }})</option>
<option value="unchecked" {% if ocr_quality == "unchecked" %}selected{% endif %} aria-label="Not yet assessed">Not yet assessed</option>
</select>
</div>
<div class="filter-item">
<label>&nbsp;</label>
<button type="submit">Apply Filters</button>
@@ -616,6 +626,12 @@
<button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
<i class="fas fa-sync" aria-hidden="true"></i> Reprocess Selected
</button>
<button type="button" onclick="bulkReprocessCloudOcr()" style="padding: 0.5rem 1rem; background-color: #805ad5; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
<i class="fas fa-cloud" aria-hidden="true"></i> Re-run Cloud OCR
</button>
<button type="button" onclick="bulkDownload()" style="padding: 0.5rem 1rem; background-color: #38a169; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
<i class="fas fa-file-archive" aria-hidden="true"></i> Download as ZIP
</button>
<button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
<i class="fas fa-trash" aria-hidden="true"></i> Delete Selected
</button>
@@ -1285,6 +1301,86 @@
});
}
function bulkReprocessCloudOcr() {
const fileIds = getSelectedFileIds();
if (fileIds.length === 0) {
alert('Please select files to re-run Cloud OCR on');
return;
}
if (!confirm(`Are you sure you want to re-run Cloud OCR on ${fileIds.length} file(s)?`)) {
return;
}
fetch('/api/files/bulk-reprocess-cloud-ocr', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(fileIds)
})
.then(response => {
if (!response.ok) {
return response.json().then(err => {
throw new Error(err.detail || 'Failed to queue Cloud OCR');
});
}
return response.json();
})
.then(data => {
if (data.errors && data.errors.length > 0) {
const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n');
alert(`${data.message}\n\nErrors:\n${errorMsg}`);
} else {
alert(data.message);
}
clearSelection();
window.location.reload();
})
.catch(error => {
console.error('Error:', error);
alert(`Error queuing Cloud OCR: ${error.message}`);
});
}
function bulkDownload() {
const fileIds = getSelectedFileIds();
if (fileIds.length === 0) {
alert('Please select files to download');
return;
}
fetch('/api/files/bulk-download', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(fileIds)
})
.then(response => {
if (!response.ok) {
return response.json().then(err => {
throw new Error(err.detail || 'Failed to create ZIP download');
});
}
return response.blob();
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `docuelevate_bulk_${Date.now()}.zip`;
document.body.appendChild(a);
a.click();
a.remove();
window.URL.revokeObjectURL(url);
})
.catch(error => {
console.error('Error:', error);
alert(`Error downloading files: ${error.message}`);
});
}
// ===== Drag-and-Drop Upload Functionality =====
const dropOverlay = document.getElementById('dropOverlay');
const uploadModal = document.getElementById('uploadModal');
+264
View File
@@ -2,6 +2,8 @@
Tests for bulk file operations (delete and reprocess).
"""
import io
import zipfile
from unittest.mock import MagicMock, patch
import pytest
@@ -293,3 +295,265 @@ class TestStatusFilter:
response = client.get("/files?status=failed")
assert response.status_code == 200
assert "failed.pdf" in response.text
@pytest.mark.integration
@pytest.mark.requires_db
class TestBulkDownload:
"""Tests for POST /api/files/bulk-download endpoint."""
def test_bulk_download_no_files_found(self, client: TestClient, db_session):
"""Test bulk download with non-existent IDs."""
response = client.post("/api/files/bulk-download", json=[99999, 99998])
assert response.status_code == 404
def test_bulk_download_files_not_on_disk(self, client: TestClient, db_session):
"""Test bulk download when files are not found on disk."""
file_record = FileRecord(
filehash="hash_dl1",
original_filename="nodisk.pdf",
local_filename="/nonexistent/nodisk.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
response = client.post("/api/files/bulk-download", json=[file_record.id])
assert response.status_code == 404
assert "None of the selected files" in response.json()["detail"]
def test_bulk_download_success(self, client: TestClient, db_session, tmp_path):
"""Test successful bulk download returns a ZIP archive."""
# Create a real file on disk
pdf_file = tmp_path / "sample.pdf"
pdf_file.write_bytes(b"PDF content")
file_record = FileRecord(
filehash="hash_dl2",
original_filename="sample.pdf",
local_filename=str(pdf_file),
processed_file_path=str(pdf_file),
file_size=len(b"PDF content"),
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
response = client.post("/api/files/bulk-download", json=[file_record.id])
assert response.status_code == 200
assert response.headers["content-type"] == "application/zip"
assert "attachment" in response.headers["content-disposition"]
assert ".zip" in response.headers["content-disposition"]
def test_bulk_download_multiple_files(self, client: TestClient, db_session, tmp_path):
"""Test bulk download with multiple files produces a valid ZIP."""
ids = []
for i in range(3):
f = tmp_path / f"file{i}.pdf"
f.write_bytes(f"content {i}".encode())
rec = FileRecord(
filehash=f"hash_multi_{i}",
original_filename=f"file{i}.pdf",
local_filename=str(f),
processed_file_path=str(f),
file_size=len(f"content {i}".encode()),
mime_type="application/pdf",
)
db_session.add(rec)
db_session.commit()
ids.append(rec.id)
response = client.post("/api/files/bulk-download", json=ids)
assert response.status_code == 200
zip_data = io.BytesIO(response.content)
with zipfile.ZipFile(zip_data) as zf:
names = zf.namelist()
assert len(names) == 3
def test_bulk_download_duplicate_filenames(self, client: TestClient, db_session, tmp_path):
"""Test bulk download disambiguates duplicate filenames."""
ids = []
for i in range(2):
f = tmp_path / f"dup_{i}.pdf"
f.write_bytes(b"data")
rec = FileRecord(
filehash=f"hash_dup_{i}",
original_filename="dup.pdf", # same name
local_filename=str(f),
processed_file_path=str(f),
file_size=4,
mime_type="application/pdf",
)
db_session.add(rec)
db_session.commit()
ids.append(rec.id)
response = client.post("/api/files/bulk-download", json=ids)
assert response.status_code == 200
zip_data = io.BytesIO(response.content)
with zipfile.ZipFile(zip_data) as zf:
names = zf.namelist()
# Names must be unique
assert len(names) == len(set(names))
@pytest.mark.integration
@pytest.mark.requires_db
class TestBulkReprocessCloudOcr:
"""Tests for POST /api/files/bulk-reprocess-cloud-ocr endpoint."""
@patch("app.api.files.process_document")
def test_bulk_reprocess_cloud_ocr_success(self, mock_delay, client: TestClient, db_session, tmp_path):
"""Test bulk Cloud OCR reprocessing queues tasks."""
mock_task = MagicMock()
mock_task.id = "task-cloud-ocr-1"
mock_delay.delay.return_value = mock_task
pdf_file = tmp_path / "ocr_test.pdf"
pdf_file.write_bytes(b"PDF data")
file_record = FileRecord(
filehash="hash_ocr1",
original_filename="ocr_test.pdf",
local_filename=str(pdf_file),
file_size=8,
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
assert response.status_code == 200
data = response.json()
assert data["status"] == "success"
assert len(data["processed_files"]) == 1
assert data["errors"] == []
# Ensure force_cloud_ocr=True was passed
call_kwargs = mock_delay.delay.call_args.kwargs
assert call_kwargs.get("force_cloud_ocr") is True
def test_bulk_reprocess_cloud_ocr_no_file_on_disk(self, client: TestClient, db_session):
"""Test Cloud OCR bulk reprocess skips files not on disk."""
file_record = FileRecord(
filehash="hash_ocr2",
original_filename="missing.pdf",
local_filename="/nonexistent/missing.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
assert response.status_code == 200
data = response.json()
assert data["status"] == "error"
assert len(data["errors"]) == 1
def test_bulk_reprocess_cloud_ocr_no_files_found(self, client: TestClient, db_session):
"""Test Cloud OCR bulk reprocess with non-existent IDs."""
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[99999])
assert response.status_code == 404
@pytest.mark.integration
@pytest.mark.requires_db
class TestOcrQualityFilter:
"""Tests for the ocr_quality filter on the /files view."""
def test_ocr_quality_poor_filter(self, client: TestClient, db_session):
"""Files with a low ocr_quality_score appear when filtering poor."""
rec_poor = FileRecord(
filehash="hash_poor1",
original_filename="poor_quality.pdf",
local_filename="/tmp/poor_quality.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=40,
)
rec_good = FileRecord(
filehash="hash_good1",
original_filename="good_quality.pdf",
local_filename="/tmp/good_quality.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=95,
)
db_session.add_all([rec_poor, rec_good])
db_session.commit()
response = client.get("/files?ocr_quality=poor")
assert response.status_code == 200
assert "poor_quality.pdf" in response.text
assert "good_quality.pdf" not in response.text
def test_ocr_quality_good_filter(self, client: TestClient, db_session):
"""Files with a high ocr_quality_score appear when filtering good."""
rec_poor = FileRecord(
filehash="hash_poor2",
original_filename="poor_quality2.pdf",
local_filename="/tmp/poor_quality2.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=40,
)
rec_good = FileRecord(
filehash="hash_good2",
original_filename="good_quality2.pdf",
local_filename="/tmp/good_quality2.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=95,
)
db_session.add_all([rec_poor, rec_good])
db_session.commit()
response = client.get("/files?ocr_quality=good")
assert response.status_code == 200
assert "good_quality2.pdf" in response.text
assert "poor_quality2.pdf" not in response.text
def test_ocr_quality_unchecked_filter(self, client: TestClient, db_session):
"""Files with no score appear when filtering unchecked."""
rec_unchecked = FileRecord(
filehash="hash_unch1",
original_filename="unchecked.pdf",
local_filename="/tmp/unchecked.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=None,
)
rec_scored = FileRecord(
filehash="hash_scored1",
original_filename="scored.pdf",
local_filename="/tmp/scored.pdf",
file_size=1024,
mime_type="application/pdf",
ocr_quality_score=90,
)
db_session.add_all([rec_unchecked, rec_scored])
db_session.commit()
response = client.get("/files?ocr_quality=unchecked")
assert response.status_code == 200
assert "unchecked.pdf" in response.text
assert "scored.pdf" not in response.text
def test_ocr_quality_no_filter(self, client: TestClient, db_session):
"""All files appear when no ocr_quality filter is applied."""
rec = FileRecord(
filehash="hash_all1",
original_filename="all_files.pdf",
local_filename="/tmp/all_files.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(rec)
db_session.commit()
response = client.get("/files")
assert response.status_code == 200
assert "all_files.pdf" in response.text