Merge pull request #439 from christianlouis/copilot/add-bulk-file-operations
feat: bulk operations (delete, reprocess, Cloud OCR, ZIP download) + persisted OCR quality score filter
This commit is contained in:
@@ -2,14 +2,17 @@
|
||||
File-related API endpoints
|
||||
"""
|
||||
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import asc, desc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -429,6 +432,163 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/files/bulk-reprocess-cloud-ocr")
|
||||
@require_login
|
||||
def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: DbSession):
|
||||
"""
|
||||
Reprocess multiple files with forced Cloud OCR (Azure Document Intelligence).
|
||||
|
||||
Useful for re-running OCR on files with poor text quality or missing OCR text.
|
||||
"""
|
||||
try:
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
|
||||
task_ids = []
|
||||
processed_files = []
|
||||
errors = []
|
||||
|
||||
for file_record in file_records:
|
||||
try:
|
||||
source_file = None
|
||||
for path in [file_record.original_file_path, file_record.local_filename]:
|
||||
if path and os.path.exists(path):
|
||||
source_file = path
|
||||
break
|
||||
|
||||
if not source_file:
|
||||
errors.append(
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"error": "File not found on disk",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
task = process_document.delay(
|
||||
source_file,
|
||||
original_filename=file_record.original_filename,
|
||||
file_id=file_record.id,
|
||||
force_cloud_ocr=True,
|
||||
)
|
||||
task_ids.append(task.id)
|
||||
processed_files.append(
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"task_id": task.id,
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
f"Bulk Cloud OCR reprocessing: ID={file_record.id}, "
|
||||
f"Filename={file_record.original_filename}, TaskID={task.id}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error queuing Cloud OCR for file {file_record.id}: {str(e)}")
|
||||
errors.append(
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success" if processed_files else "error",
|
||||
"message": f"Successfully queued {len(processed_files)} files for Cloud OCR reprocessing",
|
||||
"processed_files": processed_files,
|
||||
"errors": errors,
|
||||
"task_ids": task_ids,
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/files/bulk-download")
|
||||
@require_login
|
||||
def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
"""
|
||||
Download multiple files as a single ZIP archive.
|
||||
|
||||
For each file, the processed version is preferred; falls back to the original.
|
||||
Files not found on disk are silently skipped.
|
||||
"""
|
||||
try:
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
|
||||
zip_buffer = io.BytesIO()
|
||||
added = 0
|
||||
seen_names: dict[str, int] = {}
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
workdir = settings.workdir
|
||||
processed_dir = os.path.join(workdir, "processed")
|
||||
|
||||
for file_record in file_records:
|
||||
# Resolve file path: processed first, then original/local
|
||||
base_filename = os.path.splitext(file_record.original_filename or "file")[0]
|
||||
candidate_paths = [
|
||||
file_record.processed_file_path,
|
||||
file_record.original_file_path,
|
||||
file_record.local_filename,
|
||||
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
|
||||
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
|
||||
]
|
||||
|
||||
file_path = None
|
||||
for path in candidate_paths:
|
||||
if path and os.path.exists(path):
|
||||
file_path = path
|
||||
break
|
||||
|
||||
if not file_path:
|
||||
logger.warning(f"Skipping file {file_record.id}: no file found on disk")
|
||||
continue
|
||||
|
||||
# Build a unique archive name to avoid collisions
|
||||
archive_name = file_record.original_filename or os.path.basename(file_path)
|
||||
if archive_name in seen_names:
|
||||
seen_names[archive_name] += 1
|
||||
stem, ext = os.path.splitext(archive_name)
|
||||
archive_name = f"{stem}_{seen_names[archive_name]}{ext}"
|
||||
else:
|
||||
seen_names[archive_name] = 0
|
||||
|
||||
zf.write(file_path, archive_name)
|
||||
added += 1
|
||||
|
||||
if added == 0:
|
||||
raise HTTPException(status_code=404, detail="None of the selected files could be found on disk")
|
||||
|
||||
zip_buffer.seek(0)
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
zip_filename = f"docuelevate_bulk_{timestamp}.zip"
|
||||
|
||||
logger.info(f"Bulk download: packed {added} file(s) into {zip_filename}")
|
||||
|
||||
return StreamingResponse(
|
||||
zip_buffer,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'},
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error creating bulk download ZIP: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error creating bulk download ZIP: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/reprocess")
|
||||
@require_login
|
||||
def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
||||
|
||||
@@ -122,6 +122,12 @@ def _run_schema_migrations(engine: Any) -> None:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
|
||||
logger.info("Migration complete: 'document_title' column added to files")
|
||||
|
||||
if "ocr_quality_score" not in columns:
|
||||
logger.info("Migrating files: adding 'ocr_quality_score' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
|
||||
logger.info("Migration complete: 'ocr_quality_score' column added to files")
|
||||
|
||||
# Migration: Drop unique index on filehash to allow duplicate records
|
||||
try:
|
||||
indexes = inspector.get_indexes("files")
|
||||
|
||||
@@ -58,6 +58,9 @@ class FileRecord(Base):
|
||||
# Full OCR/extracted text for full-text search and RAG
|
||||
ocr_text = Column(Text, nullable=True)
|
||||
|
||||
# AI-assessed quality score for the OCR/extracted text (0–100; NULL = not yet assessed)
|
||||
ocr_quality_score = Column(Integer, nullable=True)
|
||||
|
||||
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
|
||||
ai_metadata = Column(Text, nullable=True)
|
||||
|
||||
|
||||
@@ -442,6 +442,14 @@ def process_document(
|
||||
f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}"
|
||||
)
|
||||
|
||||
# Persist the quality score immediately so it's available for filtering
|
||||
# even if the file is later sent to OCR for re-processing.
|
||||
with SessionLocal() as _db:
|
||||
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if _rec:
|
||||
_rec.ocr_quality_score = quality_result.quality_score
|
||||
_db.commit()
|
||||
|
||||
if not quality_result.is_good_quality:
|
||||
# Poor quality: discard embedded text and re-OCR instead.
|
||||
# Pass the original embedded text so the OCR task can compare
|
||||
|
||||
@@ -21,11 +21,13 @@ from typing import Optional
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
|
||||
from app.utils.text_quality import compare_text_quality
|
||||
from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -241,6 +243,29 @@ def process_with_ocr(self, filename: str, file_id: Optional[int] = None, origina
|
||||
f"final text length: {len(final_text)} chars",
|
||||
)
|
||||
|
||||
# Score the final embedded text — the text that will land in ocr_text.
|
||||
# We always call check_text_quality() on final_text because:
|
||||
# - merge_ocr_results() may have AI-merged output from several engines
|
||||
# - compare_text_quality() scores are relative (not the same scale)
|
||||
# - The original may have been preferred, reversing the OCR output
|
||||
# OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS
|
||||
# so the quality AI call is always made.
|
||||
if file_id is not None:
|
||||
try:
|
||||
quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS)
|
||||
logger.info(
|
||||
f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, "
|
||||
f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}"
|
||||
)
|
||||
with SessionLocal() as _db:
|
||||
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if _rec:
|
||||
_rec.ocr_quality_score = quality_result.quality_score
|
||||
_db.commit()
|
||||
logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}")
|
||||
except Exception as _score_exc:
|
||||
logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}")
|
||||
|
||||
# Continue pipeline: rotate pages (if needed), then extract metadata
|
||||
rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id)
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ def files_page(
|
||||
date_to: Optional[str] = Query(None),
|
||||
storage_provider: Optional[str] = Query(None),
|
||||
tags: Optional[str] = Query(None),
|
||||
ocr_quality: Optional[str] = Query(None),
|
||||
):
|
||||
"""
|
||||
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
||||
@@ -94,6 +95,23 @@ def files_page(
|
||||
escaped_tag = tag.replace("%", r"\%").replace("_", r"\_")
|
||||
query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
|
||||
|
||||
# Apply OCR quality filter
|
||||
if ocr_quality == "poor":
|
||||
# Files scored below the configured threshold
|
||||
threshold = settings.text_quality_threshold
|
||||
query = query.filter(
|
||||
FileRecord.ocr_quality_score.isnot(None),
|
||||
FileRecord.ocr_quality_score < threshold,
|
||||
)
|
||||
elif ocr_quality == "good":
|
||||
threshold = settings.text_quality_threshold
|
||||
query = query.filter(
|
||||
FileRecord.ocr_quality_score.isnot(None),
|
||||
FileRecord.ocr_quality_score >= threshold,
|
||||
)
|
||||
elif ocr_quality == "unchecked":
|
||||
query = query.filter(FileRecord.ocr_quality_score.is_(None))
|
||||
|
||||
# Apply status filter (before pagination for correct counts)
|
||||
query = apply_status_filter(query, db, status)
|
||||
|
||||
@@ -158,6 +176,8 @@ def files_page(
|
||||
"date_to": date_to or "",
|
||||
"storage_provider": storage_provider or "",
|
||||
"tags": tags or "",
|
||||
"ocr_quality": ocr_quality or "",
|
||||
"ocr_quality_threshold": settings.text_quality_threshold,
|
||||
"mime_types": mime_types,
|
||||
"upload_concurrency": settings.upload_concurrency,
|
||||
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
||||
|
||||
+91
@@ -295,6 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting
|
||||
- `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`)
|
||||
- `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`)
|
||||
- `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`)
|
||||
- `ocr_quality` (optional): Filter by AI-assessed OCR quality score (`poor` = score below threshold, `good` = score at or above threshold, `unchecked` = not yet assessed). The threshold is configured via `TEXT_QUALITY_THRESHOLD` (default: 85).
|
||||
|
||||
All filters are combinable using AND logic.
|
||||
|
||||
@@ -461,6 +462,96 @@ Reprocess a specific file with forced Cloud OCR, regardless of embedded text qua
|
||||
|
||||
**Note**: This endpoint forces Azure Document Intelligence OCR processing even if the PDF contains embedded text. The original file (if available) is used for reprocessing to ensure the highest quality result.
|
||||
|
||||
### Bulk Operations
|
||||
|
||||
**POST** `/api/files/bulk-delete`
|
||||
|
||||
Delete multiple file records in a single request.
|
||||
|
||||
**Request body**: JSON array of file IDs
|
||||
|
||||
```bash
|
||||
curl -X POST "http://<your-instance>/api/files/bulk-delete" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '[1, 2, 3]'
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"message": "Successfully deleted 3 file records",
|
||||
"deleted_ids": [1, 2, 3]
|
||||
}
|
||||
```
|
||||
|
||||
**Error Responses**:
|
||||
- `403`: File deletion is disabled in configuration
|
||||
- `404`: No files found with the provided IDs
|
||||
|
||||
---
|
||||
|
||||
**POST** `/api/files/bulk-reprocess`
|
||||
|
||||
Queue multiple files for full reprocessing.
|
||||
|
||||
**Request body**: JSON array of file IDs
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"message": "Successfully queued 2 files for reprocessing",
|
||||
"processed_files": [
|
||||
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"},
|
||||
{"file_id": 2, "filename": "b.pdf", "task_id": "def456"}
|
||||
],
|
||||
"errors": [],
|
||||
"task_ids": ["abc123", "def456"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**POST** `/api/files/bulk-reprocess-cloud-ocr`
|
||||
|
||||
Queue multiple files for reprocessing with forced Cloud OCR (Azure Document Intelligence). Useful for files that have missing or low-quality OCR text.
|
||||
|
||||
**Request body**: JSON array of file IDs
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"message": "Successfully queued 2 files for Cloud OCR reprocessing",
|
||||
"processed_files": [
|
||||
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"}
|
||||
],
|
||||
"errors": [],
|
||||
"task_ids": ["abc123"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**POST** `/api/files/bulk-download`
|
||||
|
||||
Download multiple files as a single ZIP archive. For each file, the processed version is preferred; falls back to the original. Files not found on disk are silently skipped.
|
||||
|
||||
**Request body**: JSON array of file IDs
|
||||
|
||||
**Response**: `application/zip` stream with `Content-Disposition: attachment; filename="docuelevate_bulk_<timestamp>.zip"`
|
||||
|
||||
```bash
|
||||
curl -X POST "http://<your-instance>/api/files/bulk-download" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '[1, 2, 3]' \
|
||||
--output bulk_download.zip
|
||||
```
|
||||
|
||||
**Error Responses**:
|
||||
- `404`: No files found with the provided IDs, or none of the selected files exist on disk
|
||||
|
||||
### File Preview
|
||||
|
||||
**GET** `/api/files/{file_id}/preview`
|
||||
|
||||
@@ -533,6 +533,16 @@
|
||||
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label for="ocr_quality">OCR Quality</label>
|
||||
<select id="ocr_quality" name="ocr_quality" aria-label="Filter by OCR quality score">
|
||||
<option value="">All Files</option>
|
||||
<option value="poor" {% if ocr_quality == "poor" %}selected{% endif %} aria-label="Poor quality, score less than {{ ocr_quality_threshold }}">Poor (score < {{ ocr_quality_threshold }})</option>
|
||||
<option value="good" {% if ocr_quality == "good" %}selected{% endif %} aria-label="Good quality, score at least {{ ocr_quality_threshold }}">Good (score ≥ {{ ocr_quality_threshold }})</option>
|
||||
<option value="unchecked" {% if ocr_quality == "unchecked" %}selected{% endif %} aria-label="Not yet assessed">Not yet assessed</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label> </label>
|
||||
<button type="submit">Apply Filters</button>
|
||||
@@ -616,6 +626,12 @@
|
||||
<button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||
<i class="fas fa-sync" aria-hidden="true"></i> Reprocess Selected
|
||||
</button>
|
||||
<button type="button" onclick="bulkReprocessCloudOcr()" style="padding: 0.5rem 1rem; background-color: #805ad5; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||
<i class="fas fa-cloud" aria-hidden="true"></i> Re-run Cloud OCR
|
||||
</button>
|
||||
<button type="button" onclick="bulkDownload()" style="padding: 0.5rem 1rem; background-color: #38a169; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||
<i class="fas fa-file-archive" aria-hidden="true"></i> Download as ZIP
|
||||
</button>
|
||||
<button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||
<i class="fas fa-trash" aria-hidden="true"></i> Delete Selected
|
||||
</button>
|
||||
@@ -1285,6 +1301,86 @@
|
||||
});
|
||||
}
|
||||
|
||||
function bulkReprocessCloudOcr() {
|
||||
const fileIds = getSelectedFileIds();
|
||||
if (fileIds.length === 0) {
|
||||
alert('Please select files to re-run Cloud OCR on');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!confirm(`Are you sure you want to re-run Cloud OCR on ${fileIds.length} file(s)?`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
fetch('/api/files/bulk-reprocess-cloud-ocr', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(fileIds)
|
||||
})
|
||||
.then(response => {
|
||||
if (!response.ok) {
|
||||
return response.json().then(err => {
|
||||
throw new Error(err.detail || 'Failed to queue Cloud OCR');
|
||||
});
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then(data => {
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n');
|
||||
alert(`${data.message}\n\nErrors:\n${errorMsg}`);
|
||||
} else {
|
||||
alert(data.message);
|
||||
}
|
||||
clearSelection();
|
||||
window.location.reload();
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error:', error);
|
||||
alert(`Error queuing Cloud OCR: ${error.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
function bulkDownload() {
|
||||
const fileIds = getSelectedFileIds();
|
||||
if (fileIds.length === 0) {
|
||||
alert('Please select files to download');
|
||||
return;
|
||||
}
|
||||
|
||||
fetch('/api/files/bulk-download', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(fileIds)
|
||||
})
|
||||
.then(response => {
|
||||
if (!response.ok) {
|
||||
return response.json().then(err => {
|
||||
throw new Error(err.detail || 'Failed to create ZIP download');
|
||||
});
|
||||
}
|
||||
return response.blob();
|
||||
})
|
||||
.then(blob => {
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `docuelevate_bulk_${Date.now()}.zip`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
window.URL.revokeObjectURL(url);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error:', error);
|
||||
alert(`Error downloading files: ${error.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ===== Drag-and-Drop Upload Functionality =====
|
||||
const dropOverlay = document.getElementById('dropOverlay');
|
||||
const uploadModal = document.getElementById('uploadModal');
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
Tests for bulk file operations (delete and reprocess).
|
||||
"""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -293,3 +295,265 @@ class TestStatusFilter:
|
||||
response = client.get("/files?status=failed")
|
||||
assert response.status_code == 200
|
||||
assert "failed.pdf" in response.text
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.requires_db
|
||||
class TestBulkDownload:
|
||||
"""Tests for POST /api/files/bulk-download endpoint."""
|
||||
|
||||
def test_bulk_download_no_files_found(self, client: TestClient, db_session):
|
||||
"""Test bulk download with non-existent IDs."""
|
||||
response = client.post("/api/files/bulk-download", json=[99999, 99998])
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_bulk_download_files_not_on_disk(self, client: TestClient, db_session):
|
||||
"""Test bulk download when files are not found on disk."""
|
||||
file_record = FileRecord(
|
||||
filehash="hash_dl1",
|
||||
original_filename="nodisk.pdf",
|
||||
local_filename="/nonexistent/nodisk.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
response = client.post("/api/files/bulk-download", json=[file_record.id])
|
||||
assert response.status_code == 404
|
||||
assert "None of the selected files" in response.json()["detail"]
|
||||
|
||||
def test_bulk_download_success(self, client: TestClient, db_session, tmp_path):
|
||||
"""Test successful bulk download returns a ZIP archive."""
|
||||
# Create a real file on disk
|
||||
pdf_file = tmp_path / "sample.pdf"
|
||||
pdf_file.write_bytes(b"PDF content")
|
||||
|
||||
file_record = FileRecord(
|
||||
filehash="hash_dl2",
|
||||
original_filename="sample.pdf",
|
||||
local_filename=str(pdf_file),
|
||||
processed_file_path=str(pdf_file),
|
||||
file_size=len(b"PDF content"),
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
response = client.post("/api/files/bulk-download", json=[file_record.id])
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "application/zip"
|
||||
assert "attachment" in response.headers["content-disposition"]
|
||||
assert ".zip" in response.headers["content-disposition"]
|
||||
|
||||
def test_bulk_download_multiple_files(self, client: TestClient, db_session, tmp_path):
|
||||
"""Test bulk download with multiple files produces a valid ZIP."""
|
||||
ids = []
|
||||
for i in range(3):
|
||||
f = tmp_path / f"file{i}.pdf"
|
||||
f.write_bytes(f"content {i}".encode())
|
||||
rec = FileRecord(
|
||||
filehash=f"hash_multi_{i}",
|
||||
original_filename=f"file{i}.pdf",
|
||||
local_filename=str(f),
|
||||
processed_file_path=str(f),
|
||||
file_size=len(f"content {i}".encode()),
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(rec)
|
||||
db_session.commit()
|
||||
ids.append(rec.id)
|
||||
|
||||
response = client.post("/api/files/bulk-download", json=ids)
|
||||
assert response.status_code == 200
|
||||
|
||||
zip_data = io.BytesIO(response.content)
|
||||
with zipfile.ZipFile(zip_data) as zf:
|
||||
names = zf.namelist()
|
||||
assert len(names) == 3
|
||||
|
||||
def test_bulk_download_duplicate_filenames(self, client: TestClient, db_session, tmp_path):
|
||||
"""Test bulk download disambiguates duplicate filenames."""
|
||||
ids = []
|
||||
for i in range(2):
|
||||
f = tmp_path / f"dup_{i}.pdf"
|
||||
f.write_bytes(b"data")
|
||||
rec = FileRecord(
|
||||
filehash=f"hash_dup_{i}",
|
||||
original_filename="dup.pdf", # same name
|
||||
local_filename=str(f),
|
||||
processed_file_path=str(f),
|
||||
file_size=4,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(rec)
|
||||
db_session.commit()
|
||||
ids.append(rec.id)
|
||||
|
||||
response = client.post("/api/files/bulk-download", json=ids)
|
||||
assert response.status_code == 200
|
||||
zip_data = io.BytesIO(response.content)
|
||||
with zipfile.ZipFile(zip_data) as zf:
|
||||
names = zf.namelist()
|
||||
# Names must be unique
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.requires_db
|
||||
class TestBulkReprocessCloudOcr:
|
||||
"""Tests for POST /api/files/bulk-reprocess-cloud-ocr endpoint."""
|
||||
|
||||
@patch("app.api.files.process_document")
|
||||
def test_bulk_reprocess_cloud_ocr_success(self, mock_delay, client: TestClient, db_session, tmp_path):
|
||||
"""Test bulk Cloud OCR reprocessing queues tasks."""
|
||||
mock_task = MagicMock()
|
||||
mock_task.id = "task-cloud-ocr-1"
|
||||
mock_delay.delay.return_value = mock_task
|
||||
|
||||
pdf_file = tmp_path / "ocr_test.pdf"
|
||||
pdf_file.write_bytes(b"PDF data")
|
||||
|
||||
file_record = FileRecord(
|
||||
filehash="hash_ocr1",
|
||||
original_filename="ocr_test.pdf",
|
||||
local_filename=str(pdf_file),
|
||||
file_size=8,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert len(data["processed_files"]) == 1
|
||||
assert data["errors"] == []
|
||||
|
||||
# Ensure force_cloud_ocr=True was passed
|
||||
call_kwargs = mock_delay.delay.call_args.kwargs
|
||||
assert call_kwargs.get("force_cloud_ocr") is True
|
||||
|
||||
def test_bulk_reprocess_cloud_ocr_no_file_on_disk(self, client: TestClient, db_session):
|
||||
"""Test Cloud OCR bulk reprocess skips files not on disk."""
|
||||
file_record = FileRecord(
|
||||
filehash="hash_ocr2",
|
||||
original_filename="missing.pdf",
|
||||
local_filename="/nonexistent/missing.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "error"
|
||||
assert len(data["errors"]) == 1
|
||||
|
||||
def test_bulk_reprocess_cloud_ocr_no_files_found(self, client: TestClient, db_session):
|
||||
"""Test Cloud OCR bulk reprocess with non-existent IDs."""
|
||||
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[99999])
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.requires_db
|
||||
class TestOcrQualityFilter:
|
||||
"""Tests for the ocr_quality filter on the /files view."""
|
||||
|
||||
def test_ocr_quality_poor_filter(self, client: TestClient, db_session):
|
||||
"""Files with a low ocr_quality_score appear when filtering poor."""
|
||||
rec_poor = FileRecord(
|
||||
filehash="hash_poor1",
|
||||
original_filename="poor_quality.pdf",
|
||||
local_filename="/tmp/poor_quality.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=40,
|
||||
)
|
||||
rec_good = FileRecord(
|
||||
filehash="hash_good1",
|
||||
original_filename="good_quality.pdf",
|
||||
local_filename="/tmp/good_quality.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=95,
|
||||
)
|
||||
db_session.add_all([rec_poor, rec_good])
|
||||
db_session.commit()
|
||||
|
||||
response = client.get("/files?ocr_quality=poor")
|
||||
assert response.status_code == 200
|
||||
assert "poor_quality.pdf" in response.text
|
||||
assert "good_quality.pdf" not in response.text
|
||||
|
||||
def test_ocr_quality_good_filter(self, client: TestClient, db_session):
|
||||
"""Files with a high ocr_quality_score appear when filtering good."""
|
||||
rec_poor = FileRecord(
|
||||
filehash="hash_poor2",
|
||||
original_filename="poor_quality2.pdf",
|
||||
local_filename="/tmp/poor_quality2.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=40,
|
||||
)
|
||||
rec_good = FileRecord(
|
||||
filehash="hash_good2",
|
||||
original_filename="good_quality2.pdf",
|
||||
local_filename="/tmp/good_quality2.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=95,
|
||||
)
|
||||
db_session.add_all([rec_poor, rec_good])
|
||||
db_session.commit()
|
||||
|
||||
response = client.get("/files?ocr_quality=good")
|
||||
assert response.status_code == 200
|
||||
assert "good_quality2.pdf" in response.text
|
||||
assert "poor_quality2.pdf" not in response.text
|
||||
|
||||
def test_ocr_quality_unchecked_filter(self, client: TestClient, db_session):
|
||||
"""Files with no score appear when filtering unchecked."""
|
||||
rec_unchecked = FileRecord(
|
||||
filehash="hash_unch1",
|
||||
original_filename="unchecked.pdf",
|
||||
local_filename="/tmp/unchecked.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=None,
|
||||
)
|
||||
rec_scored = FileRecord(
|
||||
filehash="hash_scored1",
|
||||
original_filename="scored.pdf",
|
||||
local_filename="/tmp/scored.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
ocr_quality_score=90,
|
||||
)
|
||||
db_session.add_all([rec_unchecked, rec_scored])
|
||||
db_session.commit()
|
||||
|
||||
response = client.get("/files?ocr_quality=unchecked")
|
||||
assert response.status_code == 200
|
||||
assert "unchecked.pdf" in response.text
|
||||
assert "scored.pdf" not in response.text
|
||||
|
||||
def test_ocr_quality_no_filter(self, client: TestClient, db_session):
|
||||
"""All files appear when no ocr_quality filter is applied."""
|
||||
rec = FileRecord(
|
||||
filehash="hash_all1",
|
||||
original_filename="all_files.pdf",
|
||||
local_filename="/tmp/all_files.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(rec)
|
||||
db_session.commit()
|
||||
|
||||
response = client.get("/files")
|
||||
assert response.status_code == 200
|
||||
assert "all_files.pdf" in response.text
|
||||
|
||||
Reference in New Issue
Block a user