feat: add bulk download, cloud OCR, and basic OCR quality filter
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -2,14 +2,17 @@
|
|||||||
File-related API endpoints
|
File-related API endpoints
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import io
|
||||||
import logging
|
import logging
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
|
import zipfile
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Annotated, List, Optional
|
from typing import Annotated, List, Optional
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
|
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
from sqlalchemy import asc, desc
|
from sqlalchemy import asc, desc
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -429,6 +432,163 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
|||||||
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
|
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/files/bulk-reprocess-cloud-ocr")
|
||||||
|
@require_login
|
||||||
|
def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: DbSession):
|
||||||
|
"""
|
||||||
|
Reprocess multiple files with forced Cloud OCR (Azure Document Intelligence).
|
||||||
|
|
||||||
|
Useful for re-running OCR on files with poor text quality or missing OCR text.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||||
|
|
||||||
|
if not file_records:
|
||||||
|
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||||
|
|
||||||
|
task_ids = []
|
||||||
|
processed_files = []
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
for file_record in file_records:
|
||||||
|
try:
|
||||||
|
source_file = None
|
||||||
|
for path in [file_record.original_file_path, file_record.local_filename]:
|
||||||
|
if path and os.path.exists(path):
|
||||||
|
source_file = path
|
||||||
|
break
|
||||||
|
|
||||||
|
if not source_file:
|
||||||
|
errors.append(
|
||||||
|
{
|
||||||
|
"file_id": file_record.id,
|
||||||
|
"filename": file_record.original_filename,
|
||||||
|
"error": "File not found on disk",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
task = process_document.delay(
|
||||||
|
source_file,
|
||||||
|
original_filename=file_record.original_filename,
|
||||||
|
file_id=file_record.id,
|
||||||
|
force_cloud_ocr=True,
|
||||||
|
)
|
||||||
|
task_ids.append(task.id)
|
||||||
|
processed_files.append(
|
||||||
|
{
|
||||||
|
"file_id": file_record.id,
|
||||||
|
"filename": file_record.original_filename,
|
||||||
|
"task_id": task.id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f"Bulk Cloud OCR reprocessing: ID={file_record.id}, "
|
||||||
|
f"Filename={file_record.original_filename}, TaskID={task.id}"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Error queuing Cloud OCR for file {file_record.id}: {str(e)}")
|
||||||
|
errors.append(
|
||||||
|
{
|
||||||
|
"file_id": file_record.id,
|
||||||
|
"filename": file_record.original_filename,
|
||||||
|
"error": str(e),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "success" if processed_files else "error",
|
||||||
|
"message": f"Successfully queued {len(processed_files)} files for Cloud OCR reprocessing",
|
||||||
|
"processed_files": processed_files,
|
||||||
|
"errors": errors,
|
||||||
|
"task_ids": task_ids,
|
||||||
|
}
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files with Cloud OCR: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/files/bulk-download")
|
||||||
|
@require_login
|
||||||
|
def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
|
||||||
|
"""
|
||||||
|
Download multiple files as a single ZIP archive.
|
||||||
|
|
||||||
|
For each file, the processed version is preferred; falls back to the original.
|
||||||
|
Files not found on disk are silently skipped.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||||
|
|
||||||
|
if not file_records:
|
||||||
|
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||||
|
|
||||||
|
zip_buffer = io.BytesIO()
|
||||||
|
added = 0
|
||||||
|
seen_names: dict[str, int] = {}
|
||||||
|
|
||||||
|
with zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
workdir = settings.workdir
|
||||||
|
processed_dir = os.path.join(workdir, "processed")
|
||||||
|
|
||||||
|
for file_record in file_records:
|
||||||
|
# Resolve file path: processed first, then original/local
|
||||||
|
base_filename = os.path.splitext(file_record.original_filename or "file")[0]
|
||||||
|
candidate_paths = [
|
||||||
|
file_record.processed_file_path,
|
||||||
|
file_record.original_file_path,
|
||||||
|
file_record.local_filename,
|
||||||
|
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
|
||||||
|
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
|
||||||
|
]
|
||||||
|
|
||||||
|
file_path = None
|
||||||
|
for path in candidate_paths:
|
||||||
|
if path and os.path.exists(path):
|
||||||
|
file_path = path
|
||||||
|
break
|
||||||
|
|
||||||
|
if not file_path:
|
||||||
|
logger.warning(f"Skipping file {file_record.id}: no file found on disk")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Build a unique archive name to avoid collisions
|
||||||
|
archive_name = file_record.original_filename or os.path.basename(file_path)
|
||||||
|
if archive_name in seen_names:
|
||||||
|
seen_names[archive_name] += 1
|
||||||
|
stem, ext = os.path.splitext(archive_name)
|
||||||
|
archive_name = f"{stem}_{seen_names[archive_name]}{ext}"
|
||||||
|
else:
|
||||||
|
seen_names[archive_name] = 0
|
||||||
|
|
||||||
|
zf.write(file_path, archive_name)
|
||||||
|
added += 1
|
||||||
|
|
||||||
|
if added == 0:
|
||||||
|
raise HTTPException(status_code=404, detail="None of the selected files could be found on disk")
|
||||||
|
|
||||||
|
zip_buffer.seek(0)
|
||||||
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||||
|
zip_filename = f"docuelevate_bulk_{timestamp}.zip"
|
||||||
|
|
||||||
|
logger.info(f"Bulk download: packed {added} file(s) into {zip_filename}")
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
zip_buffer,
|
||||||
|
media_type="application/zip",
|
||||||
|
headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'},
|
||||||
|
)
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Error creating bulk download ZIP: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail=f"Error creating bulk download ZIP: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/files/{file_id}/reprocess")
|
@router.post("/files/{file_id}/reprocess")
|
||||||
@require_login
|
@require_login
|
||||||
def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ def files_page(
|
|||||||
date_to: Optional[str] = Query(None),
|
date_to: Optional[str] = Query(None),
|
||||||
storage_provider: Optional[str] = Query(None),
|
storage_provider: Optional[str] = Query(None),
|
||||||
tags: Optional[str] = Query(None),
|
tags: Optional[str] = Query(None),
|
||||||
|
ocr_quality: Optional[str] = Query(None),
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
||||||
@@ -94,6 +95,15 @@ def files_page(
|
|||||||
escaped_tag = tag.replace("%", r"\%").replace("_", r"\_")
|
escaped_tag = tag.replace("%", r"\%").replace("_", r"\_")
|
||||||
query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
|
query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%"))
|
||||||
|
|
||||||
|
# Apply OCR quality filter
|
||||||
|
if ocr_quality == "no_ocr":
|
||||||
|
query = query.filter((FileRecord.ocr_text.is_(None)) | (FileRecord.ocr_text == ""))
|
||||||
|
elif ocr_quality == "has_ocr":
|
||||||
|
query = query.filter(
|
||||||
|
FileRecord.ocr_text.isnot(None),
|
||||||
|
FileRecord.ocr_text != "",
|
||||||
|
)
|
||||||
|
|
||||||
# Apply status filter (before pagination for correct counts)
|
# Apply status filter (before pagination for correct counts)
|
||||||
query = apply_status_filter(query, db, status)
|
query = apply_status_filter(query, db, status)
|
||||||
|
|
||||||
@@ -158,6 +168,7 @@ def files_page(
|
|||||||
"date_to": date_to or "",
|
"date_to": date_to or "",
|
||||||
"storage_provider": storage_provider or "",
|
"storage_provider": storage_provider or "",
|
||||||
"tags": tags or "",
|
"tags": tags or "",
|
||||||
|
"ocr_quality": ocr_quality or "",
|
||||||
"mime_types": mime_types,
|
"mime_types": mime_types,
|
||||||
"upload_concurrency": settings.upload_concurrency,
|
"upload_concurrency": settings.upload_concurrency,
|
||||||
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
||||||
|
|||||||
+91
@@ -295,6 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting
|
|||||||
- `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`)
|
- `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`)
|
||||||
- `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`)
|
- `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`)
|
||||||
- `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`)
|
- `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`)
|
||||||
|
- `ocr_quality` (optional): Filter by OCR text availability (`no_ocr` = files without OCR text, `has_ocr` = files with OCR text)
|
||||||
|
|
||||||
All filters are combinable using AND logic.
|
All filters are combinable using AND logic.
|
||||||
|
|
||||||
@@ -461,6 +462,96 @@ Reprocess a specific file with forced Cloud OCR, regardless of embedded text qua
|
|||||||
|
|
||||||
**Note**: This endpoint forces Azure Document Intelligence OCR processing even if the PDF contains embedded text. The original file (if available) is used for reprocessing to ensure the highest quality result.
|
**Note**: This endpoint forces Azure Document Intelligence OCR processing even if the PDF contains embedded text. The original file (if available) is used for reprocessing to ensure the highest quality result.
|
||||||
|
|
||||||
|
### Bulk Operations
|
||||||
|
|
||||||
|
**POST** `/api/files/bulk-delete`
|
||||||
|
|
||||||
|
Delete multiple file records in a single request.
|
||||||
|
|
||||||
|
**Request body**: JSON array of file IDs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST "http://<your-instance>/api/files/bulk-delete" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '[1, 2, 3]'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "success",
|
||||||
|
"message": "Successfully deleted 3 file records",
|
||||||
|
"deleted_ids": [1, 2, 3]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Error Responses**:
|
||||||
|
- `403`: File deletion is disabled in configuration
|
||||||
|
- `404`: No files found with the provided IDs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**POST** `/api/files/bulk-reprocess`
|
||||||
|
|
||||||
|
Queue multiple files for full reprocessing.
|
||||||
|
|
||||||
|
**Request body**: JSON array of file IDs
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "success",
|
||||||
|
"message": "Successfully queued 2 files for reprocessing",
|
||||||
|
"processed_files": [
|
||||||
|
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"},
|
||||||
|
{"file_id": 2, "filename": "b.pdf", "task_id": "def456"}
|
||||||
|
],
|
||||||
|
"errors": [],
|
||||||
|
"task_ids": ["abc123", "def456"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**POST** `/api/files/bulk-reprocess-cloud-ocr`
|
||||||
|
|
||||||
|
Queue multiple files for reprocessing with forced Cloud OCR (Azure Document Intelligence). Useful for files that have missing or low-quality OCR text.
|
||||||
|
|
||||||
|
**Request body**: JSON array of file IDs
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "success",
|
||||||
|
"message": "Successfully queued 2 files for Cloud OCR reprocessing",
|
||||||
|
"processed_files": [
|
||||||
|
{"file_id": 1, "filename": "a.pdf", "task_id": "abc123"}
|
||||||
|
],
|
||||||
|
"errors": [],
|
||||||
|
"task_ids": ["abc123"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**POST** `/api/files/bulk-download`
|
||||||
|
|
||||||
|
Download multiple files as a single ZIP archive. For each file, the processed version is preferred; falls back to the original. Files not found on disk are silently skipped.
|
||||||
|
|
||||||
|
**Request body**: JSON array of file IDs
|
||||||
|
|
||||||
|
**Response**: `application/zip` stream with `Content-Disposition: attachment; filename="docuelevate_bulk_<timestamp>.zip"`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST "http://<your-instance>/api/files/bulk-download" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '[1, 2, 3]' \
|
||||||
|
--output bulk_download.zip
|
||||||
|
```
|
||||||
|
|
||||||
|
**Error Responses**:
|
||||||
|
- `404`: No files found with the provided IDs, or none of the selected files exist on disk
|
||||||
|
|
||||||
### File Preview
|
### File Preview
|
||||||
|
|
||||||
**GET** `/api/files/{file_id}/preview`
|
**GET** `/api/files/{file_id}/preview`
|
||||||
|
|||||||
@@ -533,6 +533,15 @@
|
|||||||
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
|
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="filter-item">
|
||||||
|
<label for="ocr_quality">OCR Quality</label>
|
||||||
|
<select id="ocr_quality" name="ocr_quality" aria-label="Filter by OCR quality">
|
||||||
|
<option value="">All Files</option>
|
||||||
|
<option value="no_ocr" {% if ocr_quality == "no_ocr" %}selected{% endif %}>No OCR Text</option>
|
||||||
|
<option value="has_ocr" {% if ocr_quality == "has_ocr" %}selected{% endif %}>Has OCR Text</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="filter-item">
|
<div class="filter-item">
|
||||||
<label> </label>
|
<label> </label>
|
||||||
<button type="submit">Apply Filters</button>
|
<button type="submit">Apply Filters</button>
|
||||||
@@ -616,6 +625,12 @@
|
|||||||
<button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
<button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||||
<i class="fas fa-sync" aria-hidden="true"></i> Reprocess Selected
|
<i class="fas fa-sync" aria-hidden="true"></i> Reprocess Selected
|
||||||
</button>
|
</button>
|
||||||
|
<button type="button" onclick="bulkReprocessCloudOcr()" style="padding: 0.5rem 1rem; background-color: #805ad5; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||||
|
<i class="fas fa-cloud" aria-hidden="true"></i> Re-run Cloud OCR
|
||||||
|
</button>
|
||||||
|
<button type="button" onclick="bulkDownload()" style="padding: 0.5rem 1rem; background-color: #38a169; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||||
|
<i class="fas fa-file-archive" aria-hidden="true"></i> Download as ZIP
|
||||||
|
</button>
|
||||||
<button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
<button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600; min-height: 44px;">
|
||||||
<i class="fas fa-trash" aria-hidden="true"></i> Delete Selected
|
<i class="fas fa-trash" aria-hidden="true"></i> Delete Selected
|
||||||
</button>
|
</button>
|
||||||
@@ -1285,6 +1300,86 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function bulkReprocessCloudOcr() {
|
||||||
|
const fileIds = getSelectedFileIds();
|
||||||
|
if (fileIds.length === 0) {
|
||||||
|
alert('Please select files to re-run Cloud OCR on');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!confirm(`Are you sure you want to re-run Cloud OCR on ${fileIds.length} file(s)?`)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch('/api/files/bulk-reprocess-cloud-ocr', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(fileIds)
|
||||||
|
})
|
||||||
|
.then(response => {
|
||||||
|
if (!response.ok) {
|
||||||
|
return response.json().then(err => {
|
||||||
|
throw new Error(err.detail || 'Failed to queue Cloud OCR');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
})
|
||||||
|
.then(data => {
|
||||||
|
if (data.errors && data.errors.length > 0) {
|
||||||
|
const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n');
|
||||||
|
alert(`${data.message}\n\nErrors:\n${errorMsg}`);
|
||||||
|
} else {
|
||||||
|
alert(data.message);
|
||||||
|
}
|
||||||
|
clearSelection();
|
||||||
|
window.location.reload();
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('Error:', error);
|
||||||
|
alert(`Error queuing Cloud OCR: ${error.message}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function bulkDownload() {
|
||||||
|
const fileIds = getSelectedFileIds();
|
||||||
|
if (fileIds.length === 0) {
|
||||||
|
alert('Please select files to download');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch('/api/files/bulk-download', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(fileIds)
|
||||||
|
})
|
||||||
|
.then(response => {
|
||||||
|
if (!response.ok) {
|
||||||
|
return response.json().then(err => {
|
||||||
|
throw new Error(err.detail || 'Failed to create ZIP download');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return response.blob();
|
||||||
|
})
|
||||||
|
.then(blob => {
|
||||||
|
const url = window.URL.createObjectURL(blob);
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = url;
|
||||||
|
a.download = `docuelevate_bulk_${Date.now()}.zip`;
|
||||||
|
document.body.appendChild(a);
|
||||||
|
a.click();
|
||||||
|
a.remove();
|
||||||
|
window.URL.revokeObjectURL(url);
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('Error:', error);
|
||||||
|
alert(`Error downloading files: ${error.message}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// ===== Drag-and-Drop Upload Functionality =====
|
// ===== Drag-and-Drop Upload Functionality =====
|
||||||
const dropOverlay = document.getElementById('dropOverlay');
|
const dropOverlay = document.getElementById('dropOverlay');
|
||||||
const uploadModal = document.getElementById('uploadModal');
|
const uploadModal = document.getElementById('uploadModal');
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
Tests for bulk file operations (delete and reprocess).
|
Tests for bulk file operations (delete and reprocess).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import io
|
||||||
|
import zipfile
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -293,3 +295,239 @@ class TestStatusFilter:
|
|||||||
response = client.get("/files?status=failed")
|
response = client.get("/files?status=failed")
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert "failed.pdf" in response.text
|
assert "failed.pdf" in response.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.requires_db
|
||||||
|
class TestBulkDownload:
|
||||||
|
"""Tests for POST /api/files/bulk-download endpoint."""
|
||||||
|
|
||||||
|
def test_bulk_download_no_files_found(self, client: TestClient, db_session):
|
||||||
|
"""Test bulk download with non-existent IDs."""
|
||||||
|
response = client.post("/api/files/bulk-download", json=[99999, 99998])
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
def test_bulk_download_files_not_on_disk(self, client: TestClient, db_session):
|
||||||
|
"""Test bulk download when files are not found on disk."""
|
||||||
|
file_record = FileRecord(
|
||||||
|
filehash="hash_dl1",
|
||||||
|
original_filename="nodisk.pdf",
|
||||||
|
local_filename="/nonexistent/nodisk.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(file_record)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-download", json=[file_record.id])
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert "None of the selected files" in response.json()["detail"]
|
||||||
|
|
||||||
|
def test_bulk_download_success(self, client: TestClient, db_session, tmp_path):
|
||||||
|
"""Test successful bulk download returns a ZIP archive."""
|
||||||
|
# Create a real file on disk
|
||||||
|
pdf_file = tmp_path / "sample.pdf"
|
||||||
|
pdf_file.write_bytes(b"PDF content")
|
||||||
|
|
||||||
|
file_record = FileRecord(
|
||||||
|
filehash="hash_dl2",
|
||||||
|
original_filename="sample.pdf",
|
||||||
|
local_filename=str(pdf_file),
|
||||||
|
processed_file_path=str(pdf_file),
|
||||||
|
file_size=len(b"PDF content"),
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(file_record)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-download", json=[file_record.id])
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.headers["content-type"] == "application/zip"
|
||||||
|
assert "attachment" in response.headers["content-disposition"]
|
||||||
|
assert ".zip" in response.headers["content-disposition"]
|
||||||
|
|
||||||
|
def test_bulk_download_multiple_files(self, client: TestClient, db_session, tmp_path):
|
||||||
|
"""Test bulk download with multiple files produces a valid ZIP."""
|
||||||
|
ids = []
|
||||||
|
for i in range(3):
|
||||||
|
f = tmp_path / f"file{i}.pdf"
|
||||||
|
f.write_bytes(f"content {i}".encode())
|
||||||
|
rec = FileRecord(
|
||||||
|
filehash=f"hash_multi_{i}",
|
||||||
|
original_filename=f"file{i}.pdf",
|
||||||
|
local_filename=str(f),
|
||||||
|
processed_file_path=str(f),
|
||||||
|
file_size=len(f"content {i}".encode()),
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(rec)
|
||||||
|
db_session.commit()
|
||||||
|
ids.append(rec.id)
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-download", json=ids)
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
zip_data = io.BytesIO(response.content)
|
||||||
|
with zipfile.ZipFile(zip_data) as zf:
|
||||||
|
names = zf.namelist()
|
||||||
|
assert len(names) == 3
|
||||||
|
|
||||||
|
def test_bulk_download_duplicate_filenames(self, client: TestClient, db_session, tmp_path):
|
||||||
|
"""Test bulk download disambiguates duplicate filenames."""
|
||||||
|
ids = []
|
||||||
|
for i in range(2):
|
||||||
|
f = tmp_path / f"dup_{i}.pdf"
|
||||||
|
f.write_bytes(b"data")
|
||||||
|
rec = FileRecord(
|
||||||
|
filehash=f"hash_dup_{i}",
|
||||||
|
original_filename="dup.pdf", # same name
|
||||||
|
local_filename=str(f),
|
||||||
|
processed_file_path=str(f),
|
||||||
|
file_size=4,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(rec)
|
||||||
|
db_session.commit()
|
||||||
|
ids.append(rec.id)
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-download", json=ids)
|
||||||
|
assert response.status_code == 200
|
||||||
|
zip_data = io.BytesIO(response.content)
|
||||||
|
with zipfile.ZipFile(zip_data) as zf:
|
||||||
|
names = zf.namelist()
|
||||||
|
# Names must be unique
|
||||||
|
assert len(names) == len(set(names))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.requires_db
|
||||||
|
class TestBulkReprocessCloudOcr:
|
||||||
|
"""Tests for POST /api/files/bulk-reprocess-cloud-ocr endpoint."""
|
||||||
|
|
||||||
|
@patch("app.api.files.process_document")
|
||||||
|
def test_bulk_reprocess_cloud_ocr_success(self, mock_delay, client: TestClient, db_session, tmp_path):
|
||||||
|
"""Test bulk Cloud OCR reprocessing queues tasks."""
|
||||||
|
mock_task = MagicMock()
|
||||||
|
mock_task.id = "task-cloud-ocr-1"
|
||||||
|
mock_delay.delay.return_value = mock_task
|
||||||
|
|
||||||
|
pdf_file = tmp_path / "ocr_test.pdf"
|
||||||
|
pdf_file.write_bytes(b"PDF data")
|
||||||
|
|
||||||
|
file_record = FileRecord(
|
||||||
|
filehash="hash_ocr1",
|
||||||
|
original_filename="ocr_test.pdf",
|
||||||
|
local_filename=str(pdf_file),
|
||||||
|
file_size=8,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(file_record)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["status"] == "success"
|
||||||
|
assert len(data["processed_files"]) == 1
|
||||||
|
assert data["errors"] == []
|
||||||
|
|
||||||
|
# Ensure force_cloud_ocr=True was passed
|
||||||
|
call_kwargs = mock_delay.delay.call_args.kwargs
|
||||||
|
assert call_kwargs.get("force_cloud_ocr") is True
|
||||||
|
|
||||||
|
def test_bulk_reprocess_cloud_ocr_no_file_on_disk(self, client: TestClient, db_session):
|
||||||
|
"""Test Cloud OCR bulk reprocess skips files not on disk."""
|
||||||
|
file_record = FileRecord(
|
||||||
|
filehash="hash_ocr2",
|
||||||
|
original_filename="missing.pdf",
|
||||||
|
local_filename="/nonexistent/missing.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(file_record)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id])
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["status"] == "error"
|
||||||
|
assert len(data["errors"]) == 1
|
||||||
|
|
||||||
|
def test_bulk_reprocess_cloud_ocr_no_files_found(self, client: TestClient, db_session):
|
||||||
|
"""Test Cloud OCR bulk reprocess with non-existent IDs."""
|
||||||
|
response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[99999])
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.requires_db
|
||||||
|
class TestOcrQualityFilter:
|
||||||
|
"""Tests for the ocr_quality filter on the /files view."""
|
||||||
|
|
||||||
|
def test_ocr_quality_no_ocr_filter(self, client: TestClient, db_session):
|
||||||
|
"""Files without OCR text appear when filtering no_ocr."""
|
||||||
|
rec_no_ocr = FileRecord(
|
||||||
|
filehash="hash_noocr1",
|
||||||
|
original_filename="no_ocr.pdf",
|
||||||
|
local_filename="/tmp/no_ocr.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
ocr_text=None,
|
||||||
|
)
|
||||||
|
rec_has_ocr = FileRecord(
|
||||||
|
filehash="hash_hasocr1",
|
||||||
|
original_filename="has_ocr.pdf",
|
||||||
|
local_filename="/tmp/has_ocr.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
ocr_text="Some extracted text",
|
||||||
|
)
|
||||||
|
db_session.add_all([rec_no_ocr, rec_has_ocr])
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.get("/files?ocr_quality=no_ocr")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "no_ocr.pdf" in response.text
|
||||||
|
assert "has_ocr.pdf" not in response.text
|
||||||
|
|
||||||
|
def test_ocr_quality_has_ocr_filter(self, client: TestClient, db_session):
|
||||||
|
"""Files with OCR text appear when filtering has_ocr."""
|
||||||
|
rec_no_ocr = FileRecord(
|
||||||
|
filehash="hash_noocr2",
|
||||||
|
original_filename="no_ocr2.pdf",
|
||||||
|
local_filename="/tmp/no_ocr2.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
ocr_text=None,
|
||||||
|
)
|
||||||
|
rec_has_ocr = FileRecord(
|
||||||
|
filehash="hash_hasocr2",
|
||||||
|
original_filename="has_ocr2.pdf",
|
||||||
|
local_filename="/tmp/has_ocr2.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
ocr_text="Meaningful extracted text",
|
||||||
|
)
|
||||||
|
db_session.add_all([rec_no_ocr, rec_has_ocr])
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.get("/files?ocr_quality=has_ocr")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "has_ocr2.pdf" in response.text
|
||||||
|
assert "no_ocr2.pdf" not in response.text
|
||||||
|
|
||||||
|
def test_ocr_quality_no_filter(self, client: TestClient, db_session):
|
||||||
|
"""All files appear when no ocr_quality filter is applied."""
|
||||||
|
rec = FileRecord(
|
||||||
|
filehash="hash_all1",
|
||||||
|
original_filename="all_files.pdf",
|
||||||
|
local_filename="/tmp/all_files.pdf",
|
||||||
|
file_size=1024,
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
db_session.add(rec)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
response = client.get("/files")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "all_files.pdf" in response.text
|
||||||
|
|||||||
Reference in New Issue
Block a user