diff --git a/app/api/files.py b/app/api/files.py index bdab0f13..937d117d 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -2,14 +2,17 @@ File-related API endpoints """ +import io import logging import mimetypes import os import uuid +import zipfile from datetime import datetime, timezone from typing import Annotated, List, Optional from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile +from fastapi.responses import StreamingResponse from sqlalchemy import asc, desc from sqlalchemy.orm import Session @@ -429,6 +432,163 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}") +@router.post("/files/bulk-reprocess-cloud-ocr") +@require_login +def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: DbSession): + """ + Reprocess multiple files with forced Cloud OCR (Azure Document Intelligence). + + Useful for re-running OCR on files with poor text quality or missing OCR text. + """ + try: + file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + + if not file_records: + raise HTTPException(status_code=404, detail="No files found with the provided IDs") + + task_ids = [] + processed_files = [] + errors = [] + + for file_record in file_records: + try: + source_file = None + for path in [file_record.original_file_path, file_record.local_filename]: + if path and os.path.exists(path): + source_file = path + break + + if not source_file: + errors.append( + { + "file_id": file_record.id, + "filename": file_record.original_filename, + "error": "File not found on disk", + } + ) + continue + + task = process_document.delay( + source_file, + original_filename=file_record.original_filename, + file_id=file_record.id, + force_cloud_ocr=True, + ) + task_ids.append(task.id) + processed_files.append( + { + "file_id": file_record.id, + "filename": file_record.original_filename, + "task_id": task.id, + } + ) + logger.info( + f"Bulk Cloud OCR reprocessing: ID={file_record.id}, " + f"Filename={file_record.original_filename}, TaskID={task.id}" + ) + except Exception as e: + logger.exception(f"Error queuing Cloud OCR for file {file_record.id}: {str(e)}") + errors.append( + { + "file_id": file_record.id, + "filename": file_record.original_filename, + "error": str(e), + } + ) + + return { + "status": "success" if processed_files else "error", + "message": f"Successfully queued {len(processed_files)} files for Cloud OCR reprocessing", + "processed_files": processed_files, + "errors": errors, + "task_ids": task_ids, + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Error bulk reprocessing files with Cloud OCR: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files with Cloud OCR: {str(e)}") + + +@router.post("/files/bulk-download") +@require_login +def bulk_download_files(request: Request, file_ids: List[int], db: DbSession): + """ + Download multiple files as a single ZIP archive. + + For each file, the processed version is preferred; falls back to the original. + Files not found on disk are silently skipped. + """ + try: + file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + + if not file_records: + raise HTTPException(status_code=404, detail="No files found with the provided IDs") + + zip_buffer = io.BytesIO() + added = 0 + seen_names: dict[str, int] = {} + + with zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: + workdir = settings.workdir + processed_dir = os.path.join(workdir, "processed") + + for file_record in file_records: + # Resolve file path: processed first, then original/local + base_filename = os.path.splitext(file_record.original_filename or "file")[0] + candidate_paths = [ + file_record.processed_file_path, + file_record.original_file_path, + file_record.local_filename, + os.path.join(processed_dir, f"{file_record.filehash}.pdf"), + os.path.join(processed_dir, f"{base_filename}_processed.pdf"), + ] + + file_path = None + for path in candidate_paths: + if path and os.path.exists(path): + file_path = path + break + + if not file_path: + logger.warning(f"Skipping file {file_record.id}: no file found on disk") + continue + + # Build a unique archive name to avoid collisions + archive_name = file_record.original_filename or os.path.basename(file_path) + if archive_name in seen_names: + seen_names[archive_name] += 1 + stem, ext = os.path.splitext(archive_name) + archive_name = f"{stem}_{seen_names[archive_name]}{ext}" + else: + seen_names[archive_name] = 0 + + zf.write(file_path, archive_name) + added += 1 + + if added == 0: + raise HTTPException(status_code=404, detail="None of the selected files could be found on disk") + + zip_buffer.seek(0) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + zip_filename = f"docuelevate_bulk_{timestamp}.zip" + + logger.info(f"Bulk download: packed {added} file(s) into {zip_filename}") + + return StreamingResponse( + zip_buffer, + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{zip_filename}"'}, + ) + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Error creating bulk download ZIP: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error creating bulk download ZIP: {str(e)}") + + @router.post("/files/{file_id}/reprocess") @require_login def reprocess_single_file(request: Request, file_id: int, db: DbSession): diff --git a/app/views/files.py b/app/views/files.py index a9b69a45..b578b79a 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -34,6 +34,7 @@ def files_page( date_to: Optional[str] = Query(None), storage_provider: Optional[str] = Query(None), tags: Optional[str] = Query(None), + ocr_quality: Optional[str] = Query(None), ): """ Return the 'files.html' template with server-side pagination, sorting, and filtering @@ -94,6 +95,15 @@ def files_page( escaped_tag = tag.replace("%", r"\%").replace("_", r"\_") query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%")) + # Apply OCR quality filter + if ocr_quality == "no_ocr": + query = query.filter((FileRecord.ocr_text.is_(None)) | (FileRecord.ocr_text == "")) + elif ocr_quality == "has_ocr": + query = query.filter( + FileRecord.ocr_text.isnot(None), + FileRecord.ocr_text != "", + ) + # Apply status filter (before pagination for correct counts) query = apply_status_filter(query, db, status) @@ -158,6 +168,7 @@ def files_page( "date_to": date_to or "", "storage_provider": storage_provider or "", "tags": tags or "", + "ocr_quality": ocr_quality or "", "mime_types": mime_types, "upload_concurrency": settings.upload_concurrency, "upload_queue_delay_ms": settings.upload_queue_delay_ms, diff --git a/docs/API.md b/docs/API.md index 3613acfc..f76e9654 100644 --- a/docs/API.md +++ b/docs/API.md @@ -295,6 +295,7 @@ Retrieve a paginated list of processed files with advanced filtering and sorting - `date_to` (optional): Filter files created on or before this date (ISO 8601, e.g. `2026-12-31`) - `storage_provider` (optional): Filter by storage provider (e.g. `dropbox`, `s3`, `google_drive`, `onedrive`, `nextcloud`) - `tags` (optional): Filter by tags in AI metadata (comma-separated, AND logic, e.g. `invoice,amazon`) +- `ocr_quality` (optional): Filter by OCR text availability (`no_ocr` = files without OCR text, `has_ocr` = files with OCR text) All filters are combinable using AND logic. @@ -461,6 +462,96 @@ Reprocess a specific file with forced Cloud OCR, regardless of embedded text qua **Note**: This endpoint forces Azure Document Intelligence OCR processing even if the PDF contains embedded text. The original file (if available) is used for reprocessing to ensure the highest quality result. +### Bulk Operations + +**POST** `/api/files/bulk-delete` + +Delete multiple file records in a single request. + +**Request body**: JSON array of file IDs + +```bash +curl -X POST "http:///api/files/bulk-delete" \ + -H "Content-Type: application/json" \ + -d '[1, 2, 3]' +``` + +**Response**: +```json +{ + "status": "success", + "message": "Successfully deleted 3 file records", + "deleted_ids": [1, 2, 3] +} +``` + +**Error Responses**: +- `403`: File deletion is disabled in configuration +- `404`: No files found with the provided IDs + +--- + +**POST** `/api/files/bulk-reprocess` + +Queue multiple files for full reprocessing. + +**Request body**: JSON array of file IDs + +**Response**: +```json +{ + "status": "success", + "message": "Successfully queued 2 files for reprocessing", + "processed_files": [ + {"file_id": 1, "filename": "a.pdf", "task_id": "abc123"}, + {"file_id": 2, "filename": "b.pdf", "task_id": "def456"} + ], + "errors": [], + "task_ids": ["abc123", "def456"] +} +``` + +--- + +**POST** `/api/files/bulk-reprocess-cloud-ocr` + +Queue multiple files for reprocessing with forced Cloud OCR (Azure Document Intelligence). Useful for files that have missing or low-quality OCR text. + +**Request body**: JSON array of file IDs + +**Response**: +```json +{ + "status": "success", + "message": "Successfully queued 2 files for Cloud OCR reprocessing", + "processed_files": [ + {"file_id": 1, "filename": "a.pdf", "task_id": "abc123"} + ], + "errors": [], + "task_ids": ["abc123"] +} +``` + +--- + +**POST** `/api/files/bulk-download` + +Download multiple files as a single ZIP archive. For each file, the processed version is preferred; falls back to the original. Files not found on disk are silently skipped. + +**Request body**: JSON array of file IDs + +**Response**: `application/zip` stream with `Content-Disposition: attachment; filename="docuelevate_bulk_.zip"` + +```bash +curl -X POST "http:///api/files/bulk-download" \ + -H "Content-Type: application/json" \ + -d '[1, 2, 3]' \ + --output bulk_download.zip +``` + +**Error Responses**: +- `404`: No files found with the provided IDs, or none of the selected files exist on disk + ### File Preview **GET** `/api/files/{file_id}/preview` diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 5a353155..b706a354 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -533,6 +533,15 @@ +
+ + +
+
@@ -616,6 +625,12 @@ + + @@ -1285,6 +1300,86 @@ }); } + function bulkReprocessCloudOcr() { + const fileIds = getSelectedFileIds(); + if (fileIds.length === 0) { + alert('Please select files to re-run Cloud OCR on'); + return; + } + + if (!confirm(`Are you sure you want to re-run Cloud OCR on ${fileIds.length} file(s)?`)) { + return; + } + + fetch('/api/files/bulk-reprocess-cloud-ocr', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(fileIds) + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to queue Cloud OCR'); + }); + } + return response.json(); + }) + .then(data => { + if (data.errors && data.errors.length > 0) { + const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n'); + alert(`${data.message}\n\nErrors:\n${errorMsg}`); + } else { + alert(data.message); + } + clearSelection(); + window.location.reload(); + }) + .catch(error => { + console.error('Error:', error); + alert(`Error queuing Cloud OCR: ${error.message}`); + }); + } + + function bulkDownload() { + const fileIds = getSelectedFileIds(); + if (fileIds.length === 0) { + alert('Please select files to download'); + return; + } + + fetch('/api/files/bulk-download', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(fileIds) + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to create ZIP download'); + }); + } + return response.blob(); + }) + .then(blob => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `docuelevate_bulk_${Date.now()}.zip`; + document.body.appendChild(a); + a.click(); + a.remove(); + window.URL.revokeObjectURL(url); + }) + .catch(error => { + console.error('Error:', error); + alert(`Error downloading files: ${error.message}`); + }); + } + // ===== Drag-and-Drop Upload Functionality ===== const dropOverlay = document.getElementById('dropOverlay'); const uploadModal = document.getElementById('uploadModal'); diff --git a/tests/test_bulk_operations.py b/tests/test_bulk_operations.py index d2fbf7b9..f81a0f7b 100644 --- a/tests/test_bulk_operations.py +++ b/tests/test_bulk_operations.py @@ -2,6 +2,8 @@ Tests for bulk file operations (delete and reprocess). """ +import io +import zipfile from unittest.mock import MagicMock, patch import pytest @@ -293,3 +295,239 @@ class TestStatusFilter: response = client.get("/files?status=failed") assert response.status_code == 200 assert "failed.pdf" in response.text + + +@pytest.mark.integration +@pytest.mark.requires_db +class TestBulkDownload: + """Tests for POST /api/files/bulk-download endpoint.""" + + def test_bulk_download_no_files_found(self, client: TestClient, db_session): + """Test bulk download with non-existent IDs.""" + response = client.post("/api/files/bulk-download", json=[99999, 99998]) + assert response.status_code == 404 + + def test_bulk_download_files_not_on_disk(self, client: TestClient, db_session): + """Test bulk download when files are not found on disk.""" + file_record = FileRecord( + filehash="hash_dl1", + original_filename="nodisk.pdf", + local_filename="/nonexistent/nodisk.pdf", + file_size=1024, + mime_type="application/pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post("/api/files/bulk-download", json=[file_record.id]) + assert response.status_code == 404 + assert "None of the selected files" in response.json()["detail"] + + def test_bulk_download_success(self, client: TestClient, db_session, tmp_path): + """Test successful bulk download returns a ZIP archive.""" + # Create a real file on disk + pdf_file = tmp_path / "sample.pdf" + pdf_file.write_bytes(b"PDF content") + + file_record = FileRecord( + filehash="hash_dl2", + original_filename="sample.pdf", + local_filename=str(pdf_file), + processed_file_path=str(pdf_file), + file_size=len(b"PDF content"), + mime_type="application/pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post("/api/files/bulk-download", json=[file_record.id]) + assert response.status_code == 200 + assert response.headers["content-type"] == "application/zip" + assert "attachment" in response.headers["content-disposition"] + assert ".zip" in response.headers["content-disposition"] + + def test_bulk_download_multiple_files(self, client: TestClient, db_session, tmp_path): + """Test bulk download with multiple files produces a valid ZIP.""" + ids = [] + for i in range(3): + f = tmp_path / f"file{i}.pdf" + f.write_bytes(f"content {i}".encode()) + rec = FileRecord( + filehash=f"hash_multi_{i}", + original_filename=f"file{i}.pdf", + local_filename=str(f), + processed_file_path=str(f), + file_size=len(f"content {i}".encode()), + mime_type="application/pdf", + ) + db_session.add(rec) + db_session.commit() + ids.append(rec.id) + + response = client.post("/api/files/bulk-download", json=ids) + assert response.status_code == 200 + + zip_data = io.BytesIO(response.content) + with zipfile.ZipFile(zip_data) as zf: + names = zf.namelist() + assert len(names) == 3 + + def test_bulk_download_duplicate_filenames(self, client: TestClient, db_session, tmp_path): + """Test bulk download disambiguates duplicate filenames.""" + ids = [] + for i in range(2): + f = tmp_path / f"dup_{i}.pdf" + f.write_bytes(b"data") + rec = FileRecord( + filehash=f"hash_dup_{i}", + original_filename="dup.pdf", # same name + local_filename=str(f), + processed_file_path=str(f), + file_size=4, + mime_type="application/pdf", + ) + db_session.add(rec) + db_session.commit() + ids.append(rec.id) + + response = client.post("/api/files/bulk-download", json=ids) + assert response.status_code == 200 + zip_data = io.BytesIO(response.content) + with zipfile.ZipFile(zip_data) as zf: + names = zf.namelist() + # Names must be unique + assert len(names) == len(set(names)) + + +@pytest.mark.integration +@pytest.mark.requires_db +class TestBulkReprocessCloudOcr: + """Tests for POST /api/files/bulk-reprocess-cloud-ocr endpoint.""" + + @patch("app.api.files.process_document") + def test_bulk_reprocess_cloud_ocr_success(self, mock_delay, client: TestClient, db_session, tmp_path): + """Test bulk Cloud OCR reprocessing queues tasks.""" + mock_task = MagicMock() + mock_task.id = "task-cloud-ocr-1" + mock_delay.delay.return_value = mock_task + + pdf_file = tmp_path / "ocr_test.pdf" + pdf_file.write_bytes(b"PDF data") + + file_record = FileRecord( + filehash="hash_ocr1", + original_filename="ocr_test.pdf", + local_filename=str(pdf_file), + file_size=8, + mime_type="application/pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id]) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert len(data["processed_files"]) == 1 + assert data["errors"] == [] + + # Ensure force_cloud_ocr=True was passed + call_kwargs = mock_delay.delay.call_args.kwargs + assert call_kwargs.get("force_cloud_ocr") is True + + def test_bulk_reprocess_cloud_ocr_no_file_on_disk(self, client: TestClient, db_session): + """Test Cloud OCR bulk reprocess skips files not on disk.""" + file_record = FileRecord( + filehash="hash_ocr2", + original_filename="missing.pdf", + local_filename="/nonexistent/missing.pdf", + file_size=1024, + mime_type="application/pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[file_record.id]) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "error" + assert len(data["errors"]) == 1 + + def test_bulk_reprocess_cloud_ocr_no_files_found(self, client: TestClient, db_session): + """Test Cloud OCR bulk reprocess with non-existent IDs.""" + response = client.post("/api/files/bulk-reprocess-cloud-ocr", json=[99999]) + assert response.status_code == 404 + + +@pytest.mark.integration +@pytest.mark.requires_db +class TestOcrQualityFilter: + """Tests for the ocr_quality filter on the /files view.""" + + def test_ocr_quality_no_ocr_filter(self, client: TestClient, db_session): + """Files without OCR text appear when filtering no_ocr.""" + rec_no_ocr = FileRecord( + filehash="hash_noocr1", + original_filename="no_ocr.pdf", + local_filename="/tmp/no_ocr.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_text=None, + ) + rec_has_ocr = FileRecord( + filehash="hash_hasocr1", + original_filename="has_ocr.pdf", + local_filename="/tmp/has_ocr.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_text="Some extracted text", + ) + db_session.add_all([rec_no_ocr, rec_has_ocr]) + db_session.commit() + + response = client.get("/files?ocr_quality=no_ocr") + assert response.status_code == 200 + assert "no_ocr.pdf" in response.text + assert "has_ocr.pdf" not in response.text + + def test_ocr_quality_has_ocr_filter(self, client: TestClient, db_session): + """Files with OCR text appear when filtering has_ocr.""" + rec_no_ocr = FileRecord( + filehash="hash_noocr2", + original_filename="no_ocr2.pdf", + local_filename="/tmp/no_ocr2.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_text=None, + ) + rec_has_ocr = FileRecord( + filehash="hash_hasocr2", + original_filename="has_ocr2.pdf", + local_filename="/tmp/has_ocr2.pdf", + file_size=1024, + mime_type="application/pdf", + ocr_text="Meaningful extracted text", + ) + db_session.add_all([rec_no_ocr, rec_has_ocr]) + db_session.commit() + + response = client.get("/files?ocr_quality=has_ocr") + assert response.status_code == 200 + assert "has_ocr2.pdf" in response.text + assert "no_ocr2.pdf" not in response.text + + def test_ocr_quality_no_filter(self, client: TestClient, db_session): + """All files appear when no ocr_quality filter is applied.""" + rec = FileRecord( + filehash="hash_all1", + original_filename="all_files.pdf", + local_filename="/tmp/all_files.pdf", + file_size=1024, + mime_type="application/pdf", + ) + db_session.add(rec) + db_session.commit() + + response = client.get("/files") + assert response.status_code == 200 + assert "all_files.pdf" in response.text