From 50adfc369406b8586878111ec261ce50517b06cb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 7 Feb 2026 15:49:49 +0000 Subject: [PATCH 1/4] Initial plan From 08775459aa8ee9619566f56c4f801aaf6c893cea Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 7 Feb 2026 15:52:38 +0000 Subject: [PATCH 2/4] Fix /files view issues and add bulk operations - Fixed status filter in /files view endpoint - Added bulk delete and reprocess API endpoints - Added bulk selection UI with checkboxes - Added bulk actions bar with reprocess and delete buttons - Updated JavaScript to handle bulk operations Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/files.py | 121 ++++++++++++++++++++++++++++ app/views/files.py | 37 ++++++++- frontend/templates/files.html | 146 +++++++++++++++++++++++++++++++++- 3 files changed, 302 insertions(+), 2 deletions(-) diff --git a/app/api/files.py b/app/api/files.py index 865250a0..c2df2a41 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -275,6 +275,127 @@ def delete_file_record(request: Request, file_id: int, db: Session = Depends(get detail=f"Error deleting file record: {str(e)}" ) +@router.post("/files/bulk-delete") +@require_login +def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)): + """ + Delete multiple file records from the database. + This only removes the database entries, not the actual files. + """ + # Check if file deletion is allowed + if not settings.allow_file_delete: + raise HTTPException( + status_code=403, + detail="File deletion is disabled in the configuration" + ) + + try: + # Find all file records + file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + + if not file_records: + raise HTTPException( + status_code=404, + detail="No files found with the provided IDs" + ) + + deleted_count = len(file_records) + deleted_ids = [f.id for f in file_records] + + # Log the deletion + logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}") + + # Delete all records + for file_record in file_records: + db.delete(file_record) + + db.commit() + + return { + "status": "success", + "message": f"Successfully deleted {deleted_count} file records", + "deleted_ids": deleted_ids + } + + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.exception(f"Error bulk deleting file records: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error bulk deleting file records: {str(e)}" + ) + + +@router.post("/files/bulk-reprocess") +@require_login +def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)): + """ + Reprocess multiple files by queuing them for processing. + """ + try: + # Find all file records + file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() + + if not file_records: + raise HTTPException( + status_code=404, + detail="No files found with the provided IDs" + ) + + task_ids = [] + processed_files = [] + errors = [] + + for file_record in file_records: + try: + # Check if local file exists + if not file_record.local_filename or not os.path.exists(file_record.local_filename): + errors.append({ + "file_id": file_record.id, + "filename": file_record.original_filename, + "error": "Local file not found" + }) + continue + + # Queue the file for processing + task = process_document.delay(file_record.local_filename) + task_ids.append(task.id) + processed_files.append({ + "file_id": file_record.id, + "filename": file_record.original_filename, + "task_id": task.id + }) + + logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}") + + except Exception as e: + logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}") + errors.append({ + "file_id": file_record.id, + "filename": file_record.original_filename, + "error": str(e) + }) + + return { + "status": "success" if processed_files else "error", + "message": f"Successfully queued {len(processed_files)} files for reprocessing", + "processed_files": processed_files, + "errors": errors if errors else None, + "task_ids": task_ids + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Error bulk reprocessing files: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error bulk reprocessing files: {str(e)}" + ) + + @router.post("/ui-upload") @require_login async def ui_upload(request: Request, file: UploadFile = File(...)): diff --git a/app/views/files.py b/app/views/files.py index 8da5c545..ea793f33 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -29,7 +29,7 @@ def files_page( try: # Import the model here to avoid circular imports from app.models import FileRecord, ProcessingLog - from sqlalchemy import desc, asc + from sqlalchemy import desc, asc, or_ # Start with base query query = db.query(FileRecord) @@ -42,6 +42,41 @@ def files_page( if mime_type: query = query.filter(FileRecord.mime_type == mime_type) + # Apply status filter (before pagination for correct counts) + if status: + # Subquery to get file IDs matching the status + if status == "pending": + # Files with no logs + subq = db.query(ProcessingLog.file_id).distinct() + query = query.filter(~FileRecord.id.in_(subq)) + elif status == "processing": + # Files with in_progress logs + subq = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "in_progress" + ).distinct() + query = query.filter(FileRecord.id.in_(subq)) + elif status == "failed": + # Files with failure logs + subq = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "failure" + ).distinct() + query = query.filter(FileRecord.id.in_(subq)) + elif status == "completed": + # Files with success logs but no failures or in_progress + success_files = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "success" + ).distinct().subquery() + + failed_files = db.query(ProcessingLog.file_id).filter( + or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress") + ).distinct().subquery() + + query = query.filter( + FileRecord.id.in_(db.query(success_files.c.file_id)) + ).filter( + ~FileRecord.id.in_(db.query(failed_files.c.file_id)) + ) + # Get total count before pagination total_items = query.count() diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 5fcfcc22..09c3795f 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -297,11 +297,34 @@ + + +
+ {% for file in files %} + @@ -382,7 +408,7 @@ {% else %} - + {% endfor %} @@ -513,6 +539,124 @@ function clearFilters() { window.location.href = '/files'; } + + // Bulk selection functionality + function toggleSelectAll() { + const selectAll = document.getElementById('selectAll'); + const checkboxes = document.querySelectorAll('.file-checkbox'); + checkboxes.forEach(checkbox => { + checkbox.checked = selectAll.checked; + }); + updateBulkActionsBar(); + } + + function updateBulkActionsBar() { + const checkboxes = document.querySelectorAll('.file-checkbox:checked'); + const selectedCount = checkboxes.length; + const bulkActionsBar = document.getElementById('bulkActionsBar'); + const selectedCountEl = document.getElementById('selectedCount'); + + if (selectedCount > 0) { + bulkActionsBar.style.display = 'block'; + selectedCountEl.textContent = selectedCount; + } else { + bulkActionsBar.style.display = 'none'; + } + + // Update "select all" checkbox state + const allCheckboxes = document.querySelectorAll('.file-checkbox'); + const selectAll = document.getElementById('selectAll'); + selectAll.checked = allCheckboxes.length > 0 && selectedCount === allCheckboxes.length; + } + + function clearSelection() { + document.querySelectorAll('.file-checkbox').forEach(cb => cb.checked = false); + document.getElementById('selectAll').checked = false; + updateBulkActionsBar(); + } + + function getSelectedFileIds() { + const checkboxes = document.querySelectorAll('.file-checkbox:checked'); + return Array.from(checkboxes).map(cb => parseInt(cb.value)); + } + + function bulkDelete() { + const fileIds = getSelectedFileIds(); + if (fileIds.length === 0) { + alert('No files selected'); + return; + } + + if (!confirm(`Are you sure you want to delete ${fileIds.length} file(s)?`)) { + return; + } + + fetch('/api/files/bulk-delete', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(fileIds) + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to delete files'); + }); + } + return response.json(); + }) + .then(data => { + alert(data.message); + window.location.reload(); + }) + .catch(error => { + console.error('Error:', error); + alert(`Error deleting files: ${error.message}`); + }); + } + + function bulkReprocess() { + const fileIds = getSelectedFileIds(); + if (fileIds.length === 0) { + alert('No files selected'); + return; + } + + if (!confirm(`Are you sure you want to reprocess ${fileIds.length} file(s)?`)) { + return; + } + + fetch('/api/files/bulk-reprocess', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(fileIds) + }) + .then(response => { + if (!response.ok) { + return response.json().then(err => { + throw new Error(err.detail || 'Failed to reprocess files'); + }); + } + return response.json(); + }) + .then(data => { + if (data.errors && data.errors.length > 0) { + const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n'); + alert(`${data.message}\n\nErrors:\n${errorMsg}`); + } else { + alert(data.message); + } + clearSelection(); + window.location.reload(); + }) + .catch(error => { + console.error('Error:', error); + alert(`Error reprocessing files: ${error.message}`); + }); + } {% endblock %} From c16265798711c87f61045de52c25bdabedadc3e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 7 Feb 2026 15:53:34 +0000 Subject: [PATCH 3/4] Add tests for bulk operations and status filtering Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- tests/test_bulk_operations.py | 274 ++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 tests/test_bulk_operations.py diff --git a/tests/test_bulk_operations.py b/tests/test_bulk_operations.py new file mode 100644 index 00000000..97224d4d --- /dev/null +++ b/tests/test_bulk_operations.py @@ -0,0 +1,274 @@ +""" +Tests for bulk file operations (delete and reprocess). +""" +import pytest +from fastapi.testclient import TestClient +from app.models import FileRecord, ProcessingLog +from unittest.mock import patch, MagicMock + + +@pytest.mark.integration +@pytest.mark.requires_db +class TestBulkOperations: + """Tests for bulk file operations.""" + + def test_bulk_delete_success(self, client: TestClient, db_session): + """Test bulk deletion of files.""" + # Create sample files + file_ids = [] + for i in range(3): + file_record = FileRecord( + filehash=f"hash{i}", + original_filename=f"test{i}.pdf", + local_filename=f"/tmp/test{i}.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + file_ids.append(file_record.id) + db_session.commit() + + # Bulk delete + response = client.post( + "/api/files/bulk-delete", + json=file_ids + ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert len(data["deleted_ids"]) == 3 + + # Verify files are deleted + for file_id in file_ids: + file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first() + assert file_record is None + + def test_bulk_delete_empty_list(self, client: TestClient, db_session): + """Test bulk deletion with empty list.""" + response = client.post( + "/api/files/bulk-delete", + json=[] + ) + assert response.status_code == 404 + data = response.json() + assert "No files found" in data["detail"] + + def test_bulk_delete_nonexistent_files(self, client: TestClient, db_session): + """Test bulk deletion of non-existent files.""" + response = client.post( + "/api/files/bulk-delete", + json=[9999, 9998] + ) + assert response.status_code == 404 + + @patch('app.api.files.process_document') + def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session): + """Test bulk reprocessing of files.""" + # Setup mock + mock_task = MagicMock() + mock_task.id = "test-task-id" + mock_process_document.delay.return_value = mock_task + + # Create sample files with local files that "exist" + file_ids = [] + for i in range(2): + file_record = FileRecord( + filehash=f"hash{i}", + original_filename=f"test{i}.pdf", + local_filename=f"/tmp/test{i}.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + file_ids.append(file_record.id) + db_session.commit() + + # Mock os.path.exists to return True + with patch('os.path.exists', return_value=True): + response = client.post( + "/api/files/bulk-reprocess", + json=file_ids + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert len(data["processed_files"]) == 2 + assert len(data["task_ids"]) == 2 + + @patch('app.api.files.process_document') + def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session): + """Test bulk reprocessing when some local files are missing.""" + # Setup mock + mock_task = MagicMock() + mock_task.id = "test-task-id" + mock_process_document.delay.return_value = mock_task + + # Create sample files + file_ids = [] + for i in range(2): + file_record = FileRecord( + filehash=f"hash{i}", + original_filename=f"test{i}.pdf", + local_filename=f"/tmp/test{i}.pdf" if i == 0 else None, # Second file has no local file + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + file_ids.append(file_record.id) + db_session.commit() + + # Mock os.path.exists to return False for missing file + with patch('os.path.exists', return_value=False): + response = client.post( + "/api/files/bulk-reprocess", + json=file_ids + ) + + assert response.status_code == 200 + data = response.json() + # All files should have errors since we mocked exists to return False + assert data["errors"] is not None + assert len(data["errors"]) == 2 + + def test_bulk_reprocess_nonexistent_files(self, client: TestClient, db_session): + """Test bulk reprocessing of non-existent files.""" + response = client.post( + "/api/files/bulk-reprocess", + json=[9999, 9998] + ) + assert response.status_code == 404 + + +@pytest.mark.integration +@pytest.mark.requires_db +class TestStatusFilter: + """Tests for status filtering in files view.""" + + def test_status_filter_pending(self, client: TestClient, db_session): + """Test filtering files by pending status.""" + # Create files with different statuses + # File 1: Pending (no logs) + file1 = FileRecord( + filehash="hash1", + original_filename="pending.pdf", + local_filename="/tmp/pending.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file1) + + # File 2: Processing (has in_progress log) + file2 = FileRecord( + filehash="hash2", + original_filename="processing.pdf", + local_filename="/tmp/processing.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file2) + db_session.flush() + + log2 = ProcessingLog( + file_id=file2.id, + task_id="task2", + step_name="OCR", + status="in_progress", + message="Processing..." + ) + db_session.add(log2) + db_session.commit() + + # Test pending filter + response = client.get("/files?status=pending") + assert response.status_code == 200 + # Check that pending file is shown (HTML response) + assert "pending.pdf" in response.text + assert "processing.pdf" not in response.text + + def test_status_filter_processing(self, client: TestClient, db_session): + """Test filtering files by processing status.""" + # Create file with in_progress status + file_record = FileRecord( + filehash="hash1", + original_filename="processing.pdf", + local_filename="/tmp/processing.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + + log = ProcessingLog( + file_id=file_record.id, + task_id="task1", + step_name="OCR", + status="in_progress", + message="Processing..." + ) + db_session.add(log) + db_session.commit() + + # Test processing filter + response = client.get("/files?status=processing") + assert response.status_code == 200 + assert "processing.pdf" in response.text + + def test_status_filter_completed(self, client: TestClient, db_session): + """Test filtering files by completed status.""" + # Create file with success status + file_record = FileRecord( + filehash="hash1", + original_filename="completed.pdf", + local_filename="/tmp/completed.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + + log = ProcessingLog( + file_id=file_record.id, + task_id="task1", + step_name="OCR", + status="success", + message="Completed" + ) + db_session.add(log) + db_session.commit() + + # Test completed filter + response = client.get("/files?status=completed") + assert response.status_code == 200 + assert "completed.pdf" in response.text + + def test_status_filter_failed(self, client: TestClient, db_session): + """Test filtering files by failed status.""" + # Create file with failure status + file_record = FileRecord( + filehash="hash1", + original_filename="failed.pdf", + local_filename="/tmp/failed.pdf", + file_size=1024, + mime_type="application/pdf" + ) + db_session.add(file_record) + db_session.flush() + + log = ProcessingLog( + file_id=file_record.id, + task_id="task1", + step_name="OCR", + status="failure", + message="Failed" + ) + db_session.add(log) + db_session.commit() + + # Test failed filter + response = client.get("/files?status=failed") + assert response.status_code == 200 + assert "failed.pdf" in response.text From 34ba8a4548ac6b7230208b30a09040a400ff7272 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 7 Feb 2026 15:55:44 +0000 Subject: [PATCH 4/4] Improve error messages for bulk operations Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- frontend/templates/files.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 09c3795f..bd9acec7 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -304,13 +304,13 @@ 0 files selected
- - -
@@ -583,7 +583,7 @@ function bulkDelete() { const fileIds = getSelectedFileIds(); if (fileIds.length === 0) { - alert('No files selected'); + alert('Please select files to delete'); return; } @@ -619,7 +619,7 @@ function bulkReprocess() { const fileIds = getSelectedFileIds(); if (fileIds.length === 0) { - alert('No files selected'); + alert('Please select files to reprocess'); return; }
+ + ID @@ -359,6 +382,9 @@
+ + {{ file.id }} {{ file.original_filename }} {{ (file.file_size / 1024) | round(2) }} KB
No files foundNo files found