Merge pull request #88 from christianlouis/copilot/fix-files-view-errors

Fix /files view status filtering and add bulk operations
This commit is contained in:
Christian Krakau-Louis
2026-02-07 17:54:21 +01:00
committed by GitHub
4 changed files with 576 additions and 2 deletions
+121
View File
@@ -275,6 +275,127 @@ def delete_file_record(request: Request, file_id: int, db: Session = Depends(get
detail=f"Error deleting file record: {str(e)}"
)
@router.post("/files/bulk-delete")
@require_login
def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
"""
Delete multiple file records from the database.
This only removes the database entries, not the actual files.
"""
# Check if file deletion is allowed
if not settings.allow_file_delete:
raise HTTPException(
status_code=403,
detail="File deletion is disabled in the configuration"
)
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(
status_code=404,
detail="No files found with the provided IDs"
)
deleted_count = len(file_records)
deleted_ids = [f.id for f in file_records]
# Log the deletion
logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}")
# Delete all records
for file_record in file_records:
db.delete(file_record)
db.commit()
return {
"status": "success",
"message": f"Successfully deleted {deleted_count} file records",
"deleted_ids": deleted_ids
}
except HTTPException:
raise
except Exception as e:
db.rollback()
logger.exception(f"Error bulk deleting file records: {str(e)}")
raise HTTPException(
status_code=500,
detail=f"Error bulk deleting file records: {str(e)}"
)
@router.post("/files/bulk-reprocess")
@require_login
def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
"""
Reprocess multiple files by queuing them for processing.
"""
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(
status_code=404,
detail="No files found with the provided IDs"
)
task_ids = []
processed_files = []
errors = []
for file_record in file_records:
try:
# Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
errors.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": "Local file not found"
})
continue
# Queue the file for processing
task = process_document.delay(file_record.local_filename)
task_ids.append(task.id)
processed_files.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"task_id": task.id
})
logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}")
except Exception as e:
logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
errors.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": str(e)
})
return {
"status": "success" if processed_files else "error",
"message": f"Successfully queued {len(processed_files)} files for reprocessing",
"processed_files": processed_files,
"errors": errors if errors else None,
"task_ids": task_ids
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error bulk reprocessing files: {str(e)}")
raise HTTPException(
status_code=500,
detail=f"Error bulk reprocessing files: {str(e)}"
)
@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, file: UploadFile = File(...)):
+36 -1
View File
@@ -29,7 +29,7 @@ def files_page(
try:
# Import the model here to avoid circular imports
from app.models import FileRecord, ProcessingLog
from sqlalchemy import desc, asc
from sqlalchemy import desc, asc, or_
# Start with base query
query = db.query(FileRecord)
@@ -42,6 +42,41 @@ def files_page(
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Apply status filter (before pagination for correct counts)
if status:
# Subquery to get file IDs matching the status
if status == "pending":
# Files with no logs
subq = db.query(ProcessingLog.file_id).distinct()
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "in_progress"
).distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "failure"
).distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "success"
).distinct().subquery()
failed_files = db.query(ProcessingLog.file_id).filter(
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
).distinct().subquery()
query = query.filter(
FileRecord.id.in_(db.query(success_files.c.file_id))
).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
)
# Get total count before pagination
total_items = query.count()
+145 -1
View File
@@ -297,11 +297,34 @@
</form>
</div>
<!-- Bulk Actions Section -->
<div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
<div style="display: flex; justify-content: space-between; align-items: center;">
<div>
<strong id="selectedCount">0</strong> files selected
</div>
<div style="display: flex; gap: 1rem;">
<button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
<i class="fas fa-sync"></i> Reprocess Selected
</button>
<button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
<i class="fas fa-trash"></i> Delete Selected
</button>
<button type="button" onclick="clearSelection()" style="padding: 0.5rem 1rem; background-color: #718096; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
Clear Selection
</button>
</div>
</div>
</div>
<!-- File table -->
<div class="overflow-x-auto">
<table class="file-table" id="fileTable">
<thead>
<tr>
<th style="width: 40px;">
<input type="checkbox" id="selectAll" onclick="toggleSelectAll()" title="Select all files on this page">
</th>
<th class="sortable" onclick="sortTable('id')">
ID
<span class="sort-indicator {% if sort_by == 'id' %}active{% endif %}">
@@ -359,6 +382,9 @@
<tbody>
{% for file in files %}
<tr onclick="viewFileDetail({{ file.id }}, event)">
<td onclick="event.stopPropagation();">
<input type="checkbox" class="file-checkbox" value="{{ file.id }}" onchange="updateBulkActionsBar()">
</td>
<td>{{ file.id }}</td>
<td>{{ file.original_filename }}</td>
<td>{{ (file.file_size / 1024) | round(2) }} KB</td>
@@ -382,7 +408,7 @@
</tr>
{% else %}
<tr>
<td colspan="7" class="text-center py-4">No files found</td>
<td colspan="8" class="text-center py-4">No files found</td>
</tr>
{% endfor %}
</tbody>
@@ -513,6 +539,124 @@
function clearFilters() {
window.location.href = '/files';
}
// Bulk selection functionality
function toggleSelectAll() {
const selectAll = document.getElementById('selectAll');
const checkboxes = document.querySelectorAll('.file-checkbox');
checkboxes.forEach(checkbox => {
checkbox.checked = selectAll.checked;
});
updateBulkActionsBar();
}
function updateBulkActionsBar() {
const checkboxes = document.querySelectorAll('.file-checkbox:checked');
const selectedCount = checkboxes.length;
const bulkActionsBar = document.getElementById('bulkActionsBar');
const selectedCountEl = document.getElementById('selectedCount');
if (selectedCount > 0) {
bulkActionsBar.style.display = 'block';
selectedCountEl.textContent = selectedCount;
} else {
bulkActionsBar.style.display = 'none';
}
// Update "select all" checkbox state
const allCheckboxes = document.querySelectorAll('.file-checkbox');
const selectAll = document.getElementById('selectAll');
selectAll.checked = allCheckboxes.length > 0 && selectedCount === allCheckboxes.length;
}
function clearSelection() {
document.querySelectorAll('.file-checkbox').forEach(cb => cb.checked = false);
document.getElementById('selectAll').checked = false;
updateBulkActionsBar();
}
function getSelectedFileIds() {
const checkboxes = document.querySelectorAll('.file-checkbox:checked');
return Array.from(checkboxes).map(cb => parseInt(cb.value));
}
function bulkDelete() {
const fileIds = getSelectedFileIds();
if (fileIds.length === 0) {
alert('Please select files to delete');
return;
}
if (!confirm(`Are you sure you want to delete ${fileIds.length} file(s)?`)) {
return;
}
fetch('/api/files/bulk-delete', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(fileIds)
})
.then(response => {
if (!response.ok) {
return response.json().then(err => {
throw new Error(err.detail || 'Failed to delete files');
});
}
return response.json();
})
.then(data => {
alert(data.message);
window.location.reload();
})
.catch(error => {
console.error('Error:', error);
alert(`Error deleting files: ${error.message}`);
});
}
function bulkReprocess() {
const fileIds = getSelectedFileIds();
if (fileIds.length === 0) {
alert('Please select files to reprocess');
return;
}
if (!confirm(`Are you sure you want to reprocess ${fileIds.length} file(s)?`)) {
return;
}
fetch('/api/files/bulk-reprocess', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(fileIds)
})
.then(response => {
if (!response.ok) {
return response.json().then(err => {
throw new Error(err.detail || 'Failed to reprocess files');
});
}
return response.json();
})
.then(data => {
if (data.errors && data.errors.length > 0) {
const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n');
alert(`${data.message}\n\nErrors:\n${errorMsg}`);
} else {
alert(data.message);
}
clearSelection();
window.location.reload();
})
.catch(error => {
console.error('Error:', error);
alert(`Error reprocessing files: ${error.message}`);
});
}
</script>
</div>
{% endblock %}
+274
View File
@@ -0,0 +1,274 @@
"""
Tests for bulk file operations (delete and reprocess).
"""
import pytest
from fastapi.testclient import TestClient
from app.models import FileRecord, ProcessingLog
from unittest.mock import patch, MagicMock
@pytest.mark.integration
@pytest.mark.requires_db
class TestBulkOperations:
"""Tests for bulk file operations."""
def test_bulk_delete_success(self, client: TestClient, db_session):
"""Test bulk deletion of files."""
# Create sample files
file_ids = []
for i in range(3):
file_record = FileRecord(
filehash=f"hash{i}",
original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
file_ids.append(file_record.id)
db_session.commit()
# Bulk delete
response = client.post(
"/api/files/bulk-delete",
json=file_ids
)
assert response.status_code == 200
data = response.json()
assert data["status"] == "success"
assert len(data["deleted_ids"]) == 3
# Verify files are deleted
for file_id in file_ids:
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
assert file_record is None
def test_bulk_delete_empty_list(self, client: TestClient, db_session):
"""Test bulk deletion with empty list."""
response = client.post(
"/api/files/bulk-delete",
json=[]
)
assert response.status_code == 404
data = response.json()
assert "No files found" in data["detail"]
def test_bulk_delete_nonexistent_files(self, client: TestClient, db_session):
"""Test bulk deletion of non-existent files."""
response = client.post(
"/api/files/bulk-delete",
json=[9999, 9998]
)
assert response.status_code == 404
@patch('app.api.files.process_document')
def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing of files."""
# Setup mock
mock_task = MagicMock()
mock_task.id = "test-task-id"
mock_process_document.delay.return_value = mock_task
# Create sample files with local files that "exist"
file_ids = []
for i in range(2):
file_record = FileRecord(
filehash=f"hash{i}",
original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
file_ids.append(file_record.id)
db_session.commit()
# Mock os.path.exists to return True
with patch('os.path.exists', return_value=True):
response = client.post(
"/api/files/bulk-reprocess",
json=file_ids
)
assert response.status_code == 200
data = response.json()
assert data["status"] == "success"
assert len(data["processed_files"]) == 2
assert len(data["task_ids"]) == 2
@patch('app.api.files.process_document')
def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing when some local files are missing."""
# Setup mock
mock_task = MagicMock()
mock_task.id = "test-task-id"
mock_process_document.delay.return_value = mock_task
# Create sample files
file_ids = []
for i in range(2):
file_record = FileRecord(
filehash=f"hash{i}",
original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf" if i == 0 else None, # Second file has no local file
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
file_ids.append(file_record.id)
db_session.commit()
# Mock os.path.exists to return False for missing file
with patch('os.path.exists', return_value=False):
response = client.post(
"/api/files/bulk-reprocess",
json=file_ids
)
assert response.status_code == 200
data = response.json()
# All files should have errors since we mocked exists to return False
assert data["errors"] is not None
assert len(data["errors"]) == 2
def test_bulk_reprocess_nonexistent_files(self, client: TestClient, db_session):
"""Test bulk reprocessing of non-existent files."""
response = client.post(
"/api/files/bulk-reprocess",
json=[9999, 9998]
)
assert response.status_code == 404
@pytest.mark.integration
@pytest.mark.requires_db
class TestStatusFilter:
"""Tests for status filtering in files view."""
def test_status_filter_pending(self, client: TestClient, db_session):
"""Test filtering files by pending status."""
# Create files with different statuses
# File 1: Pending (no logs)
file1 = FileRecord(
filehash="hash1",
original_filename="pending.pdf",
local_filename="/tmp/pending.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file1)
# File 2: Processing (has in_progress log)
file2 = FileRecord(
filehash="hash2",
original_filename="processing.pdf",
local_filename="/tmp/processing.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file2)
db_session.flush()
log2 = ProcessingLog(
file_id=file2.id,
task_id="task2",
step_name="OCR",
status="in_progress",
message="Processing..."
)
db_session.add(log2)
db_session.commit()
# Test pending filter
response = client.get("/files?status=pending")
assert response.status_code == 200
# Check that pending file is shown (HTML response)
assert "pending.pdf" in response.text
assert "processing.pdf" not in response.text
def test_status_filter_processing(self, client: TestClient, db_session):
"""Test filtering files by processing status."""
# Create file with in_progress status
file_record = FileRecord(
filehash="hash1",
original_filename="processing.pdf",
local_filename="/tmp/processing.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
log = ProcessingLog(
file_id=file_record.id,
task_id="task1",
step_name="OCR",
status="in_progress",
message="Processing..."
)
db_session.add(log)
db_session.commit()
# Test processing filter
response = client.get("/files?status=processing")
assert response.status_code == 200
assert "processing.pdf" in response.text
def test_status_filter_completed(self, client: TestClient, db_session):
"""Test filtering files by completed status."""
# Create file with success status
file_record = FileRecord(
filehash="hash1",
original_filename="completed.pdf",
local_filename="/tmp/completed.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
log = ProcessingLog(
file_id=file_record.id,
task_id="task1",
step_name="OCR",
status="success",
message="Completed"
)
db_session.add(log)
db_session.commit()
# Test completed filter
response = client.get("/files?status=completed")
assert response.status_code == 200
assert "completed.pdf" in response.text
def test_status_filter_failed(self, client: TestClient, db_session):
"""Test filtering files by failed status."""
# Create file with failure status
file_record = FileRecord(
filehash="hash1",
original_filename="failed.pdf",
local_filename="/tmp/failed.pdf",
file_size=1024,
mime_type="application/pdf"
)
db_session.add(file_record)
db_session.flush()
log = ProcessingLog(
file_id=file_record.id,
task_id="task1",
step_name="OCR",
status="failure",
message="Failed"
)
db_session.add(log)
db_session.commit()
# Test failed filter
response = client.get("/files?status=failed")
assert response.status_code == 200
assert "failed.pdf" in response.text