Merge pull request #88 from christianlouis/copilot/fix-files-view-errors

Fix /files view status filtering and add bulk operations

app/api/files.py

@@ -275,6 +275,127 @@ def delete_file_record(request: Request, file_id: int, db: Session = Depends(get
             detail=f"Error deleting file record: {str(e)}"
         )
 
+@router.post("/files/bulk-delete")
+@require_login
+def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
+    """
+    Delete multiple file records from the database.
+    This only removes the database entries, not the actual files.
+    """
+    # Check if file deletion is allowed
+    if not settings.allow_file_delete:
+        raise HTTPException(
+            status_code=403,
+            detail="File deletion is disabled in the configuration"
+        )
+
+    try:
+        # Find all file records
+        file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
+        
+        if not file_records:
+            raise HTTPException(
+                status_code=404,
+                detail="No files found with the provided IDs"
+            )
+        
+        deleted_count = len(file_records)
+        deleted_ids = [f.id for f in file_records]
+        
+        # Log the deletion
+        logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}")
+        
+        # Delete all records
+        for file_record in file_records:
+            db.delete(file_record)
+        
+        db.commit()
+        
+        return {
+            "status": "success",
+            "message": f"Successfully deleted {deleted_count} file records",
+            "deleted_ids": deleted_ids
+        }
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        db.rollback()
+        logger.exception(f"Error bulk deleting file records: {str(e)}")
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error bulk deleting file records: {str(e)}"
+        )
+
+
+@router.post("/files/bulk-reprocess")
+@require_login
+def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
+    """
+    Reprocess multiple files by queuing them for processing.
+    """
+    try:
+        # Find all file records
+        file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
+        
+        if not file_records:
+            raise HTTPException(
+                status_code=404,
+                detail="No files found with the provided IDs"
+            )
+        
+        task_ids = []
+        processed_files = []
+        errors = []
+        
+        for file_record in file_records:
+            try:
+                # Check if local file exists
+                if not file_record.local_filename or not os.path.exists(file_record.local_filename):
+                    errors.append({
+                        "file_id": file_record.id,
+                        "filename": file_record.original_filename,
+                        "error": "Local file not found"
+                    })
+                    continue
+                
+                # Queue the file for processing
+                task = process_document.delay(file_record.local_filename)
+                task_ids.append(task.id)
+                processed_files.append({
+                    "file_id": file_record.id,
+                    "filename": file_record.original_filename,
+                    "task_id": task.id
+                })
+                
+                logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}")
+                
+            except Exception as e:
+                logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
+                errors.append({
+                    "file_id": file_record.id,
+                    "filename": file_record.original_filename,
+                    "error": str(e)
+                })
+        
+        return {
+            "status": "success" if processed_files else "error",
+            "message": f"Successfully queued {len(processed_files)} files for reprocessing",
+            "processed_files": processed_files,
+            "errors": errors if errors else None,
+            "task_ids": task_ids
+        }
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.exception(f"Error bulk reprocessing files: {str(e)}")
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error bulk reprocessing files: {str(e)}"
+        )
+
+
 @router.post("/ui-upload")
 @require_login
 async def ui_upload(request: Request, file: UploadFile = File(...)):

app/views/files.py

@@ -29,7 +29,7 @@ def files_page(
     try:
         # Import the model here to avoid circular imports
         from app.models import FileRecord, ProcessingLog
-        from sqlalchemy import desc, asc
+        from sqlalchemy import desc, asc, or_
         
         # Start with base query
         query = db.query(FileRecord)
@@ -42,6 +42,41 @@ def files_page(
         if mime_type:
             query = query.filter(FileRecord.mime_type == mime_type)
         
+        # Apply status filter (before pagination for correct counts)
+        if status:
+            # Subquery to get file IDs matching the status
+            if status == "pending":
+                # Files with no logs
+                subq = db.query(ProcessingLog.file_id).distinct()
+                query = query.filter(~FileRecord.id.in_(subq))
+            elif status == "processing":
+                # Files with in_progress logs
+                subq = db.query(ProcessingLog.file_id).filter(
+                    ProcessingLog.status == "in_progress"
+                ).distinct()
+                query = query.filter(FileRecord.id.in_(subq))
+            elif status == "failed":
+                # Files with failure logs
+                subq = db.query(ProcessingLog.file_id).filter(
+                    ProcessingLog.status == "failure"
+                ).distinct()
+                query = query.filter(FileRecord.id.in_(subq))
+            elif status == "completed":
+                # Files with success logs but no failures or in_progress
+                success_files = db.query(ProcessingLog.file_id).filter(
+                    ProcessingLog.status == "success"
+                ).distinct().subquery()
+                
+                failed_files = db.query(ProcessingLog.file_id).filter(
+                    or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
+                ).distinct().subquery()
+                
+                query = query.filter(
+                    FileRecord.id.in_(db.query(success_files.c.file_id))
+                ).filter(
+                    ~FileRecord.id.in_(db.query(failed_files.c.file_id))
+                )
+        
         # Get total count before pagination
         total_items = query.count()
         

frontend/templates/files.html

@@ -297,11 +297,34 @@
     </form>
   </div>
   
+  <!-- Bulk Actions Section -->
+  <div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
+    <div style="display: flex; justify-content: space-between; align-items: center;">
+      <div>
+        <strong id="selectedCount">0</strong> files selected
+      </div>
+      <div style="display: flex; gap: 1rem;">
+        <button type="button" onclick="bulkReprocess()" style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
+          <i class="fas fa-sync"></i> Reprocess Selected
+        </button>
+        <button type="button" onclick="bulkDelete()" style="padding: 0.5rem 1rem; background-color: #e53e3e; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
+          <i class="fas fa-trash"></i> Delete Selected
+        </button>
+        <button type="button" onclick="clearSelection()" style="padding: 0.5rem 1rem; background-color: #718096; color: white; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
+          Clear Selection
+        </button>
+      </div>
+    </div>
+  </div>
+  
   <!-- File table -->
   <div class="overflow-x-auto">
     <table class="file-table" id="fileTable">
       <thead>
         <tr>
+          <th style="width: 40px;">
+            <input type="checkbox" id="selectAll" onclick="toggleSelectAll()" title="Select all files on this page">
+          </th>
           <th class="sortable" onclick="sortTable('id')">
             ID
             <span class="sort-indicator {% if sort_by == 'id' %}active{% endif %}">
@@ -359,6 +382,9 @@
       <tbody>
         {% for file in files %}
         <tr onclick="viewFileDetail({{ file.id }}, event)">
+          <td onclick="event.stopPropagation();">
+            <input type="checkbox" class="file-checkbox" value="{{ file.id }}" onchange="updateBulkActionsBar()">
+          </td>
           <td>{{ file.id }}</td>
           <td>{{ file.original_filename }}</td>
           <td>{{ (file.file_size / 1024) | round(2) }} KB</td>
@@ -382,7 +408,7 @@
         </tr>
         {% else %}
         <tr>
-          <td colspan="7" class="text-center py-4">No files found</td>
+          <td colspan="8" class="text-center py-4">No files found</td>
         </tr>
         {% endfor %}
       </tbody>
@@ -513,6 +539,124 @@
     function clearFilters() {
       window.location.href = '/files';
     }
+    
+    // Bulk selection functionality
+    function toggleSelectAll() {
+      const selectAll = document.getElementById('selectAll');
+      const checkboxes = document.querySelectorAll('.file-checkbox');
+      checkboxes.forEach(checkbox => {
+        checkbox.checked = selectAll.checked;
+      });
+      updateBulkActionsBar();
+    }
+    
+    function updateBulkActionsBar() {
+      const checkboxes = document.querySelectorAll('.file-checkbox:checked');
+      const selectedCount = checkboxes.length;
+      const bulkActionsBar = document.getElementById('bulkActionsBar');
+      const selectedCountEl = document.getElementById('selectedCount');
+      
+      if (selectedCount > 0) {
+        bulkActionsBar.style.display = 'block';
+        selectedCountEl.textContent = selectedCount;
+      } else {
+        bulkActionsBar.style.display = 'none';
+      }
+      
+      // Update "select all" checkbox state
+      const allCheckboxes = document.querySelectorAll('.file-checkbox');
+      const selectAll = document.getElementById('selectAll');
+      selectAll.checked = allCheckboxes.length > 0 && selectedCount === allCheckboxes.length;
+    }
+    
+    function clearSelection() {
+      document.querySelectorAll('.file-checkbox').forEach(cb => cb.checked = false);
+      document.getElementById('selectAll').checked = false;
+      updateBulkActionsBar();
+    }
+    
+    function getSelectedFileIds() {
+      const checkboxes = document.querySelectorAll('.file-checkbox:checked');
+      return Array.from(checkboxes).map(cb => parseInt(cb.value));
+    }
+    
+    function bulkDelete() {
+      const fileIds = getSelectedFileIds();
+      if (fileIds.length === 0) {
+        alert('Please select files to delete');
+        return;
+      }
+      
+      if (!confirm(`Are you sure you want to delete ${fileIds.length} file(s)?`)) {
+        return;
+      }
+      
+      fetch('/api/files/bulk-delete', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(fileIds)
+      })
+      .then(response => {
+        if (!response.ok) {
+          return response.json().then(err => {
+            throw new Error(err.detail || 'Failed to delete files');
+          });
+        }
+        return response.json();
+      })
+      .then(data => {
+        alert(data.message);
+        window.location.reload();
+      })
+      .catch(error => {
+        console.error('Error:', error);
+        alert(`Error deleting files: ${error.message}`);
+      });
+    }
+    
+    function bulkReprocess() {
+      const fileIds = getSelectedFileIds();
+      if (fileIds.length === 0) {
+        alert('Please select files to reprocess');
+        return;
+      }
+      
+      if (!confirm(`Are you sure you want to reprocess ${fileIds.length} file(s)?`)) {
+        return;
+      }
+      
+      fetch('/api/files/bulk-reprocess', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(fileIds)
+      })
+      .then(response => {
+        if (!response.ok) {
+          return response.json().then(err => {
+            throw new Error(err.detail || 'Failed to reprocess files');
+          });
+        }
+        return response.json();
+      })
+      .then(data => {
+        if (data.errors && data.errors.length > 0) {
+          const errorMsg = data.errors.map(e => `${e.filename}: ${e.error}`).join('\n');
+          alert(`${data.message}\n\nErrors:\n${errorMsg}`);
+        } else {
+          alert(data.message);
+        }
+        clearSelection();
+        window.location.reload();
+      })
+      .catch(error => {
+        console.error('Error:', error);
+        alert(`Error reprocessing files: ${error.message}`);
+      });
+    }
   </script>
 </div>
 {% endblock %}

tests/test_bulk_operations.py

@@ -0,0 +1,274 @@
+"""
+Tests for bulk file operations (delete and reprocess).
+"""
+import pytest
+from fastapi.testclient import TestClient
+from app.models import FileRecord, ProcessingLog
+from unittest.mock import patch, MagicMock
+
+
+@pytest.mark.integration
+@pytest.mark.requires_db
+class TestBulkOperations:
+    """Tests for bulk file operations."""
+    
+    def test_bulk_delete_success(self, client: TestClient, db_session):
+        """Test bulk deletion of files."""
+        # Create sample files
+        file_ids = []
+        for i in range(3):
+            file_record = FileRecord(
+                filehash=f"hash{i}",
+                original_filename=f"test{i}.pdf",
+                local_filename=f"/tmp/test{i}.pdf",
+                file_size=1024,
+                mime_type="application/pdf"
+            )
+            db_session.add(file_record)
+            db_session.flush()
+            file_ids.append(file_record.id)
+        db_session.commit()
+        
+        # Bulk delete
+        response = client.post(
+            "/api/files/bulk-delete",
+            json=file_ids
+        )
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "success"
+        assert len(data["deleted_ids"]) == 3
+        
+        # Verify files are deleted
+        for file_id in file_ids:
+            file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
+            assert file_record is None
+    
+    def test_bulk_delete_empty_list(self, client: TestClient, db_session):
+        """Test bulk deletion with empty list."""
+        response = client.post(
+            "/api/files/bulk-delete",
+            json=[]
+        )
+        assert response.status_code == 404
+        data = response.json()
+        assert "No files found" in data["detail"]
+    
+    def test_bulk_delete_nonexistent_files(self, client: TestClient, db_session):
+        """Test bulk deletion of non-existent files."""
+        response = client.post(
+            "/api/files/bulk-delete",
+            json=[9999, 9998]
+        )
+        assert response.status_code == 404
+    
+    @patch('app.api.files.process_document')
+    def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
+        """Test bulk reprocessing of files."""
+        # Setup mock
+        mock_task = MagicMock()
+        mock_task.id = "test-task-id"
+        mock_process_document.delay.return_value = mock_task
+        
+        # Create sample files with local files that "exist"
+        file_ids = []
+        for i in range(2):
+            file_record = FileRecord(
+                filehash=f"hash{i}",
+                original_filename=f"test{i}.pdf",
+                local_filename=f"/tmp/test{i}.pdf",
+                file_size=1024,
+                mime_type="application/pdf"
+            )
+            db_session.add(file_record)
+            db_session.flush()
+            file_ids.append(file_record.id)
+        db_session.commit()
+        
+        # Mock os.path.exists to return True
+        with patch('os.path.exists', return_value=True):
+            response = client.post(
+                "/api/files/bulk-reprocess",
+                json=file_ids
+            )
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "success"
+        assert len(data["processed_files"]) == 2
+        assert len(data["task_ids"]) == 2
+    
+    @patch('app.api.files.process_document')
+    def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
+        """Test bulk reprocessing when some local files are missing."""
+        # Setup mock
+        mock_task = MagicMock()
+        mock_task.id = "test-task-id"
+        mock_process_document.delay.return_value = mock_task
+        
+        # Create sample files
+        file_ids = []
+        for i in range(2):
+            file_record = FileRecord(
+                filehash=f"hash{i}",
+                original_filename=f"test{i}.pdf",
+                local_filename=f"/tmp/test{i}.pdf" if i == 0 else None,  # Second file has no local file
+                file_size=1024,
+                mime_type="application/pdf"
+            )
+            db_session.add(file_record)
+            db_session.flush()
+            file_ids.append(file_record.id)
+        db_session.commit()
+        
+        # Mock os.path.exists to return False for missing file
+        with patch('os.path.exists', return_value=False):
+            response = client.post(
+                "/api/files/bulk-reprocess",
+                json=file_ids
+            )
+        
+        assert response.status_code == 200
+        data = response.json()
+        # All files should have errors since we mocked exists to return False
+        assert data["errors"] is not None
+        assert len(data["errors"]) == 2
+    
+    def test_bulk_reprocess_nonexistent_files(self, client: TestClient, db_session):
+        """Test bulk reprocessing of non-existent files."""
+        response = client.post(
+            "/api/files/bulk-reprocess",
+            json=[9999, 9998]
+        )
+        assert response.status_code == 404
+
+
+@pytest.mark.integration
+@pytest.mark.requires_db
+class TestStatusFilter:
+    """Tests for status filtering in files view."""
+    
+    def test_status_filter_pending(self, client: TestClient, db_session):
+        """Test filtering files by pending status."""
+        # Create files with different statuses
+        # File 1: Pending (no logs)
+        file1 = FileRecord(
+            filehash="hash1",
+            original_filename="pending.pdf",
+            local_filename="/tmp/pending.pdf",
+            file_size=1024,
+            mime_type="application/pdf"
+        )
+        db_session.add(file1)
+        
+        # File 2: Processing (has in_progress log)
+        file2 = FileRecord(
+            filehash="hash2",
+            original_filename="processing.pdf",
+            local_filename="/tmp/processing.pdf",
+            file_size=1024,
+            mime_type="application/pdf"
+        )
+        db_session.add(file2)
+        db_session.flush()
+        
+        log2 = ProcessingLog(
+            file_id=file2.id,
+            task_id="task2",
+            step_name="OCR",
+            status="in_progress",
+            message="Processing..."
+        )
+        db_session.add(log2)
+        db_session.commit()
+        
+        # Test pending filter
+        response = client.get("/files?status=pending")
+        assert response.status_code == 200
+        # Check that pending file is shown (HTML response)
+        assert "pending.pdf" in response.text
+        assert "processing.pdf" not in response.text
+    
+    def test_status_filter_processing(self, client: TestClient, db_session):
+        """Test filtering files by processing status."""
+        # Create file with in_progress status
+        file_record = FileRecord(
+            filehash="hash1",
+            original_filename="processing.pdf",
+            local_filename="/tmp/processing.pdf",
+            file_size=1024,
+            mime_type="application/pdf"
+        )
+        db_session.add(file_record)
+        db_session.flush()
+        
+        log = ProcessingLog(
+            file_id=file_record.id,
+            task_id="task1",
+            step_name="OCR",
+            status="in_progress",
+            message="Processing..."
+        )
+        db_session.add(log)
+        db_session.commit()
+        
+        # Test processing filter
+        response = client.get("/files?status=processing")
+        assert response.status_code == 200
+        assert "processing.pdf" in response.text
+    
+    def test_status_filter_completed(self, client: TestClient, db_session):
+        """Test filtering files by completed status."""
+        # Create file with success status
+        file_record = FileRecord(
+            filehash="hash1",
+            original_filename="completed.pdf",
+            local_filename="/tmp/completed.pdf",
+            file_size=1024,
+            mime_type="application/pdf"
+        )
+        db_session.add(file_record)
+        db_session.flush()
+        
+        log = ProcessingLog(
+            file_id=file_record.id,
+            task_id="task1",
+            step_name="OCR",
+            status="success",
+            message="Completed"
+        )
+        db_session.add(log)
+        db_session.commit()
+        
+        # Test completed filter
+        response = client.get("/files?status=completed")
+        assert response.status_code == 200
+        assert "completed.pdf" in response.text
+    
+    def test_status_filter_failed(self, client: TestClient, db_session):
+        """Test filtering files by failed status."""
+        # Create file with failure status
+        file_record = FileRecord(
+            filehash="hash1",
+            original_filename="failed.pdf",
+            local_filename="/tmp/failed.pdf",
+            file_size=1024,
+            mime_type="application/pdf"
+        )
+        db_session.add(file_record)
+        db_session.flush()
+        
+        log = ProcessingLog(
+            file_id=file_record.id,
+            task_id="task1",
+            step_name="OCR",
+            status="failure",
+            message="Failed"
+        )
+        db_session.add(log)
+        db_session.commit()
+        
+        # Test failed filter
+        response = client.get("/files?status=failed")
+        assert response.status_code == 200
+        assert "failed.pdf" in response.text