Merge pull request #326 from christianlouis/copilot/increase-test-coverage-files-process

test: Increase coverage for app/api/files.py and app/api/process.py
This commit is contained in:
Christian Krakau-Louis
2026-02-14 01:10:36 +01:00
committed by GitHub
3 changed files with 581 additions and 4 deletions
+2 -1
View File
File diff suppressed because one or more lines are too long
+324
View File
@@ -823,3 +823,327 @@ class TestRetryPipelineStep:
_retry_pipeline_step(file, "unsupported_step", db_session)
assert exc_info.value.status_code == 400
assert "unsupported" in exc_info.value.detail.lower()
@pytest.mark.unit
class TestDeleteFileExceptions:
"""Test exception handling in delete operations."""
def test_delete_file_database_exception(self, client: TestClient, db_session):
"""Test database exception handling during delete."""
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename="/tmp/test.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
file_id = file.id
with patch("app.config.settings") as mock_settings, patch.object(
db_session, "delete", side_effect=Exception("Database error")
):
mock_settings.allow_file_delete = True
response = client.delete(f"/api/files/{file_id}")
assert response.status_code == 500
assert "Error deleting file record" in response.json()["detail"]
def test_bulk_delete_database_exception(self, client: TestClient, db_session):
"""Test database exception during bulk delete."""
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename="/tmp/test.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
file_id = file.id
# Simulate database error after file lookup
original_commit = db_session.commit
def failing_commit():
raise Exception("Database commit error")
with patch("app.config.settings") as mock_settings, patch.object(
db_session, "commit", side_effect=failing_commit
):
mock_settings.allow_file_delete = True
response = client.post("/api/files/bulk-delete", json=[file_id])
assert response.status_code == 500
assert "Error bulk deleting" in response.json()["detail"]
@pytest.mark.unit
class TestBulkReprocessExceptions:
"""Test exception handling in bulk reprocess operations."""
def test_bulk_reprocess_file_error_handling(self, client: TestClient, db_session, tmp_path):
"""Test that file errors are collected and returned."""
# Create file that doesn't exist on disk
file = FileRecord(
filehash="hash2",
original_filename="test2.pdf",
local_filename="/nonexistent/test2.pdf", # File doesn't exist
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.tasks.process_document.process_document") as mock_task:
mock_task.delay.return_value = Mock(id="task-1")
response = client.post("/api/files/bulk-reprocess", json=[file.id])
assert response.status_code == 200
data = response.json()
# Should have error due to missing file
assert data["status"] == "error" or len(data["errors"]) > 0
def test_bulk_reprocess_general_exception(self, client: TestClient, db_session, tmp_path):
"""Test general exception handling in bulk reprocess."""
test_file = tmp_path / "test.pdf"
test_file.write_text("test content")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(test_file),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
# Cause an exception during task queuing
with patch("app.tasks.process_document.process_document") as mock_task:
mock_task.delay.side_effect = Exception("Task queue error")
response = client.post("/api/files/bulk-reprocess", json=[file.id])
assert response.status_code == 200 # Errors are collected in response
data = response.json()
assert len(data["errors"]) == 1
@pytest.mark.unit
class TestRetryPipelineSteps:
"""Test retry functionality for various pipeline steps."""
def test_retry_azure_ocr_success(self, db_session, tmp_path):
"""Test retrying Azure OCR step."""
from app.api.files import _retry_pipeline_step
test_file = tmp_path / "test.pdf"
test_file.write_text("test content")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(test_file),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.tasks.process_with_azure_document_intelligence.process_with_azure_document_intelligence") as mock_task:
mock_task.delay.return_value = Mock(id="task-azure")
result = _retry_pipeline_step(file, "process_with_azure_document_intelligence", db_session)
assert result["task_id"] == "task-azure"
assert result["subtask_name"] == "process_with_azure_document_intelligence"
def test_retry_azure_ocr_file_not_on_disk(self, db_session):
"""Test Azure OCR retry fails when file not on disk."""
from app.api.files import _retry_pipeline_step
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename="/nonexistent/test.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with pytest.raises(HTTPException) as exc_info:
_retry_pipeline_step(file, "process_with_azure_document_intelligence", db_session)
assert exc_info.value.status_code == 400
assert "Local file not found on disk" in exc_info.value.detail
def test_retry_gpt_metadata_extraction_success(self, db_session, tmp_path):
"""Test retrying GPT metadata extraction step."""
from app.api.files import _retry_pipeline_step
test_file = tmp_path / "test.pdf"
# Create a minimal PDF file for text extraction
test_file.write_bytes(b"%PDF-1.4\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\nstartxref\n0\n%%EOF")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(test_file),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.tasks.extract_metadata_with_gpt.extract_metadata_with_gpt") as mock_task, patch(
"app.api.files._extract_text_from_pdf", return_value="Sample text"
):
mock_task.delay.return_value = Mock(id="task-gpt")
result = _retry_pipeline_step(file, "extract_metadata_with_gpt", db_session)
assert result["task_id"] == "task-gpt"
assert result["subtask_name"] == "extract_metadata_with_gpt"
def test_retry_gpt_metadata_file_not_on_disk(self, db_session):
"""Test GPT metadata retry fails when file not on disk."""
from app.api.files import _retry_pipeline_step
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename="/nonexistent/test.pdf",
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with pytest.raises(HTTPException) as exc_info:
_retry_pipeline_step(file, "extract_metadata_with_gpt", db_session)
assert exc_info.value.status_code == 400
assert "Local file not found on disk" in exc_info.value.detail
def test_retry_embed_metadata_success(self, db_session, tmp_path):
"""Test retrying embed metadata step."""
from app.api.files import _retry_pipeline_step
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"%PDF-1.4\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\nstartxref\n0\n%%EOF")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(test_file),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.tasks.extract_metadata_with_gpt.extract_metadata_with_gpt") as mock_task, patch(
"app.api.files._extract_text_from_pdf", return_value="Sample text"
):
mock_task.delay.return_value = Mock(id="task-embed")
result = _retry_pipeline_step(file, "embed_metadata_into_pdf", db_session)
assert result["task_id"] == "task-embed"
assert result["subtask_name"] == "embed_metadata_into_pdf"
@pytest.mark.unit
class TestRetryUploadTasks:
"""Test retry functionality for upload tasks."""
def test_retry_upload_dropbox_finds_processed_file(self, client, db_session, tmp_path):
"""Test retrying upload finds processed file by filehash."""
processed_dir = tmp_path / "processed"
processed_dir.mkdir()
test_file = processed_dir / "abc123.pdf"
test_file.write_text("processed content")
file = FileRecord(
filehash="abc123",
original_filename="test.pdf",
local_filename=str(tmp_path / "test.pdf"),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.api.files.settings") as mock_settings, patch(
"app.tasks.upload_to_dropbox.upload_to_dropbox"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_task.delay.return_value = Mock(id="task-dropbox")
response = client.post(f"/api/files/{file.id}/retry-subtask?subtask_name=upload_to_dropbox")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "task-dropbox"
# Verify it found the file by filehash
mock_task.delay.assert_called_once()
called_path = mock_task.delay.call_args[0][0]
assert "abc123.pdf" in called_path
def test_retry_upload_file_not_found(self, client, db_session, tmp_path):
"""Test retry upload fails when processed file not found."""
processed_dir = tmp_path / "processed"
processed_dir.mkdir()
file = FileRecord(
filehash="missing",
original_filename="test.pdf",
local_filename=str(tmp_path / "test.pdf"),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.api.files.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
response = client.post(f"/api/files/{file.id}/retry-subtask?subtask_name=upload_to_nextcloud")
assert response.status_code == 400
assert "Processed file not found" in response.json()["detail"]
@pytest.mark.unit
class TestAdditionalFileOperations:
"""Test additional file operations and edge cases."""
def test_file_preview_processed_file_fallback_paths(self, client: TestClient, db_session, tmp_path):
"""Test preview tries multiple paths for processed files."""
# Create file in second fallback location
processed_dir = tmp_path / "processed"
processed_dir.mkdir()
test_file = processed_dir / "test_processed.pdf"
test_file.write_text("processed content")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(tmp_path / "test.pdf"),
file_size=1024,
mime_type="application/pdf",
)
db_session.add(file)
db_session.commit()
with patch("app.api.files.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
response = client.get(f"/api/files/{file.id}/preview?version=processed")
# Should find file in one of the fallback paths
assert response.status_code in [200, 404] # Depends on which path exists
def test_file_download_missing_mime_type(self, client: TestClient, db_session, tmp_path):
"""Test download handles missing MIME type gracefully."""
test_file = tmp_path / "test.pdf"
test_file.write_text("test content")
file = FileRecord(
filehash="hash1",
original_filename="test.pdf",
local_filename=str(test_file),
file_size=1024,
mime_type=None, # Missing MIME type
)
db_session.add(file)
db_session.commit()
response = client.get(f"/api/files/{file.id}/download")
assert response.status_code == 200
# Should default to application/pdf
+255 -3
View File
@@ -1,5 +1,9 @@
"""Tests for app/api/process.py module."""
import os
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
@@ -12,43 +16,291 @@ class TestProcessEndpoints:
response = client.post("/api/process/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_process_file_success(self, client, tmp_path):
"""Test POST /api/process/ with existing file."""
# Create a test file
test_file = tmp_path / "test.pdf"
test_file.write_text("test content")
with patch("app.api.process.process_document") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/process/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
mock_task.delay.assert_called_once_with(str(test_file))
def test_send_to_dropbox_file_not_found(self, client):
"""Test POST /api/send_to_dropbox/ with non-existent file."""
response = client.post("/api/send_to_dropbox/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_send_to_dropbox_success(self, client, tmp_path):
"""Test POST /api/send_to_dropbox/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.upload_to_dropbox") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_dropbox/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
def test_send_to_paperless_file_not_found(self, client):
"""Test POST /api/send_to_paperless/ with non-existent file."""
response = client.post("/api/send_to_paperless/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_send_to_paperless_success(self, client, tmp_path):
"""Test POST /api/send_to_paperless/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.upload_to_paperless") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_paperless/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
def test_send_to_nextcloud_file_not_found(self, client):
"""Test POST /api/send_to_nextcloud/ with non-existent file."""
response = client.post("/api/send_to_nextcloud/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_send_to_nextcloud_success(self, client, tmp_path):
"""Test POST /api/send_to_nextcloud/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.upload_to_nextcloud") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_nextcloud/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
def test_send_to_google_drive_file_not_found(self, client):
"""Test POST /api/send_to_google_drive/ with non-existent file."""
response = client.post("/api/send_to_google_drive/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_send_to_google_drive_success(self, client, tmp_path):
"""Test POST /api/send_to_google_drive/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.upload_to_google_drive") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_google_drive/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
def test_send_to_onedrive_file_not_found(self, client):
"""Test POST /api/send_to_onedrive/ with non-existent file."""
response = client.post("/api/send_to_onedrive/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_send_to_onedrive_success(self, client, tmp_path):
"""Test POST /api/send_to_onedrive/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.upload_to_onedrive") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_onedrive/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
def test_send_to_all_destinations_file_not_found(self, client):
"""Test POST /api/send_to_all_destinations/ with non-existent file."""
response = client.post("/api/send_to_all_destinations/?file_path=nonexistent.pdf")
assert response.status_code == 400
def test_processall_endpoint(self, client, tmp_path):
"""Test POST /api/processall with no PDF files in workdir."""
from unittest.mock import patch
def test_send_to_all_destinations_success(self, client, tmp_path):
"""Test POST /api/send_to_all_destinations/ with existing file."""
test_file = tmp_path / "processed" / "test.pdf"
test_file.parent.mkdir(parents=True)
test_file.write_text("test content")
with patch("app.api.process.send_to_all_destinations") as mock_task:
mock_task.delay.return_value = Mock(id="test-task-id")
response = client.post(f"/api/send_to_all_destinations/?file_path={test_file}")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "test-task-id"
assert data["status"] == "queued"
assert data["file_path"] == str(test_file)
def test_processall_endpoint_empty_dir(self, client, tmp_path):
"""Test POST /api/processall with no PDF files in workdir."""
with patch("app.api.process.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert "No PDF files found" in data["message"]
def test_processall_endpoint_nonexistent_dir(self, client, tmp_path):
"""Test POST /api/processall with nonexistent workdir."""
with patch("app.api.process.settings") as mock_settings:
mock_settings.workdir = str(tmp_path / "nonexistent")
response = client.post("/api/processall")
assert response.status_code == 400
assert "does not exist" in response.json()["detail"]
def test_processall_single_file_no_throttle(self, client, tmp_path):
"""Test POST /api/processall with single PDF file (no throttling)."""
# Create a PDF file
test_file = tmp_path / "test1.pdf"
test_file.write_text("test content")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = 10
mock_task.delay.return_value = Mock(id="task-1")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert data["message"] == "Enqueued 1 PDFs for processing"
assert len(data["pdf_files"]) == 1
assert "test1.pdf" in data["pdf_files"]
assert len(data["task_ids"]) == 1
assert data["throttled"] is False
mock_task.delay.assert_called_once()
def test_processall_multiple_files_no_throttle(self, client, tmp_path):
"""Test POST /api/processall with multiple files below threshold."""
# Create 3 PDF files
for i in range(3):
(tmp_path / f"test{i}.pdf").write_text(f"test content {i}")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = 10
mock_task.delay.return_value = Mock(id="task-id")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert "Enqueued 3 PDFs for processing" in data["message"]
assert len(data["pdf_files"]) == 3
assert len(data["task_ids"]) == 3
assert data["throttled"] is False
assert mock_task.delay.call_count == 3
def test_processall_with_throttling(self, client, tmp_path):
"""Test POST /api/processall with throttling enabled."""
# Create 12 PDF files (above threshold of 10)
for i in range(12):
(tmp_path / f"test{i}.pdf").write_text(f"test content {i}")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = 10
mock_settings.processall_throttle_delay = 5
mock_task.apply_async.return_value = Mock(id="task-id")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert "Enqueued 12 PDFs for processing" in data["message"]
assert "(throttled over 55 seconds)" in data["message"]
assert len(data["pdf_files"]) == 12
assert len(data["task_ids"]) == 12
assert data["throttled"] is True
assert mock_task.apply_async.call_count == 12
# Verify countdown values
calls = mock_task.apply_async.call_args_list
for idx, call in enumerate(calls):
assert call[1]["countdown"] == idx * 5
def test_processall_ignores_non_pdf_files(self, client, tmp_path):
"""Test that processall only processes PDF files."""
# Create mixed files
(tmp_path / "test1.pdf").write_text("pdf content")
(tmp_path / "test2.PDF").write_text("pdf content uppercase")
(tmp_path / "test.txt").write_text("text content")
(tmp_path / "test.docx").write_text("word content")
(tmp_path / "test.jpg").write_text("image content")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = 10
mock_task.delay.return_value = Mock(id="task-id")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
# Should process 2 PDF files (case-insensitive)
assert len(data["pdf_files"]) == 2
assert "test1.pdf" in data["pdf_files"]
assert "test2.PDF" in data["pdf_files"]
assert mock_task.delay.call_count == 2
def test_processall_threshold_boundary(self, client, tmp_path):
"""Test processall at throttling threshold boundary."""
threshold = 5
# Test exactly at threshold (should not throttle)
for i in range(threshold):
(tmp_path / f"at_threshold_{i}.pdf").write_text(f"content {i}")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = threshold
mock_settings.processall_throttle_delay = 2
mock_task.delay.return_value = Mock(id="task-id")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert data["throttled"] is False
assert mock_task.delay.call_count == threshold
# Clean up and test above threshold (should throttle)
for f in tmp_path.glob("*.pdf"):
f.unlink()
for i in range(threshold + 1):
(tmp_path / f"above_threshold_{i}.pdf").write_text(f"content {i}")
with patch("app.api.process.settings") as mock_settings, patch(
"app.api.process.process_document"
) as mock_task:
mock_settings.workdir = str(tmp_path)
mock_settings.processall_throttle_threshold = threshold
mock_settings.processall_throttle_delay = 2
mock_task.apply_async.return_value = Mock(id="task-id")
response = client.post("/api/processall")
assert response.status_code == 200
data = response.json()
assert data["throttled"] is True
assert mock_task.apply_async.call_count == threshold + 1