""" Tests for file status and metrics calculation using FileProcessingStep model. This test module verifies that: 1. Status calculation uses FileProcessingStep entries correctly 2. Metrics counting uses FileProcessingStep entries correctly 3. Files with completed steps show "completed" not "processing" """ import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from app.database import Base from app.models import FileRecord from app.utils.step_manager import get_file_overall_status, get_step_summary, initialize_file_steps, update_step_status @pytest.fixture def db_session(): """Create an in-memory SQLite database for testing.""" engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) SessionLocal = sessionmaker(bind=engine) session = SessionLocal() yield session session.close() Base.metadata.drop_all(engine) @pytest.mark.unit class TestFileStatusCalculation: """Test file status calculation using FileProcessingStep.""" def test_status_completed_when_all_steps_success(self, db_session): """ Test that status shows "completed" when all steps are success. """ # Create file and initialize steps file_record = FileRecord( filehash="test1", original_filename="test.pdf", local_filename="/tmp/test.pdf", file_size=1024 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) # Mark all steps as success from app.utils.step_manager import MAIN_PROCESSING_STEPS for step_name in MAIN_PROCESSING_STEPS: update_step_status(db_session, file_record.id, step_name, "success") result = get_file_overall_status(db_session, file_record.id) assert result["status"] == "completed" assert result["has_errors"] is False def test_status_processing_when_steps_in_progress(self, db_session): """ Test that status shows "processing" when there are in_progress steps. """ file_record = FileRecord( filehash="test2", original_filename="test2.pdf", local_filename="/tmp/test2.pdf", file_size=2048 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) # Mark some steps as success, one as in_progress update_step_status(db_session, file_record.id, "create_file_record", "success") update_step_status(db_session, file_record.id, "check_text", "in_progress") result = get_file_overall_status(db_session, file_record.id) assert result["status"] == "processing" assert result["has_errors"] is False def test_status_failed_when_steps_have_failure(self, db_session): """ Test that status shows "failed" when any step has failure status. """ file_record = FileRecord( filehash="test3", original_filename="test3.pdf", local_filename="/tmp/test3.pdf", file_size=3072 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) # Mark some steps as success, one as failure update_step_status(db_session, file_record.id, "create_file_record", "success") update_step_status(db_session, file_record.id, "check_text", "success") update_step_status(db_session, file_record.id, "extract_text", "failure", error_message="OCR failed") result = get_file_overall_status(db_session, file_record.id) assert result["status"] == "failed" assert result["has_errors"] is True @pytest.mark.unit class TestMetricsCounting: """Test metrics counting using FileProcessingStep.""" def test_main_steps_counted_correctly(self, db_session): """ Test that main processing steps are counted correctly. """ file_record = FileRecord( filehash="test4", original_filename="test4.pdf", local_filename="/tmp/test4.pdf", file_size=4096 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) # Mark some steps with different statuses update_step_status(db_session, file_record.id, "create_file_record", "success") update_step_status(db_session, file_record.id, "check_text", "success") update_step_status(db_session, file_record.id, "extract_text", "in_progress") summary = get_step_summary(db_session, file_record.id) # Should count each step once from app.utils.step_manager import MAIN_PROCESSING_STEPS assert summary["total_main_steps"] == len(MAIN_PROCESSING_STEPS) assert summary["main"]["success"] == 2 assert summary["main"]["in_progress"] == 1 def test_upload_tasks_counted_correctly(self, db_session): """ Test that upload tasks are counted correctly. """ file_record = FileRecord( filehash="test5", original_filename="test5.pdf", local_filename="/tmp/test5.pdf", file_size=5120 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) from app.utils.step_manager import add_upload_steps add_upload_steps(db_session, file_record.id, ["dropbox", "s3", "nextcloud"]) # Mark upload steps with different statuses update_step_status(db_session, file_record.id, "upload_to_dropbox", "success") update_step_status(db_session, file_record.id, "upload_to_s3", "failure") update_step_status(db_session, file_record.id, "upload_to_nextcloud", "in_progress") summary = get_step_summary(db_session, file_record.id) # Should count only upload_to_* steps (not queue_* steps) assert summary["total_upload_tasks"] == 3 assert summary["uploads"]["success"] == 1 assert summary["uploads"]["failure"] == 1 assert summary["uploads"]["in_progress"] == 1 def test_accurate_metrics_for_completed_file(self, db_session): """ Test metrics for a file with multiple successful uploads. """ file_record = FileRecord( filehash="test6", original_filename="test6.pdf", local_filename="/tmp/test6.pdf", file_size=6144 ) db_session.add(file_record) db_session.commit() initialize_file_steps(db_session, file_record.id) from app.utils.step_manager import add_upload_steps services = ["dropbox", "s3", "nextcloud", "google_drive", "onedrive", "webdav"] add_upload_steps(db_session, file_record.id, services) # Mark all uploads as success for service in services: update_step_status(db_session, file_record.id, f"upload_to_{service}", "success") summary = get_step_summary(db_session, file_record.id) # Should have 6 upload_to_* tasks assert summary["total_upload_tasks"] == 6 assert summary["uploads"]["success"] == 6 assert summary["uploads"]["failure"] == 0 assert summary["uploads"]["in_progress"] == 0 @pytest.mark.unit class TestGetFilesProcessingStatusEdgeCases: """Test edge cases for get_files_processing_status.""" def test_handles_empty_file_list(self, db_session): """Test with empty file list.""" from app.utils.file_status import get_files_processing_status result = get_files_processing_status(db_session, []) assert result == {} def test_handles_nonexistent_file_ids(self, db_session): """Test with file IDs that don't exist.""" from app.utils.file_status import get_files_processing_status result = get_files_processing_status(db_session, [99999, 99998]) # Should return status for these IDs even if they don't exist assert 99999 in result assert result[99999]["status"] == "pending" def test_handles_mixed_file_states(self, db_session): """Test with files in different states.""" from app.models import FileRecord from app.utils.file_status import get_files_processing_status from app.utils.step_manager import initialize_file_steps, update_step_status # Create multiple files with different states file1 = FileRecord( filehash="hash1", local_filename="/tmp/file1.pdf", original_filename="file1.pdf", file_size=1000, is_duplicate=False, ) file2 = FileRecord( filehash="hash2", local_filename="/tmp/file2.pdf", original_filename="file2.pdf", file_size=2000, is_duplicate=True, ) file3 = FileRecord( filehash="hash3", local_filename="/tmp/file3.pdf", original_filename="file3.pdf", file_size=3000, is_duplicate=False, ) db_session.add_all([file1, file2, file3]) db_session.commit() # Initialize steps for file1 and file3 initialize_file_steps(db_session, file1.id) initialize_file_steps(db_session, file3.id) # Mark file1 as failed update_step_status(db_session, file1.id, "extract_text", "failure") # Mark file3 as in progress update_step_status(db_session, file3.id, "extract_text", "in_progress") result = get_files_processing_status(db_session, [file1.id, file2.id, file3.id]) assert result[file1.id]["status"] == "failed" assert result[file2.id]["status"] == "duplicate" assert result[file3.id]["status"] == "processing" @pytest.mark.unit class TestComputeStatusFromLogsDeprecated: """Test the deprecated _compute_status_from_logs function.""" def test_empty_logs_list(self): """Test with empty logs list.""" from app.utils.file_status import _compute_status_from_logs result = _compute_status_from_logs([]) assert result["status"] == "pending" assert result["last_step"] is None assert result["has_errors"] is False def test_logs_with_multiple_steps(self, db_session): """Test logs from multiple steps.""" from app.models import ProcessingLog from app.utils.file_status import _compute_status_from_logs logs = [ ProcessingLog( file_id=1, task_id="task1", step_name="step1", status="success", message="Done", timestamp=None ), ProcessingLog( file_id=1, task_id="task2", step_name="step2", status="in_progress", message="Running", timestamp=None ), ] result = _compute_status_from_logs(logs) assert result["status"] == "processing" assert result["last_step"] == "step1" # First log in list @pytest.mark.unit class TestFileStatusMissingCoverage: """Tests for uncovered lines in file_status.py.""" @pytest.fixture def db_session(self): """Create an in-memory SQLite database for testing.""" engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) SessionLocal = sessionmaker(bind=engine) session = SessionLocal() yield session session.close() Base.metadata.drop_all(engine) def test_get_file_processing_status_duplicate(self, db_session): """Covers line 28: returns duplicate status for duplicate files.""" from app.utils.file_status import get_file_processing_status file_record = FileRecord( filehash="dup1", original_filename="dup.pdf", local_filename="/tmp/dup.pdf", file_size=100, is_duplicate=True, ) db_session.add(file_record) db_session.commit() result = get_file_processing_status(db_session, file_record.id) assert result["status"] == "duplicate" assert result["last_step"] == "check_for_duplicates" assert result["has_errors"] is False def test_get_files_processing_status_deduplication_enabled(self, db_session): """Covers line 105->109: check_for_duplicates step added when deduplication enabled.""" from unittest.mock import patch from app.utils.file_status import get_files_processing_status file_record = FileRecord( filehash="dedup1", original_filename="dedup.pdf", local_filename="/tmp/dedup.pdf", file_size=100, ) db_session.add(file_record) db_session.commit() with patch("app.config.settings") as ms: ms.enable_deduplication = True result = get_files_processing_status(db_session, [file_record.id]) # File has no steps, so should be pending assert result[file_record.id]["status"] == "pending" def test_get_files_processing_status_pending_steps(self, db_session): """Covers line 154: some steps exist but not all are completed (pending status).""" from datetime import datetime from app.models import FileProcessingStep from app.utils.file_status import get_files_processing_status file_record = FileRecord( filehash="pend1", original_filename="pending.pdf", local_filename="/tmp/pending.pdf", file_size=100, ) db_session.add(file_record) db_session.commit() # Add a step that is neither success nor failure nor in_progress step = FileProcessingStep( file_id=file_record.id, step_name="create_file_record", status="pending", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), ) db_session.add(step) db_session.commit() from unittest.mock import patch with patch("app.config.settings") as ms: ms.enable_deduplication = False result = get_files_processing_status(db_session, [file_record.id]) assert result[file_record.id]["status"] == "pending" def test_compute_status_from_logs_failed(self): """Covers lines 203: status is 'failed' when there's a failure log.""" from app.models import ProcessingLog from app.utils.file_status import _compute_status_from_logs logs = [ ProcessingLog( file_id=1, task_id="t1", step_name="extract", status="failure", message="Error", timestamp=None, ), ] result = _compute_status_from_logs(logs) assert result["status"] == "failed" assert result["has_errors"] is True def test_compute_status_from_logs_completed(self): """Covers lines 206-207: status is 'completed' when latest log is success.""" from app.models import ProcessingLog from app.utils.file_status import _compute_status_from_logs logs = [ ProcessingLog( file_id=1, task_id="t1", step_name="finalize", status="success", message="Done", timestamp=None, ), ] result = _compute_status_from_logs(logs) assert result["status"] == "completed" assert result["has_errors"] is False def test_compute_status_from_logs_pending_non_success(self): """Covers lines 208-209: status is 'pending' when latest log is not success/failure/in_progress.""" from app.models import ProcessingLog from app.utils.file_status import _compute_status_from_logs logs = [ ProcessingLog( file_id=1, task_id="t1", step_name="upload", status="queued", message="Waiting", timestamp=None, ), ] result = _compute_status_from_logs(logs) assert result["status"] == "pending" assert result["has_errors"] is False def test_get_files_processing_status_with_completed_steps(self, db_session): """Covers completed + skipped == total_steps with terminal step → completed status.""" from datetime import datetime from unittest.mock import patch from app.models import FileProcessingStep from app.utils.file_status import get_files_processing_status file_record = FileRecord( filehash="comp1", original_filename="comp.pdf", local_filename="/tmp/comp.pdf", file_size=100, ) db_session.add(file_record) db_session.commit() now = datetime.utcnow() for step_name, step_status in [ ("create_file_record", "success"), ("finalize_document_storage", "skipped"), ("send_to_all_destinations", "success"), ]: step = FileProcessingStep( file_id=file_record.id, step_name=step_name, status=step_status, created_at=now, updated_at=now, ) db_session.add(step) db_session.commit() with patch("app.config.settings") as ms: ms.enable_deduplication = False result = get_files_processing_status(db_session, [file_record.id]) assert result[file_record.id]["status"] == "completed" def test_get_files_processing_status_completed_with_pending_intermediate(self, db_session): """Test that terminal step success marks file completed even with pending intermediate steps. This tests the dynamic pipeline scenario where check_for_duplicates or extract_text might remain pending because the pipeline skipped them. """ from datetime import datetime from unittest.mock import patch from app.models import FileProcessingStep from app.utils.file_status import get_files_processing_status file_record = FileRecord( filehash="terminal_fallback", original_filename="fallback.pdf", local_filename="/tmp/fallback.pdf", file_size=100, ) db_session.add(file_record) db_session.commit() now = datetime.now() # Simulate: check_for_duplicates pending, rest succeeded, OCR skipped for step_name, step_status in [ ("check_for_duplicates", "pending"), ("create_file_record", "success"), ("check_text", "success"), ("extract_text", "success"), ("process_with_ocr", "skipped"), ("extract_metadata_with_gpt", "success"), ("embed_metadata_into_pdf", "success"), ("finalize_document_storage", "success"), ("send_to_all_destinations", "success"), ]: step = FileProcessingStep( file_id=file_record.id, step_name=step_name, status=step_status, created_at=now, updated_at=now, ) db_session.add(step) db_session.commit() with patch("app.config.settings") as ms: ms.enable_deduplication = True result = get_files_processing_status(db_session, [file_record.id]) # Terminal step succeeded → file should be completed despite pending dedup step assert result[file_record.id]["status"] == "completed"