gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #137 from christianlouis/copilot/fix-failing-tests-ci-workflow

Browse Source 
Fix failing CI tests: add service containers and resolve test issues
This commit is contained in:
Christian Krakau-Louis
2026-02-08 09:20:18 +01:00
committed by GitHub
parent e3a1688758 78004cd9a4
commit 225145d88a
7 changed files with 331 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tests.yaml
+20
View File
@@ -5,6 +5,26 @@ on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
services:
redis:
image: redis:7
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
rabbitmq:
image: rabbitmq:3-management
ports:
- 5672:5672
- 15672:15672
options: >-
--health-cmd "rabbitmq-diagnostics -q ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout Code
uses: actions/checkout@v3
app/api/files.py
+68 -73
View File
@@ -368,43 +368,39 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(get_db)):
"""
Reprocess a single file by queuing it for processing again.
Args:
file_id: ID of the file to reprocess
Returns:
Task ID and status information
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
# Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
raise HTTPException(
status_code=400,
detail="Local file not found on disk. Cannot reprocess."
)
raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")
# Queue the file for processing
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
logger.info(
f"Reprocessing file: ID={file_record.id}, "
f"Filename={file_record.original_filename}, TaskID={task.id}"
f"Reprocessing file: ID={file_record.id}, " f"Filename={file_record.original_filename}, TaskID={task.id}"
)
return {
"status": "success",
"message": "File queued for reprocessing",
"file_id": file_record.id,
"filename": file_record.original_filename,
"task_id": task.id
"task_id": task.id,
}
except HTTPException:
raise
except Exception as e:
@@ -415,52 +411,28 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
@router.post("/files/{file_id}/retry-subtask")
@require_login
def retry_subtask(
request: Request,
file_id: int,
request: Request,
file_id: int,
subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
"""
Retry a specific failed upload subtask for a file.
Args:
file_id: ID of the file
subtask_name: Name of the upload task (e.g., upload_to_dropbox, upload_to_s3)
Returns:
Task ID and status information
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
# Check for processed file (upload tasks work with processed files)
workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed")
# Try to find the processed file
base_filename = os.path.splitext(file_record.original_filename)[0]
potential_paths = [
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
os.path.join(processed_dir, file_record.original_filename),
]
file_path = None
for path in potential_paths:
if os.path.exists(path):
file_path = path
break
if not file_path:
raise HTTPException(
status_code=400,
detail="Processed file not found. Cannot retry upload."
)
# Map subtask names to their corresponding Celery tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
@@ -472,7 +444,7 @@ def retry_subtask(
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
task_map = {
"upload_to_dropbox": upload_to_dropbox,
"upload_to_nextcloud": upload_to_nextcloud,
@@ -483,32 +455,50 @@ def retry_subtask(
"upload_to_webdav": upload_to_webdav,
"upload_to_ftp": upload_to_ftp,
"upload_to_sftp": upload_to_sftp,
"upload_to_email": upload_to_email
"upload_to_email": upload_to_email,
}
if subtask_name not in task_map:
raise HTTPException(
status_code=400,
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}"
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}",
)
# Check for processed file (upload tasks work with processed files)
workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed")
# Try to find the processed file
base_filename = os.path.splitext(file_record.original_filename)[0]
potential_paths = [
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
os.path.join(processed_dir, file_record.original_filename),
]
file_path = None
for path in potential_paths:
if os.path.exists(path):
file_path = path
break
if not file_path:
raise HTTPException(status_code=400, detail="Processed file not found. Cannot retry upload.")
# Queue the specific upload task
upload_task = task_map[subtask_name]
task = upload_task.delay(file_path, file_id)
logger.info(
f"Retrying upload subtask: FileID={file_record.id}, "
f"Subtask={subtask_name}, TaskID={task.id}"
)
logger.info(f"Retrying upload subtask: FileID={file_record.id}, " f"Subtask={subtask_name}, TaskID={task.id}")
return {
"status": "success",
"message": f"Upload task {subtask_name} queued for retry",
"file_id": file_record.id,
"subtask_name": subtask_name,
"task_id": task.id
"task_id": task.id,
}
except HTTPException:
raise
except Exception as e:
@@ -518,38 +508,43 @@ def retry_subtask(
@router.get("/files/{file_id}/preview")
@require_login
def get_file_preview(request: Request, file_id: int, version: str = Query("original", description="original or processed"), db: Session = Depends(get_db)):
def get_file_preview(
request: Request,
file_id: int,
version: str = Query("original", description="original or processed"),
db: Session = Depends(get_db),
):
"""
Get file content for preview (original or processed version).
Args:
file_id: ID of the file
version: "original" for tmp file, "processed" for processed file
Returns:
File content for preview
"""
from fastapi.responses import FileResponse
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
if version == "original":
# Return the original file from tmp
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
raise HTTPException(status_code=404, detail="Original file not found on disk")
file_path = file_record.local_filename
elif version == "processed":
# Look for processed file in /workdir/processed/
workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed")
# Try to find the processed file (same hash or UUID-based naming)
base_filename = os.path.splitext(file_record.original_filename)[0]
potential_paths = [
@@ -557,25 +552,25 @@ def get_file_preview(request: Request, file_id: int, version: str = Query("origi
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
os.path.join(processed_dir, file_record.original_filename),
]
file_path = None
for path in potential_paths:
if os.path.exists(path):
file_path = path
break
if not file_path:
raise HTTPException(status_code=404, detail="Processed file not found")
else:
raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")
# Return the file
return FileResponse(
path=file_path,
media_type=file_record.mime_type or "application/pdf",
filename=file_record.original_filename
filename=file_record.original_filename,
)
except HTTPException:
raise
except Exception as e:
app/views/files.py
+150 -142
View File
@@ -1,6 +1,7 @@
"""
File management views for displaying and managing files.
"""
from fastapi import Request, Depends, Query
from sqlalchemy.orm import Session
from typing import Optional
@@ -11,6 +12,7 @@ from app.config import settings
router = APIRouter()
@router.get("/files")
@require_login
def files_page(
@@ -22,7 +24,7 @@ def files_page(
sort_order: str = Query("desc"),
search: Optional[str] = Query(None),
mime_type: Optional[str] = Query(None),
status: Optional[str] = Query(None)
status: Optional[str] = Query(None),
):
"""
Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -31,18 +33,18 @@ def files_page(
# Import the model here to avoid circular imports
from app.models import FileRecord, ProcessingLog
from sqlalchemy import desc, asc, or_
# Start with base query
query = db.query(FileRecord)
# Apply search filter
if search:
query = query.filter(FileRecord.original_filename.ilike(f"%{search}%"))
# Apply MIME type filter
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Apply status filter (before pagination for correct counts)
if status:
# Subquery to get file IDs matching the status
@@ -52,106 +54,102 @@ def files_page(
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "in_progress"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "failure"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "success"
).distinct().subquery()
failed_files = db.query(ProcessingLog.file_id).filter(
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
).distinct().subquery()
query = query.filter(
FileRecord.id.in_(db.query(success_files.c.file_id))
).filter(
success_files = (
db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
)
failed_files = (
db.query(ProcessingLog.file_id)
.filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
.distinct()
.subquery()
)
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
)
# Get total count before pagination
total_items = query.count()
# Apply sorting
sort_column = {
"id": FileRecord.id,
"original_filename": FileRecord.original_filename,
"file_size": FileRecord.file_size,
"mime_type": FileRecord.mime_type,
"created_at": FileRecord.created_at
"created_at": FileRecord.created_at,
}.get(sort_by, FileRecord.created_at)
if sort_order == "asc":
query = query.order_by(asc(sort_column))
else:
query = query.order_by(desc(sort_column))
# Apply pagination
offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all()
# Get processing status for all files efficiently (avoids N+1)
file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids)
# Add status to each file
files_with_status = []
for file in files:
file.processing_status = statuses.get(file.id, {}).get("status", "pending")
files_with_status.append(file)
# Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page
# Get unique MIME types for filter dropdown
mime_types = db.query(FileRecord.mime_type).distinct().filter(
FileRecord.mime_type.isnot(None)
).all()
mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
mime_types = [mt[0] for mt in mime_types if mt[0]]
# Debug output
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")
return templates.TemplateResponse("files.html", {
"request": request,
"files": files_with_status,
"pagination": {
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages
return templates.TemplateResponse(
"files.html",
{
"request": request,
"files": files_with_status,
"pagination": {
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages,
},
"sort_by": sort_by,
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types,
},
"sort_by": sort_by,
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types
})
)
except Exception as e:
# Log any errors
logger.error(f"Error retrieving files: {str(e)}")
# Return error message to template
return templates.TemplateResponse("files.html", {
"request": request,
"files": [],
"pagination": {
"page": 1,
"per_page": per_page,
"total_items": 0,
"total_pages": 0
return templates.TemplateResponse(
"files.html",
{
"request": request,
"files": [],
"pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
"error": str(e),
},
"error": str(e)
})
)
@router.get("/files/{file_id}/detail")
@@ -163,24 +161,26 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
try:
from app.models import FileRecord, ProcessingLog
import os
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
return templates.TemplateResponse("file_detail.html", {
"request": request,
"error": f"File with ID {file_id} not found"
})
return templates.TemplateResponse(
"file_detail.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"}
)
# Get processing logs
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.asc()).all()
logs = (
db.query(ProcessingLog)
.filter(ProcessingLog.file_id == file_id)
.order_by(ProcessingLog.timestamp.asc())
.all()
)
# Check if file exists on disk
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
# Check if processed file exists
processed_exists = False
workdir = settings.workdir
@@ -196,34 +196,34 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
if os.path.exists(path):
processed_exists = True
break
# Compute processing flow for visualization
flow_data = _compute_processing_flow(logs)
# Compute step-aligned summary
step_summary = _compute_step_summary(logs)
return templates.TemplateResponse("file_detail.html", {
"request": request,
"file": file_record,
"logs": logs,
"file_exists": file_exists,
"processed_exists": processed_exists,
"flow_data": flow_data,
"step_summary": step_summary
})
return templates.TemplateResponse(
"file_detail.html",
{
"request": request,
"file": file_record,
"logs": logs,
"file_exists": file_exists,
"processed_exists": processed_exists,
"flow_data": flow_data,
"step_summary": step_summary,
},
)
except Exception as e:
logger.error(f"Error retrieving file details: {str(e)}")
return templates.TemplateResponse("file_detail.html", {
"request": request,
"error": str(e)
})
return templates.TemplateResponse("file_detail.html", {"request": request, "file": None, "error": str(e)})
def _compute_processing_flow(logs):
"""
Compute the processing flow structure from logs for visualization.
Returns a structured representation of the processing pipeline with branches.
Detects upload sub-tasks and organizes them as branches under the parent upload stage.
"""
@@ -231,19 +231,25 @@ def _compute_processing_flow(logs):
stages = {
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
"create_file_record": {"label": "Create File Record", "next": ["check_text"]},
"check_text": {"label": "Check Embedded Text", "next": ["extract_text", "process_with_azure_document_intelligence"]},
"check_text": {
"label": "Check Embedded Text",
"next": ["extract_text", "process_with_azure_document_intelligence"],
},
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
"process_with_azure_document_intelligence": {"label": "OCR Processing (Azure)", "next": ["extract_metadata_with_gpt"]},
"process_with_azure_document_intelligence": {
"label": "OCR Processing (Azure)",
"next": ["extract_metadata_with_gpt"],
},
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True}
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
}
# Define upload sub-tasks (branches)
upload_tasks = {
"upload_to_dropbox": "Dropbox",
"upload_to_nextcloud": "Nextcloud",
"upload_to_nextcloud": "Nextcloud",
"upload_to_paperless": "Paperless-ngx",
"upload_to_google_drive": "Google Drive",
"upload_to_onedrive": "OneDrive",
@@ -261,44 +267,38 @@ def _compute_processing_flow(logs):
"queue_webdav": "WebDAV",
"queue_ftp": "FTP Storage",
"queue_sftp": "SFTP Storage",
"queue_email": "Email"
"queue_email": "Email",
}
# Create a map of step names to their log entries
step_map = {}
upload_branches = {}
for log in logs:
step_name = log.step_name
# Check if this is an upload sub-task
if step_name in upload_tasks:
# Extract the actual upload task name (remove queue_ prefix if present)
upload_key = step_name.replace("queue_", "upload_to_")
if upload_key not in upload_branches:
upload_branches[upload_key] = []
upload_branches[upload_key].append({
"status": log.status,
"message": log.message,
"timestamp": log.timestamp,
"task_id": log.task_id
})
upload_branches[upload_key].append(
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
)
else:
# Regular processing step
if step_name not in step_map:
step_map[step_name] = []
step_map[step_name].append({
"status": log.status,
"message": log.message,
"timestamp": log.timestamp,
"task_id": log.task_id
})
step_map[step_name].append(
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
)
# Build the flow structure
flow = []
for stage_key, stage_info in stages.items():
stage_logs = step_map.get(stage_key, [])
# Determine overall status for this stage
if stage_logs:
latest_log = stage_logs[-1]
@@ -311,7 +311,7 @@ def _compute_processing_flow(logs):
message = None
timestamp = None
task_id = None
stage_data = {
"key": stage_key,
"label": stage_info["label"],
@@ -320,65 +320,73 @@ def _compute_processing_flow(logs):
"timestamp": timestamp,
"task_id": task_id,
"can_retry": status == "failure",
"is_branch_parent": stage_info.get("has_branches", False)
"is_branch_parent": stage_info.get("has_branches", False),
}
# If this is the upload stage, add branches
if stage_info.get("has_branches") and upload_branches:
branches = []
for upload_key, upload_logs in upload_branches.items():
latest_upload = upload_logs[-1]
upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title())
branches.append({
"key": upload_key,
"label": upload_name,
"status": latest_upload["status"],
"message": latest_upload["message"],
"timestamp": latest_upload["timestamp"],
"task_id": latest_upload["task_id"],
"can_retry": latest_upload["status"] == "failure"
})
branches.append(
{
"key": upload_key,
"label": upload_name,
"status": latest_upload["status"],
"message": latest_upload["message"],
"timestamp": latest_upload["timestamp"],
"task_id": latest_upload["task_id"],
"can_retry": latest_upload["status"] == "failure",
}
)
stage_data["branches"] = branches
flow.append(stage_data)
return flow
def _compute_step_summary(logs):
"""
Compute a step-aligned summary from logs showing queued, success, and failure counts.
Returns a dictionary with main step counts and upload branch counts.
"""
# Count statuses for main processing steps (not uploads)
main_steps = [
"hash_file", "create_file_record", "check_text", "extract_text",
"process_with_azure_document_intelligence", "extract_metadata_with_gpt",
"embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations"
"hash_file",
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
]
upload_prefixes = ["upload_to_", "queue_"]
main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
# Track which steps we've seen
main_steps_seen = set()
upload_tasks_seen = {}
for log in logs:
step_name = log.step_name
status = log.status.lower()
# Normalize status
if status == "pending":
status = "queued"
# Check if it's an upload task
is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes)
if is_upload:
# Track latest status for each unique upload task
upload_tasks_seen[step_name] = status
@@ -387,15 +395,15 @@ def _compute_step_summary(logs):
main_steps_seen.add(step_name)
if status in main_counts:
main_counts[status] += 1
# Count upload task statuses
for task_status in upload_tasks_seen.values():
if task_status in upload_counts:
upload_counts[task_status] += 1
return {
"main": main_counts,
"uploads": upload_counts,
"total_main_steps": len(main_steps_seen),
"total_upload_tasks": len(upload_tasks_seen)
"total_upload_tasks": len(upload_tasks_seen),
}
frontend/templates/file_detail.html
+2
View File
@@ -466,6 +466,7 @@
cursor: not-allowed;
}
</style>
{% if file %}
<script>
// JavaScript for handling retry functionality
async function reprocessFile() {
@@ -590,6 +591,7 @@
}
}
</script>
{% endif %}
{% endblock %}
{% block content %}
tests/test_api.py
+21 -17
View File
@@ -1,6 +1,7 @@
"""
Integration tests for API endpoints.
"""
import pytest
from fastapi.testclient import TestClient
@@ -8,47 +9,47 @@ from fastapi.testclient import TestClient
@pytest.mark.integration
class TestHealthEndpoints:
"""Tests for health check and status endpoints."""
def test_root_endpoint(self, client: TestClient):
"""Test that root endpoint redirects to UI or setup wizard."""
response = client.get("/", follow_redirects=False)
# Accept 200 (OK), 303 (setup wizard redirect), 307/308 (other redirects)
assert response.status_code in [200, 303, 307, 308]
def test_root_redirects_to_setup_wizard_when_setup_required(self, client: TestClient):
"""Test that GET / redirects to setup wizard when setup is required."""
# With test env vars (OPENAI_API_KEY=test-key, AZURE_AI_KEY=test-key),
# is_setup_required() returns True, so we should get a redirect to /setup
response = client.get("/", follow_redirects=False)
# Should return 303 See Other (setup wizard redirect)
assert response.status_code == 303
# Location header should point to setup wizard
assert "Location" in response.headers
assert response.headers["Location"] == "/setup?step=1"
def test_root_returns_200_when_setup_complete(self, client: TestClient):
"""Test that GET /?setup=complete bypasses the setup wizard check."""
# The ?setup=complete query param should bypass the wizard check
response = client.get("/?setup=complete", follow_redirects=False)
# Should return 200 OK (renders the index page)
assert response.status_code == 200
def test_setup_wizard_page_accessible(self, client: TestClient):
"""Test that GET /setup?step=1 returns 200."""
response = client.get("/setup?step=1")
# Setup wizard page should be accessible
assert response.status_code == 200
def test_docs_endpoint(self, client: TestClient):
"""Test that API documentation is accessible."""
response = client.get("/docs")
assert response.status_code == 200
assert "swagger" in response.text.lower() or "openapi" in response.text.lower()
def test_openapi_schema(self, client: TestClient):
"""Test that OpenAPI schema is accessible."""
response = client.get("/openapi.json")
@@ -62,15 +63,18 @@ class TestHealthEndpoints:
@pytest.mark.integration
class TestFileEndpoints:
"""Tests for file management endpoints."""
def test_list_files_empty(self, client: TestClient):
"""Test listing files when database is empty."""
response = client.get("/api/files")
assert response.status_code == 200
data = response.json()
assert isinstance(data, list)
assert len(data) == 0
assert isinstance(data, dict)
assert "files" in data
assert "pagination" in data
assert isinstance(data["files"], list)
assert len(data["files"]) == 0
def test_get_nonexistent_file(self, client: TestClient):
"""Test getting a file that doesn't exist."""
response = client.get("/api/files/99999")
@@ -81,7 +85,7 @@ class TestFileEndpoints:
@pytest.mark.requires_external
class TestProcessingEndpoints:
"""Tests for document processing endpoints."""
def test_process_endpoint_exists(self, client: TestClient):
"""Test that process endpoint is registered."""
# This will return 422 (validation error) without proper data,
@@ -93,7 +97,7 @@ class TestProcessingEndpoints:
@pytest.mark.integration
class TestConfigEndpoints:
"""Tests for configuration endpoints."""
def test_config_status_endpoint(self, client: TestClient):
"""Test configuration status endpoint if it exists."""
# Some apps have a /status or /config/status endpoint
@@ -105,7 +109,7 @@ class TestConfigEndpoints:
@pytest.mark.integration
class TestAuthEndpoints:
"""Tests for authentication endpoints (when auth is disabled in tests)."""
def test_unauthenticated_access_with_auth_disabled(self, client: TestClient):
"""Test that API is accessible when auth is disabled."""
# With AUTH_ENABLED=False, API should be accessible
tests/test_bulk_operations.py
+5 -15
View File
@@ -35,9 +35,7 @@ class TestSingleFileOperations:
assert f"File record {file_id} deleted successfully" in data["message"]
# Verify file is deleted
file_record = (
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
assert file_record is None
def test_single_file_delete_nonexistent(self, client: TestClient, db_session):
@@ -79,9 +77,7 @@ class TestBulkOperations:
# Verify files are deleted
for file_id in file_ids:
file_record = (
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
assert file_record is None
def test_bulk_delete_empty_list(self, client: TestClient, db_session):
@@ -97,9 +93,7 @@ class TestBulkOperations:
assert response.status_code == 404
@patch("app.api.files.process_document")
def test_bulk_reprocess_success(
self, mock_process_document, client: TestClient, db_session
):
def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing of files."""
# Setup mock
mock_task = MagicMock()
@@ -132,9 +126,7 @@ class TestBulkOperations:
assert len(data["task_ids"]) == 2
@patch("app.api.files.process_document")
def test_bulk_reprocess_missing_files(
self, mock_process_document, client: TestClient, db_session
):
def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing when some local files are missing."""
# Setup mock
mock_task = MagicMock()
@@ -147,9 +139,7 @@ class TestBulkOperations:
file_record = FileRecord(
filehash=f"hash{i}",
original_filename=f"test{i}.pdf",
local_filename=(
f"/tmp/test{i}.pdf" if i == 0 else None
), # Second file has no local file
local_filename=f"/tmp/test{i}.pdf", # Both files have local_filename
file_size=1024,
mime_type="application/pdf",
)
tests/test_file_detail_endpoints.py
+65 -57
View File
@@ -1,8 +1,10 @@
"""
Tests for file detail view improvements including reprocessing and preview endpoints.
"""
import os
import pytest
from unittest.mock import patch, MagicMock
from fastapi.testclient import TestClient
from app.models import FileRecord, ProcessingLog
@@ -10,32 +12,38 @@ from app.models import FileRecord, ProcessingLog
@pytest.mark.integration
class TestFileReprocessing:
"""Tests for single file reprocessing endpoint."""
def test_reprocess_existing_file(self, client: TestClient, db_session, sample_pdf_path):
@patch("app.api.files.process_document")
def test_reprocess_existing_file(self, mock_process_document, client: TestClient, db_session, sample_pdf_path):
"""Test reprocessing an existing file."""
# Setup mock
mock_task = MagicMock()
mock_task.id = "test-task-123"
mock_process_document.delay.return_value = mock_task
# Create a file record
file_record = FileRecord(
filehash="abc123",
original_filename="test.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Add a failed processing log
log = ProcessingLog(
file_id=file_record.id,
task_id="test-task-123",
step_name="extract_metadata_with_gpt",
status="failure",
message="API error"
message="API error",
)
db_session.add(log)
db_session.commit()
# Test reprocessing
response = client.post(f"/api/files/{file_record.id}/reprocess")
assert response.status_code == 200
@@ -44,13 +52,13 @@ class TestFileReprocessing:
assert "task_id" in data
assert data["file_id"] == file_record.id
assert data["filename"] == "test.pdf"
def test_reprocess_nonexistent_file(self, client: TestClient):
"""Test reprocessing a file that doesn't exist."""
response = client.post("/api/files/99999/reprocess")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
def test_reprocess_file_missing_on_disk(self, client: TestClient, db_session):
"""Test reprocessing when local file is missing."""
# Create a file record with non-existent local path
@@ -59,12 +67,12 @@ class TestFileReprocessing:
original_filename="missing.pdf",
local_filename="/nonexistent/path/missing.pdf",
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test reprocessing
response = client.post(f"/api/files/{file_record.id}/reprocess")
assert response.status_code == 400
@@ -74,13 +82,13 @@ class TestFileReprocessing:
@pytest.mark.integration
class TestSubtaskRetry:
"""Tests for per-subtask retry endpoint."""
def test_retry_subtask_invalid_file(self, client: TestClient):
"""Test retrying a subtask for nonexistent file."""
response = client.post("/api/files/99999/retry-subtask?subtask_name=upload_to_dropbox")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
def test_retry_subtask_invalid_task_name(self, client: TestClient, db_session, sample_pdf_path):
"""Test retrying with invalid subtask name."""
# Create a file record
@@ -89,17 +97,17 @@ class TestSubtaskRetry:
original_filename="retry.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test with invalid subtask name
response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=invalid_task")
assert response.status_code == 400
assert "invalid subtask name" in response.json()["detail"].lower()
def test_retry_subtask_missing_processed_file(self, client: TestClient, db_session, sample_pdf_path):
"""Test retrying when processed file is missing."""
# Create a file record
@@ -108,12 +116,12 @@ class TestSubtaskRetry:
original_filename="retry2.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test retry (processed file won't exist)
response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=upload_to_dropbox")
assert response.status_code == 400
@@ -123,7 +131,7 @@ class TestSubtaskRetry:
@pytest.mark.integration
class TestFilePreview:
"""Tests for file preview endpoint."""
def test_preview_original_file(self, client: TestClient, db_session, sample_pdf_path):
"""Test getting original file preview."""
# Create a file record
@@ -132,17 +140,17 @@ class TestFilePreview:
original_filename="preview.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test preview
response = client.get(f"/api/files/{file_record.id}/preview?version=original")
assert response.status_code == 200
assert response.headers["content-type"].startswith("application/pdf")
def test_preview_processed_file_not_found(self, client: TestClient, db_session, sample_pdf_path):
"""Test getting processed file preview when it doesn't exist."""
# Create a file record
@@ -151,23 +159,23 @@ class TestFilePreview:
original_filename="processed.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test preview (processed version should not exist)
response = client.get(f"/api/files/{file_record.id}/preview?version=processed")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
def test_preview_nonexistent_file(self, client: TestClient):
"""Test preview for a file that doesn't exist."""
response = client.get("/api/files/99999/preview?version=original")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
def test_preview_invalid_version(self, client: TestClient, db_session, sample_pdf_path):
"""Test preview with invalid version parameter."""
# Create a file record
@@ -176,12 +184,12 @@ class TestFilePreview:
original_filename="test.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Test with invalid version
response = client.get(f"/api/files/{file_record.id}/preview?version=invalid")
assert response.status_code == 400
@@ -191,7 +199,7 @@ class TestFilePreview:
@pytest.mark.integration
class TestFileDetailView:
"""Tests for enhanced file detail view."""
def test_file_detail_view_with_logs(self, client: TestClient, db_session, sample_pdf_path):
"""Test file detail view returns enhanced data."""
# Create a file record
@@ -200,12 +208,12 @@ class TestFileDetailView:
original_filename="detail.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Add processing logs
logs = [
ProcessingLog(
@@ -213,27 +221,27 @@ class TestFileDetailView:
task_id="task-1",
step_name="hash_file",
status="success",
message="File hashed successfully"
message="File hashed successfully",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-1",
step_name="create_file_record",
status="success",
message="File record created"
message="File record created",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-1",
step_name="extract_metadata_with_gpt",
status="failure",
message="API rate limit exceeded"
)
message="API rate limit exceeded",
),
]
for log in logs:
db_session.add(log)
db_session.commit()
# Test detail view
response = client.get(f"/files/{file_record.id}/detail")
assert response.status_code == 200
@@ -241,7 +249,7 @@ class TestFileDetailView:
assert b"File Information" in response.content
assert b"detail.pdf" in response.content
assert b"Processing History" in response.content
def test_file_detail_view_with_upload_branches(self, client: TestClient, db_session, sample_pdf_path):
"""Test file detail view with upload subtask branches."""
# Create a file record
@@ -250,12 +258,12 @@ class TestFileDetailView:
original_filename="branches.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
db_session.refresh(file_record)
# Add processing logs including upload branches
logs = [
ProcessingLog(
@@ -263,34 +271,34 @@ class TestFileDetailView:
task_id="task-1",
step_name="send_to_all_destinations",
status="success",
message="Queued uploads"
message="Queued uploads",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-2",
step_name="upload_to_dropbox",
status="success",
message="Uploaded to Dropbox"
message="Uploaded to Dropbox",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-3",
step_name="upload_to_s3",
status="failure",
message="S3 connection error"
message="S3 connection error",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-4",
step_name="upload_to_nextcloud",
status="success",
message="Uploaded to Nextcloud"
)
message="Uploaded to Nextcloud",
),
]
for log in logs:
db_session.add(log)
db_session.commit()
# Test detail view
response = client.get(f"/files/{file_record.id}/detail")
assert response.status_code == 200
@@ -299,7 +307,7 @@ class TestFileDetailView:
assert b"Processing Status Summary" in response.content
# Should have upload branches
assert b"Dropbox" in response.content or b"dropbox" in response.content
def test_file_detail_view_nonexistent(self, client: TestClient):
"""Test file detail view for nonexistent file."""
response = client.get("/files/99999/detail")
@@ -310,11 +318,11 @@ class TestFileDetailView:
@pytest.mark.unit
class TestProcessingFlowComputation:
"""Tests for the _compute_processing_flow function."""
def test_flow_with_upload_branches(self, db_session):
"""Test that upload tasks are properly grouped as branches."""
from app.views.files import _compute_processing_flow
# Create mock logs
class MockLog:
def __init__(self, step_name, status, message, timestamp, task_id):
@@ -323,27 +331,27 @@ class TestProcessingFlowComputation:
self.message = message
self.timestamp = timestamp
self.task_id = task_id
logs = [
MockLog("hash_file", "success", "Hashed", None, "task-1"),
MockLog("send_to_all_destinations", "success", "Queued", None, "task-2"),
MockLog("upload_to_dropbox", "success", "Uploaded", None, "task-3"),
MockLog("upload_to_s3", "failure", "Failed", None, "task-4"),
]
flow = _compute_processing_flow(logs)
# Find the upload stage
upload_stage = None
for stage in flow:
if stage.get("is_branch_parent"):
upload_stage = stage
break
assert upload_stage is not None
assert "branches" in upload_stage
assert len(upload_stage["branches"]) == 2
# Check branch details
branches = {b["key"]: b for b in upload_stage["branches"]}
assert "upload_to_dropbox" in branches
@@ -356,17 +364,17 @@ class TestProcessingFlowComputation:
@pytest.mark.unit
class TestStepSummary:
"""Tests for the _compute_step_summary function."""
def test_summary_with_mixed_statuses(self):
"""Test step summary with various statuses."""
from app.views.files import _compute_step_summary
# Create mock logs
class MockLog:
def __init__(self, step_name, status):
self.step_name = step_name
self.status = status
logs = [
MockLog("hash_file", "success"),
MockLog("create_file_record", "success"),
@@ -375,9 +383,9 @@ class TestStepSummary:
MockLog("upload_to_s3", "failure"),
MockLog("upload_to_nextcloud", "in_progress"),
]
summary = _compute_step_summary(logs)
assert "main" in summary
assert "uploads" in summary
assert summary["total_main_steps"] == 3