gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: Apply Black formatting to modified files

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-08 08:16:54 +00:00
parent 0a461343e7
commit 78004cd9a4
5 changed files with 280 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+52 -57
View File
@@ -368,43 +368,39 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(get_db)): def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(get_db)):
""" """
Reprocess a single file by queuing it for processing again. Reprocess a single file by queuing it for processing again.
Args: Args:
file_id: ID of the file to reprocess file_id: ID of the file to reprocess
Returns: Returns:
Task ID and status information Task ID and status information
""" """
try: try:
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
# Check if local file exists # Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename): if not file_record.local_filename or not os.path.exists(file_record.local_filename):
raise HTTPException( raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")
status_code=400,
detail="Local file not found on disk. Cannot reprocess."
)
# Queue the file for processing # Queue the file for processing
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename) task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
logger.info( logger.info(
f"Reprocessing file: ID={file_record.id}, " f"Reprocessing file: ID={file_record.id}, " f"Filename={file_record.original_filename}, TaskID={task.id}"
f"Filename={file_record.original_filename}, TaskID={task.id}"
) )
return { return {
"status": "success", "status": "success",
"message": "File queued for reprocessing", "message": "File queued for reprocessing",
"file_id": file_record.id, "file_id": file_record.id,
"filename": file_record.original_filename, "filename": file_record.original_filename,
"task_id": task.id "task_id": task.id,
} }
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
@@ -415,28 +411,28 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
@router.post("/files/{file_id}/retry-subtask") @router.post("/files/{file_id}/retry-subtask")
@require_login @require_login
def retry_subtask( def retry_subtask(
request: Request, request: Request,
file_id: int, file_id: int,
subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"), subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
db: Session = Depends(get_db) db: Session = Depends(get_db),
): ):
""" """
Retry a specific failed upload subtask for a file. Retry a specific failed upload subtask for a file.
Args: Args:
file_id: ID of the file file_id: ID of the file
subtask_name: Name of the upload task (e.g., upload_to_dropbox, upload_to_s3) subtask_name: Name of the upload task (e.g., upload_to_dropbox, upload_to_s3)
Returns: Returns:
Task ID and status information Task ID and status information
""" """
try: try:
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
# Map subtask names to their corresponding Celery tasks # Map subtask names to their corresponding Celery tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud from app.tasks.upload_to_nextcloud import upload_to_nextcloud
@@ -448,7 +444,7 @@ def retry_subtask(
from app.tasks.upload_to_ftp import upload_to_ftp from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email from app.tasks.upload_to_email import upload_to_email
task_map = { task_map = {
"upload_to_dropbox": upload_to_dropbox, "upload_to_dropbox": upload_to_dropbox,
"upload_to_nextcloud": upload_to_nextcloud, "upload_to_nextcloud": upload_to_nextcloud,
@@ -459,19 +455,19 @@ def retry_subtask(
"upload_to_webdav": upload_to_webdav, "upload_to_webdav": upload_to_webdav,
"upload_to_ftp": upload_to_ftp, "upload_to_ftp": upload_to_ftp,
"upload_to_sftp": upload_to_sftp, "upload_to_sftp": upload_to_sftp,
"upload_to_email": upload_to_email "upload_to_email": upload_to_email,
} }
if subtask_name not in task_map: if subtask_name not in task_map:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}" detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}",
) )
# Check for processed file (upload tasks work with processed files) # Check for processed file (upload tasks work with processed files)
workdir = settings.workdir workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed") processed_dir = os.path.join(workdir, "processed")
# Try to find the processed file # Try to find the processed file
base_filename = os.path.splitext(file_record.original_filename)[0] base_filename = os.path.splitext(file_record.original_filename)[0]
potential_paths = [ potential_paths = [
@@ -479,36 +475,30 @@ def retry_subtask(
os.path.join(processed_dir, f"{base_filename}_processed.pdf"), os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
os.path.join(processed_dir, file_record.original_filename), os.path.join(processed_dir, file_record.original_filename),
] ]
file_path = None file_path = None
for path in potential_paths: for path in potential_paths:
if os.path.exists(path): if os.path.exists(path):
file_path = path file_path = path
break break
if not file_path: if not file_path:
raise HTTPException( raise HTTPException(status_code=400, detail="Processed file not found. Cannot retry upload.")
status_code=400,
detail="Processed file not found. Cannot retry upload."
)
# Queue the specific upload task # Queue the specific upload task
upload_task = task_map[subtask_name] upload_task = task_map[subtask_name]
task = upload_task.delay(file_path, file_id) task = upload_task.delay(file_path, file_id)
logger.info( logger.info(f"Retrying upload subtask: FileID={file_record.id}, " f"Subtask={subtask_name}, TaskID={task.id}")
f"Retrying upload subtask: FileID={file_record.id}, "
f"Subtask={subtask_name}, TaskID={task.id}"
)
return { return {
"status": "success", "status": "success",
"message": f"Upload task {subtask_name} queued for retry", "message": f"Upload task {subtask_name} queued for retry",
"file_id": file_record.id, "file_id": file_record.id,
"subtask_name": subtask_name, "subtask_name": subtask_name,
"task_id": task.id "task_id": task.id,
} }
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
@@ -518,38 +508,43 @@ def retry_subtask(
@router.get("/files/{file_id}/preview") @router.get("/files/{file_id}/preview")
@require_login @require_login
def get_file_preview(request: Request, file_id: int, version: str = Query("original", description="original or processed"), db: Session = Depends(get_db)): def get_file_preview(
request: Request,
file_id: int,
version: str = Query("original", description="original or processed"),
db: Session = Depends(get_db),
):
""" """
Get file content for preview (original or processed version). Get file content for preview (original or processed version).
Args: Args:
file_id: ID of the file file_id: ID of the file
version: "original" for tmp file, "processed" for processed file version: "original" for tmp file, "processed" for processed file
Returns: Returns:
File content for preview File content for preview
""" """
from fastapi.responses import FileResponse from fastapi.responses import FileResponse
try: try:
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
if version == "original": if version == "original":
# Return the original file from tmp # Return the original file from tmp
if not file_record.local_filename or not os.path.exists(file_record.local_filename): if not file_record.local_filename or not os.path.exists(file_record.local_filename):
raise HTTPException(status_code=404, detail="Original file not found on disk") raise HTTPException(status_code=404, detail="Original file not found on disk")
file_path = file_record.local_filename file_path = file_record.local_filename
elif version == "processed": elif version == "processed":
# Look for processed file in /workdir/processed/ # Look for processed file in /workdir/processed/
workdir = settings.workdir workdir = settings.workdir
processed_dir = os.path.join(workdir, "processed") processed_dir = os.path.join(workdir, "processed")
# Try to find the processed file (same hash or UUID-based naming) # Try to find the processed file (same hash or UUID-based naming)
base_filename = os.path.splitext(file_record.original_filename)[0] base_filename = os.path.splitext(file_record.original_filename)[0]
potential_paths = [ potential_paths = [
@@ -557,25 +552,25 @@ def get_file_preview(request: Request, file_id: int, version: str = Query("origi
os.path.join(processed_dir, f"{base_filename}_processed.pdf"), os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
os.path.join(processed_dir, file_record.original_filename), os.path.join(processed_dir, file_record.original_filename),
] ]
file_path = None file_path = None
for path in potential_paths: for path in potential_paths:
if os.path.exists(path): if os.path.exists(path):
file_path = path file_path = path
break break
if not file_path: if not file_path:
raise HTTPException(status_code=404, detail="Processed file not found") raise HTTPException(status_code=404, detail="Processed file not found")
else: else:
raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'") raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")
# Return the file # Return the file
return FileResponse( return FileResponse(
path=file_path, path=file_path,
media_type=file_record.mime_type or "application/pdf", media_type=file_record.mime_type or "application/pdf",
filename=file_record.original_filename filename=file_record.original_filename,
) )
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
app/views/files.py
+150 -144
View File
@@ -1,6 +1,7 @@
""" """
File management views for displaying and managing files. File management views for displaying and managing files.
""" """
from fastapi import Request, Depends, Query from fastapi import Request, Depends, Query
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from typing import Optional from typing import Optional
@@ -11,6 +12,7 @@ from app.config import settings
router = APIRouter() router = APIRouter()
@router.get("/files") @router.get("/files")
@require_login @require_login
def files_page( def files_page(
@@ -22,7 +24,7 @@ def files_page(
sort_order: str = Query("desc"), sort_order: str = Query("desc"),
search: Optional[str] = Query(None), search: Optional[str] = Query(None),
mime_type: Optional[str] = Query(None), mime_type: Optional[str] = Query(None),
status: Optional[str] = Query(None) status: Optional[str] = Query(None),
): ):
""" """
Return the 'files.html' template with server-side pagination, sorting, and filtering Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -31,18 +33,18 @@ def files_page(
# Import the model here to avoid circular imports # Import the model here to avoid circular imports
from app.models import FileRecord, ProcessingLog from app.models import FileRecord, ProcessingLog
from sqlalchemy import desc, asc, or_ from sqlalchemy import desc, asc, or_
# Start with base query # Start with base query
query = db.query(FileRecord) query = db.query(FileRecord)
# Apply search filter # Apply search filter
if search: if search:
query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) query = query.filter(FileRecord.original_filename.ilike(f"%{search}%"))
# Apply MIME type filter # Apply MIME type filter
if mime_type: if mime_type:
query = query.filter(FileRecord.mime_type == mime_type) query = query.filter(FileRecord.mime_type == mime_type)
# Apply status filter (before pagination for correct counts) # Apply status filter (before pagination for correct counts)
if status: if status:
# Subquery to get file IDs matching the status # Subquery to get file IDs matching the status
@@ -52,106 +54,102 @@ def files_page(
query = query.filter(~FileRecord.id.in_(subq)) query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing": elif status == "processing":
# Files with in_progress logs # Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter( subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
ProcessingLog.status == "in_progress"
).distinct()
query = query.filter(FileRecord.id.in_(subq)) query = query.filter(FileRecord.id.in_(subq))
elif status == "failed": elif status == "failed":
# Files with failure logs # Files with failure logs
subq = db.query(ProcessingLog.file_id).filter( subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
ProcessingLog.status == "failure"
).distinct()
query = query.filter(FileRecord.id.in_(subq)) query = query.filter(FileRecord.id.in_(subq))
elif status == "completed": elif status == "completed":
# Files with success logs but no failures or in_progress # Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter( success_files = (
ProcessingLog.status == "success" db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
).distinct().subquery() )
failed_files = db.query(ProcessingLog.file_id).filter( failed_files = (
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress") db.query(ProcessingLog.file_id)
).distinct().subquery() .filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
.distinct()
query = query.filter( .subquery()
FileRecord.id.in_(db.query(success_files.c.file_id)) )
).filter(
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id)) ~FileRecord.id.in_(db.query(failed_files.c.file_id))
) )
# Get total count before pagination # Get total count before pagination
total_items = query.count() total_items = query.count()
# Apply sorting # Apply sorting
sort_column = { sort_column = {
"id": FileRecord.id, "id": FileRecord.id,
"original_filename": FileRecord.original_filename, "original_filename": FileRecord.original_filename,
"file_size": FileRecord.file_size, "file_size": FileRecord.file_size,
"mime_type": FileRecord.mime_type, "mime_type": FileRecord.mime_type,
"created_at": FileRecord.created_at "created_at": FileRecord.created_at,
}.get(sort_by, FileRecord.created_at) }.get(sort_by, FileRecord.created_at)
if sort_order == "asc": if sort_order == "asc":
query = query.order_by(asc(sort_column)) query = query.order_by(asc(sort_column))
else: else:
query = query.order_by(desc(sort_column)) query = query.order_by(desc(sort_column))
# Apply pagination # Apply pagination
offset = (page - 1) * per_page offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all() files = query.offset(offset).limit(per_page).all()
# Get processing status for all files efficiently (avoids N+1) # Get processing status for all files efficiently (avoids N+1)
file_ids = [f.id for f in files] file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids) statuses = get_files_processing_status(db, file_ids)
# Add status to each file # Add status to each file
files_with_status = [] files_with_status = []
for file in files: for file in files:
file.processing_status = statuses.get(file.id, {}).get("status", "pending") file.processing_status = statuses.get(file.id, {}).get("status", "pending")
files_with_status.append(file) files_with_status.append(file)
# Calculate pagination info # Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page total_pages = (total_items + per_page - 1) // per_page
# Get unique MIME types for filter dropdown # Get unique MIME types for filter dropdown
mime_types = db.query(FileRecord.mime_type).distinct().filter( mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
FileRecord.mime_type.isnot(None)
).all()
mime_types = [mt[0] for mt in mime_types if mt[0]] mime_types = [mt[0] for mt in mime_types if mt[0]]
# Debug output # Debug output
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})") logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")
return templates.TemplateResponse("files.html", { return templates.TemplateResponse(
"request": request, "files.html",
"files": files_with_status, {
"pagination": { "request": request,
"page": page, "files": files_with_status,
"per_page": per_page, "pagination": {
"total_items": total_items, "page": page,
"total_pages": total_pages "per_page": per_page,
"total_items": total_items,
"total_pages": total_pages,
},
"sort_by": sort_by,
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types,
}, },
"sort_by": sort_by, )
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types
})
except Exception as e: except Exception as e:
# Log any errors # Log any errors
logger.error(f"Error retrieving files: {str(e)}") logger.error(f"Error retrieving files: {str(e)}")
# Return error message to template # Return error message to template
return templates.TemplateResponse("files.html", { return templates.TemplateResponse(
"request": request, "files.html",
"files": [], {
"pagination": { "request": request,
"page": 1, "files": [],
"per_page": per_page, "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
"total_items": 0, "error": str(e),
"total_pages": 0
}, },
"error": str(e) )
})
@router.get("/files/{file_id}/detail") @router.get("/files/{file_id}/detail")
@@ -163,25 +161,26 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
try: try:
from app.models import FileRecord, ProcessingLog from app.models import FileRecord, ProcessingLog
import os import os
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
return templates.TemplateResponse("file_detail.html", { return templates.TemplateResponse(
"request": request, "file_detail.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"}
"file": None, )
"error": f"File with ID {file_id} not found"
})
# Get processing logs # Get processing logs
logs = db.query(ProcessingLog).filter( logs = (
ProcessingLog.file_id == file_id db.query(ProcessingLog)
).order_by(ProcessingLog.timestamp.asc()).all() .filter(ProcessingLog.file_id == file_id)
.order_by(ProcessingLog.timestamp.asc())
.all()
)
# Check if file exists on disk # Check if file exists on disk
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
# Check if processed file exists # Check if processed file exists
processed_exists = False processed_exists = False
workdir = settings.workdir workdir = settings.workdir
@@ -197,35 +196,34 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
if os.path.exists(path): if os.path.exists(path):
processed_exists = True processed_exists = True
break break
# Compute processing flow for visualization # Compute processing flow for visualization
flow_data = _compute_processing_flow(logs) flow_data = _compute_processing_flow(logs)
# Compute step-aligned summary # Compute step-aligned summary
step_summary = _compute_step_summary(logs) step_summary = _compute_step_summary(logs)
return templates.TemplateResponse("file_detail.html", { return templates.TemplateResponse(
"request": request, "file_detail.html",
"file": file_record, {
"logs": logs, "request": request,
"file_exists": file_exists, "file": file_record,
"processed_exists": processed_exists, "logs": logs,
"flow_data": flow_data, "file_exists": file_exists,
"step_summary": step_summary "processed_exists": processed_exists,
}) "flow_data": flow_data,
"step_summary": step_summary,
},
)
except Exception as e: except Exception as e:
logger.error(f"Error retrieving file details: {str(e)}") logger.error(f"Error retrieving file details: {str(e)}")
return templates.TemplateResponse("file_detail.html", { return templates.TemplateResponse("file_detail.html", {"request": request, "file": None, "error": str(e)})
"request": request,
"file": None,
"error": str(e)
})
def _compute_processing_flow(logs): def _compute_processing_flow(logs):
""" """
Compute the processing flow structure from logs for visualization. Compute the processing flow structure from logs for visualization.
Returns a structured representation of the processing pipeline with branches. Returns a structured representation of the processing pipeline with branches.
Detects upload sub-tasks and organizes them as branches under the parent upload stage. Detects upload sub-tasks and organizes them as branches under the parent upload stage.
""" """
@@ -233,19 +231,25 @@ def _compute_processing_flow(logs):
stages = { stages = {
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]}, "hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
"create_file_record": {"label": "Create File Record", "next": ["check_text"]}, "create_file_record": {"label": "Create File Record", "next": ["check_text"]},
"check_text": {"label": "Check Embedded Text", "next": ["extract_text", "process_with_azure_document_intelligence"]}, "check_text": {
"label": "Check Embedded Text",
"next": ["extract_text", "process_with_azure_document_intelligence"],
},
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]}, "extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
"process_with_azure_document_intelligence": {"label": "OCR Processing (Azure)", "next": ["extract_metadata_with_gpt"]}, "process_with_azure_document_intelligence": {
"label": "OCR Processing (Azure)",
"next": ["extract_metadata_with_gpt"],
},
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]}, "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]}, "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]}, "finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True} "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
} }
# Define upload sub-tasks (branches) # Define upload sub-tasks (branches)
upload_tasks = { upload_tasks = {
"upload_to_dropbox": "Dropbox", "upload_to_dropbox": "Dropbox",
"upload_to_nextcloud": "Nextcloud", "upload_to_nextcloud": "Nextcloud",
"upload_to_paperless": "Paperless-ngx", "upload_to_paperless": "Paperless-ngx",
"upload_to_google_drive": "Google Drive", "upload_to_google_drive": "Google Drive",
"upload_to_onedrive": "OneDrive", "upload_to_onedrive": "OneDrive",
@@ -263,44 +267,38 @@ def _compute_processing_flow(logs):
"queue_webdav": "WebDAV", "queue_webdav": "WebDAV",
"queue_ftp": "FTP Storage", "queue_ftp": "FTP Storage",
"queue_sftp": "SFTP Storage", "queue_sftp": "SFTP Storage",
"queue_email": "Email" "queue_email": "Email",
} }
# Create a map of step names to their log entries # Create a map of step names to their log entries
step_map = {} step_map = {}
upload_branches = {} upload_branches = {}
for log in logs: for log in logs:
step_name = log.step_name step_name = log.step_name
# Check if this is an upload sub-task # Check if this is an upload sub-task
if step_name in upload_tasks: if step_name in upload_tasks:
# Extract the actual upload task name (remove queue_ prefix if present) # Extract the actual upload task name (remove queue_ prefix if present)
upload_key = step_name.replace("queue_", "upload_to_") upload_key = step_name.replace("queue_", "upload_to_")
if upload_key not in upload_branches: if upload_key not in upload_branches:
upload_branches[upload_key] = [] upload_branches[upload_key] = []
upload_branches[upload_key].append({ upload_branches[upload_key].append(
"status": log.status, {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
"message": log.message, )
"timestamp": log.timestamp,
"task_id": log.task_id
})
else: else:
# Regular processing step # Regular processing step
if step_name not in step_map: if step_name not in step_map:
step_map[step_name] = [] step_map[step_name] = []
step_map[step_name].append({ step_map[step_name].append(
"status": log.status, {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
"message": log.message, )
"timestamp": log.timestamp,
"task_id": log.task_id
})
# Build the flow structure # Build the flow structure
flow = [] flow = []
for stage_key, stage_info in stages.items(): for stage_key, stage_info in stages.items():
stage_logs = step_map.get(stage_key, []) stage_logs = step_map.get(stage_key, [])
# Determine overall status for this stage # Determine overall status for this stage
if stage_logs: if stage_logs:
latest_log = stage_logs[-1] latest_log = stage_logs[-1]
@@ -313,7 +311,7 @@ def _compute_processing_flow(logs):
message = None message = None
timestamp = None timestamp = None
task_id = None task_id = None
stage_data = { stage_data = {
"key": stage_key, "key": stage_key,
"label": stage_info["label"], "label": stage_info["label"],
@@ -322,65 +320,73 @@ def _compute_processing_flow(logs):
"timestamp": timestamp, "timestamp": timestamp,
"task_id": task_id, "task_id": task_id,
"can_retry": status == "failure", "can_retry": status == "failure",
"is_branch_parent": stage_info.get("has_branches", False) "is_branch_parent": stage_info.get("has_branches", False),
} }
# If this is the upload stage, add branches # If this is the upload stage, add branches
if stage_info.get("has_branches") and upload_branches: if stage_info.get("has_branches") and upload_branches:
branches = [] branches = []
for upload_key, upload_logs in upload_branches.items(): for upload_key, upload_logs in upload_branches.items():
latest_upload = upload_logs[-1] latest_upload = upload_logs[-1]
upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title()) upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title())
branches.append({ branches.append(
"key": upload_key, {
"label": upload_name, "key": upload_key,
"status": latest_upload["status"], "label": upload_name,
"message": latest_upload["message"], "status": latest_upload["status"],
"timestamp": latest_upload["timestamp"], "message": latest_upload["message"],
"task_id": latest_upload["task_id"], "timestamp": latest_upload["timestamp"],
"can_retry": latest_upload["status"] == "failure" "task_id": latest_upload["task_id"],
}) "can_retry": latest_upload["status"] == "failure",
}
)
stage_data["branches"] = branches stage_data["branches"] = branches
flow.append(stage_data) flow.append(stage_data)
return flow return flow
def _compute_step_summary(logs): def _compute_step_summary(logs):
""" """
Compute a step-aligned summary from logs showing queued, success, and failure counts. Compute a step-aligned summary from logs showing queued, success, and failure counts.
Returns a dictionary with main step counts and upload branch counts. Returns a dictionary with main step counts and upload branch counts.
""" """
# Count statuses for main processing steps (not uploads) # Count statuses for main processing steps (not uploads)
main_steps = [ main_steps = [
"hash_file", "create_file_record", "check_text", "extract_text", "hash_file",
"process_with_azure_document_intelligence", "extract_metadata_with_gpt", "create_file_record",
"embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations" "check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
] ]
upload_prefixes = ["upload_to_", "queue_"] upload_prefixes = ["upload_to_", "queue_"]
main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0} main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0} upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
# Track which steps we've seen # Track which steps we've seen
main_steps_seen = set() main_steps_seen = set()
upload_tasks_seen = {} upload_tasks_seen = {}
for log in logs: for log in logs:
step_name = log.step_name step_name = log.step_name
status = log.status.lower() status = log.status.lower()
# Normalize status # Normalize status
if status == "pending": if status == "pending":
status = "queued" status = "queued"
# Check if it's an upload task # Check if it's an upload task
is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes) is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes)
if is_upload: if is_upload:
# Track latest status for each unique upload task # Track latest status for each unique upload task
upload_tasks_seen[step_name] = status upload_tasks_seen[step_name] = status
@@ -389,15 +395,15 @@ def _compute_step_summary(logs):
main_steps_seen.add(step_name) main_steps_seen.add(step_name)
if status in main_counts: if status in main_counts:
main_counts[status] += 1 main_counts[status] += 1
# Count upload task statuses # Count upload task statuses
for task_status in upload_tasks_seen.values(): for task_status in upload_tasks_seen.values():
if task_status in upload_counts: if task_status in upload_counts:
upload_counts[task_status] += 1 upload_counts[task_status] += 1
return { return {
"main": main_counts, "main": main_counts,
"uploads": upload_counts, "uploads": upload_counts,
"total_main_steps": len(main_steps_seen), "total_main_steps": len(main_steps_seen),
"total_upload_tasks": len(upload_tasks_seen) "total_upload_tasks": len(upload_tasks_seen),
} }
tests/test_api.py
+16 -15
View File
@@ -1,6 +1,7 @@
""" """
Integration tests for API endpoints. Integration tests for API endpoints.
""" """
import pytest import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
@@ -8,47 +9,47 @@ from fastapi.testclient import TestClient
@pytest.mark.integration @pytest.mark.integration
class TestHealthEndpoints: class TestHealthEndpoints:
"""Tests for health check and status endpoints.""" """Tests for health check and status endpoints."""
def test_root_endpoint(self, client: TestClient): def test_root_endpoint(self, client: TestClient):
"""Test that root endpoint redirects to UI or setup wizard.""" """Test that root endpoint redirects to UI or setup wizard."""
response = client.get("/", follow_redirects=False) response = client.get("/", follow_redirects=False)
# Accept 200 (OK), 303 (setup wizard redirect), 307/308 (other redirects) # Accept 200 (OK), 303 (setup wizard redirect), 307/308 (other redirects)
assert response.status_code in [200, 303, 307, 308] assert response.status_code in [200, 303, 307, 308]
def test_root_redirects_to_setup_wizard_when_setup_required(self, client: TestClient): def test_root_redirects_to_setup_wizard_when_setup_required(self, client: TestClient):
"""Test that GET / redirects to setup wizard when setup is required.""" """Test that GET / redirects to setup wizard when setup is required."""
# With test env vars (OPENAI_API_KEY=test-key, AZURE_AI_KEY=test-key), # With test env vars (OPENAI_API_KEY=test-key, AZURE_AI_KEY=test-key),
# is_setup_required() returns True, so we should get a redirect to /setup # is_setup_required() returns True, so we should get a redirect to /setup
response = client.get("/", follow_redirects=False) response = client.get("/", follow_redirects=False)
# Should return 303 See Other (setup wizard redirect) # Should return 303 See Other (setup wizard redirect)
assert response.status_code == 303 assert response.status_code == 303
# Location header should point to setup wizard # Location header should point to setup wizard
assert "Location" in response.headers assert "Location" in response.headers
assert response.headers["Location"] == "/setup?step=1" assert response.headers["Location"] == "/setup?step=1"
def test_root_returns_200_when_setup_complete(self, client: TestClient): def test_root_returns_200_when_setup_complete(self, client: TestClient):
"""Test that GET /?setup=complete bypasses the setup wizard check.""" """Test that GET /?setup=complete bypasses the setup wizard check."""
# The ?setup=complete query param should bypass the wizard check # The ?setup=complete query param should bypass the wizard check
response = client.get("/?setup=complete", follow_redirects=False) response = client.get("/?setup=complete", follow_redirects=False)
# Should return 200 OK (renders the index page) # Should return 200 OK (renders the index page)
assert response.status_code == 200 assert response.status_code == 200
def test_setup_wizard_page_accessible(self, client: TestClient): def test_setup_wizard_page_accessible(self, client: TestClient):
"""Test that GET /setup?step=1 returns 200.""" """Test that GET /setup?step=1 returns 200."""
response = client.get("/setup?step=1") response = client.get("/setup?step=1")
# Setup wizard page should be accessible # Setup wizard page should be accessible
assert response.status_code == 200 assert response.status_code == 200
def test_docs_endpoint(self, client: TestClient): def test_docs_endpoint(self, client: TestClient):
"""Test that API documentation is accessible.""" """Test that API documentation is accessible."""
response = client.get("/docs") response = client.get("/docs")
assert response.status_code == 200 assert response.status_code == 200
assert "swagger" in response.text.lower() or "openapi" in response.text.lower() assert "swagger" in response.text.lower() or "openapi" in response.text.lower()
def test_openapi_schema(self, client: TestClient): def test_openapi_schema(self, client: TestClient):
"""Test that OpenAPI schema is accessible.""" """Test that OpenAPI schema is accessible."""
response = client.get("/openapi.json") response = client.get("/openapi.json")
@@ -62,7 +63,7 @@ class TestHealthEndpoints:
@pytest.mark.integration @pytest.mark.integration
class TestFileEndpoints: class TestFileEndpoints:
"""Tests for file management endpoints.""" """Tests for file management endpoints."""
def test_list_files_empty(self, client: TestClient): def test_list_files_empty(self, client: TestClient):
"""Test listing files when database is empty.""" """Test listing files when database is empty."""
response = client.get("/api/files") response = client.get("/api/files")
@@ -73,7 +74,7 @@ class TestFileEndpoints:
assert "pagination" in data assert "pagination" in data
assert isinstance(data["files"], list) assert isinstance(data["files"], list)
assert len(data["files"]) == 0 assert len(data["files"]) == 0
def test_get_nonexistent_file(self, client: TestClient): def test_get_nonexistent_file(self, client: TestClient):
"""Test getting a file that doesn't exist.""" """Test getting a file that doesn't exist."""
response = client.get("/api/files/99999") response = client.get("/api/files/99999")
@@ -84,7 +85,7 @@ class TestFileEndpoints:
@pytest.mark.requires_external @pytest.mark.requires_external
class TestProcessingEndpoints: class TestProcessingEndpoints:
"""Tests for document processing endpoints.""" """Tests for document processing endpoints."""
def test_process_endpoint_exists(self, client: TestClient): def test_process_endpoint_exists(self, client: TestClient):
"""Test that process endpoint is registered.""" """Test that process endpoint is registered."""
# This will return 422 (validation error) without proper data, # This will return 422 (validation error) without proper data,
@@ -96,7 +97,7 @@ class TestProcessingEndpoints:
@pytest.mark.integration @pytest.mark.integration
class TestConfigEndpoints: class TestConfigEndpoints:
"""Tests for configuration endpoints.""" """Tests for configuration endpoints."""
def test_config_status_endpoint(self, client: TestClient): def test_config_status_endpoint(self, client: TestClient):
"""Test configuration status endpoint if it exists.""" """Test configuration status endpoint if it exists."""
# Some apps have a /status or /config/status endpoint # Some apps have a /status or /config/status endpoint
@@ -108,7 +109,7 @@ class TestConfigEndpoints:
@pytest.mark.integration @pytest.mark.integration
class TestAuthEndpoints: class TestAuthEndpoints:
"""Tests for authentication endpoints (when auth is disabled in tests).""" """Tests for authentication endpoints (when auth is disabled in tests)."""
def test_unauthenticated_access_with_auth_disabled(self, client: TestClient): def test_unauthenticated_access_with_auth_disabled(self, client: TestClient):
"""Test that API is accessible when auth is disabled.""" """Test that API is accessible when auth is disabled."""
# With AUTH_ENABLED=False, API should be accessible # With AUTH_ENABLED=False, API should be accessible
tests/test_bulk_operations.py
+4 -12
View File
@@ -35,9 +35,7 @@ class TestSingleFileOperations:
assert f"File record {file_id} deleted successfully" in data["message"] assert f"File record {file_id} deleted successfully" in data["message"]
# Verify file is deleted # Verify file is deleted
file_record = ( file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
assert file_record is None assert file_record is None
def test_single_file_delete_nonexistent(self, client: TestClient, db_session): def test_single_file_delete_nonexistent(self, client: TestClient, db_session):
@@ -79,9 +77,7 @@ class TestBulkOperations:
# Verify files are deleted # Verify files are deleted
for file_id in file_ids: for file_id in file_ids:
file_record = ( file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
assert file_record is None assert file_record is None
def test_bulk_delete_empty_list(self, client: TestClient, db_session): def test_bulk_delete_empty_list(self, client: TestClient, db_session):
@@ -97,9 +93,7 @@ class TestBulkOperations:
assert response.status_code == 404 assert response.status_code == 404
@patch("app.api.files.process_document") @patch("app.api.files.process_document")
def test_bulk_reprocess_success( def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
self, mock_process_document, client: TestClient, db_session
):
"""Test bulk reprocessing of files.""" """Test bulk reprocessing of files."""
# Setup mock # Setup mock
mock_task = MagicMock() mock_task = MagicMock()
@@ -132,9 +126,7 @@ class TestBulkOperations:
assert len(data["task_ids"]) == 2 assert len(data["task_ids"]) == 2
@patch("app.api.files.process_document") @patch("app.api.files.process_document")
def test_bulk_reprocess_missing_files( def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
self, mock_process_document, client: TestClient, db_session
):
"""Test bulk reprocessing when some local files are missing.""" """Test bulk reprocessing when some local files are missing."""
# Setup mock # Setup mock
mock_task = MagicMock() mock_task = MagicMock()
tests/test_file_detail_endpoints.py
+58 -57
View File
@@ -1,6 +1,7 @@
""" """
Tests for file detail view improvements including reprocessing and preview endpoints. Tests for file detail view improvements including reprocessing and preview endpoints.
""" """
import os import os
import pytest import pytest
from unittest.mock import patch, MagicMock from unittest.mock import patch, MagicMock
@@ -11,7 +12,7 @@ from app.models import FileRecord, ProcessingLog
@pytest.mark.integration @pytest.mark.integration
class TestFileReprocessing: class TestFileReprocessing:
"""Tests for single file reprocessing endpoint.""" """Tests for single file reprocessing endpoint."""
@patch("app.api.files.process_document") @patch("app.api.files.process_document")
def test_reprocess_existing_file(self, mock_process_document, client: TestClient, db_session, sample_pdf_path): def test_reprocess_existing_file(self, mock_process_document, client: TestClient, db_session, sample_pdf_path):
"""Test reprocessing an existing file.""" """Test reprocessing an existing file."""
@@ -19,30 +20,30 @@ class TestFileReprocessing:
mock_task = MagicMock() mock_task = MagicMock()
mock_task.id = "test-task-123" mock_task.id = "test-task-123"
mock_process_document.delay.return_value = mock_task mock_process_document.delay.return_value = mock_task
# Create a file record # Create a file record
file_record = FileRecord( file_record = FileRecord(
filehash="abc123", filehash="abc123",
original_filename="test.pdf", original_filename="test.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Add a failed processing log # Add a failed processing log
log = ProcessingLog( log = ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="test-task-123", task_id="test-task-123",
step_name="extract_metadata_with_gpt", step_name="extract_metadata_with_gpt",
status="failure", status="failure",
message="API error" message="API error",
) )
db_session.add(log) db_session.add(log)
db_session.commit() db_session.commit()
# Test reprocessing # Test reprocessing
response = client.post(f"/api/files/{file_record.id}/reprocess") response = client.post(f"/api/files/{file_record.id}/reprocess")
assert response.status_code == 200 assert response.status_code == 200
@@ -51,13 +52,13 @@ class TestFileReprocessing:
assert "task_id" in data assert "task_id" in data
assert data["file_id"] == file_record.id assert data["file_id"] == file_record.id
assert data["filename"] == "test.pdf" assert data["filename"] == "test.pdf"
def test_reprocess_nonexistent_file(self, client: TestClient): def test_reprocess_nonexistent_file(self, client: TestClient):
"""Test reprocessing a file that doesn't exist.""" """Test reprocessing a file that doesn't exist."""
response = client.post("/api/files/99999/reprocess") response = client.post("/api/files/99999/reprocess")
assert response.status_code == 404 assert response.status_code == 404
assert "not found" in response.json()["detail"].lower() assert "not found" in response.json()["detail"].lower()
def test_reprocess_file_missing_on_disk(self, client: TestClient, db_session): def test_reprocess_file_missing_on_disk(self, client: TestClient, db_session):
"""Test reprocessing when local file is missing.""" """Test reprocessing when local file is missing."""
# Create a file record with non-existent local path # Create a file record with non-existent local path
@@ -66,12 +67,12 @@ class TestFileReprocessing:
original_filename="missing.pdf", original_filename="missing.pdf",
local_filename="/nonexistent/path/missing.pdf", local_filename="/nonexistent/path/missing.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test reprocessing # Test reprocessing
response = client.post(f"/api/files/{file_record.id}/reprocess") response = client.post(f"/api/files/{file_record.id}/reprocess")
assert response.status_code == 400 assert response.status_code == 400
@@ -81,13 +82,13 @@ class TestFileReprocessing:
@pytest.mark.integration @pytest.mark.integration
class TestSubtaskRetry: class TestSubtaskRetry:
"""Tests for per-subtask retry endpoint.""" """Tests for per-subtask retry endpoint."""
def test_retry_subtask_invalid_file(self, client: TestClient): def test_retry_subtask_invalid_file(self, client: TestClient):
"""Test retrying a subtask for nonexistent file.""" """Test retrying a subtask for nonexistent file."""
response = client.post("/api/files/99999/retry-subtask?subtask_name=upload_to_dropbox") response = client.post("/api/files/99999/retry-subtask?subtask_name=upload_to_dropbox")
assert response.status_code == 404 assert response.status_code == 404
assert "not found" in response.json()["detail"].lower() assert "not found" in response.json()["detail"].lower()
def test_retry_subtask_invalid_task_name(self, client: TestClient, db_session, sample_pdf_path): def test_retry_subtask_invalid_task_name(self, client: TestClient, db_session, sample_pdf_path):
"""Test retrying with invalid subtask name.""" """Test retrying with invalid subtask name."""
# Create a file record # Create a file record
@@ -96,17 +97,17 @@ class TestSubtaskRetry:
original_filename="retry.pdf", original_filename="retry.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test with invalid subtask name # Test with invalid subtask name
response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=invalid_task") response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=invalid_task")
assert response.status_code == 400 assert response.status_code == 400
assert "invalid subtask name" in response.json()["detail"].lower() assert "invalid subtask name" in response.json()["detail"].lower()
def test_retry_subtask_missing_processed_file(self, client: TestClient, db_session, sample_pdf_path): def test_retry_subtask_missing_processed_file(self, client: TestClient, db_session, sample_pdf_path):
"""Test retrying when processed file is missing.""" """Test retrying when processed file is missing."""
# Create a file record # Create a file record
@@ -115,12 +116,12 @@ class TestSubtaskRetry:
original_filename="retry2.pdf", original_filename="retry2.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test retry (processed file won't exist) # Test retry (processed file won't exist)
response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=upload_to_dropbox") response = client.post(f"/api/files/{file_record.id}/retry-subtask?subtask_name=upload_to_dropbox")
assert response.status_code == 400 assert response.status_code == 400
@@ -130,7 +131,7 @@ class TestSubtaskRetry:
@pytest.mark.integration @pytest.mark.integration
class TestFilePreview: class TestFilePreview:
"""Tests for file preview endpoint.""" """Tests for file preview endpoint."""
def test_preview_original_file(self, client: TestClient, db_session, sample_pdf_path): def test_preview_original_file(self, client: TestClient, db_session, sample_pdf_path):
"""Test getting original file preview.""" """Test getting original file preview."""
# Create a file record # Create a file record
@@ -139,17 +140,17 @@ class TestFilePreview:
original_filename="preview.pdf", original_filename="preview.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test preview # Test preview
response = client.get(f"/api/files/{file_record.id}/preview?version=original") response = client.get(f"/api/files/{file_record.id}/preview?version=original")
assert response.status_code == 200 assert response.status_code == 200
assert response.headers["content-type"].startswith("application/pdf") assert response.headers["content-type"].startswith("application/pdf")
def test_preview_processed_file_not_found(self, client: TestClient, db_session, sample_pdf_path): def test_preview_processed_file_not_found(self, client: TestClient, db_session, sample_pdf_path):
"""Test getting processed file preview when it doesn't exist.""" """Test getting processed file preview when it doesn't exist."""
# Create a file record # Create a file record
@@ -158,23 +159,23 @@ class TestFilePreview:
original_filename="processed.pdf", original_filename="processed.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test preview (processed version should not exist) # Test preview (processed version should not exist)
response = client.get(f"/api/files/{file_record.id}/preview?version=processed") response = client.get(f"/api/files/{file_record.id}/preview?version=processed")
assert response.status_code == 404 assert response.status_code == 404
assert "not found" in response.json()["detail"].lower() assert "not found" in response.json()["detail"].lower()
def test_preview_nonexistent_file(self, client: TestClient): def test_preview_nonexistent_file(self, client: TestClient):
"""Test preview for a file that doesn't exist.""" """Test preview for a file that doesn't exist."""
response = client.get("/api/files/99999/preview?version=original") response = client.get("/api/files/99999/preview?version=original")
assert response.status_code == 404 assert response.status_code == 404
assert "not found" in response.json()["detail"].lower() assert "not found" in response.json()["detail"].lower()
def test_preview_invalid_version(self, client: TestClient, db_session, sample_pdf_path): def test_preview_invalid_version(self, client: TestClient, db_session, sample_pdf_path):
"""Test preview with invalid version parameter.""" """Test preview with invalid version parameter."""
# Create a file record # Create a file record
@@ -183,12 +184,12 @@ class TestFilePreview:
original_filename="test.pdf", original_filename="test.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Test with invalid version # Test with invalid version
response = client.get(f"/api/files/{file_record.id}/preview?version=invalid") response = client.get(f"/api/files/{file_record.id}/preview?version=invalid")
assert response.status_code == 400 assert response.status_code == 400
@@ -198,7 +199,7 @@ class TestFilePreview:
@pytest.mark.integration @pytest.mark.integration
class TestFileDetailView: class TestFileDetailView:
"""Tests for enhanced file detail view.""" """Tests for enhanced file detail view."""
def test_file_detail_view_with_logs(self, client: TestClient, db_session, sample_pdf_path): def test_file_detail_view_with_logs(self, client: TestClient, db_session, sample_pdf_path):
"""Test file detail view returns enhanced data.""" """Test file detail view returns enhanced data."""
# Create a file record # Create a file record
@@ -207,12 +208,12 @@ class TestFileDetailView:
original_filename="detail.pdf", original_filename="detail.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Add processing logs # Add processing logs
logs = [ logs = [
ProcessingLog( ProcessingLog(
@@ -220,27 +221,27 @@ class TestFileDetailView:
task_id="task-1", task_id="task-1",
step_name="hash_file", step_name="hash_file",
status="success", status="success",
message="File hashed successfully" message="File hashed successfully",
), ),
ProcessingLog( ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="task-1", task_id="task-1",
step_name="create_file_record", step_name="create_file_record",
status="success", status="success",
message="File record created" message="File record created",
), ),
ProcessingLog( ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="task-1", task_id="task-1",
step_name="extract_metadata_with_gpt", step_name="extract_metadata_with_gpt",
status="failure", status="failure",
message="API rate limit exceeded" message="API rate limit exceeded",
) ),
] ]
for log in logs: for log in logs:
db_session.add(log) db_session.add(log)
db_session.commit() db_session.commit()
# Test detail view # Test detail view
response = client.get(f"/files/{file_record.id}/detail") response = client.get(f"/files/{file_record.id}/detail")
assert response.status_code == 200 assert response.status_code == 200
@@ -248,7 +249,7 @@ class TestFileDetailView:
assert b"File Information" in response.content assert b"File Information" in response.content
assert b"detail.pdf" in response.content assert b"detail.pdf" in response.content
assert b"Processing History" in response.content assert b"Processing History" in response.content
def test_file_detail_view_with_upload_branches(self, client: TestClient, db_session, sample_pdf_path): def test_file_detail_view_with_upload_branches(self, client: TestClient, db_session, sample_pdf_path):
"""Test file detail view with upload subtask branches.""" """Test file detail view with upload subtask branches."""
# Create a file record # Create a file record
@@ -257,12 +258,12 @@ class TestFileDetailView:
original_filename="branches.pdf", original_filename="branches.pdf",
local_filename=sample_pdf_path, local_filename=sample_pdf_path,
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
db_session.refresh(file_record) db_session.refresh(file_record)
# Add processing logs including upload branches # Add processing logs including upload branches
logs = [ logs = [
ProcessingLog( ProcessingLog(
@@ -270,34 +271,34 @@ class TestFileDetailView:
task_id="task-1", task_id="task-1",
step_name="send_to_all_destinations", step_name="send_to_all_destinations",
status="success", status="success",
message="Queued uploads" message="Queued uploads",
), ),
ProcessingLog( ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="task-2", task_id="task-2",
step_name="upload_to_dropbox", step_name="upload_to_dropbox",
status="success", status="success",
message="Uploaded to Dropbox" message="Uploaded to Dropbox",
), ),
ProcessingLog( ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="task-3", task_id="task-3",
step_name="upload_to_s3", step_name="upload_to_s3",
status="failure", status="failure",
message="S3 connection error" message="S3 connection error",
), ),
ProcessingLog( ProcessingLog(
file_id=file_record.id, file_id=file_record.id,
task_id="task-4", task_id="task-4",
step_name="upload_to_nextcloud", step_name="upload_to_nextcloud",
status="success", status="success",
message="Uploaded to Nextcloud" message="Uploaded to Nextcloud",
) ),
] ]
for log in logs: for log in logs:
db_session.add(log) db_session.add(log)
db_session.commit() db_session.commit()
# Test detail view # Test detail view
response = client.get(f"/files/{file_record.id}/detail") response = client.get(f"/files/{file_record.id}/detail")
assert response.status_code == 200 assert response.status_code == 200
@@ -306,7 +307,7 @@ class TestFileDetailView:
assert b"Processing Status Summary" in response.content assert b"Processing Status Summary" in response.content
# Should have upload branches # Should have upload branches
assert b"Dropbox" in response.content or b"dropbox" in response.content assert b"Dropbox" in response.content or b"dropbox" in response.content
def test_file_detail_view_nonexistent(self, client: TestClient): def test_file_detail_view_nonexistent(self, client: TestClient):
"""Test file detail view for nonexistent file.""" """Test file detail view for nonexistent file."""
response = client.get("/files/99999/detail") response = client.get("/files/99999/detail")
@@ -317,11 +318,11 @@ class TestFileDetailView:
@pytest.mark.unit @pytest.mark.unit
class TestProcessingFlowComputation: class TestProcessingFlowComputation:
"""Tests for the _compute_processing_flow function.""" """Tests for the _compute_processing_flow function."""
def test_flow_with_upload_branches(self, db_session): def test_flow_with_upload_branches(self, db_session):
"""Test that upload tasks are properly grouped as branches.""" """Test that upload tasks are properly grouped as branches."""
from app.views.files import _compute_processing_flow from app.views.files import _compute_processing_flow
# Create mock logs # Create mock logs
class MockLog: class MockLog:
def __init__(self, step_name, status, message, timestamp, task_id): def __init__(self, step_name, status, message, timestamp, task_id):
@@ -330,27 +331,27 @@ class TestProcessingFlowComputation:
self.message = message self.message = message
self.timestamp = timestamp self.timestamp = timestamp
self.task_id = task_id self.task_id = task_id
logs = [ logs = [
MockLog("hash_file", "success", "Hashed", None, "task-1"), MockLog("hash_file", "success", "Hashed", None, "task-1"),
MockLog("send_to_all_destinations", "success", "Queued", None, "task-2"), MockLog("send_to_all_destinations", "success", "Queued", None, "task-2"),
MockLog("upload_to_dropbox", "success", "Uploaded", None, "task-3"), MockLog("upload_to_dropbox", "success", "Uploaded", None, "task-3"),
MockLog("upload_to_s3", "failure", "Failed", None, "task-4"), MockLog("upload_to_s3", "failure", "Failed", None, "task-4"),
] ]
flow = _compute_processing_flow(logs) flow = _compute_processing_flow(logs)
# Find the upload stage # Find the upload stage
upload_stage = None upload_stage = None
for stage in flow: for stage in flow:
if stage.get("is_branch_parent"): if stage.get("is_branch_parent"):
upload_stage = stage upload_stage = stage
break break
assert upload_stage is not None assert upload_stage is not None
assert "branches" in upload_stage assert "branches" in upload_stage
assert len(upload_stage["branches"]) == 2 assert len(upload_stage["branches"]) == 2
# Check branch details # Check branch details
branches = {b["key"]: b for b in upload_stage["branches"]} branches = {b["key"]: b for b in upload_stage["branches"]}
assert "upload_to_dropbox" in branches assert "upload_to_dropbox" in branches
@@ -363,17 +364,17 @@ class TestProcessingFlowComputation:
@pytest.mark.unit @pytest.mark.unit
class TestStepSummary: class TestStepSummary:
"""Tests for the _compute_step_summary function.""" """Tests for the _compute_step_summary function."""
def test_summary_with_mixed_statuses(self): def test_summary_with_mixed_statuses(self):
"""Test step summary with various statuses.""" """Test step summary with various statuses."""
from app.views.files import _compute_step_summary from app.views.files import _compute_step_summary
# Create mock logs # Create mock logs
class MockLog: class MockLog:
def __init__(self, step_name, status): def __init__(self, step_name, status):
self.step_name = step_name self.step_name = step_name
self.status = status self.status = status
logs = [ logs = [
MockLog("hash_file", "success"), MockLog("hash_file", "success"),
MockLog("create_file_record", "success"), MockLog("create_file_record", "success"),
@@ -382,9 +383,9 @@ class TestStepSummary:
MockLog("upload_to_s3", "failure"), MockLog("upload_to_s3", "failure"),
MockLog("upload_to_nextcloud", "in_progress"), MockLog("upload_to_nextcloud", "in_progress"),
] ]
summary = _compute_step_summary(logs) summary = _compute_step_summary(logs)
assert "main" in summary assert "main" in summary
assert "uploads" in summary assert "uploads" in summary
assert summary["total_main_steps"] == 3 assert summary["total_main_steps"] == 3