style: Apply Black formatting to modified files
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+16
-21
@@ -384,17 +384,13 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
|
|||||||
|
|
||||||
# Check if local file exists
|
# Check if local file exists
|
||||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||||
raise HTTPException(
|
raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")
|
||||||
status_code=400,
|
|
||||||
detail="Local file not found on disk. Cannot reprocess."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Queue the file for processing
|
# Queue the file for processing
|
||||||
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
|
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Reprocessing file: ID={file_record.id}, "
|
f"Reprocessing file: ID={file_record.id}, " f"Filename={file_record.original_filename}, TaskID={task.id}"
|
||||||
f"Filename={file_record.original_filename}, TaskID={task.id}"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -402,7 +398,7 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
|
|||||||
"message": "File queued for reprocessing",
|
"message": "File queued for reprocessing",
|
||||||
"file_id": file_record.id,
|
"file_id": file_record.id,
|
||||||
"filename": file_record.original_filename,
|
"filename": file_record.original_filename,
|
||||||
"task_id": task.id
|
"task_id": task.id,
|
||||||
}
|
}
|
||||||
|
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
@@ -418,7 +414,7 @@ def retry_subtask(
|
|||||||
request: Request,
|
request: Request,
|
||||||
file_id: int,
|
file_id: int,
|
||||||
subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
|
subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
|
||||||
db: Session = Depends(get_db)
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Retry a specific failed upload subtask for a file.
|
Retry a specific failed upload subtask for a file.
|
||||||
@@ -459,13 +455,13 @@ def retry_subtask(
|
|||||||
"upload_to_webdav": upload_to_webdav,
|
"upload_to_webdav": upload_to_webdav,
|
||||||
"upload_to_ftp": upload_to_ftp,
|
"upload_to_ftp": upload_to_ftp,
|
||||||
"upload_to_sftp": upload_to_sftp,
|
"upload_to_sftp": upload_to_sftp,
|
||||||
"upload_to_email": upload_to_email
|
"upload_to_email": upload_to_email,
|
||||||
}
|
}
|
||||||
|
|
||||||
if subtask_name not in task_map:
|
if subtask_name not in task_map:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}"
|
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check for processed file (upload tasks work with processed files)
|
# Check for processed file (upload tasks work with processed files)
|
||||||
@@ -487,26 +483,20 @@ def retry_subtask(
|
|||||||
break
|
break
|
||||||
|
|
||||||
if not file_path:
|
if not file_path:
|
||||||
raise HTTPException(
|
raise HTTPException(status_code=400, detail="Processed file not found. Cannot retry upload.")
|
||||||
status_code=400,
|
|
||||||
detail="Processed file not found. Cannot retry upload."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Queue the specific upload task
|
# Queue the specific upload task
|
||||||
upload_task = task_map[subtask_name]
|
upload_task = task_map[subtask_name]
|
||||||
task = upload_task.delay(file_path, file_id)
|
task = upload_task.delay(file_path, file_id)
|
||||||
|
|
||||||
logger.info(
|
logger.info(f"Retrying upload subtask: FileID={file_record.id}, " f"Subtask={subtask_name}, TaskID={task.id}")
|
||||||
f"Retrying upload subtask: FileID={file_record.id}, "
|
|
||||||
f"Subtask={subtask_name}, TaskID={task.id}"
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "success",
|
"status": "success",
|
||||||
"message": f"Upload task {subtask_name} queued for retry",
|
"message": f"Upload task {subtask_name} queued for retry",
|
||||||
"file_id": file_record.id,
|
"file_id": file_record.id,
|
||||||
"subtask_name": subtask_name,
|
"subtask_name": subtask_name,
|
||||||
"task_id": task.id
|
"task_id": task.id,
|
||||||
}
|
}
|
||||||
|
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
@@ -518,7 +508,12 @@ def retry_subtask(
|
|||||||
|
|
||||||
@router.get("/files/{file_id}/preview")
|
@router.get("/files/{file_id}/preview")
|
||||||
@require_login
|
@require_login
|
||||||
def get_file_preview(request: Request, file_id: int, version: str = Query("original", description="original or processed"), db: Session = Depends(get_db)):
|
def get_file_preview(
|
||||||
|
request: Request,
|
||||||
|
file_id: int,
|
||||||
|
version: str = Query("original", description="original or processed"),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
Get file content for preview (original or processed version).
|
Get file content for preview (original or processed version).
|
||||||
|
|
||||||
@@ -573,7 +568,7 @@ def get_file_preview(request: Request, file_id: int, version: str = Query("origi
|
|||||||
return FileResponse(
|
return FileResponse(
|
||||||
path=file_path,
|
path=file_path,
|
||||||
media_type=file_record.mime_type or "application/pdf",
|
media_type=file_record.mime_type or "application/pdf",
|
||||||
filename=file_record.original_filename
|
filename=file_record.original_filename,
|
||||||
)
|
)
|
||||||
|
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
|
|||||||
+103
-97
@@ -1,6 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
File management views for displaying and managing files.
|
File management views for displaying and managing files.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from fastapi import Request, Depends, Query
|
from fastapi import Request, Depends, Query
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -11,6 +12,7 @@ from app.config import settings
|
|||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/files")
|
@router.get("/files")
|
||||||
@require_login
|
@require_login
|
||||||
def files_page(
|
def files_page(
|
||||||
@@ -22,7 +24,7 @@ def files_page(
|
|||||||
sort_order: str = Query("desc"),
|
sort_order: str = Query("desc"),
|
||||||
search: Optional[str] = Query(None),
|
search: Optional[str] = Query(None),
|
||||||
mime_type: Optional[str] = Query(None),
|
mime_type: Optional[str] = Query(None),
|
||||||
status: Optional[str] = Query(None)
|
status: Optional[str] = Query(None),
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
||||||
@@ -52,29 +54,26 @@ def files_page(
|
|||||||
query = query.filter(~FileRecord.id.in_(subq))
|
query = query.filter(~FileRecord.id.in_(subq))
|
||||||
elif status == "processing":
|
elif status == "processing":
|
||||||
# Files with in_progress logs
|
# Files with in_progress logs
|
||||||
subq = db.query(ProcessingLog.file_id).filter(
|
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
|
||||||
ProcessingLog.status == "in_progress"
|
|
||||||
).distinct()
|
|
||||||
query = query.filter(FileRecord.id.in_(subq))
|
query = query.filter(FileRecord.id.in_(subq))
|
||||||
elif status == "failed":
|
elif status == "failed":
|
||||||
# Files with failure logs
|
# Files with failure logs
|
||||||
subq = db.query(ProcessingLog.file_id).filter(
|
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
|
||||||
ProcessingLog.status == "failure"
|
|
||||||
).distinct()
|
|
||||||
query = query.filter(FileRecord.id.in_(subq))
|
query = query.filter(FileRecord.id.in_(subq))
|
||||||
elif status == "completed":
|
elif status == "completed":
|
||||||
# Files with success logs but no failures or in_progress
|
# Files with success logs but no failures or in_progress
|
||||||
success_files = db.query(ProcessingLog.file_id).filter(
|
success_files = (
|
||||||
ProcessingLog.status == "success"
|
db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
|
||||||
).distinct().subquery()
|
)
|
||||||
|
|
||||||
failed_files = db.query(ProcessingLog.file_id).filter(
|
failed_files = (
|
||||||
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
|
db.query(ProcessingLog.file_id)
|
||||||
).distinct().subquery()
|
.filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
|
||||||
|
.distinct()
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
|
||||||
query = query.filter(
|
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
|
||||||
FileRecord.id.in_(db.query(success_files.c.file_id))
|
|
||||||
).filter(
|
|
||||||
~FileRecord.id.in_(db.query(failed_files.c.file_id))
|
~FileRecord.id.in_(db.query(failed_files.c.file_id))
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -87,7 +86,7 @@ def files_page(
|
|||||||
"original_filename": FileRecord.original_filename,
|
"original_filename": FileRecord.original_filename,
|
||||||
"file_size": FileRecord.file_size,
|
"file_size": FileRecord.file_size,
|
||||||
"mime_type": FileRecord.mime_type,
|
"mime_type": FileRecord.mime_type,
|
||||||
"created_at": FileRecord.created_at
|
"created_at": FileRecord.created_at,
|
||||||
}.get(sort_by, FileRecord.created_at)
|
}.get(sort_by, FileRecord.created_at)
|
||||||
|
|
||||||
if sort_order == "asc":
|
if sort_order == "asc":
|
||||||
@@ -113,45 +112,44 @@ def files_page(
|
|||||||
total_pages = (total_items + per_page - 1) // per_page
|
total_pages = (total_items + per_page - 1) // per_page
|
||||||
|
|
||||||
# Get unique MIME types for filter dropdown
|
# Get unique MIME types for filter dropdown
|
||||||
mime_types = db.query(FileRecord.mime_type).distinct().filter(
|
mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
|
||||||
FileRecord.mime_type.isnot(None)
|
|
||||||
).all()
|
|
||||||
mime_types = [mt[0] for mt in mime_types if mt[0]]
|
mime_types = [mt[0] for mt in mime_types if mt[0]]
|
||||||
|
|
||||||
# Debug output
|
# Debug output
|
||||||
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")
|
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")
|
||||||
|
|
||||||
return templates.TemplateResponse("files.html", {
|
return templates.TemplateResponse(
|
||||||
"request": request,
|
"files.html",
|
||||||
"files": files_with_status,
|
{
|
||||||
"pagination": {
|
"request": request,
|
||||||
"page": page,
|
"files": files_with_status,
|
||||||
"per_page": per_page,
|
"pagination": {
|
||||||
"total_items": total_items,
|
"page": page,
|
||||||
"total_pages": total_pages
|
"per_page": per_page,
|
||||||
|
"total_items": total_items,
|
||||||
|
"total_pages": total_pages,
|
||||||
|
},
|
||||||
|
"sort_by": sort_by,
|
||||||
|
"sort_order": sort_order,
|
||||||
|
"search": search or "",
|
||||||
|
"mime_type": mime_type or "",
|
||||||
|
"status": status or "",
|
||||||
|
"mime_types": mime_types,
|
||||||
},
|
},
|
||||||
"sort_by": sort_by,
|
)
|
||||||
"sort_order": sort_order,
|
|
||||||
"search": search or "",
|
|
||||||
"mime_type": mime_type or "",
|
|
||||||
"status": status or "",
|
|
||||||
"mime_types": mime_types
|
|
||||||
})
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Log any errors
|
# Log any errors
|
||||||
logger.error(f"Error retrieving files: {str(e)}")
|
logger.error(f"Error retrieving files: {str(e)}")
|
||||||
# Return error message to template
|
# Return error message to template
|
||||||
return templates.TemplateResponse("files.html", {
|
return templates.TemplateResponse(
|
||||||
"request": request,
|
"files.html",
|
||||||
"files": [],
|
{
|
||||||
"pagination": {
|
"request": request,
|
||||||
"page": 1,
|
"files": [],
|
||||||
"per_page": per_page,
|
"pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
|
||||||
"total_items": 0,
|
"error": str(e),
|
||||||
"total_pages": 0
|
|
||||||
},
|
},
|
||||||
"error": str(e)
|
)
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/files/{file_id}/detail")
|
@router.get("/files/{file_id}/detail")
|
||||||
@@ -168,16 +166,17 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
|
|||||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||||
|
|
||||||
if not file_record:
|
if not file_record:
|
||||||
return templates.TemplateResponse("file_detail.html", {
|
return templates.TemplateResponse(
|
||||||
"request": request,
|
"file_detail.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"}
|
||||||
"file": None,
|
)
|
||||||
"error": f"File with ID {file_id} not found"
|
|
||||||
})
|
|
||||||
|
|
||||||
# Get processing logs
|
# Get processing logs
|
||||||
logs = db.query(ProcessingLog).filter(
|
logs = (
|
||||||
ProcessingLog.file_id == file_id
|
db.query(ProcessingLog)
|
||||||
).order_by(ProcessingLog.timestamp.asc()).all()
|
.filter(ProcessingLog.file_id == file_id)
|
||||||
|
.order_by(ProcessingLog.timestamp.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
# Check if file exists on disk
|
# Check if file exists on disk
|
||||||
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
|
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
|
||||||
@@ -204,22 +203,21 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
|
|||||||
# Compute step-aligned summary
|
# Compute step-aligned summary
|
||||||
step_summary = _compute_step_summary(logs)
|
step_summary = _compute_step_summary(logs)
|
||||||
|
|
||||||
return templates.TemplateResponse("file_detail.html", {
|
return templates.TemplateResponse(
|
||||||
"request": request,
|
"file_detail.html",
|
||||||
"file": file_record,
|
{
|
||||||
"logs": logs,
|
"request": request,
|
||||||
"file_exists": file_exists,
|
"file": file_record,
|
||||||
"processed_exists": processed_exists,
|
"logs": logs,
|
||||||
"flow_data": flow_data,
|
"file_exists": file_exists,
|
||||||
"step_summary": step_summary
|
"processed_exists": processed_exists,
|
||||||
})
|
"flow_data": flow_data,
|
||||||
|
"step_summary": step_summary,
|
||||||
|
},
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error retrieving file details: {str(e)}")
|
logger.error(f"Error retrieving file details: {str(e)}")
|
||||||
return templates.TemplateResponse("file_detail.html", {
|
return templates.TemplateResponse("file_detail.html", {"request": request, "file": None, "error": str(e)})
|
||||||
"request": request,
|
|
||||||
"file": None,
|
|
||||||
"error": str(e)
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _compute_processing_flow(logs):
|
def _compute_processing_flow(logs):
|
||||||
@@ -233,13 +231,19 @@ def _compute_processing_flow(logs):
|
|||||||
stages = {
|
stages = {
|
||||||
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
|
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
|
||||||
"create_file_record": {"label": "Create File Record", "next": ["check_text"]},
|
"create_file_record": {"label": "Create File Record", "next": ["check_text"]},
|
||||||
"check_text": {"label": "Check Embedded Text", "next": ["extract_text", "process_with_azure_document_intelligence"]},
|
"check_text": {
|
||||||
|
"label": "Check Embedded Text",
|
||||||
|
"next": ["extract_text", "process_with_azure_document_intelligence"],
|
||||||
|
},
|
||||||
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
|
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
|
||||||
"process_with_azure_document_intelligence": {"label": "OCR Processing (Azure)", "next": ["extract_metadata_with_gpt"]},
|
"process_with_azure_document_intelligence": {
|
||||||
|
"label": "OCR Processing (Azure)",
|
||||||
|
"next": ["extract_metadata_with_gpt"],
|
||||||
|
},
|
||||||
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
|
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
|
||||||
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
|
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
|
||||||
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
|
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
|
||||||
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True}
|
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Define upload sub-tasks (branches)
|
# Define upload sub-tasks (branches)
|
||||||
@@ -263,7 +267,7 @@ def _compute_processing_flow(logs):
|
|||||||
"queue_webdav": "WebDAV",
|
"queue_webdav": "WebDAV",
|
||||||
"queue_ftp": "FTP Storage",
|
"queue_ftp": "FTP Storage",
|
||||||
"queue_sftp": "SFTP Storage",
|
"queue_sftp": "SFTP Storage",
|
||||||
"queue_email": "Email"
|
"queue_email": "Email",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Create a map of step names to their log entries
|
# Create a map of step names to their log entries
|
||||||
@@ -279,22 +283,16 @@ def _compute_processing_flow(logs):
|
|||||||
upload_key = step_name.replace("queue_", "upload_to_")
|
upload_key = step_name.replace("queue_", "upload_to_")
|
||||||
if upload_key not in upload_branches:
|
if upload_key not in upload_branches:
|
||||||
upload_branches[upload_key] = []
|
upload_branches[upload_key] = []
|
||||||
upload_branches[upload_key].append({
|
upload_branches[upload_key].append(
|
||||||
"status": log.status,
|
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
|
||||||
"message": log.message,
|
)
|
||||||
"timestamp": log.timestamp,
|
|
||||||
"task_id": log.task_id
|
|
||||||
})
|
|
||||||
else:
|
else:
|
||||||
# Regular processing step
|
# Regular processing step
|
||||||
if step_name not in step_map:
|
if step_name not in step_map:
|
||||||
step_map[step_name] = []
|
step_map[step_name] = []
|
||||||
step_map[step_name].append({
|
step_map[step_name].append(
|
||||||
"status": log.status,
|
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
|
||||||
"message": log.message,
|
)
|
||||||
"timestamp": log.timestamp,
|
|
||||||
"task_id": log.task_id
|
|
||||||
})
|
|
||||||
|
|
||||||
# Build the flow structure
|
# Build the flow structure
|
||||||
flow = []
|
flow = []
|
||||||
@@ -322,7 +320,7 @@ def _compute_processing_flow(logs):
|
|||||||
"timestamp": timestamp,
|
"timestamp": timestamp,
|
||||||
"task_id": task_id,
|
"task_id": task_id,
|
||||||
"can_retry": status == "failure",
|
"can_retry": status == "failure",
|
||||||
"is_branch_parent": stage_info.get("has_branches", False)
|
"is_branch_parent": stage_info.get("has_branches", False),
|
||||||
}
|
}
|
||||||
|
|
||||||
# If this is the upload stage, add branches
|
# If this is the upload stage, add branches
|
||||||
@@ -332,15 +330,17 @@ def _compute_processing_flow(logs):
|
|||||||
latest_upload = upload_logs[-1]
|
latest_upload = upload_logs[-1]
|
||||||
upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title())
|
upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title())
|
||||||
|
|
||||||
branches.append({
|
branches.append(
|
||||||
"key": upload_key,
|
{
|
||||||
"label": upload_name,
|
"key": upload_key,
|
||||||
"status": latest_upload["status"],
|
"label": upload_name,
|
||||||
"message": latest_upload["message"],
|
"status": latest_upload["status"],
|
||||||
"timestamp": latest_upload["timestamp"],
|
"message": latest_upload["message"],
|
||||||
"task_id": latest_upload["task_id"],
|
"timestamp": latest_upload["timestamp"],
|
||||||
"can_retry": latest_upload["status"] == "failure"
|
"task_id": latest_upload["task_id"],
|
||||||
})
|
"can_retry": latest_upload["status"] == "failure",
|
||||||
|
}
|
||||||
|
)
|
||||||
stage_data["branches"] = branches
|
stage_data["branches"] = branches
|
||||||
|
|
||||||
flow.append(stage_data)
|
flow.append(stage_data)
|
||||||
@@ -356,9 +356,15 @@ def _compute_step_summary(logs):
|
|||||||
"""
|
"""
|
||||||
# Count statuses for main processing steps (not uploads)
|
# Count statuses for main processing steps (not uploads)
|
||||||
main_steps = [
|
main_steps = [
|
||||||
"hash_file", "create_file_record", "check_text", "extract_text",
|
"hash_file",
|
||||||
"process_with_azure_document_intelligence", "extract_metadata_with_gpt",
|
"create_file_record",
|
||||||
"embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations"
|
"check_text",
|
||||||
|
"extract_text",
|
||||||
|
"process_with_azure_document_intelligence",
|
||||||
|
"extract_metadata_with_gpt",
|
||||||
|
"embed_metadata_into_pdf",
|
||||||
|
"finalize_document_storage",
|
||||||
|
"send_to_all_destinations",
|
||||||
]
|
]
|
||||||
|
|
||||||
upload_prefixes = ["upload_to_", "queue_"]
|
upload_prefixes = ["upload_to_", "queue_"]
|
||||||
@@ -399,5 +405,5 @@ def _compute_step_summary(logs):
|
|||||||
"main": main_counts,
|
"main": main_counts,
|
||||||
"uploads": upload_counts,
|
"uploads": upload_counts,
|
||||||
"total_main_steps": len(main_steps_seen),
|
"total_main_steps": len(main_steps_seen),
|
||||||
"total_upload_tasks": len(upload_tasks_seen)
|
"total_upload_tasks": len(upload_tasks_seen),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
Integration tests for API endpoints.
|
Integration tests for API endpoints.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
|||||||
@@ -35,9 +35,7 @@ class TestSingleFileOperations:
|
|||||||
assert f"File record {file_id} deleted successfully" in data["message"]
|
assert f"File record {file_id} deleted successfully" in data["message"]
|
||||||
|
|
||||||
# Verify file is deleted
|
# Verify file is deleted
|
||||||
file_record = (
|
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||||
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
|
|
||||||
)
|
|
||||||
assert file_record is None
|
assert file_record is None
|
||||||
|
|
||||||
def test_single_file_delete_nonexistent(self, client: TestClient, db_session):
|
def test_single_file_delete_nonexistent(self, client: TestClient, db_session):
|
||||||
@@ -79,9 +77,7 @@ class TestBulkOperations:
|
|||||||
|
|
||||||
# Verify files are deleted
|
# Verify files are deleted
|
||||||
for file_id in file_ids:
|
for file_id in file_ids:
|
||||||
file_record = (
|
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||||
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
|
|
||||||
)
|
|
||||||
assert file_record is None
|
assert file_record is None
|
||||||
|
|
||||||
def test_bulk_delete_empty_list(self, client: TestClient, db_session):
|
def test_bulk_delete_empty_list(self, client: TestClient, db_session):
|
||||||
@@ -97,9 +93,7 @@ class TestBulkOperations:
|
|||||||
assert response.status_code == 404
|
assert response.status_code == 404
|
||||||
|
|
||||||
@patch("app.api.files.process_document")
|
@patch("app.api.files.process_document")
|
||||||
def test_bulk_reprocess_success(
|
def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
|
||||||
self, mock_process_document, client: TestClient, db_session
|
|
||||||
):
|
|
||||||
"""Test bulk reprocessing of files."""
|
"""Test bulk reprocessing of files."""
|
||||||
# Setup mock
|
# Setup mock
|
||||||
mock_task = MagicMock()
|
mock_task = MagicMock()
|
||||||
@@ -132,9 +126,7 @@ class TestBulkOperations:
|
|||||||
assert len(data["task_ids"]) == 2
|
assert len(data["task_ids"]) == 2
|
||||||
|
|
||||||
@patch("app.api.files.process_document")
|
@patch("app.api.files.process_document")
|
||||||
def test_bulk_reprocess_missing_files(
|
def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
|
||||||
self, mock_process_document, client: TestClient, db_session
|
|
||||||
):
|
|
||||||
"""Test bulk reprocessing when some local files are missing."""
|
"""Test bulk reprocessing when some local files are missing."""
|
||||||
# Setup mock
|
# Setup mock
|
||||||
mock_task = MagicMock()
|
mock_task = MagicMock()
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
Tests for file detail view improvements including reprocessing and preview endpoints.
|
Tests for file detail view improvements including reprocessing and preview endpoints.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import patch, MagicMock
|
from unittest.mock import patch, MagicMock
|
||||||
@@ -26,7 +27,7 @@ class TestFileReprocessing:
|
|||||||
original_filename="test.pdf",
|
original_filename="test.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -38,7 +39,7 @@ class TestFileReprocessing:
|
|||||||
task_id="test-task-123",
|
task_id="test-task-123",
|
||||||
step_name="extract_metadata_with_gpt",
|
step_name="extract_metadata_with_gpt",
|
||||||
status="failure",
|
status="failure",
|
||||||
message="API error"
|
message="API error",
|
||||||
)
|
)
|
||||||
db_session.add(log)
|
db_session.add(log)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -66,7 +67,7 @@ class TestFileReprocessing:
|
|||||||
original_filename="missing.pdf",
|
original_filename="missing.pdf",
|
||||||
local_filename="/nonexistent/path/missing.pdf",
|
local_filename="/nonexistent/path/missing.pdf",
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -96,7 +97,7 @@ class TestSubtaskRetry:
|
|||||||
original_filename="retry.pdf",
|
original_filename="retry.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -115,7 +116,7 @@ class TestSubtaskRetry:
|
|||||||
original_filename="retry2.pdf",
|
original_filename="retry2.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -139,7 +140,7 @@ class TestFilePreview:
|
|||||||
original_filename="preview.pdf",
|
original_filename="preview.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -158,7 +159,7 @@ class TestFilePreview:
|
|||||||
original_filename="processed.pdf",
|
original_filename="processed.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -183,7 +184,7 @@ class TestFilePreview:
|
|||||||
original_filename="test.pdf",
|
original_filename="test.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -207,7 +208,7 @@ class TestFileDetailView:
|
|||||||
original_filename="detail.pdf",
|
original_filename="detail.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -220,22 +221,22 @@ class TestFileDetailView:
|
|||||||
task_id="task-1",
|
task_id="task-1",
|
||||||
step_name="hash_file",
|
step_name="hash_file",
|
||||||
status="success",
|
status="success",
|
||||||
message="File hashed successfully"
|
message="File hashed successfully",
|
||||||
),
|
),
|
||||||
ProcessingLog(
|
ProcessingLog(
|
||||||
file_id=file_record.id,
|
file_id=file_record.id,
|
||||||
task_id="task-1",
|
task_id="task-1",
|
||||||
step_name="create_file_record",
|
step_name="create_file_record",
|
||||||
status="success",
|
status="success",
|
||||||
message="File record created"
|
message="File record created",
|
||||||
),
|
),
|
||||||
ProcessingLog(
|
ProcessingLog(
|
||||||
file_id=file_record.id,
|
file_id=file_record.id,
|
||||||
task_id="task-1",
|
task_id="task-1",
|
||||||
step_name="extract_metadata_with_gpt",
|
step_name="extract_metadata_with_gpt",
|
||||||
status="failure",
|
status="failure",
|
||||||
message="API rate limit exceeded"
|
message="API rate limit exceeded",
|
||||||
)
|
),
|
||||||
]
|
]
|
||||||
for log in logs:
|
for log in logs:
|
||||||
db_session.add(log)
|
db_session.add(log)
|
||||||
@@ -257,7 +258,7 @@ class TestFileDetailView:
|
|||||||
original_filename="branches.pdf",
|
original_filename="branches.pdf",
|
||||||
local_filename=sample_pdf_path,
|
local_filename=sample_pdf_path,
|
||||||
file_size=1024,
|
file_size=1024,
|
||||||
mime_type="application/pdf"
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
db_session.add(file_record)
|
db_session.add(file_record)
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
@@ -270,29 +271,29 @@ class TestFileDetailView:
|
|||||||
task_id="task-1",
|
task_id="task-1",
|
||||||
step_name="send_to_all_destinations",
|
step_name="send_to_all_destinations",
|
||||||
status="success",
|
status="success",
|
||||||
message="Queued uploads"
|
message="Queued uploads",
|
||||||
),
|
),
|
||||||
ProcessingLog(
|
ProcessingLog(
|
||||||
file_id=file_record.id,
|
file_id=file_record.id,
|
||||||
task_id="task-2",
|
task_id="task-2",
|
||||||
step_name="upload_to_dropbox",
|
step_name="upload_to_dropbox",
|
||||||
status="success",
|
status="success",
|
||||||
message="Uploaded to Dropbox"
|
message="Uploaded to Dropbox",
|
||||||
),
|
),
|
||||||
ProcessingLog(
|
ProcessingLog(
|
||||||
file_id=file_record.id,
|
file_id=file_record.id,
|
||||||
task_id="task-3",
|
task_id="task-3",
|
||||||
step_name="upload_to_s3",
|
step_name="upload_to_s3",
|
||||||
status="failure",
|
status="failure",
|
||||||
message="S3 connection error"
|
message="S3 connection error",
|
||||||
),
|
),
|
||||||
ProcessingLog(
|
ProcessingLog(
|
||||||
file_id=file_record.id,
|
file_id=file_record.id,
|
||||||
task_id="task-4",
|
task_id="task-4",
|
||||||
step_name="upload_to_nextcloud",
|
step_name="upload_to_nextcloud",
|
||||||
status="success",
|
status="success",
|
||||||
message="Uploaded to Nextcloud"
|
message="Uploaded to Nextcloud",
|
||||||
)
|
),
|
||||||
]
|
]
|
||||||
for log in logs:
|
for log in logs:
|
||||||
db_session.add(log)
|
db_session.add(log)
|
||||||
|
|||||||
Reference in New Issue
Block a user