Add backend endpoints and enhanced file detail view
- Added /api/files/{file_id}/reprocess endpoint for single file reprocessing
- Added /api/files/{file_id}/preview endpoint for viewing original/processed files
- Enhanced file detail view with process flow computation
- Updated frontend template with retry button, process flow visualization, and PDF previews
- Added JavaScript for async retry functionality
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -363,6 +363,122 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
|
||||
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/reprocess")
|
||||
@require_login
|
||||
def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Reprocess a single file by queuing it for processing again.
|
||||
|
||||
Args:
|
||||
file_id: ID of the file to reprocess
|
||||
|
||||
Returns:
|
||||
Task ID and status information
|
||||
"""
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
|
||||
# Check if local file exists
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Local file not found on disk. Cannot reprocess."
|
||||
)
|
||||
|
||||
# Queue the file for processing
|
||||
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
|
||||
|
||||
logger.info(
|
||||
f"Reprocessing file: ID={file_record.id}, "
|
||||
f"Filename={file_record.original_filename}, TaskID={task.id}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "File queued for reprocessing",
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"task_id": task.id
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error reprocessing file {file_id}: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reprocessing file: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/preview")
|
||||
@require_login
|
||||
def get_file_preview(request: Request, file_id: int, version: str = Query("original", description="original or processed"), db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get file content for preview (original or processed version).
|
||||
|
||||
Args:
|
||||
file_id: ID of the file
|
||||
version: "original" for tmp file, "processed" for processed file
|
||||
|
||||
Returns:
|
||||
File content for preview
|
||||
"""
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
|
||||
if version == "original":
|
||||
# Return the original file from tmp
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
raise HTTPException(status_code=404, detail="Original file not found on disk")
|
||||
|
||||
file_path = file_record.local_filename
|
||||
|
||||
elif version == "processed":
|
||||
# Look for processed file in /workdir/processed/
|
||||
workdir = settings.workdir
|
||||
processed_dir = os.path.join(workdir, "processed")
|
||||
|
||||
# Try to find the processed file (same hash or UUID-based naming)
|
||||
base_filename = os.path.splitext(file_record.original_filename)[0]
|
||||
potential_paths = [
|
||||
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
|
||||
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
|
||||
os.path.join(processed_dir, file_record.original_filename),
|
||||
]
|
||||
|
||||
file_path = None
|
||||
for path in potential_paths:
|
||||
if os.path.exists(path):
|
||||
file_path = path
|
||||
break
|
||||
|
||||
if not file_path:
|
||||
raise HTTPException(status_code=404, detail="Processed file not found")
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")
|
||||
|
||||
# Return the file
|
||||
return FileResponse(
|
||||
path=file_path,
|
||||
media_type=file_record.mime_type or "application/pdf",
|
||||
filename=file_record.original_filename
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error retrieving file preview: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error retrieving file preview: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/ui-upload")
|
||||
@require_login
|
||||
async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
|
||||
+86
-1
@@ -7,6 +7,7 @@ from typing import Optional
|
||||
|
||||
from app.views.base import APIRouter, templates, require_login, get_db, logger
|
||||
from app.utils.file_status import get_files_processing_status
|
||||
from app.config import settings
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -180,11 +181,32 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
|
||||
# Check if file exists on disk
|
||||
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
|
||||
|
||||
# Check if processed file exists
|
||||
processed_exists = False
|
||||
workdir = settings.workdir
|
||||
processed_dir = os.path.join(workdir, "processed")
|
||||
if os.path.exists(processed_dir):
|
||||
base_filename = os.path.splitext(file_record.original_filename)[0]
|
||||
potential_paths = [
|
||||
os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
|
||||
os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
|
||||
os.path.join(processed_dir, file_record.original_filename),
|
||||
]
|
||||
for path in potential_paths:
|
||||
if os.path.exists(path):
|
||||
processed_exists = True
|
||||
break
|
||||
|
||||
# Compute processing flow for visualization
|
||||
flow_data = _compute_processing_flow(logs)
|
||||
|
||||
return templates.TemplateResponse("file_detail.html", {
|
||||
"request": request,
|
||||
"file": file_record,
|
||||
"logs": logs,
|
||||
"file_exists": file_exists
|
||||
"file_exists": file_exists,
|
||||
"processed_exists": processed_exists,
|
||||
"flow_data": flow_data
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving file details: {str(e)}")
|
||||
@@ -192,3 +214,66 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
|
||||
"request": request,
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
|
||||
def _compute_processing_flow(logs):
|
||||
"""
|
||||
Compute the processing flow structure from logs for visualization.
|
||||
|
||||
Returns a structured representation of the processing pipeline with branches.
|
||||
"""
|
||||
# Define the processing stages and their relationships
|
||||
stages = {
|
||||
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
|
||||
"create_file_record": {"label": "Create File Record", "next": ["check_text"]},
|
||||
"check_text": {"label": "Check Embedded Text", "next": ["extract_text", "process_with_azure_document_intelligence"]},
|
||||
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
|
||||
"process_with_azure_document_intelligence": {"label": "OCR Processing (Azure)", "next": ["extract_metadata_with_gpt"]},
|
||||
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
|
||||
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
|
||||
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["upload_destinations"]},
|
||||
"upload_destinations": {"label": "Upload to Destinations", "next": []}
|
||||
}
|
||||
|
||||
# Create a map of step names to their log entries
|
||||
step_map = {}
|
||||
for log in logs:
|
||||
step_name = log.step_name
|
||||
if step_name not in step_map:
|
||||
step_map[step_name] = []
|
||||
step_map[step_name].append({
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp,
|
||||
"task_id": log.task_id
|
||||
})
|
||||
|
||||
# Build the flow structure
|
||||
flow = []
|
||||
for stage_key, stage_info in stages.items():
|
||||
stage_logs = step_map.get(stage_key, [])
|
||||
|
||||
# Determine overall status for this stage
|
||||
if stage_logs:
|
||||
latest_log = stage_logs[-1]
|
||||
status = latest_log["status"]
|
||||
message = latest_log["message"]
|
||||
timestamp = latest_log["timestamp"]
|
||||
task_id = latest_log["task_id"]
|
||||
else:
|
||||
status = "not_run"
|
||||
message = None
|
||||
timestamp = None
|
||||
task_id = None
|
||||
|
||||
flow.append({
|
||||
"key": stage_key,
|
||||
"label": stage_info["label"],
|
||||
"status": status,
|
||||
"message": message,
|
||||
"timestamp": timestamp,
|
||||
"task_id": task_id,
|
||||
"can_retry": status == "failure"
|
||||
})
|
||||
|
||||
return flow
|
||||
|
||||
@@ -207,6 +207,57 @@
|
||||
color: #991B1B;
|
||||
}
|
||||
</style>
|
||||
<script>
|
||||
// JavaScript for handling retry functionality
|
||||
async function reprocessFile() {
|
||||
const fileId = {{ file.id }};
|
||||
const button = document.getElementById('reprocess-btn');
|
||||
const statusDiv = document.getElementById('reprocess-status');
|
||||
|
||||
// Disable button and show loading
|
||||
button.disabled = true;
|
||||
button.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Processing...';
|
||||
statusDiv.innerHTML = '';
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/files/${fileId}/reprocess`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (response.ok) {
|
||||
statusDiv.innerHTML = `
|
||||
<div style="background-color: #D1FAE5; color: #065F46; padding: 1rem; border-radius: 0.25rem; margin-top: 1rem;">
|
||||
<i class="fas fa-check-circle"></i> File queued for reprocessing. Task ID: ${data.task_id}
|
||||
<br><small>Refresh the page in a few moments to see updated status.</small>
|
||||
</div>
|
||||
`;
|
||||
// Reload page after 3 seconds
|
||||
setTimeout(() => window.location.reload(), 3000);
|
||||
} else {
|
||||
statusDiv.innerHTML = `
|
||||
<div class="error-message" style="margin-top: 1rem;">
|
||||
<i class="fas fa-exclamation-triangle"></i> Error: ${data.detail || 'Failed to reprocess file'}
|
||||
</div>
|
||||
`;
|
||||
button.disabled = false;
|
||||
button.innerHTML = '<i class="fas fa-redo"></i> Retry Processing';
|
||||
}
|
||||
} catch (error) {
|
||||
statusDiv.innerHTML = `
|
||||
<div class="error-message" style="margin-top: 1rem;">
|
||||
<i class="fas fa-exclamation-triangle"></i> Network error: ${error.message}
|
||||
</div>
|
||||
`;
|
||||
button.disabled = false;
|
||||
button.innerHTML = '<i class="fas fa-redo"></i> Retry Processing';
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
@@ -273,7 +324,15 @@
|
||||
|
||||
<!-- Processing History Card -->
|
||||
<div class="detail-card">
|
||||
<h3>Processing History</h3>
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1rem;">
|
||||
<h3 style="margin: 0;">Processing History</h3>
|
||||
{% if file_exists %}
|
||||
<button id="reprocess-btn" onclick="reprocessFile()" style="background-color: #3182ce; color: white; padding: 0.5rem 1rem; border: none; border-radius: 0.25rem; cursor: pointer; font-weight: 600;">
|
||||
<i class="fas fa-redo"></i> Retry Processing
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div id="reprocess-status"></div>
|
||||
{% if logs %}
|
||||
<div class="timeline">
|
||||
{% for log in logs %}
|
||||
@@ -305,6 +364,97 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Process Flow Visualization Card -->
|
||||
{% if flow_data %}
|
||||
<div class="detail-card">
|
||||
<h3>Process Flow Visualization</h3>
|
||||
<div style="padding: 1rem 0;">
|
||||
<div style="position: relative;">
|
||||
{% for stage in flow_data %}
|
||||
<div style="display: flex; align-items: center; margin-bottom: 1.5rem;">
|
||||
<!-- Status indicator -->
|
||||
<div style="width: 40px; height: 40px; border-radius: 50%; display: flex; align-items: center; justify-content: center; margin-right: 1rem;
|
||||
{% if stage.status == 'success' %}background-color: #48bb78; color: white;
|
||||
{% elif stage.status == 'failure' %}background-color: #f56565; color: white;
|
||||
{% elif stage.status == 'in_progress' %}background-color: #4299e1; color: white;
|
||||
{% elif stage.status == 'pending' %}background-color: #ecc94b; color: white;
|
||||
{% else %}background-color: #e2e8f0; color: #718096;{% endif %}
|
||||
">
|
||||
{% if stage.status == 'success' %}<i class="fas fa-check"></i>
|
||||
{% elif stage.status == 'failure' %}<i class="fas fa-times"></i>
|
||||
{% elif stage.status == 'in_progress' %}<i class="fas fa-spinner fa-spin"></i>
|
||||
{% elif stage.status == 'pending' %}<i class="fas fa-clock"></i>
|
||||
{% else %}<i class="fas fa-circle"></i>{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Stage info -->
|
||||
<div style="flex: 1; background-color: #f7fafc; padding: 1rem; border-radius: 0.5rem; border-left: 3px solid
|
||||
{% if stage.status == 'success' %}#48bb78
|
||||
{% elif stage.status == 'failure' %}#f56565
|
||||
{% elif stage.status == 'in_progress' %}#4299e1
|
||||
{% elif stage.status == 'pending' %}#ecc94b
|
||||
{% else %}#e2e8f0{% endif %};">
|
||||
<div style="font-weight: 600; color: #2d3748; margin-bottom: 0.25rem;">{{ stage.label }}</div>
|
||||
{% if stage.message %}
|
||||
<div style="color: #4a5568; font-size: 0.875rem;">{{ stage.message }}</div>
|
||||
{% endif %}
|
||||
{% if stage.status == 'not_run' %}
|
||||
<div style="color: #718096; font-size: 0.875rem; font-style: italic;">Not executed</div>
|
||||
{% endif %}
|
||||
{% if stage.timestamp %}
|
||||
<div style="color: #718096; font-size: 0.75rem; margin-top: 0.25rem;">
|
||||
{{ stage.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Connector line (except for last item) -->
|
||||
{% if not loop.last %}
|
||||
<div style="width: 2px; height: 20px; background-color: #e2e8f0; margin-left: 19px; margin-bottom: 0.5rem;"></div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- File Preview Card -->
|
||||
{% if file_exists or processed_exists %}
|
||||
<div class="detail-card">
|
||||
<h3>File Previews</h3>
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); gap: 2rem;">
|
||||
{% if file_exists %}
|
||||
<div>
|
||||
<h4 style="font-weight: 600; color: #2d3748; margin-bottom: 1rem;">Original File</h4>
|
||||
<div style="border: 2px solid #e2e8f0; border-radius: 0.5rem; overflow: hidden; background-color: #f7fafc;">
|
||||
<embed src="/api/files/{{ file.id }}/preview?version=original" type="application/pdf" width="100%" height="600px" style="border: none;">
|
||||
</div>
|
||||
<div style="margin-top: 0.5rem; text-align: center;">
|
||||
<a href="/api/files/{{ file.id }}/preview?version=original" target="_blank" style="color: #3182ce; text-decoration: none; font-size: 0.875rem;">
|
||||
<i class="fas fa-external-link-alt"></i> Open in new tab
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if processed_exists %}
|
||||
<div>
|
||||
<h4 style="font-weight: 600; color: #2d3748; margin-bottom: 1rem;">Processed File</h4>
|
||||
<div style="border: 2px solid #e2e8f0; border-radius: 0.5rem; overflow: hidden; background-color: #f7fafc;">
|
||||
<embed src="/api/files/{{ file.id }}/preview?version=processed" type="application/pdf" width="100%" height="600px" style="border: none;">
|
||||
</div>
|
||||
<div style="margin-top: 0.5rem; text-align: center;">
|
||||
<a href="/api/files/{{ file.id }}/preview?version=processed" target="_blank" style="color: #3182ce; text-decoration: none; font-size: 0.875rem;">
|
||||
<i class="fas fa-external-link-alt"></i> Open in new tab
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
Reference in New Issue
Block a user