gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: Apply Black formatting to modified files

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-08 08:16:54 +00:00
parent 0a461343e7
commit 78004cd9a4
5 changed files with 280 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+16 -21
View File
@@ -384,17 +384,13 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
# Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
raise HTTPException(
status_code=400,
detail="Local file not found on disk. Cannot reprocess."
)
raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")
# Queue the file for processing
task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)
logger.info(
f"Reprocessing file: ID={file_record.id}, "
f"Filename={file_record.original_filename}, TaskID={task.id}"
f"Reprocessing file: ID={file_record.id}, " f"Filename={file_record.original_filename}, TaskID={task.id}"
)
return {
@@ -402,7 +398,7 @@ def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(
"message": "File queued for reprocessing",
"file_id": file_record.id,
"filename": file_record.original_filename,
"task_id": task.id
"task_id": task.id,
}
except HTTPException:
@@ -418,7 +414,7 @@ def retry_subtask(
request: Request,
file_id: int,
subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
"""
Retry a specific failed upload subtask for a file.
@@ -459,13 +455,13 @@ def retry_subtask(
"upload_to_webdav": upload_to_webdav,
"upload_to_ftp": upload_to_ftp,
"upload_to_sftp": upload_to_sftp,
"upload_to_email": upload_to_email
"upload_to_email": upload_to_email,
}
if subtask_name not in task_map:
raise HTTPException(
status_code=400,
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}"
detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}",
)
# Check for processed file (upload tasks work with processed files)
@@ -487,26 +483,20 @@ def retry_subtask(
break
if not file_path:
raise HTTPException(
status_code=400,
detail="Processed file not found. Cannot retry upload."
)
raise HTTPException(status_code=400, detail="Processed file not found. Cannot retry upload.")
# Queue the specific upload task
upload_task = task_map[subtask_name]
task = upload_task.delay(file_path, file_id)
logger.info(
f"Retrying upload subtask: FileID={file_record.id}, "
f"Subtask={subtask_name}, TaskID={task.id}"
)
logger.info(f"Retrying upload subtask: FileID={file_record.id}, " f"Subtask={subtask_name}, TaskID={task.id}")
return {
"status": "success",
"message": f"Upload task {subtask_name} queued for retry",
"file_id": file_record.id,
"subtask_name": subtask_name,
"task_id": task.id
"task_id": task.id,
}
except HTTPException:
@@ -518,7 +508,12 @@ def retry_subtask(
@router.get("/files/{file_id}/preview")
@require_login
def get_file_preview(request: Request, file_id: int, version: str = Query("original", description="original or processed"), db: Session = Depends(get_db)):
def get_file_preview(
request: Request,
file_id: int,
version: str = Query("original", description="original or processed"),
db: Session = Depends(get_db),
):
"""
Get file content for preview (original or processed version).
@@ -573,7 +568,7 @@ def get_file_preview(request: Request, file_id: int, version: str = Query("origi
return FileResponse(
path=file_path,
media_type=file_record.mime_type or "application/pdf",
filename=file_record.original_filename
filename=file_record.original_filename,
)
except HTTPException:
app/views/files.py
+103 -97
View File
@@ -1,6 +1,7 @@
"""
File management views for displaying and managing files.
"""
from fastapi import Request, Depends, Query
from sqlalchemy.orm import Session
from typing import Optional
@@ -11,6 +12,7 @@ from app.config import settings
router = APIRouter()
@router.get("/files")
@require_login
def files_page(
@@ -22,7 +24,7 @@ def files_page(
sort_order: str = Query("desc"),
search: Optional[str] = Query(None),
mime_type: Optional[str] = Query(None),
status: Optional[str] = Query(None)
status: Optional[str] = Query(None),
):
"""
Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -52,29 +54,26 @@ def files_page(
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "in_progress"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "failure"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "success"
).distinct().subquery()
success_files = (
db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
)
failed_files = db.query(ProcessingLog.file_id).filter(
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
).distinct().subquery()
failed_files = (
db.query(ProcessingLog.file_id)
.filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
.distinct()
.subquery()
)
query = query.filter(
FileRecord.id.in_(db.query(success_files.c.file_id))
).filter(
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
)
@@ -87,7 +86,7 @@ def files_page(
"original_filename": FileRecord.original_filename,
"file_size": FileRecord.file_size,
"mime_type": FileRecord.mime_type,
"created_at": FileRecord.created_at
"created_at": FileRecord.created_at,
}.get(sort_by, FileRecord.created_at)
if sort_order == "asc":
@@ -113,45 +112,44 @@ def files_page(
total_pages = (total_items + per_page - 1) // per_page
# Get unique MIME types for filter dropdown
mime_types = db.query(FileRecord.mime_type).distinct().filter(
FileRecord.mime_type.isnot(None)
).all()
mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
mime_types = [mt[0] for mt in mime_types if mt[0]]
# Debug output
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")
return templates.TemplateResponse("files.html", {
"request": request,
"files": files_with_status,
"pagination": {
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages
return templates.TemplateResponse(
"files.html",
{
"request": request,
"files": files_with_status,
"pagination": {
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages,
},
"sort_by": sort_by,
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types,
},
"sort_by": sort_by,
"sort_order": sort_order,
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types
})
)
except Exception as e:
# Log any errors
logger.error(f"Error retrieving files: {str(e)}")
# Return error message to template
return templates.TemplateResponse("files.html", {
"request": request,
"files": [],
"pagination": {
"page": 1,
"per_page": per_page,
"total_items": 0,
"total_pages": 0
return templates.TemplateResponse(
"files.html",
{
"request": request,
"files": [],
"pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
"error": str(e),
},
"error": str(e)
})
)
@router.get("/files/{file_id}/detail")
@@ -168,16 +166,17 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
return templates.TemplateResponse("file_detail.html", {
"request": request,
"file": None,
"error": f"File with ID {file_id} not found"
})
return templates.TemplateResponse(
"file_detail.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"}
)
# Get processing logs
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.asc()).all()
logs = (
db.query(ProcessingLog)
.filter(ProcessingLog.file_id == file_id)
.order_by(ProcessingLog.timestamp.asc())
.all()
)
# Check if file exists on disk
file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False
@@ -204,22 +203,21 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
# Compute step-aligned summary
step_summary = _compute_step_summary(logs)
return templates.TemplateResponse("file_detail.html", {
"request": request,
"file": file_record,
"logs": logs,
"file_exists": file_exists,
"processed_exists": processed_exists,
"flow_data": flow_data,
"step_summary": step_summary
})
return templates.TemplateResponse(
"file_detail.html",
{
"request": request,
"file": file_record,
"logs": logs,
"file_exists": file_exists,
"processed_exists": processed_exists,
"flow_data": flow_data,
"step_summary": step_summary,
},
)
except Exception as e:
logger.error(f"Error retrieving file details: {str(e)}")
return templates.TemplateResponse("file_detail.html", {
"request": request,
"file": None,
"error": str(e)
})
return templates.TemplateResponse("file_detail.html", {"request": request, "file": None, "error": str(e)})
def _compute_processing_flow(logs):
@@ -233,13 +231,19 @@ def _compute_processing_flow(logs):
stages = {
"hash_file": {"label": "File Upload & Hash", "next": ["create_file_record"]},
"create_file_record": {"label": "Create File Record", "next": ["check_text"]},
"check_text": {"label": "Check Embedded Text", "next": ["extract_text", "process_with_azure_document_intelligence"]},
"check_text": {
"label": "Check Embedded Text",
"next": ["extract_text", "process_with_azure_document_intelligence"],
},
"extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
"process_with_azure_document_intelligence": {"label": "OCR Processing (Azure)", "next": ["extract_metadata_with_gpt"]},
"process_with_azure_document_intelligence": {
"label": "OCR Processing (Azure)",
"next": ["extract_metadata_with_gpt"],
},
"extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
"embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True}
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
}
# Define upload sub-tasks (branches)
@@ -263,7 +267,7 @@ def _compute_processing_flow(logs):
"queue_webdav": "WebDAV",
"queue_ftp": "FTP Storage",
"queue_sftp": "SFTP Storage",
"queue_email": "Email"
"queue_email": "Email",
}
# Create a map of step names to their log entries
@@ -279,22 +283,16 @@ def _compute_processing_flow(logs):
upload_key = step_name.replace("queue_", "upload_to_")
if upload_key not in upload_branches:
upload_branches[upload_key] = []
upload_branches[upload_key].append({
"status": log.status,
"message": log.message,
"timestamp": log.timestamp,
"task_id": log.task_id
})
upload_branches[upload_key].append(
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
)
else:
# Regular processing step
if step_name not in step_map:
step_map[step_name] = []
step_map[step_name].append({
"status": log.status,
"message": log.message,
"timestamp": log.timestamp,
"task_id": log.task_id
})
step_map[step_name].append(
{"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
)
# Build the flow structure
flow = []
@@ -322,7 +320,7 @@ def _compute_processing_flow(logs):
"timestamp": timestamp,
"task_id": task_id,
"can_retry": status == "failure",
"is_branch_parent": stage_info.get("has_branches", False)
"is_branch_parent": stage_info.get("has_branches", False),
}
# If this is the upload stage, add branches
@@ -332,15 +330,17 @@ def _compute_processing_flow(logs):
latest_upload = upload_logs[-1]
upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title())
branches.append({
"key": upload_key,
"label": upload_name,
"status": latest_upload["status"],
"message": latest_upload["message"],
"timestamp": latest_upload["timestamp"],
"task_id": latest_upload["task_id"],
"can_retry": latest_upload["status"] == "failure"
})
branches.append(
{
"key": upload_key,
"label": upload_name,
"status": latest_upload["status"],
"message": latest_upload["message"],
"timestamp": latest_upload["timestamp"],
"task_id": latest_upload["task_id"],
"can_retry": latest_upload["status"] == "failure",
}
)
stage_data["branches"] = branches
flow.append(stage_data)
@@ -356,9 +356,15 @@ def _compute_step_summary(logs):
"""
# Count statuses for main processing steps (not uploads)
main_steps = [
"hash_file", "create_file_record", "check_text", "extract_text",
"process_with_azure_document_intelligence", "extract_metadata_with_gpt",
"embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations"
"hash_file",
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
]
upload_prefixes = ["upload_to_", "queue_"]
@@ -399,5 +405,5 @@ def _compute_step_summary(logs):
"main": main_counts,
"uploads": upload_counts,
"total_main_steps": len(main_steps_seen),
"total_upload_tasks": len(upload_tasks_seen)
"total_upload_tasks": len(upload_tasks_seen),
}
tests/test_api.py
+1
View File
@@ -1,6 +1,7 @@
"""
Integration tests for API endpoints.
"""
import pytest
from fastapi.testclient import TestClient
tests/test_bulk_operations.py
+4 -12
View File
@@ -35,9 +35,7 @@ class TestSingleFileOperations:
assert f"File record {file_id} deleted successfully" in data["message"]
# Verify file is deleted
file_record = (
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
assert file_record is None
def test_single_file_delete_nonexistent(self, client: TestClient, db_session):
@@ -79,9 +77,7 @@ class TestBulkOperations:
# Verify files are deleted
for file_id in file_ids:
file_record = (
db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
)
file_record = db_session.query(FileRecord).filter(FileRecord.id == file_id).first()
assert file_record is None
def test_bulk_delete_empty_list(self, client: TestClient, db_session):
@@ -97,9 +93,7 @@ class TestBulkOperations:
assert response.status_code == 404
@patch("app.api.files.process_document")
def test_bulk_reprocess_success(
self, mock_process_document, client: TestClient, db_session
):
def test_bulk_reprocess_success(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing of files."""
# Setup mock
mock_task = MagicMock()
@@ -132,9 +126,7 @@ class TestBulkOperations:
assert len(data["task_ids"]) == 2
@patch("app.api.files.process_document")
def test_bulk_reprocess_missing_files(
self, mock_process_document, client: TestClient, db_session
):
def test_bulk_reprocess_missing_files(self, mock_process_document, client: TestClient, db_session):
"""Test bulk reprocessing when some local files are missing."""
# Setup mock
mock_task = MagicMock()
tests/test_file_detail_endpoints.py
+20 -19
View File
@@ -1,6 +1,7 @@
"""
Tests for file detail view improvements including reprocessing and preview endpoints.
"""
import os
import pytest
from unittest.mock import patch, MagicMock
@@ -26,7 +27,7 @@ class TestFileReprocessing:
original_filename="test.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -38,7 +39,7 @@ class TestFileReprocessing:
task_id="test-task-123",
step_name="extract_metadata_with_gpt",
status="failure",
message="API error"
message="API error",
)
db_session.add(log)
db_session.commit()
@@ -66,7 +67,7 @@ class TestFileReprocessing:
original_filename="missing.pdf",
local_filename="/nonexistent/path/missing.pdf",
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -96,7 +97,7 @@ class TestSubtaskRetry:
original_filename="retry.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -115,7 +116,7 @@ class TestSubtaskRetry:
original_filename="retry2.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -139,7 +140,7 @@ class TestFilePreview:
original_filename="preview.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -158,7 +159,7 @@ class TestFilePreview:
original_filename="processed.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -183,7 +184,7 @@ class TestFilePreview:
original_filename="test.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -207,7 +208,7 @@ class TestFileDetailView:
original_filename="detail.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -220,22 +221,22 @@ class TestFileDetailView:
task_id="task-1",
step_name="hash_file",
status="success",
message="File hashed successfully"
message="File hashed successfully",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-1",
step_name="create_file_record",
status="success",
message="File record created"
message="File record created",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-1",
step_name="extract_metadata_with_gpt",
status="failure",
message="API rate limit exceeded"
)
message="API rate limit exceeded",
),
]
for log in logs:
db_session.add(log)
@@ -257,7 +258,7 @@ class TestFileDetailView:
original_filename="branches.pdf",
local_filename=sample_pdf_path,
file_size=1024,
mime_type="application/pdf"
mime_type="application/pdf",
)
db_session.add(file_record)
db_session.commit()
@@ -270,29 +271,29 @@ class TestFileDetailView:
task_id="task-1",
step_name="send_to_all_destinations",
status="success",
message="Queued uploads"
message="Queued uploads",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-2",
step_name="upload_to_dropbox",
status="success",
message="Uploaded to Dropbox"
message="Uploaded to Dropbox",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-3",
step_name="upload_to_s3",
status="failure",
message="S3 connection error"
message="S3 connection error",
),
ProcessingLog(
file_id=file_record.id,
task_id="task-4",
step_name="upload_to_nextcloud",
status="success",
message="Uploaded to Nextcloud"
)
message="Uploaded to Nextcloud",
),
]
for log in logs:
db_session.add(log)