gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Address code review feedback

Browse Source 
- Fixed status filtering to occur before pagination for correct counts
- Resolved N+1 query problem by batch-fetching processing statuses
- Extracted status computation logic to shared utility function
- Changed sort indicator from ⬍ to ↕ for better browser compatibility
- Updated both API and view layers to use shared status utilities

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-07 15:20:59 +00:00
parent b76fb147cb
commit 775ffa5082
4 changed files with 165 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+51 -52
View File
@@ -16,6 +16,7 @@ from app.config import settings
from app.api.common import get_db
from app.tasks.process_document import process_document
from app.tasks.convert_to_pdf import convert_to_pdf
from app.utils.file_status import get_file_processing_status, get_files_processing_status
# Set up logging
logger = logging.getLogger(__name__)
@@ -70,7 +71,42 @@ def list_files_api(
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Get total count before pagination
# Apply status filter (before pagination for correct counts)
if status:
# Subquery to get file IDs matching the status
if status == "pending":
# Files with no logs
subq = db.query(ProcessingLog.file_id).distinct()
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "in_progress"
).distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "failure"
).distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "success"
).distinct().subquery()
failed_files = db.query(ProcessingLog.file_id).filter(
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
).distinct().subquery()
query = query.filter(
FileRecord.id.in_(db.query(success_files.c.file_id))
).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
)
# Get total count before pagination (after all filters)
total_items = query.count()
# Apply sorting
@@ -91,12 +127,13 @@ def list_files_api(
offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all()
# Get processing status for all files efficiently
file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids)
# Build result with processing status
result = []
for f in files:
# Get processing status for this file
processing_status = _get_file_processing_status(db, f.id)
result.append({
"id": f.id,
"filehash": f.filehash,
@@ -105,14 +142,14 @@ def list_files_api(
"file_size": f.file_size,
"mime_type": f.mime_type,
"created_at": f.created_at.isoformat() if f.created_at else None,
"processing_status": processing_status
"processing_status": statuses.get(f.id, {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
})
})
# Filter by status if requested (after computing statuses)
if status:
result = [f for f in result if f["processing_status"]["status"] == status]
total_items = len(result) # Update total for filtered results
# Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page
@@ -129,49 +166,11 @@ def list_files_api(
def _get_file_processing_status(db: Session, file_id: int) -> dict:
"""
Get the processing status for a file by checking its processing logs.
Returns:
dict with status, last_step, and has_errors
Deprecated: Use app.utils.file_status.get_file_processing_status instead.
Kept for backward compatibility with file detail endpoint.
"""
# Get all logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.desc()).all()
if not logs:
return {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
}
# Check for failures
has_errors = any(log.status == "failure" for log in logs)
# Check if any in progress
in_progress = any(log.status == "in_progress" for log in logs)
# Get the latest log
latest_log = logs[0]
# Determine overall status
if has_errors:
status = "failed"
elif in_progress:
status = "processing"
elif latest_log.status == "success":
status = "completed"
else:
status = "pending"
return {
"status": status,
"last_step": latest_log.step_name,
"has_errors": has_errors,
"total_steps": len(logs)
}
from app.utils.file_status import get_file_processing_status
return get_file_processing_status(db, file_id)
app/utils/file_status.py
+102
View File
@@ -0,0 +1,102 @@
"""
Utility functions for file processing status determination.
"""
from typing import Dict, List
from sqlalchemy.orm import Session
from app.models import ProcessingLog
def get_file_processing_status(db: Session, file_id: int) -> Dict:
"""
Get the processing status for a file by checking its processing logs.
Args:
db: Database session
file_id: ID of the file
Returns:
dict with status, last_step, and has_errors
"""
# Get all logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.desc()).all()
return _compute_status_from_logs(logs)
def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, Dict]:
"""
Get processing status for multiple files efficiently.
Args:
db: Database session
file_ids: List of file IDs
Returns:
dict mapping file_id to status dict
"""
# Get all logs for these files in one query
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id.in_(file_ids)
).order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc()).all()
# Group logs by file_id
logs_by_file = {}
for log in logs:
if log.file_id not in logs_by_file:
logs_by_file[log.file_id] = []
logs_by_file[log.file_id].append(log)
# Compute status for each file
result = {}
for file_id in file_ids:
file_logs = logs_by_file.get(file_id, [])
result[file_id] = _compute_status_from_logs(file_logs)
return result
def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
"""
Compute processing status from a list of processing logs.
Args:
logs: List of ProcessingLog objects (should be ordered by timestamp desc)
Returns:
dict with status, last_step, has_errors, and total_steps
"""
if not logs:
return {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
}
# Check for failures
has_errors = any(log.status == "failure" for log in logs)
# Check if any in progress
in_progress = any(log.status == "in_progress" for log in logs)
# Get the latest log
latest_log = logs[0]
# Determine overall status
if has_errors:
status = "failed"
elif in_progress:
status = "processing"
elif latest_log.status == "success":
status = "completed"
else:
status = "pending"
return {
"status": status,
"last_step": latest_log.step_name,
"has_errors": has_errors,
"total_steps": len(logs)
}
app/views/files.py
+7 -25
View File
@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
from typing import Optional
from app.views.base import APIRouter, templates, require_login, get_db, logger
from app.utils.file_status import get_files_processing_status
router = APIRouter()
@@ -62,33 +63,14 @@ def files_page(
offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all()
# Add processing status to each file
# Get processing status for all files efficiently (avoids N+1)
file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids)
# Add status to each file
files_with_status = []
for file in files:
# Get processing logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file.id
).order_by(ProcessingLog.timestamp.desc()).all()
# Determine status
if not logs:
processing_status = "pending"
else:
has_errors = any(log.status == "failure" for log in logs)
in_progress = any(log.status == "in_progress" for log in logs)
latest_log = logs[0]
if has_errors:
processing_status = "failed"
elif in_progress:
processing_status = "processing"
elif latest_log.status == "success":
processing_status = "completed"
else:
processing_status = "pending"
# Add status to file object as an attribute
file.processing_status = processing_status
file.processing_status = statuses.get(file.id, {}).get("status", "pending")
files_with_status.append(file)
# Calculate pagination info
frontend/templates/files.html
+5 -5
View File
@@ -308,7 +308,7 @@
{% if sort_by == 'id' %}
{% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
{% else %}
⬍
↕
{% endif %}
</span>
</th>
@@ -318,7 +318,7 @@
{% if sort_by == 'original_filename' %}
{% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
{% else %}
⬍
↕
{% endif %}
</span>
</th>
@@ -328,7 +328,7 @@
{% if sort_by == 'file_size' %}
{% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
{% else %}
⬍
↕
{% endif %}
</span>
</th>
@@ -338,7 +338,7 @@
{% if sort_by == 'mime_type' %}
{% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
{% else %}
⬍
↕
{% endif %}
</span>
</th>
@@ -349,7 +349,7 @@
{% if sort_by == 'created_at' %}
{% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
{% else %}
⬍
↕
{% endif %}
</span>
</th>