gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance processing status tracking and UI display for file details

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-02-12 01:07:22 +01:00
parent 02e1445e01
commit 83c65f3c40
7 changed files with 287 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/file_queries.py
+70 -29
View File
@@ -2,7 +2,7 @@
Shared file query utilities for filtering files by processing status.
This module contains reusable query logic for filtering FileRecord objects
based on their processing status (pending, processing, failed, completed).
based on their processing status using the FileProcessingStep table.
"""
from typing import Optional
@@ -10,24 +10,33 @@ from typing import Optional
from sqlalchemy import or_
from sqlalchemy.orm import Query, Session
from app.models import FileRecord, ProcessingLog
from app.models import FileRecord, FileProcessingStep
def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query:
"""
Apply status filter to a FileRecord query.
Apply status filter to a FileRecord query using FileProcessingStep table.
This function modifies a SQLAlchemy query to filter files based on their
processing status by examining associated ProcessingLog entries.
processing status by examining associated FileProcessingStep entries.
Only tracks "real" processing steps that represent user-facing status:
- Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: queue_*, upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored
as they may not complete properly and don't affect the actual status.
Args:
query: The base SQLAlchemy query for FileRecord objects
db: Database session for creating subqueries
status: Status filter to apply. Valid values:
- "pending": Files with no ProcessingLog entries
- "processing": Files with in_progress logs
- "failed": Files with failure logs
- "completed": Files with success logs but no failures or in_progress
- "pending": Files with no real FileProcessingStep entries
- "processing": Files with in_progress real steps
- "failed": Files with failure real steps
- "completed": Files with all real steps success/skipped
- None: No filter applied (returns query unchanged)
Returns:
@@ -41,32 +50,64 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
if not status:
return query
# Subquery to get file IDs matching the status
if status == "pending":
# Files with no logs
subq = db.query(ProcessingLog.file_id).distinct()
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
# Define which steps are "real" status-determining steps
# Only high-level logical steps and actual upload destinations (not queue_* steps)
REAL_STEPS = {
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
"upload_to_dropbox",
"upload_to_paperless",
"upload_to_google_drive",
"upload_to_ftp",
"upload_to_onedrive",
"upload_to_webdav",
"upload_to_sftp",
"upload_to_nextcloud",
"upload_to_paperless_ngx",
"upload_to_email",
"upload_to_s3",
}
failed_files = (
db.query(ProcessingLog.file_id)
.filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
# Filter to only real steps
real_steps_subq = (
db.query(FileProcessingStep)
.filter(FileProcessingStep.step_name.in_(REAL_STEPS))
)
if status == "pending":
# Files with no real steps (never started processing)
subq = real_steps_subq.distinct().subquery()
query = query.filter(~FileRecord.id.in_(db.query(subq.c.file_id)))
elif status == "processing":
# Files with in_progress real steps
subq = real_steps_subq.filter(FileProcessingStep.status == "in_progress").distinct().subquery()
query = query.filter(FileRecord.id.in_(db.query(subq.c.file_id)))
elif status == "failed":
# Files with failure real steps
subq = real_steps_subq.filter(FileProcessingStep.status == "failure").distinct().subquery()
query = query.filter(FileRecord.id.in_(db.query(subq.c.file_id)))
elif status == "completed":
# Files where all real steps are either success or skipped (no failures or in_progress)
# Get files that have real steps
files_with_real_steps = real_steps_subq.distinct().subquery()
# Get files with failures or in_progress on real steps
files_with_issues = (
real_steps_subq
.filter(or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress"))
.distinct()
.subquery()
)
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
# Select files with real steps that don't have issues
query = query.filter(FileRecord.id.in_(db.query(files_with_real_steps.c.file_id))).filter(
~FileRecord.id.in_(db.query(files_with_issues.c.file_id))
)
return query
app/utils/file_status.py
+37 -2
View File
@@ -46,6 +46,14 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
"""
Get processing status for multiple files efficiently.
Only counts "real" processing steps that represent user-facing status:
- Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored.
Args:
db: Database session
file_ids: List of file IDs
@@ -53,8 +61,35 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
Returns:
dict mapping file_id to status dict
"""
# Get all steps for these files in one query
steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id.in_(file_ids)).all()
# Define which steps are "real" status-determining steps
REAL_STEPS = {
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
"upload_to_dropbox",
"upload_to_paperless",
"upload_to_google_drive",
"upload_to_ftp",
"upload_to_onedrive",
"upload_to_webdav",
"upload_to_sftp",
"upload_to_nextcloud",
"upload_to_paperless_ngx",
"upload_to_email",
"upload_to_s3",
}
# Get all REAL steps for these files in one query
steps = (
db.query(FileProcessingStep)
.filter(FileProcessingStep.file_id.in_(file_ids), FileProcessingStep.step_name.in_(REAL_STEPS))
.all()
)
# Group steps by file_id
steps_by_file = {}
app/utils/logging.py
+38 -1
View File
@@ -1,9 +1,10 @@
import logging
import threading
from collections import defaultdict
from datetime import datetime
from app.database import SessionLocal
from app.models import ProcessingLog
from app.models import FileProcessingStep, ProcessingLog
class TaskLogCollector(logging.Handler):
@@ -66,6 +67,8 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
If no explicit detail is provided, automatically drains any buffered
worker log output for this task ID and stores it as the detail.
Also updates the FileProcessingStep table for definitive status tracking.
Args:
task_id: The Celery task ID
step_name: Name of the processing step
@@ -83,6 +86,7 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
detail = collected
with SessionLocal() as db:
# Log to ProcessingLog (for historical viewing)
log_entry = ProcessingLog(
task_id=task_id,
step_name=step_name,
@@ -92,4 +96,37 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
detail=detail,
)
db.add(log_entry)
# Update FileProcessingStep table (for status tracking) if file_id is provided
if file_id and step_name:
# Find or create the step record
step_record = (
db.query(FileProcessingStep)
.filter(FileProcessingStep.file_id == file_id, FileProcessingStep.step_name == step_name)
.first()
)
now = datetime.utcnow()
if not step_record:
# Create new step record
step_record = FileProcessingStep(
file_id=file_id,
step_name=step_name,
status=status,
started_at=now if status == "in_progress" else None,
completed_at=now if status in ("success", "failure", "skipped") else None,
error_message=message if status == "failure" else None,
)
db.add(step_record)
else:
# Update existing step record
step_record.status = status
if status == "in_progress" and not step_record.started_at:
step_record.started_at = now
if status in ("success", "failure", "skipped"):
step_record.completed_at = now
if status == "failure":
step_record.error_message = message or detail
db.commit()
app/utils/step_manager.py
+52 -6
View File
@@ -156,6 +156,9 @@ def get_file_step_status(db: Session, file_id: int) -> Dict[str, Dict]:
def get_file_overall_status(db: Session, file_id: int) -> Dict:
"""
Get the overall processing status for a file based on its steps.
Only considers "real" processing steps that represent user-facing status.
Ignores diagnostic/internal steps like poll_task, upload_file, etc.
Args:
db: Database session
@@ -172,7 +175,28 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
"in_progress_steps": 2
}
"""
steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
# Define which steps are "real" status-determining steps
# Only high-level logical steps, not implementation sub-steps
REAL_MAIN_STEPS = {
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
}
all_steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
# Filter to only real steps
steps = [
s for s in all_steps
if s.step_name in REAL_MAIN_STEPS
or s.step_name.startswith("queue_")
or s.step_name.startswith("upload_to_")
]
if not steps:
return {
@@ -217,6 +241,9 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"""
Get a summary of main steps vs upload steps with status counts.
Only counts "real" processing steps that represent user-facing status.
Ignores diagnostic/internal steps like poll_task, upload_file, etc.
Args:
db: Database session
file_id: ID of the file
@@ -230,13 +257,24 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"total_upload_tasks": 6
}
"""
# Define which steps are "real" status-determining steps
# Only high-level logical steps, not implementation sub-steps
REAL_MAIN_STEPS = {
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
}
steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0, "skipped": 0}
upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0, "skipped": 0}
upload_prefixes = ["upload_to_", "queue_"]
main_steps_count = 0
upload_steps_count = 0
@@ -246,14 +284,21 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
if status == "pending":
status = "queued"
# Check if it's an upload task
is_upload = any(step.step_name.startswith(prefix) for prefix in upload_prefixes)
# Check if it's an upload task (only count actual upload_to_* steps, not queue_* steps)
is_upload = step.step_name.startswith("upload_to_")
# Only count "real" steps
is_real_step = step.step_name in REAL_MAIN_STEPS or is_upload or step.step_name.startswith("queue_")
if not is_real_step:
# Skip diagnostic/internal steps like poll_task, upload_file, set_custom_fields, etc.
continue
if is_upload:
if status in upload_counts:
upload_counts[status] += 1
upload_steps_count += 1
elif step.step_name in MAIN_PROCESSING_STEPS:
elif step.step_name in REAL_MAIN_STEPS:
if status in main_counts:
main_counts[status] += 1
main_steps_count += 1
@@ -264,3 +309,4 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"total_main_steps": main_steps_count,
"total_upload_tasks": upload_steps_count,
}