gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix code formatting with black, isort, and flake8

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-12 03:21:41 +00:00
parent f79bd2cb0c
commit ff9a3ff49f
87 changed files with 874 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/file_queries.py
+10 -12
View File
@@ -10,7 +10,7 @@ from typing import Optional
from sqlalchemy import or_
from sqlalchemy.orm import Query, Session
from app.models import FileRecord, FileProcessingStep
from app.models import FileProcessingStep, FileRecord
def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query:
@@ -19,13 +19,13 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
This function modifies a SQLAlchemy query to filter files based on their
processing status by examining associated FileProcessingStep entries.
Only tracks "real" processing steps that represent user-facing status:
- Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: queue_*, upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored
as they may not complete properly and don't affect the actual status.
@@ -53,7 +53,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
# Define which steps are "real" status-determining steps
# Only high-level logical steps and actual upload destinations (not queue_* steps)
from app.config import settings
REAL_STEPS = {
"create_file_record",
"check_text",
@@ -75,16 +75,13 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
"upload_to_email",
"upload_to_s3",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_STEPS.add("check_for_duplicates")
# Filter to only real steps
real_steps_subq = (
db.query(FileProcessingStep)
.filter(FileProcessingStep.step_name.in_(REAL_STEPS))
)
real_steps_subq = db.query(FileProcessingStep).filter(FileProcessingStep.step_name.in_(REAL_STEPS))
if status == "pending":
# Files with no real steps (never started processing)
@@ -102,14 +99,15 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
# Files where all real steps are either success or skipped (no failures or in_progress)
# Exclude duplicates from completed
query = query.filter(FileRecord.is_duplicate.is_(False))
# Get files that have real steps
files_with_real_steps = real_steps_subq.distinct().subquery()
# Get files with failures or in_progress on real steps
files_with_issues = (
real_steps_subq
.filter(or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress"))
real_steps_subq.filter(
or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress")
)
.distinct()
.subquery()
)
app/utils/file_status.py
+4 -4
View File
@@ -7,7 +7,7 @@ from typing import Dict, List
from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.utils.step_manager import get_file_overall_status, get_step_summary
from app.utils.step_manager import get_file_overall_status
def get_file_processing_status(db: Session, file_id: int) -> Dict:
@@ -60,7 +60,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored.
Args:
@@ -78,7 +78,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
# Define which steps are "real" status-determining steps
from app.config import settings
REAL_STEPS = {
"create_file_record",
"check_text",
@@ -100,7 +100,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
"upload_to_email",
"upload_to_s3",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_STEPS.add("check_for_duplicates")
app/utils/filename_utils.py
+7 -7
View File
@@ -73,22 +73,22 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
"""
Returns a unique filepath in the specified directory using a numeric counter suffix.
If 'base_filename.pdf' exists, it will append '-0001', '-0002', etc.
This function implements robust collision handling with zero-padded numeric suffixes
as required for document storage organization.
Args:
directory (str): Directory path where the file will be stored
base_filename (str): Base name for the file (without extension)
extension (str): File extension including the dot (default: ".pdf")
Returns:
str: Full path to a unique filename
Examples:
>>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
"/workdir/original/2024-01-01_Invoice.pdf" # If doesn't exist
>>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
"/workdir/original/2024-01-01_Invoice-0001.pdf" # If original exists
"""
@@ -96,7 +96,7 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
candidate = os.path.join(directory, base_filename + extension)
if not os.path.exists(candidate):
return candidate
# If base exists, try with counter suffix
counter = 1
while True:
@@ -106,7 +106,7 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
if not os.path.exists(candidate):
return candidate
counter += 1
# Sanity check to prevent infinite loops (very unlikely to reach)
if counter > 9999:
# Fall back to timestamp + UUID if somehow we have 10000 collisions
app/utils/logging.py
+4 -4
View File
@@ -96,7 +96,7 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
detail=detail,
)
db.add(log_entry)
# Update FileProcessingStep table (for status tracking) if file_id is provided
if file_id and step_name:
# Find or create the step record
@@ -105,9 +105,9 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
.filter(FileProcessingStep.file_id == file_id, FileProcessingStep.step_name == step_name)
.first()
)
now = datetime.utcnow()
if not step_record:
# Create new step record
step_record = FileProcessingStep(
@@ -128,5 +128,5 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
step_record.completed_at = now
if status == "failure":
step_record.error_message = message or detail
db.commit()
app/utils/migrate_logs_to_steps.py
+1 -4
View File
@@ -6,14 +6,11 @@ for files that were processed before the status tracking table was created.
"""
import logging
from datetime import datetime
from typing import Dict, List
from sqlalchemy import func
from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.utils.step_manager import MAIN_PROCESSING_STEPS
from app.models import FileProcessingStep, ProcessingLog
logger = logging.getLogger(__name__)
app/utils/step_manager.py
+10 -12
View File
@@ -167,7 +167,7 @@ def get_file_step_status(db: Session, file_id: int) -> Dict[str, Dict]:
def get_file_overall_status(db: Session, file_id: int) -> Dict:
"""
Get the overall processing status for a file based on its steps.
Only considers "real" processing steps that represent user-facing status.
Ignores diagnostic/internal steps like poll_task, upload_file, etc.
@@ -211,19 +211,18 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
"finalize_document_storage",
"send_to_all_destinations",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_MAIN_STEPS.add("check_for_duplicates")
all_steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
# Filter to only real steps
steps = [
s for s in all_steps
if s.step_name in REAL_MAIN_STEPS
or s.step_name.startswith("queue_")
or s.step_name.startswith("upload_to_")
s
for s in all_steps
if s.step_name in REAL_MAIN_STEPS or s.step_name.startswith("queue_") or s.step_name.startswith("upload_to_")
]
if not steps:
@@ -297,11 +296,11 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"finalize_document_storage",
"send_to_all_destinations",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_MAIN_STEPS.add("check_for_duplicates")
steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0, "skipped": 0}
@@ -318,7 +317,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
# Check if it's an upload task (only count actual upload_to_* steps, not queue_* steps)
is_upload = step.step_name.startswith("upload_to_")
# Only count "real" steps
is_real_step = step.step_name in REAL_MAIN_STEPS or is_upload or step.step_name.startswith("queue_")
@@ -341,4 +340,3 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"total_main_steps": main_steps_count,
"total_upload_tasks": upload_steps_count,
}
app/utils/step_timeout.py
+2 -5
View File
@@ -32,9 +32,7 @@ def get_step_timeout() -> int:
def mark_stalled_steps_as_failed(
db: Session,
timeout_seconds: Optional[int] = None,
file_id: Optional[int] = None
db: Session, timeout_seconds: Optional[int] = None, file_id: Optional[int] = None
) -> int:
"""
Find and mark any in-progress steps that have exceeded the timeout as failed.
@@ -75,8 +73,7 @@ def mark_stalled_steps_as_failed(
return 0
logger.warning(
f"Found {len(stalled_steps)} stalled step(s) that exceeded "
f"{timeout_seconds}s timeout. Marking as failed."
f"Found {len(stalled_steps)} stalled step(s) that exceeded " f"{timeout_seconds}s timeout. Marking as failed."
)
count = 0