gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply code formatting and fix linting issues

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-07 17:37:50 +00:00
parent 2db58afedf
commit 7d24b7c668
4 changed files with 288 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+163 -192
View File
@@ -1,28 +1,31 @@
"""
File-related API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File, Query
from sqlalchemy.orm import Session
from sqlalchemy import desc, asc, or_, func
from typing import Optional, List
import logging
import mimetypes
import os
import uuid
import mimetypes
from typing import List, Optional
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
from sqlalchemy import asc, desc, or_
from sqlalchemy.orm import Session
from app.auth import require_login
from app.models import FileRecord, ProcessingLog
from app.config import settings
from app.api.common import get_db
from app.tasks.process_document import process_document
from app.auth import require_login
from app.config import settings
from app.models import FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf
from app.utils.file_status import get_file_processing_status, get_files_processing_status
from app.tasks.process_document import process_document
from app.utils.file_status import get_files_processing_status
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/files")
@require_login
def list_files_api(
@@ -30,16 +33,18 @@ def list_files_api(
db: Session = Depends(get_db),
page: int = Query(1, ge=1, description="Page number"),
per_page: int = Query(50, ge=1, le=200, description="Items per page"),
sort_by: str = Query("created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"),
sort_by: str = Query(
"created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"
),
sort_order: str = Query("desc", description="Sort order: asc or desc"),
search: Optional[str] = Query(None, description="Search in filename"),
mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
status: Optional[str] = Query(None, description="Filter by processing status")
status: Optional[str] = Query(None, description="Filter by processing status"),
):
"""
Returns a paginated JSON list of FileRecord entries with processing status.
Supports server-side sorting, filtering, and searching.
Query Parameters:
- page: Page number (default: 1)
- per_page: Items per page (default: 50, max: 200)
@@ -48,7 +53,7 @@ def list_files_api(
- search: Search in filename
- mime_type: Filter by MIME type
- status: Filter by processing status (pending, processing, completed, failed)
Example response:
{
"files": [...],
@@ -62,15 +67,15 @@ def list_files_api(
"""
# Start with base query
query = db.query(FileRecord)
# Apply search filter
if search:
query = query.filter(FileRecord.original_filename.ilike(f"%{search}%"))
# Apply MIME type filter
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Apply status filter (before pagination for correct counts)
if status:
# Subquery to get file IDs matching the status
@@ -80,87 +85,78 @@ def list_files_api(
query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing":
# Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "in_progress"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "failed":
# Files with failure logs
subq = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "failure"
).distinct()
subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
query = query.filter(FileRecord.id.in_(subq))
elif status == "completed":
# Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter(
ProcessingLog.status == "success"
).distinct().subquery()
failed_files = db.query(ProcessingLog.file_id).filter(
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
).distinct().subquery()
query = query.filter(
FileRecord.id.in_(db.query(success_files.c.file_id))
).filter(
success_files = (
db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
)
failed_files = (
db.query(ProcessingLog.file_id)
.filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
.distinct()
.subquery()
)
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id))
)
# Get total count before pagination (after all filters)
total_items = query.count()
# Apply sorting
sort_column = {
"id": FileRecord.id,
"original_filename": FileRecord.original_filename,
"file_size": FileRecord.file_size,
"mime_type": FileRecord.mime_type,
"created_at": FileRecord.created_at
"created_at": FileRecord.created_at,
}.get(sort_by, FileRecord.created_at)
if sort_order == "asc":
query = query.order_by(asc(sort_column))
else:
query = query.order_by(desc(sort_column))
# Apply pagination
offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all()
# Get processing status for all files efficiently
file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids)
# Build result with processing status
result = []
for f in files:
result.append({
"id": f.id,
"filehash": f.filehash,
"original_filename": f.original_filename,
"local_filename": f.local_filename,
"file_size": f.file_size,
"mime_type": f.mime_type,
"created_at": f.created_at.isoformat() if f.created_at else None,
"processing_status": statuses.get(f.id, {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
})
})
result.append(
{
"id": f.id,
"filehash": f.filehash,
"original_filename": f.original_filename,
"local_filename": f.local_filename,
"file_size": f.file_size,
"mime_type": f.mime_type,
"created_at": f.created_at.isoformat() if f.created_at else None,
"processing_status": statuses.get(
f.id, {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
),
}
)
# Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page
return {
"files": result,
"pagination": {
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages
}
"pagination": {"page": page, "per_page": per_page, "total_items": total_items, "total_pages": total_pages},
}
@@ -170,8 +166,8 @@ def _get_file_processing_status(db: Session, file_id: int) -> dict:
Kept for backward compatibility with file detail endpoint.
"""
from app.utils.file_status import get_file_processing_status
return get_file_processing_status(db, file_id)
return get_file_processing_status(db, file_id)
@router.get("/files/{file_id}")
@@ -182,38 +178,35 @@ def get_file_details(request: Request, file_id: int, db: Session = Depends(get_d
"""
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(
status_code=404,
detail=f"File record with ID {file_id} not found"
)
raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
# Get processing logs
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.desc()).all()
logs = (
db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
)
# Build log list
log_list = []
for log in logs:
log_list.append({
"id": log.id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None
})
log_list.append(
{
"id": log.id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
}
)
# Get processing status
processing_status = _get_file_processing_status(db, file_id)
# Check if files exist on disk
files_on_disk = {
"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False
}
files_on_disk = {"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False}
return {
"file": {
"id": file_record.id,
@@ -222,13 +215,14 @@ def get_file_details(request: Request, file_id: int, db: Session = Depends(get_d
"local_filename": file_record.local_filename,
"file_size": file_record.file_size,
"mime_type": file_record.mime_type,
"created_at": file_record.created_at.isoformat() if file_record.created_at else None
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
},
"processing_status": processing_status,
"logs": log_list,
"files_on_disk": files_on_disk
"files_on_disk": files_on_disk,
}
@router.delete("/files/{file_id}")
@require_login
def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)):
@@ -238,42 +232,31 @@ def delete_file_record(request: Request, file_id: int, db: Session = Depends(get
"""
# Check if file deletion is allowed
if not settings.allow_file_delete:
raise HTTPException(
status_code=403,
detail="File deletion is disabled in the configuration"
)
raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(
status_code=404,
detail=f"File record with ID {file_id} not found"
)
raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
# Log the deletion
logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
# Delete the record
db.delete(file_record)
db.commit()
return {
"status": "success",
"message": f"File record {file_id} deleted successfully"
}
return {"status": "success", "message": f"File record {file_id} deleted successfully"}
except HTTPException:
raise
except Exception as e:
db.rollback()
logger.exception(f"Error deleting file record {file_id}: {str(e)}")
raise HTTPException(
status_code=500,
detail=f"Error deleting file record: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}")
@router.post("/files/bulk-delete")
@require_login
@@ -284,48 +267,39 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depen
"""
# Check if file deletion is allowed
if not settings.allow_file_delete:
raise HTTPException(
status_code=403,
detail="File deletion is disabled in the configuration"
)
raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(
status_code=404,
detail="No files found with the provided IDs"
)
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
deleted_count = len(file_records)
deleted_ids = [f.id for f in file_records]
# Log the deletion
logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}")
# Delete all records
for file_record in file_records:
db.delete(file_record)
db.commit()
return {
"status": "success",
"message": f"Successfully deleted {deleted_count} file records",
"deleted_ids": deleted_ids
"deleted_ids": deleted_ids,
}
except HTTPException:
raise
except Exception as e:
db.rollback()
logger.exception(f"Error bulk deleting file records: {str(e)}")
raise HTTPException(
status_code=500,
detail=f"Error bulk deleting file records: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}")
@router.post("/files/bulk-reprocess")
@@ -337,63 +311,56 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records:
raise HTTPException(
status_code=404,
detail="No files found with the provided IDs"
)
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
task_ids = []
processed_files = []
errors = []
for file_record in file_records:
try:
# Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
errors.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": "Local file not found"
})
errors.append(
{
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": "Local file not found",
}
)
continue
# Queue the file for processing
task = process_document.delay(file_record.local_filename)
task_ids.append(task.id)
processed_files.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"task_id": task.id
})
logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}")
processed_files.append(
{"file_id": file_record.id, "filename": file_record.original_filename, "task_id": task.id}
)
logger.info(
f"Reprocessing file: ID={file_record.id}, "
f"Filename={file_record.original_filename}, TaskID={task.id}"
)
except Exception as e:
logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
errors.append({
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": str(e)
})
errors.append({"file_id": file_record.id, "filename": file_record.original_filename, "error": str(e)})
return {
"status": "success" if processed_files else "error",
"message": f"Successfully queued {len(processed_files)} files for reprocessing",
"processed_files": processed_files,
"errors": errors if errors else None,
"task_ids": task_ids
"task_ids": task_ids,
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error bulk reprocessing files: {str(e)}")
raise HTTPException(
status_code=500,
detail=f"Error bulk reprocessing files: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
@router.post("/ui-upload")
@@ -401,10 +368,10 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
async def ui_upload(request: Request, file: UploadFile = File(...)):
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = settings.workdir
# Extract just the filename without any path components to prevent path traversal
safe_filename = os.path.basename(file.filename)
# Generate a unique filename with UUID to prevent overwriting and filename conflicts
unique_id = str(uuid.uuid4())
# Keep the original extension if present
@@ -413,34 +380,28 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
target_filename = f"{unique_id}.{file_extension}"
else:
target_filename = unique_id
# Store both the safe original name and the unique name
target_path = os.path.join(workdir, target_filename)
try:
with open(target_path, "wb") as f:
content = await file.read()
f.write(content)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to save file: {e}"
)
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
# Log the mapping between original and safe filename
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
# Check file size
file_size = os.path.getsize(target_path)
max_size = 500 * 1024 * 1024 # 500MB
if file_size > max_size:
# Remove the file if it's too large
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
)
raise HTTPException(status_code=413, detail=f"File too large: {file_size} bytes (max {max_size} bytes)")
# Same set of allowed file types as in the IMAP task
ALLOWED_MIME_TYPES = {
"application/pdf",
@@ -455,30 +416,40 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
"application/rtf",
"text/rtf",
}
# Image MIME types that need conversion
IMAGE_MIME_TYPES = {
'image/jpeg', 'image/jpg', 'image/png',
'image/gif', 'image/bmp', 'image/tiff',
'image/webp', 'image/svg+xml'
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
# Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower()
# Check if it's a PDF by extension or MIME type
is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
if is_pdf:
# If it's a PDF, process directly
task = process_document.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued PDF for processing: {target_path}")
elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']):
elif mime_type in IMAGE_MIME_TYPES or any(
file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
):
# If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued image for PDF conversion: {target_path}")
elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']):
elif mime_type in ALLOWED_MIME_TYPES or any(
file_ext.endswith(ext)
for ext in [".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".rtf", ".txt", ".csv"]
):
# If it's an office document, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued office document for PDF conversion: {target_path}")
@@ -486,10 +457,10 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# For any other file type, attempt conversion but log a warning
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
return {
"task_id": task.id,
"status": "queued",
"task_id": task.id,
"status": "queued",
"original_filename": safe_filename,
"stored_filename": target_filename
"stored_filename": target_filename,
}