diff --git a/app/api/files.py b/app/api/files.py index 17315ffa..e8cd14b8 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -1,15 +1,17 @@ """ File-related API endpoints """ -from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File +from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File, Query from sqlalchemy.orm import Session +from sqlalchemy import desc, asc, or_, func +from typing import Optional, List import logging import os import uuid import mimetypes from app.auth import require_login -from app.models import FileRecord +from app.models import FileRecord, ProcessingLog from app.config import settings from app.api.common import get_db from app.tasks.process_document import process_document @@ -22,29 +24,79 @@ router = APIRouter() @router.get("/files") @require_login -def list_files_api(request: Request, db: Session = Depends(get_db)): +def list_files_api( + request: Request, + db: Session = Depends(get_db), + page: int = Query(1, ge=1, description="Page number"), + per_page: int = Query(50, ge=1, le=200, description="Items per page"), + sort_by: str = Query("created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"), + sort_order: str = Query("desc", description="Sort order: asc or desc"), + search: Optional[str] = Query(None, description="Search in filename"), + mime_type: Optional[str] = Query(None, description="Filter by MIME type"), + status: Optional[str] = Query(None, description="Filter by processing status") +): """ - Returns a JSON list of all FileRecord entries. - Protected by `@require_login`, so only logged-in sessions can access. + Returns a paginated JSON list of FileRecord entries with processing status. + Supports server-side sorting, filtering, and searching. + + Query Parameters: + - page: Page number (default: 1) + - per_page: Items per page (default: 50, max: 200) + - sort_by: Field to sort by (default: created_at) + - sort_order: asc or desc (default: desc) + - search: Search in filename + - mime_type: Filter by MIME type + - status: Filter by processing status (pending, processing, completed, failed) Example response: - [ - { - "id": 123, - "filehash": "abc123...", - "original_filename": "example.pdf", - "local_filename": "/workdir/tmp/.pdf", - "file_size": 1048576, - "mime_type": "application/pdf", - "created_at": "2025-05-01T12:34:56.789000" - }, - ... - ] + { + "files": [...], + "pagination": { + "page": 1, + "per_page": 50, + "total_items": 150, + "total_pages": 3 + } + } """ - files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all() - # Return a simple list of dicts + # Start with base query + query = db.query(FileRecord) + + # Apply search filter + if search: + query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) + + # Apply MIME type filter + if mime_type: + query = query.filter(FileRecord.mime_type == mime_type) + + # Get total count before pagination + total_items = query.count() + + # Apply sorting + sort_column = { + "id": FileRecord.id, + "original_filename": FileRecord.original_filename, + "file_size": FileRecord.file_size, + "mime_type": FileRecord.mime_type, + "created_at": FileRecord.created_at + }.get(sort_by, FileRecord.created_at) + + if sort_order == "asc": + query = query.order_by(asc(sort_column)) + else: + query = query.order_by(desc(sort_column)) + + # Apply pagination + offset = (page - 1) * per_page + files = query.offset(offset).limit(per_page).all() + + # Build result with processing status result = [] for f in files: + # Get processing status for this file + processing_status = _get_file_processing_status(db, f.id) + result.append({ "id": f.id, "filehash": f.filehash, @@ -52,9 +104,131 @@ def list_files_api(request: Request, db: Session = Depends(get_db)): "local_filename": f.local_filename, "file_size": f.file_size, "mime_type": f.mime_type, - "created_at": f.created_at.isoformat() if f.created_at else None + "created_at": f.created_at.isoformat() if f.created_at else None, + "processing_status": processing_status }) - return result + + # Filter by status if requested (after computing statuses) + if status: + result = [f for f in result if f["processing_status"]["status"] == status] + total_items = len(result) # Update total for filtered results + + # Calculate pagination info + total_pages = (total_items + per_page - 1) // per_page + + return { + "files": result, + "pagination": { + "page": page, + "per_page": per_page, + "total_items": total_items, + "total_pages": total_pages + } + } + + +def _get_file_processing_status(db: Session, file_id: int) -> dict: + """ + Get the processing status for a file by checking its processing logs. + + Returns: + dict with status, last_step, and has_errors + """ + # Get all logs for this file + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id == file_id + ).order_by(ProcessingLog.timestamp.desc()).all() + + if not logs: + return { + "status": "pending", + "last_step": None, + "has_errors": False, + "total_steps": 0 + } + + # Check for failures + has_errors = any(log.status == "failure" for log in logs) + + # Check if any in progress + in_progress = any(log.status == "in_progress" for log in logs) + + # Get the latest log + latest_log = logs[0] + + # Determine overall status + if has_errors: + status = "failed" + elif in_progress: + status = "processing" + elif latest_log.status == "success": + status = "completed" + else: + status = "pending" + + return { + "status": status, + "last_step": latest_log.step_name, + "has_errors": has_errors, + "total_steps": len(logs) + } + + + +@router.get("/files/{file_id}") +@require_login +def get_file_details(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Get detailed information about a specific file including processing history. + """ + # Find the file record + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + + if not file_record: + raise HTTPException( + status_code=404, + detail=f"File record with ID {file_id} not found" + ) + + # Get processing logs + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id == file_id + ).order_by(ProcessingLog.timestamp.desc()).all() + + # Build log list + log_list = [] + for log in logs: + log_list.append({ + "id": log.id, + "task_id": log.task_id, + "step_name": log.step_name, + "status": log.status, + "message": log.message, + "timestamp": log.timestamp.isoformat() if log.timestamp else None + }) + + # Get processing status + processing_status = _get_file_processing_status(db, file_id) + + # Check if files exist on disk + files_on_disk = { + "original": os.path.exists(file_record.local_filename) if file_record.local_filename else False + } + + return { + "file": { + "id": file_record.id, + "filehash": file_record.filehash, + "original_filename": file_record.original_filename, + "local_filename": file_record.local_filename, + "file_size": file_record.file_size, + "mime_type": file_record.mime_type, + "created_at": file_record.created_at.isoformat() if file_record.created_at else None + }, + "processing_status": processing_status, + "logs": log_list, + "files_on_disk": files_on_disk + } @router.delete("/files/{file_id}") @require_login diff --git a/app/views/files.py b/app/views/files.py index 96fd7fae..dccf2110 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -1,8 +1,9 @@ """ File management views for displaying and managing files. """ -from fastapi import Request, Depends +from fastapi import Request, Depends, Query from sqlalchemy.orm import Session +from typing import Optional from app.views.base import APIRouter, templates, require_login, get_db, logger @@ -10,23 +11,113 @@ router = APIRouter() @router.get("/files") @require_login -def files_page(request: Request, db: Session = Depends(get_db)): +def files_page( + request: Request, + db: Session = Depends(get_db), + page: int = Query(1, ge=1), + per_page: int = Query(50, ge=1, le=200), + sort_by: str = Query("created_at"), + sort_order: str = Query("desc"), + search: Optional[str] = Query(None), + mime_type: Optional[str] = Query(None), + status: Optional[str] = Query(None) +): """ - Return the 'files.html' template with files from the database + Return the 'files.html' template with server-side pagination, sorting, and filtering """ try: # Import the model here to avoid circular imports - from app.models import FileRecord + from app.models import FileRecord, ProcessingLog + from sqlalchemy import desc, asc - # Fetch all files from the database - files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all() + # Start with base query + query = db.query(FileRecord) + + # Apply search filter + if search: + query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) + + # Apply MIME type filter + if mime_type: + query = query.filter(FileRecord.mime_type == mime_type) + + # Get total count before pagination + total_items = query.count() + + # Apply sorting + sort_column = { + "id": FileRecord.id, + "original_filename": FileRecord.original_filename, + "file_size": FileRecord.file_size, + "mime_type": FileRecord.mime_type, + "created_at": FileRecord.created_at + }.get(sort_by, FileRecord.created_at) + + if sort_order == "asc": + query = query.order_by(asc(sort_column)) + else: + query = query.order_by(desc(sort_column)) + + # Apply pagination + offset = (page - 1) * per_page + files = query.offset(offset).limit(per_page).all() + + # Add processing status to each file + files_with_status = [] + for file in files: + # Get processing logs for this file + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id == file.id + ).order_by(ProcessingLog.timestamp.desc()).all() + + # Determine status + if not logs: + processing_status = "pending" + else: + has_errors = any(log.status == "failure" for log in logs) + in_progress = any(log.status == "in_progress" for log in logs) + latest_log = logs[0] + + if has_errors: + processing_status = "failed" + elif in_progress: + processing_status = "processing" + elif latest_log.status == "success": + processing_status = "completed" + else: + processing_status = "pending" + + # Add status to file object as an attribute + file.processing_status = processing_status + files_with_status.append(file) + + # Calculate pagination info + total_pages = (total_items + per_page - 1) // per_page + + # Get unique MIME types for filter dropdown + mime_types = db.query(FileRecord.mime_type).distinct().filter( + FileRecord.mime_type.isnot(None) + ).all() + mime_types = [mt[0] for mt in mime_types if mt[0]] # Debug output - logger.info(f"Retrieved {len(files)} files from database") + logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})") return templates.TemplateResponse("files.html", { "request": request, - "files": files + "files": files_with_status, + "pagination": { + "page": page, + "per_page": per_page, + "total_items": total_items, + "total_pages": total_pages + }, + "sort_by": sort_by, + "sort_order": sort_order, + "search": search or "", + "mime_type": mime_type or "", + "status": status or "", + "mime_types": mime_types }) except Exception as e: # Log any errors @@ -35,5 +126,52 @@ def files_page(request: Request, db: Session = Depends(get_db)): return templates.TemplateResponse("files.html", { "request": request, "files": [], + "pagination": { + "page": 1, + "per_page": per_page, + "total_items": 0, + "total_pages": 0 + }, + "error": str(e) + }) + + +@router.get("/files/{file_id}/detail") +@require_login +def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Return the file detail page showing processing history and file information + """ + try: + from app.models import FileRecord, ProcessingLog + import os + + # Find the file record + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + + if not file_record: + return templates.TemplateResponse("file_detail.html", { + "request": request, + "error": f"File with ID {file_id} not found" + }) + + # Get processing logs + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id == file_id + ).order_by(ProcessingLog.timestamp.asc()).all() + + # Check if file exists on disk + file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False + + return templates.TemplateResponse("file_detail.html", { + "request": request, + "file": file_record, + "logs": logs, + "file_exists": file_exists + }) + except Exception as e: + logger.error(f"Error retrieving file details: {str(e)}") + return templates.TemplateResponse("file_detail.html", { + "request": request, "error": str(e) }) diff --git a/frontend/templates/file_detail.html b/frontend/templates/file_detail.html new file mode 100644 index 00000000..96153a5f --- /dev/null +++ b/frontend/templates/file_detail.html @@ -0,0 +1,310 @@ +{% extends "base.html" %} +{% block title %}File Details{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +
+ + + Back to File List + + + {% if error %} +
+

Error: {{ error }}

+
+ {% else %} + + +
+

File Information

+
+
+ File ID + {{ file.id }} +
+
+ Original Filename + {{ file.original_filename }} +
+
+ File Hash + {{ file.filehash }} +
+
+ File Size + {{ (file.file_size / 1024) | round(2) }} KB ({{ file.file_size }} bytes) +
+
+ MIME Type + {{ file.mime_type or 'N/A' }} +
+
+ Created At + {{ file.created_at.strftime('%Y-%m-%d %H:%M:%S') if file.created_at else 'N/A' }} +
+
+ Local Path + {{ file.local_filename }} +
+
+ File on Disk + + {% if file_exists %} + + File exists + + {% else %} + + File not found + + {% endif %} + +
+
+
+ + +
+

Processing History

+ {% if logs %} +
+ {% for log in logs %} +
+
+
+
+ {{ log.step_name }} - {{ log.status }} +
+ {% if log.message %} +
{{ log.message }}
+ {% endif %} +
+ {{ log.timestamp.strftime('%Y-%m-%d %H:%M:%S') if log.timestamp else 'N/A' }} + {% if log.task_id %} + Task: {{ log.task_id[:16] }}... + {% endif %} +
+
+
+ {% endfor %} +
+ {% else %} +
+ +

No processing logs found for this file.

+

The file may not have been processed yet.

+
+ {% endif %} +
+ + {% endif %} +
+{% endblock %} diff --git a/frontend/templates/files.html b/frontend/templates/files.html index c421fcc3..1174cc24 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -3,10 +3,6 @@ {% block head_extra %} - - - - {% endblock %} @@ -182,34 +251,130 @@ {% endif %} + +
+
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ + + + + +
+
- - - - - - + + + + + + + {% for file in files %} - + - + - + + {% else %} - + {% endfor %}
IDOriginal FilenameFile SizeMime TypeCreated AtActions + ID + + {% if sort_by == 'id' %} + {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} + {% else %} + ⬍ + {% endif %} + + + Original Filename + + {% if sort_by == 'original_filename' %} + {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} + {% else %} + ⬍ + {% endif %} + + + File Size + + {% if sort_by == 'file_size' %} + {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} + {% else %} + ⬍ + {% endif %} + + + MIME Type + + {% if sort_by == 'mime_type' %} + {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} + {% else %} + ⬍ + {% endif %} + + Status + Created At + + {% if sort_by == 'created_at' %} + {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} + {% else %} + ⬍ + {% endif %} + + Actions
{{ file.id }} {{ file.original_filename }}{{ (file.file_size / 1024) | round(2) }} KB{{ (file.file_size / 1024) | round(2) }} KB {{ file.mime_type }}{{ file.created_at }} + + {{ file.processing_status | title }} + + {{ file.created_at.strftime('%Y-%m-%d %H:%M:%S') if file.created_at else 'N/A' }}
- -
@@ -217,13 +382,41 @@
No files foundNo files found
+ + {% if pagination.total_pages > 1 %} + + {% endif %} + - - - - {% endblock %}