gh-christianlouis-docuelevate/app/api/files.py

"""
File-related API endpoints
"""

import logging
import mimetypes
import os
import uuid
from typing import List, Optional

from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
from sqlalchemy import asc, desc, or_
from sqlalchemy.orm import Session

from app.api.common import get_db
from app.auth import require_login
from app.config import settings
from app.models import FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.process_document import process_document
from app.utils.file_status import get_files_processing_status

# Set up logging
logger = logging.getLogger(__name__)

router = APIRouter()


@router.get("/files")
@require_login
def list_files_api(
    request: Request,
    db: Session = Depends(get_db),
    page: int = Query(1, ge=1, description="Page number"),
    per_page: int = Query(50, ge=1, le=200, description="Items per page"),
    sort_by: str = Query(
        "created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"
    ),
    sort_order: str = Query("desc", description="Sort order: asc or desc"),
    search: Optional[str] = Query(None, description="Search in filename"),
    mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
    status: Optional[str] = Query(None, description="Filter by processing status"),
):
    """
    Returns a paginated JSON list of FileRecord entries with processing status.
    Supports server-side sorting, filtering, and searching.

    Query Parameters:
    - page: Page number (default: 1)
    - per_page: Items per page (default: 50, max: 200)
    - sort_by: Field to sort by (default: created_at)
    - sort_order: asc or desc (default: desc)
    - search: Search in filename
    - mime_type: Filter by MIME type
    - status: Filter by processing status (pending, processing, completed, failed)

    Example response:
    {
      "files": [...],
      "pagination": {
        "page": 1,
        "per_page": 50,
        "total_items": 150,
        "total_pages": 3
      }
    }
    """
    # Start with base query
    query = db.query(FileRecord)

    # Apply search filter
    if search:
        query = query.filter(FileRecord.original_filename.ilike(f"%{search}%"))

    # Apply MIME type filter
    if mime_type:
        query = query.filter(FileRecord.mime_type == mime_type)

    # Apply status filter (before pagination for correct counts)
    if status:
        # Subquery to get file IDs matching the status
        if status == "pending":
            # Files with no logs
            subq = db.query(ProcessingLog.file_id).distinct()
            query = query.filter(~FileRecord.id.in_(subq))
        elif status == "processing":
            # Files with in_progress logs
            subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
            query = query.filter(FileRecord.id.in_(subq))
        elif status == "failed":
            # Files with failure logs
            subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
            query = query.filter(FileRecord.id.in_(subq))
        elif status == "completed":
            # Files with success logs but no failures or in_progress
            success_files = (
                db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
            )

            failed_files = (
                db.query(ProcessingLog.file_id)
                .filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
                .distinct()
                .subquery()
            )

            query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
                ~FileRecord.id.in_(db.query(failed_files.c.file_id))
            )

    # Get total count before pagination (after all filters)
    total_items = query.count()

    # Apply sorting
    sort_column = {
        "id": FileRecord.id,
        "original_filename": FileRecord.original_filename,
        "file_size": FileRecord.file_size,
        "mime_type": FileRecord.mime_type,
        "created_at": FileRecord.created_at,
    }.get(sort_by, FileRecord.created_at)

    if sort_order == "asc":
        query = query.order_by(asc(sort_column))
    else:
        query = query.order_by(desc(sort_column))

    # Apply pagination
    offset = (page - 1) * per_page
    files = query.offset(offset).limit(per_page).all()

    # Get processing status for all files efficiently
    file_ids = [f.id for f in files]
    statuses = get_files_processing_status(db, file_ids)

    # Build result with processing status
    result = []
    for f in files:
        result.append(
            {
                "id": f.id,
                "filehash": f.filehash,
                "original_filename": f.original_filename,
                "local_filename": f.local_filename,
                "file_size": f.file_size,
                "mime_type": f.mime_type,
                "created_at": f.created_at.isoformat() if f.created_at else None,
                "processing_status": statuses.get(
                    f.id, {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
                ),
            }
        )

    # Calculate pagination info
    total_pages = (total_items + per_page - 1) // per_page

    return {
        "files": result,
        "pagination": {"page": page, "per_page": per_page, "total_items": total_items, "total_pages": total_pages},
    }


def _get_file_processing_status(db: Session, file_id: int) -> dict:
    """
    Deprecated: Use app.utils.file_status.get_file_processing_status instead.
    Kept for backward compatibility with file detail endpoint.
    """
    from app.utils.file_status import get_file_processing_status

    return get_file_processing_status(db, file_id)


@router.get("/files/{file_id}")
@require_login
def get_file_details(request: Request, file_id: int, db: Session = Depends(get_db)):
    """
    Get detailed information about a specific file including processing history.
    """
    # Find the file record
    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

    if not file_record:
        raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")

    # Get processing logs
    logs = (
        db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
    )

    # Build log list
    log_list = []
    for log in logs:
        log_list.append(
            {
                "id": log.id,
                "task_id": log.task_id,
                "step_name": log.step_name,
                "status": log.status,
                "message": log.message,
                "timestamp": log.timestamp.isoformat() if log.timestamp else None,
            }
        )

    # Get processing status
    processing_status = _get_file_processing_status(db, file_id)

    # Check if files exist on disk
    files_on_disk = {"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False}

    return {
        "file": {
            "id": file_record.id,
            "filehash": file_record.filehash,
            "original_filename": file_record.original_filename,
            "local_filename": file_record.local_filename,
            "file_size": file_record.file_size,
            "mime_type": file_record.mime_type,
            "created_at": file_record.created_at.isoformat() if file_record.created_at else None,
        },
        "processing_status": processing_status,
        "logs": log_list,
        "files_on_disk": files_on_disk,
    }


@router.delete("/files/{file_id}")
@require_login
def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)):
    """
    Delete a file record from the database.
    This only removes the database entry, not the actual file.
    """
    # Check if file deletion is allowed
    if not settings.allow_file_delete:
        raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")

    try:
        # Find the file record
        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

        if not file_record:
            raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")

        # Log the deletion
        logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")

        # Delete the record
        db.delete(file_record)
        db.commit()

        return {"status": "success", "message": f"File record {file_id} deleted successfully"}

    except HTTPException:
        raise
    except Exception as e:
        db.rollback()
        logger.exception(f"Error deleting file record {file_id}: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}")


@router.post("/files/bulk-delete")
@require_login
def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
    """
    Delete multiple file records from the database.
    This only removes the database entries, not the actual files.
    """
    # Check if file deletion is allowed
    if not settings.allow_file_delete:
        raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")

    try:
        # Find all file records
        file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()

        if not file_records:
            raise HTTPException(status_code=404, detail="No files found with the provided IDs")

        deleted_count = len(file_records)
        deleted_ids = [f.id for f in file_records]

        # Log the deletion
        logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}")

        # Delete all records
        for file_record in file_records:
            db.delete(file_record)

        db.commit()

        return {
            "status": "success",
            "message": f"Successfully deleted {deleted_count} file records",
            "deleted_ids": deleted_ids,
        }

    except HTTPException:
        raise
    except Exception as e:
        db.rollback()
        logger.exception(f"Error bulk deleting file records: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}")


@router.post("/files/bulk-reprocess")
@require_login
def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)):
    """
    Reprocess multiple files by queuing them for processing.
    """
    try:
        # Find all file records
        file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()

        if not file_records:
            raise HTTPException(status_code=404, detail="No files found with the provided IDs")

        task_ids = []
        processed_files = []
        errors = []

        for file_record in file_records:
            try:
                # Check if local file exists
                if not file_record.local_filename or not os.path.exists(file_record.local_filename):
                    errors.append(
                        {
                            "file_id": file_record.id,
                            "filename": file_record.original_filename,
                            "error": "Local file not found",
                        }
                    )
                    continue

                # Queue the file for processing
                task = process_document.delay(file_record.local_filename)
                task_ids.append(task.id)
                processed_files.append(
                    {"file_id": file_record.id, "filename": file_record.original_filename, "task_id": task.id}
                )

                logger.info(
                    f"Reprocessing file: ID={file_record.id}, "
                    f"Filename={file_record.original_filename}, TaskID={task.id}"
                )

            except Exception as e:
                logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
                errors.append({"file_id": file_record.id, "filename": file_record.original_filename, "error": str(e)})

        return {
            "status": "success" if processed_files else "error",
            "message": f"Successfully queued {len(processed_files)} files for reprocessing",
            "processed_files": processed_files,
            "errors": errors if errors else None,
            "task_ids": task_ids,
        }

    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Error bulk reprocessing files: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")


@router.post("/files/{file_id}/reprocess")
@require_login
def reprocess_single_file(request: Request, file_id: int, db: Session = Depends(get_db)):
    """
    Reprocess a single file by queuing it for processing again.

    Args:
        file_id: ID of the file to reprocess

    Returns:
        Task ID and status information
    """
    try:
        # Find the file record
        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

        if not file_record:
            raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")

        # Check if local file exists
        if not file_record.local_filename or not os.path.exists(file_record.local_filename):
            raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")

        # Queue the file for processing
        task = process_document.delay(file_record.local_filename, original_filename=file_record.original_filename)

        logger.info(
            f"Reprocessing file: ID={file_record.id}, " f"Filename={file_record.original_filename}, TaskID={task.id}"
        )

        return {
            "status": "success",
            "message": "File queued for reprocessing",
            "file_id": file_record.id,
            "filename": file_record.original_filename,
            "task_id": task.id,
        }

    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Error reprocessing file {file_id}: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error reprocessing file: {str(e)}")


@router.post("/files/{file_id}/retry-subtask")
@require_login
def retry_subtask(
    request: Request,
    file_id: int,
    subtask_name: str = Query(..., description="Name of the upload subtask to retry (e.g., 'upload_to_dropbox')"),
    db: Session = Depends(get_db),
):
    """
    Retry a specific failed upload subtask for a file.

    Args:
        file_id: ID of the file
        subtask_name: Name of the upload task (e.g., upload_to_dropbox, upload_to_s3)

    Returns:
        Task ID and status information
    """
    try:
        # Find the file record
        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

        if not file_record:
            raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")

        # Map subtask names to their corresponding Celery tasks
        from app.tasks.upload_to_dropbox import upload_to_dropbox
        from app.tasks.upload_to_nextcloud import upload_to_nextcloud
        from app.tasks.upload_to_paperless import upload_to_paperless
        from app.tasks.upload_to_google_drive import upload_to_google_drive
        from app.tasks.upload_to_onedrive import upload_to_onedrive
        from app.tasks.upload_to_s3 import upload_to_s3
        from app.tasks.upload_to_webdav import upload_to_webdav
        from app.tasks.upload_to_ftp import upload_to_ftp
        from app.tasks.upload_to_sftp import upload_to_sftp
        from app.tasks.upload_to_email import upload_to_email

        task_map = {
            "upload_to_dropbox": upload_to_dropbox,
            "upload_to_nextcloud": upload_to_nextcloud,
            "upload_to_paperless": upload_to_paperless,
            "upload_to_google_drive": upload_to_google_drive,
            "upload_to_onedrive": upload_to_onedrive,
            "upload_to_s3": upload_to_s3,
            "upload_to_webdav": upload_to_webdav,
            "upload_to_ftp": upload_to_ftp,
            "upload_to_sftp": upload_to_sftp,
            "upload_to_email": upload_to_email,
        }

        if subtask_name not in task_map:
            raise HTTPException(
                status_code=400,
                detail=f"Invalid subtask name: {subtask_name}. Must be one of: {', '.join(task_map.keys())}",
            )

        # Check for processed file (upload tasks work with processed files)
        workdir = settings.workdir
        processed_dir = os.path.join(workdir, "processed")

        # Try to find the processed file
        base_filename = os.path.splitext(file_record.original_filename)[0]
        potential_paths = [
            os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
            os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
            os.path.join(processed_dir, file_record.original_filename),
        ]

        file_path = None
        for path in potential_paths:
            if os.path.exists(path):
                file_path = path
                break

        if not file_path:
            raise HTTPException(status_code=400, detail="Processed file not found. Cannot retry upload.")

        # Queue the specific upload task
        upload_task = task_map[subtask_name]
        task = upload_task.delay(file_path, file_id)

        logger.info(f"Retrying upload subtask: FileID={file_record.id}, " f"Subtask={subtask_name}, TaskID={task.id}")

        return {
            "status": "success",
            "message": f"Upload task {subtask_name} queued for retry",
            "file_id": file_record.id,
            "subtask_name": subtask_name,
            "task_id": task.id,
        }

    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error retrying subtask: {str(e)}")


@router.get("/files/{file_id}/preview")
@require_login
def get_file_preview(
    request: Request,
    file_id: int,
    version: str = Query("original", description="original or processed"),
    db: Session = Depends(get_db),
):
    """
    Get file content for preview (original or processed version).

    Args:
        file_id: ID of the file
        version: "original" for tmp file, "processed" for processed file

    Returns:
        File content for preview
    """
    from fastapi.responses import FileResponse

    try:
        # Find the file record
        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

        if not file_record:
            raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")

        if version == "original":
            # Return the original file from tmp
            if not file_record.local_filename or not os.path.exists(file_record.local_filename):
                raise HTTPException(status_code=404, detail="Original file not found on disk")

            file_path = file_record.local_filename

        elif version == "processed":
            # Look for processed file in /workdir/processed/
            workdir = settings.workdir
            processed_dir = os.path.join(workdir, "processed")

            # Try to find the processed file (same hash or UUID-based naming)
            base_filename = os.path.splitext(file_record.original_filename)[0]
            potential_paths = [
                os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
                os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
                os.path.join(processed_dir, file_record.original_filename),
            ]

            file_path = None
            for path in potential_paths:
                if os.path.exists(path):
                    file_path = path
                    break

            if not file_path:
                raise HTTPException(status_code=404, detail="Processed file not found")
        else:
            raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")

        # Return the file
        return FileResponse(
            path=file_path,
            media_type=file_record.mime_type or "application/pdf",
            headers={"Content-Disposition": f'inline; filename="{file_record.original_filename}"'},
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Error retrieving file preview: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error retrieving file preview: {str(e)}")


@router.get("/files/{file_id}/download")
@require_login
def download_file(
    request: Request,
    file_id: int,
    version: str = Query("original", description="original or processed"),
    db: Session = Depends(get_db),
):
    """
    Download file (original or processed version) as attachment.

    Args:
        file_id: ID of the file
        version: "original" for tmp file, "processed" for processed file

    Returns:
        File content as attachment download
    """
    from fastapi.responses import FileResponse

    try:
        # Find the file record
        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()

        if not file_record:
            raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")

        if version == "original":
            # Return the original file from tmp
            if not file_record.local_filename or not os.path.exists(file_record.local_filename):
                raise HTTPException(status_code=404, detail="Original file not found on disk")

            file_path = file_record.local_filename

        elif version == "processed":
            # Look for processed file in /workdir/processed/
            workdir = settings.workdir
            processed_dir = os.path.join(workdir, "processed")

            # Try to find the processed file (same hash or UUID-based naming)
            base_filename = os.path.splitext(file_record.original_filename)[0]
            potential_paths = [
                os.path.join(processed_dir, f"{file_record.filehash}.pdf"),
                os.path.join(processed_dir, f"{base_filename}_processed.pdf"),
                os.path.join(processed_dir, file_record.original_filename),
            ]

            file_path = None
            for path in potential_paths:
                if os.path.exists(path):
                    file_path = path
                    break

            if not file_path:
                raise HTTPException(status_code=404, detail="Processed file not found")
        else:
            raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")

        # Return the file with attachment disposition to trigger download
        return FileResponse(
            path=file_path,
            media_type=file_record.mime_type or "application/pdf",
            headers={"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'},
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Error downloading file: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")


@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, file: UploadFile = File(...)):
    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
    workdir = settings.workdir

    # Extract just the filename without any path components to prevent path traversal
    safe_filename = os.path.basename(file.filename)

    # Generate a unique filename with UUID to prevent overwriting and filename conflicts
    unique_id = str(uuid.uuid4())
    # Keep the original extension if present
    if "." in safe_filename:
        file_extension = safe_filename.rsplit(".", 1)[1]
        target_filename = f"{unique_id}.{file_extension}"
    else:
        target_filename = unique_id

    # Store both the safe original name and the unique name
    target_path = os.path.join(workdir, target_filename)

    try:
        with open(target_path, "wb") as f:
            content = await file.read()
            f.write(content)
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")

    # Log the mapping between original and safe filename
    logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")

    # Check file size
    file_size = os.path.getsize(target_path)
    max_size = 500 * 1024 * 1024  # 500MB
    if file_size > max_size:
        # Remove the file if it's too large
        os.remove(target_path)
        raise HTTPException(status_code=413, detail=f"File too large: {file_size} bytes (max {max_size} bytes)")

    # Same set of allowed file types as in the IMAP task
    ALLOWED_MIME_TYPES = {
        "application/pdf",
        "application/msword",
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        "application/vnd.ms-excel",
        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        "application/vnd.ms-powerpoint",
        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
        "text/plain",
        "text/csv",
        "application/rtf",
        "text/rtf",
    }

    # Image MIME types that need conversion
    IMAGE_MIME_TYPES = {
        "image/jpeg",
        "image/jpg",
        "image/png",
        "image/gif",
        "image/bmp",
        "image/tiff",
        "image/webp",
        "image/svg+xml",
    }

    # Determine if the file is a PDF or needs conversion
    mime_type, _ = mimetypes.guess_type(target_path)
    file_ext = os.path.splitext(target_path)[1].lower()

    # Check if it's a PDF by extension or MIME type
    is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"

    if is_pdf:
        # If it's a PDF, process directly
        task = process_document.delay(target_path, original_filename=safe_filename)
        logger.info(f"Enqueued PDF for processing: {target_path}")
    elif mime_type in IMAGE_MIME_TYPES or any(
        file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
    ):
        # If it's an image, convert to PDF first
        task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
        logger.info(f"Enqueued image for PDF conversion: {target_path}")
    elif mime_type in ALLOWED_MIME_TYPES or any(
        file_ext.endswith(ext)
        for ext in [".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".rtf", ".txt", ".csv"]
    ):
        # If it's an office document, convert to PDF first
        task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
        logger.info(f"Enqueued office document for PDF conversion: {target_path}")
    else:
        # For any other file type, attempt conversion but log a warning
        logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
        task = convert_to_pdf.delay(target_path, original_filename=safe_filename)

    return {
        "task_id": task.id,
        "status": "queued",
        "original_filename": safe_filename,
        "stored_filename": target_filename,
    }