Merge pull request #50 from christianlouis/48-feature-request-support-multi-file-uploads-file-size-type-validation

feat: implement file size check and MIME type handling for document u…

app/api/files.py

@@ -6,12 +6,14 @@ from sqlalchemy.orm import Session
 import logging
 import os
 import uuid
+import mimetypes
 
 from app.auth import require_login
 from app.models import FileRecord
 from app.config import settings
 from app.api.common import get_db
 from app.tasks.process_document import process_document
+from app.tasks.convert_to_pdf import convert_to_pdf
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -134,7 +136,63 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
     # Log the mapping between original and safe filename
     logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
     
-    task = process_document.delay(target_path)
+    # Check file size
+    file_size = os.path.getsize(target_path)
+    max_size = 500 * 1024 * 1024  # 500MB
+    if file_size > max_size:
+        # Remove the file if it's too large
+        os.remove(target_path)
+        raise HTTPException(
+            status_code=413,
+            detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
+        )
+    
+    # Same set of allowed file types as in the IMAP task
+    ALLOWED_MIME_TYPES = {
+        "application/pdf",
+        "application/msword",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.ms-excel",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.ms-powerpoint",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "text/plain",
+        "text/csv",
+        "application/rtf",
+        "text/rtf",
+    }
+    
+    # Image MIME types that need conversion
+    IMAGE_MIME_TYPES = {
+        'image/jpeg', 'image/jpg', 'image/png', 
+        'image/gif', 'image/bmp', 'image/tiff',
+        'image/webp', 'image/svg+xml'
+    }
+    
+    # Determine if the file is a PDF or needs conversion
+    mime_type, _ = mimetypes.guess_type(target_path)
+    file_ext = os.path.splitext(target_path)[1].lower()
+    
+    # Check if it's a PDF by extension or MIME type
+    is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
+    
+    if is_pdf:
+        # If it's a PDF, process directly
+        task = process_document.delay(target_path)
+        logger.info(f"Enqueued PDF for processing: {target_path}")
+    elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']):
+        # If it's an image, convert to PDF first
+        task = convert_to_pdf.delay(target_path)
+        logger.info(f"Enqueued image for PDF conversion: {target_path}")
+    elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']):
+        # If it's an office document, convert to PDF first
+        task = convert_to_pdf.delay(target_path)
+        logger.info(f"Enqueued office document for PDF conversion: {target_path}")
+    else:
+        # For any other file type, attempt conversion but log a warning
+        logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
+        task = convert_to_pdf.delay(target_path)
+    
     return {
         "task_id": task.id, 
         "status": "queued", 

app/tasks/convert_to_pdf.py

@@ -3,9 +3,10 @@ import os
 import requests
 import logging
 import mimetypes
+import json
 from celery import shared_task
 from app.config import settings
-from app.tasks.process_document import process_document  # Updated import
+from app.tasks.process_document import process_document
 
 logger = logging.getLogger(__name__)
 
@@ -14,59 +15,165 @@ def convert_to_pdf(file_path):
     """
     Converts a file to PDF using Gotenberg's API.
     Determines the appropriate Gotenberg endpoint based on the file's MIME type.
-    On success, saves the PDF locally and enqueues it for S3 upload.
+    On success, saves the PDF locally and enqueues it for processing.
     """
     gotenberg_url = getattr(settings, "gotenberg_url", None)
     if not gotenberg_url:
         logger.error("Gotenberg URL is not configured in settings.")
         return
 
-    # Try to guess the MIME type based on file content (using extension-based fallback)
+    # Try to guess the MIME type based on file content and extension
     mime_type, encoding = mimetypes.guess_type(file_path)
-    logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
+    file_ext = os.path.splitext(file_path)[1].lower()
+    logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
 
+    # Determine which Gotenberg endpoint to use
     endpoint = None
-    form_key = "files"  # Default form key for most endpoints
+    form_data = {}
-
+    files = {}
-    if mime_type:
+    
-        if mime_type == "text/html":
+    # Dictionary mapping file extensions to their handlers
-            endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+    OFFICE_EXTENSIONS = {
-            # The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
+        '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',  # Word
-            form_key = "index.html"
+        '.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw',  # Excel
-        elif mime_type.startswith("image/"):
+        '.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx',  # PowerPoint
-            # For images, we use the LibreOffice endpoint (which supports image conversion)
+        '.odt', '.ods', '.odp', '.odg', '.odf',  # OpenOffice/LibreOffice
-            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        '.rtf', '.txt', '.csv',  # Text formats
-        elif mime_type.startswith("text/plain"):
+        '.pdf',  # PDF (already in PDF format but can be processed)
-            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+    }
-        elif mime_type in ["text/markdown", "text/x-markdown"]:
+    
-            # Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
+    IMAGE_EXTENSIONS = {
-            # For now, we'll fallback to LibreOffice.
+        '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg'
-            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+    }
-        else:
+    
-            # For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
+    HTML_EXTENSIONS = {
-            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        '.html', '.htm'
+    }
+    
+    # Use LibreOffice endpoint for office documents and images
+    if (mime_type and 'office' in mime_type) or \
+       (mime_type and 'opendocument' in mime_type) or \
+       (mime_type and mime_type.startswith('image/')) or \
+       file_ext in OFFICE_EXTENSIONS or \
+       file_ext in IMAGE_EXTENSIONS:
+        endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
+        
+        # Add some quality settings for better PDF output
+        form_data = {
+            'landscape': 'false',
+            'exportBookmarks': 'true',
+            'exportNotes': 'false',
+            'losslessImageCompression': 'true',  # Use lossless compression for images
+            'pdfa': 'PDF/A-2b',  # Produce PDF/A-2b compatible output
+        }
+    
+    # Use Chromium endpoint for HTML documents
+    elif (mime_type and mime_type == 'text/html') or file_ext in HTML_EXTENSIONS:
+        endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+        # Gotenberg requires the form field to be exactly 'index.html'
+        # The content filename doesn't matter, just the form field key
+        files = {'index.html': ('index.html', open(file_path, 'rb'))}
+        
+        # Add options for better HTML to PDF conversion
+        form_data = {
+            'paperWidth': '8.27',  # A4 width in inches
+            'paperHeight': '11.7',  # A4 height in inches
+            'marginTop': '0.4',
+            'marginBottom': '0.4',
+            'marginLeft': '0.4',
+            'marginRight': '0.4',
+            'printBackground': 'true',
+            'preferCssPageSize': 'false',
+            'waitDelay': '2s',  # Wait for JavaScript to execute
+        }
+    
+    # Use Markdown route for markdown files
+    elif (mime_type and mime_type in ['text/markdown', 'text/x-markdown']) or file_ext in ['.md', '.markdown']:
+        # For Markdown, we need both the markdown file and an HTML wrapper
+        endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
+        
+        # Create a simple HTML wrapper for the markdown
+        # IMPORTANT: The filename in the template must match the key used in the files dictionary
+        markdown_filename = os.path.basename(file_path)
+        html_wrapper = f"""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Converted Markdown</title>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            line-height: 1.6;
+            margin: 2em;
+            max-width: 50em;
+        }}
+    </style>
+</head>
+<body>
+    {{{{ toHTML "{markdown_filename}" }}}}
+</body>
+</html>"""
+        
+        # Create a temporary HTML wrapper file
+        wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
+        with open(wrapper_path, 'w') as f:
+            f.write(html_wrapper)
+        
+        try:
+            files = {
+                'index.html': ('index.html', open(wrapper_path, 'rb')),
+                markdown_filename: (markdown_filename, open(file_path, 'rb'))
+            }
+            
+            form_data = {
+                'paperWidth': '8.27',  # A4 width in inches
+                'paperHeight': '11.7',  # A4 height in inches
+                'marginTop': '0.4',
+                'marginBottom': '0.4',
+                'marginLeft': '0.4',
+                'marginRight': '0.4',
+            }
+        finally:
+            # Clean up the temporary wrapper file after preparing the request
+            if os.path.exists(wrapper_path):
+                os.remove(wrapper_path)
+    
+    # Fallback to LibreOffice for everything else
     else:
-        # If MIME detection fails, fallback to extension-based detection.
+        endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
-        ext = os.path.splitext(file_path)[1].lower()
+        files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
-        if ext in [".html", ".htm"]:
+        logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
-            endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+
-            form_key = "index.html"
+    if not endpoint:
-        else:
+        logger.error(f"Could not determine Gotenberg endpoint for file type: {mime_type}")
-            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        return None
 
     try:
-        with open(file_path, "rb") as f:
+        logger.info(f"Converting {file_path} using endpoint: {endpoint}")
-            files = {form_key: f}
+        
-            response = requests.post(endpoint, files=files)
+        # Send the conversion request to Gotenberg
-
+        response = requests.post(endpoint, files=files, data=form_data)
+        
         if response.status_code == 200:
+            # Save the converted PDF
             converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
             with open(converted_file_path, "wb") as out_file:
                 out_file.write(response.content)
+            
             logger.info(f"Converted file saved as PDF: {converted_file_path}")
-            process_document.delay(converted_file_path)  # Updated function call
+            
+            # Enqueue the PDF for further processing
+            process_document.delay(converted_file_path)
+            
             return converted_file_path
         else:
-            logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
+            logger.error(
+                f"Conversion failed for {file_path}. "
+                f"Status code: {response.status_code}, "
+                f"Response: {response.text[:500]}..."
+            )
+            return None
     except Exception as e:
         logger.exception(f"Error converting {file_path} to PDF: {e}")
+        return None

frontend/templates/upload.html

@@ -3,7 +3,7 @@
 
 {% block content %}
 <div class="flex flex-col items-center justify-center p-8">
-  <h1 class="text-3xl font-bold mb-8">Upload a File</h1>
+  <h1 class="text-3xl font-bold mb-8">Upload Files</h1>
 
   <form action="/api/ui-upload" method="POST" enctype="multipart/form-data">
     <div
@@ -11,30 +11,86 @@
       class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-full max-w-lg"
       ondrop="handleDrop(event)"
       ondragover="handleDragOver(event)"
+      ondragleave="handleDragLeave(event)"
     >
       <p class="text-gray-500 mb-4">
-        Drag & drop a file here, or click to select a file.
+        Drag & drop files here, or click to select files.
       </p>
       <input
         id="fileInput"
         type="file"
         class="hidden"
         onchange="handleFileSelect(event)"
-        name="file"
+        name="files"
+        multiple
       />
+      <div class="text-sm text-gray-500 mt-2">
+        <p>Allowed types: PDF, Office documents (Word, Excel, PowerPoint, etc.), Images</p>
+        <p>Maximum size: 500MB per file</p>
+      </div>
     </div>
   </form>
 
   <div id="statusMessage" class="mt-4 text-gray-700"></div>
+  <div id="uploadProgress" class="mt-4 w-full max-w-lg"></div>
 </div>
 {% endblock %}
 
 {% block scripts %}
 <script>
+  // Configuration
+  const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
+  
+  // Allowed file types
+  const ACCEPTED_TYPES = {
+    // PDF files
+    'application/pdf': true,
+    
+    // Image formats
+    'image/jpeg': true, 'image/jpg': true, 'image/png': true,
+    'image/gif': true, 'image/bmp': true, 'image/tiff': true,
+    'image/webp': true, 'image/svg+xml': true,
+    
+    // Office document formats - Word
+    'application/msword': true,
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
+    'application/vnd.ms-word.document.macroEnabled.12': true,
+    
+    // Excel
+    'application/vnd.ms-excel': true,
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
+    'application/vnd.ms-excel.sheet.macroEnabled.12': true,
+    
+    // PowerPoint
+    'application/vnd.ms-powerpoint': true,
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
+    'application/vnd.openxmlformats-officedocument.presentationml.template': true,
+    'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
+    
+    // Other common formats
+    'text/plain': true,
+    'text/csv': true,
+    'application/rtf': true,
+    'text/rtf': true,
+    'text/html': true,
+    'application/xml': true,
+    'text/xml': true
+  };
+  
+  // File extensions that are always allowed (even if mime type is not recognized)
+  const ACCEPTED_EXTENSIONS = [
+    '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
+    '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv',
+    '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
+  ];
+
   // Drag-and-drop / file input logic
   const dropZone = document.getElementById("dropZone");
   const fileInput = document.getElementById("fileInput");
   const statusMessage = document.getElementById("statusMessage");
+  const uploadProgress = document.getElementById("uploadProgress");
 
   dropZone.addEventListener("click", () => fileInput.click());
 
@@ -44,43 +100,155 @@
     dropZone.classList.add("bg-gray-100");
   }
 
+  function handleDragLeave(e) {
+    e.preventDefault();
+    dropZone.classList.remove("bg-gray-100");
+  }
+
   function handleDrop(e) {
     e.preventDefault();
     dropZone.classList.remove("bg-gray-100");
     if (e.dataTransfer.files.length) {
-      uploadFile(e.dataTransfer.files[0]);
+      processFiles(e.dataTransfer.files);
     }
   }
 
   function handleFileSelect(e) {
     if (e.target.files.length) {
-      uploadFile(e.target.files[0]);
+      processFiles(e.target.files);
     }
   }
 
-  async function uploadFile(file) {
+  function processFiles(files) {
-    // Check file type
+    if (files.length === 0) return;
-    const acceptedTypes = ['application/pdf'];
+    
-    if (!acceptedTypes.includes(file.type)) {
+    statusMessage.textContent = `Processing ${files.length} file(s)...`;
-      statusMessage.textContent = `Error: Only PDF files are accepted.`;
+    
+    // Clear previous upload progress
+    uploadProgress.innerHTML = "";
+    
+    // Create progress container
+    const progressContainer = document.createElement("div");
+    progressContainer.className = "space-y-2";
+    uploadProgress.appendChild(progressContainer);
+    
+    // Process each file
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      validateAndUpload(file, progressContainer);
+    }
+  }
+
+  function validateAndUpload(file, progressContainer) {
+    // Create progress element for this file
+    const fileProgress = document.createElement("div");
+    fileProgress.className = "flex flex-col mb-2";
+    fileProgress.innerHTML = `
+      <div class="flex justify-between">
+        <span class="text-sm truncate" title="${file.name}">${file.name}</span>
+        <span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
+      </div>
+      <div class="w-full bg-gray-200 h-2 rounded-full mt-1">
+        <div class="file-progress-bar bg-blue-500 h-2 rounded-full" style="width: 0%"></div>
+      </div>
+      <div class="file-status text-xs text-gray-600 mt-1">Validating...</div>
+    `;
+    progressContainer.appendChild(fileProgress);
+    
+    const progressBar = fileProgress.querySelector(".file-progress-bar");
+    const statusEl = fileProgress.querySelector(".file-status");
+    
+    // Validate file type by checking both MIME type and extension
+    const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
+    const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
+    const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
+    
+    if (!isValidMimeType && !isValidExtension) {
+      statusEl.textContent = `Error: ${file.name} - Unsupported file type`;
+      statusEl.className = "text-xs text-red-500 mt-1";
       return;
     }
-    statusMessage.textContent = `Uploading ${file.name}...`;
+    
+    // Validate file size
+    if (file.size > MAX_FILE_SIZE) {
+      statusEl.textContent = `Error: ${file.name} - File size exceeds 500MB limit`;
+      statusEl.className = "text-xs text-red-500 mt-1";
+      return;
+    }
+    
+    // Upload the file
+    uploadFile(file, progressBar, statusEl);
+  }
+
+  async function uploadFile(file, progressBar, statusEl) {
+    statusEl.textContent = `Uploading...`;
     try {
       let formData = new FormData();
       formData.append("file", file);
-      const response = await fetch("/api/ui-upload", {
+      
-        method: "POST",
+      const xhr = new XMLHttpRequest();
-        body: formData,
+      xhr.open("POST", "/api/ui-upload", true);
-      });
+      
-      if (!response.ok) {
+      xhr.upload.onprogress = (e) => {
-        throw new Error(`Upload failed with status ${response.status}`);
+        if (e.lengthComputable) {
-      }
+          const percentComplete = (e.loaded / e.total) * 100;
-      const result = await response.json();
+          progressBar.style.width = percentComplete + "%";
-      statusMessage.textContent = `File ${file.name} uploaded. Task ID: ${result.task_id}`;
+          statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
+        }
+      };
+      
+      xhr.onload = function() {
+        if (xhr.status === 200) {
+          const result = JSON.parse(xhr.responseText);
+          progressBar.style.width = "100%";
+          progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
+          statusEl.textContent = `Success: Task ID: ${result.task_id}`;
+          statusEl.className = "text-xs text-green-600 mt-1";
+          updateOverallStatus();
+        } else {
+          throw new Error(`Upload failed with status ${xhr.status}`);
+        }
+      };
+      
+      xhr.onerror = function() {
+        throw new Error("Network error occurred");
+      };
+      
+      xhr.send(formData);
+      
     } catch (err) {
-      statusMessage.textContent = `Error: ${err}`;
+      statusEl.textContent = `Error: ${err.message}`;
+      statusEl.className = "text-xs text-red-500 mt-1";
+      progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+      updateOverallStatus();
     }
   }
+
+  function updateOverallStatus() {
+    // Count success/failure
+    const fileStatuses = document.querySelectorAll('.file-status');
+    let completed = 0;
+    let total = fileStatuses.length;
+    
+    fileStatuses.forEach(status => {
+      if (status.textContent.includes('Success') || status.textContent.includes('Error')) {
+        completed++;
+      }
+    });
+    
+    if (completed === total) {
+      statusMessage.textContent = `All uploads completed (${completed}/${total})`;
+    } else {
+      statusMessage.textContent = `Uploading files (${completed}/${total})`;
+    }
+  }
+
+  function formatFileSize(bytes) {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  }
 </script>
 {% endblock %}