diff --git a/app/api/files.py b/app/api/files.py
index ab17bb46..17315ffa 100644
--- a/app/api/files.py
+++ b/app/api/files.py
@@ -6,12 +6,14 @@ from sqlalchemy.orm import Session
import logging
import os
import uuid
+import mimetypes
from app.auth import require_login
from app.models import FileRecord
from app.config import settings
from app.api.common import get_db
from app.tasks.process_document import process_document
+from app.tasks.convert_to_pdf import convert_to_pdf
# Set up logging
logger = logging.getLogger(__name__)
@@ -134,7 +136,63 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# Log the mapping between original and safe filename
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
- task = process_document.delay(target_path)
+ # Check file size
+ file_size = os.path.getsize(target_path)
+ max_size = 500 * 1024 * 1024 # 500MB
+ if file_size > max_size:
+ # Remove the file if it's too large
+ os.remove(target_path)
+ raise HTTPException(
+ status_code=413,
+ detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
+ )
+
+ # Same set of allowed file types as in the IMAP task
+ ALLOWED_MIME_TYPES = {
+ "application/pdf",
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-excel",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-powerpoint",
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ "text/plain",
+ "text/csv",
+ "application/rtf",
+ "text/rtf",
+ }
+
+ # Image MIME types that need conversion
+ IMAGE_MIME_TYPES = {
+ 'image/jpeg', 'image/jpg', 'image/png',
+ 'image/gif', 'image/bmp', 'image/tiff',
+ 'image/webp', 'image/svg+xml'
+ }
+
+ # Determine if the file is a PDF or needs conversion
+ mime_type, _ = mimetypes.guess_type(target_path)
+ file_ext = os.path.splitext(target_path)[1].lower()
+
+ # Check if it's a PDF by extension or MIME type
+ is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
+
+ if is_pdf:
+ # If it's a PDF, process directly
+ task = process_document.delay(target_path)
+ logger.info(f"Enqueued PDF for processing: {target_path}")
+ elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']):
+ # If it's an image, convert to PDF first
+ task = convert_to_pdf.delay(target_path)
+ logger.info(f"Enqueued image for PDF conversion: {target_path}")
+ elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']):
+ # If it's an office document, convert to PDF first
+ task = convert_to_pdf.delay(target_path)
+ logger.info(f"Enqueued office document for PDF conversion: {target_path}")
+ else:
+ # For any other file type, attempt conversion but log a warning
+ logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
+ task = convert_to_pdf.delay(target_path)
+
return {
"task_id": task.id,
"status": "queued",
diff --git a/app/tasks/convert_to_pdf.py b/app/tasks/convert_to_pdf.py
index 2842c984..34fec4a5 100644
--- a/app/tasks/convert_to_pdf.py
+++ b/app/tasks/convert_to_pdf.py
@@ -3,9 +3,10 @@ import os
import requests
import logging
import mimetypes
+import json
from celery import shared_task
from app.config import settings
-from app.tasks.process_document import process_document # Updated import
+from app.tasks.process_document import process_document
logger = logging.getLogger(__name__)
@@ -14,59 +15,165 @@ def convert_to_pdf(file_path):
"""
Converts a file to PDF using Gotenberg's API.
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
- On success, saves the PDF locally and enqueues it for S3 upload.
+ On success, saves the PDF locally and enqueues it for processing.
"""
gotenberg_url = getattr(settings, "gotenberg_url", None)
if not gotenberg_url:
logger.error("Gotenberg URL is not configured in settings.")
return
- # Try to guess the MIME type based on file content (using extension-based fallback)
+ # Try to guess the MIME type based on file content and extension
mime_type, encoding = mimetypes.guess_type(file_path)
- logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
+ file_ext = os.path.splitext(file_path)[1].lower()
+ logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
+ # Determine which Gotenberg endpoint to use
endpoint = None
- form_key = "files" # Default form key for most endpoints
-
- if mime_type:
- if mime_type == "text/html":
- endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
- # The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
- form_key = "index.html"
- elif mime_type.startswith("image/"):
- # For images, we use the LibreOffice endpoint (which supports image conversion)
- endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
- elif mime_type.startswith("text/plain"):
- endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
- elif mime_type in ["text/markdown", "text/x-markdown"]:
- # Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
- # For now, we'll fallback to LibreOffice.
- endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
- else:
- # For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
- endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+ form_data = {}
+ files = {}
+
+ # Dictionary mapping file extensions to their handlers
+ OFFICE_EXTENSIONS = {
+ '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm', # Word
+ '.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw', # Excel
+ '.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx', # PowerPoint
+ '.odt', '.ods', '.odp', '.odg', '.odf', # OpenOffice/LibreOffice
+ '.rtf', '.txt', '.csv', # Text formats
+ '.pdf', # PDF (already in PDF format but can be processed)
+ }
+
+ IMAGE_EXTENSIONS = {
+ '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg'
+ }
+
+ HTML_EXTENSIONS = {
+ '.html', '.htm'
+ }
+
+ # Use LibreOffice endpoint for office documents and images
+ if (mime_type and 'office' in mime_type) or \
+ (mime_type and 'opendocument' in mime_type) or \
+ (mime_type and mime_type.startswith('image/')) or \
+ file_ext in OFFICE_EXTENSIONS or \
+ file_ext in IMAGE_EXTENSIONS:
+ endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+ files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
+
+ # Add some quality settings for better PDF output
+ form_data = {
+ 'landscape': 'false',
+ 'exportBookmarks': 'true',
+ 'exportNotes': 'false',
+ 'losslessImageCompression': 'true', # Use lossless compression for images
+ 'pdfa': 'PDF/A-2b', # Produce PDF/A-2b compatible output
+ }
+
+ # Use Chromium endpoint for HTML documents
+ elif (mime_type and mime_type == 'text/html') or file_ext in HTML_EXTENSIONS:
+ endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+ # Gotenberg requires the form field to be exactly 'index.html'
+ # The content filename doesn't matter, just the form field key
+ files = {'index.html': ('index.html', open(file_path, 'rb'))}
+
+ # Add options for better HTML to PDF conversion
+ form_data = {
+ 'paperWidth': '8.27', # A4 width in inches
+ 'paperHeight': '11.7', # A4 height in inches
+ 'marginTop': '0.4',
+ 'marginBottom': '0.4',
+ 'marginLeft': '0.4',
+ 'marginRight': '0.4',
+ 'printBackground': 'true',
+ 'preferCssPageSize': 'false',
+ 'waitDelay': '2s', # Wait for JavaScript to execute
+ }
+
+ # Use Markdown route for markdown files
+ elif (mime_type and mime_type in ['text/markdown', 'text/x-markdown']) or file_ext in ['.md', '.markdown']:
+ # For Markdown, we need both the markdown file and an HTML wrapper
+ endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
+
+ # Create a simple HTML wrapper for the markdown
+ # IMPORTANT: The filename in the template must match the key used in the files dictionary
+ markdown_filename = os.path.basename(file_path)
+ html_wrapper = f"""
+
+
+
+ Converted Markdown
+
+
+
+ {{{{ toHTML "{markdown_filename}" }}}}
+
+"""
+
+ # Create a temporary HTML wrapper file
+ wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
+ with open(wrapper_path, 'w') as f:
+ f.write(html_wrapper)
+
+ try:
+ files = {
+ 'index.html': ('index.html', open(wrapper_path, 'rb')),
+ markdown_filename: (markdown_filename, open(file_path, 'rb'))
+ }
+
+ form_data = {
+ 'paperWidth': '8.27', # A4 width in inches
+ 'paperHeight': '11.7', # A4 height in inches
+ 'marginTop': '0.4',
+ 'marginBottom': '0.4',
+ 'marginLeft': '0.4',
+ 'marginRight': '0.4',
+ }
+ finally:
+ # Clean up the temporary wrapper file after preparing the request
+ if os.path.exists(wrapper_path):
+ os.remove(wrapper_path)
+
+ # Fallback to LibreOffice for everything else
else:
- # If MIME detection fails, fallback to extension-based detection.
- ext = os.path.splitext(file_path)[1].lower()
- if ext in [".html", ".htm"]:
- endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
- form_key = "index.html"
- else:
- endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+ endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+ files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
+ logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
+
+ if not endpoint:
+ logger.error(f"Could not determine Gotenberg endpoint for file type: {mime_type}")
+ return None
try:
- with open(file_path, "rb") as f:
- files = {form_key: f}
- response = requests.post(endpoint, files=files)
-
+ logger.info(f"Converting {file_path} using endpoint: {endpoint}")
+
+ # Send the conversion request to Gotenberg
+ response = requests.post(endpoint, files=files, data=form_data)
+
if response.status_code == 200:
+ # Save the converted PDF
converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
with open(converted_file_path, "wb") as out_file:
out_file.write(response.content)
+
logger.info(f"Converted file saved as PDF: {converted_file_path}")
- process_document.delay(converted_file_path) # Updated function call
+
+ # Enqueue the PDF for further processing
+ process_document.delay(converted_file_path)
+
return converted_file_path
else:
- logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
+ logger.error(
+ f"Conversion failed for {file_path}. "
+ f"Status code: {response.status_code}, "
+ f"Response: {response.text[:500]}..."
+ )
+ return None
except Exception as e:
logger.exception(f"Error converting {file_path} to PDF: {e}")
+ return None
diff --git a/frontend/templates/upload.html b/frontend/templates/upload.html
index dc70734d..5fbb221a 100644
--- a/frontend/templates/upload.html
+++ b/frontend/templates/upload.html
@@ -3,7 +3,7 @@
{% block content %}