feat: implement file size check and MIME type handling for document uploads

This commit is contained in:
Christian Krakau-Louis
2025-04-04 02:46:29 +02:00
parent 40ddb88110
commit 81fd9cda14
3 changed files with 391 additions and 58 deletions
+59 -1
View File
@@ -6,12 +6,14 @@ from sqlalchemy.orm import Session
import logging
import os
import uuid
import mimetypes
from app.auth import require_login
from app.models import FileRecord
from app.config import settings
from app.api.common import get_db
from app.tasks.process_document import process_document
from app.tasks.convert_to_pdf import convert_to_pdf
# Set up logging
logger = logging.getLogger(__name__)
@@ -134,7 +136,63 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# Log the mapping between original and safe filename
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
task = process_document.delay(target_path)
# Check file size
file_size = os.path.getsize(target_path)
max_size = 500 * 1024 * 1024 # 500MB
if file_size > max_size:
# Remove the file if it's too large
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
)
# Same set of allowed file types as in the IMAP task
ALLOWED_MIME_TYPES = {
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
# Image MIME types that need conversion
IMAGE_MIME_TYPES = {
'image/jpeg', 'image/jpg', 'image/png',
'image/gif', 'image/bmp', 'image/tiff',
'image/webp', 'image/svg+xml'
}
# Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower()
# Check if it's a PDF by extension or MIME type
is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
if is_pdf:
# If it's a PDF, process directly
task = process_document.delay(target_path)
logger.info(f"Enqueued PDF for processing: {target_path}")
elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']):
# If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path)
logger.info(f"Enqueued image for PDF conversion: {target_path}")
elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']):
# If it's an office document, convert to PDF first
task = convert_to_pdf.delay(target_path)
logger.info(f"Enqueued office document for PDF conversion: {target_path}")
else:
# For any other file type, attempt conversion but log a warning
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
task = convert_to_pdf.delay(target_path)
return {
"task_id": task.id,
"status": "queued",
+143 -36
View File
@@ -3,9 +3,10 @@ import os
import requests
import logging
import mimetypes
import json
from celery import shared_task
from app.config import settings
from app.tasks.process_document import process_document # Updated import
from app.tasks.process_document import process_document
logger = logging.getLogger(__name__)
@@ -14,59 +15,165 @@ def convert_to_pdf(file_path):
"""
Converts a file to PDF using Gotenberg's API.
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
On success, saves the PDF locally and enqueues it for S3 upload.
On success, saves the PDF locally and enqueues it for processing.
"""
gotenberg_url = getattr(settings, "gotenberg_url", None)
if not gotenberg_url:
logger.error("Gotenberg URL is not configured in settings.")
return
# Try to guess the MIME type based on file content (using extension-based fallback)
# Try to guess the MIME type based on file content and extension
mime_type, encoding = mimetypes.guess_type(file_path)
logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
file_ext = os.path.splitext(file_path)[1].lower()
logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
# Determine which Gotenberg endpoint to use
endpoint = None
form_key = "files" # Default form key for most endpoints
if mime_type:
if mime_type == "text/html":
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
# The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
form_key = "index.html"
elif mime_type.startswith("image/"):
# For images, we use the LibreOffice endpoint (which supports image conversion)
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
elif mime_type.startswith("text/plain"):
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
elif mime_type in ["text/markdown", "text/x-markdown"]:
# Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
# For now, we'll fallback to LibreOffice.
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
else:
# For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
form_data = {}
files = {}
# Dictionary mapping file extensions to their handlers
OFFICE_EXTENSIONS = {
'.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm', # Word
'.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw', # Excel
'.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx', # PowerPoint
'.odt', '.ods', '.odp', '.odg', '.odf', # OpenOffice/LibreOffice
'.rtf', '.txt', '.csv', # Text formats
'.pdf', # PDF (already in PDF format but can be processed)
}
IMAGE_EXTENSIONS = {
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg'
}
HTML_EXTENSIONS = {
'.html', '.htm'
}
# Use LibreOffice endpoint for office documents and images
if (mime_type and 'office' in mime_type) or \
(mime_type and 'opendocument' in mime_type) or \
(mime_type and mime_type.startswith('image/')) or \
file_ext in OFFICE_EXTENSIONS or \
file_ext in IMAGE_EXTENSIONS:
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
# Add some quality settings for better PDF output
form_data = {
'landscape': 'false',
'exportBookmarks': 'true',
'exportNotes': 'false',
'losslessImageCompression': 'true', # Use lossless compression for images
'pdfa': 'PDF/A-2b', # Produce PDF/A-2b compatible output
}
# Use Chromium endpoint for HTML documents
elif (mime_type and mime_type == 'text/html') or file_ext in HTML_EXTENSIONS:
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
# Gotenberg requires the form field to be exactly 'index.html'
# The content filename doesn't matter, just the form field key
files = {'index.html': ('index.html', open(file_path, 'rb'))}
# Add options for better HTML to PDF conversion
form_data = {
'paperWidth': '8.27', # A4 width in inches
'paperHeight': '11.7', # A4 height in inches
'marginTop': '0.4',
'marginBottom': '0.4',
'marginLeft': '0.4',
'marginRight': '0.4',
'printBackground': 'true',
'preferCssPageSize': 'false',
'waitDelay': '2s', # Wait for JavaScript to execute
}
# Use Markdown route for markdown files
elif (mime_type and mime_type in ['text/markdown', 'text/x-markdown']) or file_ext in ['.md', '.markdown']:
# For Markdown, we need both the markdown file and an HTML wrapper
endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
# Create a simple HTML wrapper for the markdown
# IMPORTANT: The filename in the template must match the key used in the files dictionary
markdown_filename = os.path.basename(file_path)
html_wrapper = f"""<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Converted Markdown</title>
<style>
body {{
font-family: Arial, sans-serif;
line-height: 1.6;
margin: 2em;
max-width: 50em;
}}
</style>
</head>
<body>
{{{{ toHTML "{markdown_filename}" }}}}
</body>
</html>"""
# Create a temporary HTML wrapper file
wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
with open(wrapper_path, 'w') as f:
f.write(html_wrapper)
try:
files = {
'index.html': ('index.html', open(wrapper_path, 'rb')),
markdown_filename: (markdown_filename, open(file_path, 'rb'))
}
form_data = {
'paperWidth': '8.27', # A4 width in inches
'paperHeight': '11.7', # A4 height in inches
'marginTop': '0.4',
'marginBottom': '0.4',
'marginLeft': '0.4',
'marginRight': '0.4',
}
finally:
# Clean up the temporary wrapper file after preparing the request
if os.path.exists(wrapper_path):
os.remove(wrapper_path)
# Fallback to LibreOffice for everything else
else:
# If MIME detection fails, fallback to extension-based detection.
ext = os.path.splitext(file_path)[1].lower()
if ext in [".html", ".htm"]:
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
form_key = "index.html"
else:
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
if not endpoint:
logger.error(f"Could not determine Gotenberg endpoint for file type: {mime_type}")
return None
try:
with open(file_path, "rb") as f:
files = {form_key: f}
response = requests.post(endpoint, files=files)
logger.info(f"Converting {file_path} using endpoint: {endpoint}")
# Send the conversion request to Gotenberg
response = requests.post(endpoint, files=files, data=form_data)
if response.status_code == 200:
# Save the converted PDF
converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
with open(converted_file_path, "wb") as out_file:
out_file.write(response.content)
logger.info(f"Converted file saved as PDF: {converted_file_path}")
process_document.delay(converted_file_path) # Updated function call
# Enqueue the PDF for further processing
process_document.delay(converted_file_path)
return converted_file_path
else:
logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
logger.error(
f"Conversion failed for {file_path}. "
f"Status code: {response.status_code}, "
f"Response: {response.text[:500]}..."
)
return None
except Exception as e:
logger.exception(f"Error converting {file_path} to PDF: {e}")
return None
+189 -21
View File
@@ -3,7 +3,7 @@
{% block content %}
<div class="flex flex-col items-center justify-center p-8">
<h1 class="text-3xl font-bold mb-8">Upload a File</h1>
<h1 class="text-3xl font-bold mb-8">Upload Files</h1>
<form action="/api/ui-upload" method="POST" enctype="multipart/form-data">
<div
@@ -11,30 +11,86 @@
class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-full max-w-lg"
ondrop="handleDrop(event)"
ondragover="handleDragOver(event)"
ondragleave="handleDragLeave(event)"
>
<p class="text-gray-500 mb-4">
Drag & drop a file here, or click to select a file.
Drag & drop files here, or click to select files.
</p>
<input
id="fileInput"
type="file"
class="hidden"
onchange="handleFileSelect(event)"
name="file"
name="files"
multiple
/>
<div class="text-sm text-gray-500 mt-2">
<p>Allowed types: PDF, Office documents (Word, Excel, PowerPoint, etc.), Images</p>
<p>Maximum size: 500MB per file</p>
</div>
</div>
</form>
<div id="statusMessage" class="mt-4 text-gray-700"></div>
<div id="uploadProgress" class="mt-4 w-full max-w-lg"></div>
</div>
{% endblock %}
{% block scripts %}
<script>
// Configuration
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
// Allowed file types
const ACCEPTED_TYPES = {
// PDF files
'application/pdf': true,
// Image formats
'image/jpeg': true, 'image/jpg': true, 'image/png': true,
'image/gif': true, 'image/bmp': true, 'image/tiff': true,
'image/webp': true, 'image/svg+xml': true,
// Office document formats - Word
'application/msword': true,
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
'application/vnd.ms-word.document.macroEnabled.12': true,
// Excel
'application/vnd.ms-excel': true,
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
'application/vnd.ms-excel.sheet.macroEnabled.12': true,
// PowerPoint
'application/vnd.ms-powerpoint': true,
'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
'application/vnd.openxmlformats-officedocument.presentationml.template': true,
'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
// Other common formats
'text/plain': true,
'text/csv': true,
'application/rtf': true,
'text/rtf': true,
'text/html': true,
'application/xml': true,
'text/xml': true
};
// File extensions that are always allowed (even if mime type is not recognized)
const ACCEPTED_EXTENSIONS = [
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
'.odt', '.ods', '.odp', '.rtf', '.txt', '.csv',
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
];
// Drag-and-drop / file input logic
const dropZone = document.getElementById("dropZone");
const fileInput = document.getElementById("fileInput");
const statusMessage = document.getElementById("statusMessage");
const uploadProgress = document.getElementById("uploadProgress");
dropZone.addEventListener("click", () => fileInput.click());
@@ -44,43 +100,155 @@
dropZone.classList.add("bg-gray-100");
}
function handleDragLeave(e) {
e.preventDefault();
dropZone.classList.remove("bg-gray-100");
}
function handleDrop(e) {
e.preventDefault();
dropZone.classList.remove("bg-gray-100");
if (e.dataTransfer.files.length) {
uploadFile(e.dataTransfer.files[0]);
processFiles(e.dataTransfer.files);
}
}
function handleFileSelect(e) {
if (e.target.files.length) {
uploadFile(e.target.files[0]);
processFiles(e.target.files);
}
}
async function uploadFile(file) {
// Check file type
const acceptedTypes = ['application/pdf'];
if (!acceptedTypes.includes(file.type)) {
statusMessage.textContent = `Error: Only PDF files are accepted.`;
function processFiles(files) {
if (files.length === 0) return;
statusMessage.textContent = `Processing ${files.length} file(s)...`;
// Clear previous upload progress
uploadProgress.innerHTML = "";
// Create progress container
const progressContainer = document.createElement("div");
progressContainer.className = "space-y-2";
uploadProgress.appendChild(progressContainer);
// Process each file
for (let i = 0; i < files.length; i++) {
const file = files[i];
validateAndUpload(file, progressContainer);
}
}
function validateAndUpload(file, progressContainer) {
// Create progress element for this file
const fileProgress = document.createElement("div");
fileProgress.className = "flex flex-col mb-2";
fileProgress.innerHTML = `
<div class="flex justify-between">
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
</div>
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
<div class="file-progress-bar bg-blue-500 h-2 rounded-full" style="width: 0%"></div>
</div>
<div class="file-status text-xs text-gray-600 mt-1">Validating...</div>
`;
progressContainer.appendChild(fileProgress);
const progressBar = fileProgress.querySelector(".file-progress-bar");
const statusEl = fileProgress.querySelector(".file-status");
// Validate file type by checking both MIME type and extension
const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
if (!isValidMimeType && !isValidExtension) {
statusEl.textContent = `Error: ${file.name} - Unsupported file type`;
statusEl.className = "text-xs text-red-500 mt-1";
return;
}
statusMessage.textContent = `Uploading ${file.name}...`;
// Validate file size
if (file.size > MAX_FILE_SIZE) {
statusEl.textContent = `Error: ${file.name} - File size exceeds 500MB limit`;
statusEl.className = "text-xs text-red-500 mt-1";
return;
}
// Upload the file
uploadFile(file, progressBar, statusEl);
}
async function uploadFile(file, progressBar, statusEl) {
statusEl.textContent = `Uploading...`;
try {
let formData = new FormData();
formData.append("file", file);
const response = await fetch("/api/ui-upload", {
method: "POST",
body: formData,
});
if (!response.ok) {
throw new Error(`Upload failed with status ${response.status}`);
}
const result = await response.json();
statusMessage.textContent = `File ${file.name} uploaded. Task ID: ${result.task_id}`;
const xhr = new XMLHttpRequest();
xhr.open("POST", "/api/ui-upload", true);
xhr.upload.onprogress = (e) => {
if (e.lengthComputable) {
const percentComplete = (e.loaded / e.total) * 100;
progressBar.style.width = percentComplete + "%";
statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
}
};
xhr.onload = function() {
if (xhr.status === 200) {
const result = JSON.parse(xhr.responseText);
progressBar.style.width = "100%";
progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
statusEl.textContent = `Success: Task ID: ${result.task_id}`;
statusEl.className = "text-xs text-green-600 mt-1";
updateOverallStatus();
} else {
throw new Error(`Upload failed with status ${xhr.status}`);
}
};
xhr.onerror = function() {
throw new Error("Network error occurred");
};
xhr.send(formData);
} catch (err) {
statusMessage.textContent = `Error: ${err}`;
statusEl.textContent = `Error: ${err.message}`;
statusEl.className = "text-xs text-red-500 mt-1";
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
updateOverallStatus();
}
}
function updateOverallStatus() {
// Count success/failure
const fileStatuses = document.querySelectorAll('.file-status');
let completed = 0;
let total = fileStatuses.length;
fileStatuses.forEach(status => {
if (status.textContent.includes('Success') || status.textContent.includes('Error')) {
completed++;
}
});
if (completed === total) {
statusMessage.textContent = `All uploads completed (${completed}/${total})`;
} else {
statusMessage.textContent = `Uploading files (${completed}/${total})`;
}
}
function formatFileSize(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
</script>
{% endblock %}