gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upload): adaptive 429 backoff, full Gotenberg file types, directory traversal

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-26 23:24:26 +00:00
parent 35d2ed05e9
commit a05690bd5f
10 changed files with 764 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+15 -49
View File
@@ -18,6 +18,7 @@ from app.database import get_db
from app.models import FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.process_document import process_document
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
from app.utils.file_queries import apply_status_filter
from app.utils.file_status import get_files_processing_status
from app.utils.filename_utils import sanitize_filename
@@ -1034,33 +1035,6 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
file_size = written_size
# Same set of allowed file types as in the IMAP task
ALLOWED_MIME_TYPES = {
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
# Image MIME types that need conversion
IMAGE_MIME_TYPES = {
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
# Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower()
@@ -1114,32 +1088,24 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# If it's a PDF, process directly
task = process_document.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued PDF for processing: {target_path}")
elif mime_type in IMAGE_MIME_TYPES or any(
file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
):
elif mime_type in IMAGE_MIME_TYPES or file_ext in {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}:
# If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued image for PDF conversion: {target_path}")
elif mime_type in ALLOWED_MIME_TYPES or any(
file_ext.endswith(ext)
for ext in [
".doc",
".docx",
".xls",
".xlsx",
".ppt",
".pptx",
".odt",
".ods",
".odp",
".rtf",
".txt",
".csv",
]
):
# If it's an office document, convert to PDF first
elif mime_type in ALLOWED_MIME_TYPES or file_ext in ALLOWED_EXTENSIONS:
# Office document, HTML, Markdown, or other Gotenberg-supported format
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued office document for PDF conversion: {target_path}")
logger.info(f"Enqueued document for PDF conversion: {target_path}")
else:
# For any other file type, attempt conversion but log a warning
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
app/api/url_upload.py
+4 -29
View File
@@ -17,6 +17,7 @@ from pydantic import BaseModel, HttpUrl, field_validator
from app.auth import require_login
from app.config import settings
from app.tasks.process_document import process_document
from app.utils.allowed_types import ALLOWED_MIME_TYPES
from app.utils.filename_utils import sanitize_filename
# Set up logging
@@ -108,7 +109,7 @@ def validate_url_safety(url: str) -> None:
def validate_file_type(content_type: str, filename: str) -> bool:
"""
Validate that the file type is supported.
Validate that the file type is supported (i.e. processable by Gotenberg).
Args:
content_type: MIME type from response headers
@@ -117,44 +118,18 @@ def validate_file_type(content_type: str, filename: str) -> bool:
Returns:
True if file type is allowed
"""
# Same allowed types as regular upload
ALLOWED_MIME_TYPES = {
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
IMAGE_MIME_TYPES = {
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
# Check content type from header
if content_type:
# Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES:
if base_content_type in ALLOWED_MIME_TYPES:
return True
# Also check by extension as fallback
_, ext = os.path.splitext(filename)
if ext:
guessed_type, _ = mimetypes.guess_type(filename)
if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES):
if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
return True
return False
app/tasks/imap_tasks.py
+4 -16
View File
@@ -13,6 +13,7 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
logger = logging.getLogger(__name__)
@@ -268,20 +269,6 @@ def fetch_attachments_and_enqueue(email_message):
Returns True if at least one allowed attachment was processed.
"""
ALLOWED_MIME_TYPES = {
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -295,8 +282,9 @@ def fetch_attachments_and_enqueue(email_message):
is_pdf_by_extension = filename.lower().endswith(".pdf")
mime_type = part.get_content_type()
# Accept file if it has an allowed MIME type OR it's a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
file_ext = os.path.splitext(filename)[1].lower()
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
app/utils/allowed_types.py
+133
View File
@@ -0,0 +1,133 @@
"""
Canonical file-type lists for DocuElevate uploads.
All file types listed here are supported by Gotenberg (the PDF-conversion service
used by DocuElevate) via its LibreOffice, Chromium, or Markdown routes. This
module is the single source of truth consumed by:
- app/api/files.py (ui-upload endpoint)
- app/api/url_upload.py (URL-upload endpoint)
- app/tasks/imap_tasks.py (IMAP email-attachment ingestion)
- frontend/static/js/upload.js (client-side validation mirror)
Keep in sync with the OFFICE_EXTENSIONS / IMAGE_EXTENSIONS sets defined in
app/tasks/convert_to_pdf.py.
"""
# ---------------------------------------------------------------------------
# Document / office MIME types (converted via Gotenberg LibreOffice route)
# ---------------------------------------------------------------------------
DOCUMENT_MIME_TYPES: set[str] = {
# PDF
"application/pdf",
# Word
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"application/vnd.ms-word.document.macroEnabled.12",
"application/vnd.ms-word.template.macroEnabled.12",
# Excel
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"application/vnd.ms-excel.sheet.macroEnabled.12",
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
# PowerPoint
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.presentationml.template",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
# OpenDocument (LibreOffice native)
"application/vnd.oasis.opendocument.text",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.graphics",
"application/vnd.oasis.opendocument.formula",
# Plain text / data
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
# HTML (converted via Gotenberg Chromium route)
"text/html",
# Markdown (converted via Gotenberg Chromium/Markdown route)
"text/markdown",
"text/x-markdown",
}
# ---------------------------------------------------------------------------
# Image MIME types (converted via Gotenberg LibreOffice route)
# ---------------------------------------------------------------------------
IMAGE_MIME_TYPES: set[str] = {
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
# ---------------------------------------------------------------------------
# Combined set – every MIME type accepted by the upload endpoints
# ---------------------------------------------------------------------------
ALLOWED_MIME_TYPES: set[str] = DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
# ---------------------------------------------------------------------------
# File extensions (lower-case, with leading dot) accepted by Gotenberg
# ---------------------------------------------------------------------------
ALLOWED_EXTENSIONS: set[str] = {
# PDF
".pdf",
# Word
".doc",
".docx",
".docm",
".dot",
".dotx",
".dotm",
# Excel
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw",
# PowerPoint
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx",
# OpenDocument
".odt",
".ods",
".odp",
".odg",
".odf",
# Text / data
".rtf",
".txt",
".csv",
# Images
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
# Web
".html",
".htm",
# Markdown
".md",
".markdown",
}
app/views/files.py
+6 -24
View File
@@ -17,26 +17,6 @@ router = APIRouter()
_FILE_NOT_FOUND = "File not found"
def _get_upload_concurrency() -> int:
"""Return the configured upload concurrency (falls back to default on error)."""
try:
from app.config import settings
return settings.upload_concurrency
except Exception:
return 3
def _get_upload_queue_delay_ms() -> int:
"""Return the configured upload queue delay in ms (falls back to default on error)."""
try:
from app.config import settings
return settings.upload_queue_delay_ms
except Exception:
return 500
@router.get("/files")
@require_login
def files_page(
@@ -53,6 +33,8 @@ def files_page(
"""
Return the 'files.html' template with server-side pagination, sorting, and filtering
"""
from app.config import settings
try:
# Import the model here to avoid circular imports
from sqlalchemy import asc, desc
@@ -131,8 +113,8 @@ def files_page(
"mime_type": mime_type or "",
"status": status or "",
"mime_types": mime_types,
"upload_concurrency": _get_upload_concurrency(),
"upload_queue_delay_ms": _get_upload_queue_delay_ms(),
"upload_concurrency": settings.upload_concurrency,
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
},
)
except Exception as e:
@@ -146,8 +128,8 @@ def files_page(
"files": [],
"pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
"error": str(e),
"upload_concurrency": _get_upload_concurrency(),
"upload_queue_delay_ms": _get_upload_queue_delay_ms(),
"upload_concurrency": settings.upload_concurrency,
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
},
)