docs: clarify PDF splitting is page-based, not byte-based
- Add prominent documentation that splitting uses page boundaries - Update docstring with IMPORTANT note about page-level splitting - Add test to validate split PDFs are valid and readable - Update ConfigurationGuide.md to emphasize page-based approach - Update SECURITY_AUDIT.md with implementation details - Ensures users understand no risk of corrupted PDFs Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+4
-1
@@ -150,7 +150,10 @@ This document tracks security vulnerabilities found in DocuElevate and their rem
|
|||||||
**Fix:** Implemented configurable file upload size limits with the following features:
|
**Fix:** Implemented configurable file upload size limits with the following features:
|
||||||
- `MAX_UPLOAD_SIZE`: Maximum file upload size in bytes (default: 1GB)
|
- `MAX_UPLOAD_SIZE`: Maximum file upload size in bytes (default: 1GB)
|
||||||
- `MAX_SINGLE_FILE_SIZE`: Optional maximum size for a single file chunk
|
- `MAX_SINGLE_FILE_SIZE`: Optional maximum size for a single file chunk
|
||||||
- Automatic file splitting for large PDFs when max_single_file_size is configured
|
- **Automatic page-based PDF splitting** for large PDFs when max_single_file_size is configured
|
||||||
|
- Splits PDFs at **page boundaries** using PyPDF2, NOT by byte position
|
||||||
|
- Each output file is a structurally valid, complete PDF
|
||||||
|
- No risk of corrupted or broken PDF files
|
||||||
- Split files are processed sequentially to prevent overwhelming the system
|
- Split files are processed sequentially to prevent overwhelming the system
|
||||||
- Clear error messages referencing SECURITY_AUDIT.md for configuration details
|
- Clear error messages referencing SECURITY_AUDIT.md for configuration details
|
||||||
|
|
||||||
|
|||||||
@@ -19,8 +19,16 @@ def split_pdf_by_size(pdf_path: str, max_size_bytes: int, output_dir: Optional[s
|
|||||||
"""
|
"""
|
||||||
Split a PDF file into multiple smaller PDF files based on size constraints.
|
Split a PDF file into multiple smaller PDF files based on size constraints.
|
||||||
|
|
||||||
The function splits the PDF by distributing pages across multiple output files,
|
IMPORTANT: This function splits PDFs at PAGE BOUNDARIES, not by byte position.
|
||||||
ensuring each output file stays under the specified size limit.
|
It uses PyPDF2 to properly parse the PDF structure and distribute complete pages
|
||||||
|
across multiple valid output PDFs. This ensures:
|
||||||
|
- All output files are structurally valid and readable PDFs
|
||||||
|
- No corrupted or broken PDF files are created
|
||||||
|
- Each output file contains complete pages from the original document
|
||||||
|
|
||||||
|
The function adds pages to the current output file until adding another page
|
||||||
|
would exceed the size limit, then starts a new output file. This is NOT a
|
||||||
|
simple byte-level file split - it respects PDF page boundaries.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
pdf_path: Path to the PDF file to split
|
pdf_path: Path to the PDF file to split
|
||||||
@@ -28,15 +36,18 @@ def split_pdf_by_size(pdf_path: str, max_size_bytes: int, output_dir: Optional[s
|
|||||||
output_dir: Directory to save split files. If None, uses same directory as input file.
|
output_dir: Directory to save split files. If None, uses same directory as input file.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of paths to the generated PDF files (in order)
|
List of paths to the generated PDF files (in order). Each file is a valid PDF
|
||||||
|
containing complete pages from the original document.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
FileNotFoundError: If the input PDF file doesn't exist
|
FileNotFoundError: If the input PDF file doesn't exist
|
||||||
ValueError: If max_size_bytes is too small to fit even one page
|
ValueError: If max_size_bytes is too small to fit even one page
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
>>> split_files = split_pdf_by_size("large.pdf", 50 * 1024 * 1024) # 50MB max
|
>>> # Split a large PDF into chunks of max 50MB each
|
||||||
|
>>> split_files = split_pdf_by_size("large.pdf", 50 * 1024 * 1024)
|
||||||
>>> print(f"Split into {len(split_files)} files")
|
>>> print(f"Split into {len(split_files)} files")
|
||||||
|
>>> # Each file is a valid PDF with complete pages
|
||||||
"""
|
"""
|
||||||
# Validate input file exists
|
# Validate input file exists
|
||||||
if not os.path.exists(pdf_path):
|
if not os.path.exists(pdf_path):
|
||||||
|
|||||||
@@ -57,6 +57,11 @@ MAX_SINGLE_FILE_SIZE=524288000
|
|||||||
|
|
||||||
**File Splitting Behavior:**
|
**File Splitting Behavior:**
|
||||||
- When `MAX_SINGLE_FILE_SIZE` is configured and a PDF exceeds this size, it is automatically split into smaller chunks
|
- When `MAX_SINGLE_FILE_SIZE` is configured and a PDF exceeds this size, it is automatically split into smaller chunks
|
||||||
|
- **IMPORTANT:** Splitting is done at **PAGE BOUNDARIES**, not by byte position
|
||||||
|
- Uses PyPDF2 to properly parse PDF structure
|
||||||
|
- Each output file is a complete, valid PDF containing whole pages
|
||||||
|
- No risk of corrupted or broken PDF files
|
||||||
|
- Pages are distributed across output files to stay under size limit
|
||||||
- Each chunk is processed sequentially as a separate task
|
- Each chunk is processed sequentially as a separate task
|
||||||
- Only works for PDF files (images and office documents are converted to PDF first)
|
- Only works for PDF files (images and office documents are converted to PDF first)
|
||||||
- Original file is removed after successful splitting
|
- Original file is removed after successful splitting
|
||||||
|
|||||||
@@ -181,6 +181,44 @@ class TestSplitPdfBySize:
|
|||||||
if os.path.exists(split_file):
|
if os.path.exists(split_file):
|
||||||
os.remove(split_file)
|
os.remove(split_file)
|
||||||
|
|
||||||
|
def test_split_pdfs_are_valid_and_readable(self, sample_multipage_pdf):
|
||||||
|
"""Test that split PDFs are valid, complete PDFs that can be opened and read.
|
||||||
|
|
||||||
|
This test verifies that PDF splitting is done at PAGE BOUNDARIES,
|
||||||
|
not by byte position, ensuring no corrupted/broken PDFs are created.
|
||||||
|
"""
|
||||||
|
max_size = 5000 # Small size to force splitting
|
||||||
|
|
||||||
|
split_files = split_pdf_by_size(sample_multipage_pdf, max_size)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Verify each split file is a valid, readable PDF
|
||||||
|
for split_file in split_files:
|
||||||
|
assert os.path.exists(split_file), f"Split file {split_file} should exist"
|
||||||
|
|
||||||
|
# Try to open and read the PDF - this will fail if PDF is corrupted
|
||||||
|
try:
|
||||||
|
reader = PdfReader(split_file)
|
||||||
|
# Verify it has pages (not an empty or broken PDF)
|
||||||
|
assert len(reader.pages) > 0, f"Split PDF {split_file} should have pages"
|
||||||
|
|
||||||
|
# Try to access first page content to ensure PDF structure is valid
|
||||||
|
first_page = reader.pages[0]
|
||||||
|
# If the PDF was corrupted by byte-splitting, this would raise an error
|
||||||
|
_ = first_page.extract_text() # This validates PDF structure
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
pytest.fail(
|
||||||
|
f"Split PDF {split_file} is corrupted or unreadable. "
|
||||||
|
f"This indicates byte-level splitting instead of page-level splitting. Error: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Cleanup
|
||||||
|
for split_file in split_files:
|
||||||
|
if os.path.exists(split_file):
|
||||||
|
os.remove(split_file)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
class TestShouldSplitFile:
|
class TestShouldSplitFile:
|
||||||
|
|||||||
Reference in New Issue
Block a user