feat(pdfa): add PDF/A archival conversion using ocrmypdf
- Add ENABLE_PDFA_CONVERSION, PDFA_FORMAT, PDFA_UPLOAD_TO_PROVIDERS config settings - Add original_pdfa_path and processed_pdfa_path columns to FileRecord model - Create Alembic migration 011_add_pdfa_paths - Create app/tasks/convert_to_pdfa.py Celery task using ocrmypdf + Ghostscript - Integrate PDF/A conversion into finalize_document_storage pipeline - Add comprehensive unit tests (15 tests) - Update .env.demo and docs/ConfigurationGuide.md Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -346,4 +346,16 @@ ENABLE_DEDUPLICATION=True
|
||||
SHOW_DEDUPLICATION_STEP=True
|
||||
# Minimum cosine similarity score (0–1) for two documents to be flagged as
|
||||
# near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches.
|
||||
NEAR_DUPLICATE_THRESHOLD=0.85
|
||||
NEAR_DUPLICATE_THRESHOLD=0.85
|
||||
|
||||
# **PDF/A Archival Conversion**
|
||||
# When enabled, PDF/A copies of both the original ingested file and the processed
|
||||
# file are created and saved alongside the standard copies. This may double or
|
||||
# triple storage but provides better legal coverage with time-stamped archival copies.
|
||||
# Uses ocrmypdf with Ghostscript for the conversion.
|
||||
ENABLE_PDFA_CONVERSION=false
|
||||
# PDF/A format variant: 1 = PDF/A-1b, 2 = PDF/A-2b (default), 3 = PDF/A-3b
|
||||
PDFA_FORMAT=2
|
||||
# Also upload the processed PDF/A variant to all configured storage providers
|
||||
# (files are uploaded with a '-PDFA' suffix in their filename)
|
||||
PDFA_UPLOAD_TO_PROVIDERS=false
|
||||
|
||||
@@ -222,6 +222,33 @@ class Settings(BaseSettings):
|
||||
|
||||
# Feature flags
|
||||
allow_file_delete: bool = True # Default to allowing file deletion from database
|
||||
|
||||
# PDF/A archival conversion settings
|
||||
enable_pdfa_conversion: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Enable PDF/A archival variant generation. When enabled, PDF/A copies of both the "
|
||||
"original ingested file and the processed file are created and saved alongside the "
|
||||
"standard copies. Uses ocrmypdf with Ghostscript for the conversion. "
|
||||
"This may double or triple storage but provides better legal coverage. Default: False."
|
||||
),
|
||||
)
|
||||
pdfa_format: str = Field(
|
||||
default="2",
|
||||
description=(
|
||||
"PDF/A format variant to produce. Passed to ocrmypdf --output-type pdfa-N. "
|
||||
"Valid values: '1' (PDF/A-1b), '2' (PDF/A-2b), '3' (PDF/A-3b). Default: '2'."
|
||||
),
|
||||
)
|
||||
pdfa_upload_to_providers: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When enabled and PDF/A conversion is active, also upload the PDF/A variants "
|
||||
"to all configured storage providers in addition to the standard processed file. "
|
||||
"PDF/A files are uploaded with a '-PDFA' suffix in their filename. Default: False."
|
||||
),
|
||||
)
|
||||
|
||||
imap_readonly_mode: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
|
||||
@@ -67,6 +67,10 @@ class FileRecord(Base):
|
||||
# Human-readable document title from AI metadata
|
||||
document_title = Column(String, nullable=True)
|
||||
|
||||
# PDF/A archival variant paths (generated when ENABLE_PDFA_CONVERSION is True)
|
||||
original_pdfa_path = Column(String, nullable=True) # PDF/A copy of the original ingested file
|
||||
processed_pdfa_path = Column(String, nullable=True) # PDF/A copy of the processed file
|
||||
|
||||
# Pre-computed text embedding vector stored as JSON array of floats
|
||||
embedding = Column(Text, nullable=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
"""PDF/A archival conversion task.
|
||||
|
||||
Converts PDF files to PDF/A format using ocrmypdf (which relies on Ghostscript
|
||||
internally). Two variants are produced when enabled:
|
||||
|
||||
1. **Original PDF/A** – an archival copy of the ingested file, providing a
|
||||
time-stamped record of the document as it was upon ingestion.
|
||||
2. **Processed PDF/A** – an archival copy of the processed file with embedded
|
||||
metadata.
|
||||
|
||||
Both are saved under ``workdir/pdfa/`` and referenced in the database via
|
||||
``FileRecord.original_pdfa_path`` and ``FileRecord.processed_pdfa_path``.
|
||||
|
||||
.. note::
|
||||
|
||||
PDF/A conversion may alter font rendering (especially OCR text overlays
|
||||
produced by Microsoft Azure Document Intelligence). This is expected –
|
||||
the PDF/A copies are parallel archival variants, not replacements.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import get_unique_filepath_with_counter, log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectory structure under workdir for PDF/A copies
|
||||
PDFA_ORIGINAL_SUBDIR = os.path.join("pdfa", "original")
|
||||
PDFA_PROCESSED_SUBDIR = os.path.join("pdfa", "processed")
|
||||
|
||||
|
||||
def _convert_pdf_to_pdfa(input_path: str, output_path: str, pdfa_format: str = "2") -> bool:
|
||||
"""Convert a PDF file to PDF/A using ocrmypdf.
|
||||
|
||||
Uses ``ocrmypdf --skip-text --output-type pdfa-N`` so that existing text
|
||||
layers are preserved (not re-OCR'd) while the output is converted to
|
||||
PDF/A via Ghostscript.
|
||||
|
||||
Args:
|
||||
input_path: Absolute path to the source PDF file.
|
||||
output_path: Absolute path for the PDF/A output file.
|
||||
pdfa_format: PDF/A variant ('1', '2', or '3'). Defaults to '2' for PDF/A-2b.
|
||||
|
||||
Returns:
|
||||
True if conversion succeeded, False otherwise.
|
||||
"""
|
||||
ocrmypdf_bin = shutil.which("ocrmypdf")
|
||||
if not ocrmypdf_bin:
|
||||
logger.error("[convert_to_pdfa] ocrmypdf binary not found on PATH")
|
||||
return False
|
||||
|
||||
output_type = f"pdfa-{pdfa_format}"
|
||||
|
||||
cmd = [
|
||||
ocrmypdf_bin,
|
||||
"--skip-text",
|
||||
"--output-type",
|
||||
output_type,
|
||||
"--quiet",
|
||||
"--invalidate-digital-signatures",
|
||||
input_path,
|
||||
output_path,
|
||||
]
|
||||
|
||||
logger.info(f"[convert_to_pdfa] Running: {' '.join(cmd)}")
|
||||
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600, check=False) # noqa: S603
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("[convert_to_pdfa] ocrmypdf timed out after 600s")
|
||||
return False
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr_snippet = proc.stderr.strip()[:500] if proc.stderr else ""
|
||||
logger.warning(f"[convert_to_pdfa] ocrmypdf exited with code {proc.returncode}: {stderr_snippet}")
|
||||
return False
|
||||
|
||||
logger.info(f"[convert_to_pdfa] PDF/A file written to {output_path}")
|
||||
return True
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def convert_to_pdfa(self, file_id: int) -> dict:
|
||||
"""Generate PDF/A archival copies for a processed document.
|
||||
|
||||
Creates PDF/A variants of both the original ingested file and the
|
||||
processed file (with embedded metadata). Files are saved under
|
||||
``workdir/pdfa/original/`` and ``workdir/pdfa/processed/`` respectively.
|
||||
|
||||
When ``settings.pdfa_upload_to_providers`` is True, the processed PDF/A
|
||||
variant is also uploaded to all configured storage destinations.
|
||||
|
||||
Args:
|
||||
file_id: ID of the FileRecord to create PDF/A copies for.
|
||||
|
||||
Returns:
|
||||
Dictionary with status and file paths.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting PDF/A conversion for file_id={file_id}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_to_pdfa",
|
||||
"in_progress",
|
||||
"Starting PDF/A archival conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Fetch file record
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if not file_record:
|
||||
logger.error(f"[{task_id}] FileRecord {file_id} not found")
|
||||
log_task_progress(task_id, "convert_to_pdfa", "failure", "File record not found", file_id=file_id)
|
||||
return {"error": "File record not found", "file_id": file_id}
|
||||
|
||||
original_path = file_record.original_file_path
|
||||
processed_path = file_record.processed_file_path
|
||||
|
||||
pdfa_format = settings.pdfa_format
|
||||
results = {}
|
||||
|
||||
# --- Convert original file to PDF/A ---
|
||||
if original_path and os.path.exists(original_path):
|
||||
original_pdfa_dir = os.path.join(settings.workdir, PDFA_ORIGINAL_SUBDIR)
|
||||
os.makedirs(original_pdfa_dir, exist_ok=True)
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(original_path))[0]
|
||||
original_pdfa_path = get_unique_filepath_with_counter(original_pdfa_dir, base_name, ".pdf")
|
||||
|
||||
logger.info(f"[{task_id}] Converting original to PDF/A: {original_path} -> {original_pdfa_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"in_progress",
|
||||
f"Converting original to PDF/A: {os.path.basename(original_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
success = _convert_pdf_to_pdfa(original_path, original_pdfa_path, pdfa_format)
|
||||
if success:
|
||||
results["original_pdfa_path"] = original_pdfa_path
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"success",
|
||||
f"Original PDF/A saved: {os.path.basename(original_pdfa_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"failure",
|
||||
"Failed to convert original to PDF/A",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"[{task_id}] Original file not found, skipping original PDF/A conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"skipped",
|
||||
"Original file not available",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Convert processed file to PDF/A ---
|
||||
if processed_path and os.path.exists(processed_path):
|
||||
processed_pdfa_dir = os.path.join(settings.workdir, PDFA_PROCESSED_SUBDIR)
|
||||
os.makedirs(processed_pdfa_dir, exist_ok=True)
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(processed_path))[0]
|
||||
processed_pdfa_path = get_unique_filepath_with_counter(processed_pdfa_dir, f"{base_name}-PDFA", ".pdf")
|
||||
|
||||
logger.info(f"[{task_id}] Converting processed to PDF/A: {processed_path} -> {processed_pdfa_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"in_progress",
|
||||
f"Converting processed to PDF/A: {os.path.basename(processed_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
success = _convert_pdf_to_pdfa(processed_path, processed_pdfa_path, pdfa_format)
|
||||
if success:
|
||||
results["processed_pdfa_path"] = processed_pdfa_path
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"success",
|
||||
f"Processed PDF/A saved: {os.path.basename(processed_pdfa_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"failure",
|
||||
"Failed to convert processed to PDF/A",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"[{task_id}] Processed file not found, skipping processed PDF/A conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"skipped",
|
||||
"Processed file not available",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Update database with PDF/A paths ---
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if file_record:
|
||||
if "original_pdfa_path" in results:
|
||||
file_record.original_pdfa_path = results["original_pdfa_path"]
|
||||
if "processed_pdfa_path" in results:
|
||||
file_record.processed_pdfa_path = results["processed_pdfa_path"]
|
||||
db.commit()
|
||||
logger.info(f"[{task_id}] Updated database with PDF/A paths")
|
||||
|
||||
# --- Optionally upload processed PDF/A to storage providers ---
|
||||
if settings.pdfa_upload_to_providers and "processed_pdfa_path" in results:
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
logger.info(f"[{task_id}] Uploading processed PDF/A to storage providers")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_pdfa_to_providers",
|
||||
"in_progress",
|
||||
"Uploading PDF/A variant to storage providers",
|
||||
file_id=file_id,
|
||||
)
|
||||
send_to_all_destinations.delay(results["processed_pdfa_path"], True, file_id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_pdfa_to_providers",
|
||||
"success",
|
||||
"PDF/A variant queued for upload",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Final status ---
|
||||
has_any = bool(results)
|
||||
status = "success" if has_any else "failure"
|
||||
message = (
|
||||
f"PDF/A conversion complete ({len(results)} variant(s) created)" if has_any else "No PDF/A variants created"
|
||||
)
|
||||
log_task_progress(task_id, "convert_to_pdfa", status, message, file_id=file_id)
|
||||
|
||||
return {"status": status, "file_id": file_id, **results}
|
||||
@@ -76,6 +76,20 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
# We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
|
||||
send_to_all_destinations.delay(processed_file, True, file_id)
|
||||
|
||||
# 3a. Trigger PDF/A archival conversion if enabled
|
||||
if settings.enable_pdfa_conversion:
|
||||
from app.tasks.convert_to_pdfa import convert_to_pdfa
|
||||
|
||||
logger.info(f"[{task_id}] PDF/A conversion enabled, queueing archival conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"finalize_document_storage",
|
||||
"in_progress",
|
||||
"Queueing PDF/A archival conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
convert_to_pdfa.delay(file_id)
|
||||
|
||||
# 4. Send Notification (From Copilot)
|
||||
# Note: This notification is sent after processing is complete but while uploads
|
||||
# are being queued.
|
||||
|
||||
@@ -876,6 +876,56 @@ Near-duplicate detection:
|
||||
|
||||
A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity.
|
||||
|
||||
## PDF/A Archival Conversion
|
||||
|
||||
DocuElevate can optionally generate **PDF/A** archival copies of both the
|
||||
original ingested file and the processed file. PDF/A copies are saved as
|
||||
parallel variants alongside the standard files—they do **not** replace the
|
||||
originals. This provides better legal coverage by producing time-stamped,
|
||||
self-contained archival documents suitable for long-term storage and
|
||||
compliance.
|
||||
|
||||
The conversion uses **ocrmypdf** (backed by Ghostscript), which is already
|
||||
bundled in the Docker images.
|
||||
|
||||
> **Note:** PDF/A conversion may alter font rendering, especially for OCR text
|
||||
> overlays produced by Microsoft Azure Document Intelligence. This is expected
|
||||
> and is why PDF/A copies are kept as parallel variants rather than
|
||||
> replacements.
|
||||
|
||||
| Variable | Description | Default |
|
||||
|-----------------------------|------------------------------------------------------------------------------------------------------|---------|
|
||||
| `ENABLE_PDFA_CONVERSION` | Enable PDF/A archival variant generation for both original and processed files. | `false` |
|
||||
| `PDFA_FORMAT` | PDF/A format variant: `1` (PDF/A-1b), `2` (PDF/A-2b), `3` (PDF/A-3b). | `2` |
|
||||
| `PDFA_UPLOAD_TO_PROVIDERS` | Also upload the processed PDF/A variant to all configured storage providers (with `-PDFA` suffix). | `false` |
|
||||
|
||||
### Storage Layout
|
||||
|
||||
When enabled, PDF/A copies are stored under `workdir/pdfa/`:
|
||||
|
||||
```
|
||||
workdir/
|
||||
├── original/ # Immutable copy of ingested file
|
||||
├── processed/ # Processed file with embedded metadata
|
||||
├── pdfa/
|
||||
│ ├── original/ # PDF/A copy of the ingested file
|
||||
│ └── processed/ # PDF/A copy of the processed file (with -PDFA suffix)
|
||||
└── tmp/ # Temporary processing area
|
||||
```
|
||||
|
||||
### Configuration Example
|
||||
|
||||
```bash
|
||||
# Enable PDF/A archival copies
|
||||
ENABLE_PDFA_CONVERSION=true
|
||||
|
||||
# Use PDF/A-2b format (default, recommended for most use cases)
|
||||
PDFA_FORMAT=2
|
||||
|
||||
# Also upload PDF/A copies to configured storage providers
|
||||
PDFA_UPLOAD_TO_PROVIDERS=true
|
||||
```
|
||||
|
||||
## Performance & Caching
|
||||
|
||||
DocuElevate automatically optimizes database access and uses Redis as a
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Add PDF/A archival variant path columns to files table
|
||||
|
||||
Revision ID: 011_add_pdfa_paths
|
||||
Revises: 010_add_embedding_column
|
||||
Create Date: 2026-03-02
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "011_add_pdfa_paths"
|
||||
down_revision: Union[str, None] = "010_add_embedding_column"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add PDF/A variant path columns to files table."""
|
||||
op.add_column("files", sa.Column("original_pdfa_path", sa.String(), nullable=True))
|
||||
op.add_column("files", sa.Column("processed_pdfa_path", sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove PDF/A variant path columns from files table."""
|
||||
op.drop_column("files", "processed_pdfa_path")
|
||||
op.drop_column("files", "original_pdfa_path")
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Unit tests for app/tasks/convert_to_pdfa.py module."""
|
||||
|
||||
import subprocess
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.convert_to_pdfa import (
|
||||
PDFA_ORIGINAL_SUBDIR,
|
||||
PDFA_PROCESSED_SUBDIR,
|
||||
_convert_pdf_to_pdfa,
|
||||
convert_to_pdfa,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestConvertPdfToPdfa:
|
||||
"""Tests for the _convert_pdf_to_pdfa helper function."""
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.subprocess.run")
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf")
|
||||
def test_successful_conversion(self, mock_which, mock_run):
|
||||
"""Test successful PDF to PDF/A conversion."""
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr="")
|
||||
|
||||
result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf", "2")
|
||||
|
||||
assert result is True
|
||||
mock_which.assert_called_once_with("ocrmypdf")
|
||||
mock_run.assert_called_once()
|
||||
|
||||
# Verify the command arguments
|
||||
cmd = mock_run.call_args[0][0]
|
||||
assert cmd[0] == "/usr/bin/ocrmypdf"
|
||||
assert "--skip-text" in cmd
|
||||
assert "--output-type" in cmd
|
||||
assert "pdfa-2" in cmd
|
||||
assert "--quiet" in cmd
|
||||
assert "--invalidate-digital-signatures" in cmd
|
||||
assert "/input.pdf" in cmd
|
||||
assert "/output.pdf" in cmd
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value=None)
|
||||
def test_ocrmypdf_not_found(self, mock_which):
|
||||
"""Test returns False when ocrmypdf binary is not on PATH."""
|
||||
result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf")
|
||||
|
||||
assert result is False
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.subprocess.run")
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf")
|
||||
def test_conversion_failure(self, mock_which, mock_run):
|
||||
"""Test returns False when ocrmypdf exits with non-zero code."""
|
||||
mock_run.return_value = MagicMock(returncode=1, stderr="Some error occurred")
|
||||
|
||||
result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf")
|
||||
|
||||
assert result is False
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.subprocess.run")
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf")
|
||||
def test_conversion_timeout(self, mock_which, mock_run):
|
||||
"""Test returns False when ocrmypdf times out."""
|
||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd="ocrmypdf", timeout=600)
|
||||
|
||||
result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf")
|
||||
|
||||
assert result is False
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.subprocess.run")
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf")
|
||||
def test_pdfa_format_variants(self, mock_which, mock_run):
|
||||
"""Test different PDF/A format variants are passed correctly."""
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr="")
|
||||
|
||||
for fmt in ("1", "2", "3"):
|
||||
_convert_pdf_to_pdfa("/input.pdf", "/output.pdf", fmt)
|
||||
cmd = mock_run.call_args[0][0]
|
||||
assert f"pdfa-{fmt}" in cmd
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.subprocess.run")
|
||||
@patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf")
|
||||
def test_conversion_failure_empty_stderr(self, mock_which, mock_run):
|
||||
"""Test handles empty stderr on failure."""
|
||||
mock_run.return_value = MagicMock(returncode=2, stderr="")
|
||||
|
||||
result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf")
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestConvertToPdfaTask:
|
||||
"""Tests for the convert_to_pdfa Celery task."""
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter")
|
||||
@patch("app.tasks.convert_to_pdfa.os.makedirs")
|
||||
@patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_successful_conversion_both_files(
|
||||
self,
|
||||
mock_session_local,
|
||||
mock_log,
|
||||
mock_exists,
|
||||
mock_makedirs,
|
||||
mock_unique_path,
|
||||
mock_convert,
|
||||
mock_settings,
|
||||
):
|
||||
"""Test successful PDF/A conversion of both original and processed files."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = False
|
||||
|
||||
# Mock unique path to return predictable paths
|
||||
mock_unique_path.side_effect = [
|
||||
"/workdir/pdfa/original/test.pdf",
|
||||
"/workdir/pdfa/processed/test-PDFA.pdf",
|
||||
]
|
||||
|
||||
# Mock database session
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = "/workdir/original/test.pdf"
|
||||
mock_record.processed_file_path = "/workdir/processed/test.pdf"
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=1)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["file_id"] == 1
|
||||
assert "original_pdfa_path" in result
|
||||
assert "processed_pdfa_path" in result
|
||||
assert mock_convert.call_count == 2
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_file_record_not_found(self, mock_session_local, mock_log, mock_settings):
|
||||
"""Test returns error when file record is not found."""
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = None
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=999)
|
||||
|
||||
assert "error" in result
|
||||
assert result["file_id"] == 999
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=False)
|
||||
@patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter")
|
||||
@patch("app.tasks.convert_to_pdfa.os.makedirs")
|
||||
@patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_conversion_failure_both_files(
|
||||
self,
|
||||
mock_session_local,
|
||||
mock_log,
|
||||
mock_exists,
|
||||
mock_makedirs,
|
||||
mock_unique_path,
|
||||
mock_convert,
|
||||
mock_settings,
|
||||
):
|
||||
"""Test handles failure when both conversions fail."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = False
|
||||
|
||||
mock_unique_path.side_effect = [
|
||||
"/workdir/pdfa/original/test.pdf",
|
||||
"/workdir/pdfa/processed/test-PDFA.pdf",
|
||||
]
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = "/workdir/original/test.pdf"
|
||||
mock_record.processed_file_path = "/workdir/processed/test.pdf"
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=1)
|
||||
|
||||
assert result["status"] == "failure"
|
||||
assert "original_pdfa_path" not in result
|
||||
assert "processed_pdfa_path" not in result
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_skips_missing_files(self, mock_session_local, mock_log, mock_settings):
|
||||
"""Test skips conversion when original/processed files don't exist."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = False
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = None
|
||||
mock_record.processed_file_path = None
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=1)
|
||||
|
||||
assert result["status"] == "failure"
|
||||
|
||||
@patch("app.tasks.send_to_all.send_to_all_destinations")
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter")
|
||||
@patch("app.tasks.convert_to_pdfa.os.makedirs")
|
||||
@patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_uploads_pdfa_when_enabled(
|
||||
self,
|
||||
mock_session_local,
|
||||
mock_log,
|
||||
mock_exists,
|
||||
mock_makedirs,
|
||||
mock_unique_path,
|
||||
mock_convert,
|
||||
mock_settings,
|
||||
mock_send_all,
|
||||
):
|
||||
"""Test uploads processed PDF/A to providers when pdfa_upload_to_providers is True."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = True
|
||||
|
||||
mock_unique_path.side_effect = [
|
||||
"/workdir/pdfa/original/test.pdf",
|
||||
"/workdir/pdfa/processed/test-PDFA.pdf",
|
||||
]
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = "/workdir/original/test.pdf"
|
||||
mock_record.processed_file_path = "/workdir/processed/test.pdf"
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=42)
|
||||
|
||||
# Verify send_to_all_destinations was called for the PDF/A file
|
||||
mock_send_all.delay.assert_called_once_with("/workdir/pdfa/processed/test-PDFA.pdf", True, 42)
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter")
|
||||
@patch("app.tasks.convert_to_pdfa.os.makedirs")
|
||||
@patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True)
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_does_not_upload_when_disabled(
|
||||
self,
|
||||
mock_session_local,
|
||||
mock_log,
|
||||
mock_exists,
|
||||
mock_makedirs,
|
||||
mock_unique_path,
|
||||
mock_convert,
|
||||
mock_settings,
|
||||
):
|
||||
"""Test does not upload PDF/A when pdfa_upload_to_providers is False."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = False
|
||||
|
||||
mock_unique_path.side_effect = [
|
||||
"/workdir/pdfa/original/test.pdf",
|
||||
"/workdir/pdfa/processed/test-PDFA.pdf",
|
||||
]
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = "/workdir/original/test.pdf"
|
||||
mock_record.processed_file_path = "/workdir/processed/test.pdf"
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=1)
|
||||
|
||||
assert result["status"] == "success"
|
||||
|
||||
@patch("app.tasks.convert_to_pdfa.settings")
|
||||
@patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa")
|
||||
@patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter")
|
||||
@patch("app.tasks.convert_to_pdfa.os.makedirs")
|
||||
@patch("app.tasks.convert_to_pdfa.os.path.exists")
|
||||
@patch("app.tasks.convert_to_pdfa.log_task_progress")
|
||||
@patch("app.tasks.convert_to_pdfa.SessionLocal")
|
||||
def test_partial_success_original_only(
|
||||
self,
|
||||
mock_session_local,
|
||||
mock_log,
|
||||
mock_exists,
|
||||
mock_makedirs,
|
||||
mock_unique_path,
|
||||
mock_convert,
|
||||
mock_settings,
|
||||
):
|
||||
"""Test partial success when only original conversion succeeds."""
|
||||
mock_settings.workdir = "/workdir"
|
||||
mock_settings.pdfa_format = "2"
|
||||
mock_settings.pdfa_upload_to_providers = False
|
||||
|
||||
# Only original file exists
|
||||
def exists_side_effect(path):
|
||||
return "/original/" in path
|
||||
|
||||
mock_exists.side_effect = exists_side_effect
|
||||
mock_unique_path.return_value = "/workdir/pdfa/original/test.pdf"
|
||||
mock_convert.return_value = True
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_session_local.return_value.__enter__.return_value = mock_db
|
||||
mock_record = MagicMock()
|
||||
mock_record.original_file_path = "/workdir/original/test.pdf"
|
||||
mock_record.processed_file_path = "/workdir/processed/test.pdf"
|
||||
mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record
|
||||
|
||||
convert_to_pdfa.request.id = "test-task-id"
|
||||
result = convert_to_pdfa.__wrapped__(file_id=1)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert "original_pdfa_path" in result
|
||||
assert "processed_pdfa_path" not in result
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestPdfaSubdirectoryConstants:
|
||||
"""Tests for PDF/A subdirectory constants."""
|
||||
|
||||
def test_original_subdir(self):
|
||||
"""Test PDFA_ORIGINAL_SUBDIR is correct."""
|
||||
assert "pdfa" in PDFA_ORIGINAL_SUBDIR
|
||||
assert "original" in PDFA_ORIGINAL_SUBDIR
|
||||
|
||||
def test_processed_subdir(self):
|
||||
"""Test PDFA_PROCESSED_SUBDIR is correct."""
|
||||
assert "pdfa" in PDFA_PROCESSED_SUBDIR
|
||||
assert "processed" in PDFA_PROCESSED_SUBDIR
|
||||
@@ -107,6 +107,7 @@ class TestFinalizeDocumentStorage:
|
||||
):
|
||||
with patch("app.tasks.finalize_document_storage.settings") as mock_settings:
|
||||
mock_settings.workdir = "/tmp"
|
||||
mock_settings.enable_pdfa_conversion = False
|
||||
|
||||
finalize_document_storage.request.id = "test-task-id"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user