feat(pdfa): add PDF/A archival conversion using ocrmypdf
- Add ENABLE_PDFA_CONVERSION, PDFA_FORMAT, PDFA_UPLOAD_TO_PROVIDERS config settings - Add original_pdfa_path and processed_pdfa_path columns to FileRecord model - Create Alembic migration 011_add_pdfa_paths - Create app/tasks/convert_to_pdfa.py Celery task using ocrmypdf + Ghostscript - Integrate PDF/A conversion into finalize_document_storage pipeline - Add comprehensive unit tests (15 tests) - Update .env.demo and docs/ConfigurationGuide.md Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -222,6 +222,33 @@ class Settings(BaseSettings):
|
||||
|
||||
# Feature flags
|
||||
allow_file_delete: bool = True # Default to allowing file deletion from database
|
||||
|
||||
# PDF/A archival conversion settings
|
||||
enable_pdfa_conversion: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Enable PDF/A archival variant generation. When enabled, PDF/A copies of both the "
|
||||
"original ingested file and the processed file are created and saved alongside the "
|
||||
"standard copies. Uses ocrmypdf with Ghostscript for the conversion. "
|
||||
"This may double or triple storage but provides better legal coverage. Default: False."
|
||||
),
|
||||
)
|
||||
pdfa_format: str = Field(
|
||||
default="2",
|
||||
description=(
|
||||
"PDF/A format variant to produce. Passed to ocrmypdf --output-type pdfa-N. "
|
||||
"Valid values: '1' (PDF/A-1b), '2' (PDF/A-2b), '3' (PDF/A-3b). Default: '2'."
|
||||
),
|
||||
)
|
||||
pdfa_upload_to_providers: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When enabled and PDF/A conversion is active, also upload the PDF/A variants "
|
||||
"to all configured storage providers in addition to the standard processed file. "
|
||||
"PDF/A files are uploaded with a '-PDFA' suffix in their filename. Default: False."
|
||||
),
|
||||
)
|
||||
|
||||
imap_readonly_mode: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
|
||||
@@ -67,6 +67,10 @@ class FileRecord(Base):
|
||||
# Human-readable document title from AI metadata
|
||||
document_title = Column(String, nullable=True)
|
||||
|
||||
# PDF/A archival variant paths (generated when ENABLE_PDFA_CONVERSION is True)
|
||||
original_pdfa_path = Column(String, nullable=True) # PDF/A copy of the original ingested file
|
||||
processed_pdfa_path = Column(String, nullable=True) # PDF/A copy of the processed file
|
||||
|
||||
# Pre-computed text embedding vector stored as JSON array of floats
|
||||
embedding = Column(Text, nullable=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
"""PDF/A archival conversion task.
|
||||
|
||||
Converts PDF files to PDF/A format using ocrmypdf (which relies on Ghostscript
|
||||
internally). Two variants are produced when enabled:
|
||||
|
||||
1. **Original PDF/A** – an archival copy of the ingested file, providing a
|
||||
time-stamped record of the document as it was upon ingestion.
|
||||
2. **Processed PDF/A** – an archival copy of the processed file with embedded
|
||||
metadata.
|
||||
|
||||
Both are saved under ``workdir/pdfa/`` and referenced in the database via
|
||||
``FileRecord.original_pdfa_path`` and ``FileRecord.processed_pdfa_path``.
|
||||
|
||||
.. note::
|
||||
|
||||
PDF/A conversion may alter font rendering (especially OCR text overlays
|
||||
produced by Microsoft Azure Document Intelligence). This is expected –
|
||||
the PDF/A copies are parallel archival variants, not replacements.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import get_unique_filepath_with_counter, log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectory structure under workdir for PDF/A copies
|
||||
PDFA_ORIGINAL_SUBDIR = os.path.join("pdfa", "original")
|
||||
PDFA_PROCESSED_SUBDIR = os.path.join("pdfa", "processed")
|
||||
|
||||
|
||||
def _convert_pdf_to_pdfa(input_path: str, output_path: str, pdfa_format: str = "2") -> bool:
|
||||
"""Convert a PDF file to PDF/A using ocrmypdf.
|
||||
|
||||
Uses ``ocrmypdf --skip-text --output-type pdfa-N`` so that existing text
|
||||
layers are preserved (not re-OCR'd) while the output is converted to
|
||||
PDF/A via Ghostscript.
|
||||
|
||||
Args:
|
||||
input_path: Absolute path to the source PDF file.
|
||||
output_path: Absolute path for the PDF/A output file.
|
||||
pdfa_format: PDF/A variant ('1', '2', or '3'). Defaults to '2' for PDF/A-2b.
|
||||
|
||||
Returns:
|
||||
True if conversion succeeded, False otherwise.
|
||||
"""
|
||||
ocrmypdf_bin = shutil.which("ocrmypdf")
|
||||
if not ocrmypdf_bin:
|
||||
logger.error("[convert_to_pdfa] ocrmypdf binary not found on PATH")
|
||||
return False
|
||||
|
||||
output_type = f"pdfa-{pdfa_format}"
|
||||
|
||||
cmd = [
|
||||
ocrmypdf_bin,
|
||||
"--skip-text",
|
||||
"--output-type",
|
||||
output_type,
|
||||
"--quiet",
|
||||
"--invalidate-digital-signatures",
|
||||
input_path,
|
||||
output_path,
|
||||
]
|
||||
|
||||
logger.info(f"[convert_to_pdfa] Running: {' '.join(cmd)}")
|
||||
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600, check=False) # noqa: S603
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("[convert_to_pdfa] ocrmypdf timed out after 600s")
|
||||
return False
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr_snippet = proc.stderr.strip()[:500] if proc.stderr else ""
|
||||
logger.warning(f"[convert_to_pdfa] ocrmypdf exited with code {proc.returncode}: {stderr_snippet}")
|
||||
return False
|
||||
|
||||
logger.info(f"[convert_to_pdfa] PDF/A file written to {output_path}")
|
||||
return True
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def convert_to_pdfa(self, file_id: int) -> dict:
|
||||
"""Generate PDF/A archival copies for a processed document.
|
||||
|
||||
Creates PDF/A variants of both the original ingested file and the
|
||||
processed file (with embedded metadata). Files are saved under
|
||||
``workdir/pdfa/original/`` and ``workdir/pdfa/processed/`` respectively.
|
||||
|
||||
When ``settings.pdfa_upload_to_providers`` is True, the processed PDF/A
|
||||
variant is also uploaded to all configured storage destinations.
|
||||
|
||||
Args:
|
||||
file_id: ID of the FileRecord to create PDF/A copies for.
|
||||
|
||||
Returns:
|
||||
Dictionary with status and file paths.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting PDF/A conversion for file_id={file_id}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_to_pdfa",
|
||||
"in_progress",
|
||||
"Starting PDF/A archival conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Fetch file record
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if not file_record:
|
||||
logger.error(f"[{task_id}] FileRecord {file_id} not found")
|
||||
log_task_progress(task_id, "convert_to_pdfa", "failure", "File record not found", file_id=file_id)
|
||||
return {"error": "File record not found", "file_id": file_id}
|
||||
|
||||
original_path = file_record.original_file_path
|
||||
processed_path = file_record.processed_file_path
|
||||
|
||||
pdfa_format = settings.pdfa_format
|
||||
results = {}
|
||||
|
||||
# --- Convert original file to PDF/A ---
|
||||
if original_path and os.path.exists(original_path):
|
||||
original_pdfa_dir = os.path.join(settings.workdir, PDFA_ORIGINAL_SUBDIR)
|
||||
os.makedirs(original_pdfa_dir, exist_ok=True)
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(original_path))[0]
|
||||
original_pdfa_path = get_unique_filepath_with_counter(original_pdfa_dir, base_name, ".pdf")
|
||||
|
||||
logger.info(f"[{task_id}] Converting original to PDF/A: {original_path} -> {original_pdfa_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"in_progress",
|
||||
f"Converting original to PDF/A: {os.path.basename(original_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
success = _convert_pdf_to_pdfa(original_path, original_pdfa_path, pdfa_format)
|
||||
if success:
|
||||
results["original_pdfa_path"] = original_pdfa_path
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"success",
|
||||
f"Original PDF/A saved: {os.path.basename(original_pdfa_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"failure",
|
||||
"Failed to convert original to PDF/A",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"[{task_id}] Original file not found, skipping original PDF/A conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_original_to_pdfa",
|
||||
"skipped",
|
||||
"Original file not available",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Convert processed file to PDF/A ---
|
||||
if processed_path and os.path.exists(processed_path):
|
||||
processed_pdfa_dir = os.path.join(settings.workdir, PDFA_PROCESSED_SUBDIR)
|
||||
os.makedirs(processed_pdfa_dir, exist_ok=True)
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(processed_path))[0]
|
||||
processed_pdfa_path = get_unique_filepath_with_counter(processed_pdfa_dir, f"{base_name}-PDFA", ".pdf")
|
||||
|
||||
logger.info(f"[{task_id}] Converting processed to PDF/A: {processed_path} -> {processed_pdfa_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"in_progress",
|
||||
f"Converting processed to PDF/A: {os.path.basename(processed_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
success = _convert_pdf_to_pdfa(processed_path, processed_pdfa_path, pdfa_format)
|
||||
if success:
|
||||
results["processed_pdfa_path"] = processed_pdfa_path
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"success",
|
||||
f"Processed PDF/A saved: {os.path.basename(processed_pdfa_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"failure",
|
||||
"Failed to convert processed to PDF/A",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"[{task_id}] Processed file not found, skipping processed PDF/A conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"convert_processed_to_pdfa",
|
||||
"skipped",
|
||||
"Processed file not available",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Update database with PDF/A paths ---
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if file_record:
|
||||
if "original_pdfa_path" in results:
|
||||
file_record.original_pdfa_path = results["original_pdfa_path"]
|
||||
if "processed_pdfa_path" in results:
|
||||
file_record.processed_pdfa_path = results["processed_pdfa_path"]
|
||||
db.commit()
|
||||
logger.info(f"[{task_id}] Updated database with PDF/A paths")
|
||||
|
||||
# --- Optionally upload processed PDF/A to storage providers ---
|
||||
if settings.pdfa_upload_to_providers and "processed_pdfa_path" in results:
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
logger.info(f"[{task_id}] Uploading processed PDF/A to storage providers")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_pdfa_to_providers",
|
||||
"in_progress",
|
||||
"Uploading PDF/A variant to storage providers",
|
||||
file_id=file_id,
|
||||
)
|
||||
send_to_all_destinations.delay(results["processed_pdfa_path"], True, file_id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_pdfa_to_providers",
|
||||
"success",
|
||||
"PDF/A variant queued for upload",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# --- Final status ---
|
||||
has_any = bool(results)
|
||||
status = "success" if has_any else "failure"
|
||||
message = (
|
||||
f"PDF/A conversion complete ({len(results)} variant(s) created)" if has_any else "No PDF/A variants created"
|
||||
)
|
||||
log_task_progress(task_id, "convert_to_pdfa", status, message, file_id=file_id)
|
||||
|
||||
return {"status": status, "file_id": file_id, **results}
|
||||
@@ -76,6 +76,20 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
# We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
|
||||
send_to_all_destinations.delay(processed_file, True, file_id)
|
||||
|
||||
# 3a. Trigger PDF/A archival conversion if enabled
|
||||
if settings.enable_pdfa_conversion:
|
||||
from app.tasks.convert_to_pdfa import convert_to_pdfa
|
||||
|
||||
logger.info(f"[{task_id}] PDF/A conversion enabled, queueing archival conversion")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"finalize_document_storage",
|
||||
"in_progress",
|
||||
"Queueing PDF/A archival conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
convert_to_pdfa.delay(file_id)
|
||||
|
||||
# 4. Send Notification (From Copilot)
|
||||
# Note: This notification is sent after processing is complete but while uploads
|
||||
# are being queued.
|
||||
|
||||
Reference in New Issue
Block a user