diff --git a/.env.demo b/.env.demo index e06ab45b..b5b7d94f 100644 --- a/.env.demo +++ b/.env.demo @@ -346,4 +346,16 @@ ENABLE_DEDUPLICATION=True SHOW_DEDUPLICATION_STEP=True # Minimum cosine similarity score (0–1) for two documents to be flagged as # near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches. -NEAR_DUPLICATE_THRESHOLD=0.85 +NEAR_DUPLICATE_THRESHOLD=0.85 + +# **PDF/A Archival Conversion** +# When enabled, PDF/A copies of both the original ingested file and the processed +# file are created and saved alongside the standard copies. This may double or +# triple storage but provides better legal coverage with time-stamped archival copies. +# Uses ocrmypdf with Ghostscript for the conversion. +ENABLE_PDFA_CONVERSION=false +# PDF/A format variant: 1 = PDF/A-1b, 2 = PDF/A-2b (default), 3 = PDF/A-3b +PDFA_FORMAT=2 +# Also upload the processed PDF/A variant to all configured storage providers +# (files are uploaded with a '-PDFA' suffix in their filename) +PDFA_UPLOAD_TO_PROVIDERS=false diff --git a/app/config.py b/app/config.py index 5a4c2fd1..1c68024a 100644 --- a/app/config.py +++ b/app/config.py @@ -222,6 +222,33 @@ class Settings(BaseSettings): # Feature flags allow_file_delete: bool = True # Default to allowing file deletion from database + + # PDF/A archival conversion settings + enable_pdfa_conversion: bool = Field( + default=False, + description=( + "Enable PDF/A archival variant generation. When enabled, PDF/A copies of both the " + "original ingested file and the processed file are created and saved alongside the " + "standard copies. Uses ocrmypdf with Ghostscript for the conversion. " + "This may double or triple storage but provides better legal coverage. Default: False." + ), + ) + pdfa_format: str = Field( + default="2", + description=( + "PDF/A format variant to produce. Passed to ocrmypdf --output-type pdfa-N. " + "Valid values: '1' (PDF/A-1b), '2' (PDF/A-2b), '3' (PDF/A-3b). Default: '2'." + ), + ) + pdfa_upload_to_providers: bool = Field( + default=False, + description=( + "When enabled and PDF/A conversion is active, also upload the PDF/A variants " + "to all configured storage providers in addition to the standard processed file. " + "PDF/A files are uploaded with a '-PDFA' suffix in their filename. Default: False." + ), + ) + imap_readonly_mode: bool = Field( default=False, description=( diff --git a/app/models.py b/app/models.py index e09e3490..a14d5566 100644 --- a/app/models.py +++ b/app/models.py @@ -67,6 +67,10 @@ class FileRecord(Base): # Human-readable document title from AI metadata document_title = Column(String, nullable=True) + # PDF/A archival variant paths (generated when ENABLE_PDFA_CONVERSION is True) + original_pdfa_path = Column(String, nullable=True) # PDF/A copy of the original ingested file + processed_pdfa_path = Column(String, nullable=True) # PDF/A copy of the processed file + # Pre-computed text embedding vector stored as JSON array of floats embedding = Column(Text, nullable=True) diff --git a/app/tasks/convert_to_pdfa.py b/app/tasks/convert_to_pdfa.py new file mode 100644 index 00000000..c680851f --- /dev/null +++ b/app/tasks/convert_to_pdfa.py @@ -0,0 +1,261 @@ +"""PDF/A archival conversion task. + +Converts PDF files to PDF/A format using ocrmypdf (which relies on Ghostscript +internally). Two variants are produced when enabled: + +1. **Original PDF/A** – an archival copy of the ingested file, providing a + time-stamped record of the document as it was upon ingestion. +2. **Processed PDF/A** – an archival copy of the processed file with embedded + metadata. + +Both are saved under ``workdir/pdfa/`` and referenced in the database via +``FileRecord.original_pdfa_path`` and ``FileRecord.processed_pdfa_path``. + +.. note:: + + PDF/A conversion may alter font rendering (especially OCR text overlays + produced by Microsoft Azure Document Intelligence). This is expected – + the PDF/A copies are parallel archival variants, not replacements. +""" + +import logging +import os +import shutil +import subprocess + +from app.celery_app import celery +from app.config import settings +from app.database import SessionLocal +from app.models import FileRecord +from app.tasks.retry_config import BaseTaskWithRetry +from app.utils import get_unique_filepath_with_counter, log_task_progress + +logger = logging.getLogger(__name__) + +# Subdirectory structure under workdir for PDF/A copies +PDFA_ORIGINAL_SUBDIR = os.path.join("pdfa", "original") +PDFA_PROCESSED_SUBDIR = os.path.join("pdfa", "processed") + + +def _convert_pdf_to_pdfa(input_path: str, output_path: str, pdfa_format: str = "2") -> bool: + """Convert a PDF file to PDF/A using ocrmypdf. + + Uses ``ocrmypdf --skip-text --output-type pdfa-N`` so that existing text + layers are preserved (not re-OCR'd) while the output is converted to + PDF/A via Ghostscript. + + Args: + input_path: Absolute path to the source PDF file. + output_path: Absolute path for the PDF/A output file. + pdfa_format: PDF/A variant ('1', '2', or '3'). Defaults to '2' for PDF/A-2b. + + Returns: + True if conversion succeeded, False otherwise. + """ + ocrmypdf_bin = shutil.which("ocrmypdf") + if not ocrmypdf_bin: + logger.error("[convert_to_pdfa] ocrmypdf binary not found on PATH") + return False + + output_type = f"pdfa-{pdfa_format}" + + cmd = [ + ocrmypdf_bin, + "--skip-text", + "--output-type", + output_type, + "--quiet", + "--invalidate-digital-signatures", + input_path, + output_path, + ] + + logger.info(f"[convert_to_pdfa] Running: {' '.join(cmd)}") + + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600, check=False) # noqa: S603 + except subprocess.TimeoutExpired: + logger.warning("[convert_to_pdfa] ocrmypdf timed out after 600s") + return False + + if proc.returncode != 0: + stderr_snippet = proc.stderr.strip()[:500] if proc.stderr else "" + logger.warning(f"[convert_to_pdfa] ocrmypdf exited with code {proc.returncode}: {stderr_snippet}") + return False + + logger.info(f"[convert_to_pdfa] PDF/A file written to {output_path}") + return True + + +@celery.task(base=BaseTaskWithRetry, bind=True) +def convert_to_pdfa(self, file_id: int) -> dict: + """Generate PDF/A archival copies for a processed document. + + Creates PDF/A variants of both the original ingested file and the + processed file (with embedded metadata). Files are saved under + ``workdir/pdfa/original/`` and ``workdir/pdfa/processed/`` respectively. + + When ``settings.pdfa_upload_to_providers`` is True, the processed PDF/A + variant is also uploaded to all configured storage destinations. + + Args: + file_id: ID of the FileRecord to create PDF/A copies for. + + Returns: + Dictionary with status and file paths. + """ + task_id = self.request.id + logger.info(f"[{task_id}] Starting PDF/A conversion for file_id={file_id}") + log_task_progress( + task_id, + "convert_to_pdfa", + "in_progress", + "Starting PDF/A archival conversion", + file_id=file_id, + ) + + # Fetch file record + with SessionLocal() as db: + file_record = db.query(FileRecord).filter_by(id=file_id).first() + if not file_record: + logger.error(f"[{task_id}] FileRecord {file_id} not found") + log_task_progress(task_id, "convert_to_pdfa", "failure", "File record not found", file_id=file_id) + return {"error": "File record not found", "file_id": file_id} + + original_path = file_record.original_file_path + processed_path = file_record.processed_file_path + + pdfa_format = settings.pdfa_format + results = {} + + # --- Convert original file to PDF/A --- + if original_path and os.path.exists(original_path): + original_pdfa_dir = os.path.join(settings.workdir, PDFA_ORIGINAL_SUBDIR) + os.makedirs(original_pdfa_dir, exist_ok=True) + + base_name = os.path.splitext(os.path.basename(original_path))[0] + original_pdfa_path = get_unique_filepath_with_counter(original_pdfa_dir, base_name, ".pdf") + + logger.info(f"[{task_id}] Converting original to PDF/A: {original_path} -> {original_pdfa_path}") + log_task_progress( + task_id, + "convert_original_to_pdfa", + "in_progress", + f"Converting original to PDF/A: {os.path.basename(original_path)}", + file_id=file_id, + ) + + success = _convert_pdf_to_pdfa(original_path, original_pdfa_path, pdfa_format) + if success: + results["original_pdfa_path"] = original_pdfa_path + log_task_progress( + task_id, + "convert_original_to_pdfa", + "success", + f"Original PDF/A saved: {os.path.basename(original_pdfa_path)}", + file_id=file_id, + ) + else: + log_task_progress( + task_id, + "convert_original_to_pdfa", + "failure", + "Failed to convert original to PDF/A", + file_id=file_id, + ) + else: + logger.warning(f"[{task_id}] Original file not found, skipping original PDF/A conversion") + log_task_progress( + task_id, + "convert_original_to_pdfa", + "skipped", + "Original file not available", + file_id=file_id, + ) + + # --- Convert processed file to PDF/A --- + if processed_path and os.path.exists(processed_path): + processed_pdfa_dir = os.path.join(settings.workdir, PDFA_PROCESSED_SUBDIR) + os.makedirs(processed_pdfa_dir, exist_ok=True) + + base_name = os.path.splitext(os.path.basename(processed_path))[0] + processed_pdfa_path = get_unique_filepath_with_counter(processed_pdfa_dir, f"{base_name}-PDFA", ".pdf") + + logger.info(f"[{task_id}] Converting processed to PDF/A: {processed_path} -> {processed_pdfa_path}") + log_task_progress( + task_id, + "convert_processed_to_pdfa", + "in_progress", + f"Converting processed to PDF/A: {os.path.basename(processed_path)}", + file_id=file_id, + ) + + success = _convert_pdf_to_pdfa(processed_path, processed_pdfa_path, pdfa_format) + if success: + results["processed_pdfa_path"] = processed_pdfa_path + log_task_progress( + task_id, + "convert_processed_to_pdfa", + "success", + f"Processed PDF/A saved: {os.path.basename(processed_pdfa_path)}", + file_id=file_id, + ) + else: + log_task_progress( + task_id, + "convert_processed_to_pdfa", + "failure", + "Failed to convert processed to PDF/A", + file_id=file_id, + ) + else: + logger.warning(f"[{task_id}] Processed file not found, skipping processed PDF/A conversion") + log_task_progress( + task_id, + "convert_processed_to_pdfa", + "skipped", + "Processed file not available", + file_id=file_id, + ) + + # --- Update database with PDF/A paths --- + with SessionLocal() as db: + file_record = db.query(FileRecord).filter_by(id=file_id).first() + if file_record: + if "original_pdfa_path" in results: + file_record.original_pdfa_path = results["original_pdfa_path"] + if "processed_pdfa_path" in results: + file_record.processed_pdfa_path = results["processed_pdfa_path"] + db.commit() + logger.info(f"[{task_id}] Updated database with PDF/A paths") + + # --- Optionally upload processed PDF/A to storage providers --- + if settings.pdfa_upload_to_providers and "processed_pdfa_path" in results: + from app.tasks.send_to_all import send_to_all_destinations + + logger.info(f"[{task_id}] Uploading processed PDF/A to storage providers") + log_task_progress( + task_id, + "upload_pdfa_to_providers", + "in_progress", + "Uploading PDF/A variant to storage providers", + file_id=file_id, + ) + send_to_all_destinations.delay(results["processed_pdfa_path"], True, file_id) + log_task_progress( + task_id, + "upload_pdfa_to_providers", + "success", + "PDF/A variant queued for upload", + file_id=file_id, + ) + + # --- Final status --- + has_any = bool(results) + status = "success" if has_any else "failure" + message = ( + f"PDF/A conversion complete ({len(results)} variant(s) created)" if has_any else "No PDF/A variants created" + ) + log_task_progress(task_id, "convert_to_pdfa", status, message, file_id=file_id) + + return {"status": status, "file_id": file_id, **results} diff --git a/app/tasks/finalize_document_storage.py b/app/tasks/finalize_document_storage.py index 1876e73d..e42a42b5 100644 --- a/app/tasks/finalize_document_storage.py +++ b/app/tasks/finalize_document_storage.py @@ -76,6 +76,20 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements send_to_all_destinations.delay(processed_file, True, file_id) + # 3a. Trigger PDF/A archival conversion if enabled + if settings.enable_pdfa_conversion: + from app.tasks.convert_to_pdfa import convert_to_pdfa + + logger.info(f"[{task_id}] PDF/A conversion enabled, queueing archival conversion") + log_task_progress( + task_id, + "finalize_document_storage", + "in_progress", + "Queueing PDF/A archival conversion", + file_id=file_id, + ) + convert_to_pdfa.delay(file_id) + # 4. Send Notification (From Copilot) # Note: This notification is sent after processing is complete but while uploads # are being queued. diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 1d60ee4a..5151a9de 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -876,6 +876,56 @@ Near-duplicate detection: A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity. +## PDF/A Archival Conversion + +DocuElevate can optionally generate **PDF/A** archival copies of both the +original ingested file and the processed file. PDF/A copies are saved as +parallel variants alongside the standard files—they do **not** replace the +originals. This provides better legal coverage by producing time-stamped, +self-contained archival documents suitable for long-term storage and +compliance. + +The conversion uses **ocrmypdf** (backed by Ghostscript), which is already +bundled in the Docker images. + +> **Note:** PDF/A conversion may alter font rendering, especially for OCR text +> overlays produced by Microsoft Azure Document Intelligence. This is expected +> and is why PDF/A copies are kept as parallel variants rather than +> replacements. + +| Variable | Description | Default | +|-----------------------------|------------------------------------------------------------------------------------------------------|---------| +| `ENABLE_PDFA_CONVERSION` | Enable PDF/A archival variant generation for both original and processed files. | `false` | +| `PDFA_FORMAT` | PDF/A format variant: `1` (PDF/A-1b), `2` (PDF/A-2b), `3` (PDF/A-3b). | `2` | +| `PDFA_UPLOAD_TO_PROVIDERS` | Also upload the processed PDF/A variant to all configured storage providers (with `-PDFA` suffix). | `false` | + +### Storage Layout + +When enabled, PDF/A copies are stored under `workdir/pdfa/`: + +``` +workdir/ +├── original/ # Immutable copy of ingested file +├── processed/ # Processed file with embedded metadata +├── pdfa/ +│ ├── original/ # PDF/A copy of the ingested file +│ └── processed/ # PDF/A copy of the processed file (with -PDFA suffix) +└── tmp/ # Temporary processing area +``` + +### Configuration Example + +```bash +# Enable PDF/A archival copies +ENABLE_PDFA_CONVERSION=true + +# Use PDF/A-2b format (default, recommended for most use cases) +PDFA_FORMAT=2 + +# Also upload PDF/A copies to configured storage providers +PDFA_UPLOAD_TO_PROVIDERS=true +``` + ## Performance & Caching DocuElevate automatically optimizes database access and uses Redis as a diff --git a/migrations/versions/011_add_pdfa_paths.py b/migrations/versions/011_add_pdfa_paths.py new file mode 100644 index 00000000..28eeecc0 --- /dev/null +++ b/migrations/versions/011_add_pdfa_paths.py @@ -0,0 +1,29 @@ +"""Add PDF/A archival variant path columns to files table + +Revision ID: 011_add_pdfa_paths +Revises: 010_add_embedding_column +Create Date: 2026-03-02 + +""" + +from typing import Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "011_add_pdfa_paths" +down_revision: Union[str, None] = "010_add_embedding_column" +depends_on: Union[str, None] = None + + +def upgrade() -> None: + """Add PDF/A variant path columns to files table.""" + op.add_column("files", sa.Column("original_pdfa_path", sa.String(), nullable=True)) + op.add_column("files", sa.Column("processed_pdfa_path", sa.String(), nullable=True)) + + +def downgrade() -> None: + """Remove PDF/A variant path columns from files table.""" + op.drop_column("files", "processed_pdfa_path") + op.drop_column("files", "original_pdfa_path") diff --git a/tests/test_convert_to_pdfa.py b/tests/test_convert_to_pdfa.py new file mode 100644 index 00000000..29174e98 --- /dev/null +++ b/tests/test_convert_to_pdfa.py @@ -0,0 +1,356 @@ +"""Unit tests for app/tasks/convert_to_pdfa.py module.""" + +import subprocess +from unittest.mock import MagicMock, patch + +import pytest + +from app.tasks.convert_to_pdfa import ( + PDFA_ORIGINAL_SUBDIR, + PDFA_PROCESSED_SUBDIR, + _convert_pdf_to_pdfa, + convert_to_pdfa, +) + + +@pytest.mark.unit +class TestConvertPdfToPdfa: + """Tests for the _convert_pdf_to_pdfa helper function.""" + + @patch("app.tasks.convert_to_pdfa.subprocess.run") + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf") + def test_successful_conversion(self, mock_which, mock_run): + """Test successful PDF to PDF/A conversion.""" + mock_run.return_value = MagicMock(returncode=0, stderr="") + + result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf", "2") + + assert result is True + mock_which.assert_called_once_with("ocrmypdf") + mock_run.assert_called_once() + + # Verify the command arguments + cmd = mock_run.call_args[0][0] + assert cmd[0] == "/usr/bin/ocrmypdf" + assert "--skip-text" in cmd + assert "--output-type" in cmd + assert "pdfa-2" in cmd + assert "--quiet" in cmd + assert "--invalidate-digital-signatures" in cmd + assert "/input.pdf" in cmd + assert "/output.pdf" in cmd + + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value=None) + def test_ocrmypdf_not_found(self, mock_which): + """Test returns False when ocrmypdf binary is not on PATH.""" + result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf") + + assert result is False + + @patch("app.tasks.convert_to_pdfa.subprocess.run") + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf") + def test_conversion_failure(self, mock_which, mock_run): + """Test returns False when ocrmypdf exits with non-zero code.""" + mock_run.return_value = MagicMock(returncode=1, stderr="Some error occurred") + + result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf") + + assert result is False + + @patch("app.tasks.convert_to_pdfa.subprocess.run") + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf") + def test_conversion_timeout(self, mock_which, mock_run): + """Test returns False when ocrmypdf times out.""" + mock_run.side_effect = subprocess.TimeoutExpired(cmd="ocrmypdf", timeout=600) + + result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf") + + assert result is False + + @patch("app.tasks.convert_to_pdfa.subprocess.run") + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf") + def test_pdfa_format_variants(self, mock_which, mock_run): + """Test different PDF/A format variants are passed correctly.""" + mock_run.return_value = MagicMock(returncode=0, stderr="") + + for fmt in ("1", "2", "3"): + _convert_pdf_to_pdfa("/input.pdf", "/output.pdf", fmt) + cmd = mock_run.call_args[0][0] + assert f"pdfa-{fmt}" in cmd + + @patch("app.tasks.convert_to_pdfa.subprocess.run") + @patch("app.tasks.convert_to_pdfa.shutil.which", return_value="/usr/bin/ocrmypdf") + def test_conversion_failure_empty_stderr(self, mock_which, mock_run): + """Test handles empty stderr on failure.""" + mock_run.return_value = MagicMock(returncode=2, stderr="") + + result = _convert_pdf_to_pdfa("/input.pdf", "/output.pdf") + + assert result is False + + +@pytest.mark.unit +class TestConvertToPdfaTask: + """Tests for the convert_to_pdfa Celery task.""" + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True) + @patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter") + @patch("app.tasks.convert_to_pdfa.os.makedirs") + @patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True) + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_successful_conversion_both_files( + self, + mock_session_local, + mock_log, + mock_exists, + mock_makedirs, + mock_unique_path, + mock_convert, + mock_settings, + ): + """Test successful PDF/A conversion of both original and processed files.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = False + + # Mock unique path to return predictable paths + mock_unique_path.side_effect = [ + "/workdir/pdfa/original/test.pdf", + "/workdir/pdfa/processed/test-PDFA.pdf", + ] + + # Mock database session + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = "/workdir/original/test.pdf" + mock_record.processed_file_path = "/workdir/processed/test.pdf" + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=1) + + assert result["status"] == "success" + assert result["file_id"] == 1 + assert "original_pdfa_path" in result + assert "processed_pdfa_path" in result + assert mock_convert.call_count == 2 + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_file_record_not_found(self, mock_session_local, mock_log, mock_settings): + """Test returns error when file record is not found.""" + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_db.query.return_value.filter_by.return_value.first.return_value = None + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=999) + + assert "error" in result + assert result["file_id"] == 999 + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=False) + @patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter") + @patch("app.tasks.convert_to_pdfa.os.makedirs") + @patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True) + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_conversion_failure_both_files( + self, + mock_session_local, + mock_log, + mock_exists, + mock_makedirs, + mock_unique_path, + mock_convert, + mock_settings, + ): + """Test handles failure when both conversions fail.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = False + + mock_unique_path.side_effect = [ + "/workdir/pdfa/original/test.pdf", + "/workdir/pdfa/processed/test-PDFA.pdf", + ] + + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = "/workdir/original/test.pdf" + mock_record.processed_file_path = "/workdir/processed/test.pdf" + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=1) + + assert result["status"] == "failure" + assert "original_pdfa_path" not in result + assert "processed_pdfa_path" not in result + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_skips_missing_files(self, mock_session_local, mock_log, mock_settings): + """Test skips conversion when original/processed files don't exist.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = False + + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = None + mock_record.processed_file_path = None + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=1) + + assert result["status"] == "failure" + + @patch("app.tasks.send_to_all.send_to_all_destinations") + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True) + @patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter") + @patch("app.tasks.convert_to_pdfa.os.makedirs") + @patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True) + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_uploads_pdfa_when_enabled( + self, + mock_session_local, + mock_log, + mock_exists, + mock_makedirs, + mock_unique_path, + mock_convert, + mock_settings, + mock_send_all, + ): + """Test uploads processed PDF/A to providers when pdfa_upload_to_providers is True.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = True + + mock_unique_path.side_effect = [ + "/workdir/pdfa/original/test.pdf", + "/workdir/pdfa/processed/test-PDFA.pdf", + ] + + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = "/workdir/original/test.pdf" + mock_record.processed_file_path = "/workdir/processed/test.pdf" + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=42) + + # Verify send_to_all_destinations was called for the PDF/A file + mock_send_all.delay.assert_called_once_with("/workdir/pdfa/processed/test-PDFA.pdf", True, 42) + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa", return_value=True) + @patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter") + @patch("app.tasks.convert_to_pdfa.os.makedirs") + @patch("app.tasks.convert_to_pdfa.os.path.exists", return_value=True) + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_does_not_upload_when_disabled( + self, + mock_session_local, + mock_log, + mock_exists, + mock_makedirs, + mock_unique_path, + mock_convert, + mock_settings, + ): + """Test does not upload PDF/A when pdfa_upload_to_providers is False.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = False + + mock_unique_path.side_effect = [ + "/workdir/pdfa/original/test.pdf", + "/workdir/pdfa/processed/test-PDFA.pdf", + ] + + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = "/workdir/original/test.pdf" + mock_record.processed_file_path = "/workdir/processed/test.pdf" + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=1) + + assert result["status"] == "success" + + @patch("app.tasks.convert_to_pdfa.settings") + @patch("app.tasks.convert_to_pdfa._convert_pdf_to_pdfa") + @patch("app.tasks.convert_to_pdfa.get_unique_filepath_with_counter") + @patch("app.tasks.convert_to_pdfa.os.makedirs") + @patch("app.tasks.convert_to_pdfa.os.path.exists") + @patch("app.tasks.convert_to_pdfa.log_task_progress") + @patch("app.tasks.convert_to_pdfa.SessionLocal") + def test_partial_success_original_only( + self, + mock_session_local, + mock_log, + mock_exists, + mock_makedirs, + mock_unique_path, + mock_convert, + mock_settings, + ): + """Test partial success when only original conversion succeeds.""" + mock_settings.workdir = "/workdir" + mock_settings.pdfa_format = "2" + mock_settings.pdfa_upload_to_providers = False + + # Only original file exists + def exists_side_effect(path): + return "/original/" in path + + mock_exists.side_effect = exists_side_effect + mock_unique_path.return_value = "/workdir/pdfa/original/test.pdf" + mock_convert.return_value = True + + mock_db = MagicMock() + mock_session_local.return_value.__enter__.return_value = mock_db + mock_record = MagicMock() + mock_record.original_file_path = "/workdir/original/test.pdf" + mock_record.processed_file_path = "/workdir/processed/test.pdf" + mock_db.query.return_value.filter_by.return_value.first.return_value = mock_record + + convert_to_pdfa.request.id = "test-task-id" + result = convert_to_pdfa.__wrapped__(file_id=1) + + assert result["status"] == "success" + assert "original_pdfa_path" in result + assert "processed_pdfa_path" not in result + + +@pytest.mark.unit +class TestPdfaSubdirectoryConstants: + """Tests for PDF/A subdirectory constants.""" + + def test_original_subdir(self): + """Test PDFA_ORIGINAL_SUBDIR is correct.""" + assert "pdfa" in PDFA_ORIGINAL_SUBDIR + assert "original" in PDFA_ORIGINAL_SUBDIR + + def test_processed_subdir(self): + """Test PDFA_PROCESSED_SUBDIR is correct.""" + assert "pdfa" in PDFA_PROCESSED_SUBDIR + assert "processed" in PDFA_PROCESSED_SUBDIR diff --git a/tests/test_finalize_storage.py b/tests/test_finalize_storage.py index d7b54f5c..cb268b22 100644 --- a/tests/test_finalize_storage.py +++ b/tests/test_finalize_storage.py @@ -107,6 +107,7 @@ class TestFinalizeDocumentStorage: ): with patch("app.tasks.finalize_document_storage.settings") as mock_settings: mock_settings.workdir = "/tmp" + mock_settings.enable_pdfa_conversion = False finalize_document_storage.request.id = "test-task-id"