feat(pdfa): add FreeTSA timestamping, per-provider folder overrides, individual upload toggles

- Add RFC 3161 timestamping via FreeTSA (PDFA_TIMESTAMP_ENABLED, PDFA_TIMESTAMP_URL)
- Replace PDFA_UPLOAD_TO_PROVIDERS with individual PDFA_UPLOAD_ORIGINAL and PDFA_UPLOAD_PROCESSED
- Add PDFA_UPLOAD_FOLDER setting for per-provider subfolder configuration
- Add GOOGLE_DRIVE_PDFA_FOLDER_ID for Google Drive-specific folder override
- Add folder_override parameter to all 8 folder-using upload tasks
- Add folder_overrides dict parameter to send_to_all_destinations
- Add _compute_pdfa_folder_overrides() and _timestamp_file() helpers
- Expand tests to 26 (timestamping, folder overrides, individual toggles)
- Update docs/ConfigurationGuide.md and .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-02 13:55:34 +00:00
parent eea99eb01d
commit a03b3af933
14 changed files with 584 additions and 90 deletions
+195 -9
View File
@@ -11,6 +11,11 @@ internally). Two variants are produced when enabled:
Both are saved under ``workdir/pdfa/`` and referenced in the database via
``FileRecord.original_pdfa_path`` and ``FileRecord.processed_pdfa_path``.
When ``PDFA_TIMESTAMP_ENABLED`` is True, each PDF/A file also gets an RFC 3161
timestamp response (``.tsr``) from a configurable Timestamp Authority (default:
FreeTSA). This provides cryptographic proof of the file's existence at a given
point in time.
.. note::
PDF/A conversion may alter font rendering (especially OCR text overlays
@@ -23,6 +28,8 @@ import os
import shutil
import subprocess
import requests as http_requests
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
@@ -87,6 +94,110 @@ def _convert_pdf_to_pdfa(input_path: str, output_path: str, pdfa_format: str = "
return True
def _timestamp_file(file_path: str, tsa_url: str) -> str | None:
"""Create an RFC 3161 timestamp for a file using a Timestamp Authority.
Uses ``openssl ts`` to create a timestamp request (TSQ) from the file's
SHA-256 hash, submits it to the TSA via HTTP POST, and saves the timestamp
response (TSR) alongside the file.
Args:
file_path: Absolute path to the file to timestamp.
tsa_url: URL of the RFC 3161 Timestamp Authority.
Returns:
Path to the ``.tsr`` file if successful, None otherwise.
"""
openssl_bin = shutil.which("openssl")
if not openssl_bin:
logger.error("[timestamp] openssl binary not found on PATH")
return None
tsr_path = file_path + ".tsr"
tsq_path = file_path + ".tsq"
try:
# Step 1: Create timestamp request
cmd = [openssl_bin, "ts", "-query", "-data", file_path, "-sha256", "-no_nonce", "-out", tsq_path]
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False) # noqa: S603
if proc.returncode != 0:
logger.warning(f"[timestamp] openssl ts -query failed: {proc.stderr.strip()[:200]}")
return None
# Step 2: Submit TSQ to the Timestamp Authority
with open(tsq_path, "rb") as f:
tsq_data = f.read()
response = http_requests.post(
tsa_url,
data=tsq_data,
headers={"Content-Type": "application/timestamp-query"},
timeout=30,
)
if response.status_code != 200:
logger.warning(f"[timestamp] TSA returned HTTP {response.status_code} from {tsa_url}")
return None
# Step 3: Save the timestamp response
with open(tsr_path, "wb") as f:
f.write(response.content)
logger.info(f"[timestamp] RFC 3161 timestamp saved to {tsr_path}")
return tsr_path
except http_requests.RequestException as e:
logger.warning(f"[timestamp] Failed to contact TSA at {tsa_url}: {e}")
return None
except subprocess.TimeoutExpired:
logger.warning("[timestamp] openssl ts timed out")
return None
finally:
# Always clean up the TSQ file
if os.path.exists(tsq_path):
os.remove(tsq_path)
def _compute_pdfa_folder_overrides() -> dict[str, str]:
"""Compute per-provider folder overrides for PDF/A uploads.
Appends ``settings.pdfa_upload_folder`` to each provider's configured
folder. For Google Drive (which uses folder IDs), uses the dedicated
``google_drive_pdfa_folder_id`` setting.
Returns:
Dictionary mapping provider names to folder override strings.
"""
subfolder = settings.pdfa_upload_folder
overrides: dict[str, str] = {}
if not subfolder:
return overrides
# Path-based providers: append subfolder
for provider, folder_attr in [
("dropbox", "dropbox_folder"),
("nextcloud", "nextcloud_folder"),
("webdav", "webdav_folder"),
("ftp", "ftp_folder"),
("sftp", "sftp_folder"),
("onedrive", "onedrive_folder_path"),
]:
base = getattr(settings, folder_attr, "") or ""
overrides[provider] = f"{base.rstrip('/')}/{subfolder}" if base else subfolder
# S3: append subfolder to prefix
s3_prefix = getattr(settings, "s3_folder_prefix", "") or ""
overrides["s3"] = f"{s3_prefix.rstrip('/')}/{subfolder}/"
# Google Drive: use dedicated folder ID or fall back to default
gdrive_pdfa_id = settings.google_drive_pdfa_folder_id
if gdrive_pdfa_id:
overrides["google_drive"] = gdrive_pdfa_id
return overrides
@celery.task(base=BaseTaskWithRetry, bind=True)
def convert_to_pdfa(self, file_id: int) -> dict:
"""Generate PDF/A archival copies for a processed document.
@@ -95,8 +206,11 @@ def convert_to_pdfa(self, file_id: int) -> dict:
processed file (with embedded metadata). Files are saved under
``workdir/pdfa/original/`` and ``workdir/pdfa/processed/`` respectively.
When ``settings.pdfa_upload_to_providers`` is True, the processed PDF/A
variant is also uploaded to all configured storage destinations.
When timestamping is enabled, each PDF/A file also gets an RFC 3161
``.tsr`` timestamp from the configured TSA.
Upload of each variant to storage providers is controlled independently
by ``pdfa_upload_original`` and ``pdfa_upload_processed``.
Args:
file_id: ID of the FileRecord to create PDF/A copies for.
@@ -126,6 +240,8 @@ def convert_to_pdfa(self, file_id: int) -> dict:
processed_path = file_record.processed_file_path
pdfa_format = settings.pdfa_format
timestamp_enabled = settings.pdfa_timestamp_enabled
timestamp_url = settings.pdfa_timestamp_url
results = {}
# --- Convert original file to PDF/A ---
@@ -155,6 +271,25 @@ def convert_to_pdfa(self, file_id: int) -> dict:
f"Original PDF/A saved: {os.path.basename(original_pdfa_path)}",
file_id=file_id,
)
# Timestamp the original PDF/A
if timestamp_enabled:
tsr = _timestamp_file(original_pdfa_path, timestamp_url)
if tsr:
log_task_progress(
task_id,
"timestamp_original_pdfa",
"success",
f"Timestamped: {os.path.basename(tsr)}",
file_id=file_id,
)
else:
log_task_progress(
task_id,
"timestamp_original_pdfa",
"failure",
"Failed to timestamp original PDF/A",
file_id=file_id,
)
else:
log_task_progress(
task_id,
@@ -200,6 +335,25 @@ def convert_to_pdfa(self, file_id: int) -> dict:
f"Processed PDF/A saved: {os.path.basename(processed_pdfa_path)}",
file_id=file_id,
)
# Timestamp the processed PDF/A
if timestamp_enabled:
tsr = _timestamp_file(processed_pdfa_path, timestamp_url)
if tsr:
log_task_progress(
task_id,
"timestamp_processed_pdfa",
"success",
f"Timestamped: {os.path.basename(tsr)}",
file_id=file_id,
)
else:
log_task_progress(
task_id,
"timestamp_processed_pdfa",
"failure",
"Failed to timestamp processed PDF/A",
file_id=file_id,
)
else:
log_task_progress(
task_id,
@@ -229,24 +383,56 @@ def convert_to_pdfa(self, file_id: int) -> dict:
db.commit()
logger.info(f"[{task_id}] Updated database with PDF/A paths")
# --- Optionally upload processed PDF/A to storage providers ---
if settings.pdfa_upload_to_providers and "processed_pdfa_path" in results:
# --- Upload PDF/A variants to storage providers ---
folder_overrides = _compute_pdfa_folder_overrides()
if settings.pdfa_upload_original and "original_pdfa_path" in results:
from app.tasks.send_to_all import send_to_all_destinations
logger.info(f"[{task_id}] Uploading original PDF/A to storage providers")
log_task_progress(
task_id,
"upload_original_pdfa",
"in_progress",
"Uploading original PDF/A to storage providers",
file_id=file_id,
)
send_to_all_destinations.delay(
results["original_pdfa_path"],
True,
file_id,
folder_overrides=folder_overrides if folder_overrides else None,
)
log_task_progress(
task_id,
"upload_original_pdfa",
"success",
"Original PDF/A queued for upload",
file_id=file_id,
)
if settings.pdfa_upload_processed and "processed_pdfa_path" in results:
from app.tasks.send_to_all import send_to_all_destinations
logger.info(f"[{task_id}] Uploading processed PDF/A to storage providers")
log_task_progress(
task_id,
"upload_pdfa_to_providers",
"upload_processed_pdfa",
"in_progress",
"Uploading PDF/A variant to storage providers",
"Uploading processed PDF/A to storage providers",
file_id=file_id,
)
send_to_all_destinations.delay(results["processed_pdfa_path"], True, file_id)
send_to_all_destinations.delay(
results["processed_pdfa_path"],
True,
file_id,
folder_overrides=folder_overrides if folder_overrides else None,
)
log_task_progress(
task_id,
"upload_pdfa_to_providers",
"upload_processed_pdfa",
"success",
"PDF/A variant queued for upload",
"Processed PDF/A queued for upload",
file_id=file_id,
)
+9 -2
View File
@@ -106,7 +106,7 @@ def get_configured_services_from_validator():
@celery.task(base=BaseTaskWithRetry, bind=True)
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None):
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None, folder_overrides=None):
"""
Distribute a file to all configured storage destinations.
@@ -115,6 +115,10 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
use_validator: Whether to use the config validator to determine enabled services
(if False, falls back to individual checks)
file_id: Optional file ID to associate with logs
folder_overrides: Optional dict mapping provider names to folder override strings.
When set, the override is passed to the upload task which uses it
instead of the provider's default folder. Example:
{"dropbox": "/Documents/pdfa", "s3": "docs/pdfa/"}
"""
task_id = self.request.id
@@ -236,7 +240,10 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id
)
try:
task = service["upload_func"].delay(file_path, file_id=file_id)
kwargs = {"file_id": file_id}
if folder_overrides and service_name in folder_overrides:
kwargs["folder_override"] = folder_overrides[service_name]
task = service["upload_func"].delay(file_path, **kwargs)
results[f"{service_name}_task_id"] = task.id
queued_count += 1
log_task_progress(
+2 -2
View File
@@ -103,7 +103,7 @@ def get_dropbox_client():
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_dropbox(self, file_path: str, file_id: int = None):
def upload_to_dropbox(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Upload a file to Dropbox.
@@ -147,7 +147,7 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
dbx = get_dropbox_client()
# Calculate remote path based on local file structure
remote_base = settings.dropbox_folder or ""
remote_base = folder_override if folder_override is not None else (settings.dropbox_folder or "")
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Function to check if file exists in Dropbox
+5 -4
View File
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_ftp(self, file_path: str, file_id: int = None):
def upload_to_ftp(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Uploads a file to an FTP server in the configured folder.
@@ -97,10 +97,11 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
# Change to target directory if specified
if settings.ftp_folder:
ftp_folder_setting = folder_override if folder_override is not None else settings.ftp_folder
if ftp_folder_setting:
try:
# Try to navigate to the directory, create if it doesn't exist
ftp_folder = settings.ftp_folder
ftp_folder = ftp_folder_setting
# Remove leading slash if present
if ftp_folder.startswith("/"):
ftp_folder = ftp_folder[1:]
@@ -138,7 +139,7 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
"status": "Completed",
"file": file_path,
"ftp_host": settings.ftp_host,
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
"ftp_path": f"{ftp_folder_setting}/{filename}" if ftp_folder_setting else filename,
"used_tls": used_tls,
}
+6 -3
View File
@@ -153,7 +153,9 @@ def truncate_property_value(key, value, max_bytes=100):
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id: int = None):
def upload_to_google_drive(
self, file_path: str, include_metadata=True, file_id: int = None, folder_override: str = None
):
"""
Uploads a file to Google Drive in the configured folder with optional metadata.
@@ -201,8 +203,9 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
}
# If folder ID is specified, set parent folder
if settings.google_drive_folder_id:
file_metadata["parents"] = [settings.google_drive_folder_id]
gdrive_folder_id = folder_override if folder_override is not None else settings.google_drive_folder_id
if gdrive_folder_id:
file_metadata["parents"] = [gdrive_folder_id]
# Add custom properties if metadata exists
if metadata:
+4 -2
View File
@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_nextcloud(self, file_path: str, file_id: int = None):
def upload_to_nextcloud(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Upload a file to Nextcloud WebDAV.
@@ -60,7 +60,9 @@ def upload_to_nextcloud(self, file_path: str, file_id: int = None):
webdav_url += "/"
# Calculate remote path based on local file structure
remote_base = getattr(settings, "nextcloud_folder", "") or ""
remote_base = (
folder_override if folder_override is not None else (getattr(settings, "nextcloud_folder", "") or "")
)
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = f"{webdav_url}/{remote_path}"
+6 -4
View File
@@ -210,7 +210,7 @@ def upload_large_file(file_path, upload_url):
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_onedrive(self, file_path: str, file_id: int = None):
def upload_to_onedrive(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Uploads a file to OneDrive in the configured folder.
@@ -248,15 +248,17 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
# Get access token
access_token = get_onedrive_token()
onedrive_folder = folder_override if folder_override is not None else settings.onedrive_folder_path
# Create upload session
upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
upload_url = create_upload_session(filename, onedrive_folder, access_token)
# Upload the file
result = upload_large_file(file_path, upload_url)
# Log success
web_url = result.get("webUrl", "Not available")
logger.info(f"[{task_id}] Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
logger.info(f"[{task_id}] Successfully uploaded {filename} to OneDrive at path {onedrive_folder}")
logger.info(f"[{task_id}] File accessible at: {web_url}")
log_task_progress(
task_id, "upload_to_onedrive", "success", f"Uploaded to OneDrive: {filename}", file_id=file_id
@@ -265,7 +267,7 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
return {
"status": "Completed",
"file_path": file_path,
"onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
"onedrive_path": f"{onedrive_folder}/{filename}",
"web_url": web_url,
}
+4 -3
View File
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_s3(self, file_path: str, file_id: int = None):
def upload_to_s3(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Uploads a file to Amazon S3 in the configured bucket and folder.
@@ -61,9 +61,10 @@ def upload_to_s3(self, file_path: str, file_id: int = None):
)
# Construct the S3 key (path within the bucket)
if settings.s3_folder_prefix:
s3_folder = folder_override if folder_override is not None else settings.s3_folder_prefix
if s3_folder:
# Ensure folder prefix ends with a slash
folder_prefix = settings.s3_folder_prefix
folder_prefix = s3_folder
if not folder_prefix.endswith("/"):
folder_prefix += "/"
s3_key = f"{folder_prefix}{filename}"
+2 -2
View File
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_sftp(self, file_path: str, file_id: int = None):
def upload_to_sftp(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Upload a file to an SFTP server.
@@ -95,7 +95,7 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
sftp = ssh.open_sftp()
# Calculate remote path based on local file structure
remote_base = settings.sftp_folder or ""
remote_base = folder_override if folder_override is not None else (settings.sftp_folder or "")
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Ensure the remote path starts with a slash if the base folder does
+2 -2
View File
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_webdav(self, file_path: str, file_id: int = None):
def upload_to_webdav(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Uploads a file to a WebDAV server in the configured folder.
@@ -50,7 +50,7 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
raise ValueError(error_msg)
# Construct the full upload URL
webdav_folder = settings.webdav_folder or ""
webdav_folder = folder_override if folder_override is not None else (settings.webdav_folder or "")
# Ensure folder doesn't have leading slash if we're joining it to the base URL
if webdav_folder and webdav_folder.startswith("/"):
webdav_folder = webdav_folder[1:]