diff --git a/.env.demo b/.env.demo index 649f6d73..fb4752f2 100644 --- a/.env.demo +++ b/.env.demo @@ -117,6 +117,18 @@ NEXTCLOUD_PASSWORD= # Paperless-ngx PAPERLESS_HOST=https://paperless.example.com PAPERLESS_NGX_API_TOKEN= +# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value +# If set, the extracted sender information will be automatically set as a custom field in Paperless +# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender +# PAPERLESS_CUSTOM_FIELD_ABSENDER= + +# Optional: JSON mapping of metadata fields to Paperless custom field names +# This allows you to map multiple extracted metadata fields to custom fields in Paperless +# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...} +# Available metadata fields: absender, empfaenger, correspondent, document_type, language, +# kommunikationsart, kommunikationskategorie, reference_number, etc. +# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"} +# PAPERLESS_CUSTOM_FIELDS_MAPPING= # Dropbox DROPBOX_APP_KEY= diff --git a/app/config.py b/app/config.py index d08dea76..60b1a950 100644 --- a/app/config.py +++ b/app/config.py @@ -31,6 +31,10 @@ class Settings(BaseSettings): # Making Paperless optional paperless_ngx_api_token: Optional[str] = None paperless_host: Optional[str] = None + paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless + # JSON mapping of metadata field names to Paperless custom field names + # Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"} + paperless_custom_fields_mapping: Optional[str] = None azure_ai_key: str azure_region: str diff --git a/app/tasks/upload_to_paperless.py b/app/tasks/upload_to_paperless.py index f777e2ab..0a760cc3 100644 --- a/app/tasks/upload_to_paperless.py +++ b/app/tasks/upload_to_paperless.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import logging import os import time @@ -15,6 +16,8 @@ logger = logging.getLogger(__name__) POLL_MAX_ATTEMPTS = 10 POLL_INTERVAL_SEC = 3 +# Sentinel value used to indicate missing/unknown metadata that should not be set as custom fields +METADATA_UNKNOWN_PLACEHOLDER = "Unknown" def _get_headers(): @@ -33,6 +36,25 @@ def _paperless_api_url(path: str) -> str: return f"{host}{path}" +def normalize_metadata_value(value) -> str: + """ + Normalizes a metadata value to a string suitable for Paperless custom fields. + + Args: + value: The metadata value to normalize + + Returns: + Empty string if value should be excluded, otherwise the string representation + """ + if value is None: + return "" + str_value = str(value) + # Exclude empty strings and the "Unknown" placeholder + if not str_value or str_value == METADATA_UNKNOWN_PLACEHOLDER: + return "" + return str_value + + def poll_task_for_document_id(task_id: str) -> int: """ Polls /api/tasks/?task_id= until we get status=SUCCESS or FAILURE, @@ -78,10 +100,88 @@ def poll_task_for_document_id(task_id: str) -> int: raise TimeoutError(f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts.") +def get_custom_field_id(field_name: str) -> int: + """ + Retrieves the ID of a custom field by its name from Paperless-ngx. + + Args: + field_name: The name of the custom field to look up + + Returns: + The integer ID of the custom field + + Raises: + ValueError: If the custom field is not found + """ + url = _paperless_api_url("/api/custom_fields/") + try: + resp = requests.get(url, headers=_get_headers(), timeout=settings.http_request_timeout) + resp.raise_for_status() + data = resp.json() + + # Handle paginated response + results = data.get("results", []) if isinstance(data, dict) else data + + for field in results: + if field.get("name") == field_name: + return field.get("id") + + raise ValueError(f"Custom field '{field_name}' not found in Paperless-ngx") + except requests.exceptions.RequestException as exc: + logger.error(f"Failed to retrieve custom fields from Paperless: {exc}") + raise + + +def set_document_custom_fields(doc_id: int, custom_fields: dict, task_id: str) -> None: + """ + Updates custom fields for a document in Paperless-ngx using PATCH. + + Args: + doc_id: The Paperless document ID + custom_fields: Dictionary mapping field names to values + task_id: Task ID for logging + """ + if not custom_fields: + return + + # Build custom_fields array for PATCH request + custom_fields_array = [] + for field_name, value in custom_fields.items(): + normalized_value = normalize_metadata_value(value) + if normalized_value: # Only set non-empty values + try: + field_id = get_custom_field_id(field_name) + custom_fields_array.append({"field": field_id, "value": normalized_value}) + logger.info( + f"[{task_id}] Mapped custom field '{field_name}' to ID {field_id} with value '{normalized_value}'" + ) + except ValueError as e: + logger.warning(f"[{task_id}] {str(e)}, skipping this field") + continue + + if not custom_fields_array: + logger.info(f"[{task_id}] No valid custom fields to set") + return + + # PATCH the document with custom fields + url = _paperless_api_url(f"/api/documents/{doc_id}/") + payload = {"custom_fields": custom_fields_array} + + try: + logger.info(f"[{task_id}] Setting custom fields for document {doc_id}: {payload}") + resp = requests.patch(url, headers=_get_headers(), json=payload, timeout=settings.http_request_timeout) + resp.raise_for_status() + logger.info(f"[{task_id}] Successfully set custom fields for document {doc_id}") + except requests.exceptions.RequestException as exc: + logger.error(f"[{task_id}] Failed to set custom fields for document {doc_id}: {exc}") + # Don't raise - this is a non-critical failure, document is already uploaded + logger.error(f"[{task_id}] Response: {getattr(exc.response, 'text', '')}") + + @celery.task(base=BaseTaskWithRetry, bind=True) def upload_to_paperless(self, file_path: str, file_id: int = None): """ - Uploads a file to Paperless-ngx. + Uploads a file to Paperless-ngx and sets custom fields from metadata. Args: file_path: Path to the file to upload @@ -113,6 +213,20 @@ def upload_to_paperless(self, file_path: str, file_id: int = None): log_task_progress(task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id) raise ValueError(error_msg) + # Try to load metadata from accompanying JSON file + metadata = {} + json_path = os.path.splitext(file_path)[0] + ".json" + if os.path.exists(json_path): + try: + with open(json_path, "r", encoding="utf-8") as f: + metadata = json.load(f) + logger.info(f"[{task_id}] Loaded metadata from {json_path}") + except Exception as e: + logger.warning(f"[{task_id}] Failed to load metadata from {json_path}: {e}") + metadata = {} + else: + logger.info(f"[{task_id}] No metadata file found at {json_path}") + # Upload the PDF logger.info(f"[{task_id}] Posting document to Paperless") log_task_progress(task_id, "post_document", "in_progress", "Posting to Paperless API", file_id=file_id) @@ -153,6 +267,57 @@ def upload_to_paperless(self, file_path: str, file_id: int = None): task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id ) + # Set custom fields if configured and metadata available + custom_fields_to_set = {} + + # First, check for the new flexible mapping configuration + if settings.paperless_custom_fields_mapping: + try: + # Parse JSON mapping: {"metadata_field": "PaperlessFieldName", ...} + field_mapping = json.loads(settings.paperless_custom_fields_mapping) + logger.info(f"[{task_id}] Using custom fields mapping: {field_mapping}") + + # Map each metadata field to its corresponding Paperless custom field + for metadata_field, paperless_field in field_mapping.items(): + if metadata_field in metadata and metadata[metadata_field]: + # Convert to string to ensure consistent comparison with UNKNOWN_VALUE + value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else "" + if value and value != UNKNOWN_VALUE: + custom_fields_to_set[paperless_field] = value + logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'") + except json.JSONDecodeError as e: + logger.error(f"[{task_id}] Failed to parse PAPERLESS_CUSTOM_FIELDS_MAPPING: {e}") + except Exception as e: + logger.error(f"[{task_id}] Error processing custom fields mapping: {e}") + + # Fallback to legacy single-field configuration for backward compatibility + if settings.paperless_custom_field_absender and metadata.get("absender"): + # Only add if not already set by the mapping + if settings.paperless_custom_field_absender not in custom_fields_to_set: + custom_fields_to_set[settings.paperless_custom_field_absender] = metadata.get("absender") + logger.debug(f"[{task_id}] Using legacy absender field configuration") + + # Set custom fields if we have any to set + if custom_fields_to_set: + logger.info(f"[{task_id}] Setting {len(custom_fields_to_set)} custom field(s) for document {doc_id}") + log_task_progress(task_id, "set_custom_fields", "in_progress", "Setting custom fields", file_id=file_id) + + try: + set_document_custom_fields(doc_id, custom_fields_to_set, task_id) + log_task_progress( + task_id, + "set_custom_fields", + "success", + f"Set {len(custom_fields_to_set)} custom field(s)", + file_id=file_id, + ) + except Exception as e: + logger.error(f"[{task_id}] Failed to set custom fields: {e}") + log_task_progress(task_id, "set_custom_fields", "failure", f"Failed: {str(e)}", file_id=file_id) + # Don't fail the entire upload if custom fields fail + else: + logger.debug(f"[{task_id}] No custom fields configured or no metadata available") + return { "status": "Completed", "paperless_task_id": raw_task_id, diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 3498621c..f32d2b95 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -212,10 +212,45 @@ SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://trusted ### Paperless NGX -| **Variable** | **Description** | -|-------------------------------|-----------------------------------------------------| -| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. | -| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). | +| **Variable** | **Description** | +|-------------------------------------|-----------------------------------------------------------------------------------------------------| +| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. | +| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). | +| `PAPERLESS_CUSTOM_FIELD_ABSENDER` | (Optional, Legacy) Name of the custom field in Paperless-ngx to store the sender ("absender") information. If set, the extracted sender will be automatically set as a custom field after document upload. Example: `Absender` or `Sender` | +| `PAPERLESS_CUSTOM_FIELDS_MAPPING` | (Optional, Recommended) JSON mapping of extracted metadata fields to Paperless custom field names. This allows you to map multiple fields at once. Format: `{"metadata_field": "CustomFieldName", ...}`. See examples below. | + +#### Custom Fields Mapping Examples + +**Single Field (Legacy Method)**: +```bash +PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender +``` + +**Multiple Fields (Recommended Method)**: +```bash +# Map multiple metadata fields to custom fields in Paperless +PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language"}' +``` + +**All Available Metadata Fields**: +DocuElevate extracts the following fields that can be mapped to Paperless custom fields: +- `absender` - Sender/author of the document +- `empfaenger` - Recipient of the document +- `correspondent` - The issuing entity/company (shortened name) +- `document_type` - Type classification (Invoice, Contract, etc.) +- `language` - Document language (ISO 639-1 code, e.g., "de", "en") +- `kommunikationsart` - Communication type (German classification) +- `kommunikationskategorie` - Communication category (German classification) +- `reference_number` - Invoice/order/reference number if found +- `title` - Human-readable document title +- `tags` - List of thematic keywords (array) + +**Complete Example**: +```bash +PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "correspondent": "Correspondent", "language": "Language", "reference_number": "ReferenceNumber"}' +``` + +**Note**: Custom fields must be created in your Paperless-ngx instance before DocuElevate can use them. The field names in the mapping (right side of the JSON) must **exactly** match the names in Paperless (case-sensitive). ### Dropbox diff --git a/docs/UserGuide.md b/docs/UserGuide.md index d464d182..b8be88af 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -145,6 +145,50 @@ Depending on the system configuration, DocuElevate can perform: - **PDF Conversion** - Convert various file formats to PDF - **Document Distribution** - Store documents in Dropbox, Nextcloud, or Paperless NGX +### Paperless-ngx Integration + +If Paperless-ngx is configured, DocuElevate can automatically upload processed documents and set custom fields with extracted metadata: + +1. **Automatic Upload**: After processing, documents are automatically uploaded to Paperless-ngx +2. **Custom Fields Support**: DocuElevate can automatically populate custom fields in Paperless with extracted metadata +3. **Flexible Field Mapping**: Map any extracted metadata field to any custom field in Paperless + +#### Setting Up Custom Fields + +**Basic Setup (Single Field)**: +```bash +# In your .env file +PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender +``` + +**Advanced Setup (Multiple Fields)**: +```bash +# Map multiple metadata fields to Paperless custom fields +PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}' +``` + +**Available Metadata Fields**: +- `absender` - Sender/author +- `empfaenger` - Recipient +- `correspondent` - Issuing company (short name) +- `language` - Document language (e.g., "de", "en") +- `document_type` - Classification (Invoice, Contract, etc.) +- `reference_number` - Invoice/order/reference number +- `kommunikationsart` - Communication type +- `kommunikationskategorie` - Communication category +- And more... + +**Requirements**: +- Create the custom fields in Paperless-ngx first (Settings → Custom Fields) +- The field names in your configuration must exactly match the names in Paperless +- Ensure `PAPERLESS_HOST` and `PAPERLESS_NGX_API_TOKEN` are configured + +**How It Works**: +1. DocuElevate extracts metadata from your documents using AI +2. The document is uploaded to Paperless-ngx +3. After successful upload, custom fields are automatically populated +4. You can view the populated fields in your Paperless-ngx document details + ## API Access For programmatic access, DocuElevate provides a comprehensive REST API: