Merge pull request #224 from christianlouis/copilot/add-absender-custom-field

Add custom fields support for Paperless-ngx uploads

.env.demo

@@ -117,6 +117,18 @@ NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
 # Paperless-ngx
 PAPERLESS_HOST=https://paperless.example.com
 PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
+# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
+# If set, the extracted sender information will be automatically set as a custom field in Paperless
+# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
+# PAPERLESS_CUSTOM_FIELD_ABSENDER=
+
+# Optional: JSON mapping of metadata fields to Paperless custom field names
+# This allows you to map multiple extracted metadata fields to custom fields in Paperless
+# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...}
+# Available metadata fields: absender, empfaenger, correspondent, document_type, language, 
+#                            kommunikationsart, kommunikationskategorie, reference_number, etc.
+# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
+# PAPERLESS_CUSTOM_FIELDS_MAPPING=
 
 # Dropbox
 DROPBOX_APP_KEY=<DROPBOX_APP_KEY>

app/config.py

@@ -31,6 +31,10 @@ class Settings(BaseSettings):
     # Making Paperless optional
     paperless_ngx_api_token: Optional[str] = None
     paperless_host: Optional[str] = None
+    paperless_custom_field_absender: Optional[str] = None  # Name of the "absender" custom field in Paperless
+    # JSON mapping of metadata field names to Paperless custom field names
+    # Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
+    paperless_custom_fields_mapping: Optional[str] = None
 
     azure_ai_key: str
     azure_region: str

app/tasks/upload_to_paperless.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import json
 import logging
 import os
 import time
@@ -15,6 +16,8 @@ logger = logging.getLogger(__name__)
 
 POLL_MAX_ATTEMPTS = 10
 POLL_INTERVAL_SEC = 3
+# Sentinel value used to indicate missing/unknown metadata that should not be set as custom fields
+METADATA_UNKNOWN_PLACEHOLDER = "Unknown"
 
 
 def _get_headers():
@@ -33,6 +36,25 @@ def _paperless_api_url(path: str) -> str:
     return f"{host}{path}"
 
 
+def normalize_metadata_value(value) -> str:
+    """
+    Normalizes a metadata value to a string suitable for Paperless custom fields.
+
+    Args:
+        value: The metadata value to normalize
+
+    Returns:
+        Empty string if value should be excluded, otherwise the string representation
+    """
+    if value is None:
+        return ""
+    str_value = str(value)
+    # Exclude empty strings and the "Unknown" placeholder
+    if not str_value or str_value == METADATA_UNKNOWN_PLACEHOLDER:
+        return ""
+    return str_value
+
+
 def poll_task_for_document_id(task_id: str) -> int:
     """
     Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
@@ -78,10 +100,88 @@ def poll_task_for_document_id(task_id: str) -> int:
     raise TimeoutError(f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts.")
 
 
+def get_custom_field_id(field_name: str) -> int:
+    """
+    Retrieves the ID of a custom field by its name from Paperless-ngx.
+
+    Args:
+        field_name: The name of the custom field to look up
+
+    Returns:
+        The integer ID of the custom field
+
+    Raises:
+        ValueError: If the custom field is not found
+    """
+    url = _paperless_api_url("/api/custom_fields/")
+    try:
+        resp = requests.get(url, headers=_get_headers(), timeout=settings.http_request_timeout)
+        resp.raise_for_status()
+        data = resp.json()
+
+        # Handle paginated response
+        results = data.get("results", []) if isinstance(data, dict) else data
+
+        for field in results:
+            if field.get("name") == field_name:
+                return field.get("id")
+
+        raise ValueError(f"Custom field '{field_name}' not found in Paperless-ngx")
+    except requests.exceptions.RequestException as exc:
+        logger.error(f"Failed to retrieve custom fields from Paperless: {exc}")
+        raise
+
+
+def set_document_custom_fields(doc_id: int, custom_fields: dict, task_id: str) -> None:
+    """
+    Updates custom fields for a document in Paperless-ngx using PATCH.
+
+    Args:
+        doc_id: The Paperless document ID
+        custom_fields: Dictionary mapping field names to values
+        task_id: Task ID for logging
+    """
+    if not custom_fields:
+        return
+
+    # Build custom_fields array for PATCH request
+    custom_fields_array = []
+    for field_name, value in custom_fields.items():
+        normalized_value = normalize_metadata_value(value)
+        if normalized_value:  # Only set non-empty values
+            try:
+                field_id = get_custom_field_id(field_name)
+                custom_fields_array.append({"field": field_id, "value": normalized_value})
+                logger.info(
+                    f"[{task_id}] Mapped custom field '{field_name}' to ID {field_id} with value '{normalized_value}'"
+                )
+            except ValueError as e:
+                logger.warning(f"[{task_id}] {str(e)}, skipping this field")
+                continue
+
+    if not custom_fields_array:
+        logger.info(f"[{task_id}] No valid custom fields to set")
+        return
+
+    # PATCH the document with custom fields
+    url = _paperless_api_url(f"/api/documents/{doc_id}/")
+    payload = {"custom_fields": custom_fields_array}
+
+    try:
+        logger.info(f"[{task_id}] Setting custom fields for document {doc_id}: {payload}")
+        resp = requests.patch(url, headers=_get_headers(), json=payload, timeout=settings.http_request_timeout)
+        resp.raise_for_status()
+        logger.info(f"[{task_id}] Successfully set custom fields for document {doc_id}")
+    except requests.exceptions.RequestException as exc:
+        logger.error(f"[{task_id}] Failed to set custom fields for document {doc_id}: {exc}")
+        # Don't raise - this is a non-critical failure, document is already uploaded
+        logger.error(f"[{task_id}] Response: {getattr(exc.response, 'text', '<no response>')}")
+
+
 @celery.task(base=BaseTaskWithRetry, bind=True)
 def upload_to_paperless(self, file_path: str, file_id: int = None):
     """
-    Uploads a file to Paperless-ngx.
+    Uploads a file to Paperless-ngx and sets custom fields from metadata.
 
     Args:
         file_path: Path to the file to upload
@@ -113,6 +213,20 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
         log_task_progress(task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id)
         raise ValueError(error_msg)
 
+    # Try to load metadata from accompanying JSON file
+    metadata = {}
+    json_path = os.path.splitext(file_path)[0] + ".json"
+    if os.path.exists(json_path):
+        try:
+            with open(json_path, "r", encoding="utf-8") as f:
+                metadata = json.load(f)
+            logger.info(f"[{task_id}] Loaded metadata from {json_path}")
+        except Exception as e:
+            logger.warning(f"[{task_id}] Failed to load metadata from {json_path}: {e}")
+            metadata = {}
+    else:
+        logger.info(f"[{task_id}] No metadata file found at {json_path}")
+
     # Upload the PDF
     logger.info(f"[{task_id}] Posting document to Paperless")
     log_task_progress(task_id, "post_document", "in_progress", "Posting to Paperless API", file_id=file_id)
@@ -153,6 +267,57 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
         task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id
     )
 
+    # Set custom fields if configured and metadata available
+    custom_fields_to_set = {}
+
+    # First, check for the new flexible mapping configuration
+    if settings.paperless_custom_fields_mapping:
+        try:
+            # Parse JSON mapping: {"metadata_field": "PaperlessFieldName", ...}
+            field_mapping = json.loads(settings.paperless_custom_fields_mapping)
+            logger.info(f"[{task_id}] Using custom fields mapping: {field_mapping}")
+
+            # Map each metadata field to its corresponding Paperless custom field
+            for metadata_field, paperless_field in field_mapping.items():
+                if metadata_field in metadata and metadata[metadata_field]:
+                    # Convert to string to ensure consistent comparison with UNKNOWN_VALUE
+                    value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
+                    if value and value != UNKNOWN_VALUE:
+                        custom_fields_to_set[paperless_field] = value
+                        logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
+        except json.JSONDecodeError as e:
+            logger.error(f"[{task_id}] Failed to parse PAPERLESS_CUSTOM_FIELDS_MAPPING: {e}")
+        except Exception as e:
+            logger.error(f"[{task_id}] Error processing custom fields mapping: {e}")
+
+    # Fallback to legacy single-field configuration for backward compatibility
+    if settings.paperless_custom_field_absender and metadata.get("absender"):
+        # Only add if not already set by the mapping
+        if settings.paperless_custom_field_absender not in custom_fields_to_set:
+            custom_fields_to_set[settings.paperless_custom_field_absender] = metadata.get("absender")
+            logger.debug(f"[{task_id}] Using legacy absender field configuration")
+
+    # Set custom fields if we have any to set
+    if custom_fields_to_set:
+        logger.info(f"[{task_id}] Setting {len(custom_fields_to_set)} custom field(s) for document {doc_id}")
+        log_task_progress(task_id, "set_custom_fields", "in_progress", "Setting custom fields", file_id=file_id)
+
+        try:
+            set_document_custom_fields(doc_id, custom_fields_to_set, task_id)
+            log_task_progress(
+                task_id,
+                "set_custom_fields",
+                "success",
+                f"Set {len(custom_fields_to_set)} custom field(s)",
+                file_id=file_id,
+            )
+        except Exception as e:
+            logger.error(f"[{task_id}] Failed to set custom fields: {e}")
+            log_task_progress(task_id, "set_custom_fields", "failure", f"Failed: {str(e)}", file_id=file_id)
+            # Don't fail the entire upload if custom fields fail
+    else:
+        logger.debug(f"[{task_id}] No custom fields configured or no metadata available")
+
     return {
         "status": "Completed",
         "paperless_task_id": raw_task_id,

docs/ConfigurationGuide.md

@@ -213,9 +213,44 @@ SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://trusted
 ### Paperless NGX
 
 | **Variable**                        | **Description**                                                                                     |
-|-------------------------------|-----------------------------------------------------|
+|-------------------------------------|-----------------------------------------------------------------------------------------------------|
 | `PAPERLESS_NGX_API_TOKEN`           | API token for Paperless NGX.                                                                        |
 | `PAPERLESS_HOST`                    | Root URL for Paperless NGX (e.g. `https://paperless.example.com`).                                 |
+| `PAPERLESS_CUSTOM_FIELD_ABSENDER`   | (Optional, Legacy) Name of the custom field in Paperless-ngx to store the sender ("absender") information. If set, the extracted sender will be automatically set as a custom field after document upload. Example: `Absender` or `Sender` |
+| `PAPERLESS_CUSTOM_FIELDS_MAPPING`   | (Optional, Recommended) JSON mapping of extracted metadata fields to Paperless custom field names. This allows you to map multiple fields at once. Format: `{"metadata_field": "CustomFieldName", ...}`. See examples below. |
+
+#### Custom Fields Mapping Examples
+
+**Single Field (Legacy Method)**:
+```bash
+PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
+```
+
+**Multiple Fields (Recommended Method)**:
+```bash
+# Map multiple metadata fields to custom fields in Paperless
+PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language"}'
+```
+
+**All Available Metadata Fields**:
+DocuElevate extracts the following fields that can be mapped to Paperless custom fields:
+- `absender` - Sender/author of the document
+- `empfaenger` - Recipient of the document
+- `correspondent` - The issuing entity/company (shortened name)
+- `document_type` - Type classification (Invoice, Contract, etc.)
+- `language` - Document language (ISO 639-1 code, e.g., "de", "en")
+- `kommunikationsart` - Communication type (German classification)
+- `kommunikationskategorie` - Communication category (German classification)
+- `reference_number` - Invoice/order/reference number if found
+- `title` - Human-readable document title
+- `tags` - List of thematic keywords (array)
+
+**Complete Example**:
+```bash
+PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "correspondent": "Correspondent", "language": "Language", "reference_number": "ReferenceNumber"}'
+```
+
+**Note**: Custom fields must be created in your Paperless-ngx instance before DocuElevate can use them. The field names in the mapping (right side of the JSON) must **exactly** match the names in Paperless (case-sensitive).
 
 ### Dropbox
 

docs/UserGuide.md

@@ -145,6 +145,50 @@ Depending on the system configuration, DocuElevate can perform:
 - **PDF Conversion** - Convert various file formats to PDF
 - **Document Distribution** - Store documents in Dropbox, Nextcloud, or Paperless NGX
 
+### Paperless-ngx Integration
+
+If Paperless-ngx is configured, DocuElevate can automatically upload processed documents and set custom fields with extracted metadata:
+
+1. **Automatic Upload**: After processing, documents are automatically uploaded to Paperless-ngx
+2. **Custom Fields Support**: DocuElevate can automatically populate custom fields in Paperless with extracted metadata
+3. **Flexible Field Mapping**: Map any extracted metadata field to any custom field in Paperless
+
+#### Setting Up Custom Fields
+
+**Basic Setup (Single Field)**:
+```bash
+# In your .env file
+PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
+```
+
+**Advanced Setup (Multiple Fields)**:
+```bash
+# Map multiple metadata fields to Paperless custom fields
+PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}'
+```
+
+**Available Metadata Fields**:
+- `absender` - Sender/author
+- `empfaenger` - Recipient
+- `correspondent` - Issuing company (short name)
+- `language` - Document language (e.g., "de", "en")
+- `document_type` - Classification (Invoice, Contract, etc.)
+- `reference_number` - Invoice/order/reference number
+- `kommunikationsart` - Communication type
+- `kommunikationskategorie` - Communication category
+- And more...
+
+**Requirements**:
+- Create the custom fields in Paperless-ngx first (Settings → Custom Fields)
+- The field names in your configuration must exactly match the names in Paperless
+- Ensure `PAPERLESS_HOST` and `PAPERLESS_NGX_API_TOKEN` are configured
+
+**How It Works**:
+1. DocuElevate extracts metadata from your documents using AI
+2. The document is uploaded to Paperless-ngx
+3. After successful upload, custom fields are automatically populated
+4. You can view the populated fields in your Paperless-ngx document details
+
 ## API Access
 
 For programmatic access, DocuElevate provides a comprehensive REST API: