gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #224 from christianlouis/copilot/add-absender-custom-field

Browse Source 
Add custom fields support for Paperless-ngx uploads
This commit is contained in:
Christian Krakau-Louis
2026-02-10 16:06:06 +01:00
committed by GitHub
parent 8c0fc30d4c 742b4e2e8c
commit b4a17adf56
5 changed files with 265 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.demo
+12
View File
@@ -117,6 +117,18 @@ NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
# Paperless-ngx
PAPERLESS_HOST=https://paperless.example.com
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
# If set, the extracted sender information will be automatically set as a custom field in Paperless
# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
# PAPERLESS_CUSTOM_FIELD_ABSENDER=
# Optional: JSON mapping of metadata fields to Paperless custom field names
# This allows you to map multiple extracted metadata fields to custom fields in Paperless
# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...}
# Available metadata fields: absender, empfaenger, correspondent, document_type, language,
# kommunikationsart, kommunikationskategorie, reference_number, etc.
# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
# PAPERLESS_CUSTOM_FIELDS_MAPPING=
# Dropbox
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
app/config.py
+4
View File
@@ -31,6 +31,10 @@ class Settings(BaseSettings):
# Making Paperless optional
paperless_ngx_api_token: Optional[str] = None
paperless_host: Optional[str] = None
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
# JSON mapping of metadata field names to Paperless custom field names
# Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
paperless_custom_fields_mapping: Optional[str] = None
azure_ai_key: str
azure_region: str
app/tasks/upload_to_paperless.py
+166 -1
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import json
import logging
import os
import time
@@ -15,6 +16,8 @@ logger = logging.getLogger(__name__)
POLL_MAX_ATTEMPTS = 10
POLL_INTERVAL_SEC = 3
# Sentinel value used to indicate missing/unknown metadata that should not be set as custom fields
METADATA_UNKNOWN_PLACEHOLDER = "Unknown"
def _get_headers():
@@ -33,6 +36,25 @@ def _paperless_api_url(path: str) -> str:
return f"{host}{path}"
def normalize_metadata_value(value) -> str:
"""
Normalizes a metadata value to a string suitable for Paperless custom fields.
Args:
value: The metadata value to normalize
Returns:
Empty string if value should be excluded, otherwise the string representation
"""
if value is None:
return ""
str_value = str(value)
# Exclude empty strings and the "Unknown" placeholder
if not str_value or str_value == METADATA_UNKNOWN_PLACEHOLDER:
return ""
return str_value
def poll_task_for_document_id(task_id: str) -> int:
"""
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
@@ -78,10 +100,88 @@ def poll_task_for_document_id(task_id: str) -> int:
raise TimeoutError(f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts.")
def get_custom_field_id(field_name: str) -> int:
"""
Retrieves the ID of a custom field by its name from Paperless-ngx.
Args:
field_name: The name of the custom field to look up
Returns:
The integer ID of the custom field
Raises:
ValueError: If the custom field is not found
"""
url = _paperless_api_url("/api/custom_fields/")
try:
resp = requests.get(url, headers=_get_headers(), timeout=settings.http_request_timeout)
resp.raise_for_status()
data = resp.json()
# Handle paginated response
results = data.get("results", []) if isinstance(data, dict) else data
for field in results:
if field.get("name") == field_name:
return field.get("id")
raise ValueError(f"Custom field '{field_name}' not found in Paperless-ngx")
except requests.exceptions.RequestException as exc:
logger.error(f"Failed to retrieve custom fields from Paperless: {exc}")
raise
def set_document_custom_fields(doc_id: int, custom_fields: dict, task_id: str) -> None:
"""
Updates custom fields for a document in Paperless-ngx using PATCH.
Args:
doc_id: The Paperless document ID
custom_fields: Dictionary mapping field names to values
task_id: Task ID for logging
"""
if not custom_fields:
return
# Build custom_fields array for PATCH request
custom_fields_array = []
for field_name, value in custom_fields.items():
normalized_value = normalize_metadata_value(value)
if normalized_value: # Only set non-empty values
try:
field_id = get_custom_field_id(field_name)
custom_fields_array.append({"field": field_id, "value": normalized_value})
logger.info(
f"[{task_id}] Mapped custom field '{field_name}' to ID {field_id} with value '{normalized_value}'"
)
except ValueError as e:
logger.warning(f"[{task_id}] {str(e)}, skipping this field")
continue
if not custom_fields_array:
logger.info(f"[{task_id}] No valid custom fields to set")
return
# PATCH the document with custom fields
url = _paperless_api_url(f"/api/documents/{doc_id}/")
payload = {"custom_fields": custom_fields_array}
try:
logger.info(f"[{task_id}] Setting custom fields for document {doc_id}: {payload}")
resp = requests.patch(url, headers=_get_headers(), json=payload, timeout=settings.http_request_timeout)
resp.raise_for_status()
logger.info(f"[{task_id}] Successfully set custom fields for document {doc_id}")
except requests.exceptions.RequestException as exc:
logger.error(f"[{task_id}] Failed to set custom fields for document {doc_id}: {exc}")
# Don't raise - this is a non-critical failure, document is already uploaded
logger.error(f"[{task_id}] Response: {getattr(exc.response, 'text', '<no response>')}")
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_paperless(self, file_path: str, file_id: int = None):
"""
Uploads a file to Paperless-ngx.
Uploads a file to Paperless-ngx and sets custom fields from metadata.
Args:
file_path: Path to the file to upload
@@ -113,6 +213,20 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
log_task_progress(task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
# Try to load metadata from accompanying JSON file
metadata = {}
json_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(json_path):
try:
with open(json_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"[{task_id}] Loaded metadata from {json_path}")
except Exception as e:
logger.warning(f"[{task_id}] Failed to load metadata from {json_path}: {e}")
metadata = {}
else:
logger.info(f"[{task_id}] No metadata file found at {json_path}")
# Upload the PDF
logger.info(f"[{task_id}] Posting document to Paperless")
log_task_progress(task_id, "post_document", "in_progress", "Posting to Paperless API", file_id=file_id)
@@ -153,6 +267,57 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id
)
# Set custom fields if configured and metadata available
custom_fields_to_set = {}
# First, check for the new flexible mapping configuration
if settings.paperless_custom_fields_mapping:
try:
# Parse JSON mapping: {"metadata_field": "PaperlessFieldName", ...}
field_mapping = json.loads(settings.paperless_custom_fields_mapping)
logger.info(f"[{task_id}] Using custom fields mapping: {field_mapping}")
# Map each metadata field to its corresponding Paperless custom field
for metadata_field, paperless_field in field_mapping.items():
if metadata_field in metadata and metadata[metadata_field]:
# Convert to string to ensure consistent comparison with UNKNOWN_VALUE
value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
if value and value != UNKNOWN_VALUE:
custom_fields_to_set[paperless_field] = value
logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
except json.JSONDecodeError as e:
logger.error(f"[{task_id}] Failed to parse PAPERLESS_CUSTOM_FIELDS_MAPPING: {e}")
except Exception as e:
logger.error(f"[{task_id}] Error processing custom fields mapping: {e}")
# Fallback to legacy single-field configuration for backward compatibility
if settings.paperless_custom_field_absender and metadata.get("absender"):
# Only add if not already set by the mapping
if settings.paperless_custom_field_absender not in custom_fields_to_set:
custom_fields_to_set[settings.paperless_custom_field_absender] = metadata.get("absender")
logger.debug(f"[{task_id}] Using legacy absender field configuration")
# Set custom fields if we have any to set
if custom_fields_to_set:
logger.info(f"[{task_id}] Setting {len(custom_fields_to_set)} custom field(s) for document {doc_id}")
log_task_progress(task_id, "set_custom_fields", "in_progress", "Setting custom fields", file_id=file_id)
try:
set_document_custom_fields(doc_id, custom_fields_to_set, task_id)
log_task_progress(
task_id,
"set_custom_fields",
"success",
f"Set {len(custom_fields_to_set)} custom field(s)",
file_id=file_id,
)
except Exception as e:
logger.error(f"[{task_id}] Failed to set custom fields: {e}")
log_task_progress(task_id, "set_custom_fields", "failure", f"Failed: {str(e)}", file_id=file_id)
# Don't fail the entire upload if custom fields fail
else:
logger.debug(f"[{task_id}] No custom fields configured or no metadata available")
return {
"status": "Completed",
"paperless_task_id": raw_task_id,
docs/ConfigurationGuide.md
+39 -4
View File
@@ -212,10 +212,45 @@ SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://trusted
### Paperless NGX
| **Variable** | **Description** |
|-------------------------------|-----------------------------------------------------|
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. |
| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
| **Variable** | **Description** |
|-------------------------------------|-----------------------------------------------------------------------------------------------------|
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. |
| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
| `PAPERLESS_CUSTOM_FIELD_ABSENDER` | (Optional, Legacy) Name of the custom field in Paperless-ngx to store the sender ("absender") information. If set, the extracted sender will be automatically set as a custom field after document upload. Example: `Absender` or `Sender` |
| `PAPERLESS_CUSTOM_FIELDS_MAPPING` | (Optional, Recommended) JSON mapping of extracted metadata fields to Paperless custom field names. This allows you to map multiple fields at once. Format: `{"metadata_field": "CustomFieldName", ...}`. See examples below. |
#### Custom Fields Mapping Examples
**Single Field (Legacy Method)**:
```bash
PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
```
**Multiple Fields (Recommended Method)**:
```bash
# Map multiple metadata fields to custom fields in Paperless
PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language"}'
```
**All Available Metadata Fields**:
DocuElevate extracts the following fields that can be mapped to Paperless custom fields:
- `absender` - Sender/author of the document
- `empfaenger` - Recipient of the document
- `correspondent` - The issuing entity/company (shortened name)
- `document_type` - Type classification (Invoice, Contract, etc.)
- `language` - Document language (ISO 639-1 code, e.g., "de", "en")
- `kommunikationsart` - Communication type (German classification)
- `kommunikationskategorie` - Communication category (German classification)
- `reference_number` - Invoice/order/reference number if found
- `title` - Human-readable document title
- `tags` - List of thematic keywords (array)
**Complete Example**:
```bash
PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "correspondent": "Correspondent", "language": "Language", "reference_number": "ReferenceNumber"}'
```
**Note**: Custom fields must be created in your Paperless-ngx instance before DocuElevate can use them. The field names in the mapping (right side of the JSON) must **exactly** match the names in Paperless (case-sensitive).
### Dropbox
docs/UserGuide.md
+44
View File
@@ -145,6 +145,50 @@ Depending on the system configuration, DocuElevate can perform:
- **PDF Conversion** - Convert various file formats to PDF
- **Document Distribution** - Store documents in Dropbox, Nextcloud, or Paperless NGX
### Paperless-ngx Integration
If Paperless-ngx is configured, DocuElevate can automatically upload processed documents and set custom fields with extracted metadata:
1. **Automatic Upload**: After processing, documents are automatically uploaded to Paperless-ngx
2. **Custom Fields Support**: DocuElevate can automatically populate custom fields in Paperless with extracted metadata
3. **Flexible Field Mapping**: Map any extracted metadata field to any custom field in Paperless
#### Setting Up Custom Fields
**Basic Setup (Single Field)**:
```bash
# In your .env file
PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
```
**Advanced Setup (Multiple Fields)**:
```bash
# Map multiple metadata fields to Paperless custom fields
PAPERLESS_CUSTOM_FIELDS_MAPPING='{"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}'
```
**Available Metadata Fields**:
- `absender` - Sender/author
- `empfaenger` - Recipient
- `correspondent` - Issuing company (short name)
- `language` - Document language (e.g., "de", "en")
- `document_type` - Classification (Invoice, Contract, etc.)
- `reference_number` - Invoice/order/reference number
- `kommunikationsart` - Communication type
- `kommunikationskategorie` - Communication category
- And more...
**Requirements**:
- Create the custom fields in Paperless-ngx first (Settings → Custom Fields)
- The field names in your configuration must exactly match the names in Paperless
- Ensure `PAPERLESS_HOST` and `PAPERLESS_NGX_API_TOKEN` are configured
**How It Works**:
1. DocuElevate extracts metadata from your documents using AI
2. The document is uploaded to Paperless-ngx
3. After successful upload, custom fields are automatically populated
4. You can view the populated fields in your Paperless-ngx document details
## API Access
For programmatic access, DocuElevate provides a comprehensive REST API: