Merge pull request #406 from christianlouis/copilot/add-full-text-search-feature

feat: full-text search (Meilisearch) + Helm chart for Kubernetes deployment
This commit is contained in:
Christian Krakau-Louis
2026-02-25 14:46:15 +01:00
committed by GitHub
29 changed files with 2361 additions and 153 deletions
+11
View File
@@ -299,3 +299,14 @@ NOTIFY_ON_FILE_PROCESSED=True
# Uptime Kuma
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
UPTIME_KUMA_PING_INTERVAL=5
# **Full-Text Search (Meilisearch)**
# URL for the Meilisearch instance.
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
# so container-to-container networking works without extra configuration.
# Override to "http://localhost:7700" only when running the API process outside Docker.
MEILISEARCH_URL=http://meilisearch:7700
# Optional master/API key for secured Meilisearch instances
# MEILISEARCH_API_KEY=your_master_key_here
MEILISEARCH_INDEX_NAME=documents
ENABLE_SEARCH=True
+2
View File
@@ -15,6 +15,7 @@ from app.api.logs import router as logs_router
from app.api.onedrive import router as onedrive_router
from app.api.openai import router as openai_router
from app.api.process import router as process_router
from app.api.search import router as search_router
from app.api.settings import router as settings_router
from app.api.url_upload import router as url_upload_router
@@ -40,3 +41,4 @@ router.include_router(google_drive_router)
router.include_router(logs_router)
router.include_router(settings_router)
router.include_router(url_upload_router)
router.include_router(search_router)
+99
View File
@@ -0,0 +1,99 @@
"""Full-text search API endpoints.
Provides document search across OCR text, AI metadata, filenames, and tags
via Meilisearch. Designed to serve as the backend for the UI search bar on
the /files page and as a standalone API for integrations.
Future extension point: the OCR text stored in the index is also suitable
for RAG (Retrieval Augmented Generation) chatbot workflows.
"""
import logging
from typing import Optional
from fastapi import APIRouter, Query, Request
from app.auth import require_login
from app.utils.meilisearch_client import search_documents
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/search")
@require_login
def search_api(
request: Request,
q: str = Query(..., min_length=1, max_length=512, description="Full-text search query"),
mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"),
document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"),
language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"),
date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"),
date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"),
page: int = Query(1, ge=1, description="Page number (1-based)"),
per_page: int = Query(20, ge=1, le=100, description="Results per page"),
):
"""Search documents by full text, metadata, and tags.
Searches across:
- Document title and filename
- OCR / extracted text
- Tags, sender, recipient, document type
- Correspondent and reference number
Results are ranked by Meilisearch relevance and include highlighted
snippets showing where the query terms matched.
Query Parameters:
- q: Search query (required)
- mime_type: Filter by MIME type
- document_type: Filter by document type
- language: Filter by language code
- date_from: Unix timestamp lower bound
- date_to: Unix timestamp upper bound
- page: Page number (default: 1)
- per_page: Results per page (default: 20, max: 100)
Example:
```
GET /api/search?q=invoice&document_type=Invoice&date_from=1704067200&page=1&per_page=20
```
Response:
```json
{
"results": [
{
"file_id": 42,
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
"document_title": "Amazon Invoice January 2026",
"document_type": "Invoice",
"tags": ["amazon", "invoice"],
"_formatted": {
"document_title": "Amazon <mark>Invoice</mark> January 2026",
"ocr_text": "...total amount of the <mark>invoice</mark> is..."
}
}
],
"total": 42,
"page": 1,
"pages": 3,
"query": "invoice"
}
```
"""
logger.info(f"Search request: q={q!r}, mime_type={mime_type}, page={page}, per_page={per_page}")
result = search_documents(
q,
mime_type=mime_type,
document_type=document_type,
language=language,
date_from=date_from,
date_to=date_to,
page=page,
per_page=per_page,
)
return result
+9
View File
@@ -207,6 +207,15 @@ class Settings(BaseSettings):
uptime_kuma_url: Optional[str] = None
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
# Meilisearch settings (full-text search engine)
# Default uses the Docker Compose / K8s service name so container-to-container
# networking works without any extra configuration. Override to
# "http://localhost:7700" only when running the API process outside of Docker.
meilisearch_url: str = "http://meilisearch:7700"
meilisearch_api_key: Optional[str] = None # Master or API key (optional for local dev)
meilisearch_index_name: str = "documents"
enable_search: bool = True # Enable Meilisearch full-text search integration
# HTTP request settings
http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
+9
View File
@@ -55,6 +55,15 @@ class FileRecord(Base):
# If this is a duplicate, record the ID of the original file for reference
duplicate_of_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)
# Full OCR/extracted text for full-text search and RAG
ocr_text = Column(Text, nullable=True)
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
ai_metadata = Column(Text, nullable=True)
# Human-readable document title from AI metadata
document_title = Column(String, nullable=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now())
+19 -1
View File
@@ -195,8 +195,26 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
original_file_path = file_record.original_file_path
# Update the processed_file_path in the database
file_record.processed_file_path = final_file_path
# Persist extracted text and AI metadata to DB for full-text search / RAG
file_record.ocr_text = extracted_text or None
if metadata:
try:
file_record.ai_metadata = json.dumps(metadata, ensure_ascii=False)
except Exception as json_exc:
logger.warning(f"[{task_id}] Could not serialise ai_metadata: {json_exc}")
file_record.document_title = (
metadata.get("title") or metadata.get("filename") or file_record.original_filename
)
db.commit()
logger.info(f"[{task_id}] Updated database with processed_file_path: {final_file_path}")
logger.info(f"[{task_id}] Updated database with processed_file_path and search fields")
# Index into Meilisearch for full-text search (non-blocking, best-effort)
try:
from app.utils.meilisearch_client import index_document
index_document(file_record, extracted_text or "", metadata or {})
except Exception as search_exc:
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
# Persist the metadata into a JSON file with the same base name.
# Include file path references for traceability
+286
View File
@@ -0,0 +1,286 @@
"""Meilisearch client utilities for full-text document search.
This module provides functions for indexing documents into Meilisearch
and searching across OCR text, AI metadata, filenames, and tags.
The search index is designed to support future RAG (Retrieval Augmented
Generation) workflows by storing full document text alongside structured
metadata fields.
"""
import logging
from typing import Any, Optional
logger = logging.getLogger(__name__)
# Index settings applied once at index creation / first use
_INDEX_SETTINGS = {
"searchableAttributes": [
"document_title",
"original_filename",
"ocr_text",
"tags",
"sender",
"recipient",
"document_type",
"correspondent",
],
"filterableAttributes": [
"mime_type",
"document_type",
"language",
"tags",
"created_at_ts",
"file_id",
],
"sortableAttributes": [
"created_at_ts",
"file_size",
],
"displayedAttributes": [
"file_id",
"original_filename",
"document_title",
"document_type",
"tags",
"sender",
"recipient",
"correspondent",
"language",
"reference_number",
"mime_type",
"file_size",
"created_at_ts",
"ocr_text",
],
"rankingRules": [
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
],
}
def get_meilisearch_client():
"""Return a configured Meilisearch client, or None if unavailable/disabled."""
try:
import meilisearch
from app.config import settings
if not settings.enable_search:
return None
kwargs: dict[str, Any] = {"url": settings.meilisearch_url}
if settings.meilisearch_api_key:
kwargs["api_key"] = settings.meilisearch_api_key
client = meilisearch.Client(**kwargs)
return client
except ImportError:
logger.warning("meilisearch package not installed; search disabled")
return None
except Exception as exc:
logger.warning(f"Could not connect to Meilisearch: {exc}")
return None
def _get_or_create_index(client):
"""Get the documents index, creating it with settings if it doesn't exist."""
from app.config import settings
index_name = settings.meilisearch_index_name
try:
index = client.get_index(index_name)
except Exception:
# Index doesn't exist create it with file_id as primary key
task = client.create_index(index_name, {"primaryKey": "file_id"})
client.wait_for_task(task.task_uid)
index = client.get_index(index_name)
# Apply search settings
try:
task = index.update_settings(_INDEX_SETTINGS)
client.wait_for_task(task.task_uid)
except Exception as exc:
logger.warning(f"Could not update Meilisearch index settings: {exc}")
return index
def _build_document(file_record, text: str, metadata: dict) -> dict:
"""Build a Meilisearch document from a FileRecord and extracted content."""
tags = metadata.get("tags", [])
if isinstance(tags, str):
tags = [t.strip() for t in tags.split(",") if t.strip()]
# Unix timestamp for sorting/filtering
created_at_ts = 0
if file_record.created_at:
try:
created_at_ts = int(file_record.created_at.timestamp())
except Exception as ts_exc: # noqa: BLE001
logger.debug(f"Could not convert created_at to timestamp: {ts_exc}")
return {
"file_id": file_record.id,
"original_filename": file_record.original_filename or "",
"document_title": metadata.get("title") or metadata.get("filename") or file_record.original_filename or "",
"document_type": metadata.get("document_type") or metadata.get("kommunikationsart") or "",
"tags": tags,
"sender": metadata.get("absender") or "",
"recipient": metadata.get("empfaenger") or "",
"correspondent": metadata.get("correspondent") or "",
"language": metadata.get("language") or "",
"reference_number": metadata.get("reference_number") or "",
"mime_type": file_record.mime_type or "",
"file_size": file_record.file_size or 0,
"created_at_ts": created_at_ts,
"ocr_text": text or "",
}
def index_document(file_record, text: str, metadata: dict) -> bool:
"""Index a document in Meilisearch.
Args:
file_record: FileRecord ORM instance with at minimum .id set.
text: Full OCR / extracted text for the document.
metadata: AI-extracted metadata dict.
Returns:
True if indexing succeeded, False otherwise.
"""
client = get_meilisearch_client()
if client is None:
return False
try:
index = _get_or_create_index(client)
doc = _build_document(file_record, text, metadata)
task = index.add_documents([doc])
logger.info(f"Queued Meilisearch indexing for file_id={file_record.id} (task_uid={task.task_uid})")
return True
except Exception as exc:
logger.warning(f"Meilisearch indexing failed for file_id={file_record.id}: {exc}")
return False
def delete_document(file_id: int) -> bool:
"""Remove a document from the Meilisearch index.
Args:
file_id: The database ID of the file to remove.
Returns:
True if deletion succeeded, False otherwise.
"""
client = get_meilisearch_client()
if client is None:
return False
try:
from app.config import settings
index = client.get_index(settings.meilisearch_index_name)
task = index.delete_document(file_id)
logger.info(f"Queued Meilisearch deletion for file_id={file_id} (task_uid={task.task_uid})")
return True
except Exception as exc:
logger.warning(f"Meilisearch deletion failed for file_id={file_id}: {exc}")
return False
def search_documents(
query: str,
*,
mime_type: Optional[str] = None,
document_type: Optional[str] = None,
language: Optional[str] = None,
date_from: Optional[int] = None,
date_to: Optional[int] = None,
page: int = 1,
per_page: int = 20,
) -> dict:
"""Search documents in Meilisearch.
Args:
query: Full-text search query string.
mime_type: Optional MIME-type filter.
document_type: Optional document type filter.
language: Optional language filter (ISO 639-1, e.g. "de").
date_from: Optional lower bound Unix timestamp for created_at.
date_to: Optional upper bound Unix timestamp for created_at.
page: 1-based page number.
per_page: Results per page (max 100).
Returns:
Dict with keys: results, total, page, pages, query.
Returns empty results dict on any error.
"""
empty: dict = {"results": [], "total": 0, "page": page, "pages": 0, "query": query}
client = get_meilisearch_client()
if client is None:
return empty
try:
index = _get_or_create_index(client)
# Build filter expressions
filters: list[str] = []
if mime_type:
filters.append(f'mime_type = "{mime_type}"')
if document_type:
filters.append(f'document_type = "{document_type}"')
if language:
filters.append(f'language = "{language}"')
if date_from is not None:
filters.append(f"created_at_ts >= {date_from}")
if date_to is not None:
filters.append(f"created_at_ts <= {date_to}")
search_params: dict[str, Any] = {
"offset": (page - 1) * per_page,
"limit": per_page,
"attributesToHighlight": ["document_title", "original_filename", "ocr_text", "tags"],
"highlightPreTag": "<mark>",
"highlightPostTag": "</mark>",
"attributesToCrop": ["ocr_text"],
"cropLength": 200,
}
if filters:
search_params["filter"] = " AND ".join(filters)
result = index.search(query, search_params)
hits = result.get("hits", [])
total = result.get("estimatedTotalHits", result.get("nbHits", len(hits)))
# Attach highlights to each hit
formatted_results = []
for hit in hits:
formatted = dict(hit)
# Include formatted (highlighted) snippets if available
if "_formatted" in hit:
formatted["_formatted"] = hit["_formatted"]
# Exclude raw ocr_text from results (use _formatted snippet instead)
formatted.pop("ocr_text", None)
formatted_results.append(formatted)
pages = (total + per_page - 1) // per_page if total > 0 else 0
return {
"results": formatted_results,
"total": total,
"page": page,
"pages": pages,
"query": query,
}
except Exception as exc:
logger.warning(f"Meilisearch search failed for query '{query}': {exc}")
return empty
+9
View File
@@ -59,6 +59,15 @@ services:
container_name: gotenberg
restart: always
meilisearch:
image: getmeili/meilisearch:latest
container_name: document_meilisearch
restart: always
environment:
- MEILI_NO_ANALYTICS=true
volumes:
- /var/docparse/meilisearch:/meili_data
redis:
image: redis:alpine
container_name: document_redis
+310 -151
View File
@@ -1,40 +1,48 @@
# Deployment Guide
This guide provides instructions for deploying DocuElevate in various environments.
This guide covers all supported deployment methods for DocuElevate.
## Table of Contents
- [Prerequisites](#prerequisites)
- [Docker Compose Deployment](#docker-compose-deployment) *(recommended for single-server)*
- [Kubernetes / Helm Deployment](#kubernetes--helm-deployment) *(recommended for production scale-out)*
- [Production Considerations](#production-considerations)
- [Scaling](#scaling)
- [Backup Procedures](#backup-procedures)
- [Updates](#updates)
- [Troubleshooting](#troubleshooting)
## Prerequisites
- Docker and Docker Compose
- Docker and Docker Compose **or** a Kubernetes cluster with Helm 3
- Access to required external services (if configured):
- AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider)
- Azure Document Intelligence
- Dropbox API
- Nextcloud instance
- Paperless NGX instance
- SMTP server (for email notifications)
- IMAP server(s) (for email attachment processing)
- Notification services (Discord, Telegram, etc. for system alerts)
- Dropbox, Google Drive, OneDrive, S3, or other storage APIs
- SMTP / IMAP server (for email processing)
- Notification services (Discord, Telegram, etc.)
## Docker Deployment
---
Docker is the recommended deployment method for DocuElevate.
## Docker Compose Deployment
Docker Compose is the quickest way to run DocuElevate on a single server.
### Step 1: Clone the Repository
```bash
git clone https://github.com/christianlouis/document-processor.git
cd document-processor
git clone https://github.com/christianlouis/DocuElevate.git
cd DocuElevate
```
### Step 2: Configure Environment Variables
Create a `.env` file based on the example:
```bash
cp .env.example .env
cp .env.demo .env
```
Edit the `.env` file with your configuration settings. See the [Configuration Guide](ConfigurationGuide.md) for details.
Edit `.env` with your settings. See the [Configuration Guide](ConfigurationGuide.md) for all options.
### Step 3: Run with Docker Compose
@@ -42,41 +50,237 @@ Edit the `.env` file with your configuration settings. See the [Configuration Gu
docker-compose up -d
```
This will start:
- The DocuElevate API server
- A worker for background tasks
- Redis for message broker and result storage
- Gotenberg for PDF processing
This starts:
| Service | Purpose |
|---------|---------|
| `api` | FastAPI web server (port 8000) |
| `worker` | Celery background task worker |
| `redis` | Message broker for Celery |
| `gotenberg` | PDF conversion (LibreOffice headless) |
| `meilisearch` | Full-text search engine (port 7700) |
### Step 4: Verify the Installation
Access the web interface at `http://localhost:8000` and the API documentation at `http://localhost:8000/docs`.
Access the web interface at `http://localhost:8000` and the API docs at `http://localhost:8000/docs`.
---
## Kubernetes / Helm Deployment
The Helm chart at `helm/docuelevate/` packages all components into a single, configurable release. It supports:
- Multiple replicas for the API and Worker
- Horizontal Pod Autoscaling (HPA)
- Bundled or external Redis
- Persistent volumes for workdir and Meilisearch data
- Alembic database migration Job (pre-install/upgrade hook)
- TLS Ingress via any controller (nginx, Traefik, etc.)
### Prerequisites
- Kubernetes 1.24+
- Helm 3.10+
- A storage class that supports **ReadWriteMany** (e.g. NFS, CephFS, Azure Files, EFS) for the shared workdir PVC when running multiple replicas. Single-replica clusters can use `ReadWriteOnce`.
- A PostgreSQL database (strongly recommended over SQLite for multi-replica).
### Quick Start
```bash
# 1. Add the Bitnami chart repository (needed for bundled Redis)
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
# 2. Update chart dependencies
helm dependency update ./helm/docuelevate
# 3. Install with a minimal values override
helm install docuelevate ./helm/docuelevate \
--namespace docuelevate --create-namespace \
--set secrets.DATABASE_URL="postgresql://user:pass@postgres:5432/docuelevate" \
--set secrets.SESSION_SECRET="$(openssl rand -hex 32)" \
--set secrets.OPENAI_API_KEY="sk-..." \
--set secrets.AZURE_AI_KEY="..." \
--set env.AZURE_ENDPOINT="https://my-resource.cognitiveservices.azure.com/" \
--set env.EXTERNAL_HOSTNAME="docuelevate.example.com"
```
### Values Reference
The full list of configurable values is in [`helm/docuelevate/values.yaml`](../helm/docuelevate/values.yaml). Key sections:
#### Container Image
```yaml
image:
repository: ghcr.io/christianlouis/docuelevate
tag: "" # defaults to chart appVersion
pullPolicy: IfNotPresent
```
#### Non-Secret Config (`env`)
```yaml
env:
WORKDIR: /workdir
AI_PROVIDER: openai
OPENAI_MODEL: gpt-4o-mini
AZURE_REGION: eastus
AZURE_ENDPOINT: "https://my-resource.cognitiveservices.azure.com/"
MEILISEARCH_URL: http://docuelevate-meilisearch:7700 # auto-resolved from service name
ENABLE_SEARCH: "true"
AUTH_ENABLED: "true"
EXTERNAL_HOSTNAME: docuelevate.example.com
```
#### Secrets (`secrets`)
All secrets are stored in a Kubernetes `Secret` and injected as environment variables.
```yaml
secrets:
DATABASE_URL: "postgresql://user:pass@postgres:5432/docuelevate"
SESSION_SECRET: "<min-32-char-random-string>"
OPENAI_API_KEY: "sk-..."
AZURE_AI_KEY: "..."
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated dev Meilisearch
# Storage provider secrets ...
```
> **Tip:** In production use an external secret manager (Vault, ESO, Sealed Secrets) and reference the secret by name instead of embedding values in values.yaml.
#### Replicas & Autoscaling
```yaml
api:
replicaCount: 2
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 8
targetCPUUtilizationPercentage: 70
worker:
replicaCount: 2
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 75
```
#### Shared Workdir PVC
```yaml
workdir:
persistence:
enabled: true
accessMode: ReadWriteMany # RWX required for multi-replica
size: 20Gi
storageClass: "nfs-client" # or leave blank for cluster default
```
#### Ingress (nginx example)
```yaml
ingress:
enabled: true
className: nginx
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "1g"
cert-manager.io/cluster-issuer: letsencrypt-prod
hosts:
- host: docuelevate.example.com
paths:
- path: /
pathType: Prefix
tls:
- secretName: docuelevate-tls
hosts:
- docuelevate.example.com
```
#### External Redis
```yaml
redis:
enabled: false # disable bundled Redis
externalRedis:
url: "redis://my-redis-host:6379/0"
```
#### Meilisearch
The bundled Meilisearch deployment is a single-replica, persistent StatefulSet-equivalent. For production, consider [Meilisearch Cloud](https://www.meilisearch.com/cloud) and point `env.MEILISEARCH_URL` at it.
```yaml
meilisearch:
enabled: true
persistence:
enabled: true
size: 10Gi
```
### Upgrading
```bash
helm upgrade docuelevate ./helm/docuelevate \
--namespace docuelevate \
-f my-values.yaml
```
The pre-upgrade hook runs `alembic upgrade head` automatically before the new pods start.
### Uninstalling
```bash
helm uninstall docuelevate --namespace docuelevate
# PVCs are NOT deleted automatically — remove manually if desired:
kubectl delete pvc -l app.kubernetes.io/instance=docuelevate -n docuelevate
```
### Kubernetes Architecture Diagram
```
Internet
[Ingress / LoadBalancer]
[API Deployment] ─────── [Worker Deployment]
│ │ │ │
│ └── shared PVC ──┘ │
│ (workdir) │
▼ ▼
[Redis Service] [Gotenberg Service]
[Meilisearch Service]
```
---
## Production Considerations
### Security Headers
### Database
DocuElevate includes built-in support for HTTP security headers to improve browser-side security. **These headers are disabled by default** since most deployments use a reverse proxy (Traefik, Nginx, etc.) that already adds these headers.
SQLite is fine for development but **not recommended for multi-replica production** deployments because it cannot be shared safely across pods. Use PostgreSQL:
#### Supported Security Headers
- **Strict-Transport-Security (HSTS)**: Forces browsers to use HTTPS for all future requests
- **Content-Security-Policy (CSP)**: Controls which resources browsers are allowed to load
- **X-Frame-Options**: Prevents the page from being loaded in frames (clickjacking protection)
- **X-Content-Type-Options**: Prevents browsers from MIME-sniffing responses
#### Reverse Proxy Deployment (Traefik, Nginx, etc.) - DEFAULT
**Most deployments use a reverse proxy**, which is why security headers are disabled by default in DocuElevate. The reverse proxy should add these headers.
```bash
# In .env file (or omit - this is the default)
SECURITY_HEADERS_ENABLED=false
```
DATABASE_URL=postgresql://docuelevate:secret@postgres-host:5432/docuelevate
```
##### Traefik Configuration Example
### Security Headers
Traefik can add security headers using middleware. Create a `docker-compose.yaml` with Traefik labels:
DocuElevate's built-in security headers are **disabled by default** since most deployments use a reverse proxy that already adds them.
```bash
# Enable only if running without a reverse proxy
SECURITY_HEADERS_ENABLED=true
```
#### Traefik (Docker Compose) example
```yaml
services:
@@ -85,35 +289,25 @@ services:
- "traefik.enable=true"
- "traefik.http.routers.docuelevate.rule=Host(`docuelevate.example.com`)"
- "traefik.http.routers.docuelevate.entrypoints=websecure"
- "traefik.http.routers.docuelevate.tls=true"
- "traefik.http.routers.docuelevate.tls.certresolver=letsencrypt"
# Security headers middleware
- "traefik.http.routers.docuelevate.middlewares=security-headers@docker"
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
- "traefik.http.middlewares.security-headers.headers.stsIncludeSubdomains=true"
- "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
```
Then set `SECURITY_HEADERS_ENABLED=false` in your `.env` file.
##### Nginx Configuration Example
Add security headers to your Nginx configuration:
#### Nginx example
```nginx
server {
listen 443 ssl http2;
server_name docuelevate.example.com;
# SSL configuration
ssl_certificate /etc/nginx/ssl/cert.pem;
ssl_certificate_key /etc/nginx/ssl/key.pem;
# Security headers
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
@@ -123,144 +317,109 @@ server {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
client_max_body_size 1g;
}
}
```
Then keep `SECURITY_HEADERS_ENABLED=false` in your `.env` file (or omit it, as this is the default).
#### Direct Deployment (No Reverse Proxy)
If you're running DocuElevate **directly without a reverse proxy**, enable security headers:
```bash
# In .env file
SECURITY_HEADERS_ENABLED=true
```
You can also configure individual headers:
```bash
SECURITY_HEADER_HSTS_ENABLED=true
SECURITY_HEADER_CSP_ENABLED=true
SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
```
**Note**: HSTS only works when serving content over HTTPS. If using HTTP for development, you can disable it:
```bash
SECURITY_HEADER_HSTS_ENABLED=false
```
#### Customizing Security Headers
If you enable security headers, you can customize individual header values in your `.env` file:
```bash
# Customize HSTS (e.g., shorter duration for testing)
SECURITY_HEADER_HSTS_VALUE="max-age=300"
# Customize CSP (e.g., allow specific external domains)
SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://cdn.example.com; style-src 'self' 'unsafe-inline';"
# Allow framing from same origin
SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="SAMEORIGIN"
```
#### Security Considerations
1. **HSTS and HTTPS**: HSTS only works over HTTPS. Ensure you have a valid SSL certificate before enabling HSTS.
2. **CSP Testing**: The default CSP policy allows inline scripts and styles for compatibility. Test thoroughly before tightening.
3. **Content-Security-Policy**: The default policy allows `'unsafe-inline'` for scripts and styles for compatibility with Tailwind CSS and inline JavaScript. For stricter security, consider using nonces or hashes.
4. **X-Frame-Options**: Set to `DENY` by default. Change to `SAMEORIGIN` if you need to embed DocuElevate in iframes on the same domain.
See the [Configuration Guide](ConfigurationGuide.md) for all security header options.
### Reverse Proxy Setup
For production use, we recommend setting up a reverse proxy (like Nginx or Traefik) to handle HTTPS and domain routing:
```nginx
server {
listen 80;
server_name docuelevate.example.com;
location / {
proxy_pass http://localhost:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
```
### Persistent Storage
The Docker setup uses volumes for persistent storage. For production, consider:
### Storage
```yaml
# Docker Compose
volumes:
- /path/to/persistent/storage:/workdir
# Helm — use a RWX storage class for multi-replica
workdir:
persistence:
size: 20Gi
accessMode: ReadWriteMany
storageClass: "nfs-client"
```
### Security
### General Security Checklist
1. **Always use HTTPS** in production
2. Enable authentication by setting `AUTH_ENABLED=true`
3. Use strong passwords for all services
4. Limit access to the Docker host
5. Regularly update the application and dependencies
2. Set `AUTH_ENABLED=true` and use a strong `SESSION_SECRET`
3. Rotate API keys and secrets regularly — see the [Credential Rotation Guide](CredentialRotationGuide.md)
4. Limit network access to Redis and Meilisearch (both should be internal-only)
5. Regularly update the container image to pick up dependency patches
---
## Scaling
For high-volume deployments:
### Docker Compose
1. Increase worker processes by adding more worker containers:
Add more worker containers:
```yaml
worker:
image: christianlouis/document-processor:latest
deploy:
replicas: 3
```
2. Consider using dedicated Redis and database servers
3. Monitor system performance and adjust resources as needed
### Kubernetes / Helm
Enable HPA:
```yaml
api:
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 8
worker:
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 10
```
---
## Monitoring
Monitor your DocuElevate deployment using:
- **Docker Compose**: `docker-compose logs -f`, `docker stats`
- **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f`
- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed.
- **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring.
- Docker's built-in logging: `docker-compose logs -f`
- Container metrics: `docker stats`
- External monitoring tools like Prometheus and Grafana
---
## Backup Procedures
Regularly back up the following:
Regularly back up:
1. The `/workdir` directory containing all processed documents
2. The database file (if using SQLite) or database contents (if using another DBMS)
3. The `.env` configuration file
1. The `/workdir` volume (all processed documents and originals)
2. The database (PostgreSQL `pg_dump` or SQLite file)
3. The Meilisearch data directory (`/meili_data`)
4. Your `.env` / Helm values file (store securely, it contains secrets)
---
## Updates
To update DocuElevate to a newer version:
### Docker Compose
```bash
# Pull the latest changes
git pull
# Pull the latest Docker images
docker-compose pull
# Restart the services
docker-compose down
docker-compose up -d
docker-compose down && docker-compose up -d
```
### Helm
```bash
helm repo update # if using a hosted chart repository
helm upgrade docuelevate ./helm/docuelevate --namespace docuelevate -f my-values.yaml
```
The migration Job runs automatically on every `helm upgrade`.
---
## Troubleshooting
See the [Troubleshooting](Troubleshooting.md) guide for common deployment issues and solutions.
See the [Troubleshooting Guide](Troubleshooting.md) for common issues and solutions.
+152
View File
@@ -418,6 +418,47 @@
</form>
</div>
<!-- Full-Text Search Section -->
<div class="filters-section" style="margin-top: 0.75rem;">
<div style="width: 100%;">
<label for="fulltext-search" style="font-weight: 600; display: block; margin-bottom: 0.4rem;">
<i class="fas fa-search"></i> Full-Text Search (OCR text, metadata, tags)
</label>
<div style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
<input
type="text"
id="fulltext-search"
placeholder="Search document content, sender, tags, type..."
style="flex: 1; min-width: 220px; padding: 0.5rem 0.75rem; border: 1px solid #d1d5db; border-radius: 0.375rem; font-size: 0.875rem;"
oninput="debounceSearch(this.value)"
>
<button
type="button"
onclick="runFullTextSearch()"
style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
>
Search
</button>
<button
type="button"
onclick="clearFullTextSearch()"
style="padding: 0.5rem 1rem; background-color: #6b7280; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
>
Clear
</button>
</div>
</div>
</div>
<!-- Full-Text Search Results Panel -->
<div id="search-results-panel" style="display: none; margin-top: 0.5rem; border: 1px solid #e5e7eb; border-radius: 0.5rem; background: white; box-shadow: 0 1px 3px rgba(0,0,0,0.08);">
<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #e5e7eb; display: flex; justify-content: space-between; align-items: center; background: #f9fafb; border-radius: 0.5rem 0.5rem 0 0;">
<span id="search-results-summary" style="font-size: 0.875rem; color: #374151;"></span>
<div id="search-results-pagination" style="display: flex; gap: 0.5rem;"></div>
</div>
<div id="search-results-list" style="padding: 0.5rem 0;"></div>
</div>
<!-- Bulk Actions Section -->
<div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
<div class="flex flex-col sm:flex-row sm:justify-between sm:items-center gap-3">
@@ -887,6 +928,117 @@
function closeUploadModal() {
uploadModal.classList.remove('active');
}
// ---- Full-Text Search (Meilisearch) ----
let _searchDebounceTimer = null;
let _searchCurrentPage = 1;
function debounceSearch(value) {
clearTimeout(_searchDebounceTimer);
if (!value || value.trim().length < 2) {
clearFullTextSearch();
return;
}
_searchDebounceTimer = setTimeout(() => {
_searchCurrentPage = 1;
runFullTextSearch();
}, 400);
}
function runFullTextSearch(page) {
const input = document.getElementById('fulltext-search');
const q = input ? input.value.trim() : '';
if (!q) { clearFullTextSearch(); return; }
if (page) _searchCurrentPage = page;
const panel = document.getElementById('search-results-panel');
const list = document.getElementById('search-results-list');
const summary = document.getElementById('search-results-summary');
const pagination = document.getElementById('search-results-pagination');
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;"><i class="fas fa-spinner fa-spin"></i> Searching…</div>';
summary.textContent = '';
pagination.innerHTML = '';
panel.style.display = 'block';
const params = new URLSearchParams({ q, page: _searchCurrentPage, per_page: 20 });
fetch(`/api/search?${params.toString()}`)
.then(r => {
if (!r.ok) throw new Error(`Search returned ${r.status}`);
return r.json();
})
.then(data => renderSearchResults(data, q))
.catch(err => {
list.innerHTML = `<div style="padding: 1rem; color: #dc2626; font-size: 0.875rem;"><i class="fas fa-exclamation-triangle"></i> Search unavailable: ${err.message}</div>`;
summary.textContent = '';
});
}
function renderSearchResults(data, q) {
const panel = document.getElementById('search-results-panel');
const list = document.getElementById('search-results-list');
const summary = document.getElementById('search-results-summary');
const pagination = document.getElementById('search-results-pagination');
const { results, total, page, pages } = data;
summary.textContent = `${total} result${total !== 1 ? 's' : ''} for "${q}"`;
if (!results || results.length === 0) {
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;">No results found.</div>';
pagination.innerHTML = '';
return;
}
list.innerHTML = results.map(hit => {
const fmt = hit._formatted || {};
const title = fmt.document_title || hit.document_title || hit.original_filename || '(untitled)';
const filename = fmt.original_filename || hit.original_filename || '';
const snippet = fmt.ocr_text || '';
const tags = Array.isArray(hit.tags) ? hit.tags.join(', ') : (hit.tags || '');
const docType = hit.document_type || '';
return `<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #f3f4f6; display: flex; gap: 0.75rem; align-items: flex-start;">
<div style="flex-shrink: 0; color: #3b82f6; font-size: 1.25rem; padding-top: 0.1rem;">
<i class="fas fa-file-pdf"></i>
</div>
<div style="flex: 1; min-width: 0;">
<div style="font-weight: 600; font-size: 0.9rem; color: #111827;">${title}</div>
${filename ? `<div style="font-size: 0.8rem; color: #6b7280; margin-top: 0.15rem;">${filename}</div>` : ''}
${docType ? `<span style="display: inline-block; margin-top: 0.25rem; padding: 0.1rem 0.5rem; background: #eff6ff; color: #1d4ed8; border-radius: 9999px; font-size: 0.75rem;">${docType}</span>` : ''}
${tags ? `<span style="display: inline-block; margin-top: 0.25rem; margin-left: 0.25rem; padding: 0.1rem 0.5rem; background: #f0fdf4; color: #15803d; border-radius: 9999px; font-size: 0.75rem;">${tags}</span>` : ''}
${snippet ? `<div style="margin-top: 0.4rem; font-size: 0.8rem; color: #374151; white-space: pre-wrap; word-break: break-word;">…${snippet}…</div>` : ''}
</div>
<div style="flex-shrink: 0;">
<a href="/files/${hit.file_id}" style="padding: 0.25rem 0.6rem; background: #f3f4f6; color: #374151; border-radius: 0.25rem; font-size: 0.8rem; text-decoration: none; white-space: nowrap;" title="View file">
<i class="fas fa-external-link-alt"></i>
</a>
</div>
</div>`;
}).join('');
// Pagination
if (pages > 1) {
const btns = [];
if (page > 1) {
btns.push(`<button onclick="runFullTextSearch(${page - 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">« Prev</button>`);
}
btns.push(`<span style="padding: 0.25rem 0.6rem; font-size: 0.8rem; color: #6b7280;">Page ${page} / ${pages}</span>`);
if (page < pages) {
btns.push(`<button onclick="runFullTextSearch(${page + 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">Next »</button>`);
}
pagination.innerHTML = btns.join('');
} else {
pagination.innerHTML = '';
}
}
function clearFullTextSearch() {
const panel = document.getElementById('search-results-panel');
const input = document.getElementById('fulltext-search');
if (panel) panel.style.display = 'none';
if (input) input.value = '';
_searchCurrentPage = 1;
}
</script>
</div>
{% endblock %}
+10
View File
@@ -0,0 +1,10 @@
# Patterns to ignore when building packages.
.DS_Store
.git
.gitignore
*.swp
*.bak
*.tmp
*.orig
*~
.vscode
+37
View File
@@ -0,0 +1,37 @@
apiVersion: v2
name: docuelevate
description: >
DocuElevate — intelligent document processing with OCR, AI metadata
extraction, full-text search (Meilisearch), and multi-cloud storage.
type: application
# Chart version — bump on every chart change (independent of appVersion).
version: 0.1.0
# Application version — kept in sync with the VERSION file.
appVersion: "0.54.0"
keywords:
- document-management
- ocr
- ai
- meilisearch
- fastapi
- celery
home: https://github.com/christianlouis/DocuElevate
sources:
- https://github.com/christianlouis/DocuElevate
maintainers:
- name: DocuElevate Contributors
url: https://github.com/christianlouis/DocuElevate
dependencies:
# Bundled Redis (optional — disable and point to an external instance via
# externalRedis.url if you already have Redis in the cluster).
- name: redis
version: "20.x.x"
repository: "https://charts.bitnami.com/bitnami"
condition: redis.enabled
+45
View File
@@ -0,0 +1,45 @@
Thank you for installing {{ .Chart.Name }} {{ .Chart.Version }}!
{{- if .Values.ingress.enabled }}
The application will be available at:
{{- range .Values.ingress.hosts }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
{{- end }}
{{- else }}
To access DocuElevate, run:
kubectl port-forward svc/{{ include "docuelevate.fullname" . }}-api {{ .Values.api.service.port }}:{{ .Values.api.service.port }} -n {{ .Release.Namespace }}
Then open http://localhost:{{ .Values.api.service.port }}
{{- end }}
API docs are available at: <your-url>/docs
{{- if not .Values.secrets.SESSION_SECRET }}
⚠️ WARNING: secrets.SESSION_SECRET is not set.
Generate one with: openssl rand -hex 32
Then pass it via: --set secrets.SESSION_SECRET=<value>
or via a values file / external secret.
{{- end }}
{{- if not .Values.secrets.DATABASE_URL }}
⚠️ WARNING: secrets.DATABASE_URL is not set.
For production, use PostgreSQL:
postgresql://user:pass@host:5432/docuelevate
SQLite with a PVC is supported but not recommended for multi-replica
deployments (use ReadWriteOnce PVC and replicaCount=1 in that case).
{{- end }}
Useful commands:
# View API logs
kubectl logs -l app.kubernetes.io/component=api -n {{ .Release.Namespace }} -f
# View worker logs
kubectl logs -l app.kubernetes.io/component=worker -n {{ .Release.Namespace }} -f
# Run a database migration manually
kubectl create job --from=cronjob/{{ include "docuelevate.fullname" . }}-migrate migrate-manual -n {{ .Release.Namespace }}
+108
View File
@@ -0,0 +1,108 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "docuelevate.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this.
If release name contains chart name it will be used as a full name.
*/}}
{{- define "docuelevate.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart label value: "chart-name-version"
*/}}
{{- define "docuelevate.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels applied to every resource.
*/}}
{{- define "docuelevate.labels" -}}
helm.sh/chart: {{ include "docuelevate.chart" . }}
{{ include "docuelevate.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels.
*/}}
{{- define "docuelevate.selectorLabels" -}}
app.kubernetes.io/name: {{ include "docuelevate.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
ServiceAccount name.
*/}}
{{- define "docuelevate.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "docuelevate.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
Image reference, using appVersion as default tag.
*/}}
{{- define "docuelevate.image" -}}
{{- $tag := .Values.image.tag | default .Chart.AppVersion }}
{{- printf "%s:%s" .Values.image.repository $tag }}
{{- end }}
{{/*
Resolve REDIS_URL: prefer externalRedis.url, then secrets.REDIS_URL,
then fall back to the bundled Redis service URL.
*/}}
{{- define "docuelevate.redisUrl" -}}
{{- if .Values.externalRedis.url }}
{{- .Values.externalRedis.url }}
{{- else if .Values.secrets.REDIS_URL }}
{{- .Values.secrets.REDIS_URL }}
{{- else if .Values.redis.enabled }}
{{- printf "redis://%s-redis-master:6379/0" .Release.Name }}
{{- else }}
{{- "" }}
{{- end }}
{{- end }}
{{/*
Resolve GOTENBERG_URL use env override if set, otherwise build from service name.
*/}}
{{- define "docuelevate.gotenbergUrl" -}}
{{- if .Values.env.GOTENBERG_URL }}
{{- tpl .Values.env.GOTENBERG_URL . }}
{{- else }}
{{- printf "http://%s-gotenberg:3000" (include "docuelevate.fullname" .) }}
{{- end }}
{{- end }}
{{/*
Resolve MEILISEARCH_URL use env override if set, otherwise build from service name.
*/}}
{{- define "docuelevate.meilisearchUrl" -}}
{{- if .Values.env.MEILISEARCH_URL }}
{{- tpl .Values.env.MEILISEARCH_URL . }}
{{- else }}
{{- printf "http://%s-meilisearch:7700" (include "docuelevate.fullname" .) }}
{{- end }}
{{- end }}
@@ -0,0 +1,93 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "docuelevate.fullname" . }}-api
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
{{- if not .Values.api.autoscaling.enabled }}
replicas: {{ .Values.api.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: api
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: api
{{- with .Values.api.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.api.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: api
image: {{ include "docuelevate.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
command:
- uvicorn
- app.main:app
- --host
- "0.0.0.0"
- --port
- "8000"
- --proxy-headers
ports:
- name: http
containerPort: 8000
protocol: TCP
envFrom:
- configMapRef:
name: {{ include "docuelevate.fullname" . }}-config
- secretRef:
name: {{ include "docuelevate.fullname" . }}-secret
{{- with .Values.api.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.api.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.api.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.api.resources | nindent 12 }}
volumeMounts:
- name: workdir
mountPath: /workdir
volumes:
- name: workdir
{{- if .Values.workdir.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.api.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.api.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.api.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -0,0 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "docuelevate.fullname" . }}-api
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
type: {{ .Values.api.service.type }}
ports:
- port: {{ .Values.api.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: api
+40
View File
@@ -0,0 +1,40 @@
# Non-sensitive application configuration.
# All values are injected as environment variables into the API and Worker pods.
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "docuelevate.fullname" . }}-config
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
data:
WORKDIR: {{ .Values.env.WORKDIR | quote }}
PYTHONPATH: "/app"
PYTHONUNBUFFERED: "1"
# AI provider
AI_PROVIDER: {{ .Values.env.AI_PROVIDER | quote }}
OPENAI_MODEL: {{ .Values.env.OPENAI_MODEL | quote }}
# Azure
AZURE_REGION: {{ .Values.env.AZURE_REGION | quote }}
AZURE_ENDPOINT: {{ .Values.env.AZURE_ENDPOINT | quote }}
# Service URLs — resolved from helper templates so they always match the
# in-cluster service names even if values.yaml uses template expressions.
GOTENBERG_URL: {{ include "docuelevate.gotenbergUrl" . | quote }}
MEILISEARCH_URL: {{ include "docuelevate.meilisearchUrl" . | quote }}
MEILISEARCH_INDEX_NAME: {{ .Values.env.MEILISEARCH_INDEX_NAME | quote }}
ENABLE_SEARCH: {{ .Values.env.ENABLE_SEARCH | quote }}
# Auth
AUTH_ENABLED: {{ .Values.env.AUTH_ENABLED | quote }}
ADMIN_USERNAME: {{ .Values.env.ADMIN_USERNAME | quote }}
EXTERNAL_HOSTNAME: {{ .Values.env.EXTERNAL_HOSTNAME | quote }}
# Feature flags
ENABLE_DEDUPLICATION: {{ .Values.env.ENABLE_DEDUPLICATION | quote }}
ENABLE_TEXT_QUALITY_CHECK: {{ .Values.env.ENABLE_TEXT_QUALITY_CHECK | quote }}
ALLOW_FILE_DELETE: {{ .Values.env.ALLOW_FILE_DELETE | quote }}
DEBUG: {{ .Values.env.DEBUG | quote }}
+63
View File
@@ -0,0 +1,63 @@
{{- if .Values.gotenberg.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "docuelevate.fullname" . }}-gotenberg
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: gotenberg
spec:
replicas: 1
selector:
matchLabels:
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: gotenberg
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: gotenberg
spec:
containers:
- name: gotenberg
image: {{ .Values.gotenberg.image.repository }}:{{ .Values.gotenberg.image.tag }}
imagePullPolicy: {{ .Values.gotenberg.image.pullPolicy }}
ports:
- name: http
containerPort: 3000
protocol: TCP
resources:
{{- toYaml .Values.gotenberg.resources | nindent 12 }}
livenessProbe:
httpGet:
path: /health
port: 3000
initialDelaySeconds: 10
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 3000
initialDelaySeconds: 5
periodSeconds: 10
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "docuelevate.fullname" . }}-gotenberg
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: gotenberg
spec:
type: ClusterIP
ports:
- port: {{ .Values.gotenberg.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: gotenberg
{{- end }}
+51
View File
@@ -0,0 +1,51 @@
{{- if .Values.api.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "docuelevate.fullname" . }}-api
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "docuelevate.fullname" . }}-api
minReplicas: {{ .Values.api.autoscaling.minReplicas }}
maxReplicas: {{ .Values.api.autoscaling.maxReplicas }}
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.api.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
---
{{- if .Values.worker.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "docuelevate.fullname" . }}-worker
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "docuelevate.fullname" . }}-worker
minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
+36
View File
@@ -0,0 +1,36 @@
{{- if .Values.ingress.enabled -}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "docuelevate.fullname" . }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.className }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- toYaml .Values.ingress.tls | nindent 4 }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ include "docuelevate.fullname" $ }}-api
port:
number: {{ $.Values.api.service.port }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,89 @@
{{- if .Values.meilisearch.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "docuelevate.fullname" . }}-meilisearch
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: meilisearch
spec:
# Single replica — Meilisearch Community Edition is single-node only.
# Use Meilisearch Cloud or the paid Meilisearch Cluster for HA.
replicas: 1
selector:
matchLabels:
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: meilisearch
strategy:
# Recreate ensures the PVC is released before the new pod starts
type: Recreate
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: meilisearch
spec:
containers:
- name: meilisearch
image: {{ .Values.meilisearch.image.repository }}:{{ .Values.meilisearch.image.tag }}
imagePullPolicy: {{ .Values.meilisearch.image.pullPolicy }}
ports:
- name: http
containerPort: 7700
protocol: TCP
env:
- name: MEILI_NO_ANALYTICS
value: {{ .Values.meilisearch.env.MEILI_NO_ANALYTICS | quote }}
{{- if .Values.secrets.MEILISEARCH_API_KEY }}
- name: MEILI_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ include "docuelevate.fullname" . }}-secret
key: MEILISEARCH_API_KEY
{{- end }}
resources:
{{- toYaml .Values.meilisearch.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /meili_data
livenessProbe:
httpGet:
path: /health
port: 7700
initialDelaySeconds: 15
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 7700
initialDelaySeconds: 10
periodSeconds: 10
volumes:
- name: data
{{- if .Values.meilisearch.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Values.meilisearch.persistence.existingClaim | default (printf "%s-meilisearch-data" (include "docuelevate.fullname" .)) }}
{{- else }}
emptyDir: {}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "docuelevate.fullname" . }}-meilisearch
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: meilisearch
spec:
type: ClusterIP
ports:
- port: {{ .Values.meilisearch.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: meilisearch
{{- end }}
@@ -0,0 +1,69 @@
# Database migration Job — runs `alembic upgrade head` once before
# the API and Worker Deployments start. Helm hooks ensure ordering:
# the Job runs during pre-upgrade/pre-install and the Deployments wait
# for it via the `helm.sh/hook-weight` annotation.
{{- if .Values.migrations.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "docuelevate.fullname" . }}-migrate-{{ .Release.Revision }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: migrate
annotations:
"helm.sh/hook": pre-install,pre-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation
{{- if .Values.migrations.ttlSecondsAfterFinished }}
spec:
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
{{- end }}
spec:
{{- if .Values.migrations.ttlSecondsAfterFinished }}
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
{{- end }}
backoffLimit: 3
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: migrate
spec:
restartPolicy: OnFailure
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
{{- with .Values.image.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: migrate
image: {{ include "docuelevate.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: ["alembic", "upgrade", "head"]
envFrom:
- configMapRef:
name: {{ include "docuelevate.fullname" . }}-config
- secretRef:
name: {{ include "docuelevate.fullname" . }}-secret
{{- with .Values.workdir.persistence }}
volumeMounts:
- name: workdir
mountPath: /workdir
{{- end }}
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
volumes:
- name: workdir
{{- if .Values.workdir.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}
+49
View File
@@ -0,0 +1,49 @@
# Shared workdir PVC — mounted by both the API and the Worker so they can
# exchange files during processing. Requires ReadWriteMany (e.g. NFS,
# CephFS, Azure Files, EFS) when running multiple replicas.
# Set workdir.persistence.accessMode=ReadWriteOnce and replicaCount=1
# for single-node / dev clusters.
{{- if .Values.workdir.persistence.enabled }}
{{- if not .Values.workdir.persistence.existingClaim }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "docuelevate.fullname" . }}-workdir
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.workdir.persistence.accessMode }}
resources:
requests:
storage: {{ .Values.workdir.persistence.size }}
{{- if .Values.workdir.persistence.storageClass }}
storageClassName: {{ .Values.workdir.persistence.storageClass }}
{{- end }}
{{- end }}
{{- end }}
---
# Meilisearch data PVC
{{- if and .Values.meilisearch.enabled .Values.meilisearch.persistence.enabled }}
{{- if not .Values.meilisearch.persistence.existingClaim }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "docuelevate.fullname" . }}-meilisearch-data
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.meilisearch.persistence.accessMode }}
resources:
requests:
storage: {{ .Values.meilisearch.persistence.size }}
{{- if .Values.meilisearch.persistence.storageClass }}
storageClassName: {{ .Values.meilisearch.persistence.storageClass }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,14 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "docuelevate.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: false
{{- end }}
@@ -0,0 +1,82 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "docuelevate.fullname" . }}-worker
namespace: {{ .Release.Namespace }}
labels:
{{- include "docuelevate.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
{{- if not .Values.worker.autoscaling.enabled }}
replicas: {{ .Values.worker.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: worker
template:
metadata:
labels:
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: worker
{{- with .Values.worker.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.worker.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ include "docuelevate.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
command:
- celery
- -A
- app.celery_worker
- worker
- -B
- --loglevel=info
- -Q
- document_processor,default,celery
envFrom:
- configMapRef:
name: {{ include "docuelevate.fullname" . }}-config
- secretRef:
name: {{ include "docuelevate.fullname" . }}-secret
{{- with .Values.worker.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
- name: workdir
mountPath: /workdir
volumes:
- name: workdir
{{- if .Values.workdir.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.worker.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.worker.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.worker.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
+306
View File
@@ -0,0 +1,306 @@
# =============================================================================
# DocuElevate Helm Chart — values.yaml
#
# Override any value with:
# helm install docuelevate ./helm/docuelevate -f my-values.yaml
# helm install docuelevate ./helm/docuelevate --set key=value
# =============================================================================
# ---------------------------------------------------------------------------
# Global image settings
# ---------------------------------------------------------------------------
image:
repository: ghcr.io/christianlouis/docuelevate
# Defaults to the chart appVersion; pin to a specific digest in production.
tag: ""
pullPolicy: IfNotPresent
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
# ---------------------------------------------------------------------------
# Shared environment — non-secret application config
# All values map 1-to-1 onto DocuElevate environment variables.
# Sensitive values (API keys, passwords) go into `secrets` below.
# ---------------------------------------------------------------------------
env:
# Required ----------------------------------------------------------------
WORKDIR: /workdir
# AI provider (openai | azure | anthropic | gemini | ollama | openrouter)
AI_PROVIDER: openai
OPENAI_MODEL: gpt-4o-mini
# Azure Document Intelligence (required when OCR_PROVIDERS includes "azure")
AZURE_REGION: eastus
AZURE_ENDPOINT: "" # e.g. https://my-resource.cognitiveservices.azure.com/
# Gotenberg PDF conversion service
GOTENBERG_URL: http://{{ include "docuelevate.fullname" . }}-gotenberg:3000
# Full-text search — uses the in-cluster Meilisearch service by default
MEILISEARCH_URL: http://{{ include "docuelevate.fullname" . }}-meilisearch:7700
MEILISEARCH_INDEX_NAME: documents
ENABLE_SEARCH: "true"
# Authentication
AUTH_ENABLED: "true"
ADMIN_USERNAME: admin
# Feature flags
ENABLE_DEDUPLICATION: "true"
ENABLE_TEXT_QUALITY_CHECK: "true"
ALLOW_FILE_DELETE: "true"
# Logging / misc
DEBUG: "false"
EXTERNAL_HOSTNAME: localhost # set to your public hostname / Ingress host
# ---------------------------------------------------------------------------
# Secrets — values are stored in a Kubernetes Secret and injected as env vars.
# In production, use an external secret manager (Vault, ESO, Sealed Secrets)
# and leave these blank, then mount the Secret yourself.
# ---------------------------------------------------------------------------
secrets:
DATABASE_URL: "" # e.g. postgresql://user:pass@postgres:5432/docuelevate
REDIS_URL: "" # leave blank to use bundled Redis
SESSION_SECRET: "" # min 32-char random string — generate with: openssl rand -hex 32
OPENAI_API_KEY: ""
AZURE_AI_KEY: ""
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated (dev) Meilisearch
# Storage provider secrets (only the ones you use)
DROPBOX_APP_KEY: ""
DROPBOX_APP_SECRET: ""
DROPBOX_REFRESH_TOKEN: ""
GOOGLE_DRIVE_CREDENTIALS_JSON: ""
ONEDRIVE_CLIENT_ID: ""
ONEDRIVE_CLIENT_SECRET: ""
ONEDRIVE_REFRESH_TOKEN: ""
AWS_ACCESS_KEY_ID: ""
AWS_SECRET_ACCESS_KEY: ""
# OAuth / Authentik
AUTHENTIK_CLIENT_ID: ""
AUTHENTIK_CLIENT_SECRET: ""
AUTHENTIK_CONFIG_URL: ""
# ---------------------------------------------------------------------------
# External services
# Set externalRedis.url (and disable bundled redis) when you have your own.
# ---------------------------------------------------------------------------
externalRedis:
# When non-empty this value is injected as REDIS_URL, overriding secrets.REDIS_URL
# and the auto-generated bundled-Redis URL.
url: ""
# ---------------------------------------------------------------------------
# API deployment
# ---------------------------------------------------------------------------
api:
replicaCount: 2
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
# Horizontal Pod Autoscaler
autoscaling:
enabled: false
minReplicas: 2
maxReplicas: 8
targetCPUUtilizationPercentage: 70
service:
type: ClusterIP
port: 8000
# Liveness / readiness probes
livenessProbe:
httpGet:
path: /api/health
port: 8000
initialDelaySeconds: 30
periodSeconds: 20
failureThreshold: 3
readinessProbe:
httpGet:
path: /api/health
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
failureThreshold: 3
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
podSecurityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroup: 1000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false # app writes to /workdir
capabilities:
drop: ["ALL"]
# ---------------------------------------------------------------------------
# Celery worker deployment
# ---------------------------------------------------------------------------
worker:
replicaCount: 2
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
autoscaling:
enabled: false
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 75
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
podSecurityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroup: 1000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop: ["ALL"]
# ---------------------------------------------------------------------------
# Shared workdir volume (api + worker mount the same PVC)
# ---------------------------------------------------------------------------
workdir:
persistence:
enabled: true
# storageClass: "" # leave blank for cluster default
accessMode: ReadWriteMany # RWX required for multiple pods
size: 20Gi
# existingClaim: ""
# ---------------------------------------------------------------------------
# Gotenberg (PDF conversion)
# ---------------------------------------------------------------------------
gotenberg:
enabled: true
image:
repository: gotenberg/gotenberg
tag: latest
pullPolicy: IfNotPresent
service:
port: 3000
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
# ---------------------------------------------------------------------------
# Meilisearch (full-text search)
# ---------------------------------------------------------------------------
meilisearch:
enabled: true
image:
repository: getmeili/meilisearch
tag: latest
pullPolicy: IfNotPresent
service:
port: 7700
env:
MEILI_NO_ANALYTICS: "true"
# MEILI_MASTER_KEY: "" # set via secrets.MEILISEARCH_API_KEY instead
persistence:
enabled: true
# storageClass: ""
accessMode: ReadWriteOnce
size: 10Gi
# existingClaim: ""
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
# ---------------------------------------------------------------------------
# Bundled Redis (from Bitnami chart)
# Disable and set externalRedis.url to use your own.
# ---------------------------------------------------------------------------
redis:
enabled: true
architecture: standalone
auth:
enabled: false
master:
persistence:
enabled: true
size: 4Gi
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
# ---------------------------------------------------------------------------
# Ingress
# ---------------------------------------------------------------------------
ingress:
enabled: false
className: "" # e.g. nginx, traefik
annotations: {}
# nginx.ingress.kubernetes.io/proxy-body-size: "1g"
# cert-manager.io/cluster-issuer: letsencrypt-prod
hosts:
- host: docuelevate.example.com
paths:
- path: /
pathType: Prefix
tls: []
# - secretName: docuelevate-tls
# hosts:
# - docuelevate.example.com
# ---------------------------------------------------------------------------
# ServiceAccount
# ---------------------------------------------------------------------------
serviceAccount:
create: true
annotations: {}
name: ""
# ---------------------------------------------------------------------------
# Database migration job
# Runs `alembic upgrade head` before the api/worker start.
# ---------------------------------------------------------------------------
migrations:
enabled: true
# Automatically deleted after successful completion
ttlSecondsAfterFinished: 120
@@ -0,0 +1,31 @@
"""Add OCR text and AI metadata fields to files table for full-text search
Revision ID: 004_add_search_fields
Revises: 003_add_deduplication_support
Create Date: 2026-02-25
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "004_add_search_fields"
down_revision: Union[str, None] = "003_add_deduplication_support"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add ocr_text, ai_metadata, and document_title columns to files table."""
op.add_column("files", sa.Column("ocr_text", sa.Text(), nullable=True))
op.add_column("files", sa.Column("ai_metadata", sa.Text(), nullable=True))
op.add_column("files", sa.Column("document_title", sa.String(), nullable=True))
def downgrade() -> None:
"""Remove ocr_text, ai_metadata, and document_title columns from files table."""
op.drop_column("files", "document_title")
op.drop_column("files", "ai_metadata")
op.drop_column("files", "ocr_text")
+2 -1
View File
@@ -42,4 +42,5 @@ litellm>=1.0.0,<2.0.0
# Self-hosted OCR engines (optional only required when the provider is enabled)
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
meilisearch>=0.31.0 # Full-text search engine client
+312
View File
@@ -0,0 +1,312 @@
"""Tests for the full-text search API (app/api/search.py) and Meilisearch
client utilities (app/utils/meilisearch_client.py).
"""
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# app/utils/meilisearch_client tests
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestMeilisearchClientDisabled:
"""Tests when search is disabled or Meilisearch is unavailable."""
def test_get_client_returns_none_when_disabled(self):
"""get_meilisearch_client returns None when enable_search=False."""
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
from app.utils.meilisearch_client import index_document, search_documents
class _FakeRecord:
id = 1
original_filename = "test.pdf"
mime_type = "application/pdf"
file_size = 1024
created_at = None
result = index_document(_FakeRecord(), "some text", {})
assert result is False
result = search_documents("invoice")
assert result["results"] == []
assert result["total"] == 0
def test_search_documents_import_error(self):
"""search_documents returns empty dict when meilisearch not installed."""
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
from app.utils.meilisearch_client import search_documents
result = search_documents("test")
assert result["results"] == []
assert result["total"] == 0
assert result["page"] == 1
assert result["query"] == "test"
def test_delete_document_no_client(self):
"""delete_document returns False when client unavailable."""
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
from app.utils.meilisearch_client import delete_document
result = delete_document(99)
assert result is False
@pytest.mark.unit
class TestMeilisearchIndexDocument:
"""Tests for index_document function."""
def test_index_document_success(self):
"""index_document returns True when Meilisearch succeeds."""
mock_client = MagicMock()
mock_index = MagicMock()
mock_task = MagicMock()
mock_task.task_uid = 1
mock_client.get_index.return_value = mock_index
mock_index.add_documents.return_value = mock_task
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import index_document
class _FakeRecord:
id = 1
original_filename = "invoice.pdf"
mime_type = "application/pdf"
file_size = 2048
created_at = None
result = index_document(
_FakeRecord(),
"This is an invoice for services rendered",
{
"title": "Invoice January 2026",
"document_type": "Invoice",
"tags": ["invoice", "services"],
"absender": "ACME Corp",
"language": "en",
},
)
assert result is True
mock_index.add_documents.assert_called_once()
call_docs = mock_index.add_documents.call_args[0][0]
assert len(call_docs) == 1
doc = call_docs[0]
assert doc["file_id"] == 1
assert doc["document_title"] == "Invoice January 2026"
assert "invoice" in doc["tags"]
def test_index_document_meilisearch_error(self):
"""index_document returns False on Meilisearch exception."""
mock_client = MagicMock()
mock_index = MagicMock()
mock_client.get_index.return_value = mock_index
mock_index.add_documents.side_effect = RuntimeError("Meilisearch down")
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import index_document
class _FakeRecord:
id = 2
original_filename = "test.pdf"
mime_type = "application/pdf"
file_size = 512
created_at = None
result = index_document(_FakeRecord(), "some text", {})
assert result is False
@pytest.mark.unit
class TestMeilisearchSearchDocuments:
"""Tests for search_documents function."""
def _make_mock_client(self, hits=None, total=None):
mock_client = MagicMock()
mock_index = MagicMock()
mock_client.get_index.return_value = mock_index
mock_index.search.return_value = {
"hits": hits or [],
"estimatedTotalHits": total if total is not None else len(hits or []),
}
return mock_client, mock_index
def test_search_returns_results(self):
"""search_documents returns hits from Meilisearch."""
hits = [
{
"file_id": 42,
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
"document_title": "Amazon Invoice",
"document_type": "Invoice",
"tags": ["amazon", "invoice"],
"ocr_text": "Amazon invoice content here",
"_formatted": {
"document_title": "Amazon <mark>Invoice</mark>",
"ocr_text": "…Amazon <mark>invoice</mark> content here…",
},
}
]
mock_client, mock_index = self._make_mock_client(hits=hits, total=1)
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import search_documents
result = search_documents("invoice", page=1, per_page=20)
assert result["total"] == 1
assert result["pages"] == 1
assert result["query"] == "invoice"
assert len(result["results"]) == 1
# Raw ocr_text should be stripped from result (only _formatted snippet kept)
assert "ocr_text" not in result["results"][0]
assert result["results"][0]["file_id"] == 42
def test_search_with_filters(self):
"""search_documents passes filter expressions to Meilisearch."""
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import search_documents
search_documents("contract", mime_type="application/pdf", language="de", page=1, per_page=10)
call_kwargs = mock_index.search.call_args
search_params = call_kwargs[0][1]
assert "filter" in search_params
assert 'mime_type = "application/pdf"' in search_params["filter"]
assert 'language = "de"' in search_params["filter"]
def test_search_pagination(self):
"""search_documents applies correct offset for page 2."""
mock_client, mock_index = self._make_mock_client(hits=[], total=50)
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import search_documents
result = search_documents("test", page=3, per_page=10)
call_kwargs = mock_index.search.call_args
search_params = call_kwargs[0][1]
assert search_params["offset"] == 20 # (3-1) * 10
assert search_params["limit"] == 10
assert result["pages"] == 5
def test_search_empty_results(self):
"""search_documents returns proper empty structure."""
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import search_documents
result = search_documents("nothing")
assert result["results"] == []
assert result["total"] == 0
assert result["pages"] == 0
def test_search_exception_returns_empty(self):
"""search_documents returns empty dict on Meilisearch exception."""
mock_client = MagicMock()
mock_index = MagicMock()
mock_client.get_index.side_effect = RuntimeError("Meilisearch unavailable")
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
from app.utils.meilisearch_client import search_documents
result = search_documents("invoice")
assert result["results"] == []
assert result["total"] == 0
# ---------------------------------------------------------------------------
# Search API endpoint tests (GET /api/search)
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestSearchAPIEndpoint:
"""Tests for GET /api/search endpoint."""
def test_search_endpoint_success(self, client):
"""GET /api/search?q=... returns search results."""
mock_result = {
"results": [{"file_id": 1, "document_title": "Test Invoice", "document_type": "Invoice"}],
"total": 1,
"page": 1,
"pages": 1,
"query": "invoice",
}
with patch("app.api.search.search_documents", return_value=mock_result):
response = client.get("/api/search?q=invoice")
assert response.status_code == 200
data = response.json()
assert data["total"] == 1
assert data["query"] == "invoice"
assert len(data["results"]) == 1
def test_search_endpoint_missing_query(self, client):
"""GET /api/search without q returns 422."""
response = client.get("/api/search")
assert response.status_code == 422
def test_search_endpoint_empty_query(self, client):
"""GET /api/search?q= (empty) returns 422 due to min_length=1."""
response = client.get("/api/search?q=")
assert response.status_code == 422
def test_search_endpoint_with_filters(self, client):
"""GET /api/search with optional filters passes them to search_documents."""
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "invoice"}
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
response = client.get("/api/search?q=invoice&mime_type=application/pdf&language=en&page=2&per_page=10")
assert response.status_code == 200
mock_search.assert_called_once_with(
"invoice",
mime_type="application/pdf",
document_type=None,
language="en",
date_from=None,
date_to=None,
page=2,
per_page=10,
)
def test_search_endpoint_per_page_max(self, client):
"""GET /api/search with per_page > 100 returns 422."""
response = client.get("/api/search?q=test&per_page=200")
assert response.status_code == 422
def test_search_endpoint_pagination_defaults(self, client):
"""GET /api/search uses default page=1 per_page=20."""
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"}
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
response = client.get("/api/search?q=test")
assert response.status_code == 200
mock_search.assert_called_once_with(
"test",
mime_type=None,
document_type=None,
language=None,
date_from=None,
date_to=None,
page=1,
per_page=20,
)
def test_search_endpoint_date_filters(self, client):
"""GET /api/search with date_from and date_to passes them as int."""
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "contract"}
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
response = client.get("/api/search?q=contract&date_from=1704067200&date_to=1735689600")
assert response.status_code == 200
call_kwargs = mock_search.call_args
assert call_kwargs[1]["date_from"] == 1704067200
assert call_kwargs[1]["date_to"] == 1735689600