Merge pull request #406 from christianlouis/copilot/add-full-text-search-feature
feat: full-text search (Meilisearch) + Helm chart for Kubernetes deployment
This commit is contained in:
@@ -299,3 +299,14 @@ NOTIFY_ON_FILE_PROCESSED=True
|
||||
# Uptime Kuma
|
||||
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
||||
UPTIME_KUMA_PING_INTERVAL=5
|
||||
|
||||
# **Full-Text Search (Meilisearch)**
|
||||
# URL for the Meilisearch instance.
|
||||
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
|
||||
# so container-to-container networking works without extra configuration.
|
||||
# Override to "http://localhost:7700" only when running the API process outside Docker.
|
||||
MEILISEARCH_URL=http://meilisearch:7700
|
||||
# Optional master/API key for secured Meilisearch instances
|
||||
# MEILISEARCH_API_KEY=your_master_key_here
|
||||
MEILISEARCH_INDEX_NAME=documents
|
||||
ENABLE_SEARCH=True
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.api.logs import router as logs_router
|
||||
from app.api.onedrive import router as onedrive_router
|
||||
from app.api.openai import router as openai_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
@@ -40,3 +41,4 @@ router.include_router(google_drive_router)
|
||||
router.include_router(logs_router)
|
||||
router.include_router(settings_router)
|
||||
router.include_router(url_upload_router)
|
||||
router.include_router(search_router)
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Full-text search API endpoints.
|
||||
|
||||
Provides document search across OCR text, AI metadata, filenames, and tags
|
||||
via Meilisearch. Designed to serve as the backend for the UI search bar on
|
||||
the /files page and as a standalone API for integrations.
|
||||
|
||||
Future extension point: the OCR text stored in the index is also suitable
|
||||
for RAG (Retrieval Augmented Generation) chatbot workflows.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
|
||||
from app.auth import require_login
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
@require_login
|
||||
def search_api(
|
||||
request: Request,
|
||||
q: str = Query(..., min_length=1, max_length=512, description="Full-text search query"),
|
||||
mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"),
|
||||
document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"),
|
||||
language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"),
|
||||
date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"),
|
||||
date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"),
|
||||
page: int = Query(1, ge=1, description="Page number (1-based)"),
|
||||
per_page: int = Query(20, ge=1, le=100, description="Results per page"),
|
||||
):
|
||||
"""Search documents by full text, metadata, and tags.
|
||||
|
||||
Searches across:
|
||||
- Document title and filename
|
||||
- OCR / extracted text
|
||||
- Tags, sender, recipient, document type
|
||||
- Correspondent and reference number
|
||||
|
||||
Results are ranked by Meilisearch relevance and include highlighted
|
||||
snippets showing where the query terms matched.
|
||||
|
||||
Query Parameters:
|
||||
- q: Search query (required)
|
||||
- mime_type: Filter by MIME type
|
||||
- document_type: Filter by document type
|
||||
- language: Filter by language code
|
||||
- date_from: Unix timestamp lower bound
|
||||
- date_to: Unix timestamp upper bound
|
||||
- page: Page number (default: 1)
|
||||
- per_page: Results per page (default: 20, max: 100)
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/search?q=invoice&document_type=Invoice&date_from=1704067200&page=1&per_page=20
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"file_id": 42,
|
||||
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||
"document_title": "Amazon Invoice January 2026",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["amazon", "invoice"],
|
||||
"_formatted": {
|
||||
"document_title": "Amazon <mark>Invoice</mark> January 2026",
|
||||
"ocr_text": "...total amount of the <mark>invoice</mark> is..."
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": 42,
|
||||
"page": 1,
|
||||
"pages": 3,
|
||||
"query": "invoice"
|
||||
}
|
||||
```
|
||||
"""
|
||||
logger.info(f"Search request: q={q!r}, mime_type={mime_type}, page={page}, per_page={per_page}")
|
||||
|
||||
result = search_documents(
|
||||
q,
|
||||
mime_type=mime_type,
|
||||
document_type=document_type,
|
||||
language=language,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -207,6 +207,15 @@ class Settings(BaseSettings):
|
||||
uptime_kuma_url: Optional[str] = None
|
||||
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
|
||||
|
||||
# Meilisearch settings (full-text search engine)
|
||||
# Default uses the Docker Compose / K8s service name so container-to-container
|
||||
# networking works without any extra configuration. Override to
|
||||
# "http://localhost:7700" only when running the API process outside of Docker.
|
||||
meilisearch_url: str = "http://meilisearch:7700"
|
||||
meilisearch_api_key: Optional[str] = None # Master or API key (optional for local dev)
|
||||
meilisearch_index_name: str = "documents"
|
||||
enable_search: bool = True # Enable Meilisearch full-text search integration
|
||||
|
||||
# HTTP request settings
|
||||
http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
|
||||
|
||||
|
||||
@@ -55,6 +55,15 @@ class FileRecord(Base):
|
||||
# If this is a duplicate, record the ID of the original file for reference
|
||||
duplicate_of_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)
|
||||
|
||||
# Full OCR/extracted text for full-text search and RAG
|
||||
ocr_text = Column(Text, nullable=True)
|
||||
|
||||
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
|
||||
ai_metadata = Column(Text, nullable=True)
|
||||
|
||||
# Human-readable document title from AI metadata
|
||||
document_title = Column(String, nullable=True)
|
||||
|
||||
# Timestamp when we inserted this record
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
@@ -195,8 +195,26 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
original_file_path = file_record.original_file_path
|
||||
# Update the processed_file_path in the database
|
||||
file_record.processed_file_path = final_file_path
|
||||
# Persist extracted text and AI metadata to DB for full-text search / RAG
|
||||
file_record.ocr_text = extracted_text or None
|
||||
if metadata:
|
||||
try:
|
||||
file_record.ai_metadata = json.dumps(metadata, ensure_ascii=False)
|
||||
except Exception as json_exc:
|
||||
logger.warning(f"[{task_id}] Could not serialise ai_metadata: {json_exc}")
|
||||
file_record.document_title = (
|
||||
metadata.get("title") or metadata.get("filename") or file_record.original_filename
|
||||
)
|
||||
db.commit()
|
||||
logger.info(f"[{task_id}] Updated database with processed_file_path: {final_file_path}")
|
||||
logger.info(f"[{task_id}] Updated database with processed_file_path and search fields")
|
||||
|
||||
# Index into Meilisearch for full-text search (non-blocking, best-effort)
|
||||
try:
|
||||
from app.utils.meilisearch_client import index_document
|
||||
|
||||
index_document(file_record, extracted_text or "", metadata or {})
|
||||
except Exception as search_exc:
|
||||
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
|
||||
|
||||
# Persist the metadata into a JSON file with the same base name.
|
||||
# Include file path references for traceability
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Meilisearch client utilities for full-text document search.
|
||||
|
||||
This module provides functions for indexing documents into Meilisearch
|
||||
and searching across OCR text, AI metadata, filenames, and tags.
|
||||
|
||||
The search index is designed to support future RAG (Retrieval Augmented
|
||||
Generation) workflows by storing full document text alongside structured
|
||||
metadata fields.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Index settings applied once at index creation / first use
|
||||
_INDEX_SETTINGS = {
|
||||
"searchableAttributes": [
|
||||
"document_title",
|
||||
"original_filename",
|
||||
"ocr_text",
|
||||
"tags",
|
||||
"sender",
|
||||
"recipient",
|
||||
"document_type",
|
||||
"correspondent",
|
||||
],
|
||||
"filterableAttributes": [
|
||||
"mime_type",
|
||||
"document_type",
|
||||
"language",
|
||||
"tags",
|
||||
"created_at_ts",
|
||||
"file_id",
|
||||
],
|
||||
"sortableAttributes": [
|
||||
"created_at_ts",
|
||||
"file_size",
|
||||
],
|
||||
"displayedAttributes": [
|
||||
"file_id",
|
||||
"original_filename",
|
||||
"document_title",
|
||||
"document_type",
|
||||
"tags",
|
||||
"sender",
|
||||
"recipient",
|
||||
"correspondent",
|
||||
"language",
|
||||
"reference_number",
|
||||
"mime_type",
|
||||
"file_size",
|
||||
"created_at_ts",
|
||||
"ocr_text",
|
||||
],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_meilisearch_client():
|
||||
"""Return a configured Meilisearch client, or None if unavailable/disabled."""
|
||||
try:
|
||||
import meilisearch
|
||||
|
||||
from app.config import settings
|
||||
|
||||
if not settings.enable_search:
|
||||
return None
|
||||
|
||||
kwargs: dict[str, Any] = {"url": settings.meilisearch_url}
|
||||
if settings.meilisearch_api_key:
|
||||
kwargs["api_key"] = settings.meilisearch_api_key
|
||||
|
||||
client = meilisearch.Client(**kwargs)
|
||||
return client
|
||||
except ImportError:
|
||||
logger.warning("meilisearch package not installed; search disabled")
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not connect to Meilisearch: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _get_or_create_index(client):
|
||||
"""Get the documents index, creating it with settings if it doesn't exist."""
|
||||
from app.config import settings
|
||||
|
||||
index_name = settings.meilisearch_index_name
|
||||
try:
|
||||
index = client.get_index(index_name)
|
||||
except Exception:
|
||||
# Index doesn't exist – create it with file_id as primary key
|
||||
task = client.create_index(index_name, {"primaryKey": "file_id"})
|
||||
client.wait_for_task(task.task_uid)
|
||||
index = client.get_index(index_name)
|
||||
# Apply search settings
|
||||
try:
|
||||
task = index.update_settings(_INDEX_SETTINGS)
|
||||
client.wait_for_task(task.task_uid)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not update Meilisearch index settings: {exc}")
|
||||
return index
|
||||
|
||||
|
||||
def _build_document(file_record, text: str, metadata: dict) -> dict:
|
||||
"""Build a Meilisearch document from a FileRecord and extracted content."""
|
||||
|
||||
tags = metadata.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [t.strip() for t in tags.split(",") if t.strip()]
|
||||
|
||||
# Unix timestamp for sorting/filtering
|
||||
created_at_ts = 0
|
||||
if file_record.created_at:
|
||||
try:
|
||||
created_at_ts = int(file_record.created_at.timestamp())
|
||||
except Exception as ts_exc: # noqa: BLE001
|
||||
logger.debug(f"Could not convert created_at to timestamp: {ts_exc}")
|
||||
|
||||
return {
|
||||
"file_id": file_record.id,
|
||||
"original_filename": file_record.original_filename or "",
|
||||
"document_title": metadata.get("title") or metadata.get("filename") or file_record.original_filename or "",
|
||||
"document_type": metadata.get("document_type") or metadata.get("kommunikationsart") or "",
|
||||
"tags": tags,
|
||||
"sender": metadata.get("absender") or "",
|
||||
"recipient": metadata.get("empfaenger") or "",
|
||||
"correspondent": metadata.get("correspondent") or "",
|
||||
"language": metadata.get("language") or "",
|
||||
"reference_number": metadata.get("reference_number") or "",
|
||||
"mime_type": file_record.mime_type or "",
|
||||
"file_size": file_record.file_size or 0,
|
||||
"created_at_ts": created_at_ts,
|
||||
"ocr_text": text or "",
|
||||
}
|
||||
|
||||
|
||||
def index_document(file_record, text: str, metadata: dict) -> bool:
|
||||
"""Index a document in Meilisearch.
|
||||
|
||||
Args:
|
||||
file_record: FileRecord ORM instance with at minimum .id set.
|
||||
text: Full OCR / extracted text for the document.
|
||||
metadata: AI-extracted metadata dict.
|
||||
|
||||
Returns:
|
||||
True if indexing succeeded, False otherwise.
|
||||
"""
|
||||
client = get_meilisearch_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
index = _get_or_create_index(client)
|
||||
doc = _build_document(file_record, text, metadata)
|
||||
task = index.add_documents([doc])
|
||||
logger.info(f"Queued Meilisearch indexing for file_id={file_record.id} (task_uid={task.task_uid})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(f"Meilisearch indexing failed for file_id={file_record.id}: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def delete_document(file_id: int) -> bool:
|
||||
"""Remove a document from the Meilisearch index.
|
||||
|
||||
Args:
|
||||
file_id: The database ID of the file to remove.
|
||||
|
||||
Returns:
|
||||
True if deletion succeeded, False otherwise.
|
||||
"""
|
||||
client = get_meilisearch_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
from app.config import settings
|
||||
|
||||
index = client.get_index(settings.meilisearch_index_name)
|
||||
task = index.delete_document(file_id)
|
||||
logger.info(f"Queued Meilisearch deletion for file_id={file_id} (task_uid={task.task_uid})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(f"Meilisearch deletion failed for file_id={file_id}: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def search_documents(
|
||||
query: str,
|
||||
*,
|
||||
mime_type: Optional[str] = None,
|
||||
document_type: Optional[str] = None,
|
||||
language: Optional[str] = None,
|
||||
date_from: Optional[int] = None,
|
||||
date_to: Optional[int] = None,
|
||||
page: int = 1,
|
||||
per_page: int = 20,
|
||||
) -> dict:
|
||||
"""Search documents in Meilisearch.
|
||||
|
||||
Args:
|
||||
query: Full-text search query string.
|
||||
mime_type: Optional MIME-type filter.
|
||||
document_type: Optional document type filter.
|
||||
language: Optional language filter (ISO 639-1, e.g. "de").
|
||||
date_from: Optional lower bound Unix timestamp for created_at.
|
||||
date_to: Optional upper bound Unix timestamp for created_at.
|
||||
page: 1-based page number.
|
||||
per_page: Results per page (max 100).
|
||||
|
||||
Returns:
|
||||
Dict with keys: results, total, page, pages, query.
|
||||
Returns empty results dict on any error.
|
||||
"""
|
||||
empty: dict = {"results": [], "total": 0, "page": page, "pages": 0, "query": query}
|
||||
|
||||
client = get_meilisearch_client()
|
||||
if client is None:
|
||||
return empty
|
||||
|
||||
try:
|
||||
index = _get_or_create_index(client)
|
||||
|
||||
# Build filter expressions
|
||||
filters: list[str] = []
|
||||
if mime_type:
|
||||
filters.append(f'mime_type = "{mime_type}"')
|
||||
if document_type:
|
||||
filters.append(f'document_type = "{document_type}"')
|
||||
if language:
|
||||
filters.append(f'language = "{language}"')
|
||||
if date_from is not None:
|
||||
filters.append(f"created_at_ts >= {date_from}")
|
||||
if date_to is not None:
|
||||
filters.append(f"created_at_ts <= {date_to}")
|
||||
|
||||
search_params: dict[str, Any] = {
|
||||
"offset": (page - 1) * per_page,
|
||||
"limit": per_page,
|
||||
"attributesToHighlight": ["document_title", "original_filename", "ocr_text", "tags"],
|
||||
"highlightPreTag": "<mark>",
|
||||
"highlightPostTag": "</mark>",
|
||||
"attributesToCrop": ["ocr_text"],
|
||||
"cropLength": 200,
|
||||
}
|
||||
|
||||
if filters:
|
||||
search_params["filter"] = " AND ".join(filters)
|
||||
|
||||
result = index.search(query, search_params)
|
||||
|
||||
hits = result.get("hits", [])
|
||||
total = result.get("estimatedTotalHits", result.get("nbHits", len(hits)))
|
||||
|
||||
# Attach highlights to each hit
|
||||
formatted_results = []
|
||||
for hit in hits:
|
||||
formatted = dict(hit)
|
||||
# Include formatted (highlighted) snippets if available
|
||||
if "_formatted" in hit:
|
||||
formatted["_formatted"] = hit["_formatted"]
|
||||
# Exclude raw ocr_text from results (use _formatted snippet instead)
|
||||
formatted.pop("ocr_text", None)
|
||||
formatted_results.append(formatted)
|
||||
|
||||
pages = (total + per_page - 1) // per_page if total > 0 else 0
|
||||
|
||||
return {
|
||||
"results": formatted_results,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"pages": pages,
|
||||
"query": query,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning(f"Meilisearch search failed for query '{query}': {exc}")
|
||||
return empty
|
||||
@@ -59,6 +59,15 @@ services:
|
||||
container_name: gotenberg
|
||||
restart: always
|
||||
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:latest
|
||||
container_name: document_meilisearch
|
||||
restart: always
|
||||
environment:
|
||||
- MEILI_NO_ANALYTICS=true
|
||||
volumes:
|
||||
- /var/docparse/meilisearch:/meili_data
|
||||
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: document_redis
|
||||
|
||||
+310
-151
@@ -1,40 +1,48 @@
|
||||
# Deployment Guide
|
||||
|
||||
This guide provides instructions for deploying DocuElevate in various environments.
|
||||
This guide covers all supported deployment methods for DocuElevate.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Docker Compose Deployment](#docker-compose-deployment) *(recommended for single-server)*
|
||||
- [Kubernetes / Helm Deployment](#kubernetes--helm-deployment) *(recommended for production scale-out)*
|
||||
- [Production Considerations](#production-considerations)
|
||||
- [Scaling](#scaling)
|
||||
- [Backup Procedures](#backup-procedures)
|
||||
- [Updates](#updates)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker and Docker Compose
|
||||
- Docker and Docker Compose **or** a Kubernetes cluster with Helm 3
|
||||
- Access to required external services (if configured):
|
||||
- AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider)
|
||||
- Azure Document Intelligence
|
||||
- Dropbox API
|
||||
- Nextcloud instance
|
||||
- Paperless NGX instance
|
||||
- SMTP server (for email notifications)
|
||||
- IMAP server(s) (for email attachment processing)
|
||||
- Notification services (Discord, Telegram, etc. for system alerts)
|
||||
- Dropbox, Google Drive, OneDrive, S3, or other storage APIs
|
||||
- SMTP / IMAP server (for email processing)
|
||||
- Notification services (Discord, Telegram, etc.)
|
||||
|
||||
## Docker Deployment
|
||||
---
|
||||
|
||||
Docker is the recommended deployment method for DocuElevate.
|
||||
## Docker Compose Deployment
|
||||
|
||||
Docker Compose is the quickest way to run DocuElevate on a single server.
|
||||
|
||||
### Step 1: Clone the Repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/christianlouis/document-processor.git
|
||||
cd document-processor
|
||||
git clone https://github.com/christianlouis/DocuElevate.git
|
||||
cd DocuElevate
|
||||
```
|
||||
|
||||
### Step 2: Configure Environment Variables
|
||||
|
||||
Create a `.env` file based on the example:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
cp .env.demo .env
|
||||
```
|
||||
|
||||
Edit the `.env` file with your configuration settings. See the [Configuration Guide](ConfigurationGuide.md) for details.
|
||||
Edit `.env` with your settings. See the [Configuration Guide](ConfigurationGuide.md) for all options.
|
||||
|
||||
### Step 3: Run with Docker Compose
|
||||
|
||||
@@ -42,41 +50,237 @@ Edit the `.env` file with your configuration settings. See the [Configuration Gu
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
This will start:
|
||||
- The DocuElevate API server
|
||||
- A worker for background tasks
|
||||
- Redis for message broker and result storage
|
||||
- Gotenberg for PDF processing
|
||||
This starts:
|
||||
|
||||
| Service | Purpose |
|
||||
|---------|---------|
|
||||
| `api` | FastAPI web server (port 8000) |
|
||||
| `worker` | Celery background task worker |
|
||||
| `redis` | Message broker for Celery |
|
||||
| `gotenberg` | PDF conversion (LibreOffice headless) |
|
||||
| `meilisearch` | Full-text search engine (port 7700) |
|
||||
|
||||
### Step 4: Verify the Installation
|
||||
|
||||
Access the web interface at `http://localhost:8000` and the API documentation at `http://localhost:8000/docs`.
|
||||
Access the web interface at `http://localhost:8000` and the API docs at `http://localhost:8000/docs`.
|
||||
|
||||
---
|
||||
|
||||
## Kubernetes / Helm Deployment
|
||||
|
||||
The Helm chart at `helm/docuelevate/` packages all components into a single, configurable release. It supports:
|
||||
|
||||
- Multiple replicas for the API and Worker
|
||||
- Horizontal Pod Autoscaling (HPA)
|
||||
- Bundled or external Redis
|
||||
- Persistent volumes for workdir and Meilisearch data
|
||||
- Alembic database migration Job (pre-install/upgrade hook)
|
||||
- TLS Ingress via any controller (nginx, Traefik, etc.)
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Kubernetes 1.24+
|
||||
- Helm 3.10+
|
||||
- A storage class that supports **ReadWriteMany** (e.g. NFS, CephFS, Azure Files, EFS) for the shared workdir PVC when running multiple replicas. Single-replica clusters can use `ReadWriteOnce`.
|
||||
- A PostgreSQL database (strongly recommended over SQLite for multi-replica).
|
||||
|
||||
### Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Add the Bitnami chart repository (needed for bundled Redis)
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm repo update
|
||||
|
||||
# 2. Update chart dependencies
|
||||
helm dependency update ./helm/docuelevate
|
||||
|
||||
# 3. Install with a minimal values override
|
||||
helm install docuelevate ./helm/docuelevate \
|
||||
--namespace docuelevate --create-namespace \
|
||||
--set secrets.DATABASE_URL="postgresql://user:pass@postgres:5432/docuelevate" \
|
||||
--set secrets.SESSION_SECRET="$(openssl rand -hex 32)" \
|
||||
--set secrets.OPENAI_API_KEY="sk-..." \
|
||||
--set secrets.AZURE_AI_KEY="..." \
|
||||
--set env.AZURE_ENDPOINT="https://my-resource.cognitiveservices.azure.com/" \
|
||||
--set env.EXTERNAL_HOSTNAME="docuelevate.example.com"
|
||||
```
|
||||
|
||||
### Values Reference
|
||||
|
||||
The full list of configurable values is in [`helm/docuelevate/values.yaml`](../helm/docuelevate/values.yaml). Key sections:
|
||||
|
||||
#### Container Image
|
||||
|
||||
```yaml
|
||||
image:
|
||||
repository: ghcr.io/christianlouis/docuelevate
|
||||
tag: "" # defaults to chart appVersion
|
||||
pullPolicy: IfNotPresent
|
||||
```
|
||||
|
||||
#### Non-Secret Config (`env`)
|
||||
|
||||
```yaml
|
||||
env:
|
||||
WORKDIR: /workdir
|
||||
AI_PROVIDER: openai
|
||||
OPENAI_MODEL: gpt-4o-mini
|
||||
AZURE_REGION: eastus
|
||||
AZURE_ENDPOINT: "https://my-resource.cognitiveservices.azure.com/"
|
||||
MEILISEARCH_URL: http://docuelevate-meilisearch:7700 # auto-resolved from service name
|
||||
ENABLE_SEARCH: "true"
|
||||
AUTH_ENABLED: "true"
|
||||
EXTERNAL_HOSTNAME: docuelevate.example.com
|
||||
```
|
||||
|
||||
#### Secrets (`secrets`)
|
||||
|
||||
All secrets are stored in a Kubernetes `Secret` and injected as environment variables.
|
||||
|
||||
```yaml
|
||||
secrets:
|
||||
DATABASE_URL: "postgresql://user:pass@postgres:5432/docuelevate"
|
||||
SESSION_SECRET: "<min-32-char-random-string>"
|
||||
OPENAI_API_KEY: "sk-..."
|
||||
AZURE_AI_KEY: "..."
|
||||
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated dev Meilisearch
|
||||
# Storage provider secrets ...
|
||||
```
|
||||
|
||||
> **Tip:** In production use an external secret manager (Vault, ESO, Sealed Secrets) and reference the secret by name instead of embedding values in values.yaml.
|
||||
|
||||
#### Replicas & Autoscaling
|
||||
|
||||
```yaml
|
||||
api:
|
||||
replicaCount: 2
|
||||
autoscaling:
|
||||
enabled: true
|
||||
minReplicas: 2
|
||||
maxReplicas: 8
|
||||
targetCPUUtilizationPercentage: 70
|
||||
|
||||
worker:
|
||||
replicaCount: 2
|
||||
autoscaling:
|
||||
enabled: true
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
targetCPUUtilizationPercentage: 75
|
||||
```
|
||||
|
||||
#### Shared Workdir PVC
|
||||
|
||||
```yaml
|
||||
workdir:
|
||||
persistence:
|
||||
enabled: true
|
||||
accessMode: ReadWriteMany # RWX required for multi-replica
|
||||
size: 20Gi
|
||||
storageClass: "nfs-client" # or leave blank for cluster default
|
||||
```
|
||||
|
||||
#### Ingress (nginx example)
|
||||
|
||||
```yaml
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "1g"
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
hosts:
|
||||
- host: docuelevate.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls:
|
||||
- secretName: docuelevate-tls
|
||||
hosts:
|
||||
- docuelevate.example.com
|
||||
```
|
||||
|
||||
#### External Redis
|
||||
|
||||
```yaml
|
||||
redis:
|
||||
enabled: false # disable bundled Redis
|
||||
|
||||
externalRedis:
|
||||
url: "redis://my-redis-host:6379/0"
|
||||
```
|
||||
|
||||
#### Meilisearch
|
||||
|
||||
The bundled Meilisearch deployment is a single-replica, persistent StatefulSet-equivalent. For production, consider [Meilisearch Cloud](https://www.meilisearch.com/cloud) and point `env.MEILISEARCH_URL` at it.
|
||||
|
||||
```yaml
|
||||
meilisearch:
|
||||
enabled: true
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 10Gi
|
||||
```
|
||||
|
||||
### Upgrading
|
||||
|
||||
```bash
|
||||
helm upgrade docuelevate ./helm/docuelevate \
|
||||
--namespace docuelevate \
|
||||
-f my-values.yaml
|
||||
```
|
||||
|
||||
The pre-upgrade hook runs `alembic upgrade head` automatically before the new pods start.
|
||||
|
||||
### Uninstalling
|
||||
|
||||
```bash
|
||||
helm uninstall docuelevate --namespace docuelevate
|
||||
# PVCs are NOT deleted automatically — remove manually if desired:
|
||||
kubectl delete pvc -l app.kubernetes.io/instance=docuelevate -n docuelevate
|
||||
```
|
||||
|
||||
### Kubernetes Architecture Diagram
|
||||
|
||||
```
|
||||
Internet
|
||||
│
|
||||
▼
|
||||
[Ingress / LoadBalancer]
|
||||
│
|
||||
▼
|
||||
[API Deployment] ─────── [Worker Deployment]
|
||||
│ │ │ │
|
||||
│ └── shared PVC ──┘ │
|
||||
│ (workdir) │
|
||||
▼ ▼
|
||||
[Redis Service] [Gotenberg Service]
|
||||
│
|
||||
[Meilisearch Service]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Production Considerations
|
||||
|
||||
### Security Headers
|
||||
### Database
|
||||
|
||||
DocuElevate includes built-in support for HTTP security headers to improve browser-side security. **These headers are disabled by default** since most deployments use a reverse proxy (Traefik, Nginx, etc.) that already adds these headers.
|
||||
SQLite is fine for development but **not recommended for multi-replica production** deployments because it cannot be shared safely across pods. Use PostgreSQL:
|
||||
|
||||
#### Supported Security Headers
|
||||
|
||||
- **Strict-Transport-Security (HSTS)**: Forces browsers to use HTTPS for all future requests
|
||||
- **Content-Security-Policy (CSP)**: Controls which resources browsers are allowed to load
|
||||
- **X-Frame-Options**: Prevents the page from being loaded in frames (clickjacking protection)
|
||||
- **X-Content-Type-Options**: Prevents browsers from MIME-sniffing responses
|
||||
|
||||
#### Reverse Proxy Deployment (Traefik, Nginx, etc.) - DEFAULT
|
||||
|
||||
**Most deployments use a reverse proxy**, which is why security headers are disabled by default in DocuElevate. The reverse proxy should add these headers.
|
||||
|
||||
```bash
|
||||
# In .env file (or omit - this is the default)
|
||||
SECURITY_HEADERS_ENABLED=false
|
||||
```
|
||||
DATABASE_URL=postgresql://docuelevate:secret@postgres-host:5432/docuelevate
|
||||
```
|
||||
|
||||
##### Traefik Configuration Example
|
||||
### Security Headers
|
||||
|
||||
Traefik can add security headers using middleware. Create a `docker-compose.yaml` with Traefik labels:
|
||||
DocuElevate's built-in security headers are **disabled by default** since most deployments use a reverse proxy that already adds them.
|
||||
|
||||
```bash
|
||||
# Enable only if running without a reverse proxy
|
||||
SECURITY_HEADERS_ENABLED=true
|
||||
```
|
||||
|
||||
#### Traefik (Docker Compose) example
|
||||
|
||||
```yaml
|
||||
services:
|
||||
@@ -85,35 +289,25 @@ services:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.docuelevate.rule=Host(`docuelevate.example.com`)"
|
||||
- "traefik.http.routers.docuelevate.entrypoints=websecure"
|
||||
- "traefik.http.routers.docuelevate.tls=true"
|
||||
- "traefik.http.routers.docuelevate.tls.certresolver=letsencrypt"
|
||||
# Security headers middleware
|
||||
- "traefik.http.routers.docuelevate.middlewares=security-headers@docker"
|
||||
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
|
||||
- "traefik.http.middlewares.security-headers.headers.stsIncludeSubdomains=true"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
|
||||
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
||||
```
|
||||
|
||||
Then set `SECURITY_HEADERS_ENABLED=false` in your `.env` file.
|
||||
|
||||
##### Nginx Configuration Example
|
||||
|
||||
Add security headers to your Nginx configuration:
|
||||
#### Nginx example
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name docuelevate.example.com;
|
||||
|
||||
# SSL configuration
|
||||
ssl_certificate /etc/nginx/ssl/cert.pem;
|
||||
ssl_certificate_key /etc/nginx/ssl/key.pem;
|
||||
|
||||
# Security headers
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
|
||||
@@ -123,144 +317,109 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
client_max_body_size 1g;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then keep `SECURITY_HEADERS_ENABLED=false` in your `.env` file (or omit it, as this is the default).
|
||||
|
||||
#### Direct Deployment (No Reverse Proxy)
|
||||
|
||||
If you're running DocuElevate **directly without a reverse proxy**, enable security headers:
|
||||
|
||||
```bash
|
||||
# In .env file
|
||||
SECURITY_HEADERS_ENABLED=true
|
||||
```
|
||||
|
||||
You can also configure individual headers:
|
||||
|
||||
```bash
|
||||
SECURITY_HEADER_HSTS_ENABLED=true
|
||||
SECURITY_HEADER_CSP_ENABLED=true
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
```
|
||||
|
||||
**Note**: HSTS only works when serving content over HTTPS. If using HTTP for development, you can disable it:
|
||||
|
||||
```bash
|
||||
SECURITY_HEADER_HSTS_ENABLED=false
|
||||
```
|
||||
|
||||
#### Customizing Security Headers
|
||||
|
||||
If you enable security headers, you can customize individual header values in your `.env` file:
|
||||
|
||||
```bash
|
||||
# Customize HSTS (e.g., shorter duration for testing)
|
||||
SECURITY_HEADER_HSTS_VALUE="max-age=300"
|
||||
|
||||
# Customize CSP (e.g., allow specific external domains)
|
||||
SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://cdn.example.com; style-src 'self' 'unsafe-inline';"
|
||||
|
||||
# Allow framing from same origin
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="SAMEORIGIN"
|
||||
```
|
||||
|
||||
#### Security Considerations
|
||||
|
||||
1. **HSTS and HTTPS**: HSTS only works over HTTPS. Ensure you have a valid SSL certificate before enabling HSTS.
|
||||
2. **CSP Testing**: The default CSP policy allows inline scripts and styles for compatibility. Test thoroughly before tightening.
|
||||
3. **Content-Security-Policy**: The default policy allows `'unsafe-inline'` for scripts and styles for compatibility with Tailwind CSS and inline JavaScript. For stricter security, consider using nonces or hashes.
|
||||
4. **X-Frame-Options**: Set to `DENY` by default. Change to `SAMEORIGIN` if you need to embed DocuElevate in iframes on the same domain.
|
||||
|
||||
See the [Configuration Guide](ConfigurationGuide.md) for all security header options.
|
||||
|
||||
### Reverse Proxy Setup
|
||||
|
||||
For production use, we recommend setting up a reverse proxy (like Nginx or Traefik) to handle HTTPS and domain routing:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name docuelevate.example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Persistent Storage
|
||||
|
||||
The Docker setup uses volumes for persistent storage. For production, consider:
|
||||
### Storage
|
||||
|
||||
```yaml
|
||||
# Docker Compose
|
||||
volumes:
|
||||
- /path/to/persistent/storage:/workdir
|
||||
|
||||
# Helm — use a RWX storage class for multi-replica
|
||||
workdir:
|
||||
persistence:
|
||||
size: 20Gi
|
||||
accessMode: ReadWriteMany
|
||||
storageClass: "nfs-client"
|
||||
```
|
||||
|
||||
### Security
|
||||
### General Security Checklist
|
||||
|
||||
1. **Always use HTTPS** in production
|
||||
2. Enable authentication by setting `AUTH_ENABLED=true`
|
||||
3. Use strong passwords for all services
|
||||
4. Limit access to the Docker host
|
||||
5. Regularly update the application and dependencies
|
||||
2. Set `AUTH_ENABLED=true` and use a strong `SESSION_SECRET`
|
||||
3. Rotate API keys and secrets regularly — see the [Credential Rotation Guide](CredentialRotationGuide.md)
|
||||
4. Limit network access to Redis and Meilisearch (both should be internal-only)
|
||||
5. Regularly update the container image to pick up dependency patches
|
||||
|
||||
---
|
||||
|
||||
## Scaling
|
||||
|
||||
For high-volume deployments:
|
||||
### Docker Compose
|
||||
|
||||
1. Increase worker processes by adding more worker containers:
|
||||
Add more worker containers:
|
||||
|
||||
```yaml
|
||||
worker:
|
||||
image: christianlouis/document-processor:latest
|
||||
deploy:
|
||||
replicas: 3
|
||||
```
|
||||
|
||||
2. Consider using dedicated Redis and database servers
|
||||
3. Monitor system performance and adjust resources as needed
|
||||
### Kubernetes / Helm
|
||||
|
||||
Enable HPA:
|
||||
|
||||
```yaml
|
||||
api:
|
||||
autoscaling:
|
||||
enabled: true
|
||||
minReplicas: 2
|
||||
maxReplicas: 8
|
||||
|
||||
worker:
|
||||
autoscaling:
|
||||
enabled: true
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitoring
|
||||
|
||||
Monitor your DocuElevate deployment using:
|
||||
- **Docker Compose**: `docker-compose logs -f`, `docker stats`
|
||||
- **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f`
|
||||
- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed.
|
||||
- **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring.
|
||||
|
||||
- Docker's built-in logging: `docker-compose logs -f`
|
||||
- Container metrics: `docker stats`
|
||||
- External monitoring tools like Prometheus and Grafana
|
||||
---
|
||||
|
||||
## Backup Procedures
|
||||
|
||||
Regularly back up the following:
|
||||
Regularly back up:
|
||||
|
||||
1. The `/workdir` directory containing all processed documents
|
||||
2. The database file (if using SQLite) or database contents (if using another DBMS)
|
||||
3. The `.env` configuration file
|
||||
1. The `/workdir` volume (all processed documents and originals)
|
||||
2. The database (PostgreSQL `pg_dump` or SQLite file)
|
||||
3. The Meilisearch data directory (`/meili_data`)
|
||||
4. Your `.env` / Helm values file (store securely, it contains secrets)
|
||||
|
||||
---
|
||||
|
||||
## Updates
|
||||
|
||||
To update DocuElevate to a newer version:
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
# Pull the latest changes
|
||||
git pull
|
||||
|
||||
# Pull the latest Docker images
|
||||
docker-compose pull
|
||||
|
||||
# Restart the services
|
||||
docker-compose down
|
||||
docker-compose up -d
|
||||
docker-compose down && docker-compose up -d
|
||||
```
|
||||
|
||||
### Helm
|
||||
|
||||
```bash
|
||||
helm repo update # if using a hosted chart repository
|
||||
helm upgrade docuelevate ./helm/docuelevate --namespace docuelevate -f my-values.yaml
|
||||
```
|
||||
|
||||
The migration Job runs automatically on every `helm upgrade`.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
See the [Troubleshooting](Troubleshooting.md) guide for common deployment issues and solutions.
|
||||
See the [Troubleshooting Guide](Troubleshooting.md) for common issues and solutions.
|
||||
|
||||
@@ -418,6 +418,47 @@
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<!-- Full-Text Search Section -->
|
||||
<div class="filters-section" style="margin-top: 0.75rem;">
|
||||
<div style="width: 100%;">
|
||||
<label for="fulltext-search" style="font-weight: 600; display: block; margin-bottom: 0.4rem;">
|
||||
<i class="fas fa-search"></i> Full-Text Search (OCR text, metadata, tags)
|
||||
</label>
|
||||
<div style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
|
||||
<input
|
||||
type="text"
|
||||
id="fulltext-search"
|
||||
placeholder="Search document content, sender, tags, type..."
|
||||
style="flex: 1; min-width: 220px; padding: 0.5rem 0.75rem; border: 1px solid #d1d5db; border-radius: 0.375rem; font-size: 0.875rem;"
|
||||
oninput="debounceSearch(this.value)"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onclick="runFullTextSearch()"
|
||||
style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
|
||||
>
|
||||
Search
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onclick="clearFullTextSearch()"
|
||||
style="padding: 0.5rem 1rem; background-color: #6b7280; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Full-Text Search Results Panel -->
|
||||
<div id="search-results-panel" style="display: none; margin-top: 0.5rem; border: 1px solid #e5e7eb; border-radius: 0.5rem; background: white; box-shadow: 0 1px 3px rgba(0,0,0,0.08);">
|
||||
<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #e5e7eb; display: flex; justify-content: space-between; align-items: center; background: #f9fafb; border-radius: 0.5rem 0.5rem 0 0;">
|
||||
<span id="search-results-summary" style="font-size: 0.875rem; color: #374151;"></span>
|
||||
<div id="search-results-pagination" style="display: flex; gap: 0.5rem;"></div>
|
||||
</div>
|
||||
<div id="search-results-list" style="padding: 0.5rem 0;"></div>
|
||||
</div>
|
||||
|
||||
<!-- Bulk Actions Section -->
|
||||
<div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
|
||||
<div class="flex flex-col sm:flex-row sm:justify-between sm:items-center gap-3">
|
||||
@@ -887,6 +928,117 @@
|
||||
function closeUploadModal() {
|
||||
uploadModal.classList.remove('active');
|
||||
}
|
||||
|
||||
// ---- Full-Text Search (Meilisearch) ----
|
||||
let _searchDebounceTimer = null;
|
||||
let _searchCurrentPage = 1;
|
||||
|
||||
function debounceSearch(value) {
|
||||
clearTimeout(_searchDebounceTimer);
|
||||
if (!value || value.trim().length < 2) {
|
||||
clearFullTextSearch();
|
||||
return;
|
||||
}
|
||||
_searchDebounceTimer = setTimeout(() => {
|
||||
_searchCurrentPage = 1;
|
||||
runFullTextSearch();
|
||||
}, 400);
|
||||
}
|
||||
|
||||
function runFullTextSearch(page) {
|
||||
const input = document.getElementById('fulltext-search');
|
||||
const q = input ? input.value.trim() : '';
|
||||
if (!q) { clearFullTextSearch(); return; }
|
||||
if (page) _searchCurrentPage = page;
|
||||
|
||||
const panel = document.getElementById('search-results-panel');
|
||||
const list = document.getElementById('search-results-list');
|
||||
const summary = document.getElementById('search-results-summary');
|
||||
const pagination = document.getElementById('search-results-pagination');
|
||||
|
||||
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;"><i class="fas fa-spinner fa-spin"></i> Searching…</div>';
|
||||
summary.textContent = '';
|
||||
pagination.innerHTML = '';
|
||||
panel.style.display = 'block';
|
||||
|
||||
const params = new URLSearchParams({ q, page: _searchCurrentPage, per_page: 20 });
|
||||
fetch(`/api/search?${params.toString()}`)
|
||||
.then(r => {
|
||||
if (!r.ok) throw new Error(`Search returned ${r.status}`);
|
||||
return r.json();
|
||||
})
|
||||
.then(data => renderSearchResults(data, q))
|
||||
.catch(err => {
|
||||
list.innerHTML = `<div style="padding: 1rem; color: #dc2626; font-size: 0.875rem;"><i class="fas fa-exclamation-triangle"></i> Search unavailable: ${err.message}</div>`;
|
||||
summary.textContent = '';
|
||||
});
|
||||
}
|
||||
|
||||
function renderSearchResults(data, q) {
|
||||
const panel = document.getElementById('search-results-panel');
|
||||
const list = document.getElementById('search-results-list');
|
||||
const summary = document.getElementById('search-results-summary');
|
||||
const pagination = document.getElementById('search-results-pagination');
|
||||
|
||||
const { results, total, page, pages } = data;
|
||||
summary.textContent = `${total} result${total !== 1 ? 's' : ''} for "${q}"`;
|
||||
|
||||
if (!results || results.length === 0) {
|
||||
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;">No results found.</div>';
|
||||
pagination.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
|
||||
list.innerHTML = results.map(hit => {
|
||||
const fmt = hit._formatted || {};
|
||||
const title = fmt.document_title || hit.document_title || hit.original_filename || '(untitled)';
|
||||
const filename = fmt.original_filename || hit.original_filename || '';
|
||||
const snippet = fmt.ocr_text || '';
|
||||
const tags = Array.isArray(hit.tags) ? hit.tags.join(', ') : (hit.tags || '');
|
||||
const docType = hit.document_type || '';
|
||||
|
||||
return `<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #f3f4f6; display: flex; gap: 0.75rem; align-items: flex-start;">
|
||||
<div style="flex-shrink: 0; color: #3b82f6; font-size: 1.25rem; padding-top: 0.1rem;">
|
||||
<i class="fas fa-file-pdf"></i>
|
||||
</div>
|
||||
<div style="flex: 1; min-width: 0;">
|
||||
<div style="font-weight: 600; font-size: 0.9rem; color: #111827;">${title}</div>
|
||||
${filename ? `<div style="font-size: 0.8rem; color: #6b7280; margin-top: 0.15rem;">${filename}</div>` : ''}
|
||||
${docType ? `<span style="display: inline-block; margin-top: 0.25rem; padding: 0.1rem 0.5rem; background: #eff6ff; color: #1d4ed8; border-radius: 9999px; font-size: 0.75rem;">${docType}</span>` : ''}
|
||||
${tags ? `<span style="display: inline-block; margin-top: 0.25rem; margin-left: 0.25rem; padding: 0.1rem 0.5rem; background: #f0fdf4; color: #15803d; border-radius: 9999px; font-size: 0.75rem;">${tags}</span>` : ''}
|
||||
${snippet ? `<div style="margin-top: 0.4rem; font-size: 0.8rem; color: #374151; white-space: pre-wrap; word-break: break-word;">…${snippet}…</div>` : ''}
|
||||
</div>
|
||||
<div style="flex-shrink: 0;">
|
||||
<a href="/files/${hit.file_id}" style="padding: 0.25rem 0.6rem; background: #f3f4f6; color: #374151; border-radius: 0.25rem; font-size: 0.8rem; text-decoration: none; white-space: nowrap;" title="View file">
|
||||
<i class="fas fa-external-link-alt"></i>
|
||||
</a>
|
||||
</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
// Pagination
|
||||
if (pages > 1) {
|
||||
const btns = [];
|
||||
if (page > 1) {
|
||||
btns.push(`<button onclick="runFullTextSearch(${page - 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">« Prev</button>`);
|
||||
}
|
||||
btns.push(`<span style="padding: 0.25rem 0.6rem; font-size: 0.8rem; color: #6b7280;">Page ${page} / ${pages}</span>`);
|
||||
if (page < pages) {
|
||||
btns.push(`<button onclick="runFullTextSearch(${page + 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">Next »</button>`);
|
||||
}
|
||||
pagination.innerHTML = btns.join('');
|
||||
} else {
|
||||
pagination.innerHTML = '';
|
||||
}
|
||||
}
|
||||
|
||||
function clearFullTextSearch() {
|
||||
const panel = document.getElementById('search-results-panel');
|
||||
const input = document.getElementById('fulltext-search');
|
||||
if (panel) panel.style.display = 'none';
|
||||
if (input) input.value = '';
|
||||
_searchCurrentPage = 1;
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Patterns to ignore when building packages.
|
||||
.DS_Store
|
||||
.git
|
||||
.gitignore
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
.vscode
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: v2
|
||||
name: docuelevate
|
||||
description: >
|
||||
DocuElevate — intelligent document processing with OCR, AI metadata
|
||||
extraction, full-text search (Meilisearch), and multi-cloud storage.
|
||||
|
||||
type: application
|
||||
|
||||
# Chart version — bump on every chart change (independent of appVersion).
|
||||
version: 0.1.0
|
||||
|
||||
# Application version — kept in sync with the VERSION file.
|
||||
appVersion: "0.54.0"
|
||||
|
||||
keywords:
|
||||
- document-management
|
||||
- ocr
|
||||
- ai
|
||||
- meilisearch
|
||||
- fastapi
|
||||
- celery
|
||||
|
||||
home: https://github.com/christianlouis/DocuElevate
|
||||
sources:
|
||||
- https://github.com/christianlouis/DocuElevate
|
||||
|
||||
maintainers:
|
||||
- name: DocuElevate Contributors
|
||||
url: https://github.com/christianlouis/DocuElevate
|
||||
|
||||
dependencies:
|
||||
# Bundled Redis (optional — disable and point to an external instance via
|
||||
# externalRedis.url if you already have Redis in the cluster).
|
||||
- name: redis
|
||||
version: "20.x.x"
|
||||
repository: "https://charts.bitnami.com/bitnami"
|
||||
condition: redis.enabled
|
||||
@@ -0,0 +1,45 @@
|
||||
Thank you for installing {{ .Chart.Name }} {{ .Chart.Version }}!
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
The application will be available at:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
To access DocuElevate, run:
|
||||
|
||||
kubectl port-forward svc/{{ include "docuelevate.fullname" . }}-api {{ .Values.api.service.port }}:{{ .Values.api.service.port }} -n {{ .Release.Namespace }}
|
||||
|
||||
Then open http://localhost:{{ .Values.api.service.port }}
|
||||
{{- end }}
|
||||
|
||||
API docs are available at: <your-url>/docs
|
||||
|
||||
{{- if not .Values.secrets.SESSION_SECRET }}
|
||||
|
||||
⚠️ WARNING: secrets.SESSION_SECRET is not set.
|
||||
Generate one with: openssl rand -hex 32
|
||||
Then pass it via: --set secrets.SESSION_SECRET=<value>
|
||||
or via a values file / external secret.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.secrets.DATABASE_URL }}
|
||||
|
||||
⚠️ WARNING: secrets.DATABASE_URL is not set.
|
||||
For production, use PostgreSQL:
|
||||
postgresql://user:pass@host:5432/docuelevate
|
||||
SQLite with a PVC is supported but not recommended for multi-replica
|
||||
deployments (use ReadWriteOnce PVC and replicaCount=1 in that case).
|
||||
|
||||
{{- end }}
|
||||
|
||||
Useful commands:
|
||||
# View API logs
|
||||
kubectl logs -l app.kubernetes.io/component=api -n {{ .Release.Namespace }} -f
|
||||
|
||||
# View worker logs
|
||||
kubectl logs -l app.kubernetes.io/component=worker -n {{ .Release.Namespace }} -f
|
||||
|
||||
# Run a database migration manually
|
||||
kubectl create job --from=cronjob/{{ include "docuelevate.fullname" . }}-migrate migrate-manual -n {{ .Release.Namespace }}
|
||||
@@ -0,0 +1,108 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "docuelevate.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this.
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "docuelevate.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart label value: "chart-name-version"
|
||||
*/}}
|
||||
{{- define "docuelevate.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels applied to every resource.
|
||||
*/}}
|
||||
{{- define "docuelevate.labels" -}}
|
||||
helm.sh/chart: {{ include "docuelevate.chart" . }}
|
||||
{{ include "docuelevate.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels.
|
||||
*/}}
|
||||
{{- define "docuelevate.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "docuelevate.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
ServiceAccount name.
|
||||
*/}}
|
||||
{{- define "docuelevate.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
{{- default (include "docuelevate.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Image reference, using appVersion as default tag.
|
||||
*/}}
|
||||
{{- define "docuelevate.image" -}}
|
||||
{{- $tag := .Values.image.tag | default .Chart.AppVersion }}
|
||||
{{- printf "%s:%s" .Values.image.repository $tag }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Resolve REDIS_URL: prefer externalRedis.url, then secrets.REDIS_URL,
|
||||
then fall back to the bundled Redis service URL.
|
||||
*/}}
|
||||
{{- define "docuelevate.redisUrl" -}}
|
||||
{{- if .Values.externalRedis.url }}
|
||||
{{- .Values.externalRedis.url }}
|
||||
{{- else if .Values.secrets.REDIS_URL }}
|
||||
{{- .Values.secrets.REDIS_URL }}
|
||||
{{- else if .Values.redis.enabled }}
|
||||
{{- printf "redis://%s-redis-master:6379/0" .Release.Name }}
|
||||
{{- else }}
|
||||
{{- "" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Resolve GOTENBERG_URL — use env override if set, otherwise build from service name.
|
||||
*/}}
|
||||
{{- define "docuelevate.gotenbergUrl" -}}
|
||||
{{- if .Values.env.GOTENBERG_URL }}
|
||||
{{- tpl .Values.env.GOTENBERG_URL . }}
|
||||
{{- else }}
|
||||
{{- printf "http://%s-gotenberg:3000" (include "docuelevate.fullname" .) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Resolve MEILISEARCH_URL — use env override if set, otherwise build from service name.
|
||||
*/}}
|
||||
{{- define "docuelevate.meilisearchUrl" -}}
|
||||
{{- if .Values.env.MEILISEARCH_URL }}
|
||||
{{- tpl .Values.env.MEILISEARCH_URL . }}
|
||||
{{- else }}
|
||||
{{- printf "http://%s-meilisearch:7700" (include "docuelevate.fullname" .) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,93 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-api
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
{{- if not .Values.api.autoscaling.enabled }}
|
||||
replicas: {{ .Values.api.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- with .Values.api.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.api.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: api
|
||||
image: {{ include "docuelevate.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- uvicorn
|
||||
- app.main:app
|
||||
- --host
|
||||
- "0.0.0.0"
|
||||
- --port
|
||||
- "8000"
|
||||
- --proxy-headers
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-secret
|
||||
{{- with .Values.api.livenessProbe }}
|
||||
livenessProbe:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.api.readinessProbe }}
|
||||
readinessProbe:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.api.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: workdir
|
||||
mountPath: /workdir
|
||||
volumes:
|
||||
- name: workdir
|
||||
{{- if .Values.workdir.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- with .Values.api.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.api.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.api.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-api
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
type: {{ .Values.api.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.api.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
@@ -0,0 +1,40 @@
|
||||
# Non-sensitive application configuration.
|
||||
# All values are injected as environment variables into the API and Worker pods.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-config
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
data:
|
||||
WORKDIR: {{ .Values.env.WORKDIR | quote }}
|
||||
PYTHONPATH: "/app"
|
||||
PYTHONUNBUFFERED: "1"
|
||||
|
||||
# AI provider
|
||||
AI_PROVIDER: {{ .Values.env.AI_PROVIDER | quote }}
|
||||
OPENAI_MODEL: {{ .Values.env.OPENAI_MODEL | quote }}
|
||||
|
||||
# Azure
|
||||
AZURE_REGION: {{ .Values.env.AZURE_REGION | quote }}
|
||||
AZURE_ENDPOINT: {{ .Values.env.AZURE_ENDPOINT | quote }}
|
||||
|
||||
# Service URLs — resolved from helper templates so they always match the
|
||||
# in-cluster service names even if values.yaml uses template expressions.
|
||||
GOTENBERG_URL: {{ include "docuelevate.gotenbergUrl" . | quote }}
|
||||
MEILISEARCH_URL: {{ include "docuelevate.meilisearchUrl" . | quote }}
|
||||
MEILISEARCH_INDEX_NAME: {{ .Values.env.MEILISEARCH_INDEX_NAME | quote }}
|
||||
ENABLE_SEARCH: {{ .Values.env.ENABLE_SEARCH | quote }}
|
||||
|
||||
# Auth
|
||||
AUTH_ENABLED: {{ .Values.env.AUTH_ENABLED | quote }}
|
||||
ADMIN_USERNAME: {{ .Values.env.ADMIN_USERNAME | quote }}
|
||||
EXTERNAL_HOSTNAME: {{ .Values.env.EXTERNAL_HOSTNAME | quote }}
|
||||
|
||||
# Feature flags
|
||||
ENABLE_DEDUPLICATION: {{ .Values.env.ENABLE_DEDUPLICATION | quote }}
|
||||
ENABLE_TEXT_QUALITY_CHECK: {{ .Values.env.ENABLE_TEXT_QUALITY_CHECK | quote }}
|
||||
ALLOW_FILE_DELETE: {{ .Values.env.ALLOW_FILE_DELETE | quote }}
|
||||
|
||||
DEBUG: {{ .Values.env.DEBUG | quote }}
|
||||
@@ -0,0 +1,63 @@
|
||||
{{- if .Values.gotenberg.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-gotenberg
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: gotenberg
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: gotenberg
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: gotenberg
|
||||
spec:
|
||||
containers:
|
||||
- name: gotenberg
|
||||
image: {{ .Values.gotenberg.image.repository }}:{{ .Values.gotenberg.image.tag }}
|
||||
imagePullPolicy: {{ .Values.gotenberg.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
protocol: TCP
|
||||
resources:
|
||||
{{- toYaml .Values.gotenberg.resources | nindent 12 }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 3000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 3000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-gotenberg
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: gotenberg
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: {{ .Values.gotenberg.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: gotenberg
|
||||
{{- end }}
|
||||
@@ -0,0 +1,51 @@
|
||||
{{- if .Values.api.autoscaling.enabled }}
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-api
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "docuelevate.fullname" . }}-api
|
||||
minReplicas: {{ .Values.api.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.api.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: {{ .Values.api.autoscaling.targetCPUUtilizationPercentage }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
|
||||
{{- if .Values.worker.autoscaling.enabled }}
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-worker
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: worker
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "docuelevate.fullname" . }}-worker
|
||||
minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,36 @@
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- toYaml .Values.ingress.tls | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "docuelevate.fullname" $ }}-api
|
||||
port:
|
||||
number: {{ $.Values.api.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,89 @@
|
||||
{{- if .Values.meilisearch.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-meilisearch
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: meilisearch
|
||||
spec:
|
||||
# Single replica — Meilisearch Community Edition is single-node only.
|
||||
# Use Meilisearch Cloud or the paid Meilisearch Cluster for HA.
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: meilisearch
|
||||
strategy:
|
||||
# Recreate ensures the PVC is released before the new pod starts
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: meilisearch
|
||||
spec:
|
||||
containers:
|
||||
- name: meilisearch
|
||||
image: {{ .Values.meilisearch.image.repository }}:{{ .Values.meilisearch.image.tag }}
|
||||
imagePullPolicy: {{ .Values.meilisearch.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 7700
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: MEILI_NO_ANALYTICS
|
||||
value: {{ .Values.meilisearch.env.MEILI_NO_ANALYTICS | quote }}
|
||||
{{- if .Values.secrets.MEILISEARCH_API_KEY }}
|
||||
- name: MEILI_MASTER_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-secret
|
||||
key: MEILISEARCH_API_KEY
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.meilisearch.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /meili_data
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 7700
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 7700
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: data
|
||||
{{- if .Values.meilisearch.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.meilisearch.persistence.existingClaim | default (printf "%s-meilisearch-data" (include "docuelevate.fullname" .)) }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-meilisearch
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: meilisearch
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: {{ .Values.meilisearch.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: meilisearch
|
||||
{{- end }}
|
||||
@@ -0,0 +1,69 @@
|
||||
# Database migration Job — runs `alembic upgrade head` once before
|
||||
# the API and Worker Deployments start. Helm hooks ensure ordering:
|
||||
# the Job runs during pre-upgrade/pre-install and the Deployments wait
|
||||
# for it via the `helm.sh/hook-weight` annotation.
|
||||
{{- if .Values.migrations.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-migrate-{{ .Release.Revision }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: migrate
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation
|
||||
{{- if .Values.migrations.ttlSecondsAfterFinished }}
|
||||
spec:
|
||||
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.migrations.ttlSecondsAfterFinished }}
|
||||
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
|
||||
{{- end }}
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||
{{- with .Values.image.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: migrate
|
||||
image: {{ include "docuelevate.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-secret
|
||||
{{- with .Values.workdir.persistence }}
|
||||
volumeMounts:
|
||||
- name: workdir
|
||||
mountPath: /workdir
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
volumes:
|
||||
- name: workdir
|
||||
{{- if .Values.workdir.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,49 @@
|
||||
# Shared workdir PVC — mounted by both the API and the Worker so they can
|
||||
# exchange files during processing. Requires ReadWriteMany (e.g. NFS,
|
||||
# CephFS, Azure Files, EFS) when running multiple replicas.
|
||||
# Set workdir.persistence.accessMode=ReadWriteOnce and replicaCount=1
|
||||
# for single-node / dev clusters.
|
||||
{{- if .Values.workdir.persistence.enabled }}
|
||||
{{- if not .Values.workdir.persistence.existingClaim }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-workdir
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- {{ .Values.workdir.persistence.accessMode }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.workdir.persistence.size }}
|
||||
{{- if .Values.workdir.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.workdir.persistence.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
|
||||
# Meilisearch data PVC
|
||||
{{- if and .Values.meilisearch.enabled .Values.meilisearch.persistence.enabled }}
|
||||
{{- if not .Values.meilisearch.persistence.existingClaim }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-meilisearch-data
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- {{ .Values.meilisearch.persistence.accessMode }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.meilisearch.persistence.size }}
|
||||
{{- if .Values.meilisearch.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.meilisearch.persistence.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,14 @@
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "docuelevate.serviceAccountName" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: false
|
||||
{{- end }}
|
||||
@@ -0,0 +1,82 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "docuelevate.fullname" . }}-worker
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: worker
|
||||
spec:
|
||||
{{- if not .Values.worker.autoscaling.enabled }}
|
||||
replicas: {{ .Values.worker.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- with .Values.worker.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.worker.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker
|
||||
image: {{ include "docuelevate.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- celery
|
||||
- -A
|
||||
- app.celery_worker
|
||||
- worker
|
||||
- -B
|
||||
- --loglevel=info
|
||||
- -Q
|
||||
- document_processor,default,celery
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "docuelevate.fullname" . }}-secret
|
||||
{{- with .Values.worker.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: workdir
|
||||
mountPath: /workdir
|
||||
volumes:
|
||||
- name: workdir
|
||||
{{- if .Values.workdir.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- with .Values.worker.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.worker.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.worker.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,306 @@
|
||||
# =============================================================================
|
||||
# DocuElevate Helm Chart — values.yaml
|
||||
#
|
||||
# Override any value with:
|
||||
# helm install docuelevate ./helm/docuelevate -f my-values.yaml
|
||||
# helm install docuelevate ./helm/docuelevate --set key=value
|
||||
# =============================================================================
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Global image settings
|
||||
# ---------------------------------------------------------------------------
|
||||
image:
|
||||
repository: ghcr.io/christianlouis/docuelevate
|
||||
# Defaults to the chart appVersion; pin to a specific digest in production.
|
||||
tag: ""
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
imagePullSecrets: []
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared environment — non-secret application config
|
||||
# All values map 1-to-1 onto DocuElevate environment variables.
|
||||
# Sensitive values (API keys, passwords) go into `secrets` below.
|
||||
# ---------------------------------------------------------------------------
|
||||
env:
|
||||
# Required ----------------------------------------------------------------
|
||||
WORKDIR: /workdir
|
||||
|
||||
# AI provider (openai | azure | anthropic | gemini | ollama | openrouter)
|
||||
AI_PROVIDER: openai
|
||||
OPENAI_MODEL: gpt-4o-mini
|
||||
|
||||
# Azure Document Intelligence (required when OCR_PROVIDERS includes "azure")
|
||||
AZURE_REGION: eastus
|
||||
AZURE_ENDPOINT: "" # e.g. https://my-resource.cognitiveservices.azure.com/
|
||||
|
||||
# Gotenberg PDF conversion service
|
||||
GOTENBERG_URL: http://{{ include "docuelevate.fullname" . }}-gotenberg:3000
|
||||
|
||||
# Full-text search — uses the in-cluster Meilisearch service by default
|
||||
MEILISEARCH_URL: http://{{ include "docuelevate.fullname" . }}-meilisearch:7700
|
||||
MEILISEARCH_INDEX_NAME: documents
|
||||
ENABLE_SEARCH: "true"
|
||||
|
||||
# Authentication
|
||||
AUTH_ENABLED: "true"
|
||||
ADMIN_USERNAME: admin
|
||||
|
||||
# Feature flags
|
||||
ENABLE_DEDUPLICATION: "true"
|
||||
ENABLE_TEXT_QUALITY_CHECK: "true"
|
||||
ALLOW_FILE_DELETE: "true"
|
||||
|
||||
# Logging / misc
|
||||
DEBUG: "false"
|
||||
EXTERNAL_HOSTNAME: localhost # set to your public hostname / Ingress host
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Secrets — values are stored in a Kubernetes Secret and injected as env vars.
|
||||
# In production, use an external secret manager (Vault, ESO, Sealed Secrets)
|
||||
# and leave these blank, then mount the Secret yourself.
|
||||
# ---------------------------------------------------------------------------
|
||||
secrets:
|
||||
DATABASE_URL: "" # e.g. postgresql://user:pass@postgres:5432/docuelevate
|
||||
REDIS_URL: "" # leave blank to use bundled Redis
|
||||
SESSION_SECRET: "" # min 32-char random string — generate with: openssl rand -hex 32
|
||||
OPENAI_API_KEY: ""
|
||||
AZURE_AI_KEY: ""
|
||||
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated (dev) Meilisearch
|
||||
|
||||
# Storage provider secrets (only the ones you use)
|
||||
DROPBOX_APP_KEY: ""
|
||||
DROPBOX_APP_SECRET: ""
|
||||
DROPBOX_REFRESH_TOKEN: ""
|
||||
GOOGLE_DRIVE_CREDENTIALS_JSON: ""
|
||||
ONEDRIVE_CLIENT_ID: ""
|
||||
ONEDRIVE_CLIENT_SECRET: ""
|
||||
ONEDRIVE_REFRESH_TOKEN: ""
|
||||
AWS_ACCESS_KEY_ID: ""
|
||||
AWS_SECRET_ACCESS_KEY: ""
|
||||
|
||||
# OAuth / Authentik
|
||||
AUTHENTIK_CLIENT_ID: ""
|
||||
AUTHENTIK_CLIENT_SECRET: ""
|
||||
AUTHENTIK_CONFIG_URL: ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# External services
|
||||
# Set externalRedis.url (and disable bundled redis) when you have your own.
|
||||
# ---------------------------------------------------------------------------
|
||||
externalRedis:
|
||||
# When non-empty this value is injected as REDIS_URL, overriding secrets.REDIS_URL
|
||||
# and the auto-generated bundled-Redis URL.
|
||||
url: ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API deployment
|
||||
# ---------------------------------------------------------------------------
|
||||
api:
|
||||
replicaCount: 2
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
# Horizontal Pod Autoscaler
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 2
|
||||
maxReplicas: 8
|
||||
targetCPUUtilizationPercentage: 70
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8000
|
||||
|
||||
# Liveness / readiness probes
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 20
|
||||
failureThreshold: 3
|
||||
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: 8000
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
|
||||
podAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false # app writes to /workdir
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Celery worker deployment
|
||||
# ---------------------------------------------------------------------------
|
||||
worker:
|
||||
replicaCount: 2
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
targetCPUUtilizationPercentage: 75
|
||||
|
||||
podAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared workdir volume (api + worker mount the same PVC)
|
||||
# ---------------------------------------------------------------------------
|
||||
workdir:
|
||||
persistence:
|
||||
enabled: true
|
||||
# storageClass: "" # leave blank for cluster default
|
||||
accessMode: ReadWriteMany # RWX required for multiple pods
|
||||
size: 20Gi
|
||||
# existingClaim: ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gotenberg (PDF conversion)
|
||||
# ---------------------------------------------------------------------------
|
||||
gotenberg:
|
||||
enabled: true
|
||||
image:
|
||||
repository: gotenberg/gotenberg
|
||||
tag: latest
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
port: 3000
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Meilisearch (full-text search)
|
||||
# ---------------------------------------------------------------------------
|
||||
meilisearch:
|
||||
enabled: true
|
||||
image:
|
||||
repository: getmeili/meilisearch
|
||||
tag: latest
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
port: 7700
|
||||
env:
|
||||
MEILI_NO_ANALYTICS: "true"
|
||||
# MEILI_MASTER_KEY: "" # set via secrets.MEILISEARCH_API_KEY instead
|
||||
persistence:
|
||||
enabled: true
|
||||
# storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
# existingClaim: ""
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bundled Redis (from Bitnami chart)
|
||||
# Disable and set externalRedis.url to use your own.
|
||||
# ---------------------------------------------------------------------------
|
||||
redis:
|
||||
enabled: true
|
||||
architecture: standalone
|
||||
auth:
|
||||
enabled: false
|
||||
master:
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 4Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ingress
|
||||
# ---------------------------------------------------------------------------
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "" # e.g. nginx, traefik
|
||||
annotations: {}
|
||||
# nginx.ingress.kubernetes.io/proxy-body-size: "1g"
|
||||
# cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
hosts:
|
||||
- host: docuelevate.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls: []
|
||||
# - secretName: docuelevate-tls
|
||||
# hosts:
|
||||
# - docuelevate.example.com
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ServiceAccount
|
||||
# ---------------------------------------------------------------------------
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
name: ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database migration job
|
||||
# Runs `alembic upgrade head` before the api/worker start.
|
||||
# ---------------------------------------------------------------------------
|
||||
migrations:
|
||||
enabled: true
|
||||
# Automatically deleted after successful completion
|
||||
ttlSecondsAfterFinished: 120
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Add OCR text and AI metadata fields to files table for full-text search
|
||||
|
||||
Revision ID: 004_add_search_fields
|
||||
Revises: 003_add_deduplication_support
|
||||
Create Date: 2026-02-25
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "004_add_search_fields"
|
||||
down_revision: Union[str, None] = "003_add_deduplication_support"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add ocr_text, ai_metadata, and document_title columns to files table."""
|
||||
op.add_column("files", sa.Column("ocr_text", sa.Text(), nullable=True))
|
||||
op.add_column("files", sa.Column("ai_metadata", sa.Text(), nullable=True))
|
||||
op.add_column("files", sa.Column("document_title", sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove ocr_text, ai_metadata, and document_title columns from files table."""
|
||||
op.drop_column("files", "document_title")
|
||||
op.drop_column("files", "ai_metadata")
|
||||
op.drop_column("files", "ocr_text")
|
||||
+2
-1
@@ -42,4 +42,5 @@ litellm>=1.0.0,<2.0.0
|
||||
# Self-hosted OCR engines (optional – only required when the provider is enabled)
|
||||
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
|
||||
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
|
||||
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
||||
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
||||
meilisearch>=0.31.0 # Full-text search engine client
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Tests for the full-text search API (app/api/search.py) and Meilisearch
|
||||
client utilities (app/utils/meilisearch_client.py).
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# app/utils/meilisearch_client tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMeilisearchClientDisabled:
|
||||
"""Tests when search is disabled or Meilisearch is unavailable."""
|
||||
|
||||
def test_get_client_returns_none_when_disabled(self):
|
||||
"""get_meilisearch_client returns None when enable_search=False."""
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||
from app.utils.meilisearch_client import index_document, search_documents
|
||||
|
||||
class _FakeRecord:
|
||||
id = 1
|
||||
original_filename = "test.pdf"
|
||||
mime_type = "application/pdf"
|
||||
file_size = 1024
|
||||
created_at = None
|
||||
|
||||
result = index_document(_FakeRecord(), "some text", {})
|
||||
assert result is False
|
||||
|
||||
result = search_documents("invoice")
|
||||
assert result["results"] == []
|
||||
assert result["total"] == 0
|
||||
|
||||
def test_search_documents_import_error(self):
|
||||
"""search_documents returns empty dict when meilisearch not installed."""
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
result = search_documents("test")
|
||||
assert result["results"] == []
|
||||
assert result["total"] == 0
|
||||
assert result["page"] == 1
|
||||
assert result["query"] == "test"
|
||||
|
||||
def test_delete_document_no_client(self):
|
||||
"""delete_document returns False when client unavailable."""
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||
from app.utils.meilisearch_client import delete_document
|
||||
|
||||
result = delete_document(99)
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMeilisearchIndexDocument:
|
||||
"""Tests for index_document function."""
|
||||
|
||||
def test_index_document_success(self):
|
||||
"""index_document returns True when Meilisearch succeeds."""
|
||||
mock_client = MagicMock()
|
||||
mock_index = MagicMock()
|
||||
mock_task = MagicMock()
|
||||
mock_task.task_uid = 1
|
||||
|
||||
mock_client.get_index.return_value = mock_index
|
||||
mock_index.add_documents.return_value = mock_task
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import index_document
|
||||
|
||||
class _FakeRecord:
|
||||
id = 1
|
||||
original_filename = "invoice.pdf"
|
||||
mime_type = "application/pdf"
|
||||
file_size = 2048
|
||||
created_at = None
|
||||
|
||||
result = index_document(
|
||||
_FakeRecord(),
|
||||
"This is an invoice for services rendered",
|
||||
{
|
||||
"title": "Invoice January 2026",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["invoice", "services"],
|
||||
"absender": "ACME Corp",
|
||||
"language": "en",
|
||||
},
|
||||
)
|
||||
assert result is True
|
||||
mock_index.add_documents.assert_called_once()
|
||||
call_docs = mock_index.add_documents.call_args[0][0]
|
||||
assert len(call_docs) == 1
|
||||
doc = call_docs[0]
|
||||
assert doc["file_id"] == 1
|
||||
assert doc["document_title"] == "Invoice January 2026"
|
||||
assert "invoice" in doc["tags"]
|
||||
|
||||
def test_index_document_meilisearch_error(self):
|
||||
"""index_document returns False on Meilisearch exception."""
|
||||
mock_client = MagicMock()
|
||||
mock_index = MagicMock()
|
||||
mock_client.get_index.return_value = mock_index
|
||||
mock_index.add_documents.side_effect = RuntimeError("Meilisearch down")
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import index_document
|
||||
|
||||
class _FakeRecord:
|
||||
id = 2
|
||||
original_filename = "test.pdf"
|
||||
mime_type = "application/pdf"
|
||||
file_size = 512
|
||||
created_at = None
|
||||
|
||||
result = index_document(_FakeRecord(), "some text", {})
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMeilisearchSearchDocuments:
|
||||
"""Tests for search_documents function."""
|
||||
|
||||
def _make_mock_client(self, hits=None, total=None):
|
||||
mock_client = MagicMock()
|
||||
mock_index = MagicMock()
|
||||
mock_client.get_index.return_value = mock_index
|
||||
mock_index.search.return_value = {
|
||||
"hits": hits or [],
|
||||
"estimatedTotalHits": total if total is not None else len(hits or []),
|
||||
}
|
||||
return mock_client, mock_index
|
||||
|
||||
def test_search_returns_results(self):
|
||||
"""search_documents returns hits from Meilisearch."""
|
||||
hits = [
|
||||
{
|
||||
"file_id": 42,
|
||||
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||
"document_title": "Amazon Invoice",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["amazon", "invoice"],
|
||||
"ocr_text": "Amazon invoice content here",
|
||||
"_formatted": {
|
||||
"document_title": "Amazon <mark>Invoice</mark>",
|
||||
"ocr_text": "…Amazon <mark>invoice</mark> content here…",
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_client, mock_index = self._make_mock_client(hits=hits, total=1)
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
result = search_documents("invoice", page=1, per_page=20)
|
||||
|
||||
assert result["total"] == 1
|
||||
assert result["pages"] == 1
|
||||
assert result["query"] == "invoice"
|
||||
assert len(result["results"]) == 1
|
||||
# Raw ocr_text should be stripped from result (only _formatted snippet kept)
|
||||
assert "ocr_text" not in result["results"][0]
|
||||
assert result["results"][0]["file_id"] == 42
|
||||
|
||||
def test_search_with_filters(self):
|
||||
"""search_documents passes filter expressions to Meilisearch."""
|
||||
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
search_documents("contract", mime_type="application/pdf", language="de", page=1, per_page=10)
|
||||
|
||||
call_kwargs = mock_index.search.call_args
|
||||
search_params = call_kwargs[0][1]
|
||||
assert "filter" in search_params
|
||||
assert 'mime_type = "application/pdf"' in search_params["filter"]
|
||||
assert 'language = "de"' in search_params["filter"]
|
||||
|
||||
def test_search_pagination(self):
|
||||
"""search_documents applies correct offset for page 2."""
|
||||
mock_client, mock_index = self._make_mock_client(hits=[], total=50)
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
result = search_documents("test", page=3, per_page=10)
|
||||
|
||||
call_kwargs = mock_index.search.call_args
|
||||
search_params = call_kwargs[0][1]
|
||||
assert search_params["offset"] == 20 # (3-1) * 10
|
||||
assert search_params["limit"] == 10
|
||||
assert result["pages"] == 5
|
||||
|
||||
def test_search_empty_results(self):
|
||||
"""search_documents returns proper empty structure."""
|
||||
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
result = search_documents("nothing")
|
||||
|
||||
assert result["results"] == []
|
||||
assert result["total"] == 0
|
||||
assert result["pages"] == 0
|
||||
|
||||
def test_search_exception_returns_empty(self):
|
||||
"""search_documents returns empty dict on Meilisearch exception."""
|
||||
mock_client = MagicMock()
|
||||
mock_index = MagicMock()
|
||||
mock_client.get_index.side_effect = RuntimeError("Meilisearch unavailable")
|
||||
|
||||
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
result = search_documents("invoice")
|
||||
|
||||
assert result["results"] == []
|
||||
assert result["total"] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search API endpoint tests (GET /api/search)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSearchAPIEndpoint:
|
||||
"""Tests for GET /api/search endpoint."""
|
||||
|
||||
def test_search_endpoint_success(self, client):
|
||||
"""GET /api/search?q=... returns search results."""
|
||||
mock_result = {
|
||||
"results": [{"file_id": 1, "document_title": "Test Invoice", "document_type": "Invoice"}],
|
||||
"total": 1,
|
||||
"page": 1,
|
||||
"pages": 1,
|
||||
"query": "invoice",
|
||||
}
|
||||
with patch("app.api.search.search_documents", return_value=mock_result):
|
||||
response = client.get("/api/search?q=invoice")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] == 1
|
||||
assert data["query"] == "invoice"
|
||||
assert len(data["results"]) == 1
|
||||
|
||||
def test_search_endpoint_missing_query(self, client):
|
||||
"""GET /api/search without q returns 422."""
|
||||
response = client.get("/api/search")
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_search_endpoint_empty_query(self, client):
|
||||
"""GET /api/search?q= (empty) returns 422 due to min_length=1."""
|
||||
response = client.get("/api/search?q=")
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_search_endpoint_with_filters(self, client):
|
||||
"""GET /api/search with optional filters passes them to search_documents."""
|
||||
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "invoice"}
|
||||
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||
response = client.get("/api/search?q=invoice&mime_type=application/pdf&language=en&page=2&per_page=10")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_search.assert_called_once_with(
|
||||
"invoice",
|
||||
mime_type="application/pdf",
|
||||
document_type=None,
|
||||
language="en",
|
||||
date_from=None,
|
||||
date_to=None,
|
||||
page=2,
|
||||
per_page=10,
|
||||
)
|
||||
|
||||
def test_search_endpoint_per_page_max(self, client):
|
||||
"""GET /api/search with per_page > 100 returns 422."""
|
||||
response = client.get("/api/search?q=test&per_page=200")
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_search_endpoint_pagination_defaults(self, client):
|
||||
"""GET /api/search uses default page=1 per_page=20."""
|
||||
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"}
|
||||
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||
response = client.get("/api/search?q=test")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_search.assert_called_once_with(
|
||||
"test",
|
||||
mime_type=None,
|
||||
document_type=None,
|
||||
language=None,
|
||||
date_from=None,
|
||||
date_to=None,
|
||||
page=1,
|
||||
per_page=20,
|
||||
)
|
||||
|
||||
def test_search_endpoint_date_filters(self, client):
|
||||
"""GET /api/search with date_from and date_to passes them as int."""
|
||||
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "contract"}
|
||||
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||
response = client.get("/api/search?q=contract&date_from=1704067200&date_to=1735689600")
|
||||
|
||||
assert response.status_code == 200
|
||||
call_kwargs = mock_search.call_args
|
||||
assert call_kwargs[1]["date_from"] == 1704067200
|
||||
assert call_kwargs[1]["date_to"] == 1735689600
|
||||
Reference in New Issue
Block a user