Merge pull request #406 from christianlouis/copilot/add-full-text-search-feature
feat: full-text search (Meilisearch) + Helm chart for Kubernetes deployment
This commit is contained in:
@@ -299,3 +299,14 @@ NOTIFY_ON_FILE_PROCESSED=True
|
|||||||
# Uptime Kuma
|
# Uptime Kuma
|
||||||
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
||||||
UPTIME_KUMA_PING_INTERVAL=5
|
UPTIME_KUMA_PING_INTERVAL=5
|
||||||
|
|
||||||
|
# **Full-Text Search (Meilisearch)**
|
||||||
|
# URL for the Meilisearch instance.
|
||||||
|
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
|
||||||
|
# so container-to-container networking works without extra configuration.
|
||||||
|
# Override to "http://localhost:7700" only when running the API process outside Docker.
|
||||||
|
MEILISEARCH_URL=http://meilisearch:7700
|
||||||
|
# Optional master/API key for secured Meilisearch instances
|
||||||
|
# MEILISEARCH_API_KEY=your_master_key_here
|
||||||
|
MEILISEARCH_INDEX_NAME=documents
|
||||||
|
ENABLE_SEARCH=True
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from app.api.logs import router as logs_router
|
|||||||
from app.api.onedrive import router as onedrive_router
|
from app.api.onedrive import router as onedrive_router
|
||||||
from app.api.openai import router as openai_router
|
from app.api.openai import router as openai_router
|
||||||
from app.api.process import router as process_router
|
from app.api.process import router as process_router
|
||||||
|
from app.api.search import router as search_router
|
||||||
from app.api.settings import router as settings_router
|
from app.api.settings import router as settings_router
|
||||||
from app.api.url_upload import router as url_upload_router
|
from app.api.url_upload import router as url_upload_router
|
||||||
|
|
||||||
@@ -40,3 +41,4 @@ router.include_router(google_drive_router)
|
|||||||
router.include_router(logs_router)
|
router.include_router(logs_router)
|
||||||
router.include_router(settings_router)
|
router.include_router(settings_router)
|
||||||
router.include_router(url_upload_router)
|
router.include_router(url_upload_router)
|
||||||
|
router.include_router(search_router)
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
"""Full-text search API endpoints.
|
||||||
|
|
||||||
|
Provides document search across OCR text, AI metadata, filenames, and tags
|
||||||
|
via Meilisearch. Designed to serve as the backend for the UI search bar on
|
||||||
|
the /files page and as a standalone API for integrations.
|
||||||
|
|
||||||
|
Future extension point: the OCR text stored in the index is also suitable
|
||||||
|
for RAG (Retrieval Augmented Generation) chatbot workflows.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Query, Request
|
||||||
|
|
||||||
|
from app.auth import require_login
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/search")
|
||||||
|
@require_login
|
||||||
|
def search_api(
|
||||||
|
request: Request,
|
||||||
|
q: str = Query(..., min_length=1, max_length=512, description="Full-text search query"),
|
||||||
|
mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"),
|
||||||
|
document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"),
|
||||||
|
language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"),
|
||||||
|
date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"),
|
||||||
|
date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"),
|
||||||
|
page: int = Query(1, ge=1, description="Page number (1-based)"),
|
||||||
|
per_page: int = Query(20, ge=1, le=100, description="Results per page"),
|
||||||
|
):
|
||||||
|
"""Search documents by full text, metadata, and tags.
|
||||||
|
|
||||||
|
Searches across:
|
||||||
|
- Document title and filename
|
||||||
|
- OCR / extracted text
|
||||||
|
- Tags, sender, recipient, document type
|
||||||
|
- Correspondent and reference number
|
||||||
|
|
||||||
|
Results are ranked by Meilisearch relevance and include highlighted
|
||||||
|
snippets showing where the query terms matched.
|
||||||
|
|
||||||
|
Query Parameters:
|
||||||
|
- q: Search query (required)
|
||||||
|
- mime_type: Filter by MIME type
|
||||||
|
- document_type: Filter by document type
|
||||||
|
- language: Filter by language code
|
||||||
|
- date_from: Unix timestamp lower bound
|
||||||
|
- date_to: Unix timestamp upper bound
|
||||||
|
- page: Page number (default: 1)
|
||||||
|
- per_page: Results per page (default: 20, max: 100)
|
||||||
|
|
||||||
|
Example:
|
||||||
|
```
|
||||||
|
GET /api/search?q=invoice&document_type=Invoice&date_from=1704067200&page=1&per_page=20
|
||||||
|
```
|
||||||
|
|
||||||
|
Response:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"file_id": 42,
|
||||||
|
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||||
|
"document_title": "Amazon Invoice January 2026",
|
||||||
|
"document_type": "Invoice",
|
||||||
|
"tags": ["amazon", "invoice"],
|
||||||
|
"_formatted": {
|
||||||
|
"document_title": "Amazon <mark>Invoice</mark> January 2026",
|
||||||
|
"ocr_text": "...total amount of the <mark>invoice</mark> is..."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"total": 42,
|
||||||
|
"page": 1,
|
||||||
|
"pages": 3,
|
||||||
|
"query": "invoice"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
logger.info(f"Search request: q={q!r}, mime_type={mime_type}, page={page}, per_page={per_page}")
|
||||||
|
|
||||||
|
result = search_documents(
|
||||||
|
q,
|
||||||
|
mime_type=mime_type,
|
||||||
|
document_type=document_type,
|
||||||
|
language=language,
|
||||||
|
date_from=date_from,
|
||||||
|
date_to=date_to,
|
||||||
|
page=page,
|
||||||
|
per_page=per_page,
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
@@ -207,6 +207,15 @@ class Settings(BaseSettings):
|
|||||||
uptime_kuma_url: Optional[str] = None
|
uptime_kuma_url: Optional[str] = None
|
||||||
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
|
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
|
||||||
|
|
||||||
|
# Meilisearch settings (full-text search engine)
|
||||||
|
# Default uses the Docker Compose / K8s service name so container-to-container
|
||||||
|
# networking works without any extra configuration. Override to
|
||||||
|
# "http://localhost:7700" only when running the API process outside of Docker.
|
||||||
|
meilisearch_url: str = "http://meilisearch:7700"
|
||||||
|
meilisearch_api_key: Optional[str] = None # Master or API key (optional for local dev)
|
||||||
|
meilisearch_index_name: str = "documents"
|
||||||
|
enable_search: bool = True # Enable Meilisearch full-text search integration
|
||||||
|
|
||||||
# HTTP request settings
|
# HTTP request settings
|
||||||
http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
|
http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
|
||||||
|
|
||||||
|
|||||||
@@ -55,6 +55,15 @@ class FileRecord(Base):
|
|||||||
# If this is a duplicate, record the ID of the original file for reference
|
# If this is a duplicate, record the ID of the original file for reference
|
||||||
duplicate_of_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)
|
duplicate_of_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)
|
||||||
|
|
||||||
|
# Full OCR/extracted text for full-text search and RAG
|
||||||
|
ocr_text = Column(Text, nullable=True)
|
||||||
|
|
||||||
|
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
|
||||||
|
ai_metadata = Column(Text, nullable=True)
|
||||||
|
|
||||||
|
# Human-readable document title from AI metadata
|
||||||
|
document_title = Column(String, nullable=True)
|
||||||
|
|
||||||
# Timestamp when we inserted this record
|
# Timestamp when we inserted this record
|
||||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|||||||
@@ -195,8 +195,26 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
|||||||
original_file_path = file_record.original_file_path
|
original_file_path = file_record.original_file_path
|
||||||
# Update the processed_file_path in the database
|
# Update the processed_file_path in the database
|
||||||
file_record.processed_file_path = final_file_path
|
file_record.processed_file_path = final_file_path
|
||||||
|
# Persist extracted text and AI metadata to DB for full-text search / RAG
|
||||||
|
file_record.ocr_text = extracted_text or None
|
||||||
|
if metadata:
|
||||||
|
try:
|
||||||
|
file_record.ai_metadata = json.dumps(metadata, ensure_ascii=False)
|
||||||
|
except Exception as json_exc:
|
||||||
|
logger.warning(f"[{task_id}] Could not serialise ai_metadata: {json_exc}")
|
||||||
|
file_record.document_title = (
|
||||||
|
metadata.get("title") or metadata.get("filename") or file_record.original_filename
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
logger.info(f"[{task_id}] Updated database with processed_file_path: {final_file_path}")
|
logger.info(f"[{task_id}] Updated database with processed_file_path and search fields")
|
||||||
|
|
||||||
|
# Index into Meilisearch for full-text search (non-blocking, best-effort)
|
||||||
|
try:
|
||||||
|
from app.utils.meilisearch_client import index_document
|
||||||
|
|
||||||
|
index_document(file_record, extracted_text or "", metadata or {})
|
||||||
|
except Exception as search_exc:
|
||||||
|
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
|
||||||
|
|
||||||
# Persist the metadata into a JSON file with the same base name.
|
# Persist the metadata into a JSON file with the same base name.
|
||||||
# Include file path references for traceability
|
# Include file path references for traceability
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
"""Meilisearch client utilities for full-text document search.
|
||||||
|
|
||||||
|
This module provides functions for indexing documents into Meilisearch
|
||||||
|
and searching across OCR text, AI metadata, filenames, and tags.
|
||||||
|
|
||||||
|
The search index is designed to support future RAG (Retrieval Augmented
|
||||||
|
Generation) workflows by storing full document text alongside structured
|
||||||
|
metadata fields.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Index settings applied once at index creation / first use
|
||||||
|
_INDEX_SETTINGS = {
|
||||||
|
"searchableAttributes": [
|
||||||
|
"document_title",
|
||||||
|
"original_filename",
|
||||||
|
"ocr_text",
|
||||||
|
"tags",
|
||||||
|
"sender",
|
||||||
|
"recipient",
|
||||||
|
"document_type",
|
||||||
|
"correspondent",
|
||||||
|
],
|
||||||
|
"filterableAttributes": [
|
||||||
|
"mime_type",
|
||||||
|
"document_type",
|
||||||
|
"language",
|
||||||
|
"tags",
|
||||||
|
"created_at_ts",
|
||||||
|
"file_id",
|
||||||
|
],
|
||||||
|
"sortableAttributes": [
|
||||||
|
"created_at_ts",
|
||||||
|
"file_size",
|
||||||
|
],
|
||||||
|
"displayedAttributes": [
|
||||||
|
"file_id",
|
||||||
|
"original_filename",
|
||||||
|
"document_title",
|
||||||
|
"document_type",
|
||||||
|
"tags",
|
||||||
|
"sender",
|
||||||
|
"recipient",
|
||||||
|
"correspondent",
|
||||||
|
"language",
|
||||||
|
"reference_number",
|
||||||
|
"mime_type",
|
||||||
|
"file_size",
|
||||||
|
"created_at_ts",
|
||||||
|
"ocr_text",
|
||||||
|
],
|
||||||
|
"rankingRules": [
|
||||||
|
"words",
|
||||||
|
"typo",
|
||||||
|
"proximity",
|
||||||
|
"attribute",
|
||||||
|
"sort",
|
||||||
|
"exactness",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_meilisearch_client():
|
||||||
|
"""Return a configured Meilisearch client, or None if unavailable/disabled."""
|
||||||
|
try:
|
||||||
|
import meilisearch
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
if not settings.enable_search:
|
||||||
|
return None
|
||||||
|
|
||||||
|
kwargs: dict[str, Any] = {"url": settings.meilisearch_url}
|
||||||
|
if settings.meilisearch_api_key:
|
||||||
|
kwargs["api_key"] = settings.meilisearch_api_key
|
||||||
|
|
||||||
|
client = meilisearch.Client(**kwargs)
|
||||||
|
return client
|
||||||
|
except ImportError:
|
||||||
|
logger.warning("meilisearch package not installed; search disabled")
|
||||||
|
return None
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"Could not connect to Meilisearch: {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_or_create_index(client):
|
||||||
|
"""Get the documents index, creating it with settings if it doesn't exist."""
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
index_name = settings.meilisearch_index_name
|
||||||
|
try:
|
||||||
|
index = client.get_index(index_name)
|
||||||
|
except Exception:
|
||||||
|
# Index doesn't exist – create it with file_id as primary key
|
||||||
|
task = client.create_index(index_name, {"primaryKey": "file_id"})
|
||||||
|
client.wait_for_task(task.task_uid)
|
||||||
|
index = client.get_index(index_name)
|
||||||
|
# Apply search settings
|
||||||
|
try:
|
||||||
|
task = index.update_settings(_INDEX_SETTINGS)
|
||||||
|
client.wait_for_task(task.task_uid)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"Could not update Meilisearch index settings: {exc}")
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def _build_document(file_record, text: str, metadata: dict) -> dict:
|
||||||
|
"""Build a Meilisearch document from a FileRecord and extracted content."""
|
||||||
|
|
||||||
|
tags = metadata.get("tags", [])
|
||||||
|
if isinstance(tags, str):
|
||||||
|
tags = [t.strip() for t in tags.split(",") if t.strip()]
|
||||||
|
|
||||||
|
# Unix timestamp for sorting/filtering
|
||||||
|
created_at_ts = 0
|
||||||
|
if file_record.created_at:
|
||||||
|
try:
|
||||||
|
created_at_ts = int(file_record.created_at.timestamp())
|
||||||
|
except Exception as ts_exc: # noqa: BLE001
|
||||||
|
logger.debug(f"Could not convert created_at to timestamp: {ts_exc}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"file_id": file_record.id,
|
||||||
|
"original_filename": file_record.original_filename or "",
|
||||||
|
"document_title": metadata.get("title") or metadata.get("filename") or file_record.original_filename or "",
|
||||||
|
"document_type": metadata.get("document_type") or metadata.get("kommunikationsart") or "",
|
||||||
|
"tags": tags,
|
||||||
|
"sender": metadata.get("absender") or "",
|
||||||
|
"recipient": metadata.get("empfaenger") or "",
|
||||||
|
"correspondent": metadata.get("correspondent") or "",
|
||||||
|
"language": metadata.get("language") or "",
|
||||||
|
"reference_number": metadata.get("reference_number") or "",
|
||||||
|
"mime_type": file_record.mime_type or "",
|
||||||
|
"file_size": file_record.file_size or 0,
|
||||||
|
"created_at_ts": created_at_ts,
|
||||||
|
"ocr_text": text or "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def index_document(file_record, text: str, metadata: dict) -> bool:
|
||||||
|
"""Index a document in Meilisearch.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file_record: FileRecord ORM instance with at minimum .id set.
|
||||||
|
text: Full OCR / extracted text for the document.
|
||||||
|
metadata: AI-extracted metadata dict.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if indexing succeeded, False otherwise.
|
||||||
|
"""
|
||||||
|
client = get_meilisearch_client()
|
||||||
|
if client is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
index = _get_or_create_index(client)
|
||||||
|
doc = _build_document(file_record, text, metadata)
|
||||||
|
task = index.add_documents([doc])
|
||||||
|
logger.info(f"Queued Meilisearch indexing for file_id={file_record.id} (task_uid={task.task_uid})")
|
||||||
|
return True
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"Meilisearch indexing failed for file_id={file_record.id}: {exc}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def delete_document(file_id: int) -> bool:
|
||||||
|
"""Remove a document from the Meilisearch index.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file_id: The database ID of the file to remove.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if deletion succeeded, False otherwise.
|
||||||
|
"""
|
||||||
|
client = get_meilisearch_client()
|
||||||
|
if client is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
index = client.get_index(settings.meilisearch_index_name)
|
||||||
|
task = index.delete_document(file_id)
|
||||||
|
logger.info(f"Queued Meilisearch deletion for file_id={file_id} (task_uid={task.task_uid})")
|
||||||
|
return True
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"Meilisearch deletion failed for file_id={file_id}: {exc}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def search_documents(
|
||||||
|
query: str,
|
||||||
|
*,
|
||||||
|
mime_type: Optional[str] = None,
|
||||||
|
document_type: Optional[str] = None,
|
||||||
|
language: Optional[str] = None,
|
||||||
|
date_from: Optional[int] = None,
|
||||||
|
date_to: Optional[int] = None,
|
||||||
|
page: int = 1,
|
||||||
|
per_page: int = 20,
|
||||||
|
) -> dict:
|
||||||
|
"""Search documents in Meilisearch.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Full-text search query string.
|
||||||
|
mime_type: Optional MIME-type filter.
|
||||||
|
document_type: Optional document type filter.
|
||||||
|
language: Optional language filter (ISO 639-1, e.g. "de").
|
||||||
|
date_from: Optional lower bound Unix timestamp for created_at.
|
||||||
|
date_to: Optional upper bound Unix timestamp for created_at.
|
||||||
|
page: 1-based page number.
|
||||||
|
per_page: Results per page (max 100).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with keys: results, total, page, pages, query.
|
||||||
|
Returns empty results dict on any error.
|
||||||
|
"""
|
||||||
|
empty: dict = {"results": [], "total": 0, "page": page, "pages": 0, "query": query}
|
||||||
|
|
||||||
|
client = get_meilisearch_client()
|
||||||
|
if client is None:
|
||||||
|
return empty
|
||||||
|
|
||||||
|
try:
|
||||||
|
index = _get_or_create_index(client)
|
||||||
|
|
||||||
|
# Build filter expressions
|
||||||
|
filters: list[str] = []
|
||||||
|
if mime_type:
|
||||||
|
filters.append(f'mime_type = "{mime_type}"')
|
||||||
|
if document_type:
|
||||||
|
filters.append(f'document_type = "{document_type}"')
|
||||||
|
if language:
|
||||||
|
filters.append(f'language = "{language}"')
|
||||||
|
if date_from is not None:
|
||||||
|
filters.append(f"created_at_ts >= {date_from}")
|
||||||
|
if date_to is not None:
|
||||||
|
filters.append(f"created_at_ts <= {date_to}")
|
||||||
|
|
||||||
|
search_params: dict[str, Any] = {
|
||||||
|
"offset": (page - 1) * per_page,
|
||||||
|
"limit": per_page,
|
||||||
|
"attributesToHighlight": ["document_title", "original_filename", "ocr_text", "tags"],
|
||||||
|
"highlightPreTag": "<mark>",
|
||||||
|
"highlightPostTag": "</mark>",
|
||||||
|
"attributesToCrop": ["ocr_text"],
|
||||||
|
"cropLength": 200,
|
||||||
|
}
|
||||||
|
|
||||||
|
if filters:
|
||||||
|
search_params["filter"] = " AND ".join(filters)
|
||||||
|
|
||||||
|
result = index.search(query, search_params)
|
||||||
|
|
||||||
|
hits = result.get("hits", [])
|
||||||
|
total = result.get("estimatedTotalHits", result.get("nbHits", len(hits)))
|
||||||
|
|
||||||
|
# Attach highlights to each hit
|
||||||
|
formatted_results = []
|
||||||
|
for hit in hits:
|
||||||
|
formatted = dict(hit)
|
||||||
|
# Include formatted (highlighted) snippets if available
|
||||||
|
if "_formatted" in hit:
|
||||||
|
formatted["_formatted"] = hit["_formatted"]
|
||||||
|
# Exclude raw ocr_text from results (use _formatted snippet instead)
|
||||||
|
formatted.pop("ocr_text", None)
|
||||||
|
formatted_results.append(formatted)
|
||||||
|
|
||||||
|
pages = (total + per_page - 1) // per_page if total > 0 else 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"results": formatted_results,
|
||||||
|
"total": total,
|
||||||
|
"page": page,
|
||||||
|
"pages": pages,
|
||||||
|
"query": query,
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"Meilisearch search failed for query '{query}': {exc}")
|
||||||
|
return empty
|
||||||
@@ -59,6 +59,15 @@ services:
|
|||||||
container_name: gotenberg
|
container_name: gotenberg
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
|
meilisearch:
|
||||||
|
image: getmeili/meilisearch:latest
|
||||||
|
container_name: document_meilisearch
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
- MEILI_NO_ANALYTICS=true
|
||||||
|
volumes:
|
||||||
|
- /var/docparse/meilisearch:/meili_data
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:alpine
|
image: redis:alpine
|
||||||
container_name: document_redis
|
container_name: document_redis
|
||||||
|
|||||||
+310
-151
@@ -1,40 +1,48 @@
|
|||||||
# Deployment Guide
|
# Deployment Guide
|
||||||
|
|
||||||
This guide provides instructions for deploying DocuElevate in various environments.
|
This guide covers all supported deployment methods for DocuElevate.
|
||||||
|
|
||||||
|
## Table of Contents
|
||||||
|
|
||||||
|
- [Prerequisites](#prerequisites)
|
||||||
|
- [Docker Compose Deployment](#docker-compose-deployment) *(recommended for single-server)*
|
||||||
|
- [Kubernetes / Helm Deployment](#kubernetes--helm-deployment) *(recommended for production scale-out)*
|
||||||
|
- [Production Considerations](#production-considerations)
|
||||||
|
- [Scaling](#scaling)
|
||||||
|
- [Backup Procedures](#backup-procedures)
|
||||||
|
- [Updates](#updates)
|
||||||
|
- [Troubleshooting](#troubleshooting)
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- Docker and Docker Compose
|
- Docker and Docker Compose **or** a Kubernetes cluster with Helm 3
|
||||||
- Access to required external services (if configured):
|
- Access to required external services (if configured):
|
||||||
- AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider)
|
- AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider)
|
||||||
- Azure Document Intelligence
|
- Azure Document Intelligence
|
||||||
- Dropbox API
|
- Dropbox, Google Drive, OneDrive, S3, or other storage APIs
|
||||||
- Nextcloud instance
|
- SMTP / IMAP server (for email processing)
|
||||||
- Paperless NGX instance
|
- Notification services (Discord, Telegram, etc.)
|
||||||
- SMTP server (for email notifications)
|
|
||||||
- IMAP server(s) (for email attachment processing)
|
|
||||||
- Notification services (Discord, Telegram, etc. for system alerts)
|
|
||||||
|
|
||||||
## Docker Deployment
|
---
|
||||||
|
|
||||||
Docker is the recommended deployment method for DocuElevate.
|
## Docker Compose Deployment
|
||||||
|
|
||||||
|
Docker Compose is the quickest way to run DocuElevate on a single server.
|
||||||
|
|
||||||
### Step 1: Clone the Repository
|
### Step 1: Clone the Repository
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/christianlouis/document-processor.git
|
git clone https://github.com/christianlouis/DocuElevate.git
|
||||||
cd document-processor
|
cd DocuElevate
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 2: Configure Environment Variables
|
### Step 2: Configure Environment Variables
|
||||||
|
|
||||||
Create a `.env` file based on the example:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp .env.example .env
|
cp .env.demo .env
|
||||||
```
|
```
|
||||||
|
|
||||||
Edit the `.env` file with your configuration settings. See the [Configuration Guide](ConfigurationGuide.md) for details.
|
Edit `.env` with your settings. See the [Configuration Guide](ConfigurationGuide.md) for all options.
|
||||||
|
|
||||||
### Step 3: Run with Docker Compose
|
### Step 3: Run with Docker Compose
|
||||||
|
|
||||||
@@ -42,41 +50,237 @@ Edit the `.env` file with your configuration settings. See the [Configuration Gu
|
|||||||
docker-compose up -d
|
docker-compose up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
This will start:
|
This starts:
|
||||||
- The DocuElevate API server
|
|
||||||
- A worker for background tasks
|
| Service | Purpose |
|
||||||
- Redis for message broker and result storage
|
|---------|---------|
|
||||||
- Gotenberg for PDF processing
|
| `api` | FastAPI web server (port 8000) |
|
||||||
|
| `worker` | Celery background task worker |
|
||||||
|
| `redis` | Message broker for Celery |
|
||||||
|
| `gotenberg` | PDF conversion (LibreOffice headless) |
|
||||||
|
| `meilisearch` | Full-text search engine (port 7700) |
|
||||||
|
|
||||||
### Step 4: Verify the Installation
|
### Step 4: Verify the Installation
|
||||||
|
|
||||||
Access the web interface at `http://localhost:8000` and the API documentation at `http://localhost:8000/docs`.
|
Access the web interface at `http://localhost:8000` and the API docs at `http://localhost:8000/docs`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Kubernetes / Helm Deployment
|
||||||
|
|
||||||
|
The Helm chart at `helm/docuelevate/` packages all components into a single, configurable release. It supports:
|
||||||
|
|
||||||
|
- Multiple replicas for the API and Worker
|
||||||
|
- Horizontal Pod Autoscaling (HPA)
|
||||||
|
- Bundled or external Redis
|
||||||
|
- Persistent volumes for workdir and Meilisearch data
|
||||||
|
- Alembic database migration Job (pre-install/upgrade hook)
|
||||||
|
- TLS Ingress via any controller (nginx, Traefik, etc.)
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
- Kubernetes 1.24+
|
||||||
|
- Helm 3.10+
|
||||||
|
- A storage class that supports **ReadWriteMany** (e.g. NFS, CephFS, Azure Files, EFS) for the shared workdir PVC when running multiple replicas. Single-replica clusters can use `ReadWriteOnce`.
|
||||||
|
- A PostgreSQL database (strongly recommended over SQLite for multi-replica).
|
||||||
|
|
||||||
|
### Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Add the Bitnami chart repository (needed for bundled Redis)
|
||||||
|
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||||
|
helm repo update
|
||||||
|
|
||||||
|
# 2. Update chart dependencies
|
||||||
|
helm dependency update ./helm/docuelevate
|
||||||
|
|
||||||
|
# 3. Install with a minimal values override
|
||||||
|
helm install docuelevate ./helm/docuelevate \
|
||||||
|
--namespace docuelevate --create-namespace \
|
||||||
|
--set secrets.DATABASE_URL="postgresql://user:pass@postgres:5432/docuelevate" \
|
||||||
|
--set secrets.SESSION_SECRET="$(openssl rand -hex 32)" \
|
||||||
|
--set secrets.OPENAI_API_KEY="sk-..." \
|
||||||
|
--set secrets.AZURE_AI_KEY="..." \
|
||||||
|
--set env.AZURE_ENDPOINT="https://my-resource.cognitiveservices.azure.com/" \
|
||||||
|
--set env.EXTERNAL_HOSTNAME="docuelevate.example.com"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Values Reference
|
||||||
|
|
||||||
|
The full list of configurable values is in [`helm/docuelevate/values.yaml`](../helm/docuelevate/values.yaml). Key sections:
|
||||||
|
|
||||||
|
#### Container Image
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
image:
|
||||||
|
repository: ghcr.io/christianlouis/docuelevate
|
||||||
|
tag: "" # defaults to chart appVersion
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Non-Secret Config (`env`)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
env:
|
||||||
|
WORKDIR: /workdir
|
||||||
|
AI_PROVIDER: openai
|
||||||
|
OPENAI_MODEL: gpt-4o-mini
|
||||||
|
AZURE_REGION: eastus
|
||||||
|
AZURE_ENDPOINT: "https://my-resource.cognitiveservices.azure.com/"
|
||||||
|
MEILISEARCH_URL: http://docuelevate-meilisearch:7700 # auto-resolved from service name
|
||||||
|
ENABLE_SEARCH: "true"
|
||||||
|
AUTH_ENABLED: "true"
|
||||||
|
EXTERNAL_HOSTNAME: docuelevate.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Secrets (`secrets`)
|
||||||
|
|
||||||
|
All secrets are stored in a Kubernetes `Secret` and injected as environment variables.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
secrets:
|
||||||
|
DATABASE_URL: "postgresql://user:pass@postgres:5432/docuelevate"
|
||||||
|
SESSION_SECRET: "<min-32-char-random-string>"
|
||||||
|
OPENAI_API_KEY: "sk-..."
|
||||||
|
AZURE_AI_KEY: "..."
|
||||||
|
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated dev Meilisearch
|
||||||
|
# Storage provider secrets ...
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Tip:** In production use an external secret manager (Vault, ESO, Sealed Secrets) and reference the secret by name instead of embedding values in values.yaml.
|
||||||
|
|
||||||
|
#### Replicas & Autoscaling
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
api:
|
||||||
|
replicaCount: 2
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 8
|
||||||
|
targetCPUUtilizationPercentage: 70
|
||||||
|
|
||||||
|
worker:
|
||||||
|
replicaCount: 2
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 10
|
||||||
|
targetCPUUtilizationPercentage: 75
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Shared Workdir PVC
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
workdir:
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
accessMode: ReadWriteMany # RWX required for multi-replica
|
||||||
|
size: 20Gi
|
||||||
|
storageClass: "nfs-client" # or leave blank for cluster default
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Ingress (nginx example)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
className: nginx
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/proxy-body-size: "1g"
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||||
|
hosts:
|
||||||
|
- host: docuelevate.example.com
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
tls:
|
||||||
|
- secretName: docuelevate-tls
|
||||||
|
hosts:
|
||||||
|
- docuelevate.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
#### External Redis
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
redis:
|
||||||
|
enabled: false # disable bundled Redis
|
||||||
|
|
||||||
|
externalRedis:
|
||||||
|
url: "redis://my-redis-host:6379/0"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Meilisearch
|
||||||
|
|
||||||
|
The bundled Meilisearch deployment is a single-replica, persistent StatefulSet-equivalent. For production, consider [Meilisearch Cloud](https://www.meilisearch.com/cloud) and point `env.MEILISEARCH_URL` at it.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
meilisearch:
|
||||||
|
enabled: true
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
size: 10Gi
|
||||||
|
```
|
||||||
|
|
||||||
|
### Upgrading
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm upgrade docuelevate ./helm/docuelevate \
|
||||||
|
--namespace docuelevate \
|
||||||
|
-f my-values.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
The pre-upgrade hook runs `alembic upgrade head` automatically before the new pods start.
|
||||||
|
|
||||||
|
### Uninstalling
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm uninstall docuelevate --namespace docuelevate
|
||||||
|
# PVCs are NOT deleted automatically — remove manually if desired:
|
||||||
|
kubectl delete pvc -l app.kubernetes.io/instance=docuelevate -n docuelevate
|
||||||
|
```
|
||||||
|
|
||||||
|
### Kubernetes Architecture Diagram
|
||||||
|
|
||||||
|
```
|
||||||
|
Internet
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
[Ingress / LoadBalancer]
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
[API Deployment] ─────── [Worker Deployment]
|
||||||
|
│ │ │ │
|
||||||
|
│ └── shared PVC ──┘ │
|
||||||
|
│ (workdir) │
|
||||||
|
▼ ▼
|
||||||
|
[Redis Service] [Gotenberg Service]
|
||||||
|
│
|
||||||
|
[Meilisearch Service]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Production Considerations
|
## Production Considerations
|
||||||
|
|
||||||
### Security Headers
|
### Database
|
||||||
|
|
||||||
DocuElevate includes built-in support for HTTP security headers to improve browser-side security. **These headers are disabled by default** since most deployments use a reverse proxy (Traefik, Nginx, etc.) that already adds these headers.
|
SQLite is fine for development but **not recommended for multi-replica production** deployments because it cannot be shared safely across pods. Use PostgreSQL:
|
||||||
|
|
||||||
#### Supported Security Headers
|
```
|
||||||
|
DATABASE_URL=postgresql://docuelevate:secret@postgres-host:5432/docuelevate
|
||||||
- **Strict-Transport-Security (HSTS)**: Forces browsers to use HTTPS for all future requests
|
|
||||||
- **Content-Security-Policy (CSP)**: Controls which resources browsers are allowed to load
|
|
||||||
- **X-Frame-Options**: Prevents the page from being loaded in frames (clickjacking protection)
|
|
||||||
- **X-Content-Type-Options**: Prevents browsers from MIME-sniffing responses
|
|
||||||
|
|
||||||
#### Reverse Proxy Deployment (Traefik, Nginx, etc.) - DEFAULT
|
|
||||||
|
|
||||||
**Most deployments use a reverse proxy**, which is why security headers are disabled by default in DocuElevate. The reverse proxy should add these headers.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In .env file (or omit - this is the default)
|
|
||||||
SECURITY_HEADERS_ENABLED=false
|
|
||||||
```
|
```
|
||||||
|
|
||||||
##### Traefik Configuration Example
|
### Security Headers
|
||||||
|
|
||||||
Traefik can add security headers using middleware. Create a `docker-compose.yaml` with Traefik labels:
|
DocuElevate's built-in security headers are **disabled by default** since most deployments use a reverse proxy that already adds them.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Enable only if running without a reverse proxy
|
||||||
|
SECURITY_HEADERS_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Traefik (Docker Compose) example
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
@@ -85,35 +289,25 @@ services:
|
|||||||
- "traefik.enable=true"
|
- "traefik.enable=true"
|
||||||
- "traefik.http.routers.docuelevate.rule=Host(`docuelevate.example.com`)"
|
- "traefik.http.routers.docuelevate.rule=Host(`docuelevate.example.com`)"
|
||||||
- "traefik.http.routers.docuelevate.entrypoints=websecure"
|
- "traefik.http.routers.docuelevate.entrypoints=websecure"
|
||||||
- "traefik.http.routers.docuelevate.tls=true"
|
|
||||||
- "traefik.http.routers.docuelevate.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.docuelevate.tls.certresolver=letsencrypt"
|
||||||
# Security headers middleware
|
|
||||||
- "traefik.http.routers.docuelevate.middlewares=security-headers@docker"
|
- "traefik.http.routers.docuelevate.middlewares=security-headers@docker"
|
||||||
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
|
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
|
||||||
- "traefik.http.middlewares.security-headers.headers.stsIncludeSubdomains=true"
|
- "traefik.http.middlewares.security-headers.headers.stsIncludeSubdomains=true"
|
||||||
- "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
|
||||||
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
|
||||||
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
|
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
|
||||||
|
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
||||||
```
|
```
|
||||||
|
|
||||||
Then set `SECURITY_HEADERS_ENABLED=false` in your `.env` file.
|
#### Nginx example
|
||||||
|
|
||||||
##### Nginx Configuration Example
|
|
||||||
|
|
||||||
Add security headers to your Nginx configuration:
|
|
||||||
|
|
||||||
```nginx
|
```nginx
|
||||||
server {
|
server {
|
||||||
listen 443 ssl http2;
|
listen 443 ssl http2;
|
||||||
server_name docuelevate.example.com;
|
server_name docuelevate.example.com;
|
||||||
|
|
||||||
# SSL configuration
|
|
||||||
ssl_certificate /etc/nginx/ssl/cert.pem;
|
ssl_certificate /etc/nginx/ssl/cert.pem;
|
||||||
ssl_certificate_key /etc/nginx/ssl/key.pem;
|
ssl_certificate_key /etc/nginx/ssl/key.pem;
|
||||||
|
|
||||||
# Security headers
|
|
||||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||||
add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;" always;
|
|
||||||
add_header X-Frame-Options "DENY" always;
|
add_header X-Frame-Options "DENY" always;
|
||||||
add_header X-Content-Type-Options "nosniff" always;
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
|
||||||
@@ -123,144 +317,109 @@ server {
|
|||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
client_max_body_size 1g;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Then keep `SECURITY_HEADERS_ENABLED=false` in your `.env` file (or omit it, as this is the default).
|
### Storage
|
||||||
|
|
||||||
#### Direct Deployment (No Reverse Proxy)
|
|
||||||
|
|
||||||
If you're running DocuElevate **directly without a reverse proxy**, enable security headers:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In .env file
|
|
||||||
SECURITY_HEADERS_ENABLED=true
|
|
||||||
```
|
|
||||||
|
|
||||||
You can also configure individual headers:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
SECURITY_HEADER_HSTS_ENABLED=true
|
|
||||||
SECURITY_HEADER_CSP_ENABLED=true
|
|
||||||
SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
|
||||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
|
||||||
```
|
|
||||||
|
|
||||||
**Note**: HSTS only works when serving content over HTTPS. If using HTTP for development, you can disable it:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
SECURITY_HEADER_HSTS_ENABLED=false
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Customizing Security Headers
|
|
||||||
|
|
||||||
If you enable security headers, you can customize individual header values in your `.env` file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Customize HSTS (e.g., shorter duration for testing)
|
|
||||||
SECURITY_HEADER_HSTS_VALUE="max-age=300"
|
|
||||||
|
|
||||||
# Customize CSP (e.g., allow specific external domains)
|
|
||||||
SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' https://cdn.example.com; style-src 'self' 'unsafe-inline';"
|
|
||||||
|
|
||||||
# Allow framing from same origin
|
|
||||||
SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="SAMEORIGIN"
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Security Considerations
|
|
||||||
|
|
||||||
1. **HSTS and HTTPS**: HSTS only works over HTTPS. Ensure you have a valid SSL certificate before enabling HSTS.
|
|
||||||
2. **CSP Testing**: The default CSP policy allows inline scripts and styles for compatibility. Test thoroughly before tightening.
|
|
||||||
3. **Content-Security-Policy**: The default policy allows `'unsafe-inline'` for scripts and styles for compatibility with Tailwind CSS and inline JavaScript. For stricter security, consider using nonces or hashes.
|
|
||||||
4. **X-Frame-Options**: Set to `DENY` by default. Change to `SAMEORIGIN` if you need to embed DocuElevate in iframes on the same domain.
|
|
||||||
|
|
||||||
See the [Configuration Guide](ConfigurationGuide.md) for all security header options.
|
|
||||||
|
|
||||||
### Reverse Proxy Setup
|
|
||||||
|
|
||||||
For production use, we recommend setting up a reverse proxy (like Nginx or Traefik) to handle HTTPS and domain routing:
|
|
||||||
|
|
||||||
```nginx
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name docuelevate.example.com;
|
|
||||||
|
|
||||||
location / {
|
|
||||||
proxy_pass http://localhost:8000;
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Persistent Storage
|
|
||||||
|
|
||||||
The Docker setup uses volumes for persistent storage. For production, consider:
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
# Docker Compose
|
||||||
volumes:
|
volumes:
|
||||||
- /path/to/persistent/storage:/workdir
|
- /path/to/persistent/storage:/workdir
|
||||||
|
|
||||||
|
# Helm — use a RWX storage class for multi-replica
|
||||||
|
workdir:
|
||||||
|
persistence:
|
||||||
|
size: 20Gi
|
||||||
|
accessMode: ReadWriteMany
|
||||||
|
storageClass: "nfs-client"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Security
|
### General Security Checklist
|
||||||
|
|
||||||
1. **Always use HTTPS** in production
|
1. **Always use HTTPS** in production
|
||||||
2. Enable authentication by setting `AUTH_ENABLED=true`
|
2. Set `AUTH_ENABLED=true` and use a strong `SESSION_SECRET`
|
||||||
3. Use strong passwords for all services
|
3. Rotate API keys and secrets regularly — see the [Credential Rotation Guide](CredentialRotationGuide.md)
|
||||||
4. Limit access to the Docker host
|
4. Limit network access to Redis and Meilisearch (both should be internal-only)
|
||||||
5. Regularly update the application and dependencies
|
5. Regularly update the container image to pick up dependency patches
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Scaling
|
## Scaling
|
||||||
|
|
||||||
For high-volume deployments:
|
### Docker Compose
|
||||||
|
|
||||||
1. Increase worker processes by adding more worker containers:
|
Add more worker containers:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
worker:
|
worker:
|
||||||
image: christianlouis/document-processor:latest
|
|
||||||
deploy:
|
deploy:
|
||||||
replicas: 3
|
replicas: 3
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Consider using dedicated Redis and database servers
|
### Kubernetes / Helm
|
||||||
3. Monitor system performance and adjust resources as needed
|
|
||||||
|
Enable HPA:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
api:
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 8
|
||||||
|
|
||||||
|
worker:
|
||||||
|
autoscaling:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 10
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Monitoring
|
## Monitoring
|
||||||
|
|
||||||
Monitor your DocuElevate deployment using:
|
- **Docker Compose**: `docker-compose logs -f`, `docker stats`
|
||||||
|
- **Kubernetes**: `kubectl logs -l app.kubernetes.io/component=api -f`
|
||||||
|
- **Prometheus / Grafana**: Scrape the `/api/health` endpoint for readiness; add custom metrics as needed.
|
||||||
|
- **Uptime Kuma**: Set `UPTIME_KUMA_URL` to your push URL for heartbeat monitoring.
|
||||||
|
|
||||||
- Docker's built-in logging: `docker-compose logs -f`
|
---
|
||||||
- Container metrics: `docker stats`
|
|
||||||
- External monitoring tools like Prometheus and Grafana
|
|
||||||
|
|
||||||
## Backup Procedures
|
## Backup Procedures
|
||||||
|
|
||||||
Regularly back up the following:
|
Regularly back up:
|
||||||
|
|
||||||
1. The `/workdir` directory containing all processed documents
|
1. The `/workdir` volume (all processed documents and originals)
|
||||||
2. The database file (if using SQLite) or database contents (if using another DBMS)
|
2. The database (PostgreSQL `pg_dump` or SQLite file)
|
||||||
3. The `.env` configuration file
|
3. The Meilisearch data directory (`/meili_data`)
|
||||||
|
4. Your `.env` / Helm values file (store securely, it contains secrets)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Updates
|
## Updates
|
||||||
|
|
||||||
To update DocuElevate to a newer version:
|
### Docker Compose
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Pull the latest changes
|
|
||||||
git pull
|
git pull
|
||||||
|
|
||||||
# Pull the latest Docker images
|
|
||||||
docker-compose pull
|
docker-compose pull
|
||||||
|
docker-compose down && docker-compose up -d
|
||||||
# Restart the services
|
|
||||||
docker-compose down
|
|
||||||
docker-compose up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Helm
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm repo update # if using a hosted chart repository
|
||||||
|
helm upgrade docuelevate ./helm/docuelevate --namespace docuelevate -f my-values.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
The migration Job runs automatically on every `helm upgrade`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
See the [Troubleshooting](Troubleshooting.md) guide for common deployment issues and solutions.
|
See the [Troubleshooting Guide](Troubleshooting.md) for common issues and solutions.
|
||||||
|
|||||||
@@ -418,6 +418,47 @@
|
|||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Full-Text Search Section -->
|
||||||
|
<div class="filters-section" style="margin-top: 0.75rem;">
|
||||||
|
<div style="width: 100%;">
|
||||||
|
<label for="fulltext-search" style="font-weight: 600; display: block; margin-bottom: 0.4rem;">
|
||||||
|
<i class="fas fa-search"></i> Full-Text Search (OCR text, metadata, tags)
|
||||||
|
</label>
|
||||||
|
<div style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
id="fulltext-search"
|
||||||
|
placeholder="Search document content, sender, tags, type..."
|
||||||
|
style="flex: 1; min-width: 220px; padding: 0.5rem 0.75rem; border: 1px solid #d1d5db; border-radius: 0.375rem; font-size: 0.875rem;"
|
||||||
|
oninput="debounceSearch(this.value)"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onclick="runFullTextSearch()"
|
||||||
|
style="padding: 0.5rem 1rem; background-color: #3182ce; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
|
||||||
|
>
|
||||||
|
Search
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onclick="clearFullTextSearch()"
|
||||||
|
style="padding: 0.5rem 1rem; background-color: #6b7280; color: white; border: none; border-radius: 0.375rem; cursor: pointer; font-size: 0.875rem; white-space: nowrap;"
|
||||||
|
>
|
||||||
|
Clear
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Full-Text Search Results Panel -->
|
||||||
|
<div id="search-results-panel" style="display: none; margin-top: 0.5rem; border: 1px solid #e5e7eb; border-radius: 0.5rem; background: white; box-shadow: 0 1px 3px rgba(0,0,0,0.08);">
|
||||||
|
<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #e5e7eb; display: flex; justify-content: space-between; align-items: center; background: #f9fafb; border-radius: 0.5rem 0.5rem 0 0;">
|
||||||
|
<span id="search-results-summary" style="font-size: 0.875rem; color: #374151;"></span>
|
||||||
|
<div id="search-results-pagination" style="display: flex; gap: 0.5rem;"></div>
|
||||||
|
</div>
|
||||||
|
<div id="search-results-list" style="padding: 0.5rem 0;"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Bulk Actions Section -->
|
<!-- Bulk Actions Section -->
|
||||||
<div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
|
<div id="bulkActionsBar" class="filters-section" style="display: none; background-color: #e6f3ff;">
|
||||||
<div class="flex flex-col sm:flex-row sm:justify-between sm:items-center gap-3">
|
<div class="flex flex-col sm:flex-row sm:justify-between sm:items-center gap-3">
|
||||||
@@ -887,6 +928,117 @@
|
|||||||
function closeUploadModal() {
|
function closeUploadModal() {
|
||||||
uploadModal.classList.remove('active');
|
uploadModal.classList.remove('active');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- Full-Text Search (Meilisearch) ----
|
||||||
|
let _searchDebounceTimer = null;
|
||||||
|
let _searchCurrentPage = 1;
|
||||||
|
|
||||||
|
function debounceSearch(value) {
|
||||||
|
clearTimeout(_searchDebounceTimer);
|
||||||
|
if (!value || value.trim().length < 2) {
|
||||||
|
clearFullTextSearch();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
_searchDebounceTimer = setTimeout(() => {
|
||||||
|
_searchCurrentPage = 1;
|
||||||
|
runFullTextSearch();
|
||||||
|
}, 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
function runFullTextSearch(page) {
|
||||||
|
const input = document.getElementById('fulltext-search');
|
||||||
|
const q = input ? input.value.trim() : '';
|
||||||
|
if (!q) { clearFullTextSearch(); return; }
|
||||||
|
if (page) _searchCurrentPage = page;
|
||||||
|
|
||||||
|
const panel = document.getElementById('search-results-panel');
|
||||||
|
const list = document.getElementById('search-results-list');
|
||||||
|
const summary = document.getElementById('search-results-summary');
|
||||||
|
const pagination = document.getElementById('search-results-pagination');
|
||||||
|
|
||||||
|
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;"><i class="fas fa-spinner fa-spin"></i> Searching…</div>';
|
||||||
|
summary.textContent = '';
|
||||||
|
pagination.innerHTML = '';
|
||||||
|
panel.style.display = 'block';
|
||||||
|
|
||||||
|
const params = new URLSearchParams({ q, page: _searchCurrentPage, per_page: 20 });
|
||||||
|
fetch(`/api/search?${params.toString()}`)
|
||||||
|
.then(r => {
|
||||||
|
if (!r.ok) throw new Error(`Search returned ${r.status}`);
|
||||||
|
return r.json();
|
||||||
|
})
|
||||||
|
.then(data => renderSearchResults(data, q))
|
||||||
|
.catch(err => {
|
||||||
|
list.innerHTML = `<div style="padding: 1rem; color: #dc2626; font-size: 0.875rem;"><i class="fas fa-exclamation-triangle"></i> Search unavailable: ${err.message}</div>`;
|
||||||
|
summary.textContent = '';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSearchResults(data, q) {
|
||||||
|
const panel = document.getElementById('search-results-panel');
|
||||||
|
const list = document.getElementById('search-results-list');
|
||||||
|
const summary = document.getElementById('search-results-summary');
|
||||||
|
const pagination = document.getElementById('search-results-pagination');
|
||||||
|
|
||||||
|
const { results, total, page, pages } = data;
|
||||||
|
summary.textContent = `${total} result${total !== 1 ? 's' : ''} for "${q}"`;
|
||||||
|
|
||||||
|
if (!results || results.length === 0) {
|
||||||
|
list.innerHTML = '<div style="padding: 1rem; color: #6b7280; font-size: 0.875rem;">No results found.</div>';
|
||||||
|
pagination.innerHTML = '';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
list.innerHTML = results.map(hit => {
|
||||||
|
const fmt = hit._formatted || {};
|
||||||
|
const title = fmt.document_title || hit.document_title || hit.original_filename || '(untitled)';
|
||||||
|
const filename = fmt.original_filename || hit.original_filename || '';
|
||||||
|
const snippet = fmt.ocr_text || '';
|
||||||
|
const tags = Array.isArray(hit.tags) ? hit.tags.join(', ') : (hit.tags || '');
|
||||||
|
const docType = hit.document_type || '';
|
||||||
|
|
||||||
|
return `<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #f3f4f6; display: flex; gap: 0.75rem; align-items: flex-start;">
|
||||||
|
<div style="flex-shrink: 0; color: #3b82f6; font-size: 1.25rem; padding-top: 0.1rem;">
|
||||||
|
<i class="fas fa-file-pdf"></i>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1; min-width: 0;">
|
||||||
|
<div style="font-weight: 600; font-size: 0.9rem; color: #111827;">${title}</div>
|
||||||
|
${filename ? `<div style="font-size: 0.8rem; color: #6b7280; margin-top: 0.15rem;">${filename}</div>` : ''}
|
||||||
|
${docType ? `<span style="display: inline-block; margin-top: 0.25rem; padding: 0.1rem 0.5rem; background: #eff6ff; color: #1d4ed8; border-radius: 9999px; font-size: 0.75rem;">${docType}</span>` : ''}
|
||||||
|
${tags ? `<span style="display: inline-block; margin-top: 0.25rem; margin-left: 0.25rem; padding: 0.1rem 0.5rem; background: #f0fdf4; color: #15803d; border-radius: 9999px; font-size: 0.75rem;">${tags}</span>` : ''}
|
||||||
|
${snippet ? `<div style="margin-top: 0.4rem; font-size: 0.8rem; color: #374151; white-space: pre-wrap; word-break: break-word;">…${snippet}…</div>` : ''}
|
||||||
|
</div>
|
||||||
|
<div style="flex-shrink: 0;">
|
||||||
|
<a href="/files/${hit.file_id}" style="padding: 0.25rem 0.6rem; background: #f3f4f6; color: #374151; border-radius: 0.25rem; font-size: 0.8rem; text-decoration: none; white-space: nowrap;" title="View file">
|
||||||
|
<i class="fas fa-external-link-alt"></i>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>`;
|
||||||
|
}).join('');
|
||||||
|
|
||||||
|
// Pagination
|
||||||
|
if (pages > 1) {
|
||||||
|
const btns = [];
|
||||||
|
if (page > 1) {
|
||||||
|
btns.push(`<button onclick="runFullTextSearch(${page - 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">« Prev</button>`);
|
||||||
|
}
|
||||||
|
btns.push(`<span style="padding: 0.25rem 0.6rem; font-size: 0.8rem; color: #6b7280;">Page ${page} / ${pages}</span>`);
|
||||||
|
if (page < pages) {
|
||||||
|
btns.push(`<button onclick="runFullTextSearch(${page + 1})" style="padding: 0.25rem 0.6rem; border: 1px solid #d1d5db; border-radius: 0.25rem; font-size: 0.8rem; cursor: pointer; background: white;">Next »</button>`);
|
||||||
|
}
|
||||||
|
pagination.innerHTML = btns.join('');
|
||||||
|
} else {
|
||||||
|
pagination.innerHTML = '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearFullTextSearch() {
|
||||||
|
const panel = document.getElementById('search-results-panel');
|
||||||
|
const input = document.getElementById('fulltext-search');
|
||||||
|
if (panel) panel.style.display = 'none';
|
||||||
|
if (input) input.value = '';
|
||||||
|
_searchCurrentPage = 1;
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
</div>
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
.DS_Store
|
||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
.vscode
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: docuelevate
|
||||||
|
description: >
|
||||||
|
DocuElevate — intelligent document processing with OCR, AI metadata
|
||||||
|
extraction, full-text search (Meilisearch), and multi-cloud storage.
|
||||||
|
|
||||||
|
type: application
|
||||||
|
|
||||||
|
# Chart version — bump on every chart change (independent of appVersion).
|
||||||
|
version: 0.1.0
|
||||||
|
|
||||||
|
# Application version — kept in sync with the VERSION file.
|
||||||
|
appVersion: "0.54.0"
|
||||||
|
|
||||||
|
keywords:
|
||||||
|
- document-management
|
||||||
|
- ocr
|
||||||
|
- ai
|
||||||
|
- meilisearch
|
||||||
|
- fastapi
|
||||||
|
- celery
|
||||||
|
|
||||||
|
home: https://github.com/christianlouis/DocuElevate
|
||||||
|
sources:
|
||||||
|
- https://github.com/christianlouis/DocuElevate
|
||||||
|
|
||||||
|
maintainers:
|
||||||
|
- name: DocuElevate Contributors
|
||||||
|
url: https://github.com/christianlouis/DocuElevate
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
# Bundled Redis (optional — disable and point to an external instance via
|
||||||
|
# externalRedis.url if you already have Redis in the cluster).
|
||||||
|
- name: redis
|
||||||
|
version: "20.x.x"
|
||||||
|
repository: "https://charts.bitnami.com/bitnami"
|
||||||
|
condition: redis.enabled
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
Thank you for installing {{ .Chart.Name }} {{ .Chart.Version }}!
|
||||||
|
|
||||||
|
{{- if .Values.ingress.enabled }}
|
||||||
|
The application will be available at:
|
||||||
|
{{- range .Values.ingress.hosts }}
|
||||||
|
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
|
||||||
|
{{- end }}
|
||||||
|
{{- else }}
|
||||||
|
To access DocuElevate, run:
|
||||||
|
|
||||||
|
kubectl port-forward svc/{{ include "docuelevate.fullname" . }}-api {{ .Values.api.service.port }}:{{ .Values.api.service.port }} -n {{ .Release.Namespace }}
|
||||||
|
|
||||||
|
Then open http://localhost:{{ .Values.api.service.port }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
API docs are available at: <your-url>/docs
|
||||||
|
|
||||||
|
{{- if not .Values.secrets.SESSION_SECRET }}
|
||||||
|
|
||||||
|
⚠️ WARNING: secrets.SESSION_SECRET is not set.
|
||||||
|
Generate one with: openssl rand -hex 32
|
||||||
|
Then pass it via: --set secrets.SESSION_SECRET=<value>
|
||||||
|
or via a values file / external secret.
|
||||||
|
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{- if not .Values.secrets.DATABASE_URL }}
|
||||||
|
|
||||||
|
⚠️ WARNING: secrets.DATABASE_URL is not set.
|
||||||
|
For production, use PostgreSQL:
|
||||||
|
postgresql://user:pass@host:5432/docuelevate
|
||||||
|
SQLite with a PVC is supported but not recommended for multi-replica
|
||||||
|
deployments (use ReadWriteOnce PVC and replicaCount=1 in that case).
|
||||||
|
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
Useful commands:
|
||||||
|
# View API logs
|
||||||
|
kubectl logs -l app.kubernetes.io/component=api -n {{ .Release.Namespace }} -f
|
||||||
|
|
||||||
|
# View worker logs
|
||||||
|
kubectl logs -l app.kubernetes.io/component=worker -n {{ .Release.Namespace }} -f
|
||||||
|
|
||||||
|
# Run a database migration manually
|
||||||
|
kubectl create job --from=cronjob/{{ include "docuelevate.fullname" . }}-migrate migrate-manual -n {{ .Release.Namespace }}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
{{/*
|
||||||
|
Expand the name of the chart.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.name" -}}
|
||||||
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create a default fully qualified app name.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this.
|
||||||
|
If release name contains chart name it will be used as a full name.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.fullname" -}}
|
||||||
|
{{- if .Values.fullnameOverride }}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||||
|
{{- if contains $name .Release.Name }}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create chart label value: "chart-name-version"
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.chart" -}}
|
||||||
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Common labels applied to every resource.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.labels" -}}
|
||||||
|
helm.sh/chart: {{ include "docuelevate.chart" . }}
|
||||||
|
{{ include "docuelevate.selectorLabels" . }}
|
||||||
|
{{- if .Chart.AppVersion }}
|
||||||
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||||
|
{{- end }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Selector labels.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.selectorLabels" -}}
|
||||||
|
app.kubernetes.io/name: {{ include "docuelevate.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
ServiceAccount name.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.serviceAccountName" -}}
|
||||||
|
{{- if .Values.serviceAccount.create }}
|
||||||
|
{{- default (include "docuelevate.fullname" .) .Values.serviceAccount.name }}
|
||||||
|
{{- else }}
|
||||||
|
{{- default "default" .Values.serviceAccount.name }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Image reference, using appVersion as default tag.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.image" -}}
|
||||||
|
{{- $tag := .Values.image.tag | default .Chart.AppVersion }}
|
||||||
|
{{- printf "%s:%s" .Values.image.repository $tag }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Resolve REDIS_URL: prefer externalRedis.url, then secrets.REDIS_URL,
|
||||||
|
then fall back to the bundled Redis service URL.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.redisUrl" -}}
|
||||||
|
{{- if .Values.externalRedis.url }}
|
||||||
|
{{- .Values.externalRedis.url }}
|
||||||
|
{{- else if .Values.secrets.REDIS_URL }}
|
||||||
|
{{- .Values.secrets.REDIS_URL }}
|
||||||
|
{{- else if .Values.redis.enabled }}
|
||||||
|
{{- printf "redis://%s-redis-master:6379/0" .Release.Name }}
|
||||||
|
{{- else }}
|
||||||
|
{{- "" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Resolve GOTENBERG_URL — use env override if set, otherwise build from service name.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.gotenbergUrl" -}}
|
||||||
|
{{- if .Values.env.GOTENBERG_URL }}
|
||||||
|
{{- tpl .Values.env.GOTENBERG_URL . }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "http://%s-gotenberg:3000" (include "docuelevate.fullname" .) }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Resolve MEILISEARCH_URL — use env override if set, otherwise build from service name.
|
||||||
|
*/}}
|
||||||
|
{{- define "docuelevate.meilisearchUrl" -}}
|
||||||
|
{{- if .Values.env.MEILISEARCH_URL }}
|
||||||
|
{{- tpl .Values.env.MEILISEARCH_URL . }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "http://%s-meilisearch:7700" (include "docuelevate.fullname" .) }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-api
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
{{- if not .Values.api.autoscaling.enabled }}
|
||||||
|
replicas: {{ .Values.api.replicaCount }}
|
||||||
|
{{- end }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
{{- with .Values.api.podAnnotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.podSecurityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: api
|
||||||
|
image: {{ include "docuelevate.image" . }}
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
command:
|
||||||
|
- uvicorn
|
||||||
|
- app.main:app
|
||||||
|
- --host
|
||||||
|
- "0.0.0.0"
|
||||||
|
- --port
|
||||||
|
- "8000"
|
||||||
|
- --proxy-headers
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8000
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-config
|
||||||
|
- secretRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-secret
|
||||||
|
{{- with .Values.api.livenessProbe }}
|
||||||
|
livenessProbe:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.readinessProbe }}
|
||||||
|
readinessProbe:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.securityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /workdir
|
||||||
|
volumes:
|
||||||
|
- name: workdir
|
||||||
|
{{- if .Values.workdir.persistence.enabled }}
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||||
|
{{- else }}
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.nodeSelector }}
|
||||||
|
nodeSelector:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.affinity }}
|
||||||
|
affinity:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.api.tolerations }}
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-api
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.api.service.type }}
|
||||||
|
ports:
|
||||||
|
- port: {{ .Values.api.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Non-sensitive application configuration.
|
||||||
|
# All values are injected as environment variables into the API and Worker pods.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-config
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
data:
|
||||||
|
WORKDIR: {{ .Values.env.WORKDIR | quote }}
|
||||||
|
PYTHONPATH: "/app"
|
||||||
|
PYTHONUNBUFFERED: "1"
|
||||||
|
|
||||||
|
# AI provider
|
||||||
|
AI_PROVIDER: {{ .Values.env.AI_PROVIDER | quote }}
|
||||||
|
OPENAI_MODEL: {{ .Values.env.OPENAI_MODEL | quote }}
|
||||||
|
|
||||||
|
# Azure
|
||||||
|
AZURE_REGION: {{ .Values.env.AZURE_REGION | quote }}
|
||||||
|
AZURE_ENDPOINT: {{ .Values.env.AZURE_ENDPOINT | quote }}
|
||||||
|
|
||||||
|
# Service URLs — resolved from helper templates so they always match the
|
||||||
|
# in-cluster service names even if values.yaml uses template expressions.
|
||||||
|
GOTENBERG_URL: {{ include "docuelevate.gotenbergUrl" . | quote }}
|
||||||
|
MEILISEARCH_URL: {{ include "docuelevate.meilisearchUrl" . | quote }}
|
||||||
|
MEILISEARCH_INDEX_NAME: {{ .Values.env.MEILISEARCH_INDEX_NAME | quote }}
|
||||||
|
ENABLE_SEARCH: {{ .Values.env.ENABLE_SEARCH | quote }}
|
||||||
|
|
||||||
|
# Auth
|
||||||
|
AUTH_ENABLED: {{ .Values.env.AUTH_ENABLED | quote }}
|
||||||
|
ADMIN_USERNAME: {{ .Values.env.ADMIN_USERNAME | quote }}
|
||||||
|
EXTERNAL_HOSTNAME: {{ .Values.env.EXTERNAL_HOSTNAME | quote }}
|
||||||
|
|
||||||
|
# Feature flags
|
||||||
|
ENABLE_DEDUPLICATION: {{ .Values.env.ENABLE_DEDUPLICATION | quote }}
|
||||||
|
ENABLE_TEXT_QUALITY_CHECK: {{ .Values.env.ENABLE_TEXT_QUALITY_CHECK | quote }}
|
||||||
|
ALLOW_FILE_DELETE: {{ .Values.env.ALLOW_FILE_DELETE | quote }}
|
||||||
|
|
||||||
|
DEBUG: {{ .Values.env.DEBUG | quote }}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
{{- if .Values.gotenberg.enabled }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-gotenberg
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: gotenberg
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: gotenberg
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: gotenberg
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: gotenberg
|
||||||
|
image: {{ .Values.gotenberg.image.repository }}:{{ .Values.gotenberg.image.tag }}
|
||||||
|
imagePullPolicy: {{ .Values.gotenberg.image.pullPolicy }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3000
|
||||||
|
protocol: TCP
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.gotenberg.resources | nindent 12 }}
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 3000
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 30
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 3000
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 10
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-gotenberg
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: gotenberg
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: {{ .Values.gotenberg.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: gotenberg
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
{{- if .Values.api.autoscaling.enabled }}
|
||||||
|
apiVersion: autoscaling/v2
|
||||||
|
kind: HorizontalPodAutoscaler
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-api
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
scaleTargetRef:
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-api
|
||||||
|
minReplicas: {{ .Values.api.autoscaling.minReplicas }}
|
||||||
|
maxReplicas: {{ .Values.api.autoscaling.maxReplicas }}
|
||||||
|
metrics:
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: cpu
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.api.autoscaling.targetCPUUtilizationPercentage }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
{{- if .Values.worker.autoscaling.enabled }}
|
||||||
|
apiVersion: autoscaling/v2
|
||||||
|
kind: HorizontalPodAutoscaler
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-worker
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
spec:
|
||||||
|
scaleTargetRef:
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-worker
|
||||||
|
minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
|
||||||
|
maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
|
||||||
|
metrics:
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: cpu
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
{{- if .Values.ingress.enabled -}}
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.ingress.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
{{- if .Values.ingress.className }}
|
||||||
|
ingressClassName: {{ .Values.ingress.className }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.ingress.tls }}
|
||||||
|
tls:
|
||||||
|
{{- toYaml .Values.ingress.tls | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
rules:
|
||||||
|
{{- range .Values.ingress.hosts }}
|
||||||
|
- host: {{ .host | quote }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
{{- range .paths }}
|
||||||
|
- path: {{ .path }}
|
||||||
|
pathType: {{ .pathType }}
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: {{ include "docuelevate.fullname" $ }}-api
|
||||||
|
port:
|
||||||
|
number: {{ $.Values.api.service.port }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
{{- if .Values.meilisearch.enabled }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-meilisearch
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: meilisearch
|
||||||
|
spec:
|
||||||
|
# Single replica — Meilisearch Community Edition is single-node only.
|
||||||
|
# Use Meilisearch Cloud or the paid Meilisearch Cluster for HA.
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: meilisearch
|
||||||
|
strategy:
|
||||||
|
# Recreate ensures the PVC is released before the new pod starts
|
||||||
|
type: Recreate
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: meilisearch
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: meilisearch
|
||||||
|
image: {{ .Values.meilisearch.image.repository }}:{{ .Values.meilisearch.image.tag }}
|
||||||
|
imagePullPolicy: {{ .Values.meilisearch.image.pullPolicy }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 7700
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
- name: MEILI_NO_ANALYTICS
|
||||||
|
value: {{ .Values.meilisearch.env.MEILI_NO_ANALYTICS | quote }}
|
||||||
|
{{- if .Values.secrets.MEILISEARCH_API_KEY }}
|
||||||
|
- name: MEILI_MASTER_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-secret
|
||||||
|
key: MEILISEARCH_API_KEY
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.meilisearch.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /meili_data
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 7700
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 30
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 7700
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
volumes:
|
||||||
|
- name: data
|
||||||
|
{{- if .Values.meilisearch.persistence.enabled }}
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: {{ .Values.meilisearch.persistence.existingClaim | default (printf "%s-meilisearch-data" (include "docuelevate.fullname" .)) }}
|
||||||
|
{{- else }}
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-meilisearch
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: meilisearch
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: {{ .Values.meilisearch.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: meilisearch
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# Database migration Job — runs `alembic upgrade head` once before
|
||||||
|
# the API and Worker Deployments start. Helm hooks ensure ordering:
|
||||||
|
# the Job runs during pre-upgrade/pre-install and the Deployments wait
|
||||||
|
# for it via the `helm.sh/hook-weight` annotation.
|
||||||
|
{{- if .Values.migrations.enabled }}
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-migrate-{{ .Release.Revision }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: migrate
|
||||||
|
annotations:
|
||||||
|
"helm.sh/hook": pre-install,pre-upgrade
|
||||||
|
"helm.sh/hook-weight": "-5"
|
||||||
|
"helm.sh/hook-delete-policy": before-hook-creation
|
||||||
|
{{- if .Values.migrations.ttlSecondsAfterFinished }}
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
{{- if .Values.migrations.ttlSecondsAfterFinished }}
|
||||||
|
ttlSecondsAfterFinished: {{ .Values.migrations.ttlSecondsAfterFinished }}
|
||||||
|
{{- end }}
|
||||||
|
backoffLimit: 3
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: migrate
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||||
|
{{- with .Values.image.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: migrate
|
||||||
|
image: {{ include "docuelevate.image" . }}
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
command: ["alembic", "upgrade", "head"]
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-config
|
||||||
|
- secretRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-secret
|
||||||
|
{{- with .Values.workdir.persistence }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /workdir
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
volumes:
|
||||||
|
- name: workdir
|
||||||
|
{{- if .Values.workdir.persistence.enabled }}
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||||
|
{{- else }}
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# Shared workdir PVC — mounted by both the API and the Worker so they can
|
||||||
|
# exchange files during processing. Requires ReadWriteMany (e.g. NFS,
|
||||||
|
# CephFS, Azure Files, EFS) when running multiple replicas.
|
||||||
|
# Set workdir.persistence.accessMode=ReadWriteOnce and replicaCount=1
|
||||||
|
# for single-node / dev clusters.
|
||||||
|
{{- if .Values.workdir.persistence.enabled }}
|
||||||
|
{{- if not .Values.workdir.persistence.existingClaim }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-workdir
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- {{ .Values.workdir.persistence.accessMode }}
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: {{ .Values.workdir.persistence.size }}
|
||||||
|
{{- if .Values.workdir.persistence.storageClass }}
|
||||||
|
storageClassName: {{ .Values.workdir.persistence.storageClass }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Meilisearch data PVC
|
||||||
|
{{- if and .Values.meilisearch.enabled .Values.meilisearch.persistence.enabled }}
|
||||||
|
{{- if not .Values.meilisearch.persistence.existingClaim }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-meilisearch-data
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- {{ .Values.meilisearch.persistence.accessMode }}
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: {{ .Values.meilisearch.persistence.size }}
|
||||||
|
{{- if .Values.meilisearch.persistence.storageClass }}
|
||||||
|
storageClassName: {{ .Values.meilisearch.persistence.storageClass }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{{- if .Values.serviceAccount.create -}}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.serviceAccountName" . }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.serviceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-worker
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.labels" . | nindent 4 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
spec:
|
||||||
|
{{- if not .Values.worker.autoscaling.enabled }}
|
||||||
|
replicas: {{ .Values.worker.replicaCount }}
|
||||||
|
{{- end }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 6 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
{{- include "docuelevate.selectorLabels" . | nindent 8 }}
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
{{- with .Values.worker.podAnnotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
serviceAccountName: {{ include "docuelevate.serviceAccountName" . }}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.worker.podSecurityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: {{ include "docuelevate.image" . }}
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
command:
|
||||||
|
- celery
|
||||||
|
- -A
|
||||||
|
- app.celery_worker
|
||||||
|
- worker
|
||||||
|
- -B
|
||||||
|
- --loglevel=info
|
||||||
|
- -Q
|
||||||
|
- document_processor,default,celery
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-config
|
||||||
|
- secretRef:
|
||||||
|
name: {{ include "docuelevate.fullname" . }}-secret
|
||||||
|
{{- with .Values.worker.securityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /workdir
|
||||||
|
volumes:
|
||||||
|
- name: workdir
|
||||||
|
{{- if .Values.workdir.persistence.enabled }}
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: {{ .Values.workdir.persistence.existingClaim | default (printf "%s-workdir" (include "docuelevate.fullname" .)) }}
|
||||||
|
{{- else }}
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.worker.nodeSelector }}
|
||||||
|
nodeSelector:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.worker.affinity }}
|
||||||
|
affinity:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.worker.tolerations }}
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,306 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# DocuElevate Helm Chart — values.yaml
|
||||||
|
#
|
||||||
|
# Override any value with:
|
||||||
|
# helm install docuelevate ./helm/docuelevate -f my-values.yaml
|
||||||
|
# helm install docuelevate ./helm/docuelevate --set key=value
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Global image settings
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
image:
|
||||||
|
repository: ghcr.io/christianlouis/docuelevate
|
||||||
|
# Defaults to the chart appVersion; pin to a specific digest in production.
|
||||||
|
tag: ""
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
|
||||||
|
imagePullSecrets: []
|
||||||
|
|
||||||
|
nameOverride: ""
|
||||||
|
fullnameOverride: ""
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Shared environment — non-secret application config
|
||||||
|
# All values map 1-to-1 onto DocuElevate environment variables.
|
||||||
|
# Sensitive values (API keys, passwords) go into `secrets` below.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
env:
|
||||||
|
# Required ----------------------------------------------------------------
|
||||||
|
WORKDIR: /workdir
|
||||||
|
|
||||||
|
# AI provider (openai | azure | anthropic | gemini | ollama | openrouter)
|
||||||
|
AI_PROVIDER: openai
|
||||||
|
OPENAI_MODEL: gpt-4o-mini
|
||||||
|
|
||||||
|
# Azure Document Intelligence (required when OCR_PROVIDERS includes "azure")
|
||||||
|
AZURE_REGION: eastus
|
||||||
|
AZURE_ENDPOINT: "" # e.g. https://my-resource.cognitiveservices.azure.com/
|
||||||
|
|
||||||
|
# Gotenberg PDF conversion service
|
||||||
|
GOTENBERG_URL: http://{{ include "docuelevate.fullname" . }}-gotenberg:3000
|
||||||
|
|
||||||
|
# Full-text search — uses the in-cluster Meilisearch service by default
|
||||||
|
MEILISEARCH_URL: http://{{ include "docuelevate.fullname" . }}-meilisearch:7700
|
||||||
|
MEILISEARCH_INDEX_NAME: documents
|
||||||
|
ENABLE_SEARCH: "true"
|
||||||
|
|
||||||
|
# Authentication
|
||||||
|
AUTH_ENABLED: "true"
|
||||||
|
ADMIN_USERNAME: admin
|
||||||
|
|
||||||
|
# Feature flags
|
||||||
|
ENABLE_DEDUPLICATION: "true"
|
||||||
|
ENABLE_TEXT_QUALITY_CHECK: "true"
|
||||||
|
ALLOW_FILE_DELETE: "true"
|
||||||
|
|
||||||
|
# Logging / misc
|
||||||
|
DEBUG: "false"
|
||||||
|
EXTERNAL_HOSTNAME: localhost # set to your public hostname / Ingress host
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Secrets — values are stored in a Kubernetes Secret and injected as env vars.
|
||||||
|
# In production, use an external secret manager (Vault, ESO, Sealed Secrets)
|
||||||
|
# and leave these blank, then mount the Secret yourself.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
secrets:
|
||||||
|
DATABASE_URL: "" # e.g. postgresql://user:pass@postgres:5432/docuelevate
|
||||||
|
REDIS_URL: "" # leave blank to use bundled Redis
|
||||||
|
SESSION_SECRET: "" # min 32-char random string — generate with: openssl rand -hex 32
|
||||||
|
OPENAI_API_KEY: ""
|
||||||
|
AZURE_AI_KEY: ""
|
||||||
|
MEILISEARCH_API_KEY: "" # leave blank for unauthenticated (dev) Meilisearch
|
||||||
|
|
||||||
|
# Storage provider secrets (only the ones you use)
|
||||||
|
DROPBOX_APP_KEY: ""
|
||||||
|
DROPBOX_APP_SECRET: ""
|
||||||
|
DROPBOX_REFRESH_TOKEN: ""
|
||||||
|
GOOGLE_DRIVE_CREDENTIALS_JSON: ""
|
||||||
|
ONEDRIVE_CLIENT_ID: ""
|
||||||
|
ONEDRIVE_CLIENT_SECRET: ""
|
||||||
|
ONEDRIVE_REFRESH_TOKEN: ""
|
||||||
|
AWS_ACCESS_KEY_ID: ""
|
||||||
|
AWS_SECRET_ACCESS_KEY: ""
|
||||||
|
|
||||||
|
# OAuth / Authentik
|
||||||
|
AUTHENTIK_CLIENT_ID: ""
|
||||||
|
AUTHENTIK_CLIENT_SECRET: ""
|
||||||
|
AUTHENTIK_CONFIG_URL: ""
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# External services
|
||||||
|
# Set externalRedis.url (and disable bundled redis) when you have your own.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
externalRedis:
|
||||||
|
# When non-empty this value is injected as REDIS_URL, overriding secrets.REDIS_URL
|
||||||
|
# and the auto-generated bundled-Redis URL.
|
||||||
|
url: ""
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# API deployment
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
api:
|
||||||
|
replicaCount: 2
|
||||||
|
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
|
||||||
|
# Horizontal Pod Autoscaler
|
||||||
|
autoscaling:
|
||||||
|
enabled: false
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 8
|
||||||
|
targetCPUUtilizationPercentage: 70
|
||||||
|
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 8000
|
||||||
|
|
||||||
|
# Liveness / readiness probes
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 20
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 10
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
nodeSelector: {}
|
||||||
|
tolerations: []
|
||||||
|
affinity: {}
|
||||||
|
|
||||||
|
podSecurityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: false # app writes to /workdir
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Celery worker deployment
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
worker:
|
||||||
|
replicaCount: 2
|
||||||
|
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 4Gi
|
||||||
|
|
||||||
|
autoscaling:
|
||||||
|
enabled: false
|
||||||
|
minReplicas: 2
|
||||||
|
maxReplicas: 10
|
||||||
|
targetCPUUtilizationPercentage: 75
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
nodeSelector: {}
|
||||||
|
tolerations: []
|
||||||
|
affinity: {}
|
||||||
|
|
||||||
|
podSecurityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: false
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Shared workdir volume (api + worker mount the same PVC)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
workdir:
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
# storageClass: "" # leave blank for cluster default
|
||||||
|
accessMode: ReadWriteMany # RWX required for multiple pods
|
||||||
|
size: 20Gi
|
||||||
|
# existingClaim: ""
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Gotenberg (PDF conversion)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
gotenberg:
|
||||||
|
enabled: true
|
||||||
|
image:
|
||||||
|
repository: gotenberg/gotenberg
|
||||||
|
tag: latest
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
service:
|
||||||
|
port: 3000
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Meilisearch (full-text search)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
meilisearch:
|
||||||
|
enabled: true
|
||||||
|
image:
|
||||||
|
repository: getmeili/meilisearch
|
||||||
|
tag: latest
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
service:
|
||||||
|
port: 7700
|
||||||
|
env:
|
||||||
|
MEILI_NO_ANALYTICS: "true"
|
||||||
|
# MEILI_MASTER_KEY: "" # set via secrets.MEILISEARCH_API_KEY instead
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
# storageClass: ""
|
||||||
|
accessMode: ReadWriteOnce
|
||||||
|
size: 10Gi
|
||||||
|
# existingClaim: ""
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Bundled Redis (from Bitnami chart)
|
||||||
|
# Disable and set externalRedis.url to use your own.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
redis:
|
||||||
|
enabled: true
|
||||||
|
architecture: standalone
|
||||||
|
auth:
|
||||||
|
enabled: false
|
||||||
|
master:
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
size: 4Gi
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Ingress
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
ingress:
|
||||||
|
enabled: false
|
||||||
|
className: "" # e.g. nginx, traefik
|
||||||
|
annotations: {}
|
||||||
|
# nginx.ingress.kubernetes.io/proxy-body-size: "1g"
|
||||||
|
# cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||||
|
hosts:
|
||||||
|
- host: docuelevate.example.com
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
tls: []
|
||||||
|
# - secretName: docuelevate-tls
|
||||||
|
# hosts:
|
||||||
|
# - docuelevate.example.com
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ServiceAccount
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
serviceAccount:
|
||||||
|
create: true
|
||||||
|
annotations: {}
|
||||||
|
name: ""
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Database migration job
|
||||||
|
# Runs `alembic upgrade head` before the api/worker start.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
migrations:
|
||||||
|
enabled: true
|
||||||
|
# Automatically deleted after successful completion
|
||||||
|
ttlSecondsAfterFinished: 120
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""Add OCR text and AI metadata fields to files table for full-text search
|
||||||
|
|
||||||
|
Revision ID: 004_add_search_fields
|
||||||
|
Revises: 003_add_deduplication_support
|
||||||
|
Create Date: 2026-02-25
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "004_add_search_fields"
|
||||||
|
down_revision: Union[str, None] = "003_add_deduplication_support"
|
||||||
|
depends_on: Union[str, None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Add ocr_text, ai_metadata, and document_title columns to files table."""
|
||||||
|
op.add_column("files", sa.Column("ocr_text", sa.Text(), nullable=True))
|
||||||
|
op.add_column("files", sa.Column("ai_metadata", sa.Text(), nullable=True))
|
||||||
|
op.add_column("files", sa.Column("document_title", sa.String(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Remove ocr_text, ai_metadata, and document_title columns from files table."""
|
||||||
|
op.drop_column("files", "document_title")
|
||||||
|
op.drop_column("files", "ai_metadata")
|
||||||
|
op.drop_column("files", "ocr_text")
|
||||||
+2
-1
@@ -42,4 +42,5 @@ litellm>=1.0.0,<2.0.0
|
|||||||
# Self-hosted OCR engines (optional – only required when the provider is enabled)
|
# Self-hosted OCR engines (optional – only required when the provider is enabled)
|
||||||
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
|
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
|
||||||
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
|
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
|
||||||
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
ocrmypdf>=16.0.0,<17.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
||||||
|
meilisearch>=0.31.0 # Full-text search engine client
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
"""Tests for the full-text search API (app/api/search.py) and Meilisearch
|
||||||
|
client utilities (app/utils/meilisearch_client.py).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# app/utils/meilisearch_client tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestMeilisearchClientDisabled:
|
||||||
|
"""Tests when search is disabled or Meilisearch is unavailable."""
|
||||||
|
|
||||||
|
def test_get_client_returns_none_when_disabled(self):
|
||||||
|
"""get_meilisearch_client returns None when enable_search=False."""
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||||
|
from app.utils.meilisearch_client import index_document, search_documents
|
||||||
|
|
||||||
|
class _FakeRecord:
|
||||||
|
id = 1
|
||||||
|
original_filename = "test.pdf"
|
||||||
|
mime_type = "application/pdf"
|
||||||
|
file_size = 1024
|
||||||
|
created_at = None
|
||||||
|
|
||||||
|
result = index_document(_FakeRecord(), "some text", {})
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
result = search_documents("invoice")
|
||||||
|
assert result["results"] == []
|
||||||
|
assert result["total"] == 0
|
||||||
|
|
||||||
|
def test_search_documents_import_error(self):
|
||||||
|
"""search_documents returns empty dict when meilisearch not installed."""
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
result = search_documents("test")
|
||||||
|
assert result["results"] == []
|
||||||
|
assert result["total"] == 0
|
||||||
|
assert result["page"] == 1
|
||||||
|
assert result["query"] == "test"
|
||||||
|
|
||||||
|
def test_delete_document_no_client(self):
|
||||||
|
"""delete_document returns False when client unavailable."""
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=None):
|
||||||
|
from app.utils.meilisearch_client import delete_document
|
||||||
|
|
||||||
|
result = delete_document(99)
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestMeilisearchIndexDocument:
|
||||||
|
"""Tests for index_document function."""
|
||||||
|
|
||||||
|
def test_index_document_success(self):
|
||||||
|
"""index_document returns True when Meilisearch succeeds."""
|
||||||
|
mock_client = MagicMock()
|
||||||
|
mock_index = MagicMock()
|
||||||
|
mock_task = MagicMock()
|
||||||
|
mock_task.task_uid = 1
|
||||||
|
|
||||||
|
mock_client.get_index.return_value = mock_index
|
||||||
|
mock_index.add_documents.return_value = mock_task
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import index_document
|
||||||
|
|
||||||
|
class _FakeRecord:
|
||||||
|
id = 1
|
||||||
|
original_filename = "invoice.pdf"
|
||||||
|
mime_type = "application/pdf"
|
||||||
|
file_size = 2048
|
||||||
|
created_at = None
|
||||||
|
|
||||||
|
result = index_document(
|
||||||
|
_FakeRecord(),
|
||||||
|
"This is an invoice for services rendered",
|
||||||
|
{
|
||||||
|
"title": "Invoice January 2026",
|
||||||
|
"document_type": "Invoice",
|
||||||
|
"tags": ["invoice", "services"],
|
||||||
|
"absender": "ACME Corp",
|
||||||
|
"language": "en",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result is True
|
||||||
|
mock_index.add_documents.assert_called_once()
|
||||||
|
call_docs = mock_index.add_documents.call_args[0][0]
|
||||||
|
assert len(call_docs) == 1
|
||||||
|
doc = call_docs[0]
|
||||||
|
assert doc["file_id"] == 1
|
||||||
|
assert doc["document_title"] == "Invoice January 2026"
|
||||||
|
assert "invoice" in doc["tags"]
|
||||||
|
|
||||||
|
def test_index_document_meilisearch_error(self):
|
||||||
|
"""index_document returns False on Meilisearch exception."""
|
||||||
|
mock_client = MagicMock()
|
||||||
|
mock_index = MagicMock()
|
||||||
|
mock_client.get_index.return_value = mock_index
|
||||||
|
mock_index.add_documents.side_effect = RuntimeError("Meilisearch down")
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import index_document
|
||||||
|
|
||||||
|
class _FakeRecord:
|
||||||
|
id = 2
|
||||||
|
original_filename = "test.pdf"
|
||||||
|
mime_type = "application/pdf"
|
||||||
|
file_size = 512
|
||||||
|
created_at = None
|
||||||
|
|
||||||
|
result = index_document(_FakeRecord(), "some text", {})
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestMeilisearchSearchDocuments:
|
||||||
|
"""Tests for search_documents function."""
|
||||||
|
|
||||||
|
def _make_mock_client(self, hits=None, total=None):
|
||||||
|
mock_client = MagicMock()
|
||||||
|
mock_index = MagicMock()
|
||||||
|
mock_client.get_index.return_value = mock_index
|
||||||
|
mock_index.search.return_value = {
|
||||||
|
"hits": hits or [],
|
||||||
|
"estimatedTotalHits": total if total is not None else len(hits or []),
|
||||||
|
}
|
||||||
|
return mock_client, mock_index
|
||||||
|
|
||||||
|
def test_search_returns_results(self):
|
||||||
|
"""search_documents returns hits from Meilisearch."""
|
||||||
|
hits = [
|
||||||
|
{
|
||||||
|
"file_id": 42,
|
||||||
|
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||||
|
"document_title": "Amazon Invoice",
|
||||||
|
"document_type": "Invoice",
|
||||||
|
"tags": ["amazon", "invoice"],
|
||||||
|
"ocr_text": "Amazon invoice content here",
|
||||||
|
"_formatted": {
|
||||||
|
"document_title": "Amazon <mark>Invoice</mark>",
|
||||||
|
"ocr_text": "…Amazon <mark>invoice</mark> content here…",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
mock_client, mock_index = self._make_mock_client(hits=hits, total=1)
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
result = search_documents("invoice", page=1, per_page=20)
|
||||||
|
|
||||||
|
assert result["total"] == 1
|
||||||
|
assert result["pages"] == 1
|
||||||
|
assert result["query"] == "invoice"
|
||||||
|
assert len(result["results"]) == 1
|
||||||
|
# Raw ocr_text should be stripped from result (only _formatted snippet kept)
|
||||||
|
assert "ocr_text" not in result["results"][0]
|
||||||
|
assert result["results"][0]["file_id"] == 42
|
||||||
|
|
||||||
|
def test_search_with_filters(self):
|
||||||
|
"""search_documents passes filter expressions to Meilisearch."""
|
||||||
|
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
search_documents("contract", mime_type="application/pdf", language="de", page=1, per_page=10)
|
||||||
|
|
||||||
|
call_kwargs = mock_index.search.call_args
|
||||||
|
search_params = call_kwargs[0][1]
|
||||||
|
assert "filter" in search_params
|
||||||
|
assert 'mime_type = "application/pdf"' in search_params["filter"]
|
||||||
|
assert 'language = "de"' in search_params["filter"]
|
||||||
|
|
||||||
|
def test_search_pagination(self):
|
||||||
|
"""search_documents applies correct offset for page 2."""
|
||||||
|
mock_client, mock_index = self._make_mock_client(hits=[], total=50)
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
result = search_documents("test", page=3, per_page=10)
|
||||||
|
|
||||||
|
call_kwargs = mock_index.search.call_args
|
||||||
|
search_params = call_kwargs[0][1]
|
||||||
|
assert search_params["offset"] == 20 # (3-1) * 10
|
||||||
|
assert search_params["limit"] == 10
|
||||||
|
assert result["pages"] == 5
|
||||||
|
|
||||||
|
def test_search_empty_results(self):
|
||||||
|
"""search_documents returns proper empty structure."""
|
||||||
|
mock_client, mock_index = self._make_mock_client(hits=[], total=0)
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
result = search_documents("nothing")
|
||||||
|
|
||||||
|
assert result["results"] == []
|
||||||
|
assert result["total"] == 0
|
||||||
|
assert result["pages"] == 0
|
||||||
|
|
||||||
|
def test_search_exception_returns_empty(self):
|
||||||
|
"""search_documents returns empty dict on Meilisearch exception."""
|
||||||
|
mock_client = MagicMock()
|
||||||
|
mock_index = MagicMock()
|
||||||
|
mock_client.get_index.side_effect = RuntimeError("Meilisearch unavailable")
|
||||||
|
|
||||||
|
with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client):
|
||||||
|
from app.utils.meilisearch_client import search_documents
|
||||||
|
|
||||||
|
result = search_documents("invoice")
|
||||||
|
|
||||||
|
assert result["results"] == []
|
||||||
|
assert result["total"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Search API endpoint tests (GET /api/search)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestSearchAPIEndpoint:
|
||||||
|
"""Tests for GET /api/search endpoint."""
|
||||||
|
|
||||||
|
def test_search_endpoint_success(self, client):
|
||||||
|
"""GET /api/search?q=... returns search results."""
|
||||||
|
mock_result = {
|
||||||
|
"results": [{"file_id": 1, "document_title": "Test Invoice", "document_type": "Invoice"}],
|
||||||
|
"total": 1,
|
||||||
|
"page": 1,
|
||||||
|
"pages": 1,
|
||||||
|
"query": "invoice",
|
||||||
|
}
|
||||||
|
with patch("app.api.search.search_documents", return_value=mock_result):
|
||||||
|
response = client.get("/api/search?q=invoice")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["total"] == 1
|
||||||
|
assert data["query"] == "invoice"
|
||||||
|
assert len(data["results"]) == 1
|
||||||
|
|
||||||
|
def test_search_endpoint_missing_query(self, client):
|
||||||
|
"""GET /api/search without q returns 422."""
|
||||||
|
response = client.get("/api/search")
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
def test_search_endpoint_empty_query(self, client):
|
||||||
|
"""GET /api/search?q= (empty) returns 422 due to min_length=1."""
|
||||||
|
response = client.get("/api/search?q=")
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
def test_search_endpoint_with_filters(self, client):
|
||||||
|
"""GET /api/search with optional filters passes them to search_documents."""
|
||||||
|
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "invoice"}
|
||||||
|
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||||
|
response = client.get("/api/search?q=invoice&mime_type=application/pdf&language=en&page=2&per_page=10")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
mock_search.assert_called_once_with(
|
||||||
|
"invoice",
|
||||||
|
mime_type="application/pdf",
|
||||||
|
document_type=None,
|
||||||
|
language="en",
|
||||||
|
date_from=None,
|
||||||
|
date_to=None,
|
||||||
|
page=2,
|
||||||
|
per_page=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_search_endpoint_per_page_max(self, client):
|
||||||
|
"""GET /api/search with per_page > 100 returns 422."""
|
||||||
|
response = client.get("/api/search?q=test&per_page=200")
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
def test_search_endpoint_pagination_defaults(self, client):
|
||||||
|
"""GET /api/search uses default page=1 per_page=20."""
|
||||||
|
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"}
|
||||||
|
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||||
|
response = client.get("/api/search?q=test")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
mock_search.assert_called_once_with(
|
||||||
|
"test",
|
||||||
|
mime_type=None,
|
||||||
|
document_type=None,
|
||||||
|
language=None,
|
||||||
|
date_from=None,
|
||||||
|
date_to=None,
|
||||||
|
page=1,
|
||||||
|
per_page=20,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_search_endpoint_date_filters(self, client):
|
||||||
|
"""GET /api/search with date_from and date_to passes them as int."""
|
||||||
|
mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "contract"}
|
||||||
|
with patch("app.api.search.search_documents", return_value=mock_result) as mock_search:
|
||||||
|
response = client.get("/api/search?q=contract&date_from=1704067200&date_to=1735689600")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
call_kwargs = mock_search.call_args
|
||||||
|
assert call_kwargs[1]["date_from"] == 1704067200
|
||||||
|
assert call_kwargs[1]["date_to"] == 1735689600
|
||||||
Reference in New Issue
Block a user