gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into sentinel-fix-ssrf-dns-resolution-16520734505214840647

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-03-23 17:16:53 +01:00
committed by GitHub
parent 06b0fced38 88368f7f76
commit 1e7f2275d3
185 changed files with 1937 additions and 25684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/allowed_types.py
+1 -9
View File
@@ -68,8 +68,6 @@ IMAGE_MIME_TYPES: set[str] = {
"image/tiff",
"image/webp",
"image/svg+xml",
"image/heic",
"image/heif",
}
# ---------------------------------------------------------------------------
@@ -126,8 +124,6 @@ ALLOWED_EXTENSIONS: set[str] = {
".tif",
".webp",
".svg",
".heic",
".heif",
# Web
".html",
".htm",
@@ -238,7 +234,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"mime_types": frozenset(
{
"image/jpeg",
@@ -249,8 +245,6 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
"image/tiff",
"image/webp",
"image/svg+xml",
"image/heic",
"image/heif",
}
),
"extensions": frozenset(
@@ -264,8 +258,6 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
".tif",
".webp",
".svg",
".heic",
".heif",
}
),
},
app/utils/automation_hooks.py
-188
View File
@@ -1,188 +0,0 @@
"""Automation hook utilities for Zapier / Make.com integration.
Provides helpers to build Zapier-compatible flat payloads, query active
automation hook subscriptions, and fan-out event delivery to all matching
hooks via Celery tasks.
The payload format is intentionally *flat* (no nested ``data`` key) so that
Zapier and Make.com can map fields without JSONPath expressions. An ``id``
field is included for Zapier deduplication.
"""
import json
import logging
import time
import uuid
from typing import Any
from app.config import settings
from app.database import SessionLocal
from app.models import AutomationHook
from app.utils.webhook import VALID_EVENTS
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Payload helpers
# ---------------------------------------------------------------------------
def build_zapier_payload(event: str, data: dict[str, Any]) -> dict[str, Any]:
"""Build a flat, Zapier-compatible webhook payload.
Zapier works best with flat JSON objects that include an ``id`` field
for deduplication. This function merges event metadata into the
top-level object alongside the event-specific *data*.
Args:
event: The event name (e.g. ``document.processed``).
data: Event-specific key/value pairs.
Returns:
A flat dictionary suitable for Zapier / Make.com consumption.
"""
return {
"id": f"evt_{uuid.uuid4().hex[:16]}",
"event": event,
"timestamp": time.time(),
**data,
}
# ---------------------------------------------------------------------------
# Sample payloads (used by the /triggers/sample endpoint)
# ---------------------------------------------------------------------------
#: Example payloads that Zapier uses for field-mapping during Zap creation.
SAMPLE_PAYLOADS: dict[str, dict[str, Any]] = {
"document.uploaded": {
"id": "evt_sample0001",
"event": "document.uploaded",
"timestamp": 1710000000.0,
"document_id": 42,
"filename": "invoice_2024.pdf",
"content_type": "application/pdf",
"size_bytes": 204800,
"owner_id": "user@example.com",
},
"document.processed": {
"id": "evt_sample0002",
"event": "document.processed",
"timestamp": 1710000060.0,
"document_id": 42,
"filename": "invoice_2024.pdf",
"status": "processed",
"title": "Invoice #1234",
"owner_id": "user@example.com",
},
"document.failed": {
"id": "evt_sample0003",
"event": "document.failed",
"timestamp": 1710000120.0,
"document_id": 42,
"filename": "corrupt.pdf",
"status": "failed",
"error": "Unable to extract text from document",
"owner_id": "user@example.com",
},
"user.signup": {
"id": "evt_sample0004",
"event": "user.signup",
"timestamp": 1710000180.0,
"user_id": "newuser@example.com",
"display_name": "Jane Doe",
},
"user.plan_changed": {
"id": "evt_sample0005",
"event": "user.plan_changed",
"timestamp": 1710000240.0,
"user_id": "user@example.com",
"old_tier": "free",
"new_tier": "pro",
},
"user.payment_issue": {
"id": "evt_sample0006",
"event": "user.payment_issue",
"timestamp": 1710000300.0,
"user_id": "user@example.com",
"issue": "Credit card declined",
},
}
# ---------------------------------------------------------------------------
# Database queries
# ---------------------------------------------------------------------------
def get_active_hooks_for_event(event: str) -> list[dict[str, Any]]:
"""Return all active automation hooks subscribed to *event*.
Args:
event: The event name to filter on.
Returns:
A list of dicts with ``id``, ``target_url``, ``secret``, and
``events`` keys.
"""
db = SessionLocal()
try:
hooks = db.query(AutomationHook).filter(AutomationHook.is_active.is_(True)).all()
result: list[dict[str, Any]] = []
for hook in hooks:
try:
subscribed = json.loads(hook.events)
except (json.JSONDecodeError, TypeError):
subscribed = []
if event in subscribed:
result.append(
{
"id": hook.id,
"target_url": hook.target_url,
"secret": hook.secret,
"events": subscribed,
}
)
return result
finally:
db.close()
# ---------------------------------------------------------------------------
# Dispatch
# ---------------------------------------------------------------------------
def dispatch_automation_hooks(event: str, data: dict[str, Any]) -> None:
"""Fan-out an event to all matching active automation hooks.
Builds a Zapier-compatible flat payload and queues a Celery task for
each matching hook so delivery is asynchronous with automatic retries.
Args:
event: Event name (must be in :data:`VALID_EVENTS`).
data: Event-specific payload data.
"""
if not settings.automation_hooks_enabled:
return
if event not in VALID_EVENTS:
logger.warning("Ignoring unknown automation hook event: %s", event)
return
hooks = get_active_hooks_for_event(event)
if not hooks:
logger.debug("No active automation hooks for event %s", event)
return
payload = build_zapier_payload(event, data)
from app.tasks.automation_tasks import deliver_automation_hook_task
for hook in hooks:
try:
deliver_automation_hook_task.delay(hook["target_url"], payload, hook["secret"])
logger.debug("Queued automation hook delivery to %s for event %s", hook["target_url"], event)
except Exception as exc:
logger.error("Failed to queue automation hook to %s: %s", hook["target_url"], exc)
app/utils/classification_rules.py
-378
View File
@@ -1,378 +0,0 @@
"""
Rule-based document classification engine.
Provides pre-built categories and a rule matcher that classifies documents
using filename patterns, content keywords, and metadata fields. Custom
rules stored in the database are evaluated alongside the built-in defaults.
Usage::
from app.utils.classification_rules import classify_document
result = classify_document(
filename="2024-03-01_Invoice_Acme.pdf",
text="Invoice total: $1,234.56",
metadata={"absender": "Acme Corp"},
custom_rules=custom_rules_from_db,
)
# result -> ClassificationResult(category="invoice", confidence=85, matched_rules=[...])
"""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass, field
from typing import Any
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Pre-built categories
# ---------------------------------------------------------------------------
#: Canonical category names recognized by the system. Users may also define
#: their own categories via custom rules.
BUILTIN_CATEGORIES: dict[str, str] = {
"invoice": "Invoice",
"contract": "Contract",
"receipt": "Receipt",
"letter": "Letter",
"report": "Report",
"bank_statement": "Bank Statement",
"tax_document": "Tax Document",
"insurance": "Insurance Document",
"payslip": "Payslip",
"unknown": "Unknown",
}
# ---------------------------------------------------------------------------
# Rule type constants
# ---------------------------------------------------------------------------
RULE_TYPE_FILENAME = "filename_pattern"
RULE_TYPE_CONTENT = "content_keyword"
RULE_TYPE_METADATA = "metadata_match"
# ---------------------------------------------------------------------------
# Data classes
# ---------------------------------------------------------------------------
@dataclass
class ClassificationRule:
"""A single classification rule."""
name: str
category: str
rule_type: str # filename_pattern | content_keyword | metadata_match
pattern: str # regex for filename, keyword(s) for content, "field=value" for metadata
priority: int = 0 # higher = evaluated first
case_sensitive: bool = False
def __post_init__(self) -> None:
if self.rule_type not in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA):
raise ValueError(f"Invalid rule_type: {self.rule_type!r}")
@dataclass
class MatchedRule:
"""Records which rule matched and why."""
rule_name: str
rule_type: str
category: str
confidence: int
@dataclass
class ClassificationResult:
"""The outcome of running the classification engine on a document."""
category: str
confidence: int # 0 – 100
matched_rules: list[MatchedRule] = field(default_factory=list)
# ---------------------------------------------------------------------------
# Built-in rules
# ---------------------------------------------------------------------------
BUILTIN_RULES: list[ClassificationRule] = [
# ── Invoice ───────────────────────────────────────────────────────────
ClassificationRule("builtin_invoice_filename", "invoice", RULE_TYPE_FILENAME, r"(?i)invoice|rechnung|facture"),
ClassificationRule(
"builtin_invoice_content",
"invoice",
RULE_TYPE_CONTENT,
"invoice number|invoice total|amount due|rechnung|rechnungsnummer|total amount|bill to",
),
ClassificationRule("builtin_invoice_metadata", "invoice", RULE_TYPE_METADATA, "document_type=Invoice"),
ClassificationRule(
"builtin_invoice_kommunikationsart", "invoice", RULE_TYPE_METADATA, "kommunikationsart=Rechnung"
),
# ── Contract ──────────────────────────────────────────────────────────
ClassificationRule("builtin_contract_filename", "contract", RULE_TYPE_FILENAME, r"(?i)contract|vertrag|agreement"),
ClassificationRule(
"builtin_contract_content",
"contract",
RULE_TYPE_CONTENT,
"hereby agrees|terms and conditions|vertrag|agreement between|party agrees|effective date",
),
ClassificationRule("builtin_contract_metadata", "contract", RULE_TYPE_METADATA, "document_type=Contract"),
ClassificationRule(
"builtin_contract_kommunikationsart", "contract", RULE_TYPE_METADATA, "kommunikationsart=Vertrag"
),
# ── Receipt ───────────────────────────────────────────────────────────
ClassificationRule("builtin_receipt_filename", "receipt", RULE_TYPE_FILENAME, r"(?i)receipt|quittung|beleg"),
ClassificationRule(
"builtin_receipt_content",
"receipt",
RULE_TYPE_CONTENT,
"receipt|quittung|payment received|thank you for your purchase|transaction id",
),
ClassificationRule("builtin_receipt_metadata", "receipt", RULE_TYPE_METADATA, "document_type=Receipt"),
ClassificationRule(
"builtin_receipt_kommunikationsart", "receipt", RULE_TYPE_METADATA, "kommunikationsart=Quittung"
),
# ── Letter ────────────────────────────────────────────────────────────
ClassificationRule("builtin_letter_filename", "letter", RULE_TYPE_FILENAME, r"(?i)letter|brief|schreiben"),
ClassificationRule(
"builtin_letter_content",
"letter",
RULE_TYPE_CONTENT,
"dear sir|dear madam|sehr geehrte|to whom it may concern|sincerely|mit freundlichen",
),
# ── Report ────────────────────────────────────────────────────────────
ClassificationRule("builtin_report_filename", "report", RULE_TYPE_FILENAME, r"(?i)report|bericht"),
ClassificationRule(
"builtin_report_content",
"report",
RULE_TYPE_CONTENT,
"executive summary|table of contents|annual report|quarterly report|findings",
),
# ── Bank statement ────────────────────────────────────────────────────
ClassificationRule(
"builtin_bank_filename",
"bank_statement",
RULE_TYPE_FILENAME,
r"(?i)bank.?statement|kontoauszug",
),
ClassificationRule(
"builtin_bank_content",
"bank_statement",
RULE_TYPE_CONTENT,
"account statement|kontoauszug|opening balance|closing balance|account number",
),
ClassificationRule(
"builtin_bank_kommunikationsart", "bank_statement", RULE_TYPE_METADATA, "kommunikationsart=Kontoauszug"
),
# ── Tax document ──────────────────────────────────────────────────────
ClassificationRule("builtin_tax_filename", "tax_document", RULE_TYPE_FILENAME, r"(?i)tax|steuer|steuerbescheid"),
ClassificationRule(
"builtin_tax_content",
"tax_document",
RULE_TYPE_CONTENT,
"tax return|steuerbescheid|taxable income|finanzamt|tax assessment",
),
# ── Insurance ─────────────────────────────────────────────────────────
ClassificationRule(
"builtin_insurance_filename", "insurance", RULE_TYPE_FILENAME, r"(?i)insurance|versicherung|police"
),
ClassificationRule(
"builtin_insurance_content",
"insurance",
RULE_TYPE_CONTENT,
"insurance policy|versicherung|policennummer|coverage|premium|deductible",
),
# ── Payslip ───────────────────────────────────────────────────────────
ClassificationRule(
"builtin_payslip_filename", "payslip", RULE_TYPE_FILENAME, r"(?i)payslip|gehaltsabrechnung|lohnabrechnung"
),
ClassificationRule(
"builtin_payslip_content",
"payslip",
RULE_TYPE_CONTENT,
"gross salary|net salary|gehaltsabrechnung|lohnabrechnung|bruttolohn|nettolohn",
),
]
# ---------------------------------------------------------------------------
# Confidence scoring
# ---------------------------------------------------------------------------
#: Base confidence for each rule type when it matches.
_CONFIDENCE_MAP: dict[str, int] = {
RULE_TYPE_FILENAME: 60,
RULE_TYPE_CONTENT: 70,
RULE_TYPE_METADATA: 90,
}
#: Extra confidence per additional matching rule of the same category (capped).
_CONFIDENCE_BONUS_PER_EXTRA_RULE = 10
# ---------------------------------------------------------------------------
# Matching helpers
# ---------------------------------------------------------------------------
def _match_filename(rule: ClassificationRule, filename: str) -> bool:
"""Return True if *rule.pattern* (regex) matches anywhere in *filename*."""
if not filename:
return False
flags = 0 if rule.case_sensitive else re.IGNORECASE
return bool(re.search(rule.pattern, filename, flags))
def _match_content(rule: ClassificationRule, text: str) -> bool:
"""Return True if any keyword in *rule.pattern* appears in *text*.
Keywords are separated by ``|`` (pipe).
"""
if not text:
return False
keywords = [kw.strip() for kw in rule.pattern.split("|") if kw.strip()]
text_lower = text if rule.case_sensitive else text.lower()
return any((kw if rule.case_sensitive else kw.lower()) in text_lower for kw in keywords)
def _match_metadata(rule: ClassificationRule, metadata: dict[str, Any] | None) -> bool:
"""Return True if *rule.pattern* (``field=value``) matches *metadata*.
Pattern format: ``field_name=expected_value``.
"""
if not metadata:
return False
if "=" not in rule.pattern:
return False
field_name, expected_value = rule.pattern.split("=", 1)
actual = metadata.get(field_name.strip())
if actual is None:
return False
if rule.case_sensitive:
return str(actual) == expected_value.strip()
return str(actual).lower() == expected_value.strip().lower()
_MATCHERS: dict[str, tuple] = {
RULE_TYPE_FILENAME: (_match_filename, "filename"),
RULE_TYPE_CONTENT: (_match_content, "text"),
RULE_TYPE_METADATA: (_match_metadata, "metadata"),
}
def _evaluate_rule(
rule: ClassificationRule,
filename: str,
text: str,
metadata: dict[str, Any] | None,
) -> MatchedRule | None:
"""Evaluate a single rule against the document. Return a :class:`MatchedRule` on match."""
entry = _MATCHERS.get(rule.rule_type)
if entry is None:
return None
matcher, arg_key = entry
arg_map = {"filename": filename, "text": text, "metadata": metadata}
matched = matcher(rule, arg_map[arg_key])
if matched:
return MatchedRule(
rule_name=rule.name,
rule_type=rule.rule_type,
category=rule.category,
confidence=_CONFIDENCE_MAP.get(rule.rule_type, 50),
)
return None
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def classify_document(
filename: str = "",
text: str = "",
metadata: dict[str, Any] | None = None,
custom_rules: list[ClassificationRule] | None = None,
) -> ClassificationResult:
"""Classify a document by evaluating built-in and custom rules.
Rules are evaluated in priority order (highest first, then built-in before
custom for the same priority). The category with the most rule matches
wins; ties are broken by cumulative confidence.
Args:
filename: Original filename of the document.
text: Extracted / OCR text of the document.
metadata: Previously-extracted AI metadata dict (e.g. from ``ai_metadata``).
custom_rules: Optional list of user-defined :class:`ClassificationRule` objects.
Returns:
A :class:`ClassificationResult` with the best matching category,
overall confidence score, and the list of rules that fired.
"""
all_rules = list(BUILTIN_RULES)
if custom_rules:
all_rules.extend(custom_rules)
# Sort by priority descending (higher priority first)
all_rules.sort(key=lambda r: r.priority, reverse=True)
matches: list[MatchedRule] = []
for rule in all_rules:
result = _evaluate_rule(rule, filename, text, metadata)
if result is not None:
matches.append(result)
if not matches:
return ClassificationResult(category="unknown", confidence=0, matched_rules=[])
# Aggregate by category: pick the one with the most matches, then highest
# cumulative confidence as tiebreaker.
category_scores: dict[str, list[MatchedRule]] = {}
for m in matches:
category_scores.setdefault(m.category, []).append(m)
best_category = max(
category_scores,
key=lambda cat: (len(category_scores[cat]), sum(m.confidence for m in category_scores[cat])),
)
best_matches = category_scores[best_category]
base_confidence = max(m.confidence for m in best_matches)
bonus = min(
(len(best_matches) - 1) * _CONFIDENCE_BONUS_PER_EXTRA_RULE,
100 - base_confidence,
)
final_confidence = min(base_confidence + bonus, 100)
return ClassificationResult(
category=best_category,
confidence=final_confidence,
matched_rules=best_matches,
)
def db_rule_to_engine_rule(db_rule: Any) -> ClassificationRule:
"""Convert a database ``ClassificationRuleModel`` row to an engine :class:`ClassificationRule`.
Args:
db_rule: A SQLAlchemy model instance with ``name``, ``category``,
``rule_type``, ``pattern``, ``priority``, and ``case_sensitive`` attributes.
Returns:
A :class:`ClassificationRule` dataclass instance.
"""
return ClassificationRule(
name=db_rule.name,
category=db_rule.category,
rule_type=db_rule.rule_type,
pattern=db_rule.pattern,
priority=db_rule.priority,
case_sensitive=getattr(db_rule, "case_sensitive", False),
)
app/utils/db_migrate.py
+4 -3
View File
@@ -15,7 +15,7 @@ import logging
import re
from typing import Any
from sqlalchemy import MetaData, create_engine, inspect, text
from sqlalchemy import MetaData, create_engine, func, inspect, select, table
from sqlalchemy.engine import Engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.orm import sessionmaker
@@ -89,8 +89,9 @@ def preview_migration(source_url: str) -> dict[str, Any]:
logger.warning(f"Skipping table with invalid name format: {table_name}")
continue
# table_name is safe — sourced from inspect().get_table_names(), not user input
quoted_table = conn.dialect.identifier_preparer.quote(table_name)
row = conn.execute(text(f"SELECT COUNT(*) FROM {quoted_table}")).fetchone() # noqa: S608
t = table(table_name)
query = select(func.count()).select_from(t)
row = conn.execute(query).fetchone()
count = row[0] if row else 0
result.append({"name": table_name, "row_count": count})
total += count
app/utils/file_operations.py
+12 -1
View File
@@ -7,8 +7,19 @@ def hash_file(filepath: str | Path, chunk_size: int = 65536) -> str:
Returns the SHA-256 hash of the file at 'filepath'.
Reads the file in chunks to handle large files efficiently.
"""
from app.config import settings
filepath_obj = Path(filepath).resolve()
workdir_obj = Path(settings.workdir).resolve()
# Security check: Ensure the resolved path is strictly within the allowed workdir
try:
filepath_obj.relative_to(workdir_obj)
except ValueError:
raise FileNotFoundError(f"Access denied: path traversal attempt or file outside workdir '{filepath}'")
sha256 = hashlib.sha256()
with open(filepath, "rb") as f:
with open(filepath_obj, "rb") as f:
while True:
data = f.read(chunk_size)
if not data:
app/utils/settings_service.py
+45 -377
View File
@@ -39,50 +39,6 @@ SETTING_METADATA = {
"required": True,
"restart_required": True,
},
"db_pool_size": {
"category": "Core",
"description": (
"Number of persistent database connections kept in the pool per worker process. "
"Ignored for SQLite (which uses NullPool). Default: 10."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_max_overflow": {
"category": "Core",
"description": (
"Additional database connections allowed beyond db_pool_size under burst load. "
"Ignored for SQLite. Default: 20."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_timeout": {
"category": "Core",
"description": (
"Seconds to wait for a database connection from the pool before raising a TimeoutError. "
"Ignored for SQLite. Default: 30."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_recycle": {
"category": "Core",
"description": (
"Recycle (close and reopen) database connections after this many seconds "
"to avoid stale connections. Ignored for SQLite. Default: 1800."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"workdir": {
"category": "Core",
"description": "Working directory for file storage and processing",
@@ -99,18 +55,6 @@ SETTING_METADATA = {
"required": True, # Required for OAuth redirects and external URLs
"restart_required": True,
},
"public_base_url": {
"category": "Core",
"description": (
"Full public base URL including scheme (e.g., https://docuelevate.example.com). "
"When set, overrides auto-detected URLs for OAuth redirect URIs. "
"Required when behind a reverse proxy that does not forward X-Forwarded-Proto."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"debug": {
"category": "Core",
"description": "Enable debug mode for verbose logging",
@@ -206,14 +150,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"qr_login_enabled": {
"category": "Authentication",
"description": "Enable QR code-based login for mobile device authentication.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"qr_login_challenge_ttl_seconds": {
"category": "Authentication",
"description": "Time-to-live in seconds for QR login challenges (default 120).",
@@ -270,17 +206,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"sso_auto_login": {
"category": "Authentication",
"description": (
"Automatically redirect to SSO login when authentication is required. "
"Skips the login page and sends users directly to the configured SSO provider."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Social Login Providers
"social_auth_google_enabled": {
"category": "Social Login",
@@ -311,20 +236,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"social_auth_google_use_global_credentials": {
"category": "Social Login",
"description": (
"When True, Google social login uses the global GOOGLE_DRIVE_CLIENT_ID / "
"GOOGLE_DRIVE_CLIENT_SECRET credentials (the Google Drive OAuth integration) "
"instead of requiring separate SOCIAL_AUTH_GOOGLE_CLIENT_ID / "
"SOCIAL_AUTH_GOOGLE_CLIENT_SECRET values. "
"Requires SOCIAL_AUTH_GOOGLE_ENABLED=True and global Google Drive OAuth credentials to be set."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_microsoft_enabled": {
"category": "Social Login",
"description": (
@@ -367,20 +278,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"social_auth_microsoft_use_global_credentials": {
"category": "Social Login",
"description": (
"When True, Microsoft social login uses the global ONEDRIVE_CLIENT_ID / "
"ONEDRIVE_CLIENT_SECRET credentials (the OneDrive integration credentials) "
"instead of requiring separate SOCIAL_AUTH_MICROSOFT_CLIENT_ID / "
"SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET values. "
"Requires SOCIAL_AUTH_MICROSOFT_ENABLED=True and global OneDrive credentials to be set."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_apple_enabled": {
"category": "Social Login",
"description": (
@@ -429,19 +326,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"social_auth_dropbox_use_global_credentials": {
"category": "Social Login",
"description": (
"When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET "
"credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / "
"SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. "
"Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_enabled": {
"category": "Social Login",
"description": (
@@ -469,182 +353,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"social_auth_github_enabled": {
"category": "Social Login",
"description": (
"Enable GitHub Sign-In. Requires SOCIAL_AUTH_GITHUB_CLIENT_ID and "
"SOCIAL_AUTH_GITHUB_CLIENT_SECRET from GitHub Developer Settings."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
"help_link": "https://github.com/settings/developers",
"help_link_label": "GitHub Developer Settings",
},
"social_auth_github_client_id": {
"category": "Social Login",
"description": "GitHub OAuth2 client ID from GitHub Developer Settings.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_github_client_secret": {
"category": "Social Login",
"description": "GitHub OAuth2 client secret from GitHub Developer Settings.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
# Keycloak SSO
"social_auth_keycloak_enabled": {
"category": "Social Login",
"description": "Enable Keycloak SSO. Requires server URL, realm, client ID, and client secret.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_keycloak_client_id": {
"category": "Social Login",
"description": "Keycloak OAuth2 client ID.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_keycloak_client_secret": {
"category": "Social Login",
"description": "Keycloak OAuth2 client secret.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_keycloak_server_url": {
"category": "Social Login",
"description": "Keycloak server base URL (e.g. https://keycloak.example.com).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_keycloak_realm": {
"category": "Social Login",
"description": "Keycloak realm name.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
# Generic OAuth2 SSO
"social_auth_generic_oauth2_enabled": {
"category": "Social Login",
"description": "Enable a generic OAuth2 SSO provider.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_client_id": {
"category": "Social Login",
"description": "Generic OAuth2 client ID.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_client_secret": {
"category": "Social Login",
"description": "Generic OAuth2 client secret.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_authorize_url": {
"category": "Social Login",
"description": "Generic OAuth2 authorization URL.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_token_url": {
"category": "Social Login",
"description": "Generic OAuth2 token endpoint URL.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_userinfo_url": {
"category": "Social Login",
"description": "Generic OAuth2 userinfo endpoint URL.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_scope": {
"category": "Social Login",
"description": "Space-separated list of OAuth2 scopes to request (default: openid profile email).",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_generic_oauth2_name": {
"category": "Social Login",
"description": "Display name for the generic OAuth2 provider button on the login page.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
# SAML2 SSO
"social_auth_saml2_enabled": {
"category": "Social Login",
"description": "Enable SAML2 SSO authentication.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_saml2_entity_id": {
"category": "Social Login",
"description": "SAML2 Identity Provider Entity ID.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_saml2_sso_url": {
"category": "Social Login",
"description": "SAML2 Identity Provider SSO URL.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_saml2_certificate": {
"category": "Social Login",
"description": "SAML2 Identity Provider X.509 certificate (PEM format).",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": True,
},
"social_auth_saml2_name": {
"category": "Social Login",
"description": "Display name for the SAML2 provider.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
# AI Services
"openai_api_key": {
"category": "AI Services",
@@ -1011,18 +719,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"dropbox_allow_global_credentials_for_integrations": {
"category": "Storage Providers",
"description": (
"When True, users may authorize their personal Dropbox integrations using the global "
"DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without "
"needing to create their own Dropbox app."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_enabled": {
"category": "Storage Providers",
@@ -2175,30 +1871,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"telegram_enabled": {
"category": "Notifications",
"description": "Enable Telegram bot notifications.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"telegram_bot_token": {
"category": "Notifications",
"description": "Telegram Bot API token from @BotFather.",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"telegram_chat_id": {
"category": "Notifications",
"description": "Telegram chat ID to send notifications to.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Notifications Settings
"notification_urls": {
"category": "Notifications",
@@ -2297,18 +1969,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"automation_hooks_enabled": {
"category": "Feature Flags",
"description": (
"Enable Zapier / Make.com automation hook subscriptions and delivery. "
"When enabled, external automation platforms can subscribe to DocuElevate events "
"via the REST hooks protocol. Default: True."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
"compliance_enabled": {
"category": "Feature Flags",
"description": (
@@ -2897,6 +2557,51 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Database Connection Pool
"db_pool_size": {
"category": "Core",
"description": (
"Number of persistent connections kept in the SQLAlchemy QueuePool. "
"Has no effect for SQLite databases. Default: 5."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_max_overflow": {
"category": "Core",
"description": (
"Maximum extra connections that can be opened beyond db_pool_size. "
"Has no effect for SQLite databases. Default: 10."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_timeout": {
"category": "Core",
"description": (
"Seconds to wait for a connection from the pool before raising an error. "
"Has no effect for SQLite databases. Default: 30."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_recycle": {
"category": "Core",
"description": (
"Seconds after which idle connections are recycled to prevent stale connections. "
"Has no effect for SQLite databases. Default: 1800 (30 minutes)."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
# Per-user upload rate limiting
"upload_rate_limit_per_user": {
"category": "Security",
@@ -3232,43 +2937,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"sentry_js_traces_sample_rate": {
"category": "Observability",
"description": (
"Fraction of browser page-loads captured for client-side Sentry performance tracing (0.0–1.0). "
"0.0 (default) disables browser tracing; 1.0 captures every navigation. "
"Only active when SENTRY_DSN is set."
),
"type": "float",
"sensitive": False,
"required": False,
"restart_required": True,
},
"sentry_js_replay_session_sample_rate": {
"category": "Observability",
"description": (
"Fraction of sessions recorded by Sentry Session Replay (0.0–1.0). "
"0.0 (default) disables session recording; 1.0 records every session. "
"Only active when SENTRY_DSN is set."
),
"type": "float",
"sensitive": False,
"required": False,
"restart_required": True,
},
"sentry_js_replay_on_error_sample_rate": {
"category": "Observability",
"description": (
"Fraction of error sessions recorded by Sentry Session Replay (0.0–1.0). "
"Defaults to 0.1 (10%) so that errors are captured with replay context "
"even when session-level recording is disabled. "
"Only active when SENTRY_DSN is set."
),
"type": "float",
"sensitive": False,
"required": False,
"restart_required": True,
},
}
app/utils/settings_sync.py
-10
View File
@@ -71,16 +71,6 @@ def notify_settings_updated() -> None:
except Exception as exc:
logger.warning(f"Could not reload in-process settings: {exc}")
# Re-register OAuth / social-login providers so that any provider whose
# credentials were just saved (or updated) in the database is active
# immediately on the login page — no restart required.
try:
from app.auth import refresh_social_providers
refresh_social_providers()
except Exception as exc:
logger.warning(f"Could not refresh social login providers after settings update: {exc}")
# Re-check OCR language availability in the background whenever settings
# are updated. This ensures that if a user changes tesseract_language or
# easyocr_languages via the UI, the new language data is downloaded without
app/utils/user_scope.py
+5 -91
View File
@@ -11,21 +11,14 @@ import logging
from fastapi import Request
from sqlalchemy import or_
from sqlalchemy.orm import Query, Session
from sqlalchemy.orm import Query
from sqlalchemy.sql import false
from app.config import settings
from app.models import FILE_SHARE_ROLE_EDITOR, FILE_SHARE_ROLE_VIEWER, FileRecord, FileShare
from app.models import FileRecord
logger = logging.getLogger(__name__)
# Role hierarchy: higher index = more rights
_ROLE_RANK: dict[str, int] = {
FILE_SHARE_ROLE_VIEWER: 1,
FILE_SHARE_ROLE_EDITOR: 2,
"owner": 3,
}
def _owner_id_from_user(user: dict) -> str | None:
"""Extract the owner identifier from a user dict.
@@ -100,9 +93,8 @@ def apply_owner_filter(query: Query, request: Request) -> Query:
"""Conditionally filter a ``FileRecord`` query by the current user.
When multi-user mode is enabled, only files whose ``owner_id``
matches the authenticated user are returned, **plus** any files that
have been explicitly shared with the user via ``FileShare``. Admin
users bypass the filter and see all documents.
matches the authenticated user are returned. Admin users bypass
the filter and see all documents.
When ``unowned_docs_visible_to_all`` is ``True`` (default), documents
with ``owner_id IS NULL`` (unclaimed) are also included for every
@@ -130,89 +122,11 @@ def apply_owner_filter(query: Query, request: Request) -> Query:
# No authenticated user — return empty result set
return query.filter(false())
# Build filter: user's own documents + documents shared with them
# Build filter: user's own documents
conditions = [FileRecord.owner_id == owner_id]
# Include files explicitly shared with this user
from sqlalchemy import select as sa_select
conditions.append(FileRecord.id.in_(sa_select(FileShare.file_id).where(FileShare.shared_with_user_id == owner_id)))
# Optionally include unclaimed (owner_id IS NULL) documents
if settings.unowned_docs_visible_to_all:
conditions.append(FileRecord.owner_id.is_(None))
return query.filter(or_(*conditions))
def get_file_role(file_record: FileRecord, user_id: str | None, db: Session) -> str | None:
"""Return the effective role a user has on a ``FileRecord``.
Roles (in descending order of privilege):
``"owner"`` — the user's ``owner_id`` matches ``file_record.owner_id``,
or multi-user mode is disabled (everyone is effectively an
owner in single-user mode).
``"editor"`` — the user has an explicit ``FileShare`` with role=editor.
``"viewer"`` — the user has an explicit ``FileShare`` with role=viewer,
or the file is unclaimed (``owner_id IS NULL``) and
``unowned_docs_visible_to_all`` is True.
``None`` — no access.
Args:
file_record: The ``FileRecord`` to check.
user_id: The stable identifier of the requesting user.
db: An active SQLAlchemy session.
Returns:
One of ``"owner"``, ``"editor"``, ``"viewer"``, or ``None``.
"""
if not settings.multi_user_enabled:
# Single-user mode: full access for everyone
return "owner"
if user_id is None:
return None
# Owner always has full access
if file_record.owner_id == user_id:
return "owner"
# Unclaimed document — limited access when setting allows it
if file_record.owner_id is None and settings.unowned_docs_visible_to_all:
return FILE_SHARE_ROLE_VIEWER
# Check for an explicit share
share = (
db.query(FileShare)
.filter(FileShare.file_id == file_record.id, FileShare.shared_with_user_id == user_id)
.first()
)
if share:
return share.role
return None
def has_file_role(
file_record: FileRecord,
user_id: str | None,
db: Session,
minimum_role: str = FILE_SHARE_ROLE_VIEWER,
) -> bool:
"""Return ``True`` if the user's effective role meets the minimum required.
Args:
file_record: The document to check.
user_id: Requesting user's stable identifier.
db: Active SQLAlchemy session.
minimum_role: The minimum role required (``"viewer"``, ``"editor"``,
or ``"owner"``).
Returns:
``True`` when the user's role rank is >= the minimum rank.
"""
role = get_file_role(file_record, user_id, db)
if role is None:
return False
return _ROLE_RANK.get(role, 0) >= _ROLE_RANK.get(minimum_role, 0)
app/utils/webhook.py
+10 -19
View File
@@ -145,8 +145,6 @@ def dispatch_webhook_event(event: str, data: dict[str, Any]) -> None:
It delegates to :func:`deliver_webhook_task` (Celery) for each matching
webhook so delivery happens asynchronously with automatic retries.
Also dispatches to automation hooks (Zapier / Make.com) if enabled.
Args:
event: Event name (must be in :data:`VALID_EVENTS`).
data: Event-specific payload data.
@@ -158,23 +156,16 @@ def dispatch_webhook_event(event: str, data: dict[str, Any]) -> None:
webhooks = get_active_webhooks_for_event(event)
if not webhooks:
logger.debug("No active webhooks for event %s", event)
else:
payload = build_payload(event, data)
return
# Import here to avoid circular dependency with celery_app
from app.tasks.webhook_tasks import deliver_webhook_task
payload = build_payload(event, data)
for wh in webhooks:
try:
deliver_webhook_task.delay(wh["url"], payload, wh["secret"])
logger.debug("Queued webhook delivery to %s for event %s", wh["url"], event)
except Exception as exc:
logger.error("Failed to queue webhook to %s: %s", wh["url"], exc)
# Import here to avoid circular dependency with celery_app
from app.tasks.webhook_tasks import deliver_webhook_task
# Also fan-out to Zapier / Make.com automation hooks
try:
from app.utils.automation_hooks import dispatch_automation_hooks
dispatch_automation_hooks(event, data)
except Exception as exc:
logger.error("Failed to dispatch automation hooks for event %s: %s", event, exc)
for wh in webhooks:
try:
deliver_webhook_task.delay(wh["url"], payload, wh["secret"])
logger.debug("Queued webhook delivery to %s for event %s", wh["url"], event)
except Exception as exc:
logger.error("Failed to queue webhook to %s: %s", wh["url"], exc)