fix: merge main, address code review feedback for security fix PR #816
- Merge origin/main into branch (resolve conflict in integrations_dashboard.html) - Add defensive JSON parsing with try/except for integration.config - Wrap tester() call in try/except to prevent 500 errors from bad config - Add i18n key integrations.connection_test_failed_fallback in en.json - Reference i18n key in template JS fallback message - Update SECURITY_AUDIT.md: add fix date (2026-03-23), update doc date - Remove accidental revert.sh file - Fix missing MagicMock/patch imports in test file - Add tests for invalid JSON config and tester exception error paths Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/daebb70e-059a-4601-8864-88eef49f99cf
This commit is contained in:
copilot-swe-agent[bot]
@@ -68,6 +68,8 @@ IMAGE_MIME_TYPES: set[str] = {
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/heic",
|
||||
"image/heif",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -124,6 +126,8 @@ ALLOWED_EXTENSIONS: set[str] = {
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
".heic",
|
||||
".heif",
|
||||
# Web
|
||||
".html",
|
||||
".htm",
|
||||
@@ -234,7 +238,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
},
|
||||
"images": {
|
||||
"label": "Images",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"image/jpeg",
|
||||
@@ -245,6 +249,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/heic",
|
||||
"image/heif",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
@@ -258,6 +264,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
".heic",
|
||||
".heif",
|
||||
}
|
||||
),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Automation hook utilities for Zapier / Make.com integration.
|
||||
|
||||
Provides helpers to build Zapier-compatible flat payloads, query active
|
||||
automation hook subscriptions, and fan-out event delivery to all matching
|
||||
hooks via Celery tasks.
|
||||
|
||||
The payload format is intentionally *flat* (no nested ``data`` key) so that
|
||||
Zapier and Make.com can map fields without JSONPath expressions. An ``id``
|
||||
field is included for Zapier deduplication.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import AutomationHook
|
||||
from app.utils.webhook import VALID_EVENTS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Payload helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_zapier_payload(event: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Build a flat, Zapier-compatible webhook payload.
|
||||
|
||||
Zapier works best with flat JSON objects that include an ``id`` field
|
||||
for deduplication. This function merges event metadata into the
|
||||
top-level object alongside the event-specific *data*.
|
||||
|
||||
Args:
|
||||
event: The event name (e.g. ``document.processed``).
|
||||
data: Event-specific key/value pairs.
|
||||
|
||||
Returns:
|
||||
A flat dictionary suitable for Zapier / Make.com consumption.
|
||||
"""
|
||||
return {
|
||||
"id": f"evt_{uuid.uuid4().hex[:16]}",
|
||||
"event": event,
|
||||
"timestamp": time.time(),
|
||||
**data,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample payloads (used by the /triggers/sample endpoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Example payloads that Zapier uses for field-mapping during Zap creation.
|
||||
SAMPLE_PAYLOADS: dict[str, dict[str, Any]] = {
|
||||
"document.uploaded": {
|
||||
"id": "evt_sample0001",
|
||||
"event": "document.uploaded",
|
||||
"timestamp": 1710000000.0,
|
||||
"document_id": 42,
|
||||
"filename": "invoice_2024.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"size_bytes": 204800,
|
||||
"owner_id": "user@example.com",
|
||||
},
|
||||
"document.processed": {
|
||||
"id": "evt_sample0002",
|
||||
"event": "document.processed",
|
||||
"timestamp": 1710000060.0,
|
||||
"document_id": 42,
|
||||
"filename": "invoice_2024.pdf",
|
||||
"status": "processed",
|
||||
"title": "Invoice #1234",
|
||||
"owner_id": "user@example.com",
|
||||
},
|
||||
"document.failed": {
|
||||
"id": "evt_sample0003",
|
||||
"event": "document.failed",
|
||||
"timestamp": 1710000120.0,
|
||||
"document_id": 42,
|
||||
"filename": "corrupt.pdf",
|
||||
"status": "failed",
|
||||
"error": "Unable to extract text from document",
|
||||
"owner_id": "user@example.com",
|
||||
},
|
||||
"user.signup": {
|
||||
"id": "evt_sample0004",
|
||||
"event": "user.signup",
|
||||
"timestamp": 1710000180.0,
|
||||
"user_id": "newuser@example.com",
|
||||
"display_name": "Jane Doe",
|
||||
},
|
||||
"user.plan_changed": {
|
||||
"id": "evt_sample0005",
|
||||
"event": "user.plan_changed",
|
||||
"timestamp": 1710000240.0,
|
||||
"user_id": "user@example.com",
|
||||
"old_tier": "free",
|
||||
"new_tier": "pro",
|
||||
},
|
||||
"user.payment_issue": {
|
||||
"id": "evt_sample0006",
|
||||
"event": "user.payment_issue",
|
||||
"timestamp": 1710000300.0,
|
||||
"user_id": "user@example.com",
|
||||
"issue": "Credit card declined",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database queries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_active_hooks_for_event(event: str) -> list[dict[str, Any]]:
|
||||
"""Return all active automation hooks subscribed to *event*.
|
||||
|
||||
Args:
|
||||
event: The event name to filter on.
|
||||
|
||||
Returns:
|
||||
A list of dicts with ``id``, ``target_url``, ``secret``, and
|
||||
``events`` keys.
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(AutomationHook).filter(AutomationHook.is_active.is_(True)).all()
|
||||
result: list[dict[str, Any]] = []
|
||||
for hook in hooks:
|
||||
try:
|
||||
subscribed = json.loads(hook.events)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
subscribed = []
|
||||
if event in subscribed:
|
||||
result.append(
|
||||
{
|
||||
"id": hook.id,
|
||||
"target_url": hook.target_url,
|
||||
"secret": hook.secret,
|
||||
"events": subscribed,
|
||||
}
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dispatch_automation_hooks(event: str, data: dict[str, Any]) -> None:
|
||||
"""Fan-out an event to all matching active automation hooks.
|
||||
|
||||
Builds a Zapier-compatible flat payload and queues a Celery task for
|
||||
each matching hook so delivery is asynchronous with automatic retries.
|
||||
|
||||
Args:
|
||||
event: Event name (must be in :data:`VALID_EVENTS`).
|
||||
data: Event-specific payload data.
|
||||
"""
|
||||
if not settings.automation_hooks_enabled:
|
||||
return
|
||||
|
||||
if event not in VALID_EVENTS:
|
||||
logger.warning("Ignoring unknown automation hook event: %s", event)
|
||||
return
|
||||
|
||||
hooks = get_active_hooks_for_event(event)
|
||||
if not hooks:
|
||||
logger.debug("No active automation hooks for event %s", event)
|
||||
return
|
||||
|
||||
payload = build_zapier_payload(event, data)
|
||||
|
||||
from app.tasks.automation_tasks import deliver_automation_hook_task
|
||||
|
||||
for hook in hooks:
|
||||
try:
|
||||
deliver_automation_hook_task.delay(hook["target_url"], payload, hook["secret"])
|
||||
logger.debug("Queued automation hook delivery to %s for event %s", hook["target_url"], event)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to queue automation hook to %s: %s", hook["target_url"], exc)
|
||||
@@ -0,0 +1,378 @@
|
||||
"""
|
||||
Rule-based document classification engine.
|
||||
|
||||
Provides pre-built categories and a rule matcher that classifies documents
|
||||
using filename patterns, content keywords, and metadata fields. Custom
|
||||
rules stored in the database are evaluated alongside the built-in defaults.
|
||||
|
||||
Usage::
|
||||
|
||||
from app.utils.classification_rules import classify_document
|
||||
|
||||
result = classify_document(
|
||||
filename="2024-03-01_Invoice_Acme.pdf",
|
||||
text="Invoice total: $1,234.56",
|
||||
metadata={"absender": "Acme Corp"},
|
||||
custom_rules=custom_rules_from_db,
|
||||
)
|
||||
# result -> ClassificationResult(category="invoice", confidence=85, matched_rules=[...])
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-built categories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Canonical category names recognized by the system. Users may also define
|
||||
#: their own categories via custom rules.
|
||||
BUILTIN_CATEGORIES: dict[str, str] = {
|
||||
"invoice": "Invoice",
|
||||
"contract": "Contract",
|
||||
"receipt": "Receipt",
|
||||
"letter": "Letter",
|
||||
"report": "Report",
|
||||
"bank_statement": "Bank Statement",
|
||||
"tax_document": "Tax Document",
|
||||
"insurance": "Insurance Document",
|
||||
"payslip": "Payslip",
|
||||
"unknown": "Unknown",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule type constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RULE_TYPE_FILENAME = "filename_pattern"
|
||||
RULE_TYPE_CONTENT = "content_keyword"
|
||||
RULE_TYPE_METADATA = "metadata_match"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationRule:
|
||||
"""A single classification rule."""
|
||||
|
||||
name: str
|
||||
category: str
|
||||
rule_type: str # filename_pattern | content_keyword | metadata_match
|
||||
pattern: str # regex for filename, keyword(s) for content, "field=value" for metadata
|
||||
priority: int = 0 # higher = evaluated first
|
||||
case_sensitive: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.rule_type not in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA):
|
||||
raise ValueError(f"Invalid rule_type: {self.rule_type!r}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatchedRule:
|
||||
"""Records which rule matched and why."""
|
||||
|
||||
rule_name: str
|
||||
rule_type: str
|
||||
category: str
|
||||
confidence: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
"""The outcome of running the classification engine on a document."""
|
||||
|
||||
category: str
|
||||
confidence: int # 0 – 100
|
||||
matched_rules: list[MatchedRule] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BUILTIN_RULES: list[ClassificationRule] = [
|
||||
# ── Invoice ───────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_invoice_filename", "invoice", RULE_TYPE_FILENAME, r"(?i)invoice|rechnung|facture"),
|
||||
ClassificationRule(
|
||||
"builtin_invoice_content",
|
||||
"invoice",
|
||||
RULE_TYPE_CONTENT,
|
||||
"invoice number|invoice total|amount due|rechnung|rechnungsnummer|total amount|bill to",
|
||||
),
|
||||
ClassificationRule("builtin_invoice_metadata", "invoice", RULE_TYPE_METADATA, "document_type=Invoice"),
|
||||
ClassificationRule(
|
||||
"builtin_invoice_kommunikationsart", "invoice", RULE_TYPE_METADATA, "kommunikationsart=Rechnung"
|
||||
),
|
||||
# ── Contract ──────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_contract_filename", "contract", RULE_TYPE_FILENAME, r"(?i)contract|vertrag|agreement"),
|
||||
ClassificationRule(
|
||||
"builtin_contract_content",
|
||||
"contract",
|
||||
RULE_TYPE_CONTENT,
|
||||
"hereby agrees|terms and conditions|vertrag|agreement between|party agrees|effective date",
|
||||
),
|
||||
ClassificationRule("builtin_contract_metadata", "contract", RULE_TYPE_METADATA, "document_type=Contract"),
|
||||
ClassificationRule(
|
||||
"builtin_contract_kommunikationsart", "contract", RULE_TYPE_METADATA, "kommunikationsart=Vertrag"
|
||||
),
|
||||
# ── Receipt ───────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_receipt_filename", "receipt", RULE_TYPE_FILENAME, r"(?i)receipt|quittung|beleg"),
|
||||
ClassificationRule(
|
||||
"builtin_receipt_content",
|
||||
"receipt",
|
||||
RULE_TYPE_CONTENT,
|
||||
"receipt|quittung|payment received|thank you for your purchase|transaction id",
|
||||
),
|
||||
ClassificationRule("builtin_receipt_metadata", "receipt", RULE_TYPE_METADATA, "document_type=Receipt"),
|
||||
ClassificationRule(
|
||||
"builtin_receipt_kommunikationsart", "receipt", RULE_TYPE_METADATA, "kommunikationsart=Quittung"
|
||||
),
|
||||
# ── Letter ────────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_letter_filename", "letter", RULE_TYPE_FILENAME, r"(?i)letter|brief|schreiben"),
|
||||
ClassificationRule(
|
||||
"builtin_letter_content",
|
||||
"letter",
|
||||
RULE_TYPE_CONTENT,
|
||||
"dear sir|dear madam|sehr geehrte|to whom it may concern|sincerely|mit freundlichen",
|
||||
),
|
||||
# ── Report ────────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_report_filename", "report", RULE_TYPE_FILENAME, r"(?i)report|bericht"),
|
||||
ClassificationRule(
|
||||
"builtin_report_content",
|
||||
"report",
|
||||
RULE_TYPE_CONTENT,
|
||||
"executive summary|table of contents|annual report|quarterly report|findings",
|
||||
),
|
||||
# ── Bank statement ────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_bank_filename",
|
||||
"bank_statement",
|
||||
RULE_TYPE_FILENAME,
|
||||
r"(?i)bank.?statement|kontoauszug",
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_bank_content",
|
||||
"bank_statement",
|
||||
RULE_TYPE_CONTENT,
|
||||
"account statement|kontoauszug|opening balance|closing balance|account number",
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_bank_kommunikationsart", "bank_statement", RULE_TYPE_METADATA, "kommunikationsart=Kontoauszug"
|
||||
),
|
||||
# ── Tax document ──────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_tax_filename", "tax_document", RULE_TYPE_FILENAME, r"(?i)tax|steuer|steuerbescheid"),
|
||||
ClassificationRule(
|
||||
"builtin_tax_content",
|
||||
"tax_document",
|
||||
RULE_TYPE_CONTENT,
|
||||
"tax return|steuerbescheid|taxable income|finanzamt|tax assessment",
|
||||
),
|
||||
# ── Insurance ─────────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_insurance_filename", "insurance", RULE_TYPE_FILENAME, r"(?i)insurance|versicherung|police"
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_insurance_content",
|
||||
"insurance",
|
||||
RULE_TYPE_CONTENT,
|
||||
"insurance policy|versicherung|policennummer|coverage|premium|deductible",
|
||||
),
|
||||
# ── Payslip ───────────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_payslip_filename", "payslip", RULE_TYPE_FILENAME, r"(?i)payslip|gehaltsabrechnung|lohnabrechnung"
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_payslip_content",
|
||||
"payslip",
|
||||
RULE_TYPE_CONTENT,
|
||||
"gross salary|net salary|gehaltsabrechnung|lohnabrechnung|bruttolohn|nettolohn",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confidence scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Base confidence for each rule type when it matches.
|
||||
_CONFIDENCE_MAP: dict[str, int] = {
|
||||
RULE_TYPE_FILENAME: 60,
|
||||
RULE_TYPE_CONTENT: 70,
|
||||
RULE_TYPE_METADATA: 90,
|
||||
}
|
||||
|
||||
#: Extra confidence per additional matching rule of the same category (capped).
|
||||
_CONFIDENCE_BONUS_PER_EXTRA_RULE = 10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _match_filename(rule: ClassificationRule, filename: str) -> bool:
|
||||
"""Return True if *rule.pattern* (regex) matches anywhere in *filename*."""
|
||||
if not filename:
|
||||
return False
|
||||
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
||||
return bool(re.search(rule.pattern, filename, flags))
|
||||
|
||||
|
||||
def _match_content(rule: ClassificationRule, text: str) -> bool:
|
||||
"""Return True if any keyword in *rule.pattern* appears in *text*.
|
||||
|
||||
Keywords are separated by ``|`` (pipe).
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
keywords = [kw.strip() for kw in rule.pattern.split("|") if kw.strip()]
|
||||
text_lower = text if rule.case_sensitive else text.lower()
|
||||
return any((kw if rule.case_sensitive else kw.lower()) in text_lower for kw in keywords)
|
||||
|
||||
|
||||
def _match_metadata(rule: ClassificationRule, metadata: dict[str, Any] | None) -> bool:
|
||||
"""Return True if *rule.pattern* (``field=value``) matches *metadata*.
|
||||
|
||||
Pattern format: ``field_name=expected_value``.
|
||||
"""
|
||||
if not metadata:
|
||||
return False
|
||||
if "=" not in rule.pattern:
|
||||
return False
|
||||
field_name, expected_value = rule.pattern.split("=", 1)
|
||||
actual = metadata.get(field_name.strip())
|
||||
if actual is None:
|
||||
return False
|
||||
if rule.case_sensitive:
|
||||
return str(actual) == expected_value.strip()
|
||||
return str(actual).lower() == expected_value.strip().lower()
|
||||
|
||||
|
||||
_MATCHERS: dict[str, tuple] = {
|
||||
RULE_TYPE_FILENAME: (_match_filename, "filename"),
|
||||
RULE_TYPE_CONTENT: (_match_content, "text"),
|
||||
RULE_TYPE_METADATA: (_match_metadata, "metadata"),
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_rule(
|
||||
rule: ClassificationRule,
|
||||
filename: str,
|
||||
text: str,
|
||||
metadata: dict[str, Any] | None,
|
||||
) -> MatchedRule | None:
|
||||
"""Evaluate a single rule against the document. Return a :class:`MatchedRule` on match."""
|
||||
entry = _MATCHERS.get(rule.rule_type)
|
||||
if entry is None:
|
||||
return None
|
||||
|
||||
matcher, arg_key = entry
|
||||
arg_map = {"filename": filename, "text": text, "metadata": metadata}
|
||||
matched = matcher(rule, arg_map[arg_key])
|
||||
|
||||
if matched:
|
||||
return MatchedRule(
|
||||
rule_name=rule.name,
|
||||
rule_type=rule.rule_type,
|
||||
category=rule.category,
|
||||
confidence=_CONFIDENCE_MAP.get(rule.rule_type, 50),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def classify_document(
|
||||
filename: str = "",
|
||||
text: str = "",
|
||||
metadata: dict[str, Any] | None = None,
|
||||
custom_rules: list[ClassificationRule] | None = None,
|
||||
) -> ClassificationResult:
|
||||
"""Classify a document by evaluating built-in and custom rules.
|
||||
|
||||
Rules are evaluated in priority order (highest first, then built-in before
|
||||
custom for the same priority). The category with the most rule matches
|
||||
wins; ties are broken by cumulative confidence.
|
||||
|
||||
Args:
|
||||
filename: Original filename of the document.
|
||||
text: Extracted / OCR text of the document.
|
||||
metadata: Previously-extracted AI metadata dict (e.g. from ``ai_metadata``).
|
||||
custom_rules: Optional list of user-defined :class:`ClassificationRule` objects.
|
||||
|
||||
Returns:
|
||||
A :class:`ClassificationResult` with the best matching category,
|
||||
overall confidence score, and the list of rules that fired.
|
||||
"""
|
||||
all_rules = list(BUILTIN_RULES)
|
||||
if custom_rules:
|
||||
all_rules.extend(custom_rules)
|
||||
|
||||
# Sort by priority descending (higher priority first)
|
||||
all_rules.sort(key=lambda r: r.priority, reverse=True)
|
||||
|
||||
matches: list[MatchedRule] = []
|
||||
for rule in all_rules:
|
||||
result = _evaluate_rule(rule, filename, text, metadata)
|
||||
if result is not None:
|
||||
matches.append(result)
|
||||
|
||||
if not matches:
|
||||
return ClassificationResult(category="unknown", confidence=0, matched_rules=[])
|
||||
|
||||
# Aggregate by category: pick the one with the most matches, then highest
|
||||
# cumulative confidence as tiebreaker.
|
||||
category_scores: dict[str, list[MatchedRule]] = {}
|
||||
for m in matches:
|
||||
category_scores.setdefault(m.category, []).append(m)
|
||||
|
||||
best_category = max(
|
||||
category_scores,
|
||||
key=lambda cat: (len(category_scores[cat]), sum(m.confidence for m in category_scores[cat])),
|
||||
)
|
||||
|
||||
best_matches = category_scores[best_category]
|
||||
base_confidence = max(m.confidence for m in best_matches)
|
||||
bonus = min(
|
||||
(len(best_matches) - 1) * _CONFIDENCE_BONUS_PER_EXTRA_RULE,
|
||||
100 - base_confidence,
|
||||
)
|
||||
final_confidence = min(base_confidence + bonus, 100)
|
||||
|
||||
return ClassificationResult(
|
||||
category=best_category,
|
||||
confidence=final_confidence,
|
||||
matched_rules=best_matches,
|
||||
)
|
||||
|
||||
|
||||
def db_rule_to_engine_rule(db_rule: Any) -> ClassificationRule:
|
||||
"""Convert a database ``ClassificationRuleModel`` row to an engine :class:`ClassificationRule`.
|
||||
|
||||
Args:
|
||||
db_rule: A SQLAlchemy model instance with ``name``, ``category``,
|
||||
``rule_type``, ``pattern``, ``priority``, and ``case_sensitive`` attributes.
|
||||
|
||||
Returns:
|
||||
A :class:`ClassificationRule` dataclass instance.
|
||||
"""
|
||||
return ClassificationRule(
|
||||
name=db_rule.name,
|
||||
category=db_rule.category,
|
||||
rule_type=db_rule.rule_type,
|
||||
pattern=db_rule.pattern,
|
||||
priority=db_rule.priority,
|
||||
case_sensitive=getattr(db_rule, "case_sensitive", False),
|
||||
)
|
||||
@@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]:
|
||||
},
|
||||
}
|
||||
|
||||
# Check SharePoint configuration
|
||||
providers["SharePoint"] = {
|
||||
"name": "SharePoint",
|
||||
"icon": "fa-brands fa-microsoft",
|
||||
"configured": bool(
|
||||
getattr(settings, "sharepoint_client_id", None)
|
||||
and getattr(settings, "sharepoint_client_secret", None)
|
||||
and getattr(settings, "sharepoint_site_url", None)
|
||||
),
|
||||
"enabled": True,
|
||||
"description": "Store documents in Microsoft SharePoint Online",
|
||||
"details": {
|
||||
"client_id": getattr(settings, "sharepoint_client_id", "Not set"),
|
||||
"client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)),
|
||||
"tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"),
|
||||
"refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)),
|
||||
"site_url": getattr(settings, "sharepoint_site_url", "Not set"),
|
||||
"document_library": getattr(settings, "sharepoint_document_library", "Not set"),
|
||||
"folder_path": getattr(settings, "sharepoint_folder_path", "Not set"),
|
||||
},
|
||||
}
|
||||
|
||||
# Check S3 configuration
|
||||
providers["S3 Storage"] = {
|
||||
"name": "S3 Storage",
|
||||
|
||||
@@ -12,6 +12,7 @@ The utility:
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import MetaData, create_engine, inspect, text
|
||||
@@ -84,6 +85,9 @@ def preview_migration(source_url: str) -> dict[str, Any]:
|
||||
total = 0
|
||||
with src_engine.connect() as conn:
|
||||
for table_name in tables:
|
||||
if not re.match(r"^[a-zA-Z0-9_]+$", table_name):
|
||||
logger.warning(f"Skipping table with invalid name format: {table_name}")
|
||||
continue
|
||||
# table_name is safe — sourced from inspect().get_table_names(), not user input
|
||||
quoted_table = conn.dialect.identifier_preparer.quote(table_name)
|
||||
row = conn.execute(text(f"SELECT COUNT(*) FROM {quoted_table}")).fetchone() # noqa: S608
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def update_env_file(settings_to_update: Dict[str, str]) -> bool:
|
||||
"""
|
||||
Updates the .env file with the given settings (best-effort).
|
||||
Creates or modifies existing keys.
|
||||
|
||||
Args:
|
||||
settings_to_update: A dictionary mapping uppercase env var names to their new string values.
|
||||
|
||||
Returns:
|
||||
True if the file was successfully updated, False otherwise.
|
||||
"""
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if not os.path.exists(env_path):
|
||||
logger.warning(f".env file not found at {env_path}, skipping file write")
|
||||
return False
|
||||
|
||||
logger.info(f"Updating settings in {env_path}")
|
||||
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in settings_to_update.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
for key, value in settings_to_update.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated settings in .env file")
|
||||
return True
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
return False
|
||||
@@ -7,8 +7,19 @@ def hash_file(filepath: str | Path, chunk_size: int = 65536) -> str:
|
||||
Returns the SHA-256 hash of the file at 'filepath'.
|
||||
Reads the file in chunks to handle large files efficiently.
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
filepath_obj = Path(filepath).resolve()
|
||||
workdir_obj = Path(settings.workdir).resolve()
|
||||
|
||||
# Security check: Ensure the resolved path is strictly within the allowed workdir
|
||||
try:
|
||||
filepath_obj.relative_to(workdir_obj)
|
||||
except ValueError:
|
||||
raise FileNotFoundError(f"Access denied: path traversal attempt or file outside workdir '{filepath}'")
|
||||
|
||||
sha256 = hashlib.sha256()
|
||||
with open(filepath, "rb") as f:
|
||||
with open(filepath_obj, "rb") as f:
|
||||
while True:
|
||||
data = f.read(chunk_size)
|
||||
if not data:
|
||||
|
||||
+79
-77
@@ -34,89 +34,91 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_LANGUAGES: list[dict[str, str]] = [
|
||||
# --- Tier 1: Primary European languages ---
|
||||
{"code": "en", "name": "English", "native": "English", "flag": "🇬🇧"},
|
||||
{"code": "de", "name": "German", "native": "Deutsch", "flag": "🇩🇪"},
|
||||
{"code": "fr", "name": "French", "native": "Français", "flag": "🇫🇷"},
|
||||
{"code": "es", "name": "Spanish", "native": "Español", "flag": "🇪🇸"},
|
||||
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "🇮🇹"},
|
||||
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "🇵🇹"},
|
||||
# flag: lowercase ISO 3166-1 alpha-2 country code used with the flag-icons CSS library
|
||||
# (e.g. "gb" → <span class="fi fi-gb">). Regional codes like "gb-wls" are also supported.
|
||||
{"code": "en", "name": "English", "native": "English", "flag": "gb"},
|
||||
{"code": "de", "name": "German", "native": "Deutsch", "flag": "de"},
|
||||
{"code": "fr", "name": "French", "native": "Français", "flag": "fr"},
|
||||
{"code": "es", "name": "Spanish", "native": "Español", "flag": "es"},
|
||||
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "it"},
|
||||
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "pt"},
|
||||
# --- Tier 2: Western & Northern European ---
|
||||
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "🇳🇱"},
|
||||
{"code": "nb", "name": "Norwegian Bokmål", "native": "Norsk bokmål", "flag": "🇳🇴"},
|
||||
{"code": "no", "name": "Norwegian", "native": "Norsk", "flag": "🇳🇴"},
|
||||
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "🇩🇰"},
|
||||
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "🇸🇪"},
|
||||
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "🇫🇮"},
|
||||
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "🇮🇸"},
|
||||
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "🇮🇪"},
|
||||
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "🇱🇺"},
|
||||
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "🏴"},
|
||||
{"code": "cy", "name": "Welsh", "native": "Cymraeg", "flag": "🏴"}, # Wales subdivision flag (U+1F3F4 + tag chars)
|
||||
{"code": "fy", "name": "Western Frisian", "native": "Frysk", "flag": "🇳🇱"},
|
||||
{"code": "gl", "name": "Galician", "native": "Galego", "flag": "🇪🇸"},
|
||||
{"code": "li", "name": "Limburgish", "native": "Limburgs", "flag": "🇳🇱"},
|
||||
{"code": "vls", "name": "Flemish", "native": "West-Vlams", "flag": "🇧🇪"},
|
||||
{"code": "nds", "name": "Low German", "native": "Plattdüütsch", "flag": "🇩🇪"},
|
||||
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "nl"},
|
||||
{"code": "nb", "name": "Norwegian Bokmål", "native": "Norsk bokmål", "flag": "no"},
|
||||
{"code": "no", "name": "Norwegian", "native": "Norsk", "flag": "no"},
|
||||
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "dk"},
|
||||
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "se"},
|
||||
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "fi"},
|
||||
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "is"},
|
||||
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "ie"},
|
||||
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "lu"},
|
||||
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "es"}, # no dedicated ISO flag; use Spain
|
||||
{"code": "cy", "name": "Welsh", "native": "Cymraeg", "flag": "gb-wls"}, # flag-icons GB region code
|
||||
{"code": "fy", "name": "Western Frisian", "native": "Frysk", "flag": "nl"},
|
||||
{"code": "gl", "name": "Galician", "native": "Galego", "flag": "es"},
|
||||
{"code": "li", "name": "Limburgish", "native": "Limburgs", "flag": "nl"},
|
||||
{"code": "vls", "name": "Flemish", "native": "West-Vlams", "flag": "be"},
|
||||
{"code": "nds", "name": "Low German", "native": "Plattdüütsch", "flag": "de"},
|
||||
# --- Tier 3: Central & Eastern European ---
|
||||
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "🇵🇱"},
|
||||
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "🇨🇿"},
|
||||
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "🇸🇰"},
|
||||
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "🇭🇺"},
|
||||
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "🇸🇮"},
|
||||
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "🇭🇷"},
|
||||
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "🇷🇴"},
|
||||
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "🇧🇬"},
|
||||
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "🇬🇷"},
|
||||
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "🇪🇪"},
|
||||
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "🇱🇻"},
|
||||
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "🇱🇹"},
|
||||
{"code": "sr", "name": "Serbian", "native": "Српски", "flag": "🇷🇸"},
|
||||
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "pl"},
|
||||
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "cz"},
|
||||
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "sk"},
|
||||
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "hu"},
|
||||
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "si"},
|
||||
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "hr"},
|
||||
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "ro"},
|
||||
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "bg"},
|
||||
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "gr"},
|
||||
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "ee"},
|
||||
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "lv"},
|
||||
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "lt"},
|
||||
{"code": "sr", "name": "Serbian", "native": "Српски", "flag": "rs"},
|
||||
# --- Tier 4: Non-EU European, Middle Eastern & African ---
|
||||
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "🇹🇷"},
|
||||
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "🇺🇦"},
|
||||
{"code": "he", "name": "Hebrew", "native": "עברית", "flag": "🇮🇱"},
|
||||
{"code": "ar", "name": "Arabic", "native": "العربية", "flag": "🇸🇦"},
|
||||
{"code": "fa", "name": "Persian", "native": "فارسی", "flag": "🇮🇷"},
|
||||
{"code": "af", "name": "Afrikaans", "native": "Afrikaans", "flag": "🇿🇦"},
|
||||
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "tr"},
|
||||
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "ua"},
|
||||
{"code": "he", "name": "Hebrew", "native": "עברית", "flag": "il"},
|
||||
{"code": "ar", "name": "Arabic", "native": "العربية", "flag": "sa"},
|
||||
{"code": "fa", "name": "Persian", "native": "فارسی", "flag": "ir"},
|
||||
{"code": "af", "name": "Afrikaans", "native": "Afrikaans", "flag": "za"},
|
||||
# --- Tier 5: Asian languages ---
|
||||
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "🇨🇳"},
|
||||
{"code": "zh-TW", "name": "Traditional Chinese", "native": "繁體中文", "flag": "🇹🇼"},
|
||||
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "🇯🇵"},
|
||||
{"code": "ko", "name": "Korean", "native": "한국어", "flag": "🇰🇷"},
|
||||
{"code": "vi", "name": "Vietnamese", "native": "Tiếng Việt", "flag": "🇻🇳"},
|
||||
{"code": "pa", "name": "Punjabi", "native": "ਪੰਜਾਬੀ", "flag": "🇮🇳"},
|
||||
{"code": "kn", "name": "Kannada", "native": "ಕನ್ನಡ", "flag": "🇮🇳"},
|
||||
{"code": "hi", "name": "Hindi", "native": "हिन्दी", "flag": "🇮🇳"},
|
||||
{"code": "bn", "name": "Bengali", "native": "বাংলা", "flag": "🇧🇩"},
|
||||
{"code": "gu", "name": "Gujarati", "native": "ગુજરાતી", "flag": "🇮🇳"},
|
||||
{"code": "ml", "name": "Malayalam", "native": "മലയാളം", "flag": "🇮🇳"},
|
||||
{"code": "mr", "name": "Marathi", "native": "मराठी", "flag": "🇮🇳"},
|
||||
{"code": "ta", "name": "Tamil", "native": "தமிழ்", "flag": "🇮🇳"},
|
||||
{"code": "te", "name": "Telugu", "native": "తెలుగు", "flag": "🇮🇳"},
|
||||
{"code": "ur", "name": "Urdu", "native": "اردو", "flag": "🇵🇰"},
|
||||
{"code": "si", "name": "Sinhala", "native": "සිංහල", "flag": "🇱🇰"},
|
||||
{"code": "ne", "name": "Nepali", "native": "नेपाली", "flag": "🇳🇵"},
|
||||
{"code": "th", "name": "Thai", "native": "ไทย", "flag": "🇹🇭"},
|
||||
{"code": "km", "name": "Khmer", "native": "ខ្មែរ", "flag": "🇰🇭"},
|
||||
{"code": "id", "name": "Indonesian", "native": "Bahasa Indonesia", "flag": "🇮🇩"},
|
||||
{"code": "ms", "name": "Malay", "native": "Bahasa Melayu", "flag": "🇲🇾"},
|
||||
{"code": "jv", "name": "Javanese", "native": "Basa Jawa", "flag": "🇮🇩"},
|
||||
{"code": "tl", "name": "Tagalog", "native": "Filipino", "flag": "🇵🇭"},
|
||||
{"code": "mn", "name": "Mongolian", "native": "Монгол", "flag": "🇲🇳"},
|
||||
{"code": "kk", "name": "Kazakh", "native": "Қазақ тілі", "flag": "🇰🇿"},
|
||||
{"code": "uz", "name": "Uzbek", "native": "Oʻzbekcha", "flag": "🇺🇿"},
|
||||
{"code": "az", "name": "Azerbaijani", "native": "Azərbaycan dili", "flag": "🇦🇿"},
|
||||
{"code": "hy", "name": "Armenian", "native": "Հայերեն", "flag": "🇦🇲"},
|
||||
{"code": "ka", "name": "Georgian", "native": "ქართული", "flag": "🇬🇪"},
|
||||
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "cn"},
|
||||
{"code": "zh-TW", "name": "Traditional Chinese", "native": "繁體中文", "flag": "tw"},
|
||||
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "jp"},
|
||||
{"code": "ko", "name": "Korean", "native": "한국어", "flag": "kr"},
|
||||
{"code": "vi", "name": "Vietnamese", "native": "Tiếng Việt", "flag": "vn"},
|
||||
{"code": "pa", "name": "Punjabi", "native": "ਪੰਜਾਬੀ", "flag": "in"},
|
||||
{"code": "kn", "name": "Kannada", "native": "ಕನ್ನಡ", "flag": "in"},
|
||||
{"code": "hi", "name": "Hindi", "native": "हिन्दी", "flag": "in"},
|
||||
{"code": "bn", "name": "Bengali", "native": "বাংলা", "flag": "bd"},
|
||||
{"code": "gu", "name": "Gujarati", "native": "ગુજરાતી", "flag": "in"},
|
||||
{"code": "ml", "name": "Malayalam", "native": "മലയാളം", "flag": "in"},
|
||||
{"code": "mr", "name": "Marathi", "native": "मराठी", "flag": "in"},
|
||||
{"code": "ta", "name": "Tamil", "native": "தமிழ்", "flag": "in"},
|
||||
{"code": "te", "name": "Telugu", "native": "తెలుగు", "flag": "in"},
|
||||
{"code": "ur", "name": "Urdu", "native": "اردو", "flag": "pk"},
|
||||
{"code": "si", "name": "Sinhala", "native": "සිංහල", "flag": "lk"},
|
||||
{"code": "ne", "name": "Nepali", "native": "नेपाली", "flag": "np"},
|
||||
{"code": "th", "name": "Thai", "native": "ไทย", "flag": "th"},
|
||||
{"code": "km", "name": "Khmer", "native": "ខ្មែរ", "flag": "kh"},
|
||||
{"code": "id", "name": "Indonesian", "native": "Bahasa Indonesia", "flag": "id"},
|
||||
{"code": "ms", "name": "Malay", "native": "Bahasa Melayu", "flag": "my"},
|
||||
{"code": "jv", "name": "Javanese", "native": "Basa Jawa", "flag": "id"},
|
||||
{"code": "tl", "name": "Tagalog", "native": "Filipino", "flag": "ph"},
|
||||
{"code": "mn", "name": "Mongolian", "native": "Монгол", "flag": "mn"},
|
||||
{"code": "kk", "name": "Kazakh", "native": "Қазақ тілі", "flag": "kz"},
|
||||
{"code": "uz", "name": "Uzbek", "native": "Oʻzbekcha", "flag": "uz"},
|
||||
{"code": "az", "name": "Azerbaijani", "native": "Azərbaycan dili", "flag": "az"},
|
||||
{"code": "hy", "name": "Armenian", "native": "Հայերեն", "flag": "am"},
|
||||
{"code": "ka", "name": "Georgian", "native": "ქართული", "flag": "ge"},
|
||||
# --- Tier 6: African languages ---
|
||||
{"code": "sw", "name": "Swahili", "native": "Kiswahili", "flag": "🇰🇪"},
|
||||
{"code": "am", "name": "Amharic", "native": "አማርኛ", "flag": "🇪🇹"},
|
||||
{"code": "ha", "name": "Hausa", "native": "Hausa", "flag": "🇳🇬"},
|
||||
{"code": "yo", "name": "Yoruba", "native": "Yorùbá", "flag": "🇳🇬"},
|
||||
{"code": "ig", "name": "Igbo", "native": "Igbo", "flag": "🇳🇬"},
|
||||
{"code": "zu", "name": "Zulu", "native": "isiZulu", "flag": "🇿🇦"},
|
||||
{"code": "sw", "name": "Swahili", "native": "Kiswahili", "flag": "ke"},
|
||||
{"code": "am", "name": "Amharic", "native": "አማርኛ", "flag": "et"},
|
||||
{"code": "ha", "name": "Hausa", "native": "Hausa", "flag": "ng"},
|
||||
{"code": "yo", "name": "Yoruba", "native": "Yorùbá", "flag": "ng"},
|
||||
{"code": "ig", "name": "Igbo", "native": "Igbo", "flag": "ng"},
|
||||
{"code": "zu", "name": "Zulu", "native": "isiZulu", "flag": "za"},
|
||||
# --- Tier 7: Constructed & other languages ---
|
||||
{"code": "eo", "name": "Esperanto", "native": "Esperanto", "flag": "🌍"},
|
||||
{"code": "eo", "name": "Esperanto", "native": "Esperanto", "flag": "un"}, # UN flag for international language
|
||||
]
|
||||
|
||||
SUPPORTED_LANGUAGE_CODES: set[str] = {lang["code"] for lang in SUPPORTED_LANGUAGES}
|
||||
|
||||
@@ -27,8 +27,8 @@ def is_private_ip(hostname: str) -> bool:
|
||||
return True
|
||||
return False
|
||||
except (socket.gaierror, socket.error):
|
||||
# Cannot resolve - allow for testing/development
|
||||
# In production, DNS should work properly
|
||||
# Log this for debugging
|
||||
logger.warning(f"Could not resolve hostname: {hostname}")
|
||||
return False # Changed from True to False to allow external domains in tests
|
||||
# Cannot resolve.
|
||||
# Fail securely: block unresolved domains to prevent DNS rebinding
|
||||
# and SSRF bypasses via unresolvable addresses.
|
||||
logger.warning(f"Could not resolve hostname (blocking securely): {hostname}")
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
"""Server-side session management utilities.
|
||||
|
||||
Provides helpers for creating, validating, and revoking user sessions.
|
||||
Sessions are tracked in the ``user_sessions`` table and referenced by a
|
||||
cryptographically random token stored in the browser cookie. This enables
|
||||
the "log off everywhere" feature and per-session revocation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models import ApiToken, QRLoginChallenge, UserSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_tz_aware(dt: datetime | None) -> datetime | None:
|
||||
"""Return *dt* with UTC tzinfo if it is naive, or unchanged if already aware.
|
||||
|
||||
SQLite does not persist timezone information, so datetimes read back from
|
||||
the database are offset-naive. This helper normalises them for safe
|
||||
comparison with ``datetime.now(timezone.utc)``.
|
||||
"""
|
||||
if dt is not None and dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def get_session_lifetime_days() -> int:
|
||||
"""Return the effective session lifetime in days.
|
||||
|
||||
If ``session_lifetime_custom_days`` is set it takes precedence over
|
||||
``session_lifetime_days``.
|
||||
"""
|
||||
custom = getattr(settings, "session_lifetime_custom_days", None)
|
||||
if custom is not None and isinstance(custom, int) and custom > 0:
|
||||
return custom
|
||||
return max(1, getattr(settings, "session_lifetime_days", 30))
|
||||
|
||||
|
||||
def get_session_max_age_seconds() -> int:
|
||||
"""Return the session max-age in seconds for the cookie."""
|
||||
return get_session_lifetime_days() * 86400
|
||||
|
||||
|
||||
def create_session(
|
||||
db: Session,
|
||||
user_id: str,
|
||||
ip_address: str | None = None,
|
||||
user_agent: str | None = None,
|
||||
) -> UserSession:
|
||||
"""Create a new server-side session record.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: Stable owner identifier.
|
||||
ip_address: Client IP address.
|
||||
user_agent: Client User-Agent header.
|
||||
|
||||
Returns:
|
||||
The newly created ``UserSession`` instance.
|
||||
"""
|
||||
session_token = secrets.token_urlsafe(64)
|
||||
now = datetime.now(timezone.utc)
|
||||
lifetime_days = get_session_lifetime_days()
|
||||
expires_at = now + timedelta(days=lifetime_days)
|
||||
|
||||
device_info = _parse_device_info(user_agent)
|
||||
|
||||
user_session = UserSession(
|
||||
session_token=session_token,
|
||||
user_id=user_id,
|
||||
ip_address=ip_address,
|
||||
user_agent=(user_agent or "")[:512],
|
||||
device_info=device_info,
|
||||
created_at=now,
|
||||
last_active_at=now,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
try:
|
||||
db.add(user_session)
|
||||
db.commit()
|
||||
db.refresh(user_session)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create session for user_id=%s", user_id)
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[SESSION] Created session id=%s user=%s device=%r expires=%s",
|
||||
user_session.id,
|
||||
user_id,
|
||||
device_info,
|
||||
expires_at.isoformat(),
|
||||
)
|
||||
return user_session
|
||||
|
||||
|
||||
def validate_session(db: Session, session_token: str) -> UserSession | None:
|
||||
"""Validate a session token and return the session if valid.
|
||||
|
||||
A session is valid when:
|
||||
* It exists in the database.
|
||||
* ``is_revoked`` is ``False``.
|
||||
* ``expires_at`` is in the future.
|
||||
|
||||
Side-effect: updates ``last_active_at`` on valid sessions.
|
||||
|
||||
Returns:
|
||||
The ``UserSession`` if valid, else ``None``.
|
||||
"""
|
||||
if not session_token:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
user_session = db.query(UserSession).filter(UserSession.session_token == session_token).first()
|
||||
|
||||
if not user_session:
|
||||
logger.debug("[SESSION] Token not found in database")
|
||||
return None
|
||||
|
||||
if user_session.is_revoked:
|
||||
logger.debug("[SESSION] Session id=%s is revoked", user_session.id)
|
||||
return None
|
||||
|
||||
if user_session.expires_at:
|
||||
expires = _ensure_tz_aware(user_session.expires_at)
|
||||
if expires < now:
|
||||
logger.debug("[SESSION] Session id=%s has expired", user_session.id)
|
||||
return None
|
||||
|
||||
# Update last_active_at (throttled to avoid excessive writes)
|
||||
last_active = _ensure_tz_aware(user_session.last_active_at)
|
||||
if not last_active or (now - last_active).total_seconds() > 60:
|
||||
try:
|
||||
user_session.last_active_at = now
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.debug("[SESSION] Failed to update last_active_at for session id=%s", user_session.id)
|
||||
|
||||
return user_session
|
||||
|
||||
|
||||
def revoke_session(db: Session, session_id: int, user_id: str) -> bool:
|
||||
"""Revoke a single session by ID.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
session_id: The session record ID to revoke.
|
||||
user_id: The owner — ensures a user can only revoke their own sessions.
|
||||
|
||||
Returns:
|
||||
``True`` if the session was found and revoked, ``False`` otherwise.
|
||||
"""
|
||||
user_session = db.get(UserSession, session_id)
|
||||
if not user_session or user_session.user_id != user_id:
|
||||
return False
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
user_session.is_revoked = True
|
||||
user_session.revoked_at = now
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("[SESSION] Revoked session id=%s user=%s", session_id, user_id)
|
||||
return True
|
||||
|
||||
|
||||
def revoke_all_sessions(
|
||||
db: Session,
|
||||
user_id: str,
|
||||
*,
|
||||
except_session_id: int | None = None,
|
||||
revoke_api_tokens: bool = True,
|
||||
) -> int:
|
||||
"""Revoke all active sessions for a user ("log off everywhere").
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: The owner whose sessions should be revoked.
|
||||
except_session_id: If provided, keep this session active (the
|
||||
current browser session).
|
||||
revoke_api_tokens: If ``True``, also revoke all active API tokens.
|
||||
|
||||
Returns:
|
||||
Number of sessions revoked.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
query = db.query(UserSession).filter(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.is_revoked.is_(False),
|
||||
)
|
||||
if except_session_id is not None:
|
||||
query = query.filter(UserSession.id != except_session_id)
|
||||
|
||||
sessions = query.all()
|
||||
count = 0
|
||||
for s in sessions:
|
||||
s.is_revoked = True
|
||||
s.revoked_at = now
|
||||
count += 1
|
||||
|
||||
if revoke_api_tokens:
|
||||
tokens = (
|
||||
db.query(ApiToken)
|
||||
.filter(
|
||||
ApiToken.owner_id == user_id,
|
||||
ApiToken.is_active.is_(True),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for t in tokens:
|
||||
t.is_active = False
|
||||
t.revoked_at = now
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[SESSION] Revoked all sessions for user=%s (count=%d, except_session_id=%s, tokens_revoked=%s)",
|
||||
user_id,
|
||||
count,
|
||||
except_session_id,
|
||||
revoke_api_tokens,
|
||||
)
|
||||
return count
|
||||
|
||||
|
||||
def list_user_sessions(db: Session, user_id: str) -> list[UserSession]:
|
||||
"""Return all non-revoked, non-expired sessions for a user.
|
||||
|
||||
Results are ordered by most recently active first.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
sessions = (
|
||||
db.query(UserSession)
|
||||
.filter(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.is_revoked.is_(False),
|
||||
)
|
||||
.order_by(UserSession.last_active_at.desc())
|
||||
.all()
|
||||
)
|
||||
# Filter expired sessions in Python to handle timezone-naive datetimes (SQLite)
|
||||
result = []
|
||||
for s in sessions:
|
||||
expires = _ensure_tz_aware(s.expires_at)
|
||||
if expires and expires > now:
|
||||
result.append(s)
|
||||
return result
|
||||
|
||||
|
||||
def cleanup_expired_sessions(db: Session) -> int:
|
||||
"""Delete sessions that expired more than 7 days ago.
|
||||
|
||||
Intended to be called periodically (e.g. via Celery beat) to keep the
|
||||
table from growing unbounded.
|
||||
|
||||
Returns:
|
||||
Number of rows deleted.
|
||||
"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=7)
|
||||
count = db.query(UserSession).filter(UserSession.expires_at < cutoff).delete(synchronize_session=False)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
if count:
|
||||
logger.info("[SESSION] Cleaned up %d expired sessions", count)
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QR login helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_qr_challenge(db: Session, user_id: str, ip_address: str | None = None) -> QRLoginChallenge:
|
||||
"""Create a new QR login challenge.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: The authenticated web user creating the challenge.
|
||||
ip_address: IP address of the web client.
|
||||
|
||||
Returns:
|
||||
The newly created ``QRLoginChallenge``.
|
||||
"""
|
||||
token = secrets.token_urlsafe(64)
|
||||
ttl = getattr(settings, "qr_login_challenge_ttl_seconds", 120)
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + timedelta(seconds=ttl)
|
||||
|
||||
challenge = QRLoginChallenge(
|
||||
challenge_token=token,
|
||||
user_id=user_id,
|
||||
created_by_ip=ip_address,
|
||||
created_at=now,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
try:
|
||||
db.add(challenge)
|
||||
db.commit()
|
||||
db.refresh(challenge)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create QR login challenge for user_id=%s", user_id)
|
||||
raise
|
||||
|
||||
logger.info("[QR_AUTH] Challenge created: id=%s user=%s expires=%s", challenge.id, user_id, expires_at.isoformat())
|
||||
return challenge
|
||||
|
||||
|
||||
def validate_qr_challenge(db: Session, challenge_token: str) -> QRLoginChallenge | None:
|
||||
"""Validate a QR challenge token without claiming it.
|
||||
|
||||
Returns the challenge if it exists, is not expired, not claimed,
|
||||
and not cancelled. Returns ``None`` otherwise.
|
||||
"""
|
||||
if not challenge_token:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
challenge = db.query(QRLoginChallenge).filter(QRLoginChallenge.challenge_token == challenge_token).first()
|
||||
|
||||
if not challenge:
|
||||
return None
|
||||
if challenge.is_claimed or challenge.is_cancelled:
|
||||
return None
|
||||
|
||||
expires = _ensure_tz_aware(challenge.expires_at)
|
||||
if expires and expires < now:
|
||||
return None
|
||||
|
||||
return challenge
|
||||
|
||||
|
||||
def claim_qr_challenge(
|
||||
db: Session,
|
||||
challenge_token: str,
|
||||
device_name: str = "Mobile App",
|
||||
ip_address: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Claim a QR challenge and issue an API token.
|
||||
|
||||
This is the critical security path. The challenge is validated,
|
||||
marked as claimed atomically, and an API token is issued for the
|
||||
user who created the challenge.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
challenge_token: The token from the QR code.
|
||||
device_name: Name provided by the mobile app.
|
||||
ip_address: IP address of the claiming mobile device.
|
||||
|
||||
Returns:
|
||||
Dict with ``token`` (plaintext), ``token_id``, ``name``, ``owner_id``
|
||||
and ``created_at`` on success, or ``None`` if the challenge is invalid.
|
||||
"""
|
||||
from app.api.api_tokens import generate_api_token, hash_token
|
||||
|
||||
challenge = validate_qr_challenge(db, challenge_token)
|
||||
if not challenge:
|
||||
logger.warning("[QR_AUTH] Invalid or expired challenge token attempted")
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Mark as claimed first to prevent race conditions
|
||||
challenge.is_claimed = True
|
||||
challenge.claimed_at = now
|
||||
challenge.claimed_by_ip = ip_address
|
||||
challenge.device_name = device_name
|
||||
|
||||
# Generate API token for the mobile app
|
||||
token_name = f"Mobile App (QR) – {device_name}"
|
||||
plaintext = generate_api_token()
|
||||
token_hash_value = hash_token(plaintext)
|
||||
prefix = plaintext[:12]
|
||||
|
||||
db_token = ApiToken(
|
||||
owner_id=challenge.user_id,
|
||||
name=token_name,
|
||||
token_hash=token_hash_value,
|
||||
token_prefix=prefix,
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(db_token)
|
||||
db.flush()
|
||||
challenge.issued_token_id = db_token.id
|
||||
db.commit()
|
||||
db.refresh(db_token)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("[QR_AUTH] Failed to issue token for challenge id=%s", challenge.id)
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[QR_AUTH] Challenge claimed: id=%s user=%s device=%r token_id=%s",
|
||||
challenge.id,
|
||||
challenge.user_id,
|
||||
device_name,
|
||||
db_token.id,
|
||||
)
|
||||
|
||||
return {
|
||||
"token": plaintext,
|
||||
"token_id": db_token.id,
|
||||
"name": token_name,
|
||||
"owner_id": challenge.user_id,
|
||||
"created_at": db_token.created_at,
|
||||
}
|
||||
|
||||
|
||||
def get_challenge_status(db: Session, challenge_id: int, user_id: str) -> dict | None:
|
||||
"""Get the current status of a QR challenge (for polling from the web UI).
|
||||
|
||||
Returns:
|
||||
Dict with ``status`` ("pending", "claimed", "expired", "cancelled")
|
||||
and metadata, or ``None`` if the challenge doesn't belong to the user.
|
||||
"""
|
||||
challenge = db.get(QRLoginChallenge, challenge_id)
|
||||
if not challenge or challenge.user_id != user_id:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
expires = _ensure_tz_aware(challenge.expires_at)
|
||||
|
||||
if challenge.is_claimed:
|
||||
status = "claimed"
|
||||
elif challenge.is_cancelled:
|
||||
status = "cancelled"
|
||||
elif expires and expires < now:
|
||||
status = "expired"
|
||||
else:
|
||||
status = "pending"
|
||||
|
||||
return {
|
||||
"id": challenge.id,
|
||||
"status": status,
|
||||
"device_name": challenge.device_name,
|
||||
"claimed_at": challenge.claimed_at,
|
||||
"expires_at": challenge.expires_at,
|
||||
}
|
||||
|
||||
|
||||
def _parse_device_info(user_agent: str | None) -> str | None:
|
||||
"""Extract a human-readable device description from User-Agent.
|
||||
|
||||
This is a lightweight parser — not a full UA library — that covers
|
||||
the most common browsers and platforms.
|
||||
"""
|
||||
if not user_agent:
|
||||
return None
|
||||
|
||||
ua = user_agent.lower()
|
||||
|
||||
# Platform detection
|
||||
platform = "Unknown"
|
||||
if "iphone" in ua:
|
||||
platform = "iPhone"
|
||||
elif "ipad" in ua:
|
||||
platform = "iPad"
|
||||
elif "android" in ua:
|
||||
platform = "Android"
|
||||
elif "macintosh" in ua or "mac os" in ua:
|
||||
platform = "macOS"
|
||||
elif "windows" in ua:
|
||||
platform = "Windows"
|
||||
elif "linux" in ua:
|
||||
platform = "Linux"
|
||||
elif "cros" in ua:
|
||||
platform = "ChromeOS"
|
||||
|
||||
# Browser detection
|
||||
browser = "Unknown Browser"
|
||||
if "edg/" in ua or "edge/" in ua:
|
||||
browser = "Edge"
|
||||
elif "opr/" in ua or "opera" in ua:
|
||||
browser = "Opera"
|
||||
elif "chrome/" in ua and "safari/" in ua:
|
||||
browser = "Chrome"
|
||||
elif "safari/" in ua and "chrome/" not in ua:
|
||||
browser = "Safari"
|
||||
elif "firefox/" in ua:
|
||||
browser = "Firefox"
|
||||
elif "docuelevate" in ua:
|
||||
browser = "DocuElevate App"
|
||||
|
||||
return f"{browser} on {platform}"
|
||||
@@ -39,6 +39,50 @@ SETTING_METADATA = {
|
||||
"required": True,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_size": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Number of persistent database connections kept in the pool per worker process. "
|
||||
"Ignored for SQLite (which uses NullPool). Default: 10."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_max_overflow": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Additional database connections allowed beyond db_pool_size under burst load. "
|
||||
"Ignored for SQLite. Default: 20."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_timeout": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Seconds to wait for a database connection from the pool before raising a TimeoutError. "
|
||||
"Ignored for SQLite. Default: 30."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_recycle": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Recycle (close and reopen) database connections after this many seconds "
|
||||
"to avoid stale connections. Ignored for SQLite. Default: 1800."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"workdir": {
|
||||
"category": "Core",
|
||||
"description": "Working directory for file storage and processing",
|
||||
@@ -55,6 +99,18 @@ SETTING_METADATA = {
|
||||
"required": True, # Required for OAuth redirects and external URLs
|
||||
"restart_required": True,
|
||||
},
|
||||
"public_base_url": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Full public base URL including scheme (e.g., https://docuelevate.example.com). "
|
||||
"When set, overrides auto-detected URLs for OAuth redirect URIs. "
|
||||
"Required when behind a reverse proxy that does not forward X-Forwarded-Proto."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"debug": {
|
||||
"category": "Core",
|
||||
"description": "Enable debug mode for verbose logging",
|
||||
@@ -134,6 +190,38 @@ SETTING_METADATA = {
|
||||
"required": True, # Required when auth_enabled=True (validated in config.py)
|
||||
"restart_required": True,
|
||||
},
|
||||
"session_lifetime_days": {
|
||||
"category": "Authentication",
|
||||
"description": "Session lifetime in days (default 30). Determines how long a user stays logged in.",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"session_lifetime_custom_days": {
|
||||
"category": "Authentication",
|
||||
"description": "Override session_lifetime_days with a custom value. Takes precedence when set.",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"qr_login_enabled": {
|
||||
"category": "Authentication",
|
||||
"description": "Enable QR code-based login for mobile device authentication.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"qr_login_challenge_ttl_seconds": {
|
||||
"category": "Authentication",
|
||||
"description": "Time-to-live in seconds for QR login challenges (default 120).",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"admin_username": {
|
||||
"category": "Authentication",
|
||||
"description": "Admin username for local authentication",
|
||||
@@ -182,6 +270,17 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"sso_auto_login": {
|
||||
"category": "Authentication",
|
||||
"description": (
|
||||
"Automatically redirect to SSO login when authentication is required. "
|
||||
"Skips the login page and sends users directly to the configured SSO provider."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Social Login Providers
|
||||
"social_auth_google_enabled": {
|
||||
"category": "Social Login",
|
||||
@@ -212,6 +311,20 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_google_use_global_credentials": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
"When True, Google social login uses the global GOOGLE_DRIVE_CLIENT_ID / "
|
||||
"GOOGLE_DRIVE_CLIENT_SECRET credentials (the Google Drive OAuth integration) "
|
||||
"instead of requiring separate SOCIAL_AUTH_GOOGLE_CLIENT_ID / "
|
||||
"SOCIAL_AUTH_GOOGLE_CLIENT_SECRET values. "
|
||||
"Requires SOCIAL_AUTH_GOOGLE_ENABLED=True and global Google Drive OAuth credentials to be set."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_microsoft_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
@@ -254,6 +367,20 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_microsoft_use_global_credentials": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
"When True, Microsoft social login uses the global ONEDRIVE_CLIENT_ID / "
|
||||
"ONEDRIVE_CLIENT_SECRET credentials (the OneDrive integration credentials) "
|
||||
"instead of requiring separate SOCIAL_AUTH_MICROSOFT_CLIENT_ID / "
|
||||
"SOCIAL_AUTH_MICROSOFT_CLIENT_SECRET values. "
|
||||
"Requires SOCIAL_AUTH_MICROSOFT_ENABLED=True and global OneDrive credentials to be set."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_apple_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
@@ -302,6 +429,19 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_dropbox_use_global_credentials": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
"When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET "
|
||||
"credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / "
|
||||
"SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. "
|
||||
"Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_dropbox_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
@@ -329,6 +469,182 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_github_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
"Enable GitHub Sign-In. Requires SOCIAL_AUTH_GITHUB_CLIENT_ID and "
|
||||
"SOCIAL_AUTH_GITHUB_CLIENT_SECRET from GitHub Developer Settings."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
"help_link": "https://github.com/settings/developers",
|
||||
"help_link_label": "GitHub Developer Settings",
|
||||
},
|
||||
"social_auth_github_client_id": {
|
||||
"category": "Social Login",
|
||||
"description": "GitHub OAuth2 client ID from GitHub Developer Settings.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_github_client_secret": {
|
||||
"category": "Social Login",
|
||||
"description": "GitHub OAuth2 client secret from GitHub Developer Settings.",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
# Keycloak SSO
|
||||
"social_auth_keycloak_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": "Enable Keycloak SSO. Requires server URL, realm, client ID, and client secret.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_keycloak_client_id": {
|
||||
"category": "Social Login",
|
||||
"description": "Keycloak OAuth2 client ID.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_keycloak_client_secret": {
|
||||
"category": "Social Login",
|
||||
"description": "Keycloak OAuth2 client secret.",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_keycloak_server_url": {
|
||||
"category": "Social Login",
|
||||
"description": "Keycloak server base URL (e.g. https://keycloak.example.com).",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_keycloak_realm": {
|
||||
"category": "Social Login",
|
||||
"description": "Keycloak realm name.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
# Generic OAuth2 SSO
|
||||
"social_auth_generic_oauth2_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": "Enable a generic OAuth2 SSO provider.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_client_id": {
|
||||
"category": "Social Login",
|
||||
"description": "Generic OAuth2 client ID.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_client_secret": {
|
||||
"category": "Social Login",
|
||||
"description": "Generic OAuth2 client secret.",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_authorize_url": {
|
||||
"category": "Social Login",
|
||||
"description": "Generic OAuth2 authorization URL.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_token_url": {
|
||||
"category": "Social Login",
|
||||
"description": "Generic OAuth2 token endpoint URL.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_userinfo_url": {
|
||||
"category": "Social Login",
|
||||
"description": "Generic OAuth2 userinfo endpoint URL.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_scope": {
|
||||
"category": "Social Login",
|
||||
"description": "Space-separated list of OAuth2 scopes to request (default: openid profile email).",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_generic_oauth2_name": {
|
||||
"category": "Social Login",
|
||||
"description": "Display name for the generic OAuth2 provider button on the login page.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
# SAML2 SSO
|
||||
"social_auth_saml2_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": "Enable SAML2 SSO authentication.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_saml2_entity_id": {
|
||||
"category": "Social Login",
|
||||
"description": "SAML2 Identity Provider Entity ID.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_saml2_sso_url": {
|
||||
"category": "Social Login",
|
||||
"description": "SAML2 Identity Provider SSO URL.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_saml2_certificate": {
|
||||
"category": "Social Login",
|
||||
"description": "SAML2 Identity Provider X.509 certificate (PEM format).",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_saml2_name": {
|
||||
"category": "Social Login",
|
||||
"description": "Display name for the SAML2 provider.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
# AI Services
|
||||
"openai_api_key": {
|
||||
"category": "AI Services",
|
||||
@@ -522,6 +838,19 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Document Translation
|
||||
"default_document_language": {
|
||||
"category": "AI Services",
|
||||
"description": (
|
||||
"ISO 639-1 language code for the default document translation target "
|
||||
"(e.g. 'en', 'de', 'fr'). Documents whose detected language differs "
|
||||
"are automatically translated into this language after processing."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# OCR Engine Configuration
|
||||
"ocr_providers": {
|
||||
"category": "OCR Engines",
|
||||
@@ -682,6 +1011,18 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"dropbox_allow_global_credentials_for_integrations": {
|
||||
"category": "Storage Providers",
|
||||
"description": (
|
||||
"When True, users may authorize their personal Dropbox integrations using the global "
|
||||
"DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without "
|
||||
"needing to create their own Dropbox app."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - Nextcloud
|
||||
"nextcloud_enabled": {
|
||||
"category": "Storage Providers",
|
||||
@@ -862,6 +1203,63 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - SharePoint
|
||||
"sharepoint_client_id": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD application (client) ID",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_client_secret": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD client secret",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_tenant_id": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_refresh_token": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint OAuth refresh token",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_site_url": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_document_library": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint document library name (default: 'Documents')",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_folder_path": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Subfolder path inside the SharePoint document library",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - WebDAV
|
||||
"webdav_enabled": {
|
||||
"category": "Storage Providers",
|
||||
@@ -1777,6 +2175,30 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"telegram_enabled": {
|
||||
"category": "Notifications",
|
||||
"description": "Enable Telegram bot notifications.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"telegram_bot_token": {
|
||||
"category": "Notifications",
|
||||
"description": "Telegram Bot API token from @BotFather.",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"telegram_chat_id": {
|
||||
"category": "Notifications",
|
||||
"description": "Telegram chat ID to send notifications to.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Notifications Settings
|
||||
"notification_urls": {
|
||||
"category": "Notifications",
|
||||
@@ -1875,6 +2297,18 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"automation_hooks_enabled": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Enable Zapier / Make.com automation hook subscriptions and delivery. "
|
||||
"When enabled, external automation platforms can subscribe to DocuElevate events "
|
||||
"via the REST hooks protocol. Default: True."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"compliance_enabled": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
@@ -1887,6 +2321,28 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"factory_reset_on_startup": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Wipe all user data on every startup so the instance always starts fresh. "
|
||||
"Useful for demo/testing environments. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"enable_factory_reset": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Show the System Reset page in the admin UI. Allows administrators to "
|
||||
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Backup / Restore
|
||||
"backup_enabled": {
|
||||
"category": "Backup",
|
||||
@@ -1910,14 +2366,26 @@ SETTING_METADATA = {
|
||||
"category": "Backup",
|
||||
"description": (
|
||||
"Storage provider for remote backup copies. "
|
||||
"Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. "
|
||||
"Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. "
|
||||
"Leave empty to keep backups local only."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
"options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"],
|
||||
"options": [
|
||||
"",
|
||||
"s3",
|
||||
"dropbox",
|
||||
"google_drive",
|
||||
"onedrive",
|
||||
"sharepoint",
|
||||
"nextcloud",
|
||||
"webdav",
|
||||
"ftp",
|
||||
"sftp",
|
||||
"email",
|
||||
],
|
||||
},
|
||||
"backup_remote_folder": {
|
||||
"category": "Backup",
|
||||
@@ -2429,6 +2897,27 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Per-user upload rate limiting
|
||||
"upload_rate_limit_per_user": {
|
||||
"category": "Security",
|
||||
"description": (
|
||||
"Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. "
|
||||
"The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. "
|
||||
"Default: 20."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"upload_rate_limit_window": {
|
||||
"category": "Security",
|
||||
"description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Rate Limiting
|
||||
"rate_limiting_enabled": {
|
||||
"category": "Security",
|
||||
@@ -2628,6 +3117,63 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Logging
|
||||
"log_level": {
|
||||
"category": "Observability",
|
||||
"description": (
|
||||
"Python logging level for the application root logger. "
|
||||
"Accepts: DEBUG, INFO, WARNING, ERROR, CRITICAL. "
|
||||
"When DEBUG=True and LOG_LEVEL is not explicitly set, "
|
||||
"the effective level is automatically lowered to DEBUG."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"log_format": {
|
||||
"category": "Observability",
|
||||
"description": (
|
||||
"Log output format: 'text' (human-readable, default) or "
|
||||
"'json' (structured JSON lines for SIEM / log aggregation)."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"log_syslog_enabled": {
|
||||
"category": "Observability",
|
||||
"description": "Forward application logs to a syslog receiver in addition to stdout.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"log_syslog_host": {
|
||||
"category": "Observability",
|
||||
"description": "Hostname or IP of the syslog receiver for application logs.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"log_syslog_port": {
|
||||
"category": "Observability",
|
||||
"description": "Port of the syslog receiver for application logs.",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"log_syslog_protocol": {
|
||||
"category": "Observability",
|
||||
"description": "Protocol for syslog transport: 'udp' or 'tcp'.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
# Observability – Sentry
|
||||
"sentry_dsn": {
|
||||
"category": "Observability",
|
||||
@@ -2686,6 +3232,43 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"sentry_js_traces_sample_rate": {
|
||||
"category": "Observability",
|
||||
"description": (
|
||||
"Fraction of browser page-loads captured for client-side Sentry performance tracing (0.0–1.0). "
|
||||
"0.0 (default) disables browser tracing; 1.0 captures every navigation. "
|
||||
"Only active when SENTRY_DSN is set."
|
||||
),
|
||||
"type": "float",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"sentry_js_replay_session_sample_rate": {
|
||||
"category": "Observability",
|
||||
"description": (
|
||||
"Fraction of sessions recorded by Sentry Session Replay (0.0–1.0). "
|
||||
"0.0 (default) disables session recording; 1.0 records every session. "
|
||||
"Only active when SENTRY_DSN is set."
|
||||
),
|
||||
"type": "float",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"sentry_js_replay_on_error_sample_rate": {
|
||||
"category": "Observability",
|
||||
"description": (
|
||||
"Fraction of error sessions recorded by Sentry Session Replay (0.0–1.0). "
|
||||
"Defaults to 0.1 (10%) so that errors are captured with replay context "
|
||||
"even when session-level recording is disabled. "
|
||||
"Only active when SENTRY_DSN is set."
|
||||
),
|
||||
"type": "float",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,16 @@ def notify_settings_updated() -> None:
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not reload in-process settings: {exc}")
|
||||
|
||||
# Re-register OAuth / social-login providers so that any provider whose
|
||||
# credentials were just saved (or updated) in the database is active
|
||||
# immediately on the login page — no restart required.
|
||||
try:
|
||||
from app.auth import refresh_social_providers
|
||||
|
||||
refresh_social_providers()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not refresh social login providers after settings update: {exc}")
|
||||
|
||||
# Re-check OCR language availability in the background whenever settings
|
||||
# are updated. This ensures that if a user changes tesseract_language or
|
||||
# easyocr_languages via the UI, the new language data is downloaded without
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
"""
|
||||
System reset utilities for DocuElevate.
|
||||
|
||||
Provides functions to:
|
||||
- Wipe all user data (database rows + work-files on disk) for a fresh start.
|
||||
- Wipe with re-import: move original files to a dedicated folder, wipe
|
||||
everything, then let the watch-folder mechanism re-ingest the files.
|
||||
|
||||
Security: All public functions in this module require admin-level access.
|
||||
They MUST only be invoked from admin-guarded API/view endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectories inside *workdir* that contain user-generated data.
|
||||
# Everything else (app code, static assets, config) is left untouched.
|
||||
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
|
||||
|
||||
# JSON cache files written by watch-folder / ingest tasks.
|
||||
_CACHE_FILES = (
|
||||
"watch_folder_processed.json",
|
||||
"ftp_ingest_processed.json",
|
||||
"sftp_ingest_processed.json",
|
||||
"dropbox_ingest_processed.json",
|
||||
"gdrive_ingest_processed.json",
|
||||
"onedrive_ingest_processed.json",
|
||||
"nextcloud_ingest_processed.json",
|
||||
"s3_ingest_processed.json",
|
||||
"webdav_ingest_processed.json",
|
||||
"processed_mails.json",
|
||||
"credential_failures.json",
|
||||
)
|
||||
|
||||
# The folder name used for storing files prior to re-import.
|
||||
REIMPORT_FOLDER_NAME = "reimport"
|
||||
|
||||
|
||||
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
|
||||
"""Delete user data subdirectories and cache files inside *workdir*.
|
||||
|
||||
Leaves the workdir directory itself intact so the application can
|
||||
continue to write into it. Also leaves any files that do not belong
|
||||
to the known data subdirectories or caches.
|
||||
|
||||
Returns:
|
||||
A dict with counts of deleted directories and files.
|
||||
"""
|
||||
workdir_path = Path(workdir)
|
||||
deleted_dirs = 0
|
||||
deleted_files = 0
|
||||
|
||||
# Remove data subdirectories
|
||||
for subdir in _USER_DATA_SUBDIRS:
|
||||
target = workdir_path / subdir
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
logger.info("Deleted data directory: %s", target)
|
||||
deleted_dirs += 1
|
||||
|
||||
# Remove cache / state JSON files
|
||||
for cache_file in _CACHE_FILES:
|
||||
target = workdir_path / cache_file
|
||||
if target.is_file():
|
||||
target.unlink()
|
||||
logger.info("Deleted cache file: %s", target)
|
||||
deleted_files += 1
|
||||
|
||||
# Also remove user_wf_*.json files (per-user watch folder caches)
|
||||
for f in workdir_path.glob("user_wf_*.json"):
|
||||
f.unlink()
|
||||
logger.info("Deleted user watch-folder cache: %s", f)
|
||||
deleted_files += 1
|
||||
|
||||
# Remove loose files in workdir root that are user uploads (uuid-named
|
||||
# files like "a1b2c3d4-…pdf") but NOT application config files.
|
||||
for entry in workdir_path.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() in {
|
||||
".pdf",
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".tiff",
|
||||
".tif",
|
||||
".docx",
|
||||
".doc",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".pptx",
|
||||
".heic",
|
||||
".heif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".gif",
|
||||
".txt",
|
||||
".rtf",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".csv",
|
||||
".pages",
|
||||
".numbers",
|
||||
".keynote",
|
||||
}:
|
||||
entry.unlink()
|
||||
logger.info("Deleted loose workdir file: %s", entry)
|
||||
deleted_files += 1
|
||||
|
||||
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
|
||||
|
||||
|
||||
def _wipe_database(db: Session) -> dict[str, int]:
|
||||
"""Delete all user-generated rows from the database.
|
||||
|
||||
Preserves schema (tables, migrations) and system-seeded rows that will
|
||||
be re-created on the next startup (subscription plans, default pipeline,
|
||||
scheduled jobs, compliance templates).
|
||||
|
||||
Returns:
|
||||
A dict mapping table name → number of rows deleted.
|
||||
"""
|
||||
from app.models import (
|
||||
AuditLog,
|
||||
BackupRecord,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
InAppNotification,
|
||||
ProcessingLog,
|
||||
SavedSearch,
|
||||
SettingsAuditLog,
|
||||
SharedLink,
|
||||
UserImapAccount,
|
||||
UserIntegration,
|
||||
UserNotificationPreference,
|
||||
UserNotificationTarget,
|
||||
)
|
||||
|
||||
# Order matters: delete children before parents to respect FK constraints.
|
||||
tables_to_wipe: list[tuple[str, type]] = [
|
||||
("file_processing_steps", FileProcessingStep),
|
||||
("processing_logs", ProcessingLog),
|
||||
("shared_links", SharedLink),
|
||||
("in_app_notifications", InAppNotification),
|
||||
("user_notification_preferences", UserNotificationPreference),
|
||||
("user_notification_targets", UserNotificationTarget),
|
||||
("user_imap_accounts", UserImapAccount),
|
||||
("user_integrations", UserIntegration),
|
||||
("saved_searches", SavedSearch),
|
||||
("settings_audit_log", SettingsAuditLog),
|
||||
("audit_logs", AuditLog),
|
||||
("backup_records", BackupRecord),
|
||||
("document_metadata", DocumentMetadata),
|
||||
("files", FileRecord),
|
||||
]
|
||||
|
||||
result: dict[str, int] = {}
|
||||
for table_name, model in tables_to_wipe:
|
||||
try:
|
||||
count = db.query(model).delete()
|
||||
result[table_name] = count
|
||||
logger.info("Wiped %d rows from %s", count, table_name)
|
||||
except Exception:
|
||||
logger.exception("Failed to wipe table %s during system reset", table_name)
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
db.commit()
|
||||
return result
|
||||
|
||||
|
||||
def perform_full_reset(db: Session) -> dict:
|
||||
"""Perform a complete system reset: wipe database rows + work-files.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database`` and ``filesystem`` sub-dicts.
|
||||
"""
|
||||
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
|
||||
|
||||
db_result = _wipe_database(db)
|
||||
fs_result = _wipe_workdir_data(settings.workdir)
|
||||
|
||||
logger.warning(">>> SYSTEM RESET complete <<<")
|
||||
return {"database": db_result, "filesystem": fs_result}
|
||||
|
||||
|
||||
def perform_reset_and_reimport(db: Session) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, then
|
||||
configure the reimport folder as a watch folder for re-ingestion.
|
||||
|
||||
The watch-folder scanner (``scan_all_watch_folders``) will pick up
|
||||
the files on its next periodic run and process them exactly as if
|
||||
they had been freshly uploaded — respecting the same backoff
|
||||
strategy, size limits, and rate limits.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
|
||||
"""
|
||||
workdir_path = Path(settings.workdir)
|
||||
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
|
||||
original_dir = workdir_path / "original"
|
||||
|
||||
# 1. Collect original files
|
||||
files_moved = 0
|
||||
reimport_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if original_dir.is_dir():
|
||||
for entry in original_dir.iterdir():
|
||||
if entry.is_file():
|
||||
# Validate the resolved path stays within original_dir (path traversal guard)
|
||||
try:
|
||||
entry.resolve().relative_to(original_dir.resolve())
|
||||
except ValueError:
|
||||
logger.warning("Skipping file outside original dir: %s", entry)
|
||||
continue
|
||||
dest = reimport_dir / entry.name
|
||||
# Avoid overwriting: append counter if name clash
|
||||
if dest.exists():
|
||||
stem = dest.stem
|
||||
suffix = dest.suffix
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = reimport_dir / f"{stem}_{counter}{suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(str(entry), str(dest))
|
||||
files_moved += 1
|
||||
|
||||
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
|
||||
|
||||
# 2. Perform the full reset (wipe DB + other workdir data)
|
||||
reset_result = perform_full_reset(db)
|
||||
|
||||
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
|
||||
# and set up watch folder config to point at it.
|
||||
_configure_reimport_watch_folder(str(reimport_dir))
|
||||
|
||||
reset_result["reimport"] = {
|
||||
"files_moved": files_moved,
|
||||
"reimport_folder": str(reimport_dir),
|
||||
}
|
||||
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
|
||||
return reset_result
|
||||
|
||||
|
||||
def _configure_reimport_watch_folder(reimport_path: str) -> None:
|
||||
"""Append *reimport_path* to the application's watch-folder list.
|
||||
|
||||
The watch-folder scanner uses ``settings.watch_folders`` (a
|
||||
comma-separated string). We mutate the runtime setting so the
|
||||
next scan picks up the folder. We also set
|
||||
``watch_folder_delete_after_process = True`` so files are cleaned
|
||||
up after successful processing.
|
||||
"""
|
||||
current = getattr(settings, "watch_folders", None) or ""
|
||||
folders = [f.strip() for f in current.split(",") if f.strip()]
|
||||
|
||||
if reimport_path not in folders:
|
||||
folders.append(reimport_path)
|
||||
|
||||
# Mutate runtime settings (not persisted to .env — ephemeral)
|
||||
object.__setattr__(settings, "watch_folders", ",".join(folders))
|
||||
object.__setattr__(settings, "watch_folder_delete_after_process", True)
|
||||
logger.info("Configured reimport watch folder: %s", reimport_path)
|
||||
|
||||
|
||||
def perform_startup_reset() -> None:
|
||||
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
|
||||
|
||||
Wipes database and filesystem data so the instance starts completely
|
||||
fresh. Uses its own DB session so it runs before the normal lifespan
|
||||
seeding logic.
|
||||
"""
|
||||
from app.database import SessionLocal
|
||||
|
||||
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
perform_full_reset(db)
|
||||
except Exception:
|
||||
logger.exception("Factory reset on startup failed")
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
+147
-13
@@ -11,22 +11,47 @@ import logging
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Query
|
||||
from sqlalchemy.orm import Query, Session
|
||||
from sqlalchemy.sql import false
|
||||
|
||||
from app.config import settings
|
||||
from app.models import FileRecord
|
||||
from app.models import FILE_SHARE_ROLE_EDITOR, FILE_SHARE_ROLE_VIEWER, FileRecord, FileShare
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Role hierarchy: higher index = more rights
|
||||
_ROLE_RANK: dict[str, int] = {
|
||||
FILE_SHARE_ROLE_VIEWER: 1,
|
||||
FILE_SHARE_ROLE_EDITOR: 2,
|
||||
"owner": 3,
|
||||
}
|
||||
|
||||
|
||||
def _owner_id_from_user(user: dict) -> str | None:
|
||||
"""Extract the owner identifier from a user dict.
|
||||
|
||||
Priority: ``sub`` (OAuth subject) → ``preferred_username`` → ``email`` → ``id``.
|
||||
"""
|
||||
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
|
||||
|
||||
def get_current_owner_id(request: Request) -> str | None:
|
||||
"""Extract the owner identifier for the current authenticated user.
|
||||
|
||||
The owner ID is derived from the user's session data. It uses the
|
||||
``sub`` claim (OAuth subject) when available, falling back to
|
||||
``preferred_username`` or ``email``. Returns ``None`` when no user
|
||||
is authenticated.
|
||||
The owner ID is derived from the user's session data or, when no session
|
||||
is present, from a valid Bearer API token in the ``Authorization`` header.
|
||||
This ensures that both browser-based (session cookie) and mobile/API
|
||||
(Bearer token) requests are correctly identified.
|
||||
|
||||
Priority for user resolution:
|
||||
|
||||
1. Session ``user`` dict (set by OAuth or local login).
|
||||
2. ``request.state.api_token_user`` (set by ``require_login`` or an
|
||||
earlier call to this function during the same request).
|
||||
3. Direct Bearer token look-up against the database.
|
||||
|
||||
Within the resolved user dict the owner ID is chosen as:
|
||||
``sub`` → ``preferred_username`` → ``email`` → ``id``.
|
||||
|
||||
Args:
|
||||
request: The current FastAPI request with session data.
|
||||
@@ -34,19 +59,50 @@ def get_current_owner_id(request: Request) -> str | None:
|
||||
Returns:
|
||||
A stable string identifier for the user, or ``None``.
|
||||
"""
|
||||
# 1. Session-based auth (most common for web UI)
|
||||
user = request.session.get("user")
|
||||
if not user or not isinstance(user, dict):
|
||||
return None
|
||||
# Prefer 'sub' (OAuth subject), then 'preferred_username', then 'email', then 'id'
|
||||
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
|
||||
if user and isinstance(user, dict):
|
||||
return _owner_id_from_user(user)
|
||||
|
||||
# 2. Already-resolved API token user (cached by require_login or a
|
||||
# prior dependency call during this request)
|
||||
api_user = getattr(request.state, "api_token_user", None)
|
||||
if isinstance(api_user, dict):
|
||||
return _owner_id_from_user(api_user)
|
||||
|
||||
# 3. Direct Bearer token resolution – necessary when this function is
|
||||
# invoked as a FastAPI dependency (via Depends) which runs *before*
|
||||
# the @require_login decorator wrapper has had a chance to resolve
|
||||
# the token and populate request.state.api_token_user.
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if isinstance(auth_header, str) and auth_header.startswith("Bearer "):
|
||||
try:
|
||||
from app.auth import _resolve_bearer_user
|
||||
from app.database import SessionLocal
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
resolved = _resolve_bearer_user(request, db)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if resolved:
|
||||
# Cache so subsequent calls (and require_login) skip the DB
|
||||
request.state.api_token_user = resolved
|
||||
return _owner_id_from_user(resolved)
|
||||
except Exception:
|
||||
logger.debug("Bearer token resolution failed in get_current_owner_id", exc_info=True)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def apply_owner_filter(query: Query, request: Request) -> Query:
|
||||
"""Conditionally filter a ``FileRecord`` query by the current user.
|
||||
|
||||
When multi-user mode is enabled, only files whose ``owner_id``
|
||||
matches the authenticated user are returned. Admin users bypass
|
||||
the filter and see all documents.
|
||||
matches the authenticated user are returned, **plus** any files that
|
||||
have been explicitly shared with the user via ``FileShare``. Admin
|
||||
users bypass the filter and see all documents.
|
||||
|
||||
When ``unowned_docs_visible_to_all`` is ``True`` (default), documents
|
||||
with ``owner_id IS NULL`` (unclaimed) are also included for every
|
||||
@@ -74,11 +130,89 @@ def apply_owner_filter(query: Query, request: Request) -> Query:
|
||||
# No authenticated user — return empty result set
|
||||
return query.filter(false())
|
||||
|
||||
# Build filter: user's own documents
|
||||
# Build filter: user's own documents + documents shared with them
|
||||
conditions = [FileRecord.owner_id == owner_id]
|
||||
|
||||
# Include files explicitly shared with this user
|
||||
from sqlalchemy import select as sa_select
|
||||
|
||||
conditions.append(FileRecord.id.in_(sa_select(FileShare.file_id).where(FileShare.shared_with_user_id == owner_id)))
|
||||
|
||||
# Optionally include unclaimed (owner_id IS NULL) documents
|
||||
if settings.unowned_docs_visible_to_all:
|
||||
conditions.append(FileRecord.owner_id.is_(None))
|
||||
|
||||
return query.filter(or_(*conditions))
|
||||
|
||||
|
||||
def get_file_role(file_record: FileRecord, user_id: str | None, db: Session) -> str | None:
|
||||
"""Return the effective role a user has on a ``FileRecord``.
|
||||
|
||||
Roles (in descending order of privilege):
|
||||
|
||||
``"owner"`` — the user's ``owner_id`` matches ``file_record.owner_id``,
|
||||
or multi-user mode is disabled (everyone is effectively an
|
||||
owner in single-user mode).
|
||||
``"editor"`` — the user has an explicit ``FileShare`` with role=editor.
|
||||
``"viewer"`` — the user has an explicit ``FileShare`` with role=viewer,
|
||||
or the file is unclaimed (``owner_id IS NULL``) and
|
||||
``unowned_docs_visible_to_all`` is True.
|
||||
``None`` — no access.
|
||||
|
||||
Args:
|
||||
file_record: The ``FileRecord`` to check.
|
||||
user_id: The stable identifier of the requesting user.
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
One of ``"owner"``, ``"editor"``, ``"viewer"``, or ``None``.
|
||||
"""
|
||||
if not settings.multi_user_enabled:
|
||||
# Single-user mode: full access for everyone
|
||||
return "owner"
|
||||
|
||||
if user_id is None:
|
||||
return None
|
||||
|
||||
# Owner always has full access
|
||||
if file_record.owner_id == user_id:
|
||||
return "owner"
|
||||
|
||||
# Unclaimed document — limited access when setting allows it
|
||||
if file_record.owner_id is None and settings.unowned_docs_visible_to_all:
|
||||
return FILE_SHARE_ROLE_VIEWER
|
||||
|
||||
# Check for an explicit share
|
||||
share = (
|
||||
db.query(FileShare)
|
||||
.filter(FileShare.file_id == file_record.id, FileShare.shared_with_user_id == user_id)
|
||||
.first()
|
||||
)
|
||||
if share:
|
||||
return share.role
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def has_file_role(
|
||||
file_record: FileRecord,
|
||||
user_id: str | None,
|
||||
db: Session,
|
||||
minimum_role: str = FILE_SHARE_ROLE_VIEWER,
|
||||
) -> bool:
|
||||
"""Return ``True`` if the user's effective role meets the minimum required.
|
||||
|
||||
Args:
|
||||
file_record: The document to check.
|
||||
user_id: Requesting user's stable identifier.
|
||||
db: Active SQLAlchemy session.
|
||||
minimum_role: The minimum role required (``"viewer"``, ``"editor"``,
|
||||
or ``"owner"``).
|
||||
|
||||
Returns:
|
||||
``True`` when the user's role rank is >= the minimum rank.
|
||||
"""
|
||||
role = get_file_role(file_record, user_id, db)
|
||||
if role is None:
|
||||
return False
|
||||
return _ROLE_RANK.get(role, 0) >= _ROLE_RANK.get(minimum_role, 0)
|
||||
|
||||
+19
-10
@@ -145,6 +145,8 @@ def dispatch_webhook_event(event: str, data: dict[str, Any]) -> None:
|
||||
It delegates to :func:`deliver_webhook_task` (Celery) for each matching
|
||||
webhook so delivery happens asynchronously with automatic retries.
|
||||
|
||||
Also dispatches to automation hooks (Zapier / Make.com) if enabled.
|
||||
|
||||
Args:
|
||||
event: Event name (must be in :data:`VALID_EVENTS`).
|
||||
data: Event-specific payload data.
|
||||
@@ -156,16 +158,23 @@ def dispatch_webhook_event(event: str, data: dict[str, Any]) -> None:
|
||||
webhooks = get_active_webhooks_for_event(event)
|
||||
if not webhooks:
|
||||
logger.debug("No active webhooks for event %s", event)
|
||||
return
|
||||
else:
|
||||
payload = build_payload(event, data)
|
||||
|
||||
payload = build_payload(event, data)
|
||||
# Import here to avoid circular dependency with celery_app
|
||||
from app.tasks.webhook_tasks import deliver_webhook_task
|
||||
|
||||
# Import here to avoid circular dependency with celery_app
|
||||
from app.tasks.webhook_tasks import deliver_webhook_task
|
||||
for wh in webhooks:
|
||||
try:
|
||||
deliver_webhook_task.delay(wh["url"], payload, wh["secret"])
|
||||
logger.debug("Queued webhook delivery to %s for event %s", wh["url"], event)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to queue webhook to %s: %s", wh["url"], exc)
|
||||
|
||||
for wh in webhooks:
|
||||
try:
|
||||
deliver_webhook_task.delay(wh["url"], payload, wh["secret"])
|
||||
logger.debug("Queued webhook delivery to %s for event %s", wh["url"], event)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to queue webhook to %s: %s", wh["url"], exc)
|
||||
# Also fan-out to Zapier / Make.com automation hooks
|
||||
try:
|
||||
from app.utils.automation_hooks import dispatch_automation_hooks
|
||||
|
||||
dispatch_automation_hooks(event, data)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to dispatch automation hooks for event %s: %s", event, exc)
|
||||
|
||||
Reference in New Issue
Block a user