fix: merge main branch and renumber migration 037→040
Resolve all merge conflicts between our automation feature branch and current main (v0.163.0, 920 commits ahead). Conflicts resolved: - app/api/__init__.py: add automation_router alongside main's new routers (classification_rules, qr_auth, sessions, system_reset) - app/config.py: add main's new settings (dropbox_use_global_credentials, factory_reset_on_startup, enable_factory_reset) - app/models.py: add main's new models (ClassificationRuleModel, UserSession, QRLoginChallenge, SharePoint integration type) - app/utils/settings_service.py: merge automation_hooks_enabled with main's new metadata entries - docs/API.md: merge automation API docs with main's classification rules docs - docs/ConfigurationGuide.md: add factory reset settings - tests/conftest.py: import both AutomationHook and new main models Migration renumbered: - 037_add_automation_hooks → 040_add_automation_hooks - down_revision: 039_add_classification_rules (was 036_add_document_translation_fields) - Chain: 036 → 037 → 038 → 039 → 040 (automation hooks) For all non-automation files with conflicts, main's version was taken since our branch did not modify those files (conflicts were from a stale prior merge). Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/cb62f012-3b69-4415-835e-3857ce3e9f45
This commit is contained in:
@@ -68,6 +68,8 @@ IMAGE_MIME_TYPES: set[str] = {
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/heic",
|
||||
"image/heif",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -124,6 +126,8 @@ ALLOWED_EXTENSIONS: set[str] = {
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
".heic",
|
||||
".heif",
|
||||
# Web
|
||||
".html",
|
||||
".htm",
|
||||
@@ -234,7 +238,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
},
|
||||
"images": {
|
||||
"label": "Images",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"image/jpeg",
|
||||
@@ -245,6 +249,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/heic",
|
||||
"image/heif",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
@@ -258,6 +264,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
".heic",
|
||||
".heif",
|
||||
}
|
||||
),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""
|
||||
Rule-based document classification engine.
|
||||
|
||||
Provides pre-built categories and a rule matcher that classifies documents
|
||||
using filename patterns, content keywords, and metadata fields. Custom
|
||||
rules stored in the database are evaluated alongside the built-in defaults.
|
||||
|
||||
Usage::
|
||||
|
||||
from app.utils.classification_rules import classify_document
|
||||
|
||||
result = classify_document(
|
||||
filename="2024-03-01_Invoice_Acme.pdf",
|
||||
text="Invoice total: $1,234.56",
|
||||
metadata={"absender": "Acme Corp"},
|
||||
custom_rules=custom_rules_from_db,
|
||||
)
|
||||
# result -> ClassificationResult(category="invoice", confidence=85, matched_rules=[...])
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-built categories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Canonical category names recognized by the system. Users may also define
|
||||
#: their own categories via custom rules.
|
||||
BUILTIN_CATEGORIES: dict[str, str] = {
|
||||
"invoice": "Invoice",
|
||||
"contract": "Contract",
|
||||
"receipt": "Receipt",
|
||||
"letter": "Letter",
|
||||
"report": "Report",
|
||||
"bank_statement": "Bank Statement",
|
||||
"tax_document": "Tax Document",
|
||||
"insurance": "Insurance Document",
|
||||
"payslip": "Payslip",
|
||||
"unknown": "Unknown",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule type constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RULE_TYPE_FILENAME = "filename_pattern"
|
||||
RULE_TYPE_CONTENT = "content_keyword"
|
||||
RULE_TYPE_METADATA = "metadata_match"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationRule:
|
||||
"""A single classification rule."""
|
||||
|
||||
name: str
|
||||
category: str
|
||||
rule_type: str # filename_pattern | content_keyword | metadata_match
|
||||
pattern: str # regex for filename, keyword(s) for content, "field=value" for metadata
|
||||
priority: int = 0 # higher = evaluated first
|
||||
case_sensitive: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.rule_type not in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA):
|
||||
raise ValueError(f"Invalid rule_type: {self.rule_type!r}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatchedRule:
|
||||
"""Records which rule matched and why."""
|
||||
|
||||
rule_name: str
|
||||
rule_type: str
|
||||
category: str
|
||||
confidence: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
"""The outcome of running the classification engine on a document."""
|
||||
|
||||
category: str
|
||||
confidence: int # 0 – 100
|
||||
matched_rules: list[MatchedRule] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BUILTIN_RULES: list[ClassificationRule] = [
|
||||
# ── Invoice ───────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_invoice_filename", "invoice", RULE_TYPE_FILENAME, r"(?i)invoice|rechnung|facture"),
|
||||
ClassificationRule(
|
||||
"builtin_invoice_content",
|
||||
"invoice",
|
||||
RULE_TYPE_CONTENT,
|
||||
"invoice number|invoice total|amount due|rechnung|rechnungsnummer|total amount|bill to",
|
||||
),
|
||||
ClassificationRule("builtin_invoice_metadata", "invoice", RULE_TYPE_METADATA, "document_type=Invoice"),
|
||||
ClassificationRule(
|
||||
"builtin_invoice_kommunikationsart", "invoice", RULE_TYPE_METADATA, "kommunikationsart=Rechnung"
|
||||
),
|
||||
# ── Contract ──────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_contract_filename", "contract", RULE_TYPE_FILENAME, r"(?i)contract|vertrag|agreement"),
|
||||
ClassificationRule(
|
||||
"builtin_contract_content",
|
||||
"contract",
|
||||
RULE_TYPE_CONTENT,
|
||||
"hereby agrees|terms and conditions|vertrag|agreement between|party agrees|effective date",
|
||||
),
|
||||
ClassificationRule("builtin_contract_metadata", "contract", RULE_TYPE_METADATA, "document_type=Contract"),
|
||||
ClassificationRule(
|
||||
"builtin_contract_kommunikationsart", "contract", RULE_TYPE_METADATA, "kommunikationsart=Vertrag"
|
||||
),
|
||||
# ── Receipt ───────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_receipt_filename", "receipt", RULE_TYPE_FILENAME, r"(?i)receipt|quittung|beleg"),
|
||||
ClassificationRule(
|
||||
"builtin_receipt_content",
|
||||
"receipt",
|
||||
RULE_TYPE_CONTENT,
|
||||
"receipt|quittung|payment received|thank you for your purchase|transaction id",
|
||||
),
|
||||
ClassificationRule("builtin_receipt_metadata", "receipt", RULE_TYPE_METADATA, "document_type=Receipt"),
|
||||
ClassificationRule(
|
||||
"builtin_receipt_kommunikationsart", "receipt", RULE_TYPE_METADATA, "kommunikationsart=Quittung"
|
||||
),
|
||||
# ── Letter ────────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_letter_filename", "letter", RULE_TYPE_FILENAME, r"(?i)letter|brief|schreiben"),
|
||||
ClassificationRule(
|
||||
"builtin_letter_content",
|
||||
"letter",
|
||||
RULE_TYPE_CONTENT,
|
||||
"dear sir|dear madam|sehr geehrte|to whom it may concern|sincerely|mit freundlichen",
|
||||
),
|
||||
# ── Report ────────────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_report_filename", "report", RULE_TYPE_FILENAME, r"(?i)report|bericht"),
|
||||
ClassificationRule(
|
||||
"builtin_report_content",
|
||||
"report",
|
||||
RULE_TYPE_CONTENT,
|
||||
"executive summary|table of contents|annual report|quarterly report|findings",
|
||||
),
|
||||
# ── Bank statement ────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_bank_filename",
|
||||
"bank_statement",
|
||||
RULE_TYPE_FILENAME,
|
||||
r"(?i)bank.?statement|kontoauszug",
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_bank_content",
|
||||
"bank_statement",
|
||||
RULE_TYPE_CONTENT,
|
||||
"account statement|kontoauszug|opening balance|closing balance|account number",
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_bank_kommunikationsart", "bank_statement", RULE_TYPE_METADATA, "kommunikationsart=Kontoauszug"
|
||||
),
|
||||
# ── Tax document ──────────────────────────────────────────────────────
|
||||
ClassificationRule("builtin_tax_filename", "tax_document", RULE_TYPE_FILENAME, r"(?i)tax|steuer|steuerbescheid"),
|
||||
ClassificationRule(
|
||||
"builtin_tax_content",
|
||||
"tax_document",
|
||||
RULE_TYPE_CONTENT,
|
||||
"tax return|steuerbescheid|taxable income|finanzamt|tax assessment",
|
||||
),
|
||||
# ── Insurance ─────────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_insurance_filename", "insurance", RULE_TYPE_FILENAME, r"(?i)insurance|versicherung|police"
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_insurance_content",
|
||||
"insurance",
|
||||
RULE_TYPE_CONTENT,
|
||||
"insurance policy|versicherung|policennummer|coverage|premium|deductible",
|
||||
),
|
||||
# ── Payslip ───────────────────────────────────────────────────────────
|
||||
ClassificationRule(
|
||||
"builtin_payslip_filename", "payslip", RULE_TYPE_FILENAME, r"(?i)payslip|gehaltsabrechnung|lohnabrechnung"
|
||||
),
|
||||
ClassificationRule(
|
||||
"builtin_payslip_content",
|
||||
"payslip",
|
||||
RULE_TYPE_CONTENT,
|
||||
"gross salary|net salary|gehaltsabrechnung|lohnabrechnung|bruttolohn|nettolohn",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confidence scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Base confidence for each rule type when it matches.
|
||||
_CONFIDENCE_MAP: dict[str, int] = {
|
||||
RULE_TYPE_FILENAME: 60,
|
||||
RULE_TYPE_CONTENT: 70,
|
||||
RULE_TYPE_METADATA: 90,
|
||||
}
|
||||
|
||||
#: Extra confidence per additional matching rule of the same category (capped).
|
||||
_CONFIDENCE_BONUS_PER_EXTRA_RULE = 10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Matching helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _match_filename(rule: ClassificationRule, filename: str) -> bool:
|
||||
"""Return True if *rule.pattern* (regex) matches anywhere in *filename*."""
|
||||
if not filename:
|
||||
return False
|
||||
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
||||
return bool(re.search(rule.pattern, filename, flags))
|
||||
|
||||
|
||||
def _match_content(rule: ClassificationRule, text: str) -> bool:
|
||||
"""Return True if any keyword in *rule.pattern* appears in *text*.
|
||||
|
||||
Keywords are separated by ``|`` (pipe).
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
keywords = [kw.strip() for kw in rule.pattern.split("|") if kw.strip()]
|
||||
text_lower = text if rule.case_sensitive else text.lower()
|
||||
return any((kw if rule.case_sensitive else kw.lower()) in text_lower for kw in keywords)
|
||||
|
||||
|
||||
def _match_metadata(rule: ClassificationRule, metadata: dict[str, Any] | None) -> bool:
|
||||
"""Return True if *rule.pattern* (``field=value``) matches *metadata*.
|
||||
|
||||
Pattern format: ``field_name=expected_value``.
|
||||
"""
|
||||
if not metadata:
|
||||
return False
|
||||
if "=" not in rule.pattern:
|
||||
return False
|
||||
field_name, expected_value = rule.pattern.split("=", 1)
|
||||
actual = metadata.get(field_name.strip())
|
||||
if actual is None:
|
||||
return False
|
||||
if rule.case_sensitive:
|
||||
return str(actual) == expected_value.strip()
|
||||
return str(actual).lower() == expected_value.strip().lower()
|
||||
|
||||
|
||||
_MATCHERS: dict[str, tuple] = {
|
||||
RULE_TYPE_FILENAME: (_match_filename, "filename"),
|
||||
RULE_TYPE_CONTENT: (_match_content, "text"),
|
||||
RULE_TYPE_METADATA: (_match_metadata, "metadata"),
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_rule(
|
||||
rule: ClassificationRule,
|
||||
filename: str,
|
||||
text: str,
|
||||
metadata: dict[str, Any] | None,
|
||||
) -> MatchedRule | None:
|
||||
"""Evaluate a single rule against the document. Return a :class:`MatchedRule` on match."""
|
||||
entry = _MATCHERS.get(rule.rule_type)
|
||||
if entry is None:
|
||||
return None
|
||||
|
||||
matcher, arg_key = entry
|
||||
arg_map = {"filename": filename, "text": text, "metadata": metadata}
|
||||
matched = matcher(rule, arg_map[arg_key])
|
||||
|
||||
if matched:
|
||||
return MatchedRule(
|
||||
rule_name=rule.name,
|
||||
rule_type=rule.rule_type,
|
||||
category=rule.category,
|
||||
confidence=_CONFIDENCE_MAP.get(rule.rule_type, 50),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def classify_document(
|
||||
filename: str = "",
|
||||
text: str = "",
|
||||
metadata: dict[str, Any] | None = None,
|
||||
custom_rules: list[ClassificationRule] | None = None,
|
||||
) -> ClassificationResult:
|
||||
"""Classify a document by evaluating built-in and custom rules.
|
||||
|
||||
Rules are evaluated in priority order (highest first, then built-in before
|
||||
custom for the same priority). The category with the most rule matches
|
||||
wins; ties are broken by cumulative confidence.
|
||||
|
||||
Args:
|
||||
filename: Original filename of the document.
|
||||
text: Extracted / OCR text of the document.
|
||||
metadata: Previously-extracted AI metadata dict (e.g. from ``ai_metadata``).
|
||||
custom_rules: Optional list of user-defined :class:`ClassificationRule` objects.
|
||||
|
||||
Returns:
|
||||
A :class:`ClassificationResult` with the best matching category,
|
||||
overall confidence score, and the list of rules that fired.
|
||||
"""
|
||||
all_rules = list(BUILTIN_RULES)
|
||||
if custom_rules:
|
||||
all_rules.extend(custom_rules)
|
||||
|
||||
# Sort by priority descending (higher priority first)
|
||||
all_rules.sort(key=lambda r: r.priority, reverse=True)
|
||||
|
||||
matches: list[MatchedRule] = []
|
||||
for rule in all_rules:
|
||||
result = _evaluate_rule(rule, filename, text, metadata)
|
||||
if result is not None:
|
||||
matches.append(result)
|
||||
|
||||
if not matches:
|
||||
return ClassificationResult(category="unknown", confidence=0, matched_rules=[])
|
||||
|
||||
# Aggregate by category: pick the one with the most matches, then highest
|
||||
# cumulative confidence as tiebreaker.
|
||||
category_scores: dict[str, list[MatchedRule]] = {}
|
||||
for m in matches:
|
||||
category_scores.setdefault(m.category, []).append(m)
|
||||
|
||||
best_category = max(
|
||||
category_scores,
|
||||
key=lambda cat: (len(category_scores[cat]), sum(m.confidence for m in category_scores[cat])),
|
||||
)
|
||||
|
||||
best_matches = category_scores[best_category]
|
||||
base_confidence = max(m.confidence for m in best_matches)
|
||||
bonus = min(
|
||||
(len(best_matches) - 1) * _CONFIDENCE_BONUS_PER_EXTRA_RULE,
|
||||
100 - base_confidence,
|
||||
)
|
||||
final_confidence = min(base_confidence + bonus, 100)
|
||||
|
||||
return ClassificationResult(
|
||||
category=best_category,
|
||||
confidence=final_confidence,
|
||||
matched_rules=best_matches,
|
||||
)
|
||||
|
||||
|
||||
def db_rule_to_engine_rule(db_rule: Any) -> ClassificationRule:
|
||||
"""Convert a database ``ClassificationRuleModel`` row to an engine :class:`ClassificationRule`.
|
||||
|
||||
Args:
|
||||
db_rule: A SQLAlchemy model instance with ``name``, ``category``,
|
||||
``rule_type``, ``pattern``, ``priority``, and ``case_sensitive`` attributes.
|
||||
|
||||
Returns:
|
||||
A :class:`ClassificationRule` dataclass instance.
|
||||
"""
|
||||
return ClassificationRule(
|
||||
name=db_rule.name,
|
||||
category=db_rule.category,
|
||||
rule_type=db_rule.rule_type,
|
||||
pattern=db_rule.pattern,
|
||||
priority=db_rule.priority,
|
||||
case_sensitive=getattr(db_rule, "case_sensitive", False),
|
||||
)
|
||||
@@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]:
|
||||
},
|
||||
}
|
||||
|
||||
# Check SharePoint configuration
|
||||
providers["SharePoint"] = {
|
||||
"name": "SharePoint",
|
||||
"icon": "fa-brands fa-microsoft",
|
||||
"configured": bool(
|
||||
getattr(settings, "sharepoint_client_id", None)
|
||||
and getattr(settings, "sharepoint_client_secret", None)
|
||||
and getattr(settings, "sharepoint_site_url", None)
|
||||
),
|
||||
"enabled": True,
|
||||
"description": "Store documents in Microsoft SharePoint Online",
|
||||
"details": {
|
||||
"client_id": getattr(settings, "sharepoint_client_id", "Not set"),
|
||||
"client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)),
|
||||
"tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"),
|
||||
"refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)),
|
||||
"site_url": getattr(settings, "sharepoint_site_url", "Not set"),
|
||||
"document_library": getattr(settings, "sharepoint_document_library", "Not set"),
|
||||
"folder_path": getattr(settings, "sharepoint_folder_path", "Not set"),
|
||||
},
|
||||
}
|
||||
|
||||
# Check S3 configuration
|
||||
providers["S3 Storage"] = {
|
||||
"name": "S3 Storage",
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
"""Server-side session management utilities.
|
||||
|
||||
Provides helpers for creating, validating, and revoking user sessions.
|
||||
Sessions are tracked in the ``user_sessions`` table and referenced by a
|
||||
cryptographically random token stored in the browser cookie. This enables
|
||||
the "log off everywhere" feature and per-session revocation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models import ApiToken, QRLoginChallenge, UserSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_tz_aware(dt: datetime | None) -> datetime | None:
|
||||
"""Return *dt* with UTC tzinfo if it is naive, or unchanged if already aware.
|
||||
|
||||
SQLite does not persist timezone information, so datetimes read back from
|
||||
the database are offset-naive. This helper normalises them for safe
|
||||
comparison with ``datetime.now(timezone.utc)``.
|
||||
"""
|
||||
if dt is not None and dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def get_session_lifetime_days() -> int:
|
||||
"""Return the effective session lifetime in days.
|
||||
|
||||
If ``session_lifetime_custom_days`` is set it takes precedence over
|
||||
``session_lifetime_days``.
|
||||
"""
|
||||
custom = getattr(settings, "session_lifetime_custom_days", None)
|
||||
if custom is not None and isinstance(custom, int) and custom > 0:
|
||||
return custom
|
||||
return max(1, getattr(settings, "session_lifetime_days", 30))
|
||||
|
||||
|
||||
def get_session_max_age_seconds() -> int:
|
||||
"""Return the session max-age in seconds for the cookie."""
|
||||
return get_session_lifetime_days() * 86400
|
||||
|
||||
|
||||
def create_session(
|
||||
db: Session,
|
||||
user_id: str,
|
||||
ip_address: str | None = None,
|
||||
user_agent: str | None = None,
|
||||
) -> UserSession:
|
||||
"""Create a new server-side session record.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: Stable owner identifier.
|
||||
ip_address: Client IP address.
|
||||
user_agent: Client User-Agent header.
|
||||
|
||||
Returns:
|
||||
The newly created ``UserSession`` instance.
|
||||
"""
|
||||
session_token = secrets.token_urlsafe(64)
|
||||
now = datetime.now(timezone.utc)
|
||||
lifetime_days = get_session_lifetime_days()
|
||||
expires_at = now + timedelta(days=lifetime_days)
|
||||
|
||||
device_info = _parse_device_info(user_agent)
|
||||
|
||||
user_session = UserSession(
|
||||
session_token=session_token,
|
||||
user_id=user_id,
|
||||
ip_address=ip_address,
|
||||
user_agent=(user_agent or "")[:512],
|
||||
device_info=device_info,
|
||||
created_at=now,
|
||||
last_active_at=now,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
try:
|
||||
db.add(user_session)
|
||||
db.commit()
|
||||
db.refresh(user_session)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create session for user_id=%s", user_id)
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[SESSION] Created session id=%s user=%s device=%r expires=%s",
|
||||
user_session.id,
|
||||
user_id,
|
||||
device_info,
|
||||
expires_at.isoformat(),
|
||||
)
|
||||
return user_session
|
||||
|
||||
|
||||
def validate_session(db: Session, session_token: str) -> UserSession | None:
|
||||
"""Validate a session token and return the session if valid.
|
||||
|
||||
A session is valid when:
|
||||
* It exists in the database.
|
||||
* ``is_revoked`` is ``False``.
|
||||
* ``expires_at`` is in the future.
|
||||
|
||||
Side-effect: updates ``last_active_at`` on valid sessions.
|
||||
|
||||
Returns:
|
||||
The ``UserSession`` if valid, else ``None``.
|
||||
"""
|
||||
if not session_token:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
user_session = db.query(UserSession).filter(UserSession.session_token == session_token).first()
|
||||
|
||||
if not user_session:
|
||||
logger.debug("[SESSION] Token not found in database")
|
||||
return None
|
||||
|
||||
if user_session.is_revoked:
|
||||
logger.debug("[SESSION] Session id=%s is revoked", user_session.id)
|
||||
return None
|
||||
|
||||
if user_session.expires_at:
|
||||
expires = _ensure_tz_aware(user_session.expires_at)
|
||||
if expires < now:
|
||||
logger.debug("[SESSION] Session id=%s has expired", user_session.id)
|
||||
return None
|
||||
|
||||
# Update last_active_at (throttled to avoid excessive writes)
|
||||
last_active = _ensure_tz_aware(user_session.last_active_at)
|
||||
if not last_active or (now - last_active).total_seconds() > 60:
|
||||
try:
|
||||
user_session.last_active_at = now
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.debug("[SESSION] Failed to update last_active_at for session id=%s", user_session.id)
|
||||
|
||||
return user_session
|
||||
|
||||
|
||||
def revoke_session(db: Session, session_id: int, user_id: str) -> bool:
|
||||
"""Revoke a single session by ID.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
session_id: The session record ID to revoke.
|
||||
user_id: The owner — ensures a user can only revoke their own sessions.
|
||||
|
||||
Returns:
|
||||
``True`` if the session was found and revoked, ``False`` otherwise.
|
||||
"""
|
||||
user_session = db.get(UserSession, session_id)
|
||||
if not user_session or user_session.user_id != user_id:
|
||||
return False
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
user_session.is_revoked = True
|
||||
user_session.revoked_at = now
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("[SESSION] Revoked session id=%s user=%s", session_id, user_id)
|
||||
return True
|
||||
|
||||
|
||||
def revoke_all_sessions(
|
||||
db: Session,
|
||||
user_id: str,
|
||||
*,
|
||||
except_session_id: int | None = None,
|
||||
revoke_api_tokens: bool = True,
|
||||
) -> int:
|
||||
"""Revoke all active sessions for a user ("log off everywhere").
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: The owner whose sessions should be revoked.
|
||||
except_session_id: If provided, keep this session active (the
|
||||
current browser session).
|
||||
revoke_api_tokens: If ``True``, also revoke all active API tokens.
|
||||
|
||||
Returns:
|
||||
Number of sessions revoked.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
query = db.query(UserSession).filter(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.is_revoked.is_(False),
|
||||
)
|
||||
if except_session_id is not None:
|
||||
query = query.filter(UserSession.id != except_session_id)
|
||||
|
||||
sessions = query.all()
|
||||
count = 0
|
||||
for s in sessions:
|
||||
s.is_revoked = True
|
||||
s.revoked_at = now
|
||||
count += 1
|
||||
|
||||
if revoke_api_tokens:
|
||||
tokens = (
|
||||
db.query(ApiToken)
|
||||
.filter(
|
||||
ApiToken.owner_id == user_id,
|
||||
ApiToken.is_active.is_(True),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for t in tokens:
|
||||
t.is_active = False
|
||||
t.revoked_at = now
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[SESSION] Revoked all sessions for user=%s (count=%d, except_session_id=%s, tokens_revoked=%s)",
|
||||
user_id,
|
||||
count,
|
||||
except_session_id,
|
||||
revoke_api_tokens,
|
||||
)
|
||||
return count
|
||||
|
||||
|
||||
def list_user_sessions(db: Session, user_id: str) -> list[UserSession]:
|
||||
"""Return all non-revoked, non-expired sessions for a user.
|
||||
|
||||
Results are ordered by most recently active first.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
sessions = (
|
||||
db.query(UserSession)
|
||||
.filter(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.is_revoked.is_(False),
|
||||
)
|
||||
.order_by(UserSession.last_active_at.desc())
|
||||
.all()
|
||||
)
|
||||
# Filter expired sessions in Python to handle timezone-naive datetimes (SQLite)
|
||||
result = []
|
||||
for s in sessions:
|
||||
expires = _ensure_tz_aware(s.expires_at)
|
||||
if expires and expires > now:
|
||||
result.append(s)
|
||||
return result
|
||||
|
||||
|
||||
def cleanup_expired_sessions(db: Session) -> int:
|
||||
"""Delete sessions that expired more than 7 days ago.
|
||||
|
||||
Intended to be called periodically (e.g. via Celery beat) to keep the
|
||||
table from growing unbounded.
|
||||
|
||||
Returns:
|
||||
Number of rows deleted.
|
||||
"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=7)
|
||||
count = db.query(UserSession).filter(UserSession.expires_at < cutoff).delete(synchronize_session=False)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
if count:
|
||||
logger.info("[SESSION] Cleaned up %d expired sessions", count)
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QR login helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_qr_challenge(db: Session, user_id: str, ip_address: str | None = None) -> QRLoginChallenge:
|
||||
"""Create a new QR login challenge.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
user_id: The authenticated web user creating the challenge.
|
||||
ip_address: IP address of the web client.
|
||||
|
||||
Returns:
|
||||
The newly created ``QRLoginChallenge``.
|
||||
"""
|
||||
token = secrets.token_urlsafe(64)
|
||||
ttl = getattr(settings, "qr_login_challenge_ttl_seconds", 120)
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + timedelta(seconds=ttl)
|
||||
|
||||
challenge = QRLoginChallenge(
|
||||
challenge_token=token,
|
||||
user_id=user_id,
|
||||
created_by_ip=ip_address,
|
||||
created_at=now,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
try:
|
||||
db.add(challenge)
|
||||
db.commit()
|
||||
db.refresh(challenge)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Failed to create QR login challenge for user_id=%s", user_id)
|
||||
raise
|
||||
|
||||
logger.info("[QR_AUTH] Challenge created: id=%s user=%s expires=%s", challenge.id, user_id, expires_at.isoformat())
|
||||
return challenge
|
||||
|
||||
|
||||
def validate_qr_challenge(db: Session, challenge_token: str) -> QRLoginChallenge | None:
|
||||
"""Validate a QR challenge token without claiming it.
|
||||
|
||||
Returns the challenge if it exists, is not expired, not claimed,
|
||||
and not cancelled. Returns ``None`` otherwise.
|
||||
"""
|
||||
if not challenge_token:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
challenge = db.query(QRLoginChallenge).filter(QRLoginChallenge.challenge_token == challenge_token).first()
|
||||
|
||||
if not challenge:
|
||||
return None
|
||||
if challenge.is_claimed or challenge.is_cancelled:
|
||||
return None
|
||||
|
||||
expires = _ensure_tz_aware(challenge.expires_at)
|
||||
if expires and expires < now:
|
||||
return None
|
||||
|
||||
return challenge
|
||||
|
||||
|
||||
def claim_qr_challenge(
|
||||
db: Session,
|
||||
challenge_token: str,
|
||||
device_name: str = "Mobile App",
|
||||
ip_address: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Claim a QR challenge and issue an API token.
|
||||
|
||||
This is the critical security path. The challenge is validated,
|
||||
marked as claimed atomically, and an API token is issued for the
|
||||
user who created the challenge.
|
||||
|
||||
Args:
|
||||
db: Database session.
|
||||
challenge_token: The token from the QR code.
|
||||
device_name: Name provided by the mobile app.
|
||||
ip_address: IP address of the claiming mobile device.
|
||||
|
||||
Returns:
|
||||
Dict with ``token`` (plaintext), ``token_id``, ``name``, ``owner_id``
|
||||
and ``created_at`` on success, or ``None`` if the challenge is invalid.
|
||||
"""
|
||||
from app.api.api_tokens import generate_api_token, hash_token
|
||||
|
||||
challenge = validate_qr_challenge(db, challenge_token)
|
||||
if not challenge:
|
||||
logger.warning("[QR_AUTH] Invalid or expired challenge token attempted")
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Mark as claimed first to prevent race conditions
|
||||
challenge.is_claimed = True
|
||||
challenge.claimed_at = now
|
||||
challenge.claimed_by_ip = ip_address
|
||||
challenge.device_name = device_name
|
||||
|
||||
# Generate API token for the mobile app
|
||||
token_name = f"Mobile App (QR) – {device_name}"
|
||||
plaintext = generate_api_token()
|
||||
token_hash_value = hash_token(plaintext)
|
||||
prefix = plaintext[:12]
|
||||
|
||||
db_token = ApiToken(
|
||||
owner_id=challenge.user_id,
|
||||
name=token_name,
|
||||
token_hash=token_hash_value,
|
||||
token_prefix=prefix,
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(db_token)
|
||||
db.flush()
|
||||
challenge.issued_token_id = db_token.id
|
||||
db.commit()
|
||||
db.refresh(db_token)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("[QR_AUTH] Failed to issue token for challenge id=%s", challenge.id)
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"[QR_AUTH] Challenge claimed: id=%s user=%s device=%r token_id=%s",
|
||||
challenge.id,
|
||||
challenge.user_id,
|
||||
device_name,
|
||||
db_token.id,
|
||||
)
|
||||
|
||||
return {
|
||||
"token": plaintext,
|
||||
"token_id": db_token.id,
|
||||
"name": token_name,
|
||||
"owner_id": challenge.user_id,
|
||||
"created_at": db_token.created_at,
|
||||
}
|
||||
|
||||
|
||||
def get_challenge_status(db: Session, challenge_id: int, user_id: str) -> dict | None:
|
||||
"""Get the current status of a QR challenge (for polling from the web UI).
|
||||
|
||||
Returns:
|
||||
Dict with ``status`` ("pending", "claimed", "expired", "cancelled")
|
||||
and metadata, or ``None`` if the challenge doesn't belong to the user.
|
||||
"""
|
||||
challenge = db.get(QRLoginChallenge, challenge_id)
|
||||
if not challenge or challenge.user_id != user_id:
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
expires = _ensure_tz_aware(challenge.expires_at)
|
||||
|
||||
if challenge.is_claimed:
|
||||
status = "claimed"
|
||||
elif challenge.is_cancelled:
|
||||
status = "cancelled"
|
||||
elif expires and expires < now:
|
||||
status = "expired"
|
||||
else:
|
||||
status = "pending"
|
||||
|
||||
return {
|
||||
"id": challenge.id,
|
||||
"status": status,
|
||||
"device_name": challenge.device_name,
|
||||
"claimed_at": challenge.claimed_at,
|
||||
"expires_at": challenge.expires_at,
|
||||
}
|
||||
|
||||
|
||||
def _parse_device_info(user_agent: str | None) -> str | None:
|
||||
"""Extract a human-readable device description from User-Agent.
|
||||
|
||||
This is a lightweight parser — not a full UA library — that covers
|
||||
the most common browsers and platforms.
|
||||
"""
|
||||
if not user_agent:
|
||||
return None
|
||||
|
||||
ua = user_agent.lower()
|
||||
|
||||
# Platform detection
|
||||
platform = "Unknown"
|
||||
if "iphone" in ua:
|
||||
platform = "iPhone"
|
||||
elif "ipad" in ua:
|
||||
platform = "iPad"
|
||||
elif "android" in ua:
|
||||
platform = "Android"
|
||||
elif "macintosh" in ua or "mac os" in ua:
|
||||
platform = "macOS"
|
||||
elif "windows" in ua:
|
||||
platform = "Windows"
|
||||
elif "linux" in ua:
|
||||
platform = "Linux"
|
||||
elif "cros" in ua:
|
||||
platform = "ChromeOS"
|
||||
|
||||
# Browser detection
|
||||
browser = "Unknown Browser"
|
||||
if "edg/" in ua or "edge/" in ua:
|
||||
browser = "Edge"
|
||||
elif "opr/" in ua or "opera" in ua:
|
||||
browser = "Opera"
|
||||
elif "chrome/" in ua and "safari/" in ua:
|
||||
browser = "Chrome"
|
||||
elif "safari/" in ua and "chrome/" not in ua:
|
||||
browser = "Safari"
|
||||
elif "firefox/" in ua:
|
||||
browser = "Firefox"
|
||||
elif "docuelevate" in ua:
|
||||
browser = "DocuElevate App"
|
||||
|
||||
return f"{browser} on {platform}"
|
||||
@@ -39,6 +39,50 @@ SETTING_METADATA = {
|
||||
"required": True,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_size": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Number of persistent database connections kept in the pool per worker process. "
|
||||
"Ignored for SQLite (which uses NullPool). Default: 10."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_max_overflow": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Additional database connections allowed beyond db_pool_size under burst load. "
|
||||
"Ignored for SQLite. Default: 20."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_timeout": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Seconds to wait for a database connection from the pool before raising a TimeoutError. "
|
||||
"Ignored for SQLite. Default: 30."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"db_pool_recycle": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Recycle (close and reopen) database connections after this many seconds "
|
||||
"to avoid stale connections. Ignored for SQLite. Default: 1800."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"workdir": {
|
||||
"category": "Core",
|
||||
"description": "Working directory for file storage and processing",
|
||||
@@ -55,6 +99,18 @@ SETTING_METADATA = {
|
||||
"required": True, # Required for OAuth redirects and external URLs
|
||||
"restart_required": True,
|
||||
},
|
||||
"public_base_url": {
|
||||
"category": "Core",
|
||||
"description": (
|
||||
"Full public base URL including scheme (e.g., https://docuelevate.example.com). "
|
||||
"When set, overrides auto-detected URLs for OAuth redirect URIs. "
|
||||
"Required when behind a reverse proxy that does not forward X-Forwarded-Proto."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"debug": {
|
||||
"category": "Core",
|
||||
"description": "Enable debug mode for verbose logging",
|
||||
@@ -134,6 +190,30 @@ SETTING_METADATA = {
|
||||
"required": True, # Required when auth_enabled=True (validated in config.py)
|
||||
"restart_required": True,
|
||||
},
|
||||
"session_lifetime_days": {
|
||||
"category": "Authentication",
|
||||
"description": "Session lifetime in days (default 30). Determines how long a user stays logged in.",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"session_lifetime_custom_days": {
|
||||
"category": "Authentication",
|
||||
"description": "Override session_lifetime_days with a custom value. Takes precedence when set.",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"qr_login_challenge_ttl_seconds": {
|
||||
"category": "Authentication",
|
||||
"description": "Time-to-live in seconds for QR login challenges (default 120).",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"admin_username": {
|
||||
"category": "Authentication",
|
||||
"description": "Admin username for local authentication",
|
||||
@@ -302,6 +382,19 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_dropbox_use_global_credentials": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
"When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET "
|
||||
"credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / "
|
||||
"SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. "
|
||||
"Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"social_auth_dropbox_enabled": {
|
||||
"category": "Social Login",
|
||||
"description": (
|
||||
@@ -695,6 +788,18 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"dropbox_allow_global_credentials_for_integrations": {
|
||||
"category": "Storage Providers",
|
||||
"description": (
|
||||
"When True, users may authorize their personal Dropbox integrations using the global "
|
||||
"DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without "
|
||||
"needing to create their own Dropbox app."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - Nextcloud
|
||||
"nextcloud_enabled": {
|
||||
"category": "Storage Providers",
|
||||
@@ -875,6 +980,63 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - SharePoint
|
||||
"sharepoint_client_id": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD application (client) ID",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_client_secret": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD client secret",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_tenant_id": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_refresh_token": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint OAuth refresh token",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_site_url": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_document_library": {
|
||||
"category": "Storage Providers",
|
||||
"description": "SharePoint document library name (default: 'Documents')",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sharepoint_folder_path": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Subfolder path inside the SharePoint document library",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - WebDAV
|
||||
"webdav_enabled": {
|
||||
"category": "Storage Providers",
|
||||
@@ -1912,6 +2074,28 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"factory_reset_on_startup": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Wipe all user data on every startup so the instance always starts fresh. "
|
||||
"Useful for demo/testing environments. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"enable_factory_reset": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Show the System Reset page in the admin UI. Allows administrators to "
|
||||
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Backup / Restore
|
||||
"backup_enabled": {
|
||||
"category": "Backup",
|
||||
@@ -1935,14 +2119,26 @@ SETTING_METADATA = {
|
||||
"category": "Backup",
|
||||
"description": (
|
||||
"Storage provider for remote backup copies. "
|
||||
"Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. "
|
||||
"Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. "
|
||||
"Leave empty to keep backups local only."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
"options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"],
|
||||
"options": [
|
||||
"",
|
||||
"s3",
|
||||
"dropbox",
|
||||
"google_drive",
|
||||
"onedrive",
|
||||
"sharepoint",
|
||||
"nextcloud",
|
||||
"webdav",
|
||||
"ftp",
|
||||
"sftp",
|
||||
"email",
|
||||
],
|
||||
},
|
||||
"backup_remote_folder": {
|
||||
"category": "Backup",
|
||||
@@ -2454,6 +2650,27 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Per-user upload rate limiting
|
||||
"upload_rate_limit_per_user": {
|
||||
"category": "Security",
|
||||
"description": (
|
||||
"Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. "
|
||||
"The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. "
|
||||
"Default: 20."
|
||||
),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"upload_rate_limit_window": {
|
||||
"category": "Security",
|
||||
"description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."),
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Rate Limiting
|
||||
"rate_limiting_enabled": {
|
||||
"category": "Security",
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
"""
|
||||
System reset utilities for DocuElevate.
|
||||
|
||||
Provides functions to:
|
||||
- Wipe all user data (database rows + work-files on disk) for a fresh start.
|
||||
- Wipe with re-import: move original files to a dedicated folder, wipe
|
||||
everything, then let the watch-folder mechanism re-ingest the files.
|
||||
|
||||
Security: All public functions in this module require admin-level access.
|
||||
They MUST only be invoked from admin-guarded API/view endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectories inside *workdir* that contain user-generated data.
|
||||
# Everything else (app code, static assets, config) is left untouched.
|
||||
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
|
||||
|
||||
# JSON cache files written by watch-folder / ingest tasks.
|
||||
_CACHE_FILES = (
|
||||
"watch_folder_processed.json",
|
||||
"ftp_ingest_processed.json",
|
||||
"sftp_ingest_processed.json",
|
||||
"dropbox_ingest_processed.json",
|
||||
"gdrive_ingest_processed.json",
|
||||
"onedrive_ingest_processed.json",
|
||||
"nextcloud_ingest_processed.json",
|
||||
"s3_ingest_processed.json",
|
||||
"webdav_ingest_processed.json",
|
||||
"processed_mails.json",
|
||||
"credential_failures.json",
|
||||
)
|
||||
|
||||
# The folder name used for storing files prior to re-import.
|
||||
REIMPORT_FOLDER_NAME = "reimport"
|
||||
|
||||
|
||||
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
|
||||
"""Delete user data subdirectories and cache files inside *workdir*.
|
||||
|
||||
Leaves the workdir directory itself intact so the application can
|
||||
continue to write into it. Also leaves any files that do not belong
|
||||
to the known data subdirectories or caches.
|
||||
|
||||
Returns:
|
||||
A dict with counts of deleted directories and files.
|
||||
"""
|
||||
workdir_path = Path(workdir)
|
||||
deleted_dirs = 0
|
||||
deleted_files = 0
|
||||
|
||||
# Remove data subdirectories
|
||||
for subdir in _USER_DATA_SUBDIRS:
|
||||
target = workdir_path / subdir
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
logger.info("Deleted data directory: %s", target)
|
||||
deleted_dirs += 1
|
||||
|
||||
# Remove cache / state JSON files
|
||||
for cache_file in _CACHE_FILES:
|
||||
target = workdir_path / cache_file
|
||||
if target.is_file():
|
||||
target.unlink()
|
||||
logger.info("Deleted cache file: %s", target)
|
||||
deleted_files += 1
|
||||
|
||||
# Also remove user_wf_*.json files (per-user watch folder caches)
|
||||
for f in workdir_path.glob("user_wf_*.json"):
|
||||
f.unlink()
|
||||
logger.info("Deleted user watch-folder cache: %s", f)
|
||||
deleted_files += 1
|
||||
|
||||
# Remove loose files in workdir root that are user uploads (uuid-named
|
||||
# files like "a1b2c3d4-…pdf") but NOT application config files.
|
||||
for entry in workdir_path.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() in {
|
||||
".pdf",
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".tiff",
|
||||
".tif",
|
||||
".docx",
|
||||
".doc",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".pptx",
|
||||
".heic",
|
||||
".heif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".gif",
|
||||
".txt",
|
||||
".rtf",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".csv",
|
||||
".pages",
|
||||
".numbers",
|
||||
".keynote",
|
||||
}:
|
||||
entry.unlink()
|
||||
logger.info("Deleted loose workdir file: %s", entry)
|
||||
deleted_files += 1
|
||||
|
||||
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
|
||||
|
||||
|
||||
def _wipe_database(db: Session) -> dict[str, int]:
|
||||
"""Delete all user-generated rows from the database.
|
||||
|
||||
Preserves schema (tables, migrations) and system-seeded rows that will
|
||||
be re-created on the next startup (subscription plans, default pipeline,
|
||||
scheduled jobs, compliance templates).
|
||||
|
||||
Returns:
|
||||
A dict mapping table name → number of rows deleted.
|
||||
"""
|
||||
from app.models import (
|
||||
AuditLog,
|
||||
BackupRecord,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
InAppNotification,
|
||||
ProcessingLog,
|
||||
SavedSearch,
|
||||
SettingsAuditLog,
|
||||
SharedLink,
|
||||
UserImapAccount,
|
||||
UserIntegration,
|
||||
UserNotificationPreference,
|
||||
UserNotificationTarget,
|
||||
)
|
||||
|
||||
# Order matters: delete children before parents to respect FK constraints.
|
||||
tables_to_wipe: list[tuple[str, type]] = [
|
||||
("file_processing_steps", FileProcessingStep),
|
||||
("processing_logs", ProcessingLog),
|
||||
("shared_links", SharedLink),
|
||||
("in_app_notifications", InAppNotification),
|
||||
("user_notification_preferences", UserNotificationPreference),
|
||||
("user_notification_targets", UserNotificationTarget),
|
||||
("user_imap_accounts", UserImapAccount),
|
||||
("user_integrations", UserIntegration),
|
||||
("saved_searches", SavedSearch),
|
||||
("settings_audit_log", SettingsAuditLog),
|
||||
("audit_logs", AuditLog),
|
||||
("backup_records", BackupRecord),
|
||||
("document_metadata", DocumentMetadata),
|
||||
("files", FileRecord),
|
||||
]
|
||||
|
||||
result: dict[str, int] = {}
|
||||
for table_name, model in tables_to_wipe:
|
||||
try:
|
||||
count = db.query(model).delete()
|
||||
result[table_name] = count
|
||||
logger.info("Wiped %d rows from %s", count, table_name)
|
||||
except Exception:
|
||||
logger.exception("Failed to wipe table %s during system reset", table_name)
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
db.commit()
|
||||
return result
|
||||
|
||||
|
||||
def perform_full_reset(db: Session) -> dict:
|
||||
"""Perform a complete system reset: wipe database rows + work-files.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database`` and ``filesystem`` sub-dicts.
|
||||
"""
|
||||
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
|
||||
|
||||
db_result = _wipe_database(db)
|
||||
fs_result = _wipe_workdir_data(settings.workdir)
|
||||
|
||||
logger.warning(">>> SYSTEM RESET complete <<<")
|
||||
return {"database": db_result, "filesystem": fs_result}
|
||||
|
||||
|
||||
def perform_reset_and_reimport(db: Session) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, then
|
||||
configure the reimport folder as a watch folder for re-ingestion.
|
||||
|
||||
The watch-folder scanner (``scan_all_watch_folders``) will pick up
|
||||
the files on its next periodic run and process them exactly as if
|
||||
they had been freshly uploaded — respecting the same backoff
|
||||
strategy, size limits, and rate limits.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
|
||||
"""
|
||||
workdir_path = Path(settings.workdir)
|
||||
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
|
||||
original_dir = workdir_path / "original"
|
||||
|
||||
# 1. Collect original files
|
||||
files_moved = 0
|
||||
reimport_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if original_dir.is_dir():
|
||||
for entry in original_dir.iterdir():
|
||||
if entry.is_file():
|
||||
# Validate the resolved path stays within original_dir (path traversal guard)
|
||||
try:
|
||||
entry.resolve().relative_to(original_dir.resolve())
|
||||
except ValueError:
|
||||
logger.warning("Skipping file outside original dir: %s", entry)
|
||||
continue
|
||||
dest = reimport_dir / entry.name
|
||||
# Avoid overwriting: append counter if name clash
|
||||
if dest.exists():
|
||||
stem = dest.stem
|
||||
suffix = dest.suffix
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = reimport_dir / f"{stem}_{counter}{suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(str(entry), str(dest))
|
||||
files_moved += 1
|
||||
|
||||
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
|
||||
|
||||
# 2. Perform the full reset (wipe DB + other workdir data)
|
||||
reset_result = perform_full_reset(db)
|
||||
|
||||
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
|
||||
# and set up watch folder config to point at it.
|
||||
_configure_reimport_watch_folder(str(reimport_dir))
|
||||
|
||||
reset_result["reimport"] = {
|
||||
"files_moved": files_moved,
|
||||
"reimport_folder": str(reimport_dir),
|
||||
}
|
||||
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
|
||||
return reset_result
|
||||
|
||||
|
||||
def _configure_reimport_watch_folder(reimport_path: str) -> None:
|
||||
"""Append *reimport_path* to the application's watch-folder list.
|
||||
|
||||
The watch-folder scanner uses ``settings.watch_folders`` (a
|
||||
comma-separated string). We mutate the runtime setting so the
|
||||
next scan picks up the folder. We also set
|
||||
``watch_folder_delete_after_process = True`` so files are cleaned
|
||||
up after successful processing.
|
||||
"""
|
||||
current = getattr(settings, "watch_folders", None) or ""
|
||||
folders = [f.strip() for f in current.split(",") if f.strip()]
|
||||
|
||||
if reimport_path not in folders:
|
||||
folders.append(reimport_path)
|
||||
|
||||
# Mutate runtime settings (not persisted to .env — ephemeral)
|
||||
object.__setattr__(settings, "watch_folders", ",".join(folders))
|
||||
object.__setattr__(settings, "watch_folder_delete_after_process", True)
|
||||
logger.info("Configured reimport watch folder: %s", reimport_path)
|
||||
|
||||
|
||||
def perform_startup_reset() -> None:
|
||||
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
|
||||
|
||||
Wipes database and filesystem data so the instance starts completely
|
||||
fresh. Uses its own DB session so it runs before the normal lifespan
|
||||
seeding logic.
|
||||
"""
|
||||
from app.database import SessionLocal
|
||||
|
||||
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
perform_full_reset(db)
|
||||
except Exception:
|
||||
logger.exception("Factory reset on startup failed")
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
Reference in New Issue
Block a user