fix: merge main branch and renumber migration 037→040

Resolve all merge conflicts between our automation feature branch and
current main (v0.163.0, 920 commits ahead).

Conflicts resolved:
- app/api/__init__.py: add automation_router alongside main's new routers
  (classification_rules, qr_auth, sessions, system_reset)
- app/config.py: add main's new settings (dropbox_use_global_credentials,
  factory_reset_on_startup, enable_factory_reset)
- app/models.py: add main's new models (ClassificationRuleModel, UserSession,
  QRLoginChallenge, SharePoint integration type)
- app/utils/settings_service.py: merge automation_hooks_enabled with main's
  new metadata entries
- docs/API.md: merge automation API docs with main's classification rules docs
- docs/ConfigurationGuide.md: add factory reset settings
- tests/conftest.py: import both AutomationHook and new main models

Migration renumbered:
- 037_add_automation_hooks → 040_add_automation_hooks
- down_revision: 039_add_classification_rules (was 036_add_document_translation_fields)
- Chain: 036 → 037 → 038 → 039 → 040 (automation hooks)

For all non-automation files with conflicts, main's version was taken since
our branch did not modify those files (conflicts were from a stale prior merge).

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/cb62f012-3b69-4415-835e-3857ce3e9f45
This commit is contained in:
copilot-swe-agent[bot]
2026-03-20 23:54:04 +00:00
parent d1ebac74a1
commit e518bce922
153 changed files with 16306 additions and 922 deletions
+9 -1
View File
@@ -68,6 +68,8 @@ IMAGE_MIME_TYPES: set[str] = {
"image/tiff",
"image/webp",
"image/svg+xml",
"image/heic",
"image/heif",
}
# ---------------------------------------------------------------------------
@@ -124,6 +126,8 @@ ALLOWED_EXTENSIONS: set[str] = {
".tif",
".webp",
".svg",
".heic",
".heif",
# Web
".html",
".htm",
@@ -234,7 +238,7 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg, .heic, .heif)",
"mime_types": frozenset(
{
"image/jpeg",
@@ -245,6 +249,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
"image/tiff",
"image/webp",
"image/svg+xml",
"image/heic",
"image/heif",
}
),
"extensions": frozenset(
@@ -258,6 +264,8 @@ FILE_TYPE_CATEGORIES: dict[str, dict] = {
".tif",
".webp",
".svg",
".heic",
".heif",
}
),
},
+378
View File
@@ -0,0 +1,378 @@
"""
Rule-based document classification engine.
Provides pre-built categories and a rule matcher that classifies documents
using filename patterns, content keywords, and metadata fields. Custom
rules stored in the database are evaluated alongside the built-in defaults.
Usage::
from app.utils.classification_rules import classify_document
result = classify_document(
filename="2024-03-01_Invoice_Acme.pdf",
text="Invoice total: $1,234.56",
metadata={"absender": "Acme Corp"},
custom_rules=custom_rules_from_db,
)
# result -> ClassificationResult(category="invoice", confidence=85, matched_rules=[...])
"""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass, field
from typing import Any
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Pre-built categories
# ---------------------------------------------------------------------------
#: Canonical category names recognized by the system. Users may also define
#: their own categories via custom rules.
BUILTIN_CATEGORIES: dict[str, str] = {
"invoice": "Invoice",
"contract": "Contract",
"receipt": "Receipt",
"letter": "Letter",
"report": "Report",
"bank_statement": "Bank Statement",
"tax_document": "Tax Document",
"insurance": "Insurance Document",
"payslip": "Payslip",
"unknown": "Unknown",
}
# ---------------------------------------------------------------------------
# Rule type constants
# ---------------------------------------------------------------------------
RULE_TYPE_FILENAME = "filename_pattern"
RULE_TYPE_CONTENT = "content_keyword"
RULE_TYPE_METADATA = "metadata_match"
# ---------------------------------------------------------------------------
# Data classes
# ---------------------------------------------------------------------------
@dataclass
class ClassificationRule:
"""A single classification rule."""
name: str
category: str
rule_type: str # filename_pattern | content_keyword | metadata_match
pattern: str # regex for filename, keyword(s) for content, "field=value" for metadata
priority: int = 0 # higher = evaluated first
case_sensitive: bool = False
def __post_init__(self) -> None:
if self.rule_type not in (RULE_TYPE_FILENAME, RULE_TYPE_CONTENT, RULE_TYPE_METADATA):
raise ValueError(f"Invalid rule_type: {self.rule_type!r}")
@dataclass
class MatchedRule:
"""Records which rule matched and why."""
rule_name: str
rule_type: str
category: str
confidence: int
@dataclass
class ClassificationResult:
"""The outcome of running the classification engine on a document."""
category: str
confidence: int # 0 – 100
matched_rules: list[MatchedRule] = field(default_factory=list)
# ---------------------------------------------------------------------------
# Built-in rules
# ---------------------------------------------------------------------------
BUILTIN_RULES: list[ClassificationRule] = [
# ── Invoice ───────────────────────────────────────────────────────────
ClassificationRule("builtin_invoice_filename", "invoice", RULE_TYPE_FILENAME, r"(?i)invoice|rechnung|facture"),
ClassificationRule(
"builtin_invoice_content",
"invoice",
RULE_TYPE_CONTENT,
"invoice number|invoice total|amount due|rechnung|rechnungsnummer|total amount|bill to",
),
ClassificationRule("builtin_invoice_metadata", "invoice", RULE_TYPE_METADATA, "document_type=Invoice"),
ClassificationRule(
"builtin_invoice_kommunikationsart", "invoice", RULE_TYPE_METADATA, "kommunikationsart=Rechnung"
),
# ── Contract ──────────────────────────────────────────────────────────
ClassificationRule("builtin_contract_filename", "contract", RULE_TYPE_FILENAME, r"(?i)contract|vertrag|agreement"),
ClassificationRule(
"builtin_contract_content",
"contract",
RULE_TYPE_CONTENT,
"hereby agrees|terms and conditions|vertrag|agreement between|party agrees|effective date",
),
ClassificationRule("builtin_contract_metadata", "contract", RULE_TYPE_METADATA, "document_type=Contract"),
ClassificationRule(
"builtin_contract_kommunikationsart", "contract", RULE_TYPE_METADATA, "kommunikationsart=Vertrag"
),
# ── Receipt ───────────────────────────────────────────────────────────
ClassificationRule("builtin_receipt_filename", "receipt", RULE_TYPE_FILENAME, r"(?i)receipt|quittung|beleg"),
ClassificationRule(
"builtin_receipt_content",
"receipt",
RULE_TYPE_CONTENT,
"receipt|quittung|payment received|thank you for your purchase|transaction id",
),
ClassificationRule("builtin_receipt_metadata", "receipt", RULE_TYPE_METADATA, "document_type=Receipt"),
ClassificationRule(
"builtin_receipt_kommunikationsart", "receipt", RULE_TYPE_METADATA, "kommunikationsart=Quittung"
),
# ── Letter ────────────────────────────────────────────────────────────
ClassificationRule("builtin_letter_filename", "letter", RULE_TYPE_FILENAME, r"(?i)letter|brief|schreiben"),
ClassificationRule(
"builtin_letter_content",
"letter",
RULE_TYPE_CONTENT,
"dear sir|dear madam|sehr geehrte|to whom it may concern|sincerely|mit freundlichen",
),
# ── Report ────────────────────────────────────────────────────────────
ClassificationRule("builtin_report_filename", "report", RULE_TYPE_FILENAME, r"(?i)report|bericht"),
ClassificationRule(
"builtin_report_content",
"report",
RULE_TYPE_CONTENT,
"executive summary|table of contents|annual report|quarterly report|findings",
),
# ── Bank statement ────────────────────────────────────────────────────
ClassificationRule(
"builtin_bank_filename",
"bank_statement",
RULE_TYPE_FILENAME,
r"(?i)bank.?statement|kontoauszug",
),
ClassificationRule(
"builtin_bank_content",
"bank_statement",
RULE_TYPE_CONTENT,
"account statement|kontoauszug|opening balance|closing balance|account number",
),
ClassificationRule(
"builtin_bank_kommunikationsart", "bank_statement", RULE_TYPE_METADATA, "kommunikationsart=Kontoauszug"
),
# ── Tax document ──────────────────────────────────────────────────────
ClassificationRule("builtin_tax_filename", "tax_document", RULE_TYPE_FILENAME, r"(?i)tax|steuer|steuerbescheid"),
ClassificationRule(
"builtin_tax_content",
"tax_document",
RULE_TYPE_CONTENT,
"tax return|steuerbescheid|taxable income|finanzamt|tax assessment",
),
# ── Insurance ─────────────────────────────────────────────────────────
ClassificationRule(
"builtin_insurance_filename", "insurance", RULE_TYPE_FILENAME, r"(?i)insurance|versicherung|police"
),
ClassificationRule(
"builtin_insurance_content",
"insurance",
RULE_TYPE_CONTENT,
"insurance policy|versicherung|policennummer|coverage|premium|deductible",
),
# ── Payslip ───────────────────────────────────────────────────────────
ClassificationRule(
"builtin_payslip_filename", "payslip", RULE_TYPE_FILENAME, r"(?i)payslip|gehaltsabrechnung|lohnabrechnung"
),
ClassificationRule(
"builtin_payslip_content",
"payslip",
RULE_TYPE_CONTENT,
"gross salary|net salary|gehaltsabrechnung|lohnabrechnung|bruttolohn|nettolohn",
),
]
# ---------------------------------------------------------------------------
# Confidence scoring
# ---------------------------------------------------------------------------
#: Base confidence for each rule type when it matches.
_CONFIDENCE_MAP: dict[str, int] = {
RULE_TYPE_FILENAME: 60,
RULE_TYPE_CONTENT: 70,
RULE_TYPE_METADATA: 90,
}
#: Extra confidence per additional matching rule of the same category (capped).
_CONFIDENCE_BONUS_PER_EXTRA_RULE = 10
# ---------------------------------------------------------------------------
# Matching helpers
# ---------------------------------------------------------------------------
def _match_filename(rule: ClassificationRule, filename: str) -> bool:
"""Return True if *rule.pattern* (regex) matches anywhere in *filename*."""
if not filename:
return False
flags = 0 if rule.case_sensitive else re.IGNORECASE
return bool(re.search(rule.pattern, filename, flags))
def _match_content(rule: ClassificationRule, text: str) -> bool:
"""Return True if any keyword in *rule.pattern* appears in *text*.
Keywords are separated by ``|`` (pipe).
"""
if not text:
return False
keywords = [kw.strip() for kw in rule.pattern.split("|") if kw.strip()]
text_lower = text if rule.case_sensitive else text.lower()
return any((kw if rule.case_sensitive else kw.lower()) in text_lower for kw in keywords)
def _match_metadata(rule: ClassificationRule, metadata: dict[str, Any] | None) -> bool:
"""Return True if *rule.pattern* (``field=value``) matches *metadata*.
Pattern format: ``field_name=expected_value``.
"""
if not metadata:
return False
if "=" not in rule.pattern:
return False
field_name, expected_value = rule.pattern.split("=", 1)
actual = metadata.get(field_name.strip())
if actual is None:
return False
if rule.case_sensitive:
return str(actual) == expected_value.strip()
return str(actual).lower() == expected_value.strip().lower()
_MATCHERS: dict[str, tuple] = {
RULE_TYPE_FILENAME: (_match_filename, "filename"),
RULE_TYPE_CONTENT: (_match_content, "text"),
RULE_TYPE_METADATA: (_match_metadata, "metadata"),
}
def _evaluate_rule(
rule: ClassificationRule,
filename: str,
text: str,
metadata: dict[str, Any] | None,
) -> MatchedRule | None:
"""Evaluate a single rule against the document. Return a :class:`MatchedRule` on match."""
entry = _MATCHERS.get(rule.rule_type)
if entry is None:
return None
matcher, arg_key = entry
arg_map = {"filename": filename, "text": text, "metadata": metadata}
matched = matcher(rule, arg_map[arg_key])
if matched:
return MatchedRule(
rule_name=rule.name,
rule_type=rule.rule_type,
category=rule.category,
confidence=_CONFIDENCE_MAP.get(rule.rule_type, 50),
)
return None
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def classify_document(
filename: str = "",
text: str = "",
metadata: dict[str, Any] | None = None,
custom_rules: list[ClassificationRule] | None = None,
) -> ClassificationResult:
"""Classify a document by evaluating built-in and custom rules.
Rules are evaluated in priority order (highest first, then built-in before
custom for the same priority). The category with the most rule matches
wins; ties are broken by cumulative confidence.
Args:
filename: Original filename of the document.
text: Extracted / OCR text of the document.
metadata: Previously-extracted AI metadata dict (e.g. from ``ai_metadata``).
custom_rules: Optional list of user-defined :class:`ClassificationRule` objects.
Returns:
A :class:`ClassificationResult` with the best matching category,
overall confidence score, and the list of rules that fired.
"""
all_rules = list(BUILTIN_RULES)
if custom_rules:
all_rules.extend(custom_rules)
# Sort by priority descending (higher priority first)
all_rules.sort(key=lambda r: r.priority, reverse=True)
matches: list[MatchedRule] = []
for rule in all_rules:
result = _evaluate_rule(rule, filename, text, metadata)
if result is not None:
matches.append(result)
if not matches:
return ClassificationResult(category="unknown", confidence=0, matched_rules=[])
# Aggregate by category: pick the one with the most matches, then highest
# cumulative confidence as tiebreaker.
category_scores: dict[str, list[MatchedRule]] = {}
for m in matches:
category_scores.setdefault(m.category, []).append(m)
best_category = max(
category_scores,
key=lambda cat: (len(category_scores[cat]), sum(m.confidence for m in category_scores[cat])),
)
best_matches = category_scores[best_category]
base_confidence = max(m.confidence for m in best_matches)
bonus = min(
(len(best_matches) - 1) * _CONFIDENCE_BONUS_PER_EXTRA_RULE,
100 - base_confidence,
)
final_confidence = min(base_confidence + bonus, 100)
return ClassificationResult(
category=best_category,
confidence=final_confidence,
matched_rules=best_matches,
)
def db_rule_to_engine_rule(db_rule: Any) -> ClassificationRule:
"""Convert a database ``ClassificationRuleModel`` row to an engine :class:`ClassificationRule`.
Args:
db_rule: A SQLAlchemy model instance with ``name``, ``category``,
``rule_type``, ``pattern``, ``priority``, and ``case_sensitive`` attributes.
Returns:
A :class:`ClassificationRule` dataclass instance.
"""
return ClassificationRule(
name=db_rule.name,
category=db_rule.category,
rule_type=db_rule.rule_type,
pattern=db_rule.pattern,
priority=db_rule.priority,
case_sensitive=getattr(db_rule, "case_sensitive", False),
)
+22
View File
@@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]:
},
}
# Check SharePoint configuration
providers["SharePoint"] = {
"name": "SharePoint",
"icon": "fa-brands fa-microsoft",
"configured": bool(
getattr(settings, "sharepoint_client_id", None)
and getattr(settings, "sharepoint_client_secret", None)
and getattr(settings, "sharepoint_site_url", None)
),
"enabled": True,
"description": "Store documents in Microsoft SharePoint Online",
"details": {
"client_id": getattr(settings, "sharepoint_client_id", "Not set"),
"client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)),
"tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"),
"refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)),
"site_url": getattr(settings, "sharepoint_site_url", "Not set"),
"document_library": getattr(settings, "sharepoint_document_library", "Not set"),
"folder_path": getattr(settings, "sharepoint_folder_path", "Not set"),
},
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
+505
View File
@@ -0,0 +1,505 @@
"""Server-side session management utilities.
Provides helpers for creating, validating, and revoking user sessions.
Sessions are tracked in the ``user_sessions`` table and referenced by a
cryptographically random token stored in the browser cookie. This enables
the "log off everywhere" feature and per-session revocation.
"""
from __future__ import annotations
import logging
import secrets
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import Session
from app.config import settings
from app.models import ApiToken, QRLoginChallenge, UserSession
logger = logging.getLogger(__name__)
def _ensure_tz_aware(dt: datetime | None) -> datetime | None:
"""Return *dt* with UTC tzinfo if it is naive, or unchanged if already aware.
SQLite does not persist timezone information, so datetimes read back from
the database are offset-naive. This helper normalises them for safe
comparison with ``datetime.now(timezone.utc)``.
"""
if dt is not None and dt.tzinfo is None:
return dt.replace(tzinfo=timezone.utc)
return dt
def get_session_lifetime_days() -> int:
"""Return the effective session lifetime in days.
If ``session_lifetime_custom_days`` is set it takes precedence over
``session_lifetime_days``.
"""
custom = getattr(settings, "session_lifetime_custom_days", None)
if custom is not None and isinstance(custom, int) and custom > 0:
return custom
return max(1, getattr(settings, "session_lifetime_days", 30))
def get_session_max_age_seconds() -> int:
"""Return the session max-age in seconds for the cookie."""
return get_session_lifetime_days() * 86400
def create_session(
db: Session,
user_id: str,
ip_address: str | None = None,
user_agent: str | None = None,
) -> UserSession:
"""Create a new server-side session record.
Args:
db: Database session.
user_id: Stable owner identifier.
ip_address: Client IP address.
user_agent: Client User-Agent header.
Returns:
The newly created ``UserSession`` instance.
"""
session_token = secrets.token_urlsafe(64)
now = datetime.now(timezone.utc)
lifetime_days = get_session_lifetime_days()
expires_at = now + timedelta(days=lifetime_days)
device_info = _parse_device_info(user_agent)
user_session = UserSession(
session_token=session_token,
user_id=user_id,
ip_address=ip_address,
user_agent=(user_agent or "")[:512],
device_info=device_info,
created_at=now,
last_active_at=now,
expires_at=expires_at,
)
try:
db.add(user_session)
db.commit()
db.refresh(user_session)
except Exception:
db.rollback()
logger.exception("Failed to create session for user_id=%s", user_id)
raise
logger.info(
"[SESSION] Created session id=%s user=%s device=%r expires=%s",
user_session.id,
user_id,
device_info,
expires_at.isoformat(),
)
return user_session
def validate_session(db: Session, session_token: str) -> UserSession | None:
"""Validate a session token and return the session if valid.
A session is valid when:
* It exists in the database.
* ``is_revoked`` is ``False``.
* ``expires_at`` is in the future.
Side-effect: updates ``last_active_at`` on valid sessions.
Returns:
The ``UserSession`` if valid, else ``None``.
"""
if not session_token:
return None
now = datetime.now(timezone.utc)
user_session = db.query(UserSession).filter(UserSession.session_token == session_token).first()
if not user_session:
logger.debug("[SESSION] Token not found in database")
return None
if user_session.is_revoked:
logger.debug("[SESSION] Session id=%s is revoked", user_session.id)
return None
if user_session.expires_at:
expires = _ensure_tz_aware(user_session.expires_at)
if expires < now:
logger.debug("[SESSION] Session id=%s has expired", user_session.id)
return None
# Update last_active_at (throttled to avoid excessive writes)
last_active = _ensure_tz_aware(user_session.last_active_at)
if not last_active or (now - last_active).total_seconds() > 60:
try:
user_session.last_active_at = now
db.commit()
except Exception:
db.rollback()
logger.debug("[SESSION] Failed to update last_active_at for session id=%s", user_session.id)
return user_session
def revoke_session(db: Session, session_id: int, user_id: str) -> bool:
"""Revoke a single session by ID.
Args:
db: Database session.
session_id: The session record ID to revoke.
user_id: The owner — ensures a user can only revoke their own sessions.
Returns:
``True`` if the session was found and revoked, ``False`` otherwise.
"""
user_session = db.get(UserSession, session_id)
if not user_session or user_session.user_id != user_id:
return False
now = datetime.now(timezone.utc)
user_session.is_revoked = True
user_session.revoked_at = now
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info("[SESSION] Revoked session id=%s user=%s", session_id, user_id)
return True
def revoke_all_sessions(
db: Session,
user_id: str,
*,
except_session_id: int | None = None,
revoke_api_tokens: bool = True,
) -> int:
"""Revoke all active sessions for a user ("log off everywhere").
Args:
db: Database session.
user_id: The owner whose sessions should be revoked.
except_session_id: If provided, keep this session active (the
current browser session).
revoke_api_tokens: If ``True``, also revoke all active API tokens.
Returns:
Number of sessions revoked.
"""
now = datetime.now(timezone.utc)
query = db.query(UserSession).filter(
UserSession.user_id == user_id,
UserSession.is_revoked.is_(False),
)
if except_session_id is not None:
query = query.filter(UserSession.id != except_session_id)
sessions = query.all()
count = 0
for s in sessions:
s.is_revoked = True
s.revoked_at = now
count += 1
if revoke_api_tokens:
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == user_id,
ApiToken.is_active.is_(True),
)
.all()
)
for t in tokens:
t.is_active = False
t.revoked_at = now
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info(
"[SESSION] Revoked all sessions for user=%s (count=%d, except_session_id=%s, tokens_revoked=%s)",
user_id,
count,
except_session_id,
revoke_api_tokens,
)
return count
def list_user_sessions(db: Session, user_id: str) -> list[UserSession]:
"""Return all non-revoked, non-expired sessions for a user.
Results are ordered by most recently active first.
"""
now = datetime.now(timezone.utc)
sessions = (
db.query(UserSession)
.filter(
UserSession.user_id == user_id,
UserSession.is_revoked.is_(False),
)
.order_by(UserSession.last_active_at.desc())
.all()
)
# Filter expired sessions in Python to handle timezone-naive datetimes (SQLite)
result = []
for s in sessions:
expires = _ensure_tz_aware(s.expires_at)
if expires and expires > now:
result.append(s)
return result
def cleanup_expired_sessions(db: Session) -> int:
"""Delete sessions that expired more than 7 days ago.
Intended to be called periodically (e.g. via Celery beat) to keep the
table from growing unbounded.
Returns:
Number of rows deleted.
"""
cutoff = datetime.now(timezone.utc) - timedelta(days=7)
count = db.query(UserSession).filter(UserSession.expires_at < cutoff).delete(synchronize_session=False)
try:
db.commit()
except Exception:
db.rollback()
raise
if count:
logger.info("[SESSION] Cleaned up %d expired sessions", count)
return count
# ---------------------------------------------------------------------------
# QR login helpers
# ---------------------------------------------------------------------------
def create_qr_challenge(db: Session, user_id: str, ip_address: str | None = None) -> QRLoginChallenge:
"""Create a new QR login challenge.
Args:
db: Database session.
user_id: The authenticated web user creating the challenge.
ip_address: IP address of the web client.
Returns:
The newly created ``QRLoginChallenge``.
"""
token = secrets.token_urlsafe(64)
ttl = getattr(settings, "qr_login_challenge_ttl_seconds", 120)
now = datetime.now(timezone.utc)
expires_at = now + timedelta(seconds=ttl)
challenge = QRLoginChallenge(
challenge_token=token,
user_id=user_id,
created_by_ip=ip_address,
created_at=now,
expires_at=expires_at,
)
try:
db.add(challenge)
db.commit()
db.refresh(challenge)
except Exception:
db.rollback()
logger.exception("Failed to create QR login challenge for user_id=%s", user_id)
raise
logger.info("[QR_AUTH] Challenge created: id=%s user=%s expires=%s", challenge.id, user_id, expires_at.isoformat())
return challenge
def validate_qr_challenge(db: Session, challenge_token: str) -> QRLoginChallenge | None:
"""Validate a QR challenge token without claiming it.
Returns the challenge if it exists, is not expired, not claimed,
and not cancelled. Returns ``None`` otherwise.
"""
if not challenge_token:
return None
now = datetime.now(timezone.utc)
challenge = db.query(QRLoginChallenge).filter(QRLoginChallenge.challenge_token == challenge_token).first()
if not challenge:
return None
if challenge.is_claimed or challenge.is_cancelled:
return None
expires = _ensure_tz_aware(challenge.expires_at)
if expires and expires < now:
return None
return challenge
def claim_qr_challenge(
db: Session,
challenge_token: str,
device_name: str = "Mobile App",
ip_address: str | None = None,
) -> dict | None:
"""Claim a QR challenge and issue an API token.
This is the critical security path. The challenge is validated,
marked as claimed atomically, and an API token is issued for the
user who created the challenge.
Args:
db: Database session.
challenge_token: The token from the QR code.
device_name: Name provided by the mobile app.
ip_address: IP address of the claiming mobile device.
Returns:
Dict with ``token`` (plaintext), ``token_id``, ``name``, ``owner_id``
and ``created_at`` on success, or ``None`` if the challenge is invalid.
"""
from app.api.api_tokens import generate_api_token, hash_token
challenge = validate_qr_challenge(db, challenge_token)
if not challenge:
logger.warning("[QR_AUTH] Invalid or expired challenge token attempted")
return None
now = datetime.now(timezone.utc)
# Mark as claimed first to prevent race conditions
challenge.is_claimed = True
challenge.claimed_at = now
challenge.claimed_by_ip = ip_address
challenge.device_name = device_name
# Generate API token for the mobile app
token_name = f"Mobile App (QR) – {device_name}"
plaintext = generate_api_token()
token_hash_value = hash_token(plaintext)
prefix = plaintext[:12]
db_token = ApiToken(
owner_id=challenge.user_id,
name=token_name,
token_hash=token_hash_value,
token_prefix=prefix,
)
try:
db.add(db_token)
db.flush()
challenge.issued_token_id = db_token.id
db.commit()
db.refresh(db_token)
except Exception:
db.rollback()
logger.exception("[QR_AUTH] Failed to issue token for challenge id=%s", challenge.id)
raise
logger.info(
"[QR_AUTH] Challenge claimed: id=%s user=%s device=%r token_id=%s",
challenge.id,
challenge.user_id,
device_name,
db_token.id,
)
return {
"token": plaintext,
"token_id": db_token.id,
"name": token_name,
"owner_id": challenge.user_id,
"created_at": db_token.created_at,
}
def get_challenge_status(db: Session, challenge_id: int, user_id: str) -> dict | None:
"""Get the current status of a QR challenge (for polling from the web UI).
Returns:
Dict with ``status`` ("pending", "claimed", "expired", "cancelled")
and metadata, or ``None`` if the challenge doesn't belong to the user.
"""
challenge = db.get(QRLoginChallenge, challenge_id)
if not challenge or challenge.user_id != user_id:
return None
now = datetime.now(timezone.utc)
expires = _ensure_tz_aware(challenge.expires_at)
if challenge.is_claimed:
status = "claimed"
elif challenge.is_cancelled:
status = "cancelled"
elif expires and expires < now:
status = "expired"
else:
status = "pending"
return {
"id": challenge.id,
"status": status,
"device_name": challenge.device_name,
"claimed_at": challenge.claimed_at,
"expires_at": challenge.expires_at,
}
def _parse_device_info(user_agent: str | None) -> str | None:
"""Extract a human-readable device description from User-Agent.
This is a lightweight parser — not a full UA library — that covers
the most common browsers and platforms.
"""
if not user_agent:
return None
ua = user_agent.lower()
# Platform detection
platform = "Unknown"
if "iphone" in ua:
platform = "iPhone"
elif "ipad" in ua:
platform = "iPad"
elif "android" in ua:
platform = "Android"
elif "macintosh" in ua or "mac os" in ua:
platform = "macOS"
elif "windows" in ua:
platform = "Windows"
elif "linux" in ua:
platform = "Linux"
elif "cros" in ua:
platform = "ChromeOS"
# Browser detection
browser = "Unknown Browser"
if "edg/" in ua or "edge/" in ua:
browser = "Edge"
elif "opr/" in ua or "opera" in ua:
browser = "Opera"
elif "chrome/" in ua and "safari/" in ua:
browser = "Chrome"
elif "safari/" in ua and "chrome/" not in ua:
browser = "Safari"
elif "firefox/" in ua:
browser = "Firefox"
elif "docuelevate" in ua:
browser = "DocuElevate App"
return f"{browser} on {platform}"
+219 -2
View File
@@ -39,6 +39,50 @@ SETTING_METADATA = {
"required": True,
"restart_required": True,
},
"db_pool_size": {
"category": "Core",
"description": (
"Number of persistent database connections kept in the pool per worker process. "
"Ignored for SQLite (which uses NullPool). Default: 10."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_max_overflow": {
"category": "Core",
"description": (
"Additional database connections allowed beyond db_pool_size under burst load. "
"Ignored for SQLite. Default: 20."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_timeout": {
"category": "Core",
"description": (
"Seconds to wait for a database connection from the pool before raising a TimeoutError. "
"Ignored for SQLite. Default: 30."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_recycle": {
"category": "Core",
"description": (
"Recycle (close and reopen) database connections after this many seconds "
"to avoid stale connections. Ignored for SQLite. Default: 1800."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"workdir": {
"category": "Core",
"description": "Working directory for file storage and processing",
@@ -55,6 +99,18 @@ SETTING_METADATA = {
"required": True, # Required for OAuth redirects and external URLs
"restart_required": True,
},
"public_base_url": {
"category": "Core",
"description": (
"Full public base URL including scheme (e.g., https://docuelevate.example.com). "
"When set, overrides auto-detected URLs for OAuth redirect URIs. "
"Required when behind a reverse proxy that does not forward X-Forwarded-Proto."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"debug": {
"category": "Core",
"description": "Enable debug mode for verbose logging",
@@ -134,6 +190,30 @@ SETTING_METADATA = {
"required": True, # Required when auth_enabled=True (validated in config.py)
"restart_required": True,
},
"session_lifetime_days": {
"category": "Authentication",
"description": "Session lifetime in days (default 30). Determines how long a user stays logged in.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"session_lifetime_custom_days": {
"category": "Authentication",
"description": "Override session_lifetime_days with a custom value. Takes precedence when set.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"qr_login_challenge_ttl_seconds": {
"category": "Authentication",
"description": "Time-to-live in seconds for QR login challenges (default 120).",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"admin_username": {
"category": "Authentication",
"description": "Admin username for local authentication",
@@ -302,6 +382,19 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
"social_auth_dropbox_use_global_credentials": {
"category": "Social Login",
"description": (
"When True, Dropbox social login uses the global DROPBOX_APP_KEY / DROPBOX_APP_SECRET "
"credentials instead of requiring separate SOCIAL_AUTH_DROPBOX_CLIENT_ID / "
"SOCIAL_AUTH_DROPBOX_CLIENT_SECRET values. "
"Requires SOCIAL_AUTH_DROPBOX_ENABLED=True and global Dropbox credentials to be set."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"social_auth_dropbox_enabled": {
"category": "Social Login",
"description": (
@@ -695,6 +788,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"dropbox_allow_global_credentials_for_integrations": {
"category": "Storage Providers",
"description": (
"When True, users may authorize their personal Dropbox integrations using the global "
"DROPBOX_APP_KEY / DROPBOX_APP_SECRET credentials configured by the admin, without "
"needing to create their own Dropbox app."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_enabled": {
"category": "Storage Providers",
@@ -875,6 +980,63 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - SharePoint
"sharepoint_client_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD application (client) ID",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_client_secret": {
"category": "Storage Providers",
"description": "SharePoint Azure AD client secret",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_tenant_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_refresh_token": {
"category": "Storage Providers",
"description": "SharePoint OAuth refresh token",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_site_url": {
"category": "Storage Providers",
"description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_document_library": {
"category": "Storage Providers",
"description": "SharePoint document library name (default: 'Documents')",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_folder_path": {
"category": "Storage Providers",
"description": "Subfolder path inside the SharePoint document library",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_enabled": {
"category": "Storage Providers",
@@ -1912,6 +2074,28 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"factory_reset_on_startup": {
"category": "Feature Flags",
"description": (
"Wipe all user data on every startup so the instance always starts fresh. "
"Useful for demo/testing environments. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"enable_factory_reset": {
"category": "Feature Flags",
"description": (
"Show the System Reset page in the admin UI. Allows administrators to "
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Backup / Restore
"backup_enabled": {
"category": "Backup",
@@ -1935,14 +2119,26 @@ SETTING_METADATA = {
"category": "Backup",
"description": (
"Storage provider for remote backup copies. "
"Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. "
"Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. "
"Leave empty to keep backups local only."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"],
"options": [
"",
"s3",
"dropbox",
"google_drive",
"onedrive",
"sharepoint",
"nextcloud",
"webdav",
"ftp",
"sftp",
"email",
],
},
"backup_remote_folder": {
"category": "Backup",
@@ -2454,6 +2650,27 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Per-user upload rate limiting
"upload_rate_limit_per_user": {
"category": "Security",
"description": (
"Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. "
"The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. "
"Default: 20."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"upload_rate_limit_window": {
"category": "Security",
"description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Rate Limiting
"rate_limiting_enabled": {
"category": "Security",
+296
View File
@@ -0,0 +1,296 @@
"""
System reset utilities for DocuElevate.
Provides functions to:
- Wipe all user data (database rows + work-files on disk) for a fresh start.
- Wipe with re-import: move original files to a dedicated folder, wipe
everything, then let the watch-folder mechanism re-ingest the files.
Security: All public functions in this module require admin-level access.
They MUST only be invoked from admin-guarded API/view endpoints.
"""
import logging
import shutil
from pathlib import Path
from sqlalchemy.orm import Session
from app.config import settings
logger = logging.getLogger(__name__)
# Subdirectories inside *workdir* that contain user-generated data.
# Everything else (app code, static assets, config) is left untouched.
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
# JSON cache files written by watch-folder / ingest tasks.
_CACHE_FILES = (
"watch_folder_processed.json",
"ftp_ingest_processed.json",
"sftp_ingest_processed.json",
"dropbox_ingest_processed.json",
"gdrive_ingest_processed.json",
"onedrive_ingest_processed.json",
"nextcloud_ingest_processed.json",
"s3_ingest_processed.json",
"webdav_ingest_processed.json",
"processed_mails.json",
"credential_failures.json",
)
# The folder name used for storing files prior to re-import.
REIMPORT_FOLDER_NAME = "reimport"
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
"""Delete user data subdirectories and cache files inside *workdir*.
Leaves the workdir directory itself intact so the application can
continue to write into it. Also leaves any files that do not belong
to the known data subdirectories or caches.
Returns:
A dict with counts of deleted directories and files.
"""
workdir_path = Path(workdir)
deleted_dirs = 0
deleted_files = 0
# Remove data subdirectories
for subdir in _USER_DATA_SUBDIRS:
target = workdir_path / subdir
if target.is_dir():
shutil.rmtree(target)
logger.info("Deleted data directory: %s", target)
deleted_dirs += 1
# Remove cache / state JSON files
for cache_file in _CACHE_FILES:
target = workdir_path / cache_file
if target.is_file():
target.unlink()
logger.info("Deleted cache file: %s", target)
deleted_files += 1
# Also remove user_wf_*.json files (per-user watch folder caches)
for f in workdir_path.glob("user_wf_*.json"):
f.unlink()
logger.info("Deleted user watch-folder cache: %s", f)
deleted_files += 1
# Remove loose files in workdir root that are user uploads (uuid-named
# files like "a1b2c3d4-…pdf") but NOT application config files.
for entry in workdir_path.iterdir():
if entry.is_file() and entry.suffix.lower() in {
".pdf",
".png",
".jpg",
".jpeg",
".tiff",
".tif",
".docx",
".doc",
".xlsx",
".xls",
".pptx",
".heic",
".heif",
".webp",
".bmp",
".gif",
".txt",
".rtf",
".odt",
".ods",
".odp",
".csv",
".pages",
".numbers",
".keynote",
}:
entry.unlink()
logger.info("Deleted loose workdir file: %s", entry)
deleted_files += 1
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
def _wipe_database(db: Session) -> dict[str, int]:
"""Delete all user-generated rows from the database.
Preserves schema (tables, migrations) and system-seeded rows that will
be re-created on the next startup (subscription plans, default pipeline,
scheduled jobs, compliance templates).
Returns:
A dict mapping table name → number of rows deleted.
"""
from app.models import (
AuditLog,
BackupRecord,
DocumentMetadata,
FileProcessingStep,
FileRecord,
InAppNotification,
ProcessingLog,
SavedSearch,
SettingsAuditLog,
SharedLink,
UserImapAccount,
UserIntegration,
UserNotificationPreference,
UserNotificationTarget,
)
# Order matters: delete children before parents to respect FK constraints.
tables_to_wipe: list[tuple[str, type]] = [
("file_processing_steps", FileProcessingStep),
("processing_logs", ProcessingLog),
("shared_links", SharedLink),
("in_app_notifications", InAppNotification),
("user_notification_preferences", UserNotificationPreference),
("user_notification_targets", UserNotificationTarget),
("user_imap_accounts", UserImapAccount),
("user_integrations", UserIntegration),
("saved_searches", SavedSearch),
("settings_audit_log", SettingsAuditLog),
("audit_logs", AuditLog),
("backup_records", BackupRecord),
("document_metadata", DocumentMetadata),
("files", FileRecord),
]
result: dict[str, int] = {}
for table_name, model in tables_to_wipe:
try:
count = db.query(model).delete()
result[table_name] = count
logger.info("Wiped %d rows from %s", count, table_name)
except Exception:
logger.exception("Failed to wipe table %s during system reset", table_name)
db.rollback()
raise
db.commit()
return result
def perform_full_reset(db: Session) -> dict:
"""Perform a complete system reset: wipe database rows + work-files.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database`` and ``filesystem`` sub-dicts.
"""
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
db_result = _wipe_database(db)
fs_result = _wipe_workdir_data(settings.workdir)
logger.warning(">>> SYSTEM RESET complete <<<")
return {"database": db_result, "filesystem": fs_result}
def perform_reset_and_reimport(db: Session) -> dict:
"""Move original files to a reimport folder, wipe everything, then
configure the reimport folder as a watch folder for re-ingestion.
The watch-folder scanner (``scan_all_watch_folders``) will pick up
the files on its next periodic run and process them exactly as if
they had been freshly uploaded — respecting the same backoff
strategy, size limits, and rate limits.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
"""
workdir_path = Path(settings.workdir)
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
original_dir = workdir_path / "original"
# 1. Collect original files
files_moved = 0
reimport_dir.mkdir(parents=True, exist_ok=True)
if original_dir.is_dir():
for entry in original_dir.iterdir():
if entry.is_file():
# Validate the resolved path stays within original_dir (path traversal guard)
try:
entry.resolve().relative_to(original_dir.resolve())
except ValueError:
logger.warning("Skipping file outside original dir: %s", entry)
continue
dest = reimport_dir / entry.name
# Avoid overwriting: append counter if name clash
if dest.exists():
stem = dest.stem
suffix = dest.suffix
counter = 1
while dest.exists():
dest = reimport_dir / f"{stem}_{counter}{suffix}"
counter += 1
shutil.copy2(str(entry), str(dest))
files_moved += 1
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
# 2. Perform the full reset (wipe DB + other workdir data)
reset_result = perform_full_reset(db)
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
# and set up watch folder config to point at it.
_configure_reimport_watch_folder(str(reimport_dir))
reset_result["reimport"] = {
"files_moved": files_moved,
"reimport_folder": str(reimport_dir),
}
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
return reset_result
def _configure_reimport_watch_folder(reimport_path: str) -> None:
"""Append *reimport_path* to the application's watch-folder list.
The watch-folder scanner uses ``settings.watch_folders`` (a
comma-separated string). We mutate the runtime setting so the
next scan picks up the folder. We also set
``watch_folder_delete_after_process = True`` so files are cleaned
up after successful processing.
"""
current = getattr(settings, "watch_folders", None) or ""
folders = [f.strip() for f in current.split(",") if f.strip()]
if reimport_path not in folders:
folders.append(reimport_path)
# Mutate runtime settings (not persisted to .env — ephemeral)
object.__setattr__(settings, "watch_folders", ",".join(folders))
object.__setattr__(settings, "watch_folder_delete_after_process", True)
logger.info("Configured reimport watch folder: %s", reimport_path)
def perform_startup_reset() -> None:
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
Wipes database and filesystem data so the instance starts completely
fresh. Uses its own DB session so it runs before the normal lifespan
seeding logic.
"""
from app.database import SessionLocal
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
db = SessionLocal()
try:
perform_full_reset(db)
except Exception:
logger.exception("Factory reset on startup failed")
db.rollback()
finally:
db.close()