gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: merge main, resolve conflicts, address review feedback

Browse Source 
- Resolve merge conflicts in app/api/onedrive.py and tests/test_api_google_drive_final.py
- Fix legacy Dict[str, str] type hints in update_env_file functions to use dict[str, str]
- Add admin-only access (_require_admin dependency) to save-settings endpoints
  in google_drive.py, onedrive.py, and dropbox.py
- Fix in_memory_only response field to reflect actual env_write_success status
- Update tests to override _require_admin dependency for save-settings endpoint tests
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 16:29:46 +00:00
parent 47595818b4 78bd5b5904
commit ef5528dcef
232 changed files with 114245 additions and 911 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+8
View File
@@ -33,16 +33,20 @@ from app.api.pipelines import router as pipelines_router
from app.api.plans import router as plans_router
from app.api.process import router as process_router
from app.api.profile import router as profile_router
from app.api.qr_auth import router as qr_auth_router
from app.api.queue import router as queue_router
from app.api.routing_rules import router as routing_rules_router
from app.api.saved_searches import router as saved_searches_router
from app.api.scheduled_jobs import router as scheduled_jobs_router
from app.api.search import router as search_router
from app.api.sessions import router as sessions_router
from app.api.settings import router as settings_router
from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router
from app.api.system_reset import router as system_reset_router
from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router
# Import all the individual routers
@@ -95,4 +99,8 @@ router.include_router(scheduled_jobs_router)
router.include_router(audit_logs_router)
router.include_router(i18n_router)
router.include_router(mobile_router)
router.include_router(sessions_router)
router.include_router(qr_auth_router)
router.include_router(compliance_router)
router.include_router(system_reset_router)
router.include_router(translation_router)
app/api/api_tokens.py
+55 -15
View File
@@ -42,6 +42,9 @@ TOKEN_HASH_ITERATIONS = 100_000
#: PBKDF2 salt for API token hashing (not secret, but fixed for determinism).
TOKEN_HASH_SALT = b"api-token-v1"
#: Name prefix used for tokens created by the mobile app flow.
MOBILE_TOKEN_PREFIX = "Mobile App"
# ---------------------------------------------------------------------------
# Auth helper
@@ -91,6 +94,20 @@ def hash_token(token: str) -> str:
return dk.hex()
def _token_to_dict(t: ApiToken) -> dict[str, Any]:
"""Convert an ``ApiToken`` ORM instance to a serialisable dict."""
return {
"id": t.id,
"name": t.name,
"token_prefix": t.token_prefix,
"is_active": t.is_active,
"last_used_at": t.last_used_at,
"last_used_ip": t.last_used_ip,
"created_at": t.created_at,
"revoked_at": t.revoked_at,
}
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
@@ -177,21 +194,44 @@ async def list_tokens(
owner_id: CurrentOwner,
db: DbSession,
) -> list[dict[str, Any]]:
"""List all API tokens for the authenticated user."""
tokens = db.query(ApiToken).filter(ApiToken.owner_id == owner_id).order_by(ApiToken.created_at.desc()).all()
return [
{
"id": t.id,
"name": t.name,
"token_prefix": t.token_prefix,
"is_active": t.is_active,
"last_used_at": t.last_used_at,
"last_used_ip": t.last_used_ip,
"created_at": t.created_at,
"revoked_at": t.revoked_at,
}
for t in tokens
]
"""List non-mobile API tokens for the authenticated user.
Mobile tokens (whose names start with ``"Mobile App"``) are excluded
from this list; they are managed on the dedicated Devices page via
``GET /api/api-tokens/mobile``.
"""
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == owner_id,
~ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
)
.order_by(ApiToken.created_at.desc())
.all()
)
return [_token_to_dict(t) for t in tokens]
@router.get("/mobile", response_model=list[TokenResponse])
async def list_mobile_tokens(
owner_id: CurrentOwner,
db: DbSession,
) -> list[dict[str, Any]]:
"""List mobile API tokens for the authenticated user.
Returns tokens whose names start with ``"Mobile App"`` — these are
created via the mobile SSO flow or QR code login.
"""
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == owner_id,
ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
)
.order_by(ApiToken.created_at.desc())
.all()
)
return [_token_to_dict(t) for t in tokens]
@router.delete("/{token_id}", status_code=status.HTTP_200_OK)
app/api/billing.py
+2 -3
View File
@@ -168,9 +168,8 @@ async def create_checkout_session(
checkout_session = client.checkout.sessions.create(params=session_params)
logger.info(
"Created Stripe checkout session %s for user %s plan %s",
"Created Stripe checkout session %s for plan %s",
checkout_session.id,
owner_id,
body.plan_id,
)
return {"checkout_url": checkout_session.url, "session_id": checkout_session.id}
@@ -213,7 +212,7 @@ async def create_portal_session(
}
)
logger.info("Created Stripe portal session for user %s", owner_id)
logger.info("Created Stripe portal session for user")
return {"portal_url": portal.url}
app/api/dropbox.py
+61 -47
View File
@@ -6,7 +6,7 @@ import logging
import os
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/dropbox/exchange-token")
@require_login
async def exchange_dropbox_token(
@@ -132,57 +143,60 @@ async def test_dropbox_token(request: Request):
"message": "Dropbox credentials are not fully configured",
}
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = requests.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = requests.post(
async with httpx.AsyncClient() as client:
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get account info
account_info = response.json()
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = await client.post(
refresh_url, data=refresh_data, timeout=settings.http_request_timeout
)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# Get account info
account_info = response.json()
account_email = account_info.get("email", "Unknown account")
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
@@ -208,9 +222,9 @@ async def test_dropbox_token(request: Request):
@router.post("/dropbox/save-settings")
@require_login
async def save_dropbox_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
app_key: Annotated[Optional[str], Form()] = None,
app_secret: Annotated[Optional[str], Form()] = None,
app/api/duplicates.py
+28 -23
View File
@@ -73,33 +73,38 @@ def list_duplicate_groups(
groups = []
total_duplicate_files = 0
for filehash in dup_hashes:
# Find the original (non-duplicate) record with this hash
original = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
if dup_hashes:
# Fetch all matching files (both original and duplicates) in a single batch query
all_records = (
db.query(FileRecord).filter(FileRecord.filehash.in_(dup_hashes)).order_by(FileRecord.id.asc()).all()
)
# Find all duplicate records for this hash
duplicates = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
.order_by(FileRecord.id.asc())
.all()
)
# Group records by hash
originals_by_hash = {}
duplicates_by_hash = {h: [] for h in dup_hashes}
total_duplicate_files += len(duplicates)
for record in all_records:
h = record.filehash
if not record.is_duplicate:
# Store only the first original record per hash, matching the old .first() behaviour
if h not in originals_by_hash:
originals_by_hash[h] = record
else:
duplicates_by_hash[h].append(record)
total_duplicate_files += 1
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
for filehash in dup_hashes:
original = originals_by_hash.get(filehash)
duplicates = duplicates_by_hash.get(filehash, [])
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
app/api/files.py
+95 -39
View File
@@ -11,6 +11,7 @@ import zipfile
from datetime import datetime, timezone
from typing import Annotated, List, Optional
import aiofiles
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status
from fastapi.responses import StreamingResponse
from sqlalchemy import asc, desc
@@ -345,7 +346,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -384,7 +387,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
"""
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -456,7 +461,9 @@ def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: Db
Useful for re-running OCR on files with poor text quality or missing OCR text.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -536,7 +543,9 @@ def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
Files not found on disk are silently skipped.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -618,7 +627,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -674,7 +685,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -937,7 +950,9 @@ def retry_subtask(
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1079,7 +1094,9 @@ def get_file_preview(
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1159,7 +1176,9 @@ def download_file(
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1217,6 +1236,66 @@ def download_file(
raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")
async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: int) -> int:
"""Save an uploaded file in chunks and enforce the maximum size limit."""
try:
written_size = 0
with open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
if not chunk:
break
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
return written_size
except HTTPException:
raise
except Exception as e:
if os.path.exists(target_path):
os.remove(target_path)
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
if not settings.enable_deduplication:
return None
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
return {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
return None
@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
@@ -1277,7 +1356,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# enforcing the size limit during the read so memory usage stays bounded.
try:
written_size = 0
with open(target_path, "wb") as f:
async with aiofiles.open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
@@ -1286,14 +1365,14 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
await f.write(chunk)
except HTTPException:
raise
except Exception as e:
@@ -1384,29 +1463,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# Check for exact duplicates (same SHA-256 hash) before returning.
# This gives the caller an immediate warning without waiting for the pipeline.
# Only performed when deduplication is enabled in settings.
exact_duplicate_warning = None
if settings.enable_deduplication:
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
exact_duplicate_warning = {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
response: dict = {
"task_id": task.id,
@@ -1458,7 +1515,7 @@ def claim_file(request: Request, file_id: int, db: DbSession):
logger.exception(f"Error claiming file {file_id}: {e}")
raise HTTPException(status_code=500, detail="Failed to claim document")
logger.info(f"File {file_id} claimed by user '{owner_id}'")
logger.info("File %d claimed by user", file_id)
return {"status": "success", "message": "Document claimed successfully", "file_id": file_id, "owner_id": owner_id}
@@ -1498,7 +1555,7 @@ def bulk_claim_files(request: Request, file_ids: list[int], db: DbSession):
logger.exception(f"Error during bulk claim: {e}")
raise HTTPException(status_code=500, detail="Failed to claim documents")
logger.info(f"Bulk claim by '{owner_id}': claimed={claimed}, skipped={[s['file_id'] for s in skipped]}")
logger.info("Bulk claim: claimed=%s, skipped=%s", claimed, [s["file_id"] for s in skipped])
return {
"status": "success",
"claimed_count": len(claimed),
@@ -1551,8 +1608,7 @@ def assign_owner(request: Request, db: DbSession, owner_id: str = Query(...), fi
logger.exception(f"Error assigning owner: {e}")
raise HTTPException(status_code=500, detail="Failed to assign owner")
admin_name = get_current_owner_id(request) or "admin"
logger.info(f"Admin '{admin_name}' assigned owner_id='{owner_id}' to {updated} file(s)")
logger.info("Admin assigned owner to %d file(s)", updated)
return {
"status": "success",
"message": f"Assigned owner to {updated} document(s)",
app/api/google_drive.py
+15 -3
View File
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/google-drive/exchange-token")
@require_login
async def exchange_google_drive_token(
@@ -362,9 +373,9 @@ def format_time_remaining(time_delta):
@router.post("/google-drive/save-settings")
@require_login
async def save_google_drive_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
client_secret: Annotated[Optional[str], Form()] = None,
@@ -404,7 +415,8 @@ async def save_google_drive_settings(
drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
# Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
if not update_env_file(env_path, drive_settings):
env_write_success = update_env_file(env_path, drive_settings)
if not env_write_success:
logger.info("Continuing with in-memory update despite .env file update failure or skip")
# Update the settings in memory (this always happens)
@@ -443,7 +455,7 @@ async def save_google_drive_settings(
return {
"status": "success",
"message": "Google Drive settings have been saved",
"in_memory_only": not os.path.exists(env_path),
"in_memory_only": not env_write_success,
}
except Exception as e:
app/api/imap_accounts.py
+6
View File
@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
from app.database import get_db
from app.models import UserImapAccount
from app.utils.encryption import decrypt_value, encrypt_value
from app.utils.network import is_private_ip
from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id
@@ -187,6 +188,11 @@ def _test_imap_connection(host: str, port: int, username: str, password: str, us
Returns a dict with ``{"success": bool, "message": str}``.
"""
# Security: Prevent SSRF by blocking connections to internal IPs
if is_private_ip(host):
logger.warning("SSRF blocked: Attempt to connect to private IP %s", host)
return {"success": False, "message": "Connection error: Invalid hostname or IP address"}
try:
if use_ssl:
mail = imaplib.IMAP4_SSL(host, port)
app/api/notifications.py
+9 -10
View File
@@ -452,17 +452,16 @@ async def update_preferences(
)
try:
# Pre-fetch existing preferences for this user to avoid N+1 queries
existing_prefs = (
db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
)
# Build a fast lookup dictionary keyed by (event_type, channel_type, target_id)
prefs_dict = {(pref.event_type, pref.channel_type, pref.target_id): pref for pref in existing_prefs}
for item in body.preferences:
existing = (
db.query(UserNotificationPreference)
.filter(
UserNotificationPreference.owner_id == owner_id,
UserNotificationPreference.event_type == item.event_type,
UserNotificationPreference.channel_type == item.channel_type,
UserNotificationPreference.target_id == item.target_id,
)
.first()
)
existing = prefs_dict.get((item.event_type, item.channel_type, item.target_id))
if existing:
existing.is_enabled = item.is_enabled
else:
app/api/onedrive.py
+46 -61
View File
@@ -7,7 +7,7 @@ import os
from datetime import datetime, timedelta
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -24,6 +24,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/onedrive/exchange-token")
@require_login
async def exchange_onedrive_token(
@@ -92,17 +103,18 @@ async def test_onedrive_token(request: Request):
"scope": "offline_access Files.ReadWrite",
}
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
response = await client.post(token_url, data=refresh_data)
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_data = response.json()
token_data = response.json()
access_token = token_data.get("access_token")
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
@@ -115,32 +127,7 @@ async def test_onedrive_token(request: Request):
settings.onedrive_refresh_token = new_refresh_token
# Also try to update .env file if it exists
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if os.path.exists(env_path):
with open(env_path, "r") as f:
env_lines = f.readlines()
updated_lines = []
updated = False
for line in env_lines:
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
updated = True
else:
updated_lines.append(line)
if not updated:
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
with open(env_path, "w") as f:
f.writelines(updated_lines)
logger.info("Updated refresh token in .env file")
except Exception as e:
logger.warning(f"Failed to update refresh token in .env file: {e}")
update_env_file({"ONEDRIVE_REFRESH_TOKEN": new_refresh_token})
# Persist the rotated refresh token to the database
try:
@@ -164,17 +151,18 @@ async def test_onedrive_token(request: Request):
user_info_url = "https://graph.microsoft.com/v1.0/me"
headers = {"Authorization": f"Bearer {access_token}"}
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
user_response = await client.get(user_info_url, headers=headers)
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
# Get user info
user_info = user_response.json()
# Get user info
user_info = user_response.json()
display_name = user_info.get("displayName", "Unknown user")
email = user_info.get("userPrincipalName", "Unknown email")
@@ -227,9 +215,9 @@ def format_time_remaining(time_delta):
@router.post("/onedrive/save-settings")
@require_login
async def save_onedrive_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
client_secret: Annotated[Optional[str], Form()] = None,
@@ -247,22 +235,19 @@ async def save_onedrive_settings(
)
# Best-effort .env file write
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
if not update_env_file(env_path, onedrive_settings):
logger.info("Continuing with in-memory update despite .env file update failure or skip")
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
if not update_env_file(env_path, onedrive_settings):
logger.info("Continuing with in-memory update despite .env file update failure or skip")
# Update the settings in memory
if refresh_token:
app/api/plans.py
+11 -2
View File
@@ -196,11 +196,20 @@ def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
updated = 0
for sort_order, plan_id in enumerate(body.order):
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
# Fetch all requested plans in a single query to avoid N+1
plan_ids = body.order
plans = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id.in_(plan_ids)).all()
# Build a map for fast O(1) lookup
plan_map = {p.plan_id: p for p in plans}
for sort_order, plan_id in enumerate(plan_ids):
plan = plan_map.get(plan_id)
if plan:
plan.sort_order = sort_order
updated += 1
try:
db.commit()
except Exception:
app/api/profile.py
+18
View File
@@ -99,6 +99,8 @@ class ProfileResponse(BaseModel):
contact_email: str | None
preferred_language: str | None
preferred_theme: str | None
default_document_language: str | None
"""ISO 639-1 code for the user's preferred document translation target language."""
avatar_url: str
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
is_local_user: bool
@@ -112,6 +114,10 @@ class ProfileUpdateRequest(BaseModel):
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
default_document_language: str | None = Field(
default=None,
description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'",
)
class ChangePasswordRequest(BaseModel):
@@ -149,6 +155,7 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
@@ -201,6 +208,16 @@ async def update_profile(
)
profile.preferred_theme = theme or None # type: ignore[assignment]
# Validate default document language
if body.default_document_language is not None:
doc_lang = body.default_document_language.lower().strip()
if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported language code: {doc_lang}",
)
profile.default_document_language = doc_lang or None # type: ignore[assignment]
if body.display_name is not None:
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
@@ -225,6 +242,7 @@ async def update_profile(
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
app/api/qr_auth.py
+233
View File
@@ -0,0 +1,233 @@
"""QR code login API endpoints for mobile app authentication.
Provides a secure challenge-response flow for logging into the mobile app
by scanning a QR code displayed in the web interface:
1. **Web user** calls ``POST /qr-auth/challenge`` → receives a time-limited
challenge token (encoded in the QR code).
2. **Web UI** polls ``GET /qr-auth/challenge/{id}/status`` to detect when
the mobile app has claimed the challenge.
3. **Mobile app** scans the QR code and calls ``POST /qr-auth/claim`` with
the challenge token + device name → receives an API token.
Security properties:
* Challenges expire after a configurable TTL (default 2 minutes).
* Single-use: once claimed, a challenge cannot be reused (replay-safe).
* Cryptographically random 64-byte tokens.
* IP addresses are logged for audit.
"""
from __future__ import annotations
import base64
import io
import logging
from datetime import datetime
from typing import Annotated, Any
import segno
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.middleware.audit_log import get_client_ip
from app.utils.session_manager import (
claim_qr_challenge,
create_qr_challenge,
get_challenge_status,
)
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/qr-auth", tags=["qr-auth"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helper
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if not owner_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Request / Response schemas
# ---------------------------------------------------------------------------
class CreateChallengeResponse(BaseModel):
"""Response after creating a QR login challenge."""
challenge_id: int
challenge_token: str
expires_at: datetime
ttl_seconds: int = Field(description="Seconds until the challenge expires (use for client-side countdown).")
qr_payload: str = Field(description="The string to encode in the QR code.")
qr_code_svg: str = Field(description="Base64-encoded SVG data URI of the QR code, ready for use in an <img> src.")
class ChallengeStatusResponse(BaseModel):
"""Response for polling the status of a QR challenge."""
id: int
status: str # "pending", "claimed", "expired", "cancelled"
device_name: str | None = None
claimed_at: datetime | None = None
expires_at: datetime
class ClaimChallengeRequest(BaseModel):
"""Request body for claiming a QR login challenge."""
challenge_token: str = Field(min_length=1, max_length=256)
device_name: str = Field(
default="Mobile App",
min_length=1,
max_length=120,
description="Human-readable device name.",
)
class ClaimChallengeResponse(BaseModel):
"""Response after successfully claiming a QR challenge."""
token: str
token_id: int
name: str
owner_id: str
created_at: datetime
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
# QR code rendering parameters
_QR_ERROR_LEVEL = "M" # Medium error correction (~15% recovery); sufficient for on-screen display
_QR_SCALE = 4 # Each QR module is rendered as 4×4 SVG pixels
def _generate_qr_svg(payload: str) -> str:
"""Generate a QR code for *payload* and return it as a base64 SVG data URI.
Using ``segno`` (pure-Python, no Pillow dependency) and SVG output so the
QR code scales crisply at any resolution without requiring a canvas or any
client-side JavaScript library.
"""
qr = segno.make(payload, error=_QR_ERROR_LEVEL)
buf = io.BytesIO()
qr.save(buf, kind="svg", scale=_QR_SCALE, xmldecl=False, svgclass=None, lineclass=None, omitsize=True)
svg_bytes = buf.getvalue()
return "data:image/svg+xml;base64," + base64.b64encode(svg_bytes).decode("ascii")
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.post("/challenge", status_code=status.HTTP_201_CREATED, response_model=CreateChallengeResponse)
@require_login
async def create_challenge(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Create a new QR login challenge.
The returned ``qr_payload`` should be encoded into a QR code and
displayed to the user. The mobile app scans this QR code and
calls the ``/claim`` endpoint.
"""
ip = get_client_ip(request)
challenge = create_qr_challenge(db, owner_id, ip_address=ip)
# The QR payload is a JSON-like string with enough info for the mobile
# app to know the server URL and challenge token.
base_url = str(request.base_url).rstrip("/")
qr_payload = f"docuelevate://qr-login?token={challenge.challenge_token}&server={base_url}"
# Compute the TTL in seconds so the client can run a countdown timer
# without comparing absolute timestamps (which breaks when client and
# server clocks are out of sync).
ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds()))
return {
"challenge_id": challenge.id,
"challenge_token": challenge.challenge_token,
"expires_at": challenge.expires_at,
"ttl_seconds": ttl_seconds,
"qr_payload": qr_payload,
"qr_code_svg": _generate_qr_svg(qr_payload),
}
@router.get("/challenge/{challenge_id}/status", response_model=ChallengeStatusResponse)
@require_login
async def poll_challenge_status(
request: Request,
challenge_id: int,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Poll the status of a QR login challenge.
The web UI calls this endpoint every few seconds to check if the
mobile app has scanned the QR code and claimed the challenge.
"""
result = get_challenge_status(db, challenge_id, owner_id)
if not result:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Challenge not found")
return result
@router.post("/claim", response_model=ClaimChallengeResponse)
async def claim_challenge(
request: Request,
body: ClaimChallengeRequest,
db: DbSession,
) -> dict[str, Any]:
"""Claim a QR login challenge and receive an API token.
This endpoint is called by the mobile app after scanning a QR code.
It does **not** require authentication — the challenge token itself
serves as proof that the user authorized this login from their web
session.
"""
ip = get_client_ip(request)
result = claim_qr_challenge(db, body.challenge_token, device_name=body.device_name, ip_address=ip)
if not result:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid, expired, or already claimed challenge.",
)
try:
from app.utils.audit_service import record_event
record_event(
db,
action="qr_login_claimed",
user=result["owner_id"],
resource_type="session",
ip_address=ip,
details={"device_name": body.device_name, "token_id": result["token_id"]},
severity="info",
)
except Exception:
logger.debug("Failed to write QR login audit event", exc_info=True)
return result
app/api/sessions.py
+196
View File
@@ -0,0 +1,196 @@
"""API endpoints for managing user sessions.
Provides endpoints for listing active sessions, revoking individual sessions,
and the "log off everywhere" feature that invalidates all sessions and API
tokens across all devices.
"""
from __future__ import annotations
import logging
from datetime import datetime
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.middleware.audit_log import get_client_ip
from app.utils.session_manager import (
get_session_lifetime_days,
list_user_sessions,
revoke_all_sessions,
revoke_session,
)
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/sessions", tags=["sessions"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helper
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if not owner_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Response schemas
# ---------------------------------------------------------------------------
class SessionResponse(BaseModel):
"""Serialised user session for the management UI."""
id: int
device_info: str | None
ip_address: str | None
created_at: datetime
last_active_at: datetime
expires_at: datetime
is_current: bool = False
class SessionListResponse(BaseModel):
"""Response for listing active sessions."""
sessions: list[SessionResponse]
session_lifetime_days: int
class RevokeAllResponse(BaseModel):
"""Response after revoking all sessions."""
revoked_count: int
message: str
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/", response_model=SessionListResponse)
@require_login
async def list_sessions(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""List all active sessions for the current user."""
sessions = list_user_sessions(db, owner_id)
# Determine which session is the current one
current_token = request.session.get("_session_token")
session_list = []
for s in sessions:
session_list.append(
{
"id": s.id,
"device_info": s.device_info,
"ip_address": s.ip_address,
"created_at": s.created_at,
"last_active_at": s.last_active_at,
"expires_at": s.expires_at,
"is_current": s.session_token == current_token if current_token else False,
}
)
return {
"sessions": session_list,
"session_lifetime_days": get_session_lifetime_days(),
}
@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT)
@require_login
async def revoke_single_session(
request: Request,
session_id: int,
owner_id: CurrentOwner,
db: DbSession,
) -> None:
"""Revoke a specific session by ID."""
success = revoke_session(db, session_id, owner_id)
if not success:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found")
try:
from app.utils.audit_service import record_event
record_event(
db,
action="session_revoked",
user=owner_id,
resource_type="session",
resource_id=str(session_id),
ip_address=get_client_ip(request),
severity="info",
)
except Exception:
logger.debug("Failed to write session revocation audit event", exc_info=True)
@router.post("/revoke-all", response_model=RevokeAllResponse)
@require_login
async def revoke_all(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Revoke all sessions except the current one ("log off everywhere").
Also revokes all active API tokens for the user, which invalidates
mobile app sessions and any programmatic access.
"""
# Find current session to preserve it
current_token = request.session.get("_session_token")
current_session_id = None
if current_token:
from app.models import UserSession
current = db.query(UserSession).filter(UserSession.session_token == current_token).first()
if current:
current_session_id = current.id
count = revoke_all_sessions(
db,
owner_id,
except_session_id=current_session_id,
revoke_api_tokens=True,
)
try:
from app.utils.audit_service import record_event
record_event(
db,
action="revoke_all_sessions",
user=owner_id,
resource_type="session",
ip_address=get_client_ip(request),
details={"revoked_count": count},
severity="warning",
)
except Exception:
logger.debug("Failed to write revoke-all audit event", exc_info=True)
return {
"revoked_count": count,
"message": f"Successfully revoked {count} session(s) and all API tokens.",
}
app/api/shared_links.py
+7 -5
View File
@@ -313,16 +313,18 @@ async def list_shared_links(
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
) -> list[dict[str, Any]]:
"""List all shared links created by the authenticated user."""
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
q = (
db.query(SharedLink, FileRecord.original_filename)
.outerjoin(FileRecord, SharedLink.file_id == FileRecord.id)
.filter(SharedLink.owner_id == owner_id)
)
if active_only:
q = q.filter(SharedLink.is_active.is_(True))
links = q.order_by(SharedLink.created_at.desc()).all()
links_with_filenames = q.order_by(SharedLink.created_at.desc()).all()
base_url = str(request.base_url).rstrip("/")
result = []
for link in links:
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
filename = file_record.original_filename if file_record else None
for link, filename in links_with_filenames:
result.append(_link_to_dict(link, base_url, filename))
return result
app/api/system_reset.py
+124
View File
@@ -0,0 +1,124 @@
"""
System reset API endpoints for DocuElevate.
Provides admin-only REST endpoints for:
- Full system reset (wipe all user data)
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
admin privileges.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
def _require_admin(request: Request) -> dict:
"""Ensure the caller is an admin. Raises 403 otherwise."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
def _require_feature_enabled() -> None:
"""Raise 404 when the factory-reset feature flag is off."""
if not settings.enable_factory_reset:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
)
class ResetRequest(BaseModel):
"""Body for system reset endpoints. Requires explicit confirmation."""
confirmation: str
@router.post("/full")
async def full_reset(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Wipe all user data (database + work-files).
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "DELETE":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
)
from app.utils.system_reset import perform_full_reset
try:
result = perform_full_reset(db)
except Exception as exc:
logger.exception("Full system reset failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"System reset failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.post("/reimport")
async def reset_and_reimport(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Move original files to a reimport folder, wipe everything, and
configure the reimport folder as a watch folder for automatic
re-ingestion.
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "REIMPORT":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
)
from app.utils.system_reset import perform_reset_and_reimport
try:
result = perform_reset_and_reimport(db)
except Exception as exc:
logger.exception("Reset-and-reimport failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Reset and reimport failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.get("/status")
async def reset_status(_admin: AdminUser) -> dict:
"""Return whether the system reset feature is enabled."""
return {
"enabled": settings.enable_factory_reset,
"factory_reset_on_startup": settings.factory_reset_on_startup,
}
app/api/translation.py
+156
View File
@@ -0,0 +1,156 @@
"""
API endpoints for document translation.
Provides on-the-fly translation via the AI provider and access to the
persisted default-language translation.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord
from app.utils.ai_provider import get_ai_provider
from app.utils.user_scope import apply_owner_filter
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
# Maximum characters sent to the AI provider for a single translation request.
_MAX_TRANSLATION_INPUT = 50_000
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
"""Fetch a FileRecord visible to the current user or raise 404."""
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
record = query.first()
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
return record
@router.get("/files/{file_id}/translation/default")
@require_login
def get_default_translation(
request: Request,
file_id: int,
db: DbSession,
) -> JSONResponse:
"""Return the persisted default-language translation for a document.
Returns 404 if no default-language translation has been generated yet
(e.g. because the document is already in the default language).
"""
record = _get_file_or_404(db, file_id, request)
if not record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available for this file",
)
return JSONResponse(
content={
"file_id": record.id,
"detected_language": record.detected_language,
"default_language_code": record.default_language_code,
"text": record.default_language_text,
}
)
@router.get("/files/{file_id}/translate")
@require_login
def translate_on_the_fly(
request: Request,
file_id: int,
db: DbSession,
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
) -> JSONResponse:
"""Translate a document's extracted text into an arbitrary language on the fly.
The translation is generated via the configured AI provider and is **not**
persisted. For the default-language translation, use the
``/files/{file_id}/translation/default`` endpoint instead.
"""
record = _get_file_or_404(db, file_id, request)
source_text = record.ocr_text
if not source_text:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No extracted text available for this file — translation requires OCR text",
)
# If the requested language matches what is already stored, return it directly.
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
return JSONResponse(
content={
"file_id": record.id,
"source_language": record.detected_language,
"target_language": lang,
"text": record.default_language_text,
"cached": True,
}
)
# If the detected language already matches, return the original text.
detected = record.detected_language
if detected and detected == lang:
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected,
"target_language": lang,
"text": source_text,
"cached": True,
}
)
# Truncate to keep AI costs bounded.
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"into {lang}. Preserve the original formatting, paragraph structure, "
f"and meaning. Do not add any commentary — output ONLY the translated text."
),
},
{"role": "user", "content": text_to_translate},
],
model=model,
temperature=0.3,
)
except Exception as exc:
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Translation failed — the AI provider returned an error",
)
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected or "unknown",
"target_language": lang,
"text": translated,
"cached": False,
}
)
app/api/url_upload.py
+63 -56
View File
@@ -9,7 +9,8 @@ import urllib.parse
import uuid
from typing import Optional
import requests
import aiofiles
import httpx
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, HttpUrl, field_validator
@@ -153,67 +154,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
logger.info(f"Downloading file from URL: {url}")
# Use configured timeout to prevent hanging
response = requests.get(
url,
async with httpx.AsyncClient(
timeout=settings.http_request_timeout,
stream=True, # Stream to handle large files
allow_redirects=True, # Follow redirects
follow_redirects=True,
headers={
"User-Agent": "DocuElevate/1.0", # Identify ourselves
},
)
response.raise_for_status()
) as client:
async with client.stream("GET", url) as response:
response.raise_for_status()
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
if "." in safe_filename:
file_extension = safe_filename.rsplit(".", 1)[1]
target_filename = f"{unique_id}.{file_extension}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
with open(target_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
f.close()
os.remove(target_path)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
# Check for extension using original_filename to avoid any CodeQL issues
# with safe_filename which is derived from the URL directly.
if "." in original_filename:
_, ext = os.path.splitext(original_filename)
# Strip out the leading dot and any non-alphanumeric chars
clean_ext = "".join(c for c in ext if c.isalnum())
if not clean_ext:
clean_ext = "bin"
target_filename = f"{unique_id}.{clean_ext}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
async with aiofiles.open(target_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=8192):
if chunk:
await f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
)
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
# Enqueue for processing
@@ -227,19 +234,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
"size": downloaded_size,
}
except requests.exceptions.Timeout:
except httpx.TimeoutException:
logger.error(f"Timeout while downloading file from URL: {url}")
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
except requests.exceptions.ConnectionError as e:
except httpx.ConnectError as e:
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
except requests.exceptions.HTTPError as e:
except httpx.HTTPStatusError as e:
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
except requests.exceptions.RequestException as e:
except httpx.RequestError as e:
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
app/auth.py
+183 -2
View File
@@ -125,8 +125,36 @@ def get_current_user(request: Request):
# Check for Bearer token auth first (API tokens)
api_user = getattr(request.state, "api_token_user", None)
if isinstance(api_user, dict):
logger.debug("[AUTH] get_current_user: resolved from API token (user_id=%s)", api_user.get("id"))
return api_user
return request.session.get("user")
session_user = request.session.get("user")
if session_user:
# Validate server-side session if a session token is present
session_token = request.session.get("_session_token")
if session_token:
try:
from app.database import SessionLocal
from app.utils.session_manager import validate_session
db = SessionLocal()
try:
valid = validate_session(db, session_token)
if not valid:
logger.debug("[AUTH] get_current_user: server-side session invalid — clearing")
request.session.pop("user", None)
request.session.pop("_session_token", None)
return None
finally:
db.close()
except Exception:
logger.debug("[AUTH] get_current_user: session validation error", exc_info=True)
logger.debug(
"[AUTH] get_current_user: resolved from session (user=%s)",
session_user.get("preferred_username") or session_user.get("email") or session_user.get("id"),
)
else:
logger.debug("[AUTH] get_current_user: no user in session or API token")
return session_user
def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
@@ -141,10 +169,12 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
"""
auth_header = request.headers.get("authorization", "")
if not isinstance(auth_header, str) or not auth_header.startswith("Bearer "):
logger.debug("[AUTH] _resolve_bearer_user: no Bearer token in Authorization header")
return None
raw_token = auth_header[7:]
if not raw_token or not isinstance(raw_token, str):
logger.debug("[AUTH] _resolve_bearer_user: empty or invalid token after 'Bearer ' prefix")
return None
from app.api.api_tokens import hash_token
@@ -153,8 +183,15 @@ def _resolve_bearer_user(request: Request, db: Session) -> dict | None:
token_hash = hash_token(raw_token)
db_token = db.query(ApiToken).filter(ApiToken.token_hash == token_hash, ApiToken.is_active.is_(True)).first()
if db_token is None:
logger.debug("[AUTH] _resolve_bearer_user: no active API token matched the provided hash")
return None
logger.debug(
"[AUTH] _resolve_bearer_user: matched API token id=%s owner=%s",
db_token.id,
db_token.owner_id,
)
# Update usage tracking
try:
db_token.last_used_at = datetime.now(timezone.utc)
@@ -205,16 +242,18 @@ def require_login(func):
@wraps(func)
async def wrapper(request: Request, *args, **kwargs):
url_path = urlparse(str(request.url)).path
# Check session auth first
if request.session.get("user"):
logger.debug("[AUTH] require_login: session auth OK for %s", url_path)
if inspect.iscoroutinefunction(func):
return await func(*args, request=request, **kwargs)
else:
return func(*args, request=request, **kwargs)
# Fall back to Bearer token auth for API endpoints
url_path = urlparse(str(request.url)).path
if url_path.startswith("/api/"):
logger.debug("[AUTH] require_login: no session, trying Bearer token for %s", url_path)
try:
from app.database import SessionLocal
@@ -228,17 +267,22 @@ def require_login(func):
if api_user:
request.state.api_token_user = api_user
logger.debug(
"[AUTH] require_login: Bearer token auth OK for %s (user=%s)", url_path, api_user.get("id")
)
if inspect.iscoroutinefunction(func):
return await func(*args, request=request, **kwargs)
else:
return func(*args, request=request, **kwargs)
logger.debug("[AUTH] require_login: no valid auth for API endpoint %s — returning 401", url_path)
return JSONResponse(
status_code=status.HTTP_401_UNAUTHORIZED,
content={"error": "Not authenticated"},
)
# Non-API endpoint with no session — redirect to login
logger.debug("[AUTH] require_login: no session for %s — redirecting to /login", url_path)
request.session["redirect_after_login"] = str(request.url)
return RedirectResponse(url="/login", status_code=status.HTTP_302_FOUND)
@@ -307,9 +351,15 @@ async def login(request: Request):
async def oauth_login(request: Request):
"""Handle OAuth login flow"""
if not OAUTH_CONFIGURED:
logger.debug("[AUTH] oauth_login: OAuth not configured — redirecting to /login")
return RedirectResponse(url="/login?error=OAuth+not+configured", status_code=status.HTTP_302_FOUND)
redirect_uri = request.url_for("oauth_callback")
logger.debug(
"[AUTH] oauth_login: initiating Authentik OAuth redirect_uri=%s session_keys=%s",
redirect_uri,
list(request.session.keys()),
)
return await oauth.authentik.authorize_redirect(request, redirect_uri)
@@ -324,13 +374,23 @@ async def social_login(request: Request, provider: str):
A redirect to the provider's authorization page, or back to /login on error.
"""
if provider not in SOCIAL_PROVIDERS:
logger.debug(
"[AUTH] social_login: unknown provider=%r (registered=%s)", provider, list(SOCIAL_PROVIDERS.keys())
)
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
redirect_uri = request.url_for("social_callback", provider=provider)
oauth_client = getattr(oauth, provider, None)
if oauth_client is None:
logger.debug("[AUTH] social_login: provider=%r registered but OAuth client not configured", provider)
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
logger.debug(
"[AUTH] social_login: initiating %s OAuth, redirect_uri=%s session_keys=%s",
provider,
redirect_uri,
list(request.session.keys()),
)
return await oauth_client.authorize_redirect(request, redirect_uri)
@@ -389,27 +449,39 @@ async def social_callback(request: Request, provider: str, db: Session = Depends
A redirect to the user's original destination or the upload page.
"""
if provider not in SOCIAL_PROVIDERS:
logger.debug("[AUTH] social_callback: unknown provider=%r", provider)
return RedirectResponse(url="/login?error=Unknown+social+provider", status_code=status.HTTP_302_FOUND)
oauth_client = getattr(oauth, provider, None)
if oauth_client is None:
logger.debug("[AUTH] social_callback: provider=%r not configured", provider)
return RedirectResponse(url="/login?error=Provider+not+configured", status_code=status.HTTP_302_FOUND)
try:
logger.debug("[AUTH] social_callback: exchanging auth code for provider=%s", provider)
token = await oauth_client.authorize_access_token(request)
# Try standard OIDC userinfo first, fall back to token-embedded userinfo
raw_userinfo = token.get("userinfo")
if not raw_userinfo:
logger.debug("[AUTH] social_callback: no userinfo in token, fetching from userinfo endpoint")
try:
resp = await oauth_client.userinfo(token=token)
raw_userinfo = resp if isinstance(resp, dict) else resp.json() if hasattr(resp, "json") else {}
except Exception:
logger.debug("[AUTH] social_callback: userinfo endpoint failed, using empty dict", exc_info=True)
raw_userinfo = {}
user_data = _normalize_social_userinfo(provider, token, raw_userinfo)
logger.debug(
"[AUTH] social_callback: normalized user_data email=%s sub=%s provider=%s",
user_data.get("email"),
user_data.get("sub"),
provider,
)
if not user_data.get("email"):
logger.debug("[AUTH] social_callback: no email in user_data — aborting")
return RedirectResponse(
url="/login?error=Could+not+retrieve+email+from+provider",
status_code=status.HTTP_302_FOUND,
@@ -428,6 +500,27 @@ async def social_callback(request: Request, provider: str, db: Session = Depends
request.session["user"] = user_data
# Create server-side session for tracking and revocation
try:
from app.utils.session_manager import create_session
_session_user_id = (
user_data.get("sub")
or user_data.get("preferred_username")
or user_data.get("email")
or user_data.get("id")
)
if _session_user_id:
user_session = create_session(
db,
user_id=_session_user_id,
ip_address=get_client_ip(request),
user_agent=request.headers.get("user-agent"),
)
request.session["_session_token"] = user_session.session_token
except Exception:
logger.debug("[AUTH] Failed to create server-side session for social user", exc_info=True)
# Auto-create or update UserProfile
_ensure_user_profile(db, user_data, is_admin=False)
@@ -442,21 +535,29 @@ async def social_callback(request: Request, provider: str, db: Session = Depends
)
# Mobile app flow: issue an inline API token and redirect back to the app.
logger.debug(
"[MOBILE] social_callback: checking for mobile redirect (session has mobile_redirect_uri=%s)",
"mobile_redirect_uri" in request.session,
)
mobile_resp = _create_mobile_redirect(request, db)
if mobile_resp:
logger.info("[MOBILE] social_callback: returning mobile redirect response for provider=%s", provider)
return mobile_resp
if user_id:
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
if profile and not profile.onboarding_completed:
logger.debug("[AUTH] social_callback: user=%s needs onboarding, redirecting", user_id)
post_onboarding = request.session.pop("redirect_after_login", "/upload")
request.session["post_onboarding_redirect"] = post_onboarding
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
redirect_url = request.session.pop("redirect_after_login", "/upload")
logger.debug("[AUTH] social_callback: login complete, redirecting to %s", redirect_url)
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
except Exception as e:
logger.warning("[SECURITY] SOCIAL_LOGIN_FAILURE provider=%s error=%s", provider, type(e).__name__)
logger.debug("[AUTH] social_callback: full exception for provider=%s", provider, exc_info=True)
return RedirectResponse(
url="/login?error=Social+login+failed.+Please+try+again.", status_code=status.HTTP_302_FOUND
)
@@ -561,15 +662,23 @@ def _ensure_user_profile(db: Session, user_data: dict, is_admin: bool = False) -
async def oauth_callback(request: Request, db: Session = Depends(get_db)):
"""Handle OAuth callback from provider"""
try:
logger.debug("[AUTH] oauth_callback: exchanging authorization code for token")
token = await oauth.authentik.authorize_access_token(request)
userinfo = token.get("userinfo")
if not userinfo:
logger.debug("[AUTH] oauth_callback: no userinfo in token response — aborting")
return RedirectResponse(
url="/login?error=Failed+to+retrieve+user+information", status_code=status.HTTP_302_FOUND
)
# Store user info in session
user_data = dict(userinfo)
logger.debug(
"[AUTH] oauth_callback: received userinfo email=%s sub=%s groups=%s",
user_data.get("email"),
user_data.get("sub"),
user_data.get("groups", []),
)
# Add Gravatar picture if no picture is provided
if not user_data.get("picture") and user_data.get("email"):
@@ -584,12 +693,39 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)):
groups = user_data.get("groups", [])
admin_group = (settings.admin_group_name or "admin").strip().lower()
is_admin = admin_group in [group.lower() for group in groups]
logger.debug(
"[AUTH] oauth_callback: admin group check — looking for %r in %s → is_admin=%s",
admin_group,
[g.lower() for g in groups],
is_admin,
)
# Set is_admin flag (defaults to False for OAuth users unless they're in admin group)
user_data["is_admin"] = is_admin
request.session["user"] = user_data
# Create server-side session for tracking and revocation
try:
from app.utils.session_manager import create_session
_session_user_id = (
user_data.get("sub")
or user_data.get("preferred_username")
or user_data.get("email")
or user_data.get("id")
)
if _session_user_id:
user_session = create_session(
db,
user_id=_session_user_id,
ip_address=get_client_ip(request),
user_agent=request.headers.get("user-agent"),
)
request.session["_session_token"] = user_session.session_token
except Exception:
logger.debug("[AUTH] Failed to create server-side session for OAuth user", exc_info=True)
# Auto-create or update UserProfile so the user appears in admin user management
_ensure_user_profile(db, user_data, is_admin=is_admin)
@@ -623,15 +759,18 @@ async def oauth_callback(request: Request, db: Session = Depends(get_db)):
if user_id:
profile = db.query(_UserProfile).filter(_UserProfile.user_id == user_id).first()
if profile and not profile.onboarding_completed:
logger.debug("[AUTH] oauth_callback: user=%s needs onboarding, redirecting", user_id)
post_onboarding = request.session.pop("redirect_after_login", "/upload")
request.session["post_onboarding_redirect"] = post_onboarding
return RedirectResponse(url="/onboarding", status_code=status.HTTP_302_FOUND)
# Redirect to original destination or default
redirect_url = request.session.pop("redirect_after_login", "/upload")
logger.debug("[AUTH] oauth_callback: login complete, redirecting to %s", redirect_url)
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
except Exception as e:
logger.warning(f"[SECURITY] OAUTH_LOGIN_FAILURE error={type(e).__name__}")
logger.debug("[AUTH] oauth_callback: full exception details", exc_info=True)
return RedirectResponse(url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND)
@@ -840,6 +979,19 @@ async def auth(request: Request, db: Session = Depends(get_db)):
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
user_data = _build_session_user(local_user)
request.session["user"] = user_data
# Create server-side session for tracking and revocation
try:
from app.utils.session_manager import create_session
user_session = create_session(
db,
user_id=local_user.email,
ip_address=get_client_ip(request),
user_agent=request.headers.get("user-agent"),
)
request.session["_session_token"] = user_session.session_token
except Exception:
logger.debug("[AUTH] Failed to create server-side session", exc_info=True)
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", local_user.email)
_record_login_event(db, request, local_user.email, success=True)
_ensure_user_profile(db, user_data, is_admin=bool(local_user.is_admin))
@@ -896,6 +1048,20 @@ async def auth(request: Request, db: Session = Depends(get_db)):
"is_admin": True,
}
request.session["user"] = admin_user_data
# Create server-side session for tracking and revocation
try:
from app.utils.session_manager import create_session
admin_user_id = settings.admin_username or "admin"
user_session = create_session(
db,
user_id=admin_user_id,
ip_address=get_client_ip(request),
user_agent=request.headers.get("user-agent"),
)
request.session["_session_token"] = user_session.session_token
except Exception:
logger.debug("[AUTH] Failed to create server-side session for admin", exc_info=True)
logger.info("[SECURITY] LOCAL_LOGIN_SUCCESS user=%s", username)
_record_login_event(db, request, username, success=True)
_ensure_user_profile(db, admin_user_data, is_admin=True)
@@ -931,6 +1097,7 @@ async def logout(request: Request, db: Session = Depends(get_db)):
username = "unknown"
if isinstance(user, dict):
username = user.get("preferred_username") or user.get("email") or "unknown"
logger.debug("[AUTH] logout: clearing session for user=%s client_ip=%s", username, get_client_ip(request))
logger.info(f"[SECURITY] LOGOUT user={username}")
try:
from app.utils.audit_service import record_event
@@ -945,6 +1112,20 @@ async def logout(request: Request, db: Session = Depends(get_db)):
)
except Exception:
logger.debug("Failed to write logout audit event for user=%s", username, exc_info=True)
# Revoke server-side session
session_token = request.session.get("_session_token")
if session_token:
try:
from app.utils.session_manager import validate_session
user_session = validate_session(db, session_token)
if user_session:
user_session.is_revoked = True
user_session.revoked_at = datetime.now(timezone.utc)
db.commit()
except Exception:
logger.debug("[AUTH] Failed to revoke server-side session", exc_info=True)
request.session.pop("_session_token", None)
request.session.pop("user", None)
return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302)
app/celery_app.py
+69 -2
View File
@@ -1,10 +1,15 @@
# app/celery_app.py
import logging
import os
from celery import Celery
from celery.signals import task_failure, worker_ready
from app.config import settings
logger = logging.getLogger(__name__)
celery = Celery(
"document_processor",
broker=settings.redis_url,
@@ -21,6 +26,64 @@ celery.conf.task_routes = {
"app.tasks.*": {"queue": "document_processor"},
}
# Mapping of document pipeline task names to the positional index of ``file_id``
# in their ``args`` tuple. These indices correspond to the task signatures:
# process_with_ocr(filename, file_id, ...) → index 1
# extract_metadata_with_gpt(filename, text, file_id) → index 2
# embed_metadata_into_pdf(path, text, metadata, file_id) → index 3
# Tasks that always pass ``file_id`` as a keyword argument
# (e.g. ``process_document``, ``finalize_document_storage``) are not listed
# here — their ``file_id`` is found via ``kwargs`` instead.
_FILE_ID_ARG_INDEX: dict[str, int] = {
"app.tasks.process_with_ocr.process_with_ocr": 1,
"app.tasks.extract_metadata_with_gpt.extract_metadata_with_gpt": 2,
"app.tasks.embed_metadata_into_pdf.embed_metadata_into_pdf": 3,
}
def _dispatch_user_failure_notification(sender, exception, args: list | None, kwargs: dict | None) -> None:
"""Best-effort per-user failure notification for document pipeline tasks.
Extracts ``file_id`` from the failed task's arguments, looks up the owning
user from the database, and dispatches a ``document.failed`` notification.
"""
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.user_notification import notify_user_document_failed
task_name = sender.name if sender else ""
if not task_name.startswith("app.tasks."):
return
# 1. Resolve file_id from kwargs or positional args
file_id = (kwargs or {}).get("file_id")
if file_id is None:
idx = _FILE_ID_ARG_INDEX.get(task_name)
if idx is not None and args and len(args) > idx:
val = args[idx]
if isinstance(val, int):
file_id = val
if file_id is None:
return
# 2. Look up owner from the database
with SessionLocal() as db:
record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not record or not record.owner_id:
return
owner_id = record.owner_id
filename = record.original_filename or record.local_filename or "unknown"
# 3. Dispatch per-user notification
error_msg = f"{type(exception).__name__}: {exception}" if exception else "Unknown error"
notify_user_document_failed(
owner_id=owner_id,
filename=os.path.basename(filename),
error=error_msg,
file_id=file_id,
)
@worker_ready.connect
def init_sentry_on_worker_ready(**kwargs):
@@ -48,6 +111,10 @@ def task_failure_handler(
kwargs=kwargs or {},
)
except Exception as e:
import logging
logger.exception(f"Failed to send task failure notification: {e}")
logging.exception(f"Failed to send task failure notification: {e}")
# Also dispatch a per-user failure notification for document pipeline tasks
try:
_dispatch_user_failure_notification(sender, exception, args, kwargs)
except Exception:
logger.warning("Could not dispatch per-user failure notification", exc_info=True)
app/celery_worker.py
+2
View File
@@ -39,6 +39,7 @@ from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
from app.tasks.subscription_tasks import apply_pending_subscription_changes_all # noqa: F401
from app.tasks.translate_to_default_language import translate_to_default_language # noqa: F401
# Import new send tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
@@ -51,6 +52,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
from app.tasks.upload_to_sharepoint import upload_to_sharepoint # noqa: F401
from app.tasks.upload_to_user_integration import upload_to_user_integration # noqa: F401
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401
app/config.py
+151
View File
@@ -48,6 +48,51 @@ class Settings(BaseSettings):
workdir: str
debug: bool = False # Default to False
# Logging level for the application. Accepts standard Python level names:
# DEBUG, INFO, WARNING, ERROR, CRITICAL. When *debug* is True and
# *log_level* has not been explicitly set, the effective level is forced to
# DEBUG so that all ``logger.debug()`` calls produce output.
log_level: str = Field(
default="INFO",
description=(
"Python logging level for the application root logger. "
"Accepts: DEBUG, INFO, WARNING, ERROR, CRITICAL. "
"When DEBUG=True and LOG_LEVEL is not explicitly set, "
"the effective level is automatically lowered to DEBUG."
),
)
# Log output format. ``text`` is the human-readable default.
# ``json`` emits one JSON object per line, ideal for log collectors
# (Promtail, Fluentd, Filebeat, Datadog agent) and SIEM ingestion.
log_format: str = Field(
default="text",
description=(
"Log output format: 'text' (human-readable, default) or "
"'json' (structured JSON lines for SIEM / log aggregation)."
),
)
# Optional syslog forwarding for application logs (not just audit events).
# When enabled, a Python SysLogHandler is added to the root logger so that
# every log message is also sent to the configured syslog receiver.
log_syslog_enabled: bool = Field(
default=False,
description="Forward application logs to a syslog receiver in addition to stdout.",
)
log_syslog_host: str = Field(
default="localhost",
description="Hostname or IP of the syslog receiver for application logs.",
)
log_syslog_port: int = Field(
default=514,
description="Port of the syslog receiver for application logs.",
)
log_syslog_protocol: str = Field(
default="udp",
description="Protocol for syslog transport: 'udp' or 'tcp'.",
)
# Making Dropbox optional
dropbox_enabled: bool = Field(
default=True,
@@ -121,11 +166,50 @@ class Settings(BaseSettings):
google_docai_location: str = "us" # Processor location, e.g. "us" or "eu"
external_hostname: str = "localhost" # Default to localhost
# ---------------------------------------------------------------------------
# Document Translation Settings
# ---------------------------------------------------------------------------
# Default target language for automatic document translation (ISO 639-1 code).
# After OCR / metadata extraction, if the detected document language differs
# from this value the system translates the extracted text into this language
# and stores it alongside the original. Other language translations are
# generated on the fly via the AI provider and are NOT persisted.
# Per-user overrides are stored in UserProfile.default_document_language.
default_document_language: str = Field(
default="en",
description=(
"ISO 639-1 language code for the default translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language "
"differs are automatically translated into this language after "
"processing. Default: 'en' (English)."
),
)
# Authentication settings
auth_enabled: bool = True # Default to enabled
admin_username: Optional[str] = None
admin_password: Optional[str] = None
session_secret: Optional[str] = None
session_lifetime_days: int = Field(
default=30,
description=(
"Session lifetime in days. Common values: 30, 60, 90. "
"Determines how long a user stays logged in before being required to re-authenticate. "
"Applies to both browser sessions and the session cookie max_age."
),
)
session_lifetime_custom_days: int | None = Field(
default=None,
description=(
"Override session_lifetime_days with a custom value. "
"When set, this takes precedence over session_lifetime_days. "
"Useful for admin-configured non-standard durations."
),
)
qr_login_challenge_ttl_seconds: int = Field(
default=120,
description="Time-to-live in seconds for QR login challenges (default: 2 minutes).",
)
admin_group_name: str = "admin"
# Multi-user settings
@@ -525,6 +609,15 @@ class Settings(BaseSettings):
onedrive_refresh_token: Optional[str] = None # Required for personal accounts
onedrive_folder_path: Optional[str] = None
# SharePoint settings
sharepoint_client_id: Optional[str] = None
sharepoint_client_secret: Optional[str] = None
sharepoint_tenant_id: Optional[str] = "common"
sharepoint_refresh_token: Optional[str] = None
sharepoint_site_url: Optional[str] = None # e.g. https://tenant.sharepoint.com/sites/sitename
sharepoint_document_library: Optional[str] = "Documents" # Document library name
sharepoint_folder_path: Optional[str] = None # Subfolder inside the library
# AWS S3 settings
s3_enabled: bool = Field(
default=True,
@@ -575,6 +668,25 @@ class Settings(BaseSettings):
),
)
# System reset / factory reset settings
factory_reset_on_startup: bool = Field(
default=False,
description=(
"When enabled, DocuElevate wipes all user data (database rows and "
"work-files on disk) on every startup so the instance always comes "
"up in a clean, fresh state. Useful for demo or testing environments. "
"Default: False."
),
)
enable_factory_reset: bool = Field(
default=False,
description=(
"Show the 'System Reset' page in the admin UI. When enabled, "
"administrators can trigger a full data wipe or a wipe-and-reimport "
"directly from the web interface. Default: False."
),
)
# PDF/A archival conversion settings
enable_pdfa_conversion: bool = Field(
default=False,
@@ -1003,6 +1115,45 @@ class Settings(BaseSettings):
),
)
# Database Connection Pool Configuration
# Controls SQLAlchemy QueuePool behaviour for PostgreSQL/MySQL.
# SQLite uses NullPool and ignores these settings.
db_pool_size: int = Field(
default=5,
description="Number of persistent connections kept in the pool. Ignored for SQLite.",
)
db_max_overflow: int = Field(
default=10,
description=("Maximum number of connections that can be opened beyond db_pool_size. Ignored for SQLite."),
)
db_pool_timeout: int = Field(
default=30,
description="Seconds to wait for a connection from the pool before raising an error. Ignored for SQLite.",
)
db_pool_recycle: int = Field(
default=1800,
description=(
"Seconds after which a connection is recycled to prevent stale connections. "
"Ignored for SQLite. Default: 1800 (30 minutes)."
),
)
# Per-user upload rate limiting (health-aware limiter)
# Controls how many uploads a single user may submit within a sliding window.
upload_rate_limit_per_user: int = Field(
default=20,
description=(
"Maximum number of uploads allowed per user within the upload_rate_limit_window. "
"The limiter may dynamically reduce this value when Redis queue depth or CPU load is high."
),
)
upload_rate_limit_window: int = Field(
default=60,
description=(
"Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60 seconds."
),
)
# Rate Limiting Configuration (see SECURITY_AUDIT.md and docs/API.md)
# Protects against DoS attacks and API abuse
rate_limiting_enabled: bool = Field(
app/database.py
+16 -1
View File
@@ -10,6 +10,7 @@ from typing import Any
from sqlalchemy import create_engine, exc
from sqlalchemy.engine.url import make_url
from sqlalchemy.orm import Session, declarative_base, sessionmaker
from sqlalchemy.pool import NullPool, QueuePool
from app.config import settings
@@ -19,7 +20,20 @@ Base = declarative_base()
# Parse the DATABASE_URL
DB_URL = settings.database_url
engine = create_engine(DB_URL, connect_args={"check_same_thread": False})
_db_url = make_url(DB_URL)
if _db_url.get_backend_name() == "sqlite":
# SQLite does not benefit from connection pooling; NullPool avoids contention.
engine = create_engine(DB_URL, connect_args={"check_same_thread": False}, poolclass=NullPool)
else:
# PostgreSQL / MySQL / other: use a configurable QueuePool.
engine = create_engine(
DB_URL,
poolclass=QueuePool,
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout,
pool_recycle=settings.db_pool_recycle,
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@@ -271,6 +285,7 @@ def _ensure_indexes(engine: Any, inspector: Any) -> None:
if table not in columns_by_table:
columns_by_table[table] = {col["name"] for col in inspector.get_columns(table)}
if column in columns_by_table[table]:
# SECURITY: Quoted identifiers to prevent SQL injection during index creation
quoted_idx = preparer.quote(idx_name)
quoted_table = preparer.quote(table)
quoted_col = preparer.quote(column)
app/main.py
+129 -1
View File
@@ -1,8 +1,11 @@
#!/usr/bin/env python3
import json as _json_mod
import logging
import os
import pathlib
from contextlib import asynccontextmanager
from datetime import datetime as _dt
from datetime import timezone as _tz
from fastapi import FastAPI, HTTPException, Request, status
from fastapi.middleware.cors import CORSMiddleware
@@ -36,6 +39,114 @@ from app.views import router as frontend_router
# Explicitly include the files router
from app.views.files import router as files_router
# ---------------------------------------------------------------------------
# Configure Python root logging level early so that *all* loggers (including
# those already created via ``logging.getLogger(__name__)`` in other modules)
# respect the configured level.
#
# Standard behaviour (matches Django, Flask, 12-factor conventions):
# • ``LOG_LEVEL`` env var takes precedence when explicitly set.
# • When ``DEBUG=True`` and ``LOG_LEVEL`` is **not** set, the effective
# level is automatically lowered to ``DEBUG``.
# • Default (neither flag set): ``INFO``.
#
# ``LOG_FORMAT=json`` enables structured JSON lines on stdout, suitable for
# Promtail, Fluentd, Filebeat, Datadog, Splunk UF, or any log collector.
#
# ``LOG_SYSLOG_ENABLED=true`` adds a Python SysLogHandler so that every log
# message is also forwarded to the configured syslog receiver — useful for
# traditional (non-container) deployments and centralised SIEM ingestion.
#
# Noisy third-party loggers (httpx, httpcore, authlib, etc.) are pinned to
# WARNING when the app-level is DEBUG to keep output useful.
# ---------------------------------------------------------------------------
_explicit_log_level = os.environ.get("LOG_LEVEL")
if settings.debug and _explicit_log_level is None:
_effective_level = "DEBUG"
else:
_effective_level = settings.log_level.upper()
_effective_level_int = getattr(logging, _effective_level, logging.INFO)
class _JsonFormatter(logging.Formatter):
"""Emit one JSON object per log line for machine consumption.
Fields emitted: ``timestamp``, ``level``, ``logger``, ``message``,
``module``, ``funcName``, ``lineno``, and — when present — ``exc_info``.
Compatible with Grafana Loki, Splunk, ELK, Datadog, and most SIEM tools.
"""
def format(self, record: logging.LogRecord) -> str:
log_entry: dict = {
"timestamp": _dt.fromtimestamp(record.created, tz=_tz.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
"module": record.module,
"funcName": record.funcName,
"lineno": record.lineno,
}
if record.exc_info and record.exc_info[1] is not None:
log_entry["exc_info"] = self.formatException(record.exc_info)
return _json_mod.dumps(log_entry, default=str)
# Choose formatter based on LOG_FORMAT setting
if settings.log_format.lower() == "json":
_handler = logging.StreamHandler()
_handler.setFormatter(_JsonFormatter())
logging.root.handlers = [_handler]
logging.root.setLevel(_effective_level_int)
else:
logging.basicConfig(
level=_effective_level_int,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
# Optional: forward application logs to a syslog receiver
if settings.log_syslog_enabled:
import logging.handlers as _lh
import socket as _socket
_proto = settings.log_syslog_protocol.lower()
_socktype = _socket.SOCK_STREAM if _proto == "tcp" else _socket.SOCK_DGRAM
_syslog_handler = _lh.SysLogHandler(
address=(settings.log_syslog_host, settings.log_syslog_port),
socktype=_socktype,
)
_syslog_handler.setLevel(_effective_level_int)
# Use the same formatter as stdout (text or JSON)
if settings.log_format.lower() == "json":
_syslog_handler.setFormatter(_JsonFormatter())
else:
_syslog_handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s"))
logging.root.addHandler(_syslog_handler)
# Keep noisy third-party loggers quiet at DEBUG level
if _effective_level_int <= logging.DEBUG:
for _noisy in (
"httpx",
"httpcore",
"authlib",
"urllib3",
"hpack",
"multipart",
"watchfiles",
):
logging.getLogger(_noisy).setLevel(logging.WARNING)
_startup_logger = logging.getLogger(__name__)
_startup_logger.info(
"Root logging level set to %s (debug=%s, format=%s, syslog=%s)",
_effective_level,
settings.debug,
settings.log_format,
settings.log_syslog_enabled,
)
# Load configuration from .env for the session key
config = Config(".env")
# Use settings.session_secret which has proper validation
@@ -59,6 +170,12 @@ async def lifespan(app: FastAPI):
# Startup: Initialize database
init_db() # Create tables if they don't exist
# Factory reset on startup — wipe all user data before anything else
if settings.factory_reset_on_startup:
from app.utils.system_reset import perform_startup_reset
perform_startup_reset()
# Load settings from database after DB initialization
from app.database import SessionLocal
from app.utils.config_loader import load_settings_from_db
@@ -207,8 +324,19 @@ app.add_middleware(CSRFMiddleware, config=settings)
# See SECURITY_AUDIT.md – Infrastructure Security section
app.add_middleware(AuditLogMiddleware, config=settings)
# 3) Session Middleware (for request.session to work)
app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET)
def _get_session_max_age() -> int:
"""Compute session max-age at startup time."""
try:
from app.utils.session_manager import get_session_max_age_seconds
return get_session_max_age_seconds()
except Exception:
return 30 * 86400 # 30 days default fallback
app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET, max_age=_get_session_max_age())
# 3a) CORS Middleware - handles cross-origin requests and preflight (OPTIONS) responses.
# Disabled by default: set CORS_ENABLED=True only when NOT using a reverse proxy
app/middleware/csrf.py
+7
View File
@@ -20,6 +20,9 @@ How it works:
Exempt paths (CSRF is not checked even for state-changing methods):
- ``/oauth-callback`` – OAuth 2.0 callback; protected by the ``state`` parameter.
- ``/api/qr-auth/claim`` – Called by the unauthenticated mobile app; the
cryptographically-random, single-use challenge token provides equivalent
protection.
"""
import logging
@@ -39,6 +42,10 @@ CSRF_PROTECTED_METHODS = {"POST", "PUT", "DELETE", "PATCH"}
# their own replay-protection mechanism).
CSRF_EXEMPT_PATHS = {
"/oauth-callback",
# The mobile app calls this endpoint without a browser session/CSRF token.
# The cryptographically-random, single-use challenge token already provides
# equivalent protection against cross-site request forgery.
"/api/qr-auth/claim",
}
app/models.py
+101
View File
@@ -84,6 +84,19 @@ class FileRecord(Base):
# Processing pipeline assigned to this file (NULL = use system default)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True)
# Detected document language (ISO 639-1 code, e.g. "de", "en", "fr")
# Extracted from AI metadata during processing; cached here for fast access.
detected_language = Column(String(10), nullable=True)
# Default-language translation of the extracted text.
# Stored when the detected language differs from the user's/system default
# document language. Only the original text and this translation are persisted;
# other languages are translated on the fly via the AI provider.
default_language_text = Column(Text, nullable=True)
# ISO 639-1 code of the default-language translation stored above (e.g. "en").
default_language_code = Column(String(10), nullable=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
@@ -282,6 +295,12 @@ class UserProfile(Base):
# NULL means "auto-detect from browser Accept-Language header"
preferred_language = Column(String(10), nullable=True)
# Default document language for translated versions (ISO 639-1 code).
# When a document's detected language differs from this value, the system
# automatically generates and stores a translation into this language.
# NULL means "use the global DEFAULT_DOCUMENT_LANGUAGE setting".
default_document_language = Column(String(10), nullable=True)
# UI colour scheme preference: "light" | "dark" | "system" (NULL = "system")
preferred_theme = Column(String(10), nullable=True)
@@ -584,6 +603,7 @@ class IntegrationType:
EMAIL = "EMAIL"
PAPERLESS = "PAPERLESS"
RCLONE = "RCLONE"
SHAREPOINT = "SHAREPOINT"
ICLOUD = "ICLOUD"
ALL = {
@@ -601,6 +621,7 @@ class IntegrationType:
EMAIL,
PAPERLESS,
RCLONE,
SHAREPOINT,
ICLOUD,
}
@@ -950,6 +971,86 @@ class MobileDevice(Base):
__table_args__ = (UniqueConstraint("owner_id", "push_token", name="uq_mobile_device_owner_token"),)
class UserSession(Base):
"""Server-side session tracking for invalidation and device management.
Each row represents an active browser or app session. The ``session_token``
is stored in the user's cookie and validated on every authenticated request.
Revoking a row (``is_revoked=True``) immediately terminates that session
on the next request.
"""
__tablename__ = "user_sessions"
id = Column(Integer, primary_key=True, index=True)
# Cryptographically random token stored in the session cookie.
session_token = Column(String(128), unique=True, nullable=False, index=True)
# Stable owner identifier — matches FileRecord.owner_id.
user_id = Column(String, nullable=False, index=True)
# Client metadata for display in the session management UI.
ip_address = Column(String(45), nullable=True)
user_agent = Column(String(512), nullable=True)
device_info = Column(String(255), nullable=True)
is_revoked = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
last_active_at = Column(DateTime(timezone=True), server_default=func.now())
expires_at = Column(DateTime(timezone=True), nullable=False)
revoked_at = Column(DateTime(timezone=True), nullable=True)
class QRLoginChallenge(Base):
"""Time-limited QR code login challenge for mobile app authentication.
A logged-in web user generates a challenge that produces a QR code. The
mobile app scans the QR code and calls the claim endpoint with the
``challenge_token``. The server verifies the challenge is still valid,
unclaimed, and unexpired, then issues an API token for the mobile app.
Security properties:
* Time-bound (default 2 minutes).
* Single-use (``is_claimed`` prevents replay).
* Cryptographically random 64-byte token.
* Bound to the creating user — only that user's mobile device receives a
token.
"""
__tablename__ = "qr_login_challenges"
id = Column(Integer, primary_key=True, index=True)
# Cryptographically random token encoded in the QR code.
challenge_token = Column(String(128), unique=True, nullable=False, index=True)
# The user who created this challenge (from the web session).
user_id = Column(String, nullable=False, index=True)
# Whether the challenge has been successfully claimed by a mobile app.
is_claimed = Column(Boolean, nullable=False, default=False)
# Whether the challenge has been explicitly cancelled or expired.
is_cancelled = Column(Boolean, nullable=False, default=False)
# IP address of the web client that created the challenge.
created_by_ip = Column(String(45), nullable=True)
# IP address of the mobile client that claimed the challenge.
claimed_by_ip = Column(String(45), nullable=True)
# Device name provided by the mobile app when claiming.
device_name = Column(String(255), nullable=True)
# The API token ID that was issued to the mobile app (for audit trail).
issued_token_id = Column(Integer, nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
expires_at = Column(DateTime(timezone=True), nullable=False)
claimed_at = Column(DateTime(timezone=True), nullable=True)
class ComplianceTemplate(Base):
"""Pre-built compliance configuration templates (GDPR, HIPAA, SOC2).
app/tasks/embed_metadata_into_pdf.py
+24
View File
@@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as search_exc:
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
# Cache the detected language on the FileRecord and trigger
# default-language translation when the document is in a
# different language.
detected_lang = metadata.get("language") if metadata else None
if detected_lang and extracted_text:
try:
file_record.detected_language = detected_lang
db.commit()
from app.tasks.translate_to_default_language import translate_to_default_language
translate_to_default_language.delay(
file_id,
extracted_text,
detected_lang,
owner_id=file_record.owner_id,
)
logger.info(
f"[{task_id}] Queued default-language translation for file {file_id} "
f"(detected: {detected_lang})"
)
except Exception as trans_exc:
logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}")
# Persist the metadata into a JSON file with the same base name.
# Include file path references for traceability
logger.info(f"[{task_id}] Persisting metadata to JSON")
app/tasks/finalize_document_storage.py
+13 -1
View File
@@ -21,8 +21,9 @@ from app.tasks.send_to_all import (
# Import database and logging utils from main
from app.utils import log_task_progress
# Import notification utility
# Import notification utilities
from app.utils.notification import notify_file_processed
from app.utils.user_notification import notify_user_document_processed
logger = logging.getLogger(__name__)
@@ -139,4 +140,15 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
except Exception as e:
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
# 6. Send per-user notification
if owner_id:
try:
notify_user_document_processed(
owner_id=owner_id,
filename=os.path.basename(processed_file),
file_id=file_id,
)
except Exception as e:
logger.warning(f"[WARNING] Failed to send per-user processed notification: {e}")
return {"status": "Completed", "file": processed_file}
app/tasks/imap_tasks.py
+6
View File
@@ -18,6 +18,7 @@ from app.utils.allowed_types import (
DEFAULT_CATEGORIES,
get_allowed_types_for_categories,
)
from app.utils.network import is_private_ip
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -405,6 +406,11 @@ def pull_inbox(
)
processed_emails = load_processed_emails()
# Security: Prevent SSRF by blocking connections to internal IPs
if is_private_ip(host):
logger.warning("SSRF blocked: Attempt to pull mailbox from private IP %s", host)
return
try:
mail = imaplib.IMAP4_SSL(host, port) if use_ssl else imaplib.IMAP4(host, port)
mail.login(username, password)
app/tasks/send_to_all.py
+19
View File
@@ -18,6 +18,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
from app.tasks.upload_to_webdav import upload_to_webdav
from app.utils.config_validator import get_provider_status
from app.utils.logging import log_task_progress
@@ -121,6 +122,18 @@ def _should_upload_to_icloud():
return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password)
def _should_upload_to_sharepoint():
return bool(
settings.sharepoint_client_id
and settings.sharepoint_client_secret
and settings.sharepoint_site_url
and (
settings.sharepoint_refresh_token
or (settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common")
)
)
def get_configured_services_from_validator():
"""
Use the config validator to determine which services are configured and enabled.
@@ -140,6 +153,7 @@ def get_configured_services_from_validator():
"Email": "email",
"OneDrive": "onedrive",
"S3 Storage": "s3",
"SharePoint": "sharepoint",
"iCloud Drive": "icloud",
}
@@ -250,6 +264,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
"should_upload": _should_upload_to_s3,
"upload_func": upload_to_s3,
},
{
"name": "sharepoint",
"should_upload": _should_upload_to_sharepoint,
"upload_func": upload_to_sharepoint,
},
{
"name": "icloud",
"should_upload": _should_upload_to_icloud,
app/tasks/translate_to_default_language.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Celery task to translate extracted document text into the default target language.
This task is triggered after metadata extraction when the detected document
language differs from the user's (or system) default document language. The
translated text is persisted in ``FileRecord.default_language_text`` so that
users can always read a reference copy in their preferred language.
Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate``
endpoint and are NOT persisted.
"""
import logging
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord, UserProfile
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.ai_provider import get_ai_provider
logger = logging.getLogger(__name__)
def _resolve_default_language(owner_id: str | None) -> str:
"""Return the default document language for the given owner.
Resolution order:
1. ``UserProfile.default_document_language`` (per-user override)
2. ``settings.default_document_language`` (global setting)
"""
if owner_id:
with SessionLocal() as db:
profile = db.query(UserProfile).filter_by(user_id=owner_id).first()
if profile and profile.default_document_language:
return profile.default_document_language
return settings.default_document_language
@celery.task(base=BaseTaskWithRetry, bind=True)
def translate_to_default_language(
self,
file_id: int,
extracted_text: str,
detected_language: str,
owner_id: str | None = None,
) -> dict:
"""Translate *extracted_text* into the default document language and persist the result.
Args:
file_id: Primary key of the :class:`FileRecord`.
extracted_text: The OCR / refined text in the document's original language.
detected_language: ISO 639-1 code of the document's detected language.
owner_id: Owner identifier used to resolve per-user language preference.
Returns:
A dict with ``status``, ``target_language``, and the translated text length.
"""
task_id = self.request.id
target_language = _resolve_default_language(owner_id)
# Nothing to do when the document is already in the target language.
if detected_language == target_language:
logger.info(
f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation"
)
log_task_progress(
task_id,
"translate_to_default_language",
"skipped",
f"Document already in {target_language}",
file_id=file_id,
)
return {"status": "skipped", "reason": "already_in_target_language"}
logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'")
log_task_progress(
task_id,
"translate_to_default_language",
"in_progress",
f"Translating from {detected_language} to {target_language}",
file_id=file_id,
)
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated_text = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"from {detected_language} to {target_language}. "
f"Preserve the original formatting, paragraph structure, and meaning. "
f"Do not add any commentary or explanation — output ONLY the translated text."
),
},
{"role": "user", "content": extracted_text},
],
model=model,
temperature=0.3,
)
# Persist the translation.
with SessionLocal() as db:
record = db.query(FileRecord).filter_by(id=file_id).first()
if record:
record.default_language_text = translated_text
record.default_language_code = target_language
record.detected_language = detected_language
db.commit()
logger.info(
f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}"
)
log_task_progress(
task_id,
"translate_to_default_language",
"success",
f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})",
file_id=file_id,
)
return {
"status": "success",
"target_language": target_language,
"translated_length": len(translated_text),
}
except Exception as exc:
logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}")
log_task_progress(
task_id,
"translate_to_default_language",
"failure",
f"Exception: {exc}",
file_id=file_id,
)
raise
app/tasks/upload_to_email.py
+34 -5
View File
@@ -11,6 +11,7 @@ from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import pypdf
from jinja2 import Environment, FileSystemLoader, select_autoescape
from app.celery_app import celery
@@ -23,6 +24,15 @@ logger = logging.getLogger(__name__)
# Constants
_LOGO_FILENAME = "logo.png"
# Mapping from PDF metadata keys (with leading slash stripped) to application-specific names.
# This mirrors the inverse of the mapping used in app/tasks/embed_metadata_into_pdf.py.
_PDF_METADATA_KEY_MAP = {
"Title": "filename",
"Author": "absender",
"Subject": "document_type",
"Keywords": "tags",
}
def get_email_template(template_name="default.html"):
"""
@@ -63,9 +73,12 @@ def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
2. Extract metadata from PDF if it's embedded
2. Extract embedded metadata from PDF using pypdf
Returns a dictionary of metadata or None if not found
JSON metadata takes precedence; embedded PDF metadata fills in any missing
fields using the application's standard key mapping (e.g., /Title → filename).
Returns a dictionary of metadata (may be empty if none found).
"""
metadata = {}
@@ -76,12 +89,28 @@ def extract_metadata_from_file(file_path):
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
# This would require additional dependencies, so for now we'll just check for external JSON
# Try to extract embedded metadata from PDF
if file_path.lower().endswith(".pdf") and os.path.exists(file_path):
try:
with open(file_path, "rb") as f:
pdf_reader = pypdf.PdfReader(f)
pdf_metadata = pdf_reader.metadata
if pdf_metadata:
for key, value in pdf_metadata.items():
# Remove the leading slash from PDF metadata keys (e.g., '/Title' -> 'Title')
clean_key = key[1:] if key.startswith("/") else key
# Map to application-specific key names where possible
mapped_key = _PDF_METADATA_KEY_MAP.get(clean_key, clean_key)
# Only set if not already present (JSON metadata takes precedence)
if mapped_key not in metadata:
metadata[mapped_key] = str(value)
logger.info(f"Extracted embedded metadata from PDF: {file_path}")
except Exception as e:
logger.warning(f"Failed to extract metadata from PDF {file_path}: {str(e)}")
return metadata
app/tasks/upload_to_sharepoint.py
+338
View File
@@ -0,0 +1,338 @@
#!/usr/bin/env python3
"""Upload documents to Microsoft SharePoint via the Microsoft Graph API.
This module authenticates using MSAL (same OAuth2 flow as OneDrive) and
uploads files to a configurable SharePoint Online document library using
the chunked upload session approach for reliability with large files.
Key differences from the OneDrive provider:
- Uses ``/sites/{siteId}/drives/{driveId}`` instead of ``/me/drive``
- Requires a SharePoint site URL to resolve the site and drive IDs
- Targets a named document library (default: ``Documents``)
"""
import logging
import os
import time
import urllib.parse
import msal
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import UploadTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_sharepoint_token() -> str:
"""Acquire a Microsoft Graph API access token for SharePoint.
Uses MSAL ``ConfidentialClientApplication`` with the refresh-token flow
(delegated permissions) or the client-credentials flow (application
permissions) depending on configuration.
Returns:
A valid access token string.
Raises:
ValueError: When required settings are missing or token acquisition fails.
"""
if not settings.sharepoint_client_id or not settings.sharepoint_client_secret:
raise ValueError("SharePoint client ID and client secret must be configured")
tenant = settings.sharepoint_tenant_id or "common"
logger.info("Using SharePoint tenant: %s", tenant)
scopes = ["https://graph.microsoft.com/.default"]
if settings.sharepoint_refresh_token:
app = msal.ConfidentialClientApplication(
client_id=settings.sharepoint_client_id,
client_credential=settings.sharepoint_client_secret,
authority=f"https://login.microsoftonline.com/{tenant}",
)
logger.info("Attempting to acquire SharePoint token using refresh token")
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.sharepoint_refresh_token, scopes=scopes
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
logger.error("Failed to get SharePoint access token: %s - %s", error, error_desc)
raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}")
if "refresh_token" in token_response:
settings.sharepoint_refresh_token = token_response["refresh_token"]
logger.info("Updated SharePoint refresh token in memory")
return token_response["access_token"]
elif settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common":
authority = f"https://login.microsoftonline.com/{settings.sharepoint_tenant_id}"
app = msal.ConfidentialClientApplication(
client_id=settings.sharepoint_client_id,
client_credential=settings.sharepoint_client_secret,
authority=authority,
)
token_response = app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}")
return token_response["access_token"]
else:
raise ValueError("For SharePoint, either a refresh token or a non-'common' tenant ID is required")
def resolve_sharepoint_drive(access_token: str, site_url: str, library_name: str) -> tuple[str, str]:
"""Resolve the Graph API site ID and drive ID for a SharePoint site.
Args:
access_token: Valid Microsoft Graph API token.
site_url: Full SharePoint site URL, e.g.
``https://tenant.sharepoint.com/sites/sitename``.
library_name: Display name of the document library (e.g. ``Documents``).
Returns:
A ``(site_id, drive_id)`` tuple.
Raises:
ValueError: When the site URL cannot be parsed.
RuntimeError: When the Graph API call fails.
"""
parsed = urllib.parse.urlparse(site_url)
hostname = parsed.hostname
site_path = parsed.path.rstrip("/")
if not hostname or not site_path:
raise ValueError(
f"Invalid SharePoint site URL '{site_url}'. Expected format: https://tenant.sharepoint.com/sites/sitename"
)
headers = {"Authorization": f"Bearer {access_token}"}
# Resolve site ID
site_api_url = f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}"
logger.info("Resolving SharePoint site: %s", site_api_url)
resp = requests.get(site_api_url, headers=headers, timeout=settings.http_request_timeout)
if resp.status_code != 200:
raise RuntimeError(f"Failed to resolve SharePoint site: {resp.status_code} - {resp.text}")
site_id = resp.json()["id"]
logger.info("Resolved SharePoint site ID: %s", site_id)
# Resolve drive ID from the document library name
drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives"
resp = requests.get(drives_url, headers=headers, timeout=settings.http_request_timeout)
if resp.status_code != 200:
raise RuntimeError(f"Failed to list SharePoint drives: {resp.status_code} - {resp.text}")
drives = resp.json().get("value", [])
drive_id = None
for drive in drives:
if drive.get("name", "").lower() == library_name.lower():
drive_id = drive["id"]
break
if not drive_id:
available = [d.get("name") for d in drives]
raise RuntimeError(f"Document library '{library_name}' not found on site. Available libraries: {available}")
logger.info("Resolved SharePoint drive ID: %s (library: %s)", drive_id, library_name)
return site_id, drive_id
def create_sharepoint_upload_session(
filename: str, folder_path: str | None, drive_id: str, site_id: str, access_token: str
) -> str:
"""Create a resumable upload session on a SharePoint document library.
Args:
filename: Name of the file to upload.
folder_path: Optional subfolder path inside the library.
drive_id: Graph API drive ID of the document library.
site_id: Graph API site ID.
access_token: Valid access token.
Returns:
The upload session URL for chunked PUT requests.
Raises:
RuntimeError: When session creation fails.
"""
base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}"
if folder_path:
folder_path = folder_path.strip("/")
path_components = folder_path.split("/")
encoded_path = "/".join(urllib.parse.quote(component) for component in path_components)
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession"
else:
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_filename}:/createUploadSession"
url = f"{base_url}{item_path}"
request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}}
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
logger.info("Creating SharePoint upload session for %s at path %s", filename, folder_path)
response = requests.post(url, headers=headers, json=request_body, timeout=settings.http_request_timeout)
if response.status_code == 200:
upload_url = response.json().get("uploadUrl")
logger.info("SharePoint upload session created for %s", filename)
return upload_url
else:
raise RuntimeError(f"Failed to create SharePoint upload session: {response.status_code} - {response.text}")
def upload_large_file_sharepoint(file_path: str, upload_url: str) -> dict:
"""Upload a file to SharePoint using a chunked upload session.
Args:
file_path: Local path to the file.
upload_url: The upload session URL from ``create_sharepoint_upload_session``.
Returns:
The Graph API response dict containing file metadata.
Raises:
RuntimeError: When a chunk upload fails after retries.
"""
file_size = os.path.getsize(file_path)
chunk_size = 10 * 1024 * 1024 # 10 MB
response = None
with open(file_path, "rb") as f:
chunk_number = 0
while True:
chunk = f.read(chunk_size)
if not chunk:
break
chunk_start = chunk_number * chunk_size
chunk_end = chunk_start + len(chunk) - 1
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range}
max_retries = 3
retry_delay = 2
for attempt in range(max_retries):
try:
response = requests.put(
upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout
)
if response.status_code in (201, 202):
break
else:
logger.warning(
"SharePoint chunk upload failed (attempt %d): %d", attempt + 1, response.status_code
)
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
except Exception as e:
logger.warning("SharePoint chunk upload error (attempt %d): %s", attempt + 1, str(e))
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
if response is None or response.status_code not in (201, 202):
status = response.status_code if response else "no response"
text = response.text if response else ""
raise RuntimeError(f"Failed to upload chunk after {max_retries} attempts: {status} - {text}")
chunk_number += 1
return response.json() if response else {}
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_sharepoint(self, file_path: str, file_id: int = None, folder_override: str = None):
"""Upload a file to SharePoint Online.
Args:
file_path: Path to the file to upload.
file_id: Optional file ID to associate with logs.
folder_override: Optional folder path override.
Returns:
A dict with upload status and file details.
Raises:
FileNotFoundError: When the file does not exist.
ValueError: When SharePoint is not configured.
RuntimeError: When the upload fails.
"""
task_id = self.request.id
logger.info("[%s] Starting SharePoint upload: %s", task_id, file_path)
log_task_progress(
task_id,
"upload_to_sharepoint",
"in_progress",
f"Uploading to SharePoint: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
filename = os.path.basename(file_path)
if not settings.sharepoint_client_id:
error_msg = "SharePoint client ID is not configured"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
if not settings.sharepoint_site_url:
error_msg = "SharePoint site URL is not configured"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
try:
access_token = get_sharepoint_token()
library_name = settings.sharepoint_document_library or "Documents"
site_id, drive_id = resolve_sharepoint_drive(access_token, settings.sharepoint_site_url, library_name)
folder_path = folder_override if folder_override is not None else settings.sharepoint_folder_path
upload_url = create_sharepoint_upload_session(filename, folder_path, drive_id, site_id, access_token)
result = upload_large_file_sharepoint(file_path, upload_url)
web_url = result.get("webUrl", "Not available")
logger.info("[%s] Successfully uploaded %s to SharePoint", task_id, filename)
logger.info("[%s] File accessible at: %s", task_id, web_url)
log_task_progress(
task_id, "upload_to_sharepoint", "success", f"Uploaded to SharePoint: {filename}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"sharepoint_path": f"{folder_path or ''}/{filename}",
"web_url": web_url,
}
except Exception as e:
error_msg = f"Failed to upload {filename} to SharePoint: {str(e)}"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise RuntimeError(error_msg) from e
app/tasks/upload_to_user_integration.py
+108
View File
@@ -571,6 +571,113 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t
return {"status": "Completed", "rclone_dest": dest}
def _upload_sharepoint(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]:
"""Upload *file_path* to SharePoint using per-user MSAL credentials."""
import urllib.parse
import msal
import requests as _requests
client_id = creds.get("client_id") or ""
client_secret = creds.get("client_secret") or ""
refresh_token = creds.get("refresh_token") or ""
tenant = cfg.get("tenant_id") or "common"
site_url = cfg.get("site_url") or ""
library_name = cfg.get("document_library") or "Documents"
folder_path = cfg.get("folder_path") or ""
if not (client_id and client_secret):
raise ValueError("SharePoint integration is missing client_id or client_secret in credentials")
if not site_url:
raise ValueError("SharePoint integration is missing site_url in config")
scopes = ["https://graph.microsoft.com/.default"]
msal_app = msal.ConfidentialClientApplication(
client_id=client_id,
client_credential=client_secret,
authority=f"https://login.microsoftonline.com/{tenant}",
)
if refresh_token:
token_resp = msal_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=scopes)
else:
token_resp = msal_app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_resp:
raise ValueError(f"SharePoint token acquisition failed: {token_resp.get('error_description', 'unknown')}")
access_token = token_resp["access_token"]
headers = {"Authorization": f"Bearer {access_token}"}
# Resolve site ID
parsed = urllib.parse.urlparse(site_url)
hostname = parsed.hostname
site_path = parsed.path.rstrip("/")
if not hostname or not site_path:
raise ValueError(f"Invalid SharePoint site URL: {site_url}")
resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}", headers=headers, timeout=30)
resp.raise_for_status()
site_id = resp.json()["id"]
# Resolve drive ID
resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives", headers=headers, timeout=30)
resp.raise_for_status()
drive_id = None
for drive in resp.json().get("value", []):
if drive.get("name", "").lower() == library_name.lower():
drive_id = drive["id"]
break
if not drive_id:
raise RuntimeError(f"Document library '{library_name}' not found on SharePoint site")
filename = os.path.basename(file_path)
# Build upload-session URL
base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}"
if folder_path:
folder_path = folder_path.strip("/")
encoded_path = "/".join(urllib.parse.quote(p) for p in folder_path.split("/"))
encoded_file = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_path}/{encoded_file}:/createUploadSession"
else:
encoded_file = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_file}:/createUploadSession"
session_url = f"{base_url}{item_path}"
session_headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
resp = _requests.post(
session_url,
headers=session_headers,
json={"item": {"@microsoft.graph.conflictBehavior": "replace"}},
timeout=30,
)
resp.raise_for_status()
upload_url = resp.json()["uploadUrl"]
file_size = os.path.getsize(file_path)
chunk_size = 10 * 1024 * 1024
with open(file_path, "rb") as fh:
chunk_num = 0
while True:
chunk = fh.read(chunk_size)
if not chunk:
break
start = chunk_num * chunk_size
end = start + len(chunk) - 1
upload_headers = {
"Content-Length": str(len(chunk)),
"Content-Range": f"bytes {start}-{end}/{file_size}",
}
upload_resp = _requests.put(upload_url, headers=upload_headers, data=chunk, timeout=120)
if upload_resp.status_code not in (201, 202):
raise RuntimeError(f"SharePoint chunk upload failed: {upload_resp.status_code}")
chunk_num += 1
logger.info("[%s] SharePoint upload complete: %s/%s", task_id, folder_path, filename)
return {"status": "Completed", "sharepoint_folder": folder_path, "filename": filename}
def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]:
"""Upload *file_path* to iCloud Drive using per-user credentials.
@@ -615,6 +722,7 @@ _UPLOAD_HANDLERS = {
IntegrationType.PAPERLESS: _upload_paperless,
IntegrationType.EMAIL: _upload_email,
IntegrationType.RCLONE: _upload_rclone,
IntegrationType.SHAREPOINT: _upload_sharepoint,
IntegrationType.ICLOUD: _upload_icloud,
}
app/tasks/upload_with_rclone.py
+3 -3
View File
@@ -55,12 +55,12 @@ def upload_with_rclone(self, file_path: str, destination: str):
try:
# Ensure the remote path exists (create folders if needed)
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, destination]
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, "--", destination]
subprocess.run(mkdir_cmd, check=True, capture_output=True) # noqa: S603
# Construct the upload command
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, file_path, destination, "--progress"]
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, "--progress", "--", file_path, destination]
log_task_progress(task_id, "rclone_upload", "in_progress", f"Executing rclone copy to {destination}")
@@ -71,7 +71,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
if result.returncode == 0:
# Try to get a public link if possible
try:
link_cmd = ["rclone", "link", "--config", rclone_config_path, f"{destination}/{filename}"]
link_cmd = ["rclone", "link", "--config", rclone_config_path, "--", f"{destination}/{filename}"]
link_result = subprocess.run(link_cmd, capture_output=True, text=True, check=False) # noqa: S603
public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
except (subprocess.SubprocessError, OSError) as e:
app/utils/config_validator/providers.py
+22
View File
@@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]:
},
}
# Check SharePoint configuration
providers["SharePoint"] = {
"name": "SharePoint",
"icon": "fa-brands fa-microsoft",
"configured": bool(
getattr(settings, "sharepoint_client_id", None)
and getattr(settings, "sharepoint_client_secret", None)
and getattr(settings, "sharepoint_site_url", None)
),
"enabled": True,
"description": "Store documents in Microsoft SharePoint Online",
"details": {
"client_id": getattr(settings, "sharepoint_client_id", "Not set"),
"client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)),
"tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"),
"refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)),
"site_url": getattr(settings, "sharepoint_site_url", "Not set"),
"document_library": getattr(settings, "sharepoint_document_library", "Not set"),
"folder_path": getattr(settings, "sharepoint_folder_path", "Not set"),
},
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
app/utils/db_migrate.py
+4
View File
@@ -12,6 +12,7 @@ The utility:
"""
import logging
import re
from typing import Any
from sqlalchemy import MetaData, create_engine, inspect, text
@@ -84,6 +85,9 @@ def preview_migration(source_url: str) -> dict[str, Any]:
total = 0
with src_engine.connect() as conn:
for table_name in tables:
if not re.match(r"^[a-zA-Z0-9_]+$", table_name):
logger.warning(f"Skipping table with invalid name format: {table_name}")
continue
# table_name is safe — sourced from inspect().get_table_names(), not user input
quoted_table = conn.dialect.identifier_preparer.quote(table_name)
row = conn.execute(text(f"SELECT COUNT(*) FROM {quoted_table}")).fetchone() # noqa: S608
app/utils/env_utils.py
+54
View File
@@ -0,0 +1,54 @@
import logging
import os
logger = logging.getLogger(__name__)
def update_env_file(settings_to_update: dict[str, str]) -> bool:
"""
Updates the .env file with the given settings (best-effort).
Creates or modifies existing keys.
Args:
settings_to_update: A dictionary mapping uppercase env var names to their new string values.
Returns:
True if the file was successfully updated, False otherwise.
"""
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if not os.path.exists(env_path):
logger.warning(f".env file not found at {env_path}, skipping file write")
return False
logger.info(f"Updating settings in {env_path}")
with open(env_path, "r") as f:
env_lines = f.readlines()
updated = set()
new_env_lines = []
for line in env_lines:
stripped_line = line.rstrip()
is_updated = False
for key, value in settings_to_update.items():
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
new_env_lines.append(f"{key}={value}")
updated.add(key)
is_updated = True
break
if not is_updated:
new_env_lines.append(stripped_line)
for key, value in settings_to_update.items():
if key not in updated:
new_env_lines.append(f"{key}={value}")
with open(env_path, "w") as f:
f.write("\n".join(new_env_lines) + "\n")
logger.info("Successfully updated settings in .env file")
return True
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
return False
app/utils/file_operations.py
+12 -1
View File
@@ -7,8 +7,19 @@ def hash_file(filepath: str | Path, chunk_size: int = 65536) -> str:
Returns the SHA-256 hash of the file at 'filepath'.
Reads the file in chunks to handle large files efficiently.
"""
from app.config import settings
filepath_obj = Path(filepath).resolve()
workdir_obj = Path(settings.workdir).resolve()
# Security check: Ensure the resolved path is strictly within the allowed workdir
try:
filepath_obj.relative_to(workdir_obj)
except ValueError:
raise FileNotFoundError(f"Access denied: path traversal attempt or file outside workdir '{filepath}'")
sha256 = hashlib.sha256()
with open(filepath, "rb") as f:
with open(filepath_obj, "rb") as f:
while True:
data = f.read(chunk_size)
if not data:
app/utils/i18n.py
+79 -77
View File
@@ -34,89 +34,91 @@ logger = logging.getLogger(__name__)
SUPPORTED_LANGUAGES: list[dict[str, str]] = [
# --- Tier 1: Primary European languages ---
{"code": "en", "name": "English", "native": "English", "flag": "🇬🇧"},
{"code": "de", "name": "German", "native": "Deutsch", "flag": "🇩🇪"},
{"code": "fr", "name": "French", "native": "Français", "flag": "🇫🇷"},
{"code": "es", "name": "Spanish", "native": "Español", "flag": "🇪🇸"},
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "🇮🇹"},
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "🇵🇹"},
# flag: lowercase ISO 3166-1 alpha-2 country code used with the flag-icons CSS library
# (e.g. "gb" → <span class="fi fi-gb">). Regional codes like "gb-wls" are also supported.
{"code": "en", "name": "English", "native": "English", "flag": "gb"},
{"code": "de", "name": "German", "native": "Deutsch", "flag": "de"},
{"code": "fr", "name": "French", "native": "Français", "flag": "fr"},
{"code": "es", "name": "Spanish", "native": "Español", "flag": "es"},
{"code": "it", "name": "Italian", "native": "Italiano", "flag": "it"},
{"code": "pt", "name": "Portuguese", "native": "Português", "flag": "pt"},
# --- Tier 2: Western & Northern European ---
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "🇳🇱"},
{"code": "nb", "name": "Norwegian Bokmål", "native": "Norsk bokmål", "flag": "🇳🇴"},
{"code": "no", "name": "Norwegian", "native": "Norsk", "flag": "🇳🇴"},
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "🇩🇰"},
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "🇸🇪"},
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "🇫🇮"},
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "🇮🇸"},
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "🇮🇪"},
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "🇱🇺"},
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "🏴"},
{"code": "cy", "name": "Welsh", "native": "Cymraeg", "flag": "🏴󠁧󠁢󠁷󠁬󠁳󠁿"}, # Wales subdivision flag (U+1F3F4 + tag chars)
{"code": "fy", "name": "Western Frisian", "native": "Frysk", "flag": "🇳🇱"},
{"code": "gl", "name": "Galician", "native": "Galego", "flag": "🇪🇸"},
{"code": "li", "name": "Limburgish", "native": "Limburgs", "flag": "🇳🇱"},
{"code": "vls", "name": "Flemish", "native": "West-Vlams", "flag": "🇧🇪"},
{"code": "nds", "name": "Low German", "native": "Plattdüütsch", "flag": "🇩🇪"},
{"code": "nl", "name": "Dutch", "native": "Nederlands", "flag": "nl"},
{"code": "nb", "name": "Norwegian Bokmål", "native": "Norsk bokmål", "flag": "no"},
{"code": "no", "name": "Norwegian", "native": "Norsk", "flag": "no"},
{"code": "da", "name": "Danish", "native": "Dansk", "flag": "dk"},
{"code": "sv", "name": "Swedish", "native": "Svenska", "flag": "se"},
{"code": "fi", "name": "Finnish", "native": "Suomi", "flag": "fi"},
{"code": "is", "name": "Icelandic", "native": "Íslenska", "flag": "is"},
{"code": "ga", "name": "Irish", "native": "Gaeilge", "flag": "ie"},
{"code": "lb", "name": "Luxembourgish", "native": "Lëtzebuergesch", "flag": "lu"},
{"code": "ca", "name": "Catalan", "native": "Català", "flag": "es"}, # no dedicated ISO flag; use Spain
{"code": "cy", "name": "Welsh", "native": "Cymraeg", "flag": "gb-wls"}, # flag-icons GB region code
{"code": "fy", "name": "Western Frisian", "native": "Frysk", "flag": "nl"},
{"code": "gl", "name": "Galician", "native": "Galego", "flag": "es"},
{"code": "li", "name": "Limburgish", "native": "Limburgs", "flag": "nl"},
{"code": "vls", "name": "Flemish", "native": "West-Vlams", "flag": "be"},
{"code": "nds", "name": "Low German", "native": "Plattdüütsch", "flag": "de"},
# --- Tier 3: Central & Eastern European ---
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "🇵🇱"},
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "🇨🇿"},
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "🇸🇰"},
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "🇭🇺"},
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "🇸🇮"},
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "🇭🇷"},
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "🇷🇴"},
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "🇧🇬"},
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "🇬🇷"},
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "🇪🇪"},
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "🇱🇻"},
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "🇱🇹"},
{"code": "sr", "name": "Serbian", "native": "Српски", "flag": "🇷🇸"},
{"code": "pl", "name": "Polish", "native": "Polski", "flag": "pl"},
{"code": "cs", "name": "Czech", "native": "Čeština", "flag": "cz"},
{"code": "sk", "name": "Slovak", "native": "Slovenčina", "flag": "sk"},
{"code": "hu", "name": "Hungarian", "native": "Magyar", "flag": "hu"},
{"code": "sl", "name": "Slovenian", "native": "Slovenščina", "flag": "si"},
{"code": "hr", "name": "Croatian", "native": "Hrvatski", "flag": "hr"},
{"code": "ro", "name": "Romanian", "native": "Română", "flag": "ro"},
{"code": "bg", "name": "Bulgarian", "native": "Български", "flag": "bg"},
{"code": "el", "name": "Greek", "native": "Ελληνικά", "flag": "gr"},
{"code": "et", "name": "Estonian", "native": "Eesti", "flag": "ee"},
{"code": "lv", "name": "Latvian", "native": "Latviešu", "flag": "lv"},
{"code": "lt", "name": "Lithuanian", "native": "Lietuvių", "flag": "lt"},
{"code": "sr", "name": "Serbian", "native": "Српски", "flag": "rs"},
# --- Tier 4: Non-EU European, Middle Eastern & African ---
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "🇹🇷"},
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "🇺🇦"},
{"code": "he", "name": "Hebrew", "native": "עברית", "flag": "🇮🇱"},
{"code": "ar", "name": "Arabic", "native": "العربية", "flag": "🇸🇦"},
{"code": "fa", "name": "Persian", "native": "فارسی", "flag": "🇮🇷"},
{"code": "af", "name": "Afrikaans", "native": "Afrikaans", "flag": "🇿🇦"},
{"code": "tr", "name": "Turkish", "native": "Türkçe", "flag": "tr"},
{"code": "uk", "name": "Ukrainian", "native": "Українська", "flag": "ua"},
{"code": "he", "name": "Hebrew", "native": "עברית", "flag": "il"},
{"code": "ar", "name": "Arabic", "native": "العربية", "flag": "sa"},
{"code": "fa", "name": "Persian", "native": "فارسی", "flag": "ir"},
{"code": "af", "name": "Afrikaans", "native": "Afrikaans", "flag": "za"},
# --- Tier 5: Asian languages ---
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "🇨🇳"},
{"code": "zh-TW", "name": "Traditional Chinese", "native": "繁體中文", "flag": "🇹🇼"},
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "🇯🇵"},
{"code": "ko", "name": "Korean", "native": "한국어", "flag": "🇰🇷"},
{"code": "vi", "name": "Vietnamese", "native": "Tiếng Việt", "flag": "🇻🇳"},
{"code": "pa", "name": "Punjabi", "native": "ਪੰਜਾਬੀ", "flag": "🇮🇳"},
{"code": "kn", "name": "Kannada", "native": "ಕನ್ನಡ", "flag": "🇮🇳"},
{"code": "hi", "name": "Hindi", "native": "हिन्दी", "flag": "🇮🇳"},
{"code": "bn", "name": "Bengali", "native": "বাংলা", "flag": "🇧🇩"},
{"code": "gu", "name": "Gujarati", "native": "ગુજરાતી", "flag": "🇮🇳"},
{"code": "ml", "name": "Malayalam", "native": "മലയാളം", "flag": "🇮🇳"},
{"code": "mr", "name": "Marathi", "native": "मराठी", "flag": "🇮🇳"},
{"code": "ta", "name": "Tamil", "native": "தமிழ்", "flag": "🇮🇳"},
{"code": "te", "name": "Telugu", "native": "తెలుగు", "flag": "🇮🇳"},
{"code": "ur", "name": "Urdu", "native": "اردو", "flag": "🇵🇰"},
{"code": "si", "name": "Sinhala", "native": "සිංහල", "flag": "🇱🇰"},
{"code": "ne", "name": "Nepali", "native": "नेपाली", "flag": "🇳🇵"},
{"code": "th", "name": "Thai", "native": "ไทย", "flag": "🇹🇭"},
{"code": "km", "name": "Khmer", "native": "ខ្មែរ", "flag": "🇰🇭"},
{"code": "id", "name": "Indonesian", "native": "Bahasa Indonesia", "flag": "🇮🇩"},
{"code": "ms", "name": "Malay", "native": "Bahasa Melayu", "flag": "🇲🇾"},
{"code": "jv", "name": "Javanese", "native": "Basa Jawa", "flag": "🇮🇩"},
{"code": "tl", "name": "Tagalog", "native": "Filipino", "flag": "🇵🇭"},
{"code": "mn", "name": "Mongolian", "native": "Монгол", "flag": "🇲🇳"},
{"code": "kk", "name": "Kazakh", "native": "Қазақ тілі", "flag": "🇰🇿"},
{"code": "uz", "name": "Uzbek", "native": "Oʻzbekcha", "flag": "🇺🇿"},
{"code": "az", "name": "Azerbaijani", "native": "Azərbaycan dili", "flag": "🇦🇿"},
{"code": "hy", "name": "Armenian", "native": "Հայերեն", "flag": "🇦🇲"},
{"code": "ka", "name": "Georgian", "native": "ქართული", "flag": "🇬🇪"},
{"code": "zh", "name": "Chinese", "native": "中文", "flag": "cn"},
{"code": "zh-TW", "name": "Traditional Chinese", "native": "繁體中文", "flag": "tw"},
{"code": "ja", "name": "Japanese", "native": "日本語", "flag": "jp"},
{"code": "ko", "name": "Korean", "native": "한국어", "flag": "kr"},
{"code": "vi", "name": "Vietnamese", "native": "Tiếng Việt", "flag": "vn"},
{"code": "pa", "name": "Punjabi", "native": "ਪੰਜਾਬੀ", "flag": "in"},
{"code": "kn", "name": "Kannada", "native": "ಕನ್ನಡ", "flag": "in"},
{"code": "hi", "name": "Hindi", "native": "हिन्दी", "flag": "in"},
{"code": "bn", "name": "Bengali", "native": "বাংলা", "flag": "bd"},
{"code": "gu", "name": "Gujarati", "native": "ગુજરાતી", "flag": "in"},
{"code": "ml", "name": "Malayalam", "native": "മലയാളം", "flag": "in"},
{"code": "mr", "name": "Marathi", "native": "मराठी", "flag": "in"},
{"code": "ta", "name": "Tamil", "native": "தமிழ்", "flag": "in"},
{"code": "te", "name": "Telugu", "native": "తెలుగు", "flag": "in"},
{"code": "ur", "name": "Urdu", "native": "اردو", "flag": "pk"},
{"code": "si", "name": "Sinhala", "native": "සිංහල", "flag": "lk"},
{"code": "ne", "name": "Nepali", "native": "नेपाली", "flag": "np"},
{"code": "th", "name": "Thai", "native": "ไทย", "flag": "th"},
{"code": "km", "name": "Khmer", "native": "ខ្មែរ", "flag": "kh"},
{"code": "id", "name": "Indonesian", "native": "Bahasa Indonesia", "flag": "id"},
{"code": "ms", "name": "Malay", "native": "Bahasa Melayu", "flag": "my"},
{"code": "jv", "name": "Javanese", "native": "Basa Jawa", "flag": "id"},
{"code": "tl", "name": "Tagalog", "native": "Filipino", "flag": "ph"},
{"code": "mn", "name": "Mongolian", "native": "Монгол", "flag": "mn"},
{"code": "kk", "name": "Kazakh", "native": "Қазақ тілі", "flag": "kz"},
{"code": "uz", "name": "Uzbek", "native": "Oʻzbekcha", "flag": "uz"},
{"code": "az", "name": "Azerbaijani", "native": "Azərbaycan dili", "flag": "az"},
{"code": "hy", "name": "Armenian", "native": "Հայերեն", "flag": "am"},
{"code": "ka", "name": "Georgian", "native": "ქართული", "flag": "ge"},
# --- Tier 6: African languages ---
{"code": "sw", "name": "Swahili", "native": "Kiswahili", "flag": "🇰🇪"},
{"code": "am", "name": "Amharic", "native": "አማርኛ", "flag": "🇪🇹"},
{"code": "ha", "name": "Hausa", "native": "Hausa", "flag": "🇳🇬"},
{"code": "yo", "name": "Yoruba", "native": "Yorùbá", "flag": "🇳🇬"},
{"code": "ig", "name": "Igbo", "native": "Igbo", "flag": "🇳🇬"},
{"code": "zu", "name": "Zulu", "native": "isiZulu", "flag": "🇿🇦"},
{"code": "sw", "name": "Swahili", "native": "Kiswahili", "flag": "ke"},
{"code": "am", "name": "Amharic", "native": "አማርኛ", "flag": "et"},
{"code": "ha", "name": "Hausa", "native": "Hausa", "flag": "ng"},
{"code": "yo", "name": "Yoruba", "native": "Yorùbá", "flag": "ng"},
{"code": "ig", "name": "Igbo", "native": "Igbo", "flag": "ng"},
{"code": "zu", "name": "Zulu", "native": "isiZulu", "flag": "za"},
# --- Tier 7: Constructed & other languages ---
{"code": "eo", "name": "Esperanto", "native": "Esperanto", "flag": "🌍"},
{"code": "eo", "name": "Esperanto", "native": "Esperanto", "flag": "un"}, # UN flag for international language
]
SUPPORTED_LANGUAGE_CODES: set[str] = {lang["code"] for lang in SUPPORTED_LANGUAGES}
app/utils/session_manager.py
+505
View File
@@ -0,0 +1,505 @@
"""Server-side session management utilities.
Provides helpers for creating, validating, and revoking user sessions.
Sessions are tracked in the ``user_sessions`` table and referenced by a
cryptographically random token stored in the browser cookie. This enables
the "log off everywhere" feature and per-session revocation.
"""
from __future__ import annotations
import logging
import secrets
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import Session
from app.config import settings
from app.models import ApiToken, QRLoginChallenge, UserSession
logger = logging.getLogger(__name__)
def _ensure_tz_aware(dt: datetime | None) -> datetime | None:
"""Return *dt* with UTC tzinfo if it is naive, or unchanged if already aware.
SQLite does not persist timezone information, so datetimes read back from
the database are offset-naive. This helper normalises them for safe
comparison with ``datetime.now(timezone.utc)``.
"""
if dt is not None and dt.tzinfo is None:
return dt.replace(tzinfo=timezone.utc)
return dt
def get_session_lifetime_days() -> int:
"""Return the effective session lifetime in days.
If ``session_lifetime_custom_days`` is set it takes precedence over
``session_lifetime_days``.
"""
custom = getattr(settings, "session_lifetime_custom_days", None)
if custom is not None and isinstance(custom, int) and custom > 0:
return custom
return max(1, getattr(settings, "session_lifetime_days", 30))
def get_session_max_age_seconds() -> int:
"""Return the session max-age in seconds for the cookie."""
return get_session_lifetime_days() * 86400
def create_session(
db: Session,
user_id: str,
ip_address: str | None = None,
user_agent: str | None = None,
) -> UserSession:
"""Create a new server-side session record.
Args:
db: Database session.
user_id: Stable owner identifier.
ip_address: Client IP address.
user_agent: Client User-Agent header.
Returns:
The newly created ``UserSession`` instance.
"""
session_token = secrets.token_urlsafe(64)
now = datetime.now(timezone.utc)
lifetime_days = get_session_lifetime_days()
expires_at = now + timedelta(days=lifetime_days)
device_info = _parse_device_info(user_agent)
user_session = UserSession(
session_token=session_token,
user_id=user_id,
ip_address=ip_address,
user_agent=(user_agent or "")[:512],
device_info=device_info,
created_at=now,
last_active_at=now,
expires_at=expires_at,
)
try:
db.add(user_session)
db.commit()
db.refresh(user_session)
except Exception:
db.rollback()
logger.exception("Failed to create session for user_id=%s", user_id)
raise
logger.info(
"[SESSION] Created session id=%s user=%s device=%r expires=%s",
user_session.id,
user_id,
device_info,
expires_at.isoformat(),
)
return user_session
def validate_session(db: Session, session_token: str) -> UserSession | None:
"""Validate a session token and return the session if valid.
A session is valid when:
* It exists in the database.
* ``is_revoked`` is ``False``.
* ``expires_at`` is in the future.
Side-effect: updates ``last_active_at`` on valid sessions.
Returns:
The ``UserSession`` if valid, else ``None``.
"""
if not session_token:
return None
now = datetime.now(timezone.utc)
user_session = db.query(UserSession).filter(UserSession.session_token == session_token).first()
if not user_session:
logger.debug("[SESSION] Token not found in database")
return None
if user_session.is_revoked:
logger.debug("[SESSION] Session id=%s is revoked", user_session.id)
return None
if user_session.expires_at:
expires = _ensure_tz_aware(user_session.expires_at)
if expires < now:
logger.debug("[SESSION] Session id=%s has expired", user_session.id)
return None
# Update last_active_at (throttled to avoid excessive writes)
last_active = _ensure_tz_aware(user_session.last_active_at)
if not last_active or (now - last_active).total_seconds() > 60:
try:
user_session.last_active_at = now
db.commit()
except Exception:
db.rollback()
logger.debug("[SESSION] Failed to update last_active_at for session id=%s", user_session.id)
return user_session
def revoke_session(db: Session, session_id: int, user_id: str) -> bool:
"""Revoke a single session by ID.
Args:
db: Database session.
session_id: The session record ID to revoke.
user_id: The owner — ensures a user can only revoke their own sessions.
Returns:
``True`` if the session was found and revoked, ``False`` otherwise.
"""
user_session = db.get(UserSession, session_id)
if not user_session or user_session.user_id != user_id:
return False
now = datetime.now(timezone.utc)
user_session.is_revoked = True
user_session.revoked_at = now
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info("[SESSION] Revoked session id=%s user=%s", session_id, user_id)
return True
def revoke_all_sessions(
db: Session,
user_id: str,
*,
except_session_id: int | None = None,
revoke_api_tokens: bool = True,
) -> int:
"""Revoke all active sessions for a user ("log off everywhere").
Args:
db: Database session.
user_id: The owner whose sessions should be revoked.
except_session_id: If provided, keep this session active (the
current browser session).
revoke_api_tokens: If ``True``, also revoke all active API tokens.
Returns:
Number of sessions revoked.
"""
now = datetime.now(timezone.utc)
query = db.query(UserSession).filter(
UserSession.user_id == user_id,
UserSession.is_revoked.is_(False),
)
if except_session_id is not None:
query = query.filter(UserSession.id != except_session_id)
sessions = query.all()
count = 0
for s in sessions:
s.is_revoked = True
s.revoked_at = now
count += 1
if revoke_api_tokens:
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == user_id,
ApiToken.is_active.is_(True),
)
.all()
)
for t in tokens:
t.is_active = False
t.revoked_at = now
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info(
"[SESSION] Revoked all sessions for user=%s (count=%d, except_session_id=%s, tokens_revoked=%s)",
user_id,
count,
except_session_id,
revoke_api_tokens,
)
return count
def list_user_sessions(db: Session, user_id: str) -> list[UserSession]:
"""Return all non-revoked, non-expired sessions for a user.
Results are ordered by most recently active first.
"""
now = datetime.now(timezone.utc)
sessions = (
db.query(UserSession)
.filter(
UserSession.user_id == user_id,
UserSession.is_revoked.is_(False),
)
.order_by(UserSession.last_active_at.desc())
.all()
)
# Filter expired sessions in Python to handle timezone-naive datetimes (SQLite)
result = []
for s in sessions:
expires = _ensure_tz_aware(s.expires_at)
if expires and expires > now:
result.append(s)
return result
def cleanup_expired_sessions(db: Session) -> int:
"""Delete sessions that expired more than 7 days ago.
Intended to be called periodically (e.g. via Celery beat) to keep the
table from growing unbounded.
Returns:
Number of rows deleted.
"""
cutoff = datetime.now(timezone.utc) - timedelta(days=7)
count = db.query(UserSession).filter(UserSession.expires_at < cutoff).delete(synchronize_session=False)
try:
db.commit()
except Exception:
db.rollback()
raise
if count:
logger.info("[SESSION] Cleaned up %d expired sessions", count)
return count
# ---------------------------------------------------------------------------
# QR login helpers
# ---------------------------------------------------------------------------
def create_qr_challenge(db: Session, user_id: str, ip_address: str | None = None) -> QRLoginChallenge:
"""Create a new QR login challenge.
Args:
db: Database session.
user_id: The authenticated web user creating the challenge.
ip_address: IP address of the web client.
Returns:
The newly created ``QRLoginChallenge``.
"""
token = secrets.token_urlsafe(64)
ttl = getattr(settings, "qr_login_challenge_ttl_seconds", 120)
now = datetime.now(timezone.utc)
expires_at = now + timedelta(seconds=ttl)
challenge = QRLoginChallenge(
challenge_token=token,
user_id=user_id,
created_by_ip=ip_address,
created_at=now,
expires_at=expires_at,
)
try:
db.add(challenge)
db.commit()
db.refresh(challenge)
except Exception:
db.rollback()
logger.exception("Failed to create QR login challenge for user_id=%s", user_id)
raise
logger.info("[QR_AUTH] Challenge created: id=%s user=%s expires=%s", challenge.id, user_id, expires_at.isoformat())
return challenge
def validate_qr_challenge(db: Session, challenge_token: str) -> QRLoginChallenge | None:
"""Validate a QR challenge token without claiming it.
Returns the challenge if it exists, is not expired, not claimed,
and not cancelled. Returns ``None`` otherwise.
"""
if not challenge_token:
return None
now = datetime.now(timezone.utc)
challenge = db.query(QRLoginChallenge).filter(QRLoginChallenge.challenge_token == challenge_token).first()
if not challenge:
return None
if challenge.is_claimed or challenge.is_cancelled:
return None
expires = _ensure_tz_aware(challenge.expires_at)
if expires and expires < now:
return None
return challenge
def claim_qr_challenge(
db: Session,
challenge_token: str,
device_name: str = "Mobile App",
ip_address: str | None = None,
) -> dict | None:
"""Claim a QR challenge and issue an API token.
This is the critical security path. The challenge is validated,
marked as claimed atomically, and an API token is issued for the
user who created the challenge.
Args:
db: Database session.
challenge_token: The token from the QR code.
device_name: Name provided by the mobile app.
ip_address: IP address of the claiming mobile device.
Returns:
Dict with ``token`` (plaintext), ``token_id``, ``name``, ``owner_id``
and ``created_at`` on success, or ``None`` if the challenge is invalid.
"""
from app.api.api_tokens import generate_api_token, hash_token
challenge = validate_qr_challenge(db, challenge_token)
if not challenge:
logger.warning("[QR_AUTH] Invalid or expired challenge token attempted")
return None
now = datetime.now(timezone.utc)
# Mark as claimed first to prevent race conditions
challenge.is_claimed = True
challenge.claimed_at = now
challenge.claimed_by_ip = ip_address
challenge.device_name = device_name
# Generate API token for the mobile app
token_name = f"Mobile App (QR) – {device_name}"
plaintext = generate_api_token()
token_hash_value = hash_token(plaintext)
prefix = plaintext[:12]
db_token = ApiToken(
owner_id=challenge.user_id,
name=token_name,
token_hash=token_hash_value,
token_prefix=prefix,
)
try:
db.add(db_token)
db.flush()
challenge.issued_token_id = db_token.id
db.commit()
db.refresh(db_token)
except Exception:
db.rollback()
logger.exception("[QR_AUTH] Failed to issue token for challenge id=%s", challenge.id)
raise
logger.info(
"[QR_AUTH] Challenge claimed: id=%s user=%s device=%r token_id=%s",
challenge.id,
challenge.user_id,
device_name,
db_token.id,
)
return {
"token": plaintext,
"token_id": db_token.id,
"name": token_name,
"owner_id": challenge.user_id,
"created_at": db_token.created_at,
}
def get_challenge_status(db: Session, challenge_id: int, user_id: str) -> dict | None:
"""Get the current status of a QR challenge (for polling from the web UI).
Returns:
Dict with ``status`` ("pending", "claimed", "expired", "cancelled")
and metadata, or ``None`` if the challenge doesn't belong to the user.
"""
challenge = db.get(QRLoginChallenge, challenge_id)
if not challenge or challenge.user_id != user_id:
return None
now = datetime.now(timezone.utc)
expires = _ensure_tz_aware(challenge.expires_at)
if challenge.is_claimed:
status = "claimed"
elif challenge.is_cancelled:
status = "cancelled"
elif expires and expires < now:
status = "expired"
else:
status = "pending"
return {
"id": challenge.id,
"status": status,
"device_name": challenge.device_name,
"claimed_at": challenge.claimed_at,
"expires_at": challenge.expires_at,
}
def _parse_device_info(user_agent: str | None) -> str | None:
"""Extract a human-readable device description from User-Agent.
This is a lightweight parser — not a full UA library — that covers
the most common browsers and platforms.
"""
if not user_agent:
return None
ua = user_agent.lower()
# Platform detection
platform = "Unknown"
if "iphone" in ua:
platform = "iPhone"
elif "ipad" in ua:
platform = "iPad"
elif "android" in ua:
platform = "Android"
elif "macintosh" in ua or "mac os" in ua:
platform = "macOS"
elif "windows" in ua:
platform = "Windows"
elif "linux" in ua:
platform = "Linux"
elif "cros" in ua:
platform = "ChromeOS"
# Browser detection
browser = "Unknown Browser"
if "edg/" in ua or "edge/" in ua:
browser = "Edge"
elif "opr/" in ua or "opera" in ua:
browser = "Opera"
elif "chrome/" in ua and "safari/" in ua:
browser = "Chrome"
elif "safari/" in ua and "chrome/" not in ua:
browser = "Safari"
elif "firefox/" in ua:
browser = "Firefox"
elif "docuelevate" in ua:
browser = "DocuElevate App"
return f"{browser} on {platform}"
app/utils/settings_service.py
+254 -3
View File
@@ -135,6 +135,30 @@ SETTING_METADATA = {
"required": True, # Required when auth_enabled=True (validated in config.py)
"restart_required": True,
},
"session_lifetime_days": {
"category": "Authentication",
"description": "Session lifetime in days (default 30). Determines how long a user stays logged in.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"session_lifetime_custom_days": {
"category": "Authentication",
"description": "Override session_lifetime_days with a custom value. Takes precedence when set.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"qr_login_challenge_ttl_seconds": {
"category": "Authentication",
"description": "Time-to-live in seconds for QR login challenges (default 120).",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"admin_username": {
"category": "Authentication",
"description": "Admin username for local authentication",
@@ -523,6 +547,19 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Document Translation
"default_document_language": {
"category": "AI Services",
"description": (
"ISO 639-1 language code for the default document translation target "
"(e.g. 'en', 'de', 'fr'). Documents whose detected language differs "
"are automatically translated into this language after processing."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# OCR Engine Configuration
"ocr_providers": {
"category": "OCR Engines",
@@ -863,6 +900,63 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - SharePoint
"sharepoint_client_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD application (client) ID",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_client_secret": {
"category": "Storage Providers",
"description": "SharePoint Azure AD client secret",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_tenant_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_refresh_token": {
"category": "Storage Providers",
"description": "SharePoint OAuth refresh token",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_site_url": {
"category": "Storage Providers",
"description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_document_library": {
"category": "Storage Providers",
"description": "SharePoint document library name (default: 'Documents')",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_folder_path": {
"category": "Storage Providers",
"description": "Subfolder path inside the SharePoint document library",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_enabled": {
"category": "Storage Providers",
@@ -1888,6 +1982,28 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"factory_reset_on_startup": {
"category": "Feature Flags",
"description": (
"Wipe all user data on every startup so the instance always starts fresh. "
"Useful for demo/testing environments. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"enable_factory_reset": {
"category": "Feature Flags",
"description": (
"Show the System Reset page in the admin UI. Allows administrators to "
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Backup / Restore
"backup_enabled": {
"category": "Backup",
@@ -1911,14 +2027,26 @@ SETTING_METADATA = {
"category": "Backup",
"description": (
"Storage provider for remote backup copies. "
"Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. "
"Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. "
"Leave empty to keep backups local only."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"],
"options": [
"",
"s3",
"dropbox",
"google_drive",
"onedrive",
"sharepoint",
"nextcloud",
"webdav",
"ftp",
"sftp",
"email",
],
},
"backup_remote_folder": {
"category": "Backup",
@@ -2430,6 +2558,72 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Database Connection Pool
"db_pool_size": {
"category": "Core",
"description": (
"Number of persistent connections kept in the SQLAlchemy QueuePool. "
"Has no effect for SQLite databases. Default: 5."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_max_overflow": {
"category": "Core",
"description": (
"Maximum extra connections that can be opened beyond db_pool_size. "
"Has no effect for SQLite databases. Default: 10."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_timeout": {
"category": "Core",
"description": (
"Seconds to wait for a connection from the pool before raising an error. "
"Has no effect for SQLite databases. Default: 30."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"db_pool_recycle": {
"category": "Core",
"description": (
"Seconds after which idle connections are recycled to prevent stale connections. "
"Has no effect for SQLite databases. Default: 1800 (30 minutes)."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
# Per-user upload rate limiting
"upload_rate_limit_per_user": {
"category": "Security",
"description": (
"Maximum number of uploads a single user may submit within upload_rate_limit_window seconds. "
"The health-aware limiter may reduce this dynamically under high Redis queue depth or CPU load. "
"Default: 20."
),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
"upload_rate_limit_window": {
"category": "Security",
"description": ("Sliding window in seconds over which upload_rate_limit_per_user is enforced. Default: 60."),
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Rate Limiting
"rate_limiting_enabled": {
"category": "Security",
@@ -2629,6 +2823,63 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Logging
"log_level": {
"category": "Observability",
"description": (
"Python logging level for the application root logger. "
"Accepts: DEBUG, INFO, WARNING, ERROR, CRITICAL. "
"When DEBUG=True and LOG_LEVEL is not explicitly set, "
"the effective level is automatically lowered to DEBUG."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_format": {
"category": "Observability",
"description": (
"Log output format: 'text' (human-readable, default) or "
"'json' (structured JSON lines for SIEM / log aggregation)."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_enabled": {
"category": "Observability",
"description": "Forward application logs to a syslog receiver in addition to stdout.",
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_host": {
"category": "Observability",
"description": "Hostname or IP of the syslog receiver for application logs.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_port": {
"category": "Observability",
"description": "Port of the syslog receiver for application logs.",
"type": "integer",
"sensitive": False,
"required": False,
"restart_required": True,
},
"log_syslog_protocol": {
"category": "Observability",
"description": "Protocol for syslog transport: 'udp' or 'tcp'.",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": True,
},
# Observability – Sentry
"sentry_dsn": {
"category": "Observability",
@@ -3123,7 +3374,7 @@ def get_settings_for_export(db: Session, source: str = "db") -> Dict[str, str]:
return {k.upper(): v for k, v in sorted(db_settings.items()) if v is not None}
def update_env_file(env_path: str, settings_to_update: Dict[str, str]) -> bool:
def update_env_file(env_path: str, settings_to_update: dict[str, str]) -> bool:
"""
Update an .env file with new settings.
app/utils/system_reset.py
+296
View File
@@ -0,0 +1,296 @@
"""
System reset utilities for DocuElevate.
Provides functions to:
- Wipe all user data (database rows + work-files on disk) for a fresh start.
- Wipe with re-import: move original files to a dedicated folder, wipe
everything, then let the watch-folder mechanism re-ingest the files.
Security: All public functions in this module require admin-level access.
They MUST only be invoked from admin-guarded API/view endpoints.
"""
import logging
import shutil
from pathlib import Path
from sqlalchemy.orm import Session
from app.config import settings
logger = logging.getLogger(__name__)
# Subdirectories inside *workdir* that contain user-generated data.
# Everything else (app code, static assets, config) is left untouched.
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
# JSON cache files written by watch-folder / ingest tasks.
_CACHE_FILES = (
"watch_folder_processed.json",
"ftp_ingest_processed.json",
"sftp_ingest_processed.json",
"dropbox_ingest_processed.json",
"gdrive_ingest_processed.json",
"onedrive_ingest_processed.json",
"nextcloud_ingest_processed.json",
"s3_ingest_processed.json",
"webdav_ingest_processed.json",
"processed_mails.json",
"credential_failures.json",
)
# The folder name used for storing files prior to re-import.
REIMPORT_FOLDER_NAME = "reimport"
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
"""Delete user data subdirectories and cache files inside *workdir*.
Leaves the workdir directory itself intact so the application can
continue to write into it. Also leaves any files that do not belong
to the known data subdirectories or caches.
Returns:
A dict with counts of deleted directories and files.
"""
workdir_path = Path(workdir)
deleted_dirs = 0
deleted_files = 0
# Remove data subdirectories
for subdir in _USER_DATA_SUBDIRS:
target = workdir_path / subdir
if target.is_dir():
shutil.rmtree(target)
logger.info("Deleted data directory: %s", target)
deleted_dirs += 1
# Remove cache / state JSON files
for cache_file in _CACHE_FILES:
target = workdir_path / cache_file
if target.is_file():
target.unlink()
logger.info("Deleted cache file: %s", target)
deleted_files += 1
# Also remove user_wf_*.json files (per-user watch folder caches)
for f in workdir_path.glob("user_wf_*.json"):
f.unlink()
logger.info("Deleted user watch-folder cache: %s", f)
deleted_files += 1
# Remove loose files in workdir root that are user uploads (uuid-named
# files like "a1b2c3d4-…pdf") but NOT application config files.
for entry in workdir_path.iterdir():
if entry.is_file() and entry.suffix.lower() in {
".pdf",
".png",
".jpg",
".jpeg",
".tiff",
".tif",
".docx",
".doc",
".xlsx",
".xls",
".pptx",
".heic",
".heif",
".webp",
".bmp",
".gif",
".txt",
".rtf",
".odt",
".ods",
".odp",
".csv",
".pages",
".numbers",
".keynote",
}:
entry.unlink()
logger.info("Deleted loose workdir file: %s", entry)
deleted_files += 1
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
def _wipe_database(db: Session) -> dict[str, int]:
"""Delete all user-generated rows from the database.
Preserves schema (tables, migrations) and system-seeded rows that will
be re-created on the next startup (subscription plans, default pipeline,
scheduled jobs, compliance templates).
Returns:
A dict mapping table name → number of rows deleted.
"""
from app.models import (
AuditLog,
BackupRecord,
DocumentMetadata,
FileProcessingStep,
FileRecord,
InAppNotification,
ProcessingLog,
SavedSearch,
SettingsAuditLog,
SharedLink,
UserImapAccount,
UserIntegration,
UserNotificationPreference,
UserNotificationTarget,
)
# Order matters: delete children before parents to respect FK constraints.
tables_to_wipe: list[tuple[str, type]] = [
("file_processing_steps", FileProcessingStep),
("processing_logs", ProcessingLog),
("shared_links", SharedLink),
("in_app_notifications", InAppNotification),
("user_notification_preferences", UserNotificationPreference),
("user_notification_targets", UserNotificationTarget),
("user_imap_accounts", UserImapAccount),
("user_integrations", UserIntegration),
("saved_searches", SavedSearch),
("settings_audit_log", SettingsAuditLog),
("audit_logs", AuditLog),
("backup_records", BackupRecord),
("document_metadata", DocumentMetadata),
("files", FileRecord),
]
result: dict[str, int] = {}
for table_name, model in tables_to_wipe:
try:
count = db.query(model).delete()
result[table_name] = count
logger.info("Wiped %d rows from %s", count, table_name)
except Exception:
logger.exception("Failed to wipe table %s during system reset", table_name)
db.rollback()
raise
db.commit()
return result
def perform_full_reset(db: Session) -> dict:
"""Perform a complete system reset: wipe database rows + work-files.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database`` and ``filesystem`` sub-dicts.
"""
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
db_result = _wipe_database(db)
fs_result = _wipe_workdir_data(settings.workdir)
logger.warning(">>> SYSTEM RESET complete <<<")
return {"database": db_result, "filesystem": fs_result}
def perform_reset_and_reimport(db: Session) -> dict:
"""Move original files to a reimport folder, wipe everything, then
configure the reimport folder as a watch folder for re-ingestion.
The watch-folder scanner (``scan_all_watch_folders``) will pick up
the files on its next periodic run and process them exactly as if
they had been freshly uploaded — respecting the same backoff
strategy, size limits, and rate limits.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
"""
workdir_path = Path(settings.workdir)
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
original_dir = workdir_path / "original"
# 1. Collect original files
files_moved = 0
reimport_dir.mkdir(parents=True, exist_ok=True)
if original_dir.is_dir():
for entry in original_dir.iterdir():
if entry.is_file():
# Validate the resolved path stays within original_dir (path traversal guard)
try:
entry.resolve().relative_to(original_dir.resolve())
except ValueError:
logger.warning("Skipping file outside original dir: %s", entry)
continue
dest = reimport_dir / entry.name
# Avoid overwriting: append counter if name clash
if dest.exists():
stem = dest.stem
suffix = dest.suffix
counter = 1
while dest.exists():
dest = reimport_dir / f"{stem}_{counter}{suffix}"
counter += 1
shutil.copy2(str(entry), str(dest))
files_moved += 1
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
# 2. Perform the full reset (wipe DB + other workdir data)
reset_result = perform_full_reset(db)
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
# and set up watch folder config to point at it.
_configure_reimport_watch_folder(str(reimport_dir))
reset_result["reimport"] = {
"files_moved": files_moved,
"reimport_folder": str(reimport_dir),
}
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
return reset_result
def _configure_reimport_watch_folder(reimport_path: str) -> None:
"""Append *reimport_path* to the application's watch-folder list.
The watch-folder scanner uses ``settings.watch_folders`` (a
comma-separated string). We mutate the runtime setting so the
next scan picks up the folder. We also set
``watch_folder_delete_after_process = True`` so files are cleaned
up after successful processing.
"""
current = getattr(settings, "watch_folders", None) or ""
folders = [f.strip() for f in current.split(",") if f.strip()]
if reimport_path not in folders:
folders.append(reimport_path)
# Mutate runtime settings (not persisted to .env — ephemeral)
object.__setattr__(settings, "watch_folders", ",".join(folders))
object.__setattr__(settings, "watch_folder_delete_after_process", True)
logger.info("Configured reimport watch folder: %s", reimport_path)
def perform_startup_reset() -> None:
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
Wipes database and filesystem data so the instance starts completely
fresh. Uses its own DB session so it runs before the normal lifespan
seeding logic.
"""
from app.database import SessionLocal
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
db = SessionLocal()
try:
perform_full_reset(db)
except Exception:
logger.exception("Factory reset on startup failed")
db.rollback()
finally:
db.close()
app/utils/user_scope.py
+56 -8
View File
@@ -20,13 +20,31 @@ from app.models import FileRecord
logger = logging.getLogger(__name__)
def _owner_id_from_user(user: dict) -> str | None:
"""Extract the owner identifier from a user dict.
Priority: ``sub`` (OAuth subject) → ``preferred_username`` → ``email`` → ``id``.
"""
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
def get_current_owner_id(request: Request) -> str | None:
"""Extract the owner identifier for the current authenticated user.
The owner ID is derived from the user's session data. It uses the
``sub`` claim (OAuth subject) when available, falling back to
``preferred_username`` or ``email``. Returns ``None`` when no user
is authenticated.
The owner ID is derived from the user's session data or, when no session
is present, from a valid Bearer API token in the ``Authorization`` header.
This ensures that both browser-based (session cookie) and mobile/API
(Bearer token) requests are correctly identified.
Priority for user resolution:
1. Session ``user`` dict (set by OAuth or local login).
2. ``request.state.api_token_user`` (set by ``require_login`` or an
earlier call to this function during the same request).
3. Direct Bearer token look-up against the database.
Within the resolved user dict the owner ID is chosen as:
``sub`` → ``preferred_username`` → ``email`` → ``id``.
Args:
request: The current FastAPI request with session data.
@@ -34,11 +52,41 @@ def get_current_owner_id(request: Request) -> str | None:
Returns:
A stable string identifier for the user, or ``None``.
"""
# 1. Session-based auth (most common for web UI)
user = request.session.get("user")
if not user or not isinstance(user, dict):
return None
# Prefer 'sub' (OAuth subject), then 'preferred_username', then 'email', then 'id'
return user.get("sub") or user.get("preferred_username") or user.get("email") or user.get("id")
if user and isinstance(user, dict):
return _owner_id_from_user(user)
# 2. Already-resolved API token user (cached by require_login or a
# prior dependency call during this request)
api_user = getattr(request.state, "api_token_user", None)
if isinstance(api_user, dict):
return _owner_id_from_user(api_user)
# 3. Direct Bearer token resolution – necessary when this function is
# invoked as a FastAPI dependency (via Depends) which runs *before*
# the @require_login decorator wrapper has had a chance to resolve
# the token and populate request.state.api_token_user.
auth_header = request.headers.get("authorization", "")
if isinstance(auth_header, str) and auth_header.startswith("Bearer "):
try:
from app.auth import _resolve_bearer_user
from app.database import SessionLocal
db = SessionLocal()
try:
resolved = _resolve_bearer_user(request, db)
finally:
db.close()
if resolved:
# Cache so subsequent calls (and require_login) skip the DB
request.state.api_token_user = resolved
return _owner_id_from_user(resolved)
except Exception:
logger.debug("Bearer token resolution failed in get_current_owner_id", exc_info=True)
return None
def apply_owner_filter(query: Query, request: Request) -> Query:
app/views/__init__.py
+6
View File
@@ -10,6 +10,7 @@ from app.views.audit_logs import router as audit_logs_router
from app.views.backup import router as backup_router
from app.views.compliance import router as compliance_router
from app.views.db_wizard import router as db_wizard_router
from app.views.devices import router as devices_router # Mobile devices dashboard
from app.views.dropbox import router as dropbox_router
from app.views.filemanager import router as filemanager_router
@@ -26,6 +27,7 @@ from app.views.onedrive import router as onedrive_router
from app.views.pipelines import router as pipelines_router # Processing pipelines
from app.views.plans import router as plans_router # Admin Plan Designer
from app.views.profile import router as profile_router # User self-service profile
from app.views.qr_login import router as qr_login_router # QR code mobile login
from app.views.queue import router as queue_router
from app.views.scheduled_jobs import router as scheduled_jobs_router # Scheduled batch jobs
from app.views.search import router as search_router
@@ -34,6 +36,7 @@ from app.views.share import router as share_router
from app.views.shared_links import router as shared_links_router
from app.views.status import router as status_router
from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages
from app.views.system_reset import router as system_reset_router # System reset / factory reset
from app.views.wizard import router as wizard_router
# Create a main router that includes all the view routers
@@ -60,6 +63,7 @@ router.include_router(plans_router) # Admin Plan Designer
router.include_router(onboarding_router) # User onboarding wizard
router.include_router(pipelines_router) # Processing pipelines
router.include_router(profile_router) # User self-service profile settings
router.include_router(qr_login_router) # QR code mobile login page
router.include_router(imap_accounts_router) # Per-user IMAP ingestion accounts
router.include_router(integrations_router) # Unified integrations dashboard
router.include_router(notifications_router) # User notification dashboard
@@ -67,3 +71,5 @@ router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs
router.include_router(audit_logs_router) # Comprehensive audit log viewer
router.include_router(help_router) # Built-in help / How-To docs
router.include_router(compliance_router) # Compliance templates dashboard
router.include_router(devices_router) # Mobile devices dashboard
router.include_router(system_reset_router) # System reset / factory reset
app/views/base.py
+1
View File
@@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None:
"allow_signup",
getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False),
)
ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False))
req = ctx.get("request")
if req is not None:
app/views/devices.py
+25
View File
@@ -0,0 +1,25 @@
"""View route for the Devices management page.
Renders the ``devices.html`` template where users can see their registered
mobile devices, mobile API tokens (created via the mobile SSO flow or QR
code login), and revoke access per-device.
"""
import logging
from fastapi import APIRouter, Request
from app.views.base import require_login, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/devices", include_in_schema=False)
@require_login
async def devices_page(request: Request):
"""Render the Devices management page."""
return templates.TemplateResponse(
"devices.html",
{"request": request, "page_title": "Devices"},
)
app/views/files.py
+28
View File
@@ -832,6 +832,34 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
)
@router.get("/files/{file_id}/text/default-language")
@require_login
def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)):
"""Return the persisted default-language translation for the file view."""
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND)
if not file_record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available",
)
return JSONResponse(
content={
"text": file_record.default_language_text,
"language_code": file_record.default_language_code,
"detected_language": file_record.detected_language,
}
)
@router.get("/duplicates")
@require_login
def duplicates_page(
app/views/qr_login.py
+26
View File
@@ -0,0 +1,26 @@
"""View route for the QR code mobile login page.
Route:
GET /qr-login — renders the QR login page (requires login)
"""
from __future__ import annotations
import logging
from fastapi import Request
from app.views.base import APIRouter, require_login, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/qr-login", include_in_schema=False)
@require_login
async def qr_login_page(request: Request):
"""Serve the QR code login page for mobile app authentication."""
return templates.TemplateResponse(
"qr_login.html",
{"request": request},
)
app/views/system_reset.py
+40
View File
@@ -0,0 +1,40 @@
"""
System reset view — admin-only UI page.
Renders a confirmation-heavy page that allows administrators to:
1. **Full Reset** — wipe all user data (DB + disk) for a fresh start.
2. **Reset & Re-import** — move originals to a reimport folder, wipe,
and let the watch-folder mechanism re-ingest them.
Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag.
"""
import logging
from fastapi import Depends, Request
from fastapi.responses import RedirectResponse, Response
from sqlalchemy.orm import Session
from app.config import settings
from app.views.base import APIRouter, get_db, require_login, templates
from app.views.settings import require_admin_access
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/admin/system-reset")
@require_login
@require_admin_access
async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response:
"""Render the system reset administration page."""
if not settings.enable_factory_reset:
return RedirectResponse(url="/settings", status_code=302)
return templates.TemplateResponse(
"system_reset.html",
{
"request": request,
"factory_reset_on_startup": settings.factory_reset_on_startup,
},
)