fix: merge main, resolve conflicts, address review feedback

- Resolve merge conflicts in app/api/onedrive.py and tests/test_api_google_drive_final.py
- Fix legacy Dict[str, str] type hints in update_env_file functions to use dict[str, str]
- Add admin-only access (_require_admin dependency) to save-settings endpoints
  in google_drive.py, onedrive.py, and dropbox.py
- Fix in_memory_only response field to reflect actual env_write_success status
- Update tests to override _require_admin dependency for save-settings endpoint tests
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 16:29:46 +00:00
232 changed files with 114245 additions and 911 deletions
+8
View File
@@ -33,16 +33,20 @@ from app.api.pipelines import router as pipelines_router
from app.api.plans import router as plans_router
from app.api.process import router as process_router
from app.api.profile import router as profile_router
from app.api.qr_auth import router as qr_auth_router
from app.api.queue import router as queue_router
from app.api.routing_rules import router as routing_rules_router
from app.api.saved_searches import router as saved_searches_router
from app.api.scheduled_jobs import router as scheduled_jobs_router
from app.api.search import router as search_router
from app.api.sessions import router as sessions_router
from app.api.settings import router as settings_router
from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router
from app.api.system_reset import router as system_reset_router
from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router
# Import all the individual routers
@@ -95,4 +99,8 @@ router.include_router(scheduled_jobs_router)
router.include_router(audit_logs_router)
router.include_router(i18n_router)
router.include_router(mobile_router)
router.include_router(sessions_router)
router.include_router(qr_auth_router)
router.include_router(compliance_router)
router.include_router(system_reset_router)
router.include_router(translation_router)
+55 -15
View File
@@ -42,6 +42,9 @@ TOKEN_HASH_ITERATIONS = 100_000
#: PBKDF2 salt for API token hashing (not secret, but fixed for determinism).
TOKEN_HASH_SALT = b"api-token-v1"
#: Name prefix used for tokens created by the mobile app flow.
MOBILE_TOKEN_PREFIX = "Mobile App"
# ---------------------------------------------------------------------------
# Auth helper
@@ -91,6 +94,20 @@ def hash_token(token: str) -> str:
return dk.hex()
def _token_to_dict(t: ApiToken) -> dict[str, Any]:
"""Convert an ``ApiToken`` ORM instance to a serialisable dict."""
return {
"id": t.id,
"name": t.name,
"token_prefix": t.token_prefix,
"is_active": t.is_active,
"last_used_at": t.last_used_at,
"last_used_ip": t.last_used_ip,
"created_at": t.created_at,
"revoked_at": t.revoked_at,
}
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
@@ -177,21 +194,44 @@ async def list_tokens(
owner_id: CurrentOwner,
db: DbSession,
) -> list[dict[str, Any]]:
"""List all API tokens for the authenticated user."""
tokens = db.query(ApiToken).filter(ApiToken.owner_id == owner_id).order_by(ApiToken.created_at.desc()).all()
return [
{
"id": t.id,
"name": t.name,
"token_prefix": t.token_prefix,
"is_active": t.is_active,
"last_used_at": t.last_used_at,
"last_used_ip": t.last_used_ip,
"created_at": t.created_at,
"revoked_at": t.revoked_at,
}
for t in tokens
]
"""List non-mobile API tokens for the authenticated user.
Mobile tokens (whose names start with ``"Mobile App"``) are excluded
from this list; they are managed on the dedicated Devices page via
``GET /api/api-tokens/mobile``.
"""
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == owner_id,
~ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
)
.order_by(ApiToken.created_at.desc())
.all()
)
return [_token_to_dict(t) for t in tokens]
@router.get("/mobile", response_model=list[TokenResponse])
async def list_mobile_tokens(
owner_id: CurrentOwner,
db: DbSession,
) -> list[dict[str, Any]]:
"""List mobile API tokens for the authenticated user.
Returns tokens whose names start with ``"Mobile App"`` — these are
created via the mobile SSO flow or QR code login.
"""
tokens = (
db.query(ApiToken)
.filter(
ApiToken.owner_id == owner_id,
ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
)
.order_by(ApiToken.created_at.desc())
.all()
)
return [_token_to_dict(t) for t in tokens]
@router.delete("/{token_id}", status_code=status.HTTP_200_OK)
+2 -3
View File
@@ -168,9 +168,8 @@ async def create_checkout_session(
checkout_session = client.checkout.sessions.create(params=session_params)
logger.info(
"Created Stripe checkout session %s for user %s plan %s",
"Created Stripe checkout session %s for plan %s",
checkout_session.id,
owner_id,
body.plan_id,
)
return {"checkout_url": checkout_session.url, "session_id": checkout_session.id}
@@ -213,7 +212,7 @@ async def create_portal_session(
}
)
logger.info("Created Stripe portal session for user %s", owner_id)
logger.info("Created Stripe portal session for user")
return {"portal_url": portal.url}
+61 -47
View File
@@ -6,7 +6,7 @@ import logging
import os
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/dropbox/exchange-token")
@require_login
async def exchange_dropbox_token(
@@ -132,57 +143,60 @@ async def test_dropbox_token(request: Request):
"message": "Dropbox credentials are not fully configured",
}
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = requests.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = requests.post(
async with httpx.AsyncClient() as client:
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get account info
account_info = response.json()
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = await client.post(
refresh_url, data=refresh_data, timeout=settings.http_request_timeout
)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# Get account info
account_info = response.json()
account_email = account_info.get("email", "Unknown account")
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
@@ -208,9 +222,9 @@ async def test_dropbox_token(request: Request):
@router.post("/dropbox/save-settings")
@require_login
async def save_dropbox_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
app_key: Annotated[Optional[str], Form()] = None,
app_secret: Annotated[Optional[str], Form()] = None,
+28 -23
View File
@@ -73,33 +73,38 @@ def list_duplicate_groups(
groups = []
total_duplicate_files = 0
for filehash in dup_hashes:
# Find the original (non-duplicate) record with this hash
original = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
if dup_hashes:
# Fetch all matching files (both original and duplicates) in a single batch query
all_records = (
db.query(FileRecord).filter(FileRecord.filehash.in_(dup_hashes)).order_by(FileRecord.id.asc()).all()
)
# Find all duplicate records for this hash
duplicates = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
.order_by(FileRecord.id.asc())
.all()
)
# Group records by hash
originals_by_hash = {}
duplicates_by_hash = {h: [] for h in dup_hashes}
total_duplicate_files += len(duplicates)
for record in all_records:
h = record.filehash
if not record.is_duplicate:
# Store only the first original record per hash, matching the old .first() behaviour
if h not in originals_by_hash:
originals_by_hash[h] = record
else:
duplicates_by_hash[h].append(record)
total_duplicate_files += 1
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
for filehash in dup_hashes:
original = originals_by_hash.get(filehash)
duplicates = duplicates_by_hash.get(filehash, [])
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
+95 -39
View File
@@ -11,6 +11,7 @@ import zipfile
from datetime import datetime, timezone
from typing import Annotated, List, Optional
import aiofiles
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status
from fastapi.responses import StreamingResponse
from sqlalchemy import asc, desc
@@ -345,7 +346,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -384,7 +387,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
"""
try:
# Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -456,7 +461,9 @@ def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: Db
Useful for re-running OCR on files with poor text quality or missing OCR text.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -536,7 +543,9 @@ def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
Files not found on disk are silently skipped.
"""
try:
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
query = apply_owner_filter(query, request)
file_records = query.all()
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
@@ -618,7 +627,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -674,7 +685,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -937,7 +950,9 @@ def retry_subtask(
"""
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1079,7 +1094,9 @@ def get_file_preview(
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1159,7 +1176,9 @@ def download_file(
try:
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
file_record = query.first()
if not file_record:
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
@@ -1217,6 +1236,66 @@ def download_file(
raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")
async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: int) -> int:
"""Save an uploaded file in chunks and enforce the maximum size limit."""
try:
written_size = 0
with open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
if not chunk:
break
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
return written_size
except HTTPException:
raise
except Exception as e:
if os.path.exists(target_path):
os.remove(target_path)
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
if not settings.enable_deduplication:
return None
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
return {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
return None
@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
@@ -1277,7 +1356,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# enforcing the size limit during the read so memory usage stays bounded.
try:
written_size = 0
with open(target_path, "wb") as f:
async with aiofiles.open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
@@ -1286,14 +1365,14 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
await f.write(chunk)
except HTTPException:
raise
except Exception as e:
@@ -1384,29 +1463,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# Check for exact duplicates (same SHA-256 hash) before returning.
# This gives the caller an immediate warning without waiting for the pipeline.
# Only performed when deduplication is enabled in settings.
exact_duplicate_warning = None
if settings.enable_deduplication:
try:
filehash = hash_file(target_path)
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
)
if existing:
exact_duplicate_warning = {
"duplicate_type": "exact",
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
),
}
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
except Exception as e:
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
response: dict = {
"task_id": task.id,
@@ -1458,7 +1515,7 @@ def claim_file(request: Request, file_id: int, db: DbSession):
logger.exception(f"Error claiming file {file_id}: {e}")
raise HTTPException(status_code=500, detail="Failed to claim document")
logger.info(f"File {file_id} claimed by user '{owner_id}'")
logger.info("File %d claimed by user", file_id)
return {"status": "success", "message": "Document claimed successfully", "file_id": file_id, "owner_id": owner_id}
@@ -1498,7 +1555,7 @@ def bulk_claim_files(request: Request, file_ids: list[int], db: DbSession):
logger.exception(f"Error during bulk claim: {e}")
raise HTTPException(status_code=500, detail="Failed to claim documents")
logger.info(f"Bulk claim by '{owner_id}': claimed={claimed}, skipped={[s['file_id'] for s in skipped]}")
logger.info("Bulk claim: claimed=%s, skipped=%s", claimed, [s["file_id"] for s in skipped])
return {
"status": "success",
"claimed_count": len(claimed),
@@ -1551,8 +1608,7 @@ def assign_owner(request: Request, db: DbSession, owner_id: str = Query(...), fi
logger.exception(f"Error assigning owner: {e}")
raise HTTPException(status_code=500, detail="Failed to assign owner")
admin_name = get_current_owner_id(request) or "admin"
logger.info(f"Admin '{admin_name}' assigned owner_id='{owner_id}' to {updated} file(s)")
logger.info("Admin assigned owner to %d file(s)", updated)
return {
"status": "success",
"message": f"Assigned owner to {updated} document(s)",
+15 -3
View File
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/google-drive/exchange-token")
@require_login
async def exchange_google_drive_token(
@@ -362,9 +373,9 @@ def format_time_remaining(time_delta):
@router.post("/google-drive/save-settings")
@require_login
async def save_google_drive_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
client_secret: Annotated[Optional[str], Form()] = None,
@@ -404,7 +415,8 @@ async def save_google_drive_settings(
drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
# Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
if not update_env_file(env_path, drive_settings):
env_write_success = update_env_file(env_path, drive_settings)
if not env_write_success:
logger.info("Continuing with in-memory update despite .env file update failure or skip")
# Update the settings in memory (this always happens)
@@ -443,7 +455,7 @@ async def save_google_drive_settings(
return {
"status": "success",
"message": "Google Drive settings have been saved",
"in_memory_only": not os.path.exists(env_path),
"in_memory_only": not env_write_success,
}
except Exception as e:
+6
View File
@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
from app.database import get_db
from app.models import UserImapAccount
from app.utils.encryption import decrypt_value, encrypt_value
from app.utils.network import is_private_ip
from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id
@@ -187,6 +188,11 @@ def _test_imap_connection(host: str, port: int, username: str, password: str, us
Returns a dict with ``{"success": bool, "message": str}``.
"""
# Security: Prevent SSRF by blocking connections to internal IPs
if is_private_ip(host):
logger.warning("SSRF blocked: Attempt to connect to private IP %s", host)
return {"success": False, "message": "Connection error: Invalid hostname or IP address"}
try:
if use_ssl:
mail = imaplib.IMAP4_SSL(host, port)
+9 -10
View File
@@ -452,17 +452,16 @@ async def update_preferences(
)
try:
# Pre-fetch existing preferences for this user to avoid N+1 queries
existing_prefs = (
db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
)
# Build a fast lookup dictionary keyed by (event_type, channel_type, target_id)
prefs_dict = {(pref.event_type, pref.channel_type, pref.target_id): pref for pref in existing_prefs}
for item in body.preferences:
existing = (
db.query(UserNotificationPreference)
.filter(
UserNotificationPreference.owner_id == owner_id,
UserNotificationPreference.event_type == item.event_type,
UserNotificationPreference.channel_type == item.channel_type,
UserNotificationPreference.target_id == item.target_id,
)
.first()
)
existing = prefs_dict.get((item.event_type, item.channel_type, item.target_id))
if existing:
existing.is_enabled = item.is_enabled
else:
+46 -61
View File
@@ -7,7 +7,7 @@ import os
from datetime import datetime, timedelta
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -24,6 +24,17 @@ logger = logging.getLogger(__name__)
router = APIRouter()
def _require_admin(request: Request) -> dict:
"""Dependency to ensure the current user is an admin."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
@router.post("/onedrive/exchange-token")
@require_login
async def exchange_onedrive_token(
@@ -92,17 +103,18 @@ async def test_onedrive_token(request: Request):
"scope": "offline_access Files.ReadWrite",
}
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
response = await client.post(token_url, data=refresh_data)
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
if response.status_code != 200:
logger.error(f"Failed to refresh OneDrive token: {response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_data = response.json()
token_data = response.json()
access_token = token_data.get("access_token")
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
@@ -115,32 +127,7 @@ async def test_onedrive_token(request: Request):
settings.onedrive_refresh_token = new_refresh_token
# Also try to update .env file if it exists
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if os.path.exists(env_path):
with open(env_path, "r") as f:
env_lines = f.readlines()
updated_lines = []
updated = False
for line in env_lines:
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
updated = True
else:
updated_lines.append(line)
if not updated:
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
with open(env_path, "w") as f:
f.writelines(updated_lines)
logger.info("Updated refresh token in .env file")
except Exception as e:
logger.warning(f"Failed to update refresh token in .env file: {e}")
update_env_file({"ONEDRIVE_REFRESH_TOKEN": new_refresh_token})
# Persist the rotated refresh token to the database
try:
@@ -164,17 +151,18 @@ async def test_onedrive_token(request: Request):
user_info_url = "https://graph.microsoft.com/v1.0/me"
headers = {"Authorization": f"Bearer {access_token}"}
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
user_response = await client.get(user_info_url, headers=headers)
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
if user_response.status_code != 200:
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
}
# Get user info
user_info = user_response.json()
# Get user info
user_info = user_response.json()
display_name = user_info.get("displayName", "Unknown user")
email = user_info.get("userPrincipalName", "Unknown email")
@@ -227,9 +215,9 @@ def format_time_remaining(time_delta):
@router.post("/onedrive/save-settings")
@require_login
async def save_onedrive_settings(
request: Request,
_admin: AdminUser,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
client_secret: Annotated[Optional[str], Form()] = None,
@@ -247,22 +235,19 @@ async def save_onedrive_settings(
)
# Best-effort .env file write
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
if not update_env_file(env_path, onedrive_settings):
logger.info("Continuing with in-memory update despite .env file update failure or skip")
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
if not update_env_file(env_path, onedrive_settings):
logger.info("Continuing with in-memory update despite .env file update failure or skip")
# Update the settings in memory
if refresh_token:
+11 -2
View File
@@ -196,11 +196,20 @@ def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
updated = 0
for sort_order, plan_id in enumerate(body.order):
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
# Fetch all requested plans in a single query to avoid N+1
plan_ids = body.order
plans = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id.in_(plan_ids)).all()
# Build a map for fast O(1) lookup
plan_map = {p.plan_id: p for p in plans}
for sort_order, plan_id in enumerate(plan_ids):
plan = plan_map.get(plan_id)
if plan:
plan.sort_order = sort_order
updated += 1
try:
db.commit()
except Exception:
+18
View File
@@ -99,6 +99,8 @@ class ProfileResponse(BaseModel):
contact_email: str | None
preferred_language: str | None
preferred_theme: str | None
default_document_language: str | None
"""ISO 639-1 code for the user's preferred document translation target language."""
avatar_url: str
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
is_local_user: bool
@@ -112,6 +114,10 @@ class ProfileUpdateRequest(BaseModel):
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
default_document_language: str | None = Field(
default=None,
description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'",
)
class ChangePasswordRequest(BaseModel):
@@ -149,6 +155,7 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
@@ -201,6 +208,16 @@ async def update_profile(
)
profile.preferred_theme = theme or None # type: ignore[assignment]
# Validate default document language
if body.default_document_language is not None:
doc_lang = body.default_document_language.lower().strip()
if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported language code: {doc_lang}",
)
profile.default_document_language = doc_lang or None # type: ignore[assignment]
if body.display_name is not None:
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
@@ -225,6 +242,7 @@ async def update_profile(
contact_email=profile.contact_email, # type: ignore[arg-type]
preferred_language=profile.preferred_language, # type: ignore[arg-type]
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
default_document_language=profile.default_document_language, # type: ignore[arg-type]
avatar_url=avatar_url,
is_local_user=is_local,
)
+233
View File
@@ -0,0 +1,233 @@
"""QR code login API endpoints for mobile app authentication.
Provides a secure challenge-response flow for logging into the mobile app
by scanning a QR code displayed in the web interface:
1. **Web user** calls ``POST /qr-auth/challenge`` → receives a time-limited
challenge token (encoded in the QR code).
2. **Web UI** polls ``GET /qr-auth/challenge/{id}/status`` to detect when
the mobile app has claimed the challenge.
3. **Mobile app** scans the QR code and calls ``POST /qr-auth/claim`` with
the challenge token + device name → receives an API token.
Security properties:
* Challenges expire after a configurable TTL (default 2 minutes).
* Single-use: once claimed, a challenge cannot be reused (replay-safe).
* Cryptographically random 64-byte tokens.
* IP addresses are logged for audit.
"""
from __future__ import annotations
import base64
import io
import logging
from datetime import datetime
from typing import Annotated, Any
import segno
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.middleware.audit_log import get_client_ip
from app.utils.session_manager import (
claim_qr_challenge,
create_qr_challenge,
get_challenge_status,
)
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/qr-auth", tags=["qr-auth"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helper
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if not owner_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Request / Response schemas
# ---------------------------------------------------------------------------
class CreateChallengeResponse(BaseModel):
"""Response after creating a QR login challenge."""
challenge_id: int
challenge_token: str
expires_at: datetime
ttl_seconds: int = Field(description="Seconds until the challenge expires (use for client-side countdown).")
qr_payload: str = Field(description="The string to encode in the QR code.")
qr_code_svg: str = Field(description="Base64-encoded SVG data URI of the QR code, ready for use in an <img> src.")
class ChallengeStatusResponse(BaseModel):
"""Response for polling the status of a QR challenge."""
id: int
status: str # "pending", "claimed", "expired", "cancelled"
device_name: str | None = None
claimed_at: datetime | None = None
expires_at: datetime
class ClaimChallengeRequest(BaseModel):
"""Request body for claiming a QR login challenge."""
challenge_token: str = Field(min_length=1, max_length=256)
device_name: str = Field(
default="Mobile App",
min_length=1,
max_length=120,
description="Human-readable device name.",
)
class ClaimChallengeResponse(BaseModel):
"""Response after successfully claiming a QR challenge."""
token: str
token_id: int
name: str
owner_id: str
created_at: datetime
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
# QR code rendering parameters
_QR_ERROR_LEVEL = "M" # Medium error correction (~15% recovery); sufficient for on-screen display
_QR_SCALE = 4 # Each QR module is rendered as 4×4 SVG pixels
def _generate_qr_svg(payload: str) -> str:
"""Generate a QR code for *payload* and return it as a base64 SVG data URI.
Using ``segno`` (pure-Python, no Pillow dependency) and SVG output so the
QR code scales crisply at any resolution without requiring a canvas or any
client-side JavaScript library.
"""
qr = segno.make(payload, error=_QR_ERROR_LEVEL)
buf = io.BytesIO()
qr.save(buf, kind="svg", scale=_QR_SCALE, xmldecl=False, svgclass=None, lineclass=None, omitsize=True)
svg_bytes = buf.getvalue()
return "data:image/svg+xml;base64," + base64.b64encode(svg_bytes).decode("ascii")
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.post("/challenge", status_code=status.HTTP_201_CREATED, response_model=CreateChallengeResponse)
@require_login
async def create_challenge(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Create a new QR login challenge.
The returned ``qr_payload`` should be encoded into a QR code and
displayed to the user. The mobile app scans this QR code and
calls the ``/claim`` endpoint.
"""
ip = get_client_ip(request)
challenge = create_qr_challenge(db, owner_id, ip_address=ip)
# The QR payload is a JSON-like string with enough info for the mobile
# app to know the server URL and challenge token.
base_url = str(request.base_url).rstrip("/")
qr_payload = f"docuelevate://qr-login?token={challenge.challenge_token}&server={base_url}"
# Compute the TTL in seconds so the client can run a countdown timer
# without comparing absolute timestamps (which breaks when client and
# server clocks are out of sync).
ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds()))
return {
"challenge_id": challenge.id,
"challenge_token": challenge.challenge_token,
"expires_at": challenge.expires_at,
"ttl_seconds": ttl_seconds,
"qr_payload": qr_payload,
"qr_code_svg": _generate_qr_svg(qr_payload),
}
@router.get("/challenge/{challenge_id}/status", response_model=ChallengeStatusResponse)
@require_login
async def poll_challenge_status(
request: Request,
challenge_id: int,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Poll the status of a QR login challenge.
The web UI calls this endpoint every few seconds to check if the
mobile app has scanned the QR code and claimed the challenge.
"""
result = get_challenge_status(db, challenge_id, owner_id)
if not result:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Challenge not found")
return result
@router.post("/claim", response_model=ClaimChallengeResponse)
async def claim_challenge(
request: Request,
body: ClaimChallengeRequest,
db: DbSession,
) -> dict[str, Any]:
"""Claim a QR login challenge and receive an API token.
This endpoint is called by the mobile app after scanning a QR code.
It does **not** require authentication — the challenge token itself
serves as proof that the user authorized this login from their web
session.
"""
ip = get_client_ip(request)
result = claim_qr_challenge(db, body.challenge_token, device_name=body.device_name, ip_address=ip)
if not result:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid, expired, or already claimed challenge.",
)
try:
from app.utils.audit_service import record_event
record_event(
db,
action="qr_login_claimed",
user=result["owner_id"],
resource_type="session",
ip_address=ip,
details={"device_name": body.device_name, "token_id": result["token_id"]},
severity="info",
)
except Exception:
logger.debug("Failed to write QR login audit event", exc_info=True)
return result
+196
View File
@@ -0,0 +1,196 @@
"""API endpoints for managing user sessions.
Provides endpoints for listing active sessions, revoking individual sessions,
and the "log off everywhere" feature that invalidates all sessions and API
tokens across all devices.
"""
from __future__ import annotations
import logging
from datetime import datetime
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.auth import require_login
from app.database import get_db
from app.middleware.audit_log import get_client_ip
from app.utils.session_manager import (
get_session_lifetime_days,
list_user_sessions,
revoke_all_sessions,
revoke_session,
)
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/sessions", tags=["sessions"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helper
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if not owner_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Response schemas
# ---------------------------------------------------------------------------
class SessionResponse(BaseModel):
"""Serialised user session for the management UI."""
id: int
device_info: str | None
ip_address: str | None
created_at: datetime
last_active_at: datetime
expires_at: datetime
is_current: bool = False
class SessionListResponse(BaseModel):
"""Response for listing active sessions."""
sessions: list[SessionResponse]
session_lifetime_days: int
class RevokeAllResponse(BaseModel):
"""Response after revoking all sessions."""
revoked_count: int
message: str
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/", response_model=SessionListResponse)
@require_login
async def list_sessions(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""List all active sessions for the current user."""
sessions = list_user_sessions(db, owner_id)
# Determine which session is the current one
current_token = request.session.get("_session_token")
session_list = []
for s in sessions:
session_list.append(
{
"id": s.id,
"device_info": s.device_info,
"ip_address": s.ip_address,
"created_at": s.created_at,
"last_active_at": s.last_active_at,
"expires_at": s.expires_at,
"is_current": s.session_token == current_token if current_token else False,
}
)
return {
"sessions": session_list,
"session_lifetime_days": get_session_lifetime_days(),
}
@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT)
@require_login
async def revoke_single_session(
request: Request,
session_id: int,
owner_id: CurrentOwner,
db: DbSession,
) -> None:
"""Revoke a specific session by ID."""
success = revoke_session(db, session_id, owner_id)
if not success:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found")
try:
from app.utils.audit_service import record_event
record_event(
db,
action="session_revoked",
user=owner_id,
resource_type="session",
resource_id=str(session_id),
ip_address=get_client_ip(request),
severity="info",
)
except Exception:
logger.debug("Failed to write session revocation audit event", exc_info=True)
@router.post("/revoke-all", response_model=RevokeAllResponse)
@require_login
async def revoke_all(
request: Request,
owner_id: CurrentOwner,
db: DbSession,
) -> dict[str, Any]:
"""Revoke all sessions except the current one ("log off everywhere").
Also revokes all active API tokens for the user, which invalidates
mobile app sessions and any programmatic access.
"""
# Find current session to preserve it
current_token = request.session.get("_session_token")
current_session_id = None
if current_token:
from app.models import UserSession
current = db.query(UserSession).filter(UserSession.session_token == current_token).first()
if current:
current_session_id = current.id
count = revoke_all_sessions(
db,
owner_id,
except_session_id=current_session_id,
revoke_api_tokens=True,
)
try:
from app.utils.audit_service import record_event
record_event(
db,
action="revoke_all_sessions",
user=owner_id,
resource_type="session",
ip_address=get_client_ip(request),
details={"revoked_count": count},
severity="warning",
)
except Exception:
logger.debug("Failed to write revoke-all audit event", exc_info=True)
return {
"revoked_count": count,
"message": f"Successfully revoked {count} session(s) and all API tokens.",
}
+7 -5
View File
@@ -313,16 +313,18 @@ async def list_shared_links(
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
) -> list[dict[str, Any]]:
"""List all shared links created by the authenticated user."""
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
q = (
db.query(SharedLink, FileRecord.original_filename)
.outerjoin(FileRecord, SharedLink.file_id == FileRecord.id)
.filter(SharedLink.owner_id == owner_id)
)
if active_only:
q = q.filter(SharedLink.is_active.is_(True))
links = q.order_by(SharedLink.created_at.desc()).all()
links_with_filenames = q.order_by(SharedLink.created_at.desc()).all()
base_url = str(request.base_url).rstrip("/")
result = []
for link in links:
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
filename = file_record.original_filename if file_record else None
for link, filename in links_with_filenames:
result.append(_link_to_dict(link, base_url, filename))
return result
+124
View File
@@ -0,0 +1,124 @@
"""
System reset API endpoints for DocuElevate.
Provides admin-only REST endpoints for:
- Full system reset (wipe all user data)
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
admin privileges.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
def _require_admin(request: Request) -> dict:
"""Ensure the caller is an admin. Raises 403 otherwise."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
def _require_feature_enabled() -> None:
"""Raise 404 when the factory-reset feature flag is off."""
if not settings.enable_factory_reset:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
)
class ResetRequest(BaseModel):
"""Body for system reset endpoints. Requires explicit confirmation."""
confirmation: str
@router.post("/full")
async def full_reset(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Wipe all user data (database + work-files).
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "DELETE":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
)
from app.utils.system_reset import perform_full_reset
try:
result = perform_full_reset(db)
except Exception as exc:
logger.exception("Full system reset failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"System reset failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.post("/reimport")
async def reset_and_reimport(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Move original files to a reimport folder, wipe everything, and
configure the reimport folder as a watch folder for automatic
re-ingestion.
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "REIMPORT":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
)
from app.utils.system_reset import perform_reset_and_reimport
try:
result = perform_reset_and_reimport(db)
except Exception as exc:
logger.exception("Reset-and-reimport failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Reset and reimport failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.get("/status")
async def reset_status(_admin: AdminUser) -> dict:
"""Return whether the system reset feature is enabled."""
return {
"enabled": settings.enable_factory_reset,
"factory_reset_on_startup": settings.factory_reset_on_startup,
}
+156
View File
@@ -0,0 +1,156 @@
"""
API endpoints for document translation.
Provides on-the-fly translation via the AI provider and access to the
persisted default-language translation.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord
from app.utils.ai_provider import get_ai_provider
from app.utils.user_scope import apply_owner_filter
logger = logging.getLogger(__name__)
router = APIRouter()
DbSession = Annotated[Session, Depends(get_db)]
# Maximum characters sent to the AI provider for a single translation request.
_MAX_TRANSLATION_INPUT = 50_000
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
"""Fetch a FileRecord visible to the current user or raise 404."""
query = db.query(FileRecord).filter(FileRecord.id == file_id)
query = apply_owner_filter(query, request)
record = query.first()
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
return record
@router.get("/files/{file_id}/translation/default")
@require_login
def get_default_translation(
request: Request,
file_id: int,
db: DbSession,
) -> JSONResponse:
"""Return the persisted default-language translation for a document.
Returns 404 if no default-language translation has been generated yet
(e.g. because the document is already in the default language).
"""
record = _get_file_or_404(db, file_id, request)
if not record.default_language_text:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No default-language translation available for this file",
)
return JSONResponse(
content={
"file_id": record.id,
"detected_language": record.detected_language,
"default_language_code": record.default_language_code,
"text": record.default_language_text,
}
)
@router.get("/files/{file_id}/translate")
@require_login
def translate_on_the_fly(
request: Request,
file_id: int,
db: DbSession,
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
) -> JSONResponse:
"""Translate a document's extracted text into an arbitrary language on the fly.
The translation is generated via the configured AI provider and is **not**
persisted. For the default-language translation, use the
``/files/{file_id}/translation/default`` endpoint instead.
"""
record = _get_file_or_404(db, file_id, request)
source_text = record.ocr_text
if not source_text:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No extracted text available for this file — translation requires OCR text",
)
# If the requested language matches what is already stored, return it directly.
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
return JSONResponse(
content={
"file_id": record.id,
"source_language": record.detected_language,
"target_language": lang,
"text": record.default_language_text,
"cached": True,
}
)
# If the detected language already matches, return the original text.
detected = record.detected_language
if detected and detected == lang:
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected,
"target_language": lang,
"text": source_text,
"cached": True,
}
)
# Truncate to keep AI costs bounded.
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
try:
provider = get_ai_provider()
model = settings.ai_model or settings.openai_model
translated = provider.chat_completion(
messages=[
{
"role": "system",
"content": (
f"You are a professional translator. Translate the following text "
f"into {lang}. Preserve the original formatting, paragraph structure, "
f"and meaning. Do not add any commentary — output ONLY the translated text."
),
},
{"role": "user", "content": text_to_translate},
],
model=model,
temperature=0.3,
)
except Exception as exc:
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Translation failed — the AI provider returned an error",
)
return JSONResponse(
content={
"file_id": record.id,
"source_language": detected or "unknown",
"target_language": lang,
"text": translated,
"cached": False,
}
)
+63 -56
View File
@@ -9,7 +9,8 @@ import urllib.parse
import uuid
from typing import Optional
import requests
import aiofiles
import httpx
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, HttpUrl, field_validator
@@ -153,67 +154,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
logger.info(f"Downloading file from URL: {url}")
# Use configured timeout to prevent hanging
response = requests.get(
url,
async with httpx.AsyncClient(
timeout=settings.http_request_timeout,
stream=True, # Stream to handle large files
allow_redirects=True, # Follow redirects
follow_redirects=True,
headers={
"User-Agent": "DocuElevate/1.0", # Identify ourselves
},
)
response.raise_for_status()
) as client:
async with client.stream("GET", url) as response:
response.raise_for_status()
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Validate content type
content_type = response.headers.get("Content-Type", "")
if not validate_file_type(content_type, safe_filename):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {content_type}. "
"Supported types: PDF, Office documents, images, plain text",
)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
if "." in safe_filename:
file_extension = safe_filename.rsplit(".", 1)[1]
target_filename = f"{unique_id}.{file_extension}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
with open(target_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
f.close()
os.remove(target_path)
# Check content length before downloading
content_length = response.headers.get("Content-Length")
if content_length:
file_size = int(content_length)
max_size = settings.max_upload_size
if file_size > max_size:
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
)
# Generate unique filename
unique_id = str(uuid.uuid4())
# Check for extension using original_filename to avoid any CodeQL issues
# with safe_filename which is derived from the URL directly.
if "." in original_filename:
_, ext = os.path.splitext(original_filename)
# Strip out the leading dot and any non-alphanumeric chars
clean_ext = "".join(c for c in ext if c.isalnum())
if not clean_ext:
clean_ext = "bin"
target_filename = f"{unique_id}.{clean_ext}"
else:
target_filename = unique_id
target_path = os.path.join(settings.workdir, target_filename)
# Download file in chunks to handle large files
downloaded_size = 0
max_size = settings.max_upload_size
async with aiofiles.open(target_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=8192):
if chunk:
await f.write(chunk)
downloaded_size += len(chunk)
# Check size during download
if downloaded_size > max_size:
# Remove partial file
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during download",
)
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
# Enqueue for processing
@@ -227,19 +234,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
"size": downloaded_size,
}
except requests.exceptions.Timeout:
except httpx.TimeoutException:
logger.error(f"Timeout while downloading file from URL: {url}")
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
except requests.exceptions.ConnectionError as e:
except httpx.ConnectError as e:
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
except requests.exceptions.HTTPError as e:
except httpx.HTTPStatusError as e:
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
except requests.exceptions.RequestException as e:
except httpx.RequestError as e:
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")