fix: merge main, resolve conflicts, address review feedback
- Resolve merge conflicts in app/api/onedrive.py and tests/test_api_google_drive_final.py - Fix legacy Dict[str, str] type hints in update_env_file functions to use dict[str, str] - Add admin-only access (_require_admin dependency) to save-settings endpoints in google_drive.py, onedrive.py, and dropbox.py - Fix in_memory_only response field to reflect actual env_write_success status - Update tests to override _require_admin dependency for save-settings endpoint tests
This commit is contained in:
@@ -33,16 +33,20 @@ from app.api.pipelines import router as pipelines_router
|
||||
from app.api.plans import router as plans_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.profile import router as profile_router
|
||||
from app.api.qr_auth import router as qr_auth_router
|
||||
from app.api.queue import router as queue_router
|
||||
from app.api.routing_rules import router as routing_rules_router
|
||||
from app.api.saved_searches import router as saved_searches_router
|
||||
from app.api.scheduled_jobs import router as scheduled_jobs_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.sessions import router as sessions_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.shared_links import public_router as shared_links_public_router
|
||||
from app.api.shared_links import router as shared_links_router
|
||||
from app.api.similarity import router as similarity_router
|
||||
from app.api.subscriptions import router as subscriptions_router
|
||||
from app.api.system_reset import router as system_reset_router
|
||||
from app.api.translation import router as translation_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
# Import all the individual routers
|
||||
@@ -95,4 +99,8 @@ router.include_router(scheduled_jobs_router)
|
||||
router.include_router(audit_logs_router)
|
||||
router.include_router(i18n_router)
|
||||
router.include_router(mobile_router)
|
||||
router.include_router(sessions_router)
|
||||
router.include_router(qr_auth_router)
|
||||
router.include_router(compliance_router)
|
||||
router.include_router(system_reset_router)
|
||||
router.include_router(translation_router)
|
||||
|
||||
+55
-15
@@ -42,6 +42,9 @@ TOKEN_HASH_ITERATIONS = 100_000
|
||||
#: PBKDF2 salt for API token hashing (not secret, but fixed for determinism).
|
||||
TOKEN_HASH_SALT = b"api-token-v1"
|
||||
|
||||
#: Name prefix used for tokens created by the mobile app flow.
|
||||
MOBILE_TOKEN_PREFIX = "Mobile App"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
@@ -91,6 +94,20 @@ def hash_token(token: str) -> str:
|
||||
return dk.hex()
|
||||
|
||||
|
||||
def _token_to_dict(t: ApiToken) -> dict[str, Any]:
|
||||
"""Convert an ``ApiToken`` ORM instance to a serialisable dict."""
|
||||
return {
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"token_prefix": t.token_prefix,
|
||||
"is_active": t.is_active,
|
||||
"last_used_at": t.last_used_at,
|
||||
"last_used_ip": t.last_used_ip,
|
||||
"created_at": t.created_at,
|
||||
"revoked_at": t.revoked_at,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -177,21 +194,44 @@ async def list_tokens(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all API tokens for the authenticated user."""
|
||||
tokens = db.query(ApiToken).filter(ApiToken.owner_id == owner_id).order_by(ApiToken.created_at.desc()).all()
|
||||
return [
|
||||
{
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"token_prefix": t.token_prefix,
|
||||
"is_active": t.is_active,
|
||||
"last_used_at": t.last_used_at,
|
||||
"last_used_ip": t.last_used_ip,
|
||||
"created_at": t.created_at,
|
||||
"revoked_at": t.revoked_at,
|
||||
}
|
||||
for t in tokens
|
||||
]
|
||||
"""List non-mobile API tokens for the authenticated user.
|
||||
|
||||
Mobile tokens (whose names start with ``"Mobile App"``) are excluded
|
||||
from this list; they are managed on the dedicated Devices page via
|
||||
``GET /api/api-tokens/mobile``.
|
||||
"""
|
||||
tokens = (
|
||||
db.query(ApiToken)
|
||||
.filter(
|
||||
ApiToken.owner_id == owner_id,
|
||||
~ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
|
||||
)
|
||||
.order_by(ApiToken.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
return [_token_to_dict(t) for t in tokens]
|
||||
|
||||
|
||||
@router.get("/mobile", response_model=list[TokenResponse])
|
||||
async def list_mobile_tokens(
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List mobile API tokens for the authenticated user.
|
||||
|
||||
Returns tokens whose names start with ``"Mobile App"`` — these are
|
||||
created via the mobile SSO flow or QR code login.
|
||||
"""
|
||||
tokens = (
|
||||
db.query(ApiToken)
|
||||
.filter(
|
||||
ApiToken.owner_id == owner_id,
|
||||
ApiToken.name.startswith(MOBILE_TOKEN_PREFIX),
|
||||
)
|
||||
.order_by(ApiToken.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
return [_token_to_dict(t) for t in tokens]
|
||||
|
||||
|
||||
@router.delete("/{token_id}", status_code=status.HTTP_200_OK)
|
||||
|
||||
+2
-3
@@ -168,9 +168,8 @@ async def create_checkout_session(
|
||||
checkout_session = client.checkout.sessions.create(params=session_params)
|
||||
|
||||
logger.info(
|
||||
"Created Stripe checkout session %s for user %s plan %s",
|
||||
"Created Stripe checkout session %s for plan %s",
|
||||
checkout_session.id,
|
||||
owner_id,
|
||||
body.plan_id,
|
||||
)
|
||||
return {"checkout_url": checkout_session.url, "session_id": checkout_session.id}
|
||||
@@ -213,7 +212,7 @@ async def create_portal_session(
|
||||
}
|
||||
)
|
||||
|
||||
logger.info("Created Stripe portal session for user %s", owner_id)
|
||||
logger.info("Created Stripe portal session for user")
|
||||
return {"portal_url": portal.url}
|
||||
|
||||
|
||||
|
||||
+61
-47
@@ -6,7 +6,7 @@ import logging
|
||||
import os
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Dependency to ensure the current user is an admin."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
@router.post("/dropbox/exchange-token")
|
||||
@require_login
|
||||
async def exchange_dropbox_token(
|
||||
@@ -132,57 +143,60 @@ async def test_dropbox_token(request: Request):
|
||||
"message": "Dropbox credentials are not fully configured",
|
||||
}
|
||||
|
||||
# Check token validity by getting current account info
|
||||
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
|
||||
response = requests.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
)
|
||||
|
||||
# If token is invalid, try refreshing it
|
||||
if response.status_code == 401:
|
||||
logger.info("Dropbox access token invalid or expired, trying to refresh")
|
||||
|
||||
# Get a new access token using the refresh token
|
||||
refresh_url = "https://api.dropbox.com/oauth2/token"
|
||||
refresh_data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
}
|
||||
|
||||
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
|
||||
if refresh_response.status_code != 200:
|
||||
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
|
||||
token_info = refresh_response.json()
|
||||
access_token = token_info.get("access_token")
|
||||
|
||||
# Try again with the new access token
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = requests.post(
|
||||
async with httpx.AsyncClient() as client:
|
||||
# Check token validity by getting current account info
|
||||
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
|
||||
response = await client.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {response.status_code}: {response.text}",
|
||||
}
|
||||
# If token is invalid, try refreshing it
|
||||
if response.status_code == 401:
|
||||
logger.info("Dropbox access token invalid or expired, trying to refresh")
|
||||
|
||||
# Get account info
|
||||
account_info = response.json()
|
||||
# Get a new access token using the refresh token
|
||||
refresh_url = "https://api.dropbox.com/oauth2/token"
|
||||
refresh_data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
}
|
||||
|
||||
refresh_response = await client.post(
|
||||
refresh_url, data=refresh_data, timeout=settings.http_request_timeout
|
||||
)
|
||||
|
||||
if refresh_response.status_code != 200:
|
||||
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
|
||||
token_info = refresh_response.json()
|
||||
access_token = token_info.get("access_token")
|
||||
|
||||
# Try again with the new access token
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = await client.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {response.status_code}: {response.text}",
|
||||
}
|
||||
|
||||
# Get account info
|
||||
account_info = response.json()
|
||||
account_email = account_info.get("email", "Unknown account")
|
||||
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
|
||||
|
||||
@@ -208,9 +222,9 @@ async def test_dropbox_token(request: Request):
|
||||
|
||||
|
||||
@router.post("/dropbox/save-settings")
|
||||
@require_login
|
||||
async def save_dropbox_settings(
|
||||
request: Request,
|
||||
_admin: AdminUser,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
app_key: Annotated[Optional[str], Form()] = None,
|
||||
app_secret: Annotated[Optional[str], Form()] = None,
|
||||
|
||||
+28
-23
@@ -73,33 +73,38 @@ def list_duplicate_groups(
|
||||
groups = []
|
||||
total_duplicate_files = 0
|
||||
|
||||
for filehash in dup_hashes:
|
||||
# Find the original (non-duplicate) record with this hash
|
||||
original = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
if dup_hashes:
|
||||
# Fetch all matching files (both original and duplicates) in a single batch query
|
||||
all_records = (
|
||||
db.query(FileRecord).filter(FileRecord.filehash.in_(dup_hashes)).order_by(FileRecord.id.asc()).all()
|
||||
)
|
||||
|
||||
# Find all duplicate records for this hash
|
||||
duplicates = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
# Group records by hash
|
||||
originals_by_hash = {}
|
||||
duplicates_by_hash = {h: [] for h in dup_hashes}
|
||||
|
||||
total_duplicate_files += len(duplicates)
|
||||
for record in all_records:
|
||||
h = record.filehash
|
||||
if not record.is_duplicate:
|
||||
# Store only the first original record per hash, matching the old .first() behaviour
|
||||
if h not in originals_by_hash:
|
||||
originals_by_hash[h] = record
|
||||
else:
|
||||
duplicates_by_hash[h].append(record)
|
||||
total_duplicate_files += 1
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"filehash": filehash,
|
||||
"original": _file_record_to_dict(original) if original else None,
|
||||
"duplicates": [_file_record_to_dict(d) for d in duplicates],
|
||||
"duplicate_count": len(duplicates),
|
||||
}
|
||||
)
|
||||
for filehash in dup_hashes:
|
||||
original = originals_by_hash.get(filehash)
|
||||
duplicates = duplicates_by_hash.get(filehash, [])
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"filehash": filehash,
|
||||
"original": _file_record_to_dict(original) if original else None,
|
||||
"duplicates": [_file_record_to_dict(d) for d in duplicates],
|
||||
"duplicate_count": len(duplicates),
|
||||
}
|
||||
)
|
||||
|
||||
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
|
||||
|
||||
|
||||
+95
-39
@@ -11,6 +11,7 @@ import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, List, Optional
|
||||
|
||||
import aiofiles
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import asc, desc
|
||||
@@ -345,7 +346,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
|
||||
try:
|
||||
# Find all file records
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
|
||||
query = apply_owner_filter(query, request)
|
||||
file_records = query.all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
@@ -384,7 +387,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
"""
|
||||
try:
|
||||
# Find all file records
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
|
||||
query = apply_owner_filter(query, request)
|
||||
file_records = query.all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
@@ -456,7 +461,9 @@ def bulk_reprocess_files_cloud_ocr(request: Request, file_ids: List[int], db: Db
|
||||
Useful for re-running OCR on files with poor text quality or missing OCR text.
|
||||
"""
|
||||
try:
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
|
||||
query = apply_owner_filter(query, request)
|
||||
file_records = query.all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
@@ -536,7 +543,9 @@ def bulk_download_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
Files not found on disk are silently skipped.
|
||||
"""
|
||||
try:
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
query = db.query(FileRecord).filter(FileRecord.id.in_(file_ids))
|
||||
query = apply_owner_filter(query, request)
|
||||
file_records = query.all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
@@ -618,7 +627,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
||||
"""
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
file_record = query.first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
@@ -674,7 +685,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
|
||||
"""
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
file_record = query.first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
@@ -937,7 +950,9 @@ def retry_subtask(
|
||||
"""
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
file_record = query.first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
@@ -1079,7 +1094,9 @@ def get_file_preview(
|
||||
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
file_record = query.first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
@@ -1159,7 +1176,9 @@ def download_file(
|
||||
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
file_record = query.first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
@@ -1217,6 +1236,66 @@ def download_file(
|
||||
raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")
|
||||
|
||||
|
||||
async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: int) -> int:
|
||||
"""Save an uploaded file in chunks and enforce the maximum size limit."""
|
||||
try:
|
||||
written_size = 0
|
||||
with open(target_path, "wb") as f:
|
||||
chunk_size = 65536 # 64 KB chunks
|
||||
while True:
|
||||
chunk = await file.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
written_size += len(chunk)
|
||||
if written_size > max_size:
|
||||
# Exceeded limit mid-stream; clean up and reject
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during upload. "
|
||||
f"See SECURITY_AUDIT.md for configuration details.",
|
||||
)
|
||||
f.write(chunk)
|
||||
return written_size
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
if os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
|
||||
|
||||
|
||||
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
|
||||
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
|
||||
if not settings.enable_deduplication:
|
||||
return None
|
||||
|
||||
try:
|
||||
filehash = hash_file(target_path)
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
|
||||
return {
|
||||
"duplicate_type": "exact",
|
||||
"original_file_id": existing.id,
|
||||
"original_filename": existing.original_filename,
|
||||
"message": (
|
||||
"This file appears to be an exact duplicate of an already-processed document. "
|
||||
"It will still be queued but will be flagged as a duplicate."
|
||||
),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/ui-upload")
|
||||
@require_login
|
||||
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
|
||||
@@ -1277,7 +1356,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
|
||||
# enforcing the size limit during the read so memory usage stays bounded.
|
||||
try:
|
||||
written_size = 0
|
||||
with open(target_path, "wb") as f:
|
||||
async with aiofiles.open(target_path, "wb") as f:
|
||||
chunk_size = 65536 # 64 KB chunks
|
||||
while True:
|
||||
chunk = await file.read(chunk_size)
|
||||
@@ -1286,14 +1365,14 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
|
||||
written_size += len(chunk)
|
||||
if written_size > max_size:
|
||||
# Exceeded limit mid-stream; clean up and reject
|
||||
f.close()
|
||||
await f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during upload. "
|
||||
f"See SECURITY_AUDIT.md for configuration details.",
|
||||
)
|
||||
f.write(chunk)
|
||||
await f.write(chunk)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -1384,29 +1463,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
|
||||
# Check for exact duplicates (same SHA-256 hash) before returning.
|
||||
# This gives the caller an immediate warning without waiting for the pipeline.
|
||||
# Only performed when deduplication is enabled in settings.
|
||||
exact_duplicate_warning = None
|
||||
if settings.enable_deduplication:
|
||||
try:
|
||||
filehash = hash_file(target_path)
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
exact_duplicate_warning = {
|
||||
"duplicate_type": "exact",
|
||||
"original_file_id": existing.id,
|
||||
"original_filename": existing.original_filename,
|
||||
"message": (
|
||||
"This file appears to be an exact duplicate of an already-processed document. "
|
||||
"It will still be queued but will be flagged as a duplicate."
|
||||
),
|
||||
}
|
||||
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
|
||||
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
|
||||
|
||||
response: dict = {
|
||||
"task_id": task.id,
|
||||
@@ -1458,7 +1515,7 @@ def claim_file(request: Request, file_id: int, db: DbSession):
|
||||
logger.exception(f"Error claiming file {file_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to claim document")
|
||||
|
||||
logger.info(f"File {file_id} claimed by user '{owner_id}'")
|
||||
logger.info("File %d claimed by user", file_id)
|
||||
return {"status": "success", "message": "Document claimed successfully", "file_id": file_id, "owner_id": owner_id}
|
||||
|
||||
|
||||
@@ -1498,7 +1555,7 @@ def bulk_claim_files(request: Request, file_ids: list[int], db: DbSession):
|
||||
logger.exception(f"Error during bulk claim: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to claim documents")
|
||||
|
||||
logger.info(f"Bulk claim by '{owner_id}': claimed={claimed}, skipped={[s['file_id'] for s in skipped]}")
|
||||
logger.info("Bulk claim: claimed=%s, skipped=%s", claimed, [s["file_id"] for s in skipped])
|
||||
return {
|
||||
"status": "success",
|
||||
"claimed_count": len(claimed),
|
||||
@@ -1551,8 +1608,7 @@ def assign_owner(request: Request, db: DbSession, owner_id: str = Query(...), fi
|
||||
logger.exception(f"Error assigning owner: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to assign owner")
|
||||
|
||||
admin_name = get_current_owner_id(request) or "admin"
|
||||
logger.info(f"Admin '{admin_name}' assigned owner_id='{owner_id}' to {updated} file(s)")
|
||||
logger.info("Admin assigned owner to %d file(s)", updated)
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Assigned owner to {updated} document(s)",
|
||||
|
||||
+15
-3
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Dependency to ensure the current user is an admin."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
@router.post("/google-drive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_google_drive_token(
|
||||
@@ -362,9 +373,9 @@ def format_time_remaining(time_delta):
|
||||
|
||||
|
||||
@router.post("/google-drive/save-settings")
|
||||
@require_login
|
||||
async def save_google_drive_settings(
|
||||
request: Request,
|
||||
_admin: AdminUser,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
@@ -404,7 +415,8 @@ async def save_google_drive_settings(
|
||||
drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
|
||||
|
||||
# Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
|
||||
if not update_env_file(env_path, drive_settings):
|
||||
env_write_success = update_env_file(env_path, drive_settings)
|
||||
if not env_write_success:
|
||||
logger.info("Continuing with in-memory update despite .env file update failure or skip")
|
||||
|
||||
# Update the settings in memory (this always happens)
|
||||
@@ -443,7 +455,7 @@ async def save_google_drive_settings(
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Google Drive settings have been saved",
|
||||
"in_memory_only": not os.path.exists(env_path),
|
||||
"in_memory_only": not env_write_success,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
from app.models import UserImapAccount
|
||||
from app.utils.encryption import decrypt_value, encrypt_value
|
||||
from app.utils.network import is_private_ip
|
||||
from app.utils.subscription import get_tier, get_user_tier_id
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
@@ -187,6 +188,11 @@ def _test_imap_connection(host: str, port: int, username: str, password: str, us
|
||||
|
||||
Returns a dict with ``{"success": bool, "message": str}``.
|
||||
"""
|
||||
|
||||
# Security: Prevent SSRF by blocking connections to internal IPs
|
||||
if is_private_ip(host):
|
||||
logger.warning("SSRF blocked: Attempt to connect to private IP %s", host)
|
||||
return {"success": False, "message": "Connection error: Invalid hostname or IP address"}
|
||||
try:
|
||||
if use_ssl:
|
||||
mail = imaplib.IMAP4_SSL(host, port)
|
||||
|
||||
@@ -452,17 +452,16 @@ async def update_preferences(
|
||||
)
|
||||
|
||||
try:
|
||||
# Pre-fetch existing preferences for this user to avoid N+1 queries
|
||||
existing_prefs = (
|
||||
db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
|
||||
)
|
||||
|
||||
# Build a fast lookup dictionary keyed by (event_type, channel_type, target_id)
|
||||
prefs_dict = {(pref.event_type, pref.channel_type, pref.target_id): pref for pref in existing_prefs}
|
||||
|
||||
for item in body.preferences:
|
||||
existing = (
|
||||
db.query(UserNotificationPreference)
|
||||
.filter(
|
||||
UserNotificationPreference.owner_id == owner_id,
|
||||
UserNotificationPreference.event_type == item.event_type,
|
||||
UserNotificationPreference.channel_type == item.channel_type,
|
||||
UserNotificationPreference.target_id == item.target_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
existing = prefs_dict.get((item.event_type, item.channel_type, item.target_id))
|
||||
if existing:
|
||||
existing.is_enabled = item.is_enabled
|
||||
else:
|
||||
|
||||
+46
-61
@@ -7,7 +7,7 @@ import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -24,6 +24,17 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Dependency to ensure the current user is an admin."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
@router.post("/onedrive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_onedrive_token(
|
||||
@@ -92,17 +103,18 @@ async def test_onedrive_token(request: Request):
|
||||
"scope": "offline_access Files.ReadWrite",
|
||||
}
|
||||
|
||||
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
|
||||
response = await client.post(token_url, data=refresh_data)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Failed to refresh OneDrive token: {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Failed to refresh OneDrive token: {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
|
||||
token_data = response.json()
|
||||
token_data = response.json()
|
||||
access_token = token_data.get("access_token")
|
||||
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
|
||||
|
||||
@@ -115,32 +127,7 @@ async def test_onedrive_token(request: Request):
|
||||
settings.onedrive_refresh_token = new_refresh_token
|
||||
|
||||
# Also try to update .env file if it exists
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if os.path.exists(env_path):
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
updated_lines = []
|
||||
updated = False
|
||||
|
||||
for line in env_lines:
|
||||
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
updated = True
|
||||
else:
|
||||
updated_lines.append(line)
|
||||
|
||||
if not updated:
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.writelines(updated_lines)
|
||||
|
||||
logger.info("Updated refresh token in .env file")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update refresh token in .env file: {e}")
|
||||
update_env_file({"ONEDRIVE_REFRESH_TOKEN": new_refresh_token})
|
||||
|
||||
# Persist the rotated refresh token to the database
|
||||
try:
|
||||
@@ -164,17 +151,18 @@ async def test_onedrive_token(request: Request):
|
||||
user_info_url = "https://graph.microsoft.com/v1.0/me"
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
|
||||
async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
|
||||
user_response = await client.get(user_info_url, headers=headers)
|
||||
|
||||
if user_response.status_code != 200:
|
||||
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
|
||||
}
|
||||
if user_response.status_code != 200:
|
||||
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
|
||||
}
|
||||
|
||||
# Get user info
|
||||
user_info = user_response.json()
|
||||
# Get user info
|
||||
user_info = user_response.json()
|
||||
display_name = user_info.get("displayName", "Unknown user")
|
||||
email = user_info.get("userPrincipalName", "Unknown email")
|
||||
|
||||
@@ -227,9 +215,9 @@ def format_time_remaining(time_delta):
|
||||
|
||||
|
||||
@router.post("/onedrive/save-settings")
|
||||
@require_login
|
||||
async def save_onedrive_settings(
|
||||
request: Request,
|
||||
_admin: AdminUser,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
@@ -247,22 +235,19 @@ async def save_onedrive_settings(
|
||||
)
|
||||
|
||||
# Best-effort .env file write
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
|
||||
if client_id:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
|
||||
if tenant_id:
|
||||
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
|
||||
if folder_path:
|
||||
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
|
||||
if client_id:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
|
||||
if tenant_id:
|
||||
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
|
||||
if folder_path:
|
||||
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
|
||||
|
||||
if not update_env_file(env_path, onedrive_settings):
|
||||
logger.info("Continuing with in-memory update despite .env file update failure or skip")
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
if not update_env_file(env_path, onedrive_settings):
|
||||
logger.info("Continuing with in-memory update despite .env file update failure or skip")
|
||||
|
||||
# Update the settings in memory
|
||||
if refresh_token:
|
||||
|
||||
+11
-2
@@ -196,11 +196,20 @@ def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
|
||||
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
|
||||
updated = 0
|
||||
for sort_order, plan_id in enumerate(body.order):
|
||||
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
|
||||
|
||||
# Fetch all requested plans in a single query to avoid N+1
|
||||
plan_ids = body.order
|
||||
plans = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id.in_(plan_ids)).all()
|
||||
|
||||
# Build a map for fast O(1) lookup
|
||||
plan_map = {p.plan_id: p for p in plans}
|
||||
|
||||
for sort_order, plan_id in enumerate(plan_ids):
|
||||
plan = plan_map.get(plan_id)
|
||||
if plan:
|
||||
plan.sort_order = sort_order
|
||||
updated += 1
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
|
||||
@@ -99,6 +99,8 @@ class ProfileResponse(BaseModel):
|
||||
contact_email: str | None
|
||||
preferred_language: str | None
|
||||
preferred_theme: str | None
|
||||
default_document_language: str | None
|
||||
"""ISO 639-1 code for the user's preferred document translation target language."""
|
||||
avatar_url: str
|
||||
"""Gravatar URL or ``data:`` URI for a custom uploaded avatar."""
|
||||
is_local_user: bool
|
||||
@@ -112,6 +114,10 @@ class ProfileUpdateRequest(BaseModel):
|
||||
contact_email: str | None = Field(default=None, max_length=255, description="Contact / notification e-mail")
|
||||
preferred_language: str | None = Field(default=None, description="ISO 639-1 language code, e.g. 'en', 'de'")
|
||||
preferred_theme: str | None = Field(default=None, description="Colour scheme: 'light', 'dark', or 'system'")
|
||||
default_document_language: str | None = Field(
|
||||
default=None,
|
||||
description="ISO 639-1 code for the default document translation target language, e.g. 'en', 'de'",
|
||||
)
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
@@ -149,6 +155,7 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
|
||||
contact_email=profile.contact_email, # type: ignore[arg-type]
|
||||
preferred_language=profile.preferred_language, # type: ignore[arg-type]
|
||||
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
|
||||
default_document_language=profile.default_document_language, # type: ignore[arg-type]
|
||||
avatar_url=avatar_url,
|
||||
is_local_user=is_local,
|
||||
)
|
||||
@@ -201,6 +208,16 @@ async def update_profile(
|
||||
)
|
||||
profile.preferred_theme = theme or None # type: ignore[assignment]
|
||||
|
||||
# Validate default document language
|
||||
if body.default_document_language is not None:
|
||||
doc_lang = body.default_document_language.lower().strip()
|
||||
if doc_lang and doc_lang not in SUPPORTED_LANGUAGE_CODES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unsupported language code: {doc_lang}",
|
||||
)
|
||||
profile.default_document_language = doc_lang or None # type: ignore[assignment]
|
||||
|
||||
if body.display_name is not None:
|
||||
profile.display_name = body.display_name.strip() or None # type: ignore[assignment]
|
||||
|
||||
@@ -225,6 +242,7 @@ async def update_profile(
|
||||
contact_email=profile.contact_email, # type: ignore[arg-type]
|
||||
preferred_language=profile.preferred_language, # type: ignore[arg-type]
|
||||
preferred_theme=profile.preferred_theme, # type: ignore[arg-type]
|
||||
default_document_language=profile.default_document_language, # type: ignore[arg-type]
|
||||
avatar_url=avatar_url,
|
||||
is_local_user=is_local,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
"""QR code login API endpoints for mobile app authentication.
|
||||
|
||||
Provides a secure challenge-response flow for logging into the mobile app
|
||||
by scanning a QR code displayed in the web interface:
|
||||
|
||||
1. **Web user** calls ``POST /qr-auth/challenge`` → receives a time-limited
|
||||
challenge token (encoded in the QR code).
|
||||
2. **Web UI** polls ``GET /qr-auth/challenge/{id}/status`` to detect when
|
||||
the mobile app has claimed the challenge.
|
||||
3. **Mobile app** scans the QR code and calls ``POST /qr-auth/claim`` with
|
||||
the challenge token + device name → receives an API token.
|
||||
|
||||
Security properties:
|
||||
* Challenges expire after a configurable TTL (default 2 minutes).
|
||||
* Single-use: once claimed, a challenge cannot be reused (replay-safe).
|
||||
* Cryptographically random 64-byte tokens.
|
||||
* IP addresses are logged for audit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any
|
||||
|
||||
import segno
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.middleware.audit_log import get_client_ip
|
||||
from app.utils.session_manager import (
|
||||
claim_qr_challenge,
|
||||
create_qr_challenge,
|
||||
get_challenge_status,
|
||||
)
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/qr-auth", tags=["qr-auth"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CreateChallengeResponse(BaseModel):
|
||||
"""Response after creating a QR login challenge."""
|
||||
|
||||
challenge_id: int
|
||||
challenge_token: str
|
||||
expires_at: datetime
|
||||
ttl_seconds: int = Field(description="Seconds until the challenge expires (use for client-side countdown).")
|
||||
qr_payload: str = Field(description="The string to encode in the QR code.")
|
||||
qr_code_svg: str = Field(description="Base64-encoded SVG data URI of the QR code, ready for use in an <img> src.")
|
||||
|
||||
|
||||
class ChallengeStatusResponse(BaseModel):
|
||||
"""Response for polling the status of a QR challenge."""
|
||||
|
||||
id: int
|
||||
status: str # "pending", "claimed", "expired", "cancelled"
|
||||
device_name: str | None = None
|
||||
claimed_at: datetime | None = None
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
class ClaimChallengeRequest(BaseModel):
|
||||
"""Request body for claiming a QR login challenge."""
|
||||
|
||||
challenge_token: str = Field(min_length=1, max_length=256)
|
||||
device_name: str = Field(
|
||||
default="Mobile App",
|
||||
min_length=1,
|
||||
max_length=120,
|
||||
description="Human-readable device name.",
|
||||
)
|
||||
|
||||
|
||||
class ClaimChallengeResponse(BaseModel):
|
||||
"""Response after successfully claiming a QR challenge."""
|
||||
|
||||
token: str
|
||||
token_id: int
|
||||
name: str
|
||||
owner_id: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# QR code rendering parameters
|
||||
_QR_ERROR_LEVEL = "M" # Medium error correction (~15% recovery); sufficient for on-screen display
|
||||
_QR_SCALE = 4 # Each QR module is rendered as 4×4 SVG pixels
|
||||
|
||||
|
||||
def _generate_qr_svg(payload: str) -> str:
|
||||
"""Generate a QR code for *payload* and return it as a base64 SVG data URI.
|
||||
|
||||
Using ``segno`` (pure-Python, no Pillow dependency) and SVG output so the
|
||||
QR code scales crisply at any resolution without requiring a canvas or any
|
||||
client-side JavaScript library.
|
||||
"""
|
||||
qr = segno.make(payload, error=_QR_ERROR_LEVEL)
|
||||
buf = io.BytesIO()
|
||||
qr.save(buf, kind="svg", scale=_QR_SCALE, xmldecl=False, svgclass=None, lineclass=None, omitsize=True)
|
||||
svg_bytes = buf.getvalue()
|
||||
return "data:image/svg+xml;base64," + base64.b64encode(svg_bytes).decode("ascii")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/challenge", status_code=status.HTTP_201_CREATED, response_model=CreateChallengeResponse)
|
||||
@require_login
|
||||
async def create_challenge(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new QR login challenge.
|
||||
|
||||
The returned ``qr_payload`` should be encoded into a QR code and
|
||||
displayed to the user. The mobile app scans this QR code and
|
||||
calls the ``/claim`` endpoint.
|
||||
"""
|
||||
ip = get_client_ip(request)
|
||||
challenge = create_qr_challenge(db, owner_id, ip_address=ip)
|
||||
|
||||
# The QR payload is a JSON-like string with enough info for the mobile
|
||||
# app to know the server URL and challenge token.
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
qr_payload = f"docuelevate://qr-login?token={challenge.challenge_token}&server={base_url}"
|
||||
|
||||
# Compute the TTL in seconds so the client can run a countdown timer
|
||||
# without comparing absolute timestamps (which breaks when client and
|
||||
# server clocks are out of sync).
|
||||
ttl_seconds = max(0, int((challenge.expires_at - challenge.created_at).total_seconds()))
|
||||
|
||||
return {
|
||||
"challenge_id": challenge.id,
|
||||
"challenge_token": challenge.challenge_token,
|
||||
"expires_at": challenge.expires_at,
|
||||
"ttl_seconds": ttl_seconds,
|
||||
"qr_payload": qr_payload,
|
||||
"qr_code_svg": _generate_qr_svg(qr_payload),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/challenge/{challenge_id}/status", response_model=ChallengeStatusResponse)
|
||||
@require_login
|
||||
async def poll_challenge_status(
|
||||
request: Request,
|
||||
challenge_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Poll the status of a QR login challenge.
|
||||
|
||||
The web UI calls this endpoint every few seconds to check if the
|
||||
mobile app has scanned the QR code and claimed the challenge.
|
||||
"""
|
||||
result = get_challenge_status(db, challenge_id, owner_id)
|
||||
if not result:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Challenge not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/claim", response_model=ClaimChallengeResponse)
|
||||
async def claim_challenge(
|
||||
request: Request,
|
||||
body: ClaimChallengeRequest,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Claim a QR login challenge and receive an API token.
|
||||
|
||||
This endpoint is called by the mobile app after scanning a QR code.
|
||||
It does **not** require authentication — the challenge token itself
|
||||
serves as proof that the user authorized this login from their web
|
||||
session.
|
||||
"""
|
||||
ip = get_client_ip(request)
|
||||
result = claim_qr_challenge(db, body.challenge_token, device_name=body.device_name, ip_address=ip)
|
||||
|
||||
if not result:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid, expired, or already claimed challenge.",
|
||||
)
|
||||
|
||||
try:
|
||||
from app.utils.audit_service import record_event
|
||||
|
||||
record_event(
|
||||
db,
|
||||
action="qr_login_claimed",
|
||||
user=result["owner_id"],
|
||||
resource_type="session",
|
||||
ip_address=ip,
|
||||
details={"device_name": body.device_name, "token_id": result["token_id"]},
|
||||
severity="info",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to write QR login audit event", exc_info=True)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,196 @@
|
||||
"""API endpoints for managing user sessions.
|
||||
|
||||
Provides endpoints for listing active sessions, revoking individual sessions,
|
||||
and the "log off everywhere" feature that invalidates all sessions and API
|
||||
tokens across all devices.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.middleware.audit_log import get_client_ip
|
||||
from app.utils.session_manager import (
|
||||
get_session_lifetime_days,
|
||||
list_user_sessions,
|
||||
revoke_all_sessions,
|
||||
revoke_session,
|
||||
)
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/sessions", tags=["sessions"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if not owner_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SessionResponse(BaseModel):
|
||||
"""Serialised user session for the management UI."""
|
||||
|
||||
id: int
|
||||
device_info: str | None
|
||||
ip_address: str | None
|
||||
created_at: datetime
|
||||
last_active_at: datetime
|
||||
expires_at: datetime
|
||||
is_current: bool = False
|
||||
|
||||
|
||||
class SessionListResponse(BaseModel):
|
||||
"""Response for listing active sessions."""
|
||||
|
||||
sessions: list[SessionResponse]
|
||||
session_lifetime_days: int
|
||||
|
||||
|
||||
class RevokeAllResponse(BaseModel):
|
||||
"""Response after revoking all sessions."""
|
||||
|
||||
revoked_count: int
|
||||
message: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/", response_model=SessionListResponse)
|
||||
@require_login
|
||||
async def list_sessions(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""List all active sessions for the current user."""
|
||||
sessions = list_user_sessions(db, owner_id)
|
||||
|
||||
# Determine which session is the current one
|
||||
current_token = request.session.get("_session_token")
|
||||
|
||||
session_list = []
|
||||
for s in sessions:
|
||||
session_list.append(
|
||||
{
|
||||
"id": s.id,
|
||||
"device_info": s.device_info,
|
||||
"ip_address": s.ip_address,
|
||||
"created_at": s.created_at,
|
||||
"last_active_at": s.last_active_at,
|
||||
"expires_at": s.expires_at,
|
||||
"is_current": s.session_token == current_token if current_token else False,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"sessions": session_list,
|
||||
"session_lifetime_days": get_session_lifetime_days(),
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
async def revoke_single_session(
|
||||
request: Request,
|
||||
session_id: int,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> None:
|
||||
"""Revoke a specific session by ID."""
|
||||
success = revoke_session(db, session_id, owner_id)
|
||||
if not success:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found")
|
||||
|
||||
try:
|
||||
from app.utils.audit_service import record_event
|
||||
|
||||
record_event(
|
||||
db,
|
||||
action="session_revoked",
|
||||
user=owner_id,
|
||||
resource_type="session",
|
||||
resource_id=str(session_id),
|
||||
ip_address=get_client_ip(request),
|
||||
severity="info",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to write session revocation audit event", exc_info=True)
|
||||
|
||||
|
||||
@router.post("/revoke-all", response_model=RevokeAllResponse)
|
||||
@require_login
|
||||
async def revoke_all(
|
||||
request: Request,
|
||||
owner_id: CurrentOwner,
|
||||
db: DbSession,
|
||||
) -> dict[str, Any]:
|
||||
"""Revoke all sessions except the current one ("log off everywhere").
|
||||
|
||||
Also revokes all active API tokens for the user, which invalidates
|
||||
mobile app sessions and any programmatic access.
|
||||
"""
|
||||
# Find current session to preserve it
|
||||
current_token = request.session.get("_session_token")
|
||||
current_session_id = None
|
||||
if current_token:
|
||||
from app.models import UserSession
|
||||
|
||||
current = db.query(UserSession).filter(UserSession.session_token == current_token).first()
|
||||
if current:
|
||||
current_session_id = current.id
|
||||
|
||||
count = revoke_all_sessions(
|
||||
db,
|
||||
owner_id,
|
||||
except_session_id=current_session_id,
|
||||
revoke_api_tokens=True,
|
||||
)
|
||||
|
||||
try:
|
||||
from app.utils.audit_service import record_event
|
||||
|
||||
record_event(
|
||||
db,
|
||||
action="revoke_all_sessions",
|
||||
user=owner_id,
|
||||
resource_type="session",
|
||||
ip_address=get_client_ip(request),
|
||||
details={"revoked_count": count},
|
||||
severity="warning",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to write revoke-all audit event", exc_info=True)
|
||||
|
||||
return {
|
||||
"revoked_count": count,
|
||||
"message": f"Successfully revoked {count} session(s) and all API tokens.",
|
||||
}
|
||||
@@ -313,16 +313,18 @@ async def list_shared_links(
|
||||
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all shared links created by the authenticated user."""
|
||||
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
|
||||
q = (
|
||||
db.query(SharedLink, FileRecord.original_filename)
|
||||
.outerjoin(FileRecord, SharedLink.file_id == FileRecord.id)
|
||||
.filter(SharedLink.owner_id == owner_id)
|
||||
)
|
||||
if active_only:
|
||||
q = q.filter(SharedLink.is_active.is_(True))
|
||||
links = q.order_by(SharedLink.created_at.desc()).all()
|
||||
links_with_filenames = q.order_by(SharedLink.created_at.desc()).all()
|
||||
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
result = []
|
||||
for link in links:
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
|
||||
filename = file_record.original_filename if file_record else None
|
||||
for link, filename in links_with_filenames:
|
||||
result.append(_link_to_dict(link, base_url, filename))
|
||||
return result
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
System reset API endpoints for DocuElevate.
|
||||
|
||||
Provides admin-only REST endpoints for:
|
||||
- Full system reset (wipe all user data)
|
||||
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
|
||||
|
||||
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
|
||||
admin privileges.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
def _require_feature_enabled() -> None:
|
||||
"""Raise 404 when the factory-reset feature flag is off."""
|
||||
if not settings.enable_factory_reset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
|
||||
)
|
||||
|
||||
|
||||
class ResetRequest(BaseModel):
|
||||
"""Body for system reset endpoints. Requires explicit confirmation."""
|
||||
|
||||
confirmation: str
|
||||
|
||||
|
||||
@router.post("/full")
|
||||
async def full_reset(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Wipe all user data (database + work-files).
|
||||
|
||||
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "DELETE":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_full_reset
|
||||
|
||||
try:
|
||||
result = perform_full_reset(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Full system reset failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"System reset failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.post("/reimport")
|
||||
async def reset_and_reimport(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, and
|
||||
configure the reimport folder as a watch folder for automatic
|
||||
re-ingestion.
|
||||
|
||||
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "REIMPORT":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_reset_and_reimport
|
||||
|
||||
try:
|
||||
result = perform_reset_and_reimport(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Reset-and-reimport failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Reset and reimport failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def reset_status(_admin: AdminUser) -> dict:
|
||||
"""Return whether the system reset feature is enabled."""
|
||||
return {
|
||||
"enabled": settings.enable_factory_reset,
|
||||
"factory_reset_on_startup": settings.factory_reset_on_startup,
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
"""
|
||||
API endpoints for document translation.
|
||||
|
||||
Provides on-the-fly translation via the AI provider and access to the
|
||||
persisted default-language translation.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
from app.utils.user_scope import apply_owner_filter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# Maximum characters sent to the AI provider for a single translation request.
|
||||
_MAX_TRANSLATION_INPUT = 50_000
|
||||
|
||||
|
||||
def _get_file_or_404(db: Session, file_id: int, request: Request) -> FileRecord:
|
||||
"""Fetch a FileRecord visible to the current user or raise 404."""
|
||||
query = db.query(FileRecord).filter(FileRecord.id == file_id)
|
||||
query = apply_owner_filter(query, request)
|
||||
record = query.first()
|
||||
if not record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
return record
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/translation/default")
|
||||
@require_login
|
||||
def get_default_translation(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
) -> JSONResponse:
|
||||
"""Return the persisted default-language translation for a document.
|
||||
|
||||
Returns 404 if no default-language translation has been generated yet
|
||||
(e.g. because the document is already in the default language).
|
||||
"""
|
||||
record = _get_file_or_404(db, file_id, request)
|
||||
|
||||
if not record.default_language_text:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="No default-language translation available for this file",
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"file_id": record.id,
|
||||
"detected_language": record.detected_language,
|
||||
"default_language_code": record.default_language_code,
|
||||
"text": record.default_language_text,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/translate")
|
||||
@require_login
|
||||
def translate_on_the_fly(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
lang: str = Query(..., min_length=2, max_length=10, description="Target language ISO 639-1 code"),
|
||||
) -> JSONResponse:
|
||||
"""Translate a document's extracted text into an arbitrary language on the fly.
|
||||
|
||||
The translation is generated via the configured AI provider and is **not**
|
||||
persisted. For the default-language translation, use the
|
||||
``/files/{file_id}/translation/default`` endpoint instead.
|
||||
"""
|
||||
record = _get_file_or_404(db, file_id, request)
|
||||
|
||||
source_text = record.ocr_text
|
||||
if not source_text:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="No extracted text available for this file — translation requires OCR text",
|
||||
)
|
||||
|
||||
# If the requested language matches what is already stored, return it directly.
|
||||
if record.default_language_code and lang == record.default_language_code and record.default_language_text:
|
||||
return JSONResponse(
|
||||
content={
|
||||
"file_id": record.id,
|
||||
"source_language": record.detected_language,
|
||||
"target_language": lang,
|
||||
"text": record.default_language_text,
|
||||
"cached": True,
|
||||
}
|
||||
)
|
||||
|
||||
# If the detected language already matches, return the original text.
|
||||
detected = record.detected_language
|
||||
if detected and detected == lang:
|
||||
return JSONResponse(
|
||||
content={
|
||||
"file_id": record.id,
|
||||
"source_language": detected,
|
||||
"target_language": lang,
|
||||
"text": source_text,
|
||||
"cached": True,
|
||||
}
|
||||
)
|
||||
|
||||
# Truncate to keep AI costs bounded.
|
||||
text_to_translate = source_text[:_MAX_TRANSLATION_INPUT]
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
translated = provider.chat_completion(
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
f"You are a professional translator. Translate the following text "
|
||||
f"into {lang}. Preserve the original formatting, paragraph structure, "
|
||||
f"and meaning. Do not add any commentary — output ONLY the translated text."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": text_to_translate},
|
||||
],
|
||||
model=model,
|
||||
temperature=0.3,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception(f"On-the-fly translation failed for file {file_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="Translation failed — the AI provider returned an error",
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"file_id": record.id,
|
||||
"source_language": detected or "unknown",
|
||||
"target_language": lang,
|
||||
"text": translated,
|
||||
"cached": False,
|
||||
}
|
||||
)
|
||||
+63
-56
@@ -9,7 +9,8 @@ import urllib.parse
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
import aiofiles
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, HttpUrl, field_validator
|
||||
|
||||
@@ -153,67 +154,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
logger.info(f"Downloading file from URL: {url}")
|
||||
|
||||
# Use configured timeout to prevent hanging
|
||||
response = requests.get(
|
||||
url,
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_request_timeout,
|
||||
stream=True, # Stream to handle large files
|
||||
allow_redirects=True, # Follow redirects
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": "DocuElevate/1.0", # Identify ourselves
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
if "." in safe_filename:
|
||||
file_extension = safe_filename.rsplit(".", 1)[1]
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
with open(target_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
|
||||
# Check for extension using original_filename to avoid any CodeQL issues
|
||||
# with safe_filename which is derived from the URL directly.
|
||||
if "." in original_filename:
|
||||
_, ext = os.path.splitext(original_filename)
|
||||
# Strip out the leading dot and any non-alphanumeric chars
|
||||
clean_ext = "".join(c for c in ext if c.isalnum())
|
||||
if not clean_ext:
|
||||
clean_ext = "bin"
|
||||
target_filename = f"{unique_id}.{clean_ext}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
async with aiofiles.open(target_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=8192):
|
||||
if chunk:
|
||||
await f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
await f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
)
|
||||
|
||||
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
|
||||
|
||||
# Enqueue for processing
|
||||
@@ -227,19 +234,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
"size": downloaded_size,
|
||||
}
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
except httpx.TimeoutException:
|
||||
logger.error(f"Timeout while downloading file from URL: {url}")
|
||||
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
except httpx.ConnectError as e:
|
||||
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
except httpx.RequestError as e:
|
||||
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user