gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into refactor/audit-logs-annotated-b008-6104828608171783200

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-03-16 12:10:32 +01:00
committed by GitHub
parent 8ad90d7da9 0f312160bc
commit e10f0bff42
193 changed files with 189978 additions and 42248 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/dropbox.py
+49 -46
View File
@@ -6,7 +6,7 @@ import logging
import os
from typing import Annotated, Optional
import requests
import httpx
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from sqlalchemy.orm import Session
@@ -132,57 +132,60 @@ async def test_dropbox_token(request: Request):
"message": "Dropbox credentials are not fully configured",
}
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = requests.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = requests.post(
async with httpx.AsyncClient() as client:
# Check token validity by getting current account info
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# If token is invalid, try refreshing it
if response.status_code == 401:
logger.info("Dropbox access token invalid or expired, trying to refresh")
# Get account info
account_info = response.json()
# Get a new access token using the refresh token
refresh_url = "https://api.dropbox.com/oauth2/token"
refresh_data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
refresh_response = await client.post(
refresh_url, data=refresh_data, timeout=settings.http_request_timeout
)
if refresh_response.status_code != 200:
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
return {
"status": "error",
"message": "Refresh token has expired or is invalid",
"needs_reauth": True,
}
token_info = refresh_response.json()
access_token = token_info.get("access_token")
# Try again with the new access token
headers = {"Authorization": f"Bearer {access_token}"}
response = await client.post(
"https://api.dropboxapi.com/2/users/get_current_account",
headers=headers,
timeout=settings.http_request_timeout,
)
if response.status_code != 200:
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
return {
"status": "error",
"message": f"Token validation failed with status {response.status_code}: {response.text}",
}
# Get account info
account_info = response.json()
account_email = account_info.get("email", "Unknown account")
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
app/api/duplicates.py
+28 -23
View File
@@ -73,33 +73,38 @@ def list_duplicate_groups(
groups = []
total_duplicate_files = 0
for filehash in dup_hashes:
# Find the original (non-duplicate) record with this hash
original = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.id.asc())
.first()
if dup_hashes:
# Fetch all matching files (both original and duplicates) in a single batch query
all_records = (
db.query(FileRecord).filter(FileRecord.filehash.in_(dup_hashes)).order_by(FileRecord.id.asc()).all()
)
# Find all duplicate records for this hash
duplicates = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
.order_by(FileRecord.id.asc())
.all()
)
# Group records by hash
originals_by_hash = {}
duplicates_by_hash = {h: [] for h in dup_hashes}
total_duplicate_files += len(duplicates)
for record in all_records:
h = record.filehash
if not record.is_duplicate:
# Store only the first original record per hash, matching the old .first() behaviour
if h not in originals_by_hash:
originals_by_hash[h] = record
else:
duplicates_by_hash[h].append(record)
total_duplicate_files += 1
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
for filehash in dup_hashes:
original = originals_by_hash.get(filehash)
duplicates = duplicates_by_hash.get(filehash, [])
groups.append(
{
"filehash": filehash,
"original": _file_record_to_dict(original) if original else None,
"duplicates": [_file_record_to_dict(d) for d in duplicates],
"duplicate_count": len(duplicates),
}
)
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
app/api/files.py
+4 -3
View File
@@ -11,6 +11,7 @@ import zipfile
from datetime import datetime, timezone
from typing import Annotated, List, Optional
import aiofiles
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status
from fastapi.responses import StreamingResponse
from sqlalchemy import asc, desc
@@ -1277,7 +1278,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
# enforcing the size limit during the read so memory usage stays bounded.
try:
written_size = 0
with open(target_path, "wb") as f:
async with aiofiles.open(target_path, "wb") as f:
chunk_size = 65536 # 64 KB chunks
while True:
chunk = await file.read(chunk_size)
@@ -1286,14 +1287,14 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
written_size += len(chunk)
if written_size > max_size:
# Exceeded limit mid-stream; clean up and reject
f.close()
await f.close()
os.remove(target_path)
raise HTTPException(
status_code=413,
detail=f"File too large: exceeded {max_size} bytes during upload. "
f"See SECURITY_AUDIT.md for configuration details.",
)
f.write(chunk)
await f.write(chunk)
except HTTPException:
raise
except Exception as e:
app/api/google_drive.py
+1 -1
View File
@@ -363,7 +363,7 @@ def format_time_remaining(time_delta):
@router.post("/google-drive/save-settings")
@require_login
async def save_dropbox_settings(
async def save_google_drive_settings(
request: Request,
refresh_token: Annotated[str, Form(...)],
client_id: Annotated[Optional[str], Form()] = None,
app/api/integrations.py
+4 -9
View File
@@ -564,7 +564,6 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str
return {"success": False, "message": "Missing required field: url"}
# Only allow http/https to prevent file:// or other custom scheme attacks
import ipaddress
from urllib.parse import urlparse
parsed = urlparse(url)
@@ -574,14 +573,10 @@ def _test_webdav_connection(config: dict[str, Any] | None, credentials: dict[str
# Block requests to private/internal IPs to prevent SSRF
hostname = parsed.hostname or ""
if hostname:
try:
addr = ipaddress.ip_address(hostname)
if addr.is_private or addr.is_loopback or addr.is_link_local:
return {"success": False, "message": "URLs pointing to internal or private networks are not allowed"}
except ValueError:
# Hostname is not an IP literal — allow DNS names through
if hostname in ("localhost", "localhost.localdomain"):
return {"success": False, "message": "URLs pointing to localhost are not allowed"}
from app.utils.network import is_private_ip
if is_private_ip(hostname):
return {"success": False, "message": "URLs pointing to internal or private networks are not allowed"}
try:
import base64
app/api/notifications.py
+9 -10
View File
@@ -452,17 +452,16 @@ async def update_preferences(
)
try:
# Pre-fetch existing preferences for this user to avoid N+1 queries
existing_prefs = (
db.query(UserNotificationPreference).filter(UserNotificationPreference.owner_id == owner_id).all()
)
# Build a fast lookup dictionary keyed by (event_type, channel_type, target_id)
prefs_dict = {(pref.event_type, pref.channel_type, pref.target_id): pref for pref in existing_prefs}
for item in body.preferences:
existing = (
db.query(UserNotificationPreference)
.filter(
UserNotificationPreference.owner_id == owner_id,
UserNotificationPreference.event_type == item.event_type,
UserNotificationPreference.channel_type == item.channel_type,
UserNotificationPreference.target_id == item.target_id,
)
.first()
)
existing = prefs_dict.get((item.event_type, item.channel_type, item.target_id))
if existing:
existing.is_enabled = item.is_enabled
else:
app/api/onedrive.py
+19 -93
View File
@@ -3,7 +3,6 @@ OneDrive API endpoints
"""
import logging
import os
from datetime import datetime, timedelta
from typing import Annotated, Optional
@@ -14,6 +13,7 @@ from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.utils.env_utils import update_env_file
from app.utils.oauth_helper import exchange_oauth_token
from app.utils.settings_service import save_setting_to_db
from app.utils.settings_sync import notify_settings_updated
@@ -115,32 +115,7 @@ async def test_onedrive_token(request: Request):
settings.onedrive_refresh_token = new_refresh_token
# Also try to update .env file if it exists
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if os.path.exists(env_path):
with open(env_path, "r") as f:
env_lines = f.readlines()
updated_lines = []
updated = False
for line in env_lines:
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
updated = True
else:
updated_lines.append(line)
if not updated:
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
with open(env_path, "w") as f:
f.writelines(updated_lines)
logger.info("Updated refresh token in .env file")
except Exception as e:
logger.warning(f"Failed to update refresh token in .env file: {e}")
update_env_file({"ONEDRIVE_REFRESH_TOKEN": new_refresh_token})
# Persist the rotated refresh token to the database
try:
@@ -246,75 +221,26 @@ async def save_onedrive_settings(
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
)
# Best-effort .env file write
try:
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if not os.path.exists(env_path):
logger.warning(f".env file not found at {env_path}, skipping file write")
else:
logger.info(f"Updating OneDrive settings in {env_path}")
# Build settings dictionary mapped to database/memory keys
onedrive_settings = {
"onedrive_refresh_token": refresh_token,
"onedrive_client_id": client_id,
"onedrive_client_secret": client_secret,
"onedrive_tenant_id": tenant_id,
"onedrive_folder_path": folder_path,
}
with open(env_path, "r") as f:
env_lines = f.readlines()
# Filter out None values
onedrive_settings = {k: v for k, v in onedrive_settings.items() if v is not None}
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
# Best-effort .env file write using the new utility
env_settings = {k.upper(): v for k, v in onedrive_settings.items()}
update_env_file(env_settings)
updated = set()
new_env_lines = []
for line in env_lines:
stripped_line = line.rstrip()
is_updated = False
for key, value in onedrive_settings.items():
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
new_env_lines.append(f"{key}={value}")
updated.add(key)
is_updated = True
break
if not is_updated:
new_env_lines.append(stripped_line)
for key, value in onedrive_settings.items():
if key not in updated:
new_env_lines.append(f"{key}={value}")
with open(env_path, "w") as f:
f.write("\n".join(new_env_lines) + "\n")
logger.info("Successfully updated OneDrive settings in .env file")
except Exception as env_err:
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
# Update the settings in memory
if refresh_token:
settings.onedrive_refresh_token = refresh_token
if client_id:
settings.onedrive_client_id = client_id
if client_secret:
settings.onedrive_client_secret = client_secret
if tenant_id:
settings.onedrive_tenant_id = tenant_id
if folder_path:
settings.onedrive_folder_path = folder_path
# Persist to database (primary)
if refresh_token:
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
if client_id:
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
if client_secret:
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
if tenant_id:
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
if folder_path:
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
# Update in-memory settings and persist to database dynamically
for key, value in onedrive_settings.items():
setattr(settings, key, value)
save_setting_to_db(db, key, value, changed_by=changed_by)
notify_settings_updated()
app/api/plans.py
+11 -2
View File
@@ -196,11 +196,20 @@ def seed_plans(db: DbSession, _admin: AdminUser) -> dict[str, Any]:
def reorder_plans(body: ReorderBody, db: DbSession, _admin: AdminUser) -> dict[str, Any]:
"""Update sort_order for each plan_id in *body.order* (position = index in list)."""
updated = 0
for sort_order, plan_id in enumerate(body.order):
plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id == plan_id).first()
# Fetch all requested plans in a single query to avoid N+1
plan_ids = body.order
plans = db.query(SubscriptionPlan).filter(SubscriptionPlan.plan_id.in_(plan_ids)).all()
# Build a map for fast O(1) lookup
plan_map = {p.plan_id: p for p in plans}
for sort_order, plan_id in enumerate(plan_ids):
plan = plan_map.get(plan_id)
if plan:
plan.sort_order = sort_order
updated += 1
try:
db.commit()
except Exception:
app/api/profile.py
+22 -2
View File
@@ -18,7 +18,7 @@ import logging
from hashlib import md5
from typing import Annotated
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile, status
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -156,7 +156,9 @@ async def get_profile(request: Request, db: DbSession) -> ProfileResponse:
@router.patch("", response_model=ProfileResponse)
@require_login
async def update_profile(body: ProfileUpdateRequest, request: Request, db: DbSession) -> ProfileResponse:
async def update_profile(
body: ProfileUpdateRequest, request: Request, response: Response, db: DbSession
) -> ProfileResponse:
"""Update the current user's editable profile settings."""
user_id = _get_user_id(request)
profile = _get_or_create_profile(db, user_id)
@@ -171,6 +173,24 @@ async def update_profile(body: ProfileUpdateRequest, request: Request, db: DbSes
)
profile.preferred_language = lang or None # type: ignore[assignment]
# Keep session and cookie in sync so detect_language() picks up
# the new preference immediately (without a DB round-trip).
if hasattr(request, "session"):
if lang:
request.session["preferred_language"] = lang
else:
request.session.pop("preferred_language", None)
if lang:
response.set_cookie(
key="docuelevate_lang",
value=lang,
max_age=30 * 24 * 60 * 60,
httponly=False,
samesite="lax",
)
else:
response.delete_cookie(key="docuelevate_lang")
# Validate theme
if body.preferred_theme is not None:
theme = body.preferred_theme.lower().strip()
app/api/shared_links.py
+7 -5
View File
@@ -313,16 +313,18 @@ async def list_shared_links(
active_only: bool = Query(False, description="When true, only return active (non-revoked) links"),
) -> list[dict[str, Any]]:
"""List all shared links created by the authenticated user."""
q = db.query(SharedLink).filter(SharedLink.owner_id == owner_id)
q = (
db.query(SharedLink, FileRecord.original_filename)
.outerjoin(FileRecord, SharedLink.file_id == FileRecord.id)
.filter(SharedLink.owner_id == owner_id)
)
if active_only:
q = q.filter(SharedLink.is_active.is_(True))
links = q.order_by(SharedLink.created_at.desc()).all()
links_with_filenames = q.order_by(SharedLink.created_at.desc()).all()
base_url = str(request.base_url).rstrip("/")
result = []
for link in links:
file_record = db.query(FileRecord).filter(FileRecord.id == link.file_id).first()
filename = file_record.original_filename if file_record else None
for link, filename in links_with_filenames:
result.append(_link_to_dict(link, base_url, filename))
return result
app/api/url_upload.py
+1 -32
View File
@@ -2,7 +2,6 @@
API endpoint for processing files from URLs
"""
import ipaddress
import logging
import mimetypes
import os
@@ -19,6 +18,7 @@ from app.config import settings
from app.tasks.process_document import process_document
from app.utils.allowed_types import ALLOWED_MIME_TYPES
from app.utils.filename_utils import sanitize_filename
from app.utils.network import is_private_ip
# Set up logging
logger = logging.getLogger(__name__)
@@ -42,37 +42,6 @@ class URLUploadRequest(BaseModel):
return v
def is_private_ip(hostname: str) -> bool:
"""
Check if a hostname resolves to a private/internal IP address.
Protects against SSRF attacks by blocking access to internal networks.
"""
try:
# Try to parse as IP address directly
ip = ipaddress.ip_address(hostname)
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
except ValueError:
# Not a direct IP, try to resolve hostname
try:
import socket
# Get all IP addresses for this hostname
addr_info = socket.getaddrinfo(hostname, None)
for info in addr_info:
ip_str = info[4][0]
ip = ipaddress.ip_address(ip_str)
# Block if ANY resolved IP is private/internal
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
return True
return False
except (socket.gaierror, socket.error):
# Cannot resolve - allow for testing/development
# In production, DNS should work properly
# Log this for debugging
logger.warning(f"Could not resolve hostname: {hostname}")
return False # Changed from True to False to allow external domains in tests
def validate_url_safety(url: str) -> None:
"""
Validate that URL is safe to fetch (SSRF protection).