Merge remote-tracking branch 'origin/main' into copilot/fix-mobile-app-login

2026-03-16 11:58:14 +00:00
parent 9a256741d5 0e72a965d5
commit 8cc817930d
22 changed files with 1016 additions and 261 deletions
@@ -20,14 +20,16 @@ logger = logging.getLogger(__name__)

 router = APIRouter()

-DbSession = Annotated[Session, Depends(get_db)]
+# Module-level dependency singleton to satisfy Ruff B008 while maintaining default values for manual calls (e.g. in decorators).
+_db_dep = Depends(get_db)
+DbSession = Annotated[Session, _db_dep]


@router.get("/audit-logs")
@require_login
 async def list_audit_logs(
    request: Request,
-    db: DbSession,
+    db: DbSession = _db_dep,
    action: Annotated[str | None, Query(description="Filter by action (exact match)")] = None,
    user: Annotated[str | None, Query(description="Filter by username")] = None,
    resource_type: Annotated[str | None, Query(description="Filter by resource type")] = None,
@@ -73,7 +75,7 @@ async def list_audit_logs(
@require_login
 async def list_distinct_actions(
    request: Request,
-    db: DbSession,
+    db: DbSession = _db_dep,
 ) -> list[str]:
    """Return the distinct action values present in the audit log."""
    from app.models import AuditLog
@@ -86,7 +88,7 @@ async def list_distinct_actions(
@require_login
 async def list_distinct_users(
    request: Request,
-    db: DbSession,
+    db: DbSession = _db_dep,
 ) -> list[str]:
    """Return the distinct user values present in the audit log."""
    from app.models import AuditLog
@@ -1218,6 +1218,66 @@ def download_file(
        raise HTTPException(status_code=500, detail=f"Error downloading file: {str(e)}")


+async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size: int) -> int:
+    """Save an uploaded file in chunks and enforce the maximum size limit."""
+    try:
+        written_size = 0
+        with open(target_path, "wb") as f:
+            chunk_size = 65536  # 64 KB chunks
+            while True:
+                chunk = await file.read(chunk_size)
+                if not chunk:
+                    break
+                written_size += len(chunk)
+                if written_size > max_size:
+                    # Exceeded limit mid-stream; clean up and reject
+                    f.close()
+                    os.remove(target_path)
+                    raise HTTPException(
+                        status_code=413,
+                        detail=f"File too large: exceeded {max_size} bytes during upload. "
+                        f"See SECURITY_AUDIT.md for configuration details.",
+                    )
+                f.write(chunk)
+        return written_size
+    except HTTPException:
+        raise
+    except Exception as e:
+        if os.path.exists(target_path):
+            os.remove(target_path)
+        raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
+
+
+def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
+    """Check for an exact duplicate of the uploaded file and return a warning if found."""
+    if not settings.enable_deduplication:
+        return None
+
+    try:
+        filehash = hash_file(target_path)
+        existing = (
+            db.query(FileRecord)
+            .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
+            .order_by(FileRecord.id.asc())
+            .first()
+        )
+        if existing:
+            logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
+            return {
+                "duplicate_type": "exact",
+                "original_file_id": existing.id,
+                "original_filename": existing.original_filename,
+                "message": (
+                    "This file appears to be an exact duplicate of an already-processed document. "
+                    "It will still be queued but will be flagged as a duplicate."
+                ),
+            }
+    except Exception as e:
+        logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
+
+    return None
+
+
@router.post("/ui-upload")
@require_login
 async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
@@ -1385,29 +1445,7 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
    # Check for exact duplicates (same SHA-256 hash) before returning.
    # This gives the caller an immediate warning without waiting for the pipeline.
    # Only performed when deduplication is enabled in settings.
-    exact_duplicate_warning = None
-    if settings.enable_deduplication:
-        try:
-            filehash = hash_file(target_path)
-            existing = (
-                db.query(FileRecord)
-                .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
-                .order_by(FileRecord.id.asc())
-                .first()
-            )
-            if existing:
-                exact_duplicate_warning = {
-                    "duplicate_type": "exact",
-                    "original_file_id": existing.id,
-                    "original_filename": existing.original_filename,
-                    "message": (
-                        "This file appears to be an exact duplicate of an already-processed document. "
-                        "It will still be queued but will be flagged as a duplicate."
-                    ),
-                }
-                logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
-        except Exception as e:
-            logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
+    exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)

    response: dict = {
        "task_id": task.id,
@@ -6,7 +6,7 @@ import logging
 from datetime import datetime, timedelta
 from typing import Annotated, Optional

-import requests
+import httpx
 from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
 from sqlalchemy.orm import Session

@@ -92,17 +92,18 @@ async def test_onedrive_token(request: Request):
            "scope": "offline_access Files.ReadWrite",
        }

-        response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
+        async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
+            response = await client.post(token_url, data=refresh_data)

-        if response.status_code != 200:
-            logger.error(f"Failed to refresh OneDrive token: {response.text}")
-            return {
-                "status": "error",
-                "message": "Refresh token has expired or is invalid",
-                "needs_reauth": True,
-            }
+            if response.status_code != 200:
+                logger.error(f"Failed to refresh OneDrive token: {response.text}")
+                return {
+                    "status": "error",
+                    "message": "Refresh token has expired or is invalid",
+                    "needs_reauth": True,
+                }

-        token_data = response.json()
+            token_data = response.json()
        access_token = token_data.get("access_token")
        expires_in = token_data.get("expires_in", 3600)  # Default to 1 hour if not specified

@@ -139,17 +140,18 @@ async def test_onedrive_token(request: Request):
        user_info_url = "https://graph.microsoft.com/v1.0/me"
        headers = {"Authorization": f"Bearer {access_token}"}

-        user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
+        async with httpx.AsyncClient(timeout=settings.http_request_timeout) as client:
+            user_response = await client.get(user_info_url, headers=headers)

-        if user_response.status_code != 200:
-            logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
-            return {
-                "status": "error",
-                "message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
-            }
+            if user_response.status_code != 200:
+                logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
+                return {
+                    "status": "error",
+                    "message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
+                }

-        # Get user info
-        user_info = user_response.json()
+            # Get user info
+            user_info = user_response.json()
        display_name = user_info.get("displayName", "Unknown user")
        email = user_info.get("userPrincipalName", "Unknown email")

@@ -9,7 +9,8 @@ import urllib.parse
 import uuid
 from typing import Optional

-import requests
+import aiofiles
+import httpx
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, HttpUrl, field_validator

@@ -153,67 +154,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
        logger.info(f"Downloading file from URL: {url}")

        # Use configured timeout to prevent hanging
-        response = requests.get(
-            url,
+        async with httpx.AsyncClient(
            timeout=settings.http_request_timeout,
-            stream=True,  # Stream to handle large files
-            allow_redirects=True,  # Follow redirects
+            follow_redirects=True,
            headers={
                "User-Agent": "DocuElevate/1.0",  # Identify ourselves
            },
-        )
-        response.raise_for_status()
+        ) as client:
+            async with client.stream("GET", url) as response:
+                response.raise_for_status()

-        # Validate content type
-        content_type = response.headers.get("Content-Type", "")
-        if not validate_file_type(content_type, safe_filename):
-            raise HTTPException(
-                status_code=400,
-                detail=f"Unsupported file type: {content_type}. "
-                "Supported types: PDF, Office documents, images, plain text",
-            )
+                # Validate content type
+                content_type = response.headers.get("Content-Type", "")
+                if not validate_file_type(content_type, safe_filename):
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Unsupported file type: {content_type}. "
+                        "Supported types: PDF, Office documents, images, plain text",
+                    )

-        # Check content length before downloading
-        content_length = response.headers.get("Content-Length")
-        if content_length:
-            file_size = int(content_length)
-            max_size = settings.max_upload_size
-            if file_size > max_size:
-                raise HTTPException(
-                    status_code=413,
-                    detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
-                )
-
-        # Generate unique filename
-        unique_id = str(uuid.uuid4())
-        if "." in safe_filename:
-            file_extension = safe_filename.rsplit(".", 1)[1]
-            target_filename = f"{unique_id}.{file_extension}"
-        else:
-            target_filename = unique_id
-
-        target_path = os.path.join(settings.workdir, target_filename)
-
-        # Download file in chunks to handle large files
-        downloaded_size = 0
-        max_size = settings.max_upload_size
-
-        with open(target_path, "wb") as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:
-                    f.write(chunk)
-                    downloaded_size += len(chunk)
-
-                    # Check size during download
-                    if downloaded_size > max_size:
-                        # Remove partial file
-                        f.close()
-                        os.remove(target_path)
+                # Check content length before downloading
+                content_length = response.headers.get("Content-Length")
+                if content_length:
+                    file_size = int(content_length)
+                    max_size = settings.max_upload_size
+                    if file_size > max_size:
                        raise HTTPException(
                            status_code=413,
-                            detail=f"File too large: exceeded {max_size} bytes during download",
+                            detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
                        )

+                # Generate unique filename
+                unique_id = str(uuid.uuid4())
+
+                # Check for extension using original_filename to avoid any CodeQL issues
+                # with safe_filename which is derived from the URL directly.
+                if "." in original_filename:
+                    _, ext = os.path.splitext(original_filename)
+                    # Strip out the leading dot and any non-alphanumeric chars
+                    clean_ext = "".join(c for c in ext if c.isalnum())
+                    if not clean_ext:
+                        clean_ext = "bin"
+                    target_filename = f"{unique_id}.{clean_ext}"
+                else:
+                    target_filename = unique_id
+
+                target_path = os.path.join(settings.workdir, target_filename)
+
+                # Download file in chunks to handle large files
+                downloaded_size = 0
+                max_size = settings.max_upload_size
+
+                async with aiofiles.open(target_path, "wb") as f:
+                    async for chunk in response.aiter_bytes(chunk_size=8192):
+                        if chunk:
+                            await f.write(chunk)
+                            downloaded_size += len(chunk)
+
+                            # Check size during download
+                            if downloaded_size > max_size:
+                                # Remove partial file
+                                await f.close()
+                                os.remove(target_path)
+                                raise HTTPException(
+                                    status_code=413,
+                                    detail=f"File too large: exceeded {max_size} bytes during download",
+                                )
+
        logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")

        # Enqueue for processing
@@ -227,19 +234,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
            "size": downloaded_size,
        }

-    except requests.exceptions.Timeout:
+    except httpx.TimeoutException:
        logger.error(f"Timeout while downloading file from URL: {url}")
        raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")

-    except requests.exceptions.ConnectionError as e:
+    except httpx.ConnectError as e:
        logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")

-    except requests.exceptions.HTTPError as e:
+    except httpx.HTTPStatusError as e:
        logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")

-    except requests.exceptions.RequestException as e:
+    except httpx.RequestError as e:
        logger.error(f"Error downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")

@@ -24,6 +24,15 @@ logger = logging.getLogger(__name__)
 # Constants
 _LOGO_FILENAME = "logo.png"

+# Mapping from PDF metadata keys (with leading slash stripped) to application-specific names.
+# This mirrors the inverse of the mapping used in app/tasks/embed_metadata_into_pdf.py.
+_PDF_METADATA_KEY_MAP = {
+    "Title": "filename",
+    "Author": "absender",
+    "Subject": "document_type",
+    "Keywords": "tags",
+}
+

 def get_email_template(template_name="default.html"):
    """
@@ -64,9 +73,12 @@ def extract_metadata_from_file(file_path):
    """
    Try to extract metadata from a file using several methods:
    1. Check for a .json metadata file with the same name
-    2. Extract metadata from PDF if it's embedded
+    2. Extract embedded metadata from PDF using pypdf

-    Returns a dictionary of metadata or None if not found
+    JSON metadata takes precedence; embedded PDF metadata fills in any missing
+    fields using the application's standard key mapping (e.g., /Title → filename).
+
+    Returns a dictionary of metadata (may be empty if none found).
    """
    metadata = {}

@@ -77,7 +89,6 @@ def extract_metadata_from_file(file_path):
            with open(metadata_path, "r", encoding="utf-8") as f:
                metadata = json.load(f)
                logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
-                return metadata
        except Exception as e:
            logger.warning(f"Failed to load metadata from JSON file: {str(e)}")

@@ -88,11 +99,14 @@ def extract_metadata_from_file(file_path):
                pdf_reader = pypdf.PdfReader(f)
                pdf_metadata = pdf_reader.metadata
                if pdf_metadata:
-                    # Convert metadata to a standard dictionary
                    for key, value in pdf_metadata.items():
                        # Remove the leading slash from PDF metadata keys (e.g., '/Title' -> 'Title')
                        clean_key = key[1:] if key.startswith("/") else key
-                        metadata[clean_key] = str(value)
+                        # Map to application-specific key names where possible
+                        mapped_key = _PDF_METADATA_KEY_MAP.get(clean_key, clean_key)
+                        # Only set if not already present (JSON metadata takes precedence)
+                        if mapped_key not in metadata:
+                            metadata[mapped_key] = str(value)

                    logger.info(f"Extracted embedded metadata from PDF: {file_path}")
        except Exception as e: