gh-christianlouis-docuelevate/app/api/url_upload.py

"""
API endpoint for processing files from URLs
"""

import ipaddress
import logging
import mimetypes
import os
import urllib.parse
import uuid
from typing import Optional

import requests
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, HttpUrl, validator

from app.auth import require_login
from app.config import settings
from app.tasks.process_document import process_document
from app.utils.filename_utils import sanitize_filename

# Set up logging
logger = logging.getLogger(__name__)

router = APIRouter()


class URLUploadRequest(BaseModel):
    """Request model for URL-based file upload"""

    url: HttpUrl
    filename: Optional[str] = None

    @validator("url")
    def validate_url_scheme(cls, v):
        """Ensure only HTTP/HTTPS schemes are allowed"""
        parsed = urllib.parse.urlparse(str(v))
        if parsed.scheme not in ["http", "https"]:
            raise ValueError("Only HTTP and HTTPS URLs are allowed")
        return v


def is_private_ip(hostname: str) -> bool:
    """
    Check if a hostname resolves to a private/internal IP address.
    Protects against SSRF attacks by blocking access to internal networks.
    """
    try:
        # Try to parse as IP address directly
        ip = ipaddress.ip_address(hostname)
        return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
    except ValueError:
        # Not a direct IP, try to resolve hostname
        try:
            import socket

            # Get all IP addresses for this hostname
            addr_info = socket.getaddrinfo(hostname, None)
            for info in addr_info:
                ip_str = info[4][0]
                ip = ipaddress.ip_address(ip_str)
                # Block if ANY resolved IP is private/internal
                if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
                    return True
            return False
        except (socket.gaierror, socket.error):
            # Cannot resolve - allow for testing/development
            # In production, DNS should work properly
            # Log this for debugging
            logger.warning(f"Could not resolve hostname: {hostname}")
            return False  # Changed from True to False to allow external domains in tests


def validate_url_safety(url: str) -> None:
    """
    Validate that URL is safe to fetch (SSRF protection).

    Raises:
        HTTPException: If URL is unsafe
    """
    parsed = urllib.parse.urlparse(url)

    # Check scheme
    if parsed.scheme not in ["http", "https"]:
        raise HTTPException(status_code=400, detail="Only HTTP and HTTPS URLs are supported")

    # Check hostname exists
    if not parsed.hostname:
        raise HTTPException(status_code=400, detail="Invalid URL: no hostname")

    # Block private/internal IPs (SSRF protection)
    if is_private_ip(parsed.hostname):
        raise HTTPException(
            status_code=400,
            detail="Access to private/internal IP addresses is not allowed for security reasons",
        )

    # Block well-known metadata endpoints (cloud provider SSRF)
    metadata_endpoints = [
        "169.254.169.254",  # AWS, Azure, GCP metadata
        "metadata.google.internal",  # GCP
        "169.254.169.253",  # AWS link-local
    ]
    if parsed.hostname in metadata_endpoints:
        raise HTTPException(status_code=400, detail="Access to cloud metadata endpoints is not allowed")


def validate_file_type(content_type: str, filename: str) -> bool:
    """
    Validate that the file type is supported.

    Args:
        content_type: MIME type from response headers
        filename: Filename to check extension

    Returns:
        True if file type is allowed
    """
    # Same allowed types as regular upload
    ALLOWED_MIME_TYPES = {
        "application/pdf",
        "application/msword",
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        "application/vnd.ms-excel",
        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        "application/vnd.ms-powerpoint",
        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
        "text/plain",
        "text/csv",
        "application/rtf",
        "text/rtf",
    }

    IMAGE_MIME_TYPES = {
        "image/jpeg",
        "image/jpg",
        "image/png",
        "image/gif",
        "image/bmp",
        "image/tiff",
        "image/webp",
        "image/svg+xml",
    }

    # Check content type from header
    if content_type:
        # Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
        base_content_type = content_type.split(";")[0].strip().lower()
        if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES:
            return True

    # Also check by extension as fallback
    _, ext = os.path.splitext(filename)
    if ext:
        guessed_type, _ = mimetypes.guess_type(filename)
        if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES):
            return True

    return False


@router.post("/process-url")
@require_login
async def process_url(request: URLUploadRequest):
    """
    Download a file from a URL and enqueue it for processing.

    Security features:
    - SSRF protection: blocks private IPs, localhost, cloud metadata endpoints
    - File type validation: only allows supported document/image types
    - File size limits: enforces maximum upload size
    - Timeout protection: prevents hanging on slow/malicious servers

    Args:
        request: URLUploadRequest with url and optional filename

    Returns:
        JSON with task_id and status

    Raises:
        HTTPException: If URL is invalid, unsafe, or file cannot be processed
    """
    url = str(request.url)

    # Validate URL safety (SSRF protection)
    validate_url_safety(url)

    # Parse URL to extract filename if not provided
    if request.filename:
        original_filename = request.filename
    else:
        # Extract filename from URL path
        parsed = urllib.parse.urlparse(url)
        path = parsed.path
        original_filename = os.path.basename(path) if path else "download"

    # Sanitize filename
    safe_filename = sanitize_filename(original_filename)
    if not safe_filename:
        safe_filename = "download"

    # Download file with security measures
    # Initialize target_path to None to prevent UnboundLocalError in exception handlers
    # that may execute before target_path is assigned during error cases
    target_path = None
    try:
        logger.info(f"Downloading file from URL: {url}")

        # Use configured timeout to prevent hanging
        response = requests.get(
            url,
            timeout=settings.http_request_timeout,
            stream=True,  # Stream to handle large files
            allow_redirects=True,  # Follow redirects
            headers={
                "User-Agent": "DocuElevate/1.0",  # Identify ourselves
            },
        )
        response.raise_for_status()

        # Validate content type
        content_type = response.headers.get("Content-Type", "")
        if not validate_file_type(content_type, safe_filename):
            raise HTTPException(
                status_code=400,
                detail=f"Unsupported file type: {content_type}. "
                "Supported types: PDF, Office documents, images, plain text",
            )

        # Check content length before downloading
        content_length = response.headers.get("Content-Length")
        if content_length:
            file_size = int(content_length)
            max_size = settings.max_upload_size
            if file_size > max_size:
                raise HTTPException(
                    status_code=413,
                    detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
                )

        # Generate unique filename
        unique_id = str(uuid.uuid4())
        if "." in safe_filename:
            file_extension = safe_filename.rsplit(".", 1)[1]
            target_filename = f"{unique_id}.{file_extension}"
        else:
            target_filename = unique_id

        target_path = os.path.join(settings.workdir, target_filename)

        # Download file in chunks to handle large files
        downloaded_size = 0
        max_size = settings.max_upload_size

        with open(target_path, "wb") as f:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)
                    downloaded_size += len(chunk)

                    # Check size during download
                    if downloaded_size > max_size:
                        # Remove partial file
                        f.close()
                        os.remove(target_path)
                        raise HTTPException(
                            status_code=413,
                            detail=f"File too large: exceeded {max_size} bytes during download",
                        )

        logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")

        # Enqueue for processing
        task = process_document.delay(target_path, original_filename=safe_filename)

        return {
            "task_id": task.id,
            "status": "queued",
            "message": f"File downloaded from URL and queued for processing",
            "filename": safe_filename,
            "size": downloaded_size,
        }

    except requests.exceptions.Timeout:
        logger.error(f"Timeout while downloading file from URL: {url}")
        raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")

    except requests.exceptions.ConnectionError as e:
        logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")

    except requests.exceptions.HTTPError as e:
        logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")

    except requests.exceptions.RequestException as e:
        logger.error(f"Error downloading file from URL: {url} - {str(e)}")
        raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")

    except HTTPException:
        # Re-raise FastAPI HTTPExceptions (validation errors, file too large, etc.)
        raise

    except OSError as e:
        logger.error(f"Error saving file from URL: {url} - {str(e)}")
        # Clean up partial file if it exists
        if target_path and os.path.exists(target_path):
            os.remove(target_path)
        raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")

    except Exception as e:
        logger.exception(f"Unexpected error processing URL: {url}")
        # Clean up partial file if it exists
        if target_path and os.path.exists(target_path):
            os.remove(target_path)
        raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")