Add security fixes, testing infrastructure, and CI/CD improvements
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,41 @@
|
|||||||
|
name: "CodeQL Security Scanning"
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ "main", "develop" ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ "main", "develop" ]
|
||||||
|
schedule:
|
||||||
|
- cron: '0 0 * * 1' # Run every Monday at midnight
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
analyze:
|
||||||
|
name: Analyze
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
contents: read
|
||||||
|
security-events: write
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
language: [ 'python', 'javascript' ]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Initialize CodeQL
|
||||||
|
uses: github/codeql-action/init@v2
|
||||||
|
with:
|
||||||
|
languages: ${{ matrix.language }}
|
||||||
|
queries: security-and-quality
|
||||||
|
|
||||||
|
- name: Autobuild
|
||||||
|
uses: github/codeql-action/autobuild@v2
|
||||||
|
|
||||||
|
- name: Perform CodeQL Analysis
|
||||||
|
uses: github/codeql-action/analyze@v2
|
||||||
|
with:
|
||||||
|
category: "/language:${{matrix.language}}"
|
||||||
@@ -17,24 +17,40 @@ jobs:
|
|||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip
|
python -m pip install --upgrade pip
|
||||||
pip install -r requirements.txt
|
pip install -r requirements-dev.txt
|
||||||
pip install pytest flake8 black mypy pylint
|
|
||||||
|
|
||||||
# - name: Run Tests
|
- name: Run Tests
|
||||||
# run: pytest tests/
|
run: pytest tests/ -v --cov=app --cov-report=xml --cov-report=term
|
||||||
|
|
||||||
|
- name: Upload Coverage to Codecov
|
||||||
|
uses: codecov/codecov-action@v3
|
||||||
|
with:
|
||||||
|
file: ./coverage.xml
|
||||||
|
fail_ci_if_error: false
|
||||||
|
|
||||||
- name: Run Linter (Flake8)
|
- name: Run Linter (Flake8)
|
||||||
run: flake8 app/
|
run: flake8 app/ --max-line-length=120 --extend-ignore=E203,W503
|
||||||
continue-on-error: true
|
continue-on-error: false
|
||||||
|
|
||||||
- name: Run Code Formatter (Black)
|
- name: Run Code Formatter (Black)
|
||||||
run: black --check app/
|
run: black --check app/ --line-length=120
|
||||||
continue-on-error: true
|
continue-on-error: false
|
||||||
|
|
||||||
- name: Run Type Checker (Mypy)
|
- name: Run Type Checker (Mypy)
|
||||||
run: mypy app/
|
run: mypy app/ --ignore-missing-imports
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Run Linter (Pylint)
|
- name: Run Linter (Pylint)
|
||||||
run: pylint app/
|
run: pylint app/ --max-line-length=120 --disable=C0111,C0103,R0903
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Run Security Linter (Bandit)
|
||||||
|
run: bandit -r app/ -ll -f json -o bandit-report.json
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Upload Bandit Report
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
if: always()
|
||||||
|
with:
|
||||||
|
name: bandit-report
|
||||||
|
path: bandit-report.json
|
||||||
|
|||||||
+27
-2
@@ -25,7 +25,34 @@ share/python-wheels/
|
|||||||
.installed.cfg
|
.installed.cfg
|
||||||
*.egg
|
*.egg
|
||||||
MANIFEST
|
MANIFEST
|
||||||
|
|
||||||
|
# Environment files - NEVER commit these!
|
||||||
.env
|
.env
|
||||||
|
.env.local
|
||||||
|
.env.*.local
|
||||||
|
*.env
|
||||||
|
|
||||||
|
# Secrets and credentials
|
||||||
|
*secret*
|
||||||
|
*credentials*.json
|
||||||
|
!frontend/static/* # Allow static files even if they match patterns
|
||||||
|
!docs/* # Allow documentation files
|
||||||
|
|
||||||
|
# Private keys
|
||||||
|
*.pem
|
||||||
|
*.key
|
||||||
|
*.p12
|
||||||
|
*.pfx
|
||||||
|
id_rsa*
|
||||||
|
ssh_host_*
|
||||||
|
|
||||||
|
# Database files - may contain sensitive data
|
||||||
|
*.db
|
||||||
|
*.sqlite
|
||||||
|
*.sqlite3
|
||||||
|
database.db
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
# PyInstaller
|
# PyInstaller
|
||||||
# Usually these files are written by a python script from a template
|
# Usually these files are written by a python script from a template
|
||||||
@@ -59,8 +86,6 @@ cover/
|
|||||||
# Django stuff:
|
# Django stuff:
|
||||||
*.log
|
*.log
|
||||||
local_settings.py
|
local_settings.py
|
||||||
db.sqlite3
|
|
||||||
db.sqlite3-journal
|
|
||||||
|
|
||||||
# Flask stuff:
|
# Flask stuff:
|
||||||
instance/
|
instance/
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
# Pre-commit hooks for code quality and security
|
||||||
|
# Install: pip install pre-commit
|
||||||
|
# Setup: pre-commit install
|
||||||
|
# Run manually: pre-commit run --all-files
|
||||||
|
|
||||||
|
repos:
|
||||||
|
# General file checks
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.5.0
|
||||||
|
hooks:
|
||||||
|
- id: trailing-whitespace
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
- id: check-yaml
|
||||||
|
- id: check-json
|
||||||
|
- id: check-added-large-files
|
||||||
|
args: ['--maxkb=1000']
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: detect-private-key
|
||||||
|
- id: detect-aws-credentials
|
||||||
|
args: ['--allow-missing-credentials']
|
||||||
|
|
||||||
|
# Python code formatting
|
||||||
|
- repo: https://github.com/psf/black
|
||||||
|
rev: 24.1.1
|
||||||
|
hooks:
|
||||||
|
- id: black
|
||||||
|
args: ['--line-length=120']
|
||||||
|
language_version: python3.11
|
||||||
|
|
||||||
|
# Import sorting
|
||||||
|
- repo: https://github.com/PyCQA/isort
|
||||||
|
rev: 5.13.2
|
||||||
|
hooks:
|
||||||
|
- id: isort
|
||||||
|
args: ['--profile=black', '--line-length=120']
|
||||||
|
|
||||||
|
# Linting
|
||||||
|
- repo: https://github.com/PyCQA/flake8
|
||||||
|
rev: 7.0.0
|
||||||
|
hooks:
|
||||||
|
- id: flake8
|
||||||
|
args: ['--max-line-length=120', '--extend-ignore=E203,W503']
|
||||||
|
|
||||||
|
# Security linting
|
||||||
|
- repo: https://github.com/PyCQA/bandit
|
||||||
|
rev: 1.7.6
|
||||||
|
hooks:
|
||||||
|
- id: bandit
|
||||||
|
args: ['-ll', '-r', 'app/']
|
||||||
|
exclude: 'tests/'
|
||||||
|
|
||||||
|
# Type checking
|
||||||
|
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||||
|
rev: v1.8.0
|
||||||
|
hooks:
|
||||||
|
- id: mypy
|
||||||
|
args: ['--ignore-missing-imports']
|
||||||
|
additional_dependencies: ['types-requests']
|
||||||
|
|
||||||
|
# Secret detection
|
||||||
|
- repo: https://github.com/Yelp/detect-secrets
|
||||||
|
rev: v1.4.0
|
||||||
|
hooks:
|
||||||
|
- id: detect-secrets
|
||||||
|
args: ['--baseline', '.secrets.baseline']
|
||||||
|
exclude: |
|
||||||
|
(?x)^(
|
||||||
|
.+\.lock|
|
||||||
|
.+\.json|
|
||||||
|
.env.demo
|
||||||
|
)$
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
# Security Audit Report
|
||||||
|
|
||||||
|
**Date:** 2026-02-06
|
||||||
|
**Status:** Completed Initial Assessment
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
This document tracks security vulnerabilities found in DocuElevate and their remediation status.
|
||||||
|
|
||||||
|
## Critical Vulnerabilities (Fixed) ✅
|
||||||
|
|
||||||
|
### 1. Outdated Authlib with Known Vulnerabilities
|
||||||
|
**Status:** ✅ FIXED
|
||||||
|
**Severity:** HIGH
|
||||||
|
**Description:** Authlib version 1.3.2 had two critical vulnerabilities:
|
||||||
|
- CVE: Denial of Service via Oversized JOSE Segments
|
||||||
|
- CVE: JWS/JWT accepts unknown crit headers (RFC violation → possible authz bypass)
|
||||||
|
|
||||||
|
**Fix:** Updated `requirements.txt` to require `authlib>=1.6.5`
|
||||||
|
|
||||||
|
### 2. Starlette DoS Vulnerability
|
||||||
|
**Status:** ✅ FIXED
|
||||||
|
**Severity:** MEDIUM
|
||||||
|
**Description:** Starlette 0.41.3 vulnerable to O(n^2) DoS via Range header merging in `FileResponse`
|
||||||
|
|
||||||
|
**Fix:** Updated `requirements.txt` to require `starlette>=0.49.1`
|
||||||
|
|
||||||
|
### 3. Weak SESSION_SECRET Default
|
||||||
|
**Status:** ✅ FIXED
|
||||||
|
**Severity:** HIGH
|
||||||
|
**Description:** Default SESSION_SECRET value in `app/main.py` was a predictable string that could be exploited if not overridden
|
||||||
|
|
||||||
|
**Fix:**
|
||||||
|
- Enhanced validation in `app/main.py` to raise error if auth is enabled without proper secret
|
||||||
|
- Updated default to be clearly marked as insecure for development only
|
||||||
|
- Added generation instructions in error message
|
||||||
|
|
||||||
|
## Medium Risk Issues (Fixed) ✅
|
||||||
|
|
||||||
|
### 4. Insufficient .gitignore Protection
|
||||||
|
**Status:** ✅ FIXED
|
||||||
|
**Severity:** MEDIUM
|
||||||
|
**Description:** .gitignore didn't adequately protect against accidentally committing sensitive files (credentials, private keys, secrets)
|
||||||
|
|
||||||
|
**Fix:** Enhanced `.gitignore` with comprehensive patterns for:
|
||||||
|
- Various environment file formats
|
||||||
|
- Credential JSON files
|
||||||
|
- Private keys (.pem, .key, .pfx, etc.)
|
||||||
|
- SSH keys
|
||||||
|
- Explicit exclusion of patterns where needed
|
||||||
|
|
||||||
|
## Best Practices Implemented
|
||||||
|
|
||||||
|
### Dependency Management
|
||||||
|
- ✅ Version pinning for security-critical packages (authlib, starlette)
|
||||||
|
- ✅ Advisory database checks integrated into development workflow
|
||||||
|
- ⏳ TODO: Add automated dependency vulnerability scanning in CI/CD
|
||||||
|
|
||||||
|
### Authentication & Secrets
|
||||||
|
- ✅ Strong validation for SESSION_SECRET (minimum 32 characters)
|
||||||
|
- ✅ Error-on-missing for critical security settings when auth enabled
|
||||||
|
- ✅ Clear documentation of secret generation methods
|
||||||
|
- ✅ .env.demo file for configuration examples (no real secrets)
|
||||||
|
|
||||||
|
### Configuration Security
|
||||||
|
- ✅ All secrets loaded from environment variables
|
||||||
|
- ✅ No hardcoded credentials in codebase
|
||||||
|
- ✅ Proper masking in configuration validators
|
||||||
|
|
||||||
|
## Ongoing Security Measures
|
||||||
|
|
||||||
|
### CI/CD Security
|
||||||
|
- ⏳ **TODO:** Add CodeQL scanning to GitHub Actions
|
||||||
|
- ⏳ **TODO:** Add Bandit (Python security linter) to CI pipeline
|
||||||
|
- ⏳ **TODO:** Add dependency vulnerability scanning (Safety, pip-audit)
|
||||||
|
- ⏳ **TODO:** Make security scans blocking (fail on critical issues)
|
||||||
|
|
||||||
|
### Code Security
|
||||||
|
- ⏳ **TODO:** Implement rate limiting on API endpoints
|
||||||
|
- ⏳ **TODO:** Add CSRF protection for state-changing operations
|
||||||
|
- ⏳ **TODO:** Implement request size limits
|
||||||
|
- ⏳ **TODO:** Add input sanitization for all user inputs
|
||||||
|
- ⏳ **TODO:** Implement proper API key rotation mechanisms
|
||||||
|
|
||||||
|
### Infrastructure Security
|
||||||
|
- ✅ TrustedHostMiddleware configured
|
||||||
|
- ✅ ProxyHeadersMiddleware for reverse proxy setup
|
||||||
|
- ⏳ **TODO:** Add security headers (HSTS, CSP, X-Frame-Options)
|
||||||
|
- ⏳ **TODO:** Implement proper CORS configuration
|
||||||
|
- ⏳ **TODO:** Add request logging with sensitive data masking
|
||||||
|
|
||||||
|
## Recommendations
|
||||||
|
|
||||||
|
### High Priority
|
||||||
|
1. **Enable CodeQL scanning** - Automated security vulnerability detection
|
||||||
|
2. **Implement rate limiting** - Prevent abuse and DoS attacks
|
||||||
|
3. **Add comprehensive input validation** - Prevent injection attacks
|
||||||
|
4. **Implement API authentication** - Secure all API endpoints properly
|
||||||
|
|
||||||
|
### Medium Priority
|
||||||
|
1. **Add security headers** - Improve browser-side security
|
||||||
|
2. **Implement audit logging** - Track security-relevant events
|
||||||
|
3. **Add automated security testing** - Integration with CI/CD
|
||||||
|
4. **Document security architecture** - Security design decisions
|
||||||
|
|
||||||
|
### Low Priority
|
||||||
|
1. **Security training documentation** - For contributors
|
||||||
|
2. **Penetration testing** - Professional security assessment
|
||||||
|
3. **Bug bounty program** - Community security contributions
|
||||||
|
|
||||||
|
## Security Contact
|
||||||
|
|
||||||
|
For security issues, please follow the guidelines in [SECURITY.md](SECURITY.md).
|
||||||
|
|
||||||
|
## Audit History
|
||||||
|
|
||||||
|
| Date | Auditor | Scope | Critical Issues | Status |
|
||||||
|
|------|---------|-------|-----------------|--------|
|
||||||
|
| 2026-02-06 | Automated Agent | Dependencies, Auth, Config | 3 | Fixed |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Next Audit Due:** 2026-05-06 (Quarterly)
|
||||||
+5
-4
@@ -27,10 +27,11 @@ from app.views.files import router as files_router
|
|||||||
|
|
||||||
# Load configuration from .env for the session key
|
# Load configuration from .env for the session key
|
||||||
config = Config(".env")
|
config = Config(".env")
|
||||||
SESSION_SECRET = config(
|
# Use settings.session_secret which has proper validation
|
||||||
"SESSION_SECRET",
|
# Fallback to raising an error if not set when auth is enabled
|
||||||
default="YOUR_DEFAULT_SESSION_SECRET_MUST_BE_32_CHARS_OR_MORE"
|
if settings.auth_enabled and not settings.session_secret:
|
||||||
)
|
raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True. Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'")
|
||||||
|
SESSION_SECRET = settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
|
||||||
|
|
||||||
app = FastAPI(title="DocuElevate")
|
app = FastAPI(title="DocuElevate")
|
||||||
|
|
||||||
|
|||||||
+64
@@ -0,0 +1,64 @@
|
|||||||
|
[tool:pytest]
|
||||||
|
# Pytest configuration
|
||||||
|
testpaths = tests
|
||||||
|
python_files = test_*.py
|
||||||
|
python_classes = Test*
|
||||||
|
python_functions = test_*
|
||||||
|
|
||||||
|
# Output options
|
||||||
|
addopts =
|
||||||
|
--verbose
|
||||||
|
--strict-markers
|
||||||
|
--strict-config
|
||||||
|
--cov=app
|
||||||
|
--cov-report=term-missing
|
||||||
|
--cov-report=html
|
||||||
|
--cov-report=xml
|
||||||
|
--cov-branch
|
||||||
|
--cov-fail-under=0
|
||||||
|
# Note: Coverage threshold set to 0 initially, should be increased gradually
|
||||||
|
# Target: 80% coverage for production code
|
||||||
|
|
||||||
|
# Markers for organizing tests
|
||||||
|
markers =
|
||||||
|
unit: Unit tests for individual functions/methods
|
||||||
|
integration: Integration tests for API endpoints and workflows
|
||||||
|
slow: Tests that take significant time to run
|
||||||
|
security: Security-related tests
|
||||||
|
requires_external: Tests requiring external services (OpenAI, Azure, etc.)
|
||||||
|
requires_db: Tests requiring database
|
||||||
|
requires_redis: Tests requiring Redis
|
||||||
|
|
||||||
|
# Ignore patterns
|
||||||
|
norecursedirs =
|
||||||
|
.git
|
||||||
|
.tox
|
||||||
|
dist
|
||||||
|
build
|
||||||
|
*.egg
|
||||||
|
__pycache__
|
||||||
|
.venv
|
||||||
|
venv
|
||||||
|
env
|
||||||
|
|
||||||
|
# Coverage options
|
||||||
|
[coverage:run]
|
||||||
|
source = app
|
||||||
|
omit =
|
||||||
|
*/tests/*
|
||||||
|
*/test_*.py
|
||||||
|
*/__pycache__/*
|
||||||
|
*/venv/*
|
||||||
|
*/env/*
|
||||||
|
*/.venv/*
|
||||||
|
|
||||||
|
[coverage:report]
|
||||||
|
exclude_lines =
|
||||||
|
pragma: no cover
|
||||||
|
def __repr__
|
||||||
|
raise AssertionError
|
||||||
|
raise NotImplementedError
|
||||||
|
if __name__ == .__main__.:
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
@abstractmethod
|
||||||
|
@abc.abstractmethod
|
||||||
@@ -1 +1,26 @@
|
|||||||
|
# Development and testing dependencies
|
||||||
|
-r requirements.txt
|
||||||
|
|
||||||
|
# Testing
|
||||||
|
pytest>=8.0.0
|
||||||
|
pytest-cov>=4.1.0
|
||||||
|
pytest-asyncio>=0.23.0
|
||||||
|
pytest-mock>=3.12.0
|
||||||
|
httpx>=0.26.0 # For async test client
|
||||||
|
|
||||||
|
# Code quality
|
||||||
|
flake8>=7.0.0
|
||||||
|
black>=24.0.0
|
||||||
|
mypy>=1.8.0
|
||||||
|
pylint>=3.0.0
|
||||||
|
isort>=5.13.0
|
||||||
|
|
||||||
|
# Security scanning
|
||||||
|
bandit>=1.7.6
|
||||||
|
safety>=3.0.0
|
||||||
|
|
||||||
|
# Pre-commit hooks
|
||||||
|
pre-commit>=3.6.0
|
||||||
|
|
||||||
|
# License compliance
|
||||||
pip-licenses==5.0.0 # For license compliance checking
|
pip-licenses==5.0.0 # For license compliance checking
|
||||||
+2
-2
@@ -9,9 +9,9 @@ PyPDF2>=3.0.0 # PDF processing for text extraction, metadata editing and rotati
|
|||||||
requests # HTTP client
|
requests # HTTP client
|
||||||
dropbox>=11.36.0 # Dropbox integration
|
dropbox>=11.36.0 # Dropbox integration
|
||||||
azure-ai-documentintelligence # Azure OCR service
|
azure-ai-documentintelligence # Azure OCR service
|
||||||
authlib # Authentication
|
authlib>=1.6.5 # Authentication - fixed security vulnerabilities (GHSA-xxx)
|
||||||
python-dotenv # Environment variables
|
python-dotenv # Environment variables
|
||||||
starlette # ASGI toolkit (used by FastAPI)
|
starlette>=0.49.1 # ASGI toolkit (used by FastAPI) - fixed DoS vulnerability
|
||||||
alembic # Database migrations
|
alembic # Database migrations
|
||||||
|
|
||||||
# Google Drive API
|
# Google Drive API
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
"""
|
||||||
|
Pytest configuration and shared fixtures for DocuElevate tests.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import pytest
|
||||||
|
from typing import Generator
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
|
||||||
|
# Set test environment variables before importing app
|
||||||
|
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
|
||||||
|
os.environ["REDIS_URL"] = "redis://localhost:6379/1"
|
||||||
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
||||||
|
os.environ["AZURE_AI_KEY"] = "test-key"
|
||||||
|
os.environ["AZURE_REGION"] = "test"
|
||||||
|
os.environ["AZURE_ENDPOINT"] = "https://test.cognitiveservices.azure.com/"
|
||||||
|
os.environ["GOTENBERG_URL"] = "http://localhost:3000"
|
||||||
|
os.environ["WORKDIR"] = "/tmp"
|
||||||
|
os.environ["AUTH_ENABLED"] = "False"
|
||||||
|
os.environ["SESSION_SECRET"] = "test_secret_key_for_testing_must_be_at_least_32_characters_long"
|
||||||
|
|
||||||
|
from app.database import Base, get_db
|
||||||
|
from app.main import app as fastapi_app
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def test_workdir() -> Generator[str, None, None]:
|
||||||
|
"""Create a temporary work directory for tests."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
yield tmpdir
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def db_session():
|
||||||
|
"""Create a fresh database session for each test."""
|
||||||
|
# Create an in-memory SQLite database
|
||||||
|
engine = create_engine(
|
||||||
|
"sqlite:///:memory:",
|
||||||
|
connect_args={"check_same_thread": False},
|
||||||
|
poolclass=StaticPool,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create all tables
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
# Create a session
|
||||||
|
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
session = TestingSessionLocal()
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield session
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
Base.metadata.drop_all(bind=engine)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def client(db_session) -> TestClient:
|
||||||
|
"""Create a test client with a fresh database."""
|
||||||
|
|
||||||
|
# Override the get_db dependency to use our test database
|
||||||
|
def override_get_db():
|
||||||
|
try:
|
||||||
|
yield db_session
|
||||||
|
finally:
|
||||||
|
pass
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[get_db] = override_get_db
|
||||||
|
|
||||||
|
with TestClient(fastapi_app) as test_client:
|
||||||
|
yield test_client
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sample_pdf_path(test_workdir) -> str:
|
||||||
|
"""Create a sample PDF file for testing."""
|
||||||
|
pdf_path = os.path.join(test_workdir, "test.pdf")
|
||||||
|
|
||||||
|
# Create a minimal valid PDF
|
||||||
|
pdf_content = b"""%PDF-1.4
|
||||||
|
1 0 obj
|
||||||
|
<<
|
||||||
|
/Type /Catalog
|
||||||
|
/Pages 2 0 R
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
2 0 obj
|
||||||
|
<<
|
||||||
|
/Type /Pages
|
||||||
|
/Kids [3 0 R]
|
||||||
|
/Count 1
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
3 0 obj
|
||||||
|
<<
|
||||||
|
/Type /Page
|
||||||
|
/Parent 2 0 R
|
||||||
|
/MediaBox [0 0 612 792]
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
xref
|
||||||
|
0 4
|
||||||
|
0000000000 65535 f
|
||||||
|
0000000009 00000 n
|
||||||
|
0000000058 00000 n
|
||||||
|
0000000115 00000 n
|
||||||
|
trailer
|
||||||
|
<<
|
||||||
|
/Size 4
|
||||||
|
/Root 1 0 R
|
||||||
|
>>
|
||||||
|
startxref
|
||||||
|
197
|
||||||
|
%%EOF
|
||||||
|
"""
|
||||||
|
|
||||||
|
with open(pdf_path, 'wb') as f:
|
||||||
|
f.write(pdf_content)
|
||||||
|
|
||||||
|
return pdf_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sample_text_file(test_workdir) -> str:
|
||||||
|
"""Create a sample text file for testing."""
|
||||||
|
text_path = os.path.join(test_workdir, "test.txt")
|
||||||
|
|
||||||
|
with open(text_path, 'w') as f:
|
||||||
|
f.write("This is a test document.\nWith multiple lines.\n")
|
||||||
|
|
||||||
|
return text_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_openai_response():
|
||||||
|
"""Mock OpenAI API response for testing."""
|
||||||
|
return {
|
||||||
|
"choices": [{
|
||||||
|
"message": {
|
||||||
|
"content": '{"document_type": "invoice", "summary": "Test invoice", "tags": ["test", "invoice"]}'
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_azure_response():
|
||||||
|
"""Mock Azure Document Intelligence API response for testing."""
|
||||||
|
return {
|
||||||
|
"analyzeResult": {
|
||||||
|
"content": "Test document content extracted by OCR",
|
||||||
|
"pages": [{"pageNumber": 1}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Markers for categorizing tests
|
||||||
|
def pytest_configure(config):
|
||||||
|
"""Configure custom pytest markers."""
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "unit: Unit tests for individual functions/methods"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "integration: Integration tests for API endpoints and workflows"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "slow: Tests that take significant time to run"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "security: Security-related tests"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "requires_external: Tests requiring external services"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "requires_db: Tests requiring database"
|
||||||
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers", "requires_redis: Tests requiring Redis"
|
||||||
|
)
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""
|
||||||
|
Integration tests for API endpoints.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
class TestHealthEndpoints:
|
||||||
|
"""Tests for health check and status endpoints."""
|
||||||
|
|
||||||
|
def test_root_endpoint(self, client: TestClient):
|
||||||
|
"""Test that root endpoint redirects to UI."""
|
||||||
|
response = client.get("/", follow_redirects=False)
|
||||||
|
assert response.status_code in [200, 307, 308] # OK or redirect
|
||||||
|
|
||||||
|
def test_docs_endpoint(self, client: TestClient):
|
||||||
|
"""Test that API documentation is accessible."""
|
||||||
|
response = client.get("/docs")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "swagger" in response.text.lower() or "openapi" in response.text.lower()
|
||||||
|
|
||||||
|
def test_openapi_schema(self, client: TestClient):
|
||||||
|
"""Test that OpenAPI schema is accessible."""
|
||||||
|
response = client.get("/openapi.json")
|
||||||
|
assert response.status_code == 200
|
||||||
|
schema = response.json()
|
||||||
|
assert "openapi" in schema
|
||||||
|
assert "info" in schema
|
||||||
|
assert schema["info"]["title"] == "DocuElevate"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
class TestFileEndpoints:
|
||||||
|
"""Tests for file management endpoints."""
|
||||||
|
|
||||||
|
def test_list_files_empty(self, client: TestClient):
|
||||||
|
"""Test listing files when database is empty."""
|
||||||
|
response = client.get("/api/files")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert isinstance(data, list)
|
||||||
|
assert len(data) == 0
|
||||||
|
|
||||||
|
def test_get_nonexistent_file(self, client: TestClient):
|
||||||
|
"""Test getting a file that doesn't exist."""
|
||||||
|
response = client.get("/api/files/99999")
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.requires_external
|
||||||
|
class TestProcessingEndpoints:
|
||||||
|
"""Tests for document processing endpoints."""
|
||||||
|
|
||||||
|
def test_process_endpoint_exists(self, client: TestClient):
|
||||||
|
"""Test that process endpoint is registered."""
|
||||||
|
# This will return 422 (validation error) without proper data,
|
||||||
|
# but confirms the endpoint exists
|
||||||
|
response = client.post("/api/process")
|
||||||
|
assert response.status_code in [400, 422] # Bad request or validation error
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
class TestConfigEndpoints:
|
||||||
|
"""Tests for configuration endpoints."""
|
||||||
|
|
||||||
|
def test_config_status_endpoint(self, client: TestClient):
|
||||||
|
"""Test configuration status endpoint if it exists."""
|
||||||
|
# Some apps have a /status or /config/status endpoint
|
||||||
|
response = client.get("/api/status")
|
||||||
|
# Endpoint may not exist, which is fine
|
||||||
|
assert response.status_code in [200, 404]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
class TestAuthEndpoints:
|
||||||
|
"""Tests for authentication endpoints (when auth is disabled in tests)."""
|
||||||
|
|
||||||
|
def test_unauthenticated_access_with_auth_disabled(self, client: TestClient):
|
||||||
|
"""Test that API is accessible when auth is disabled."""
|
||||||
|
# With AUTH_ENABLED=False, API should be accessible
|
||||||
|
response = client.get("/api/files")
|
||||||
|
assert response.status_code == 200
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
"""
|
||||||
|
Unit tests for configuration and security validation.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
import os
|
||||||
|
from pydantic import ValidationError
|
||||||
|
from app.config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestConfigurationValidation:
|
||||||
|
"""Tests for configuration validation."""
|
||||||
|
|
||||||
|
def test_session_secret_required_with_auth(self):
|
||||||
|
"""Test that SESSION_SECRET is required when auth is enabled."""
|
||||||
|
with pytest.raises(ValidationError) as exc_info:
|
||||||
|
Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=True,
|
||||||
|
session_secret=None
|
||||||
|
)
|
||||||
|
assert "SESSION_SECRET must be set" in str(exc_info.value)
|
||||||
|
|
||||||
|
def test_session_secret_minimum_length(self):
|
||||||
|
"""Test that SESSION_SECRET must be at least 32 characters."""
|
||||||
|
with pytest.raises(ValidationError) as exc_info:
|
||||||
|
Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=True,
|
||||||
|
session_secret="short"
|
||||||
|
)
|
||||||
|
assert "at least 32 characters" in str(exc_info.value)
|
||||||
|
|
||||||
|
def test_valid_configuration(self):
|
||||||
|
"""Test that valid configuration is accepted."""
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test_key",
|
||||||
|
azure_ai_key="test_key",
|
||||||
|
azure_region="eastus",
|
||||||
|
azure_endpoint="https://test.cognitiveservices.azure.com/",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=True,
|
||||||
|
session_secret="a" * 32 # 32 character secret
|
||||||
|
)
|
||||||
|
assert config.auth_enabled is True
|
||||||
|
assert len(config.session_secret) == 32
|
||||||
|
|
||||||
|
def test_auth_disabled_no_session_secret_required(self):
|
||||||
|
"""Test that SESSION_SECRET is not required when auth is disabled."""
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test_key",
|
||||||
|
azure_ai_key="test_key",
|
||||||
|
azure_region="eastus",
|
||||||
|
azure_endpoint="https://test.cognitiveservices.azure.com/",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=False,
|
||||||
|
session_secret=None
|
||||||
|
)
|
||||||
|
assert config.auth_enabled is False
|
||||||
|
assert config.session_secret is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestNotificationConfiguration:
|
||||||
|
"""Tests for notification configuration parsing."""
|
||||||
|
|
||||||
|
def test_notification_urls_from_string(self):
|
||||||
|
"""Test parsing notification URLs from comma-separated string."""
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=False,
|
||||||
|
notification_urls="discord://webhook1,telegram://webhook2"
|
||||||
|
)
|
||||||
|
assert len(config.notification_urls) == 2
|
||||||
|
assert "discord://webhook1" in config.notification_urls
|
||||||
|
assert "telegram://webhook2" in config.notification_urls
|
||||||
|
|
||||||
|
def test_notification_urls_from_list(self):
|
||||||
|
"""Test that notification URLs can be provided as a list."""
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=False,
|
||||||
|
notification_urls=["discord://webhook1", "telegram://webhook2"]
|
||||||
|
)
|
||||||
|
assert len(config.notification_urls) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
@pytest.mark.security
|
||||||
|
class TestSecurityConfiguration:
|
||||||
|
"""Tests for security-related configuration."""
|
||||||
|
|
||||||
|
def test_no_default_credentials_in_config(self):
|
||||||
|
"""Test that no default credentials are present in configuration."""
|
||||||
|
# This test ensures we don't accidentally have hardcoded credentials
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=False
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ensure optional credentials are actually optional (None)
|
||||||
|
assert config.dropbox_app_key is None
|
||||||
|
assert config.dropbox_app_secret is None
|
||||||
|
assert config.nextcloud_password is None
|
||||||
|
assert config.paperless_ngx_api_token is None
|
||||||
|
|
||||||
|
def test_optional_services_dont_require_credentials(self):
|
||||||
|
"""Test that application can start without optional service credentials."""
|
||||||
|
config = Settings(
|
||||||
|
database_url="sqlite:///test.db",
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
openai_api_key="test",
|
||||||
|
azure_ai_key="test",
|
||||||
|
azure_region="test",
|
||||||
|
azure_endpoint="https://test.example.com",
|
||||||
|
gotenberg_url="http://localhost:3000",
|
||||||
|
workdir="/tmp",
|
||||||
|
auth_enabled=False
|
||||||
|
)
|
||||||
|
|
||||||
|
# Should not raise an error
|
||||||
|
assert config.database_url == "sqlite:///test.db"
|
||||||
Reference in New Issue
Block a user