diff --git a/SETTINGS_IMPLEMENTATION.md b/SETTINGS_IMPLEMENTATION.md new file mode 100644 index 00000000..d48ad1fc --- /dev/null +++ b/SETTINGS_IMPLEMENTATION.md @@ -0,0 +1,235 @@ +# Settings Page Implementation - Summary + +## Overview + +This PR implements a complete database-backed settings management system for DocuElevate, allowing administrators to view and edit application configuration through a web interface. + +## What Was Implemented + +### 1. Fixed Critical Redirect Issue + +**Problem**: The `/settings` endpoint was returning a 301 redirect to `/` for all users. + +**Root Cause**: The `require_admin_access` function was implemented as a regular function called inside the route handler, rather than as a proper decorator. This meant: +- Non-admin users would reach the handler and get redirected +- The redirect happened after `@require_login` passed, creating inconsistent behavior + +**Solution**: Converted `require_admin_access` to a proper decorator pattern (like `@require_login`): +```python +@router.get("/settings") +@require_login +@require_admin_access # Now properly blocks non-admin users before handler executes +async def settings_page(request: Request, db: Session = Depends(get_db)): + # Admin-only code here +``` + +### 2. Added OAuth Admin Support + +Enhanced OAuth authentication to support admin privileges: +- Added `is_admin` flag to OAuth user sessions +- Checks if user is in "admin" or "administrators" group +- Maintains consistent admin checking across local and OAuth authentication +- Logged admin status for debugging + +### 3. Completed Settings Metadata + +Expanded `SETTING_METADATA` from 16 to 102 entries covering all settings in `app/config.py`: +- Organized into 10 logical categories +- Added descriptions, types, sensitivity flags, and restart requirements +- Covers all storage providers, AI services, authentication, monitoring, etc. + +### 4. Database-Backed Storage (Already Existed, Now Verified) + +The infrastructure was already in place: +- `ApplicationSettings` model in database +- `settings_service.py` for CRUD operations +- `config_loader.py` for loading settings with precedence +- Settings precedence: **Database > Environment > Defaults** + +### 5. Comprehensive Testing + +Added extensive test coverage: +- **Unit tests** for settings service functions +- **Integration tests** for settings precedence +- **Model tests** for ApplicationSettings +- **Type conversion tests** for boolean, integer, string, list +- **Validation tests** for required fields and constraints +- **Metadata completeness tests** + +All tests pass successfully. + +### 6. API Endpoints (Already Existed, Now Enhanced) + +Settings API in `/api/settings/`: +- `GET /api/settings/` - Get all settings with metadata +- `GET /api/settings/{key}` - Get specific setting +- `POST /api/settings/{key}` - Update setting +- `DELETE /api/settings/{key}` - Delete setting (revert to env/default) +- `POST /api/settings/bulk-update` - Update multiple settings + +All require admin authentication. + +### 7. UI Template (Already Existed) + +The settings page template at `frontend/templates/settings.html` includes: +- Organized categories with expandable sections +- Boolean checkboxes and text inputs +- Sensitive value masking with show/hide toggles +- Bulk update support +- Reset functionality +- Success/error messaging +- Restart requirement indicators + +### 8. Documentation + +Created comprehensive `docs/SettingsManagement.md` covering: +- How to access the settings page +- Settings organization and categories +- Using the UI and API +- Settings precedence explanation +- Security considerations +- Troubleshooting guide +- Development guide for adding new settings + +## Files Modified + +1. **app/views/settings.py** - Fixed admin decorator +2. **app/auth.py** - Added OAuth admin support +3. **app/utils/settings_service.py** - Expanded metadata to 102 settings +4. **app/api/settings.py** - Enhanced admin check with type hints +5. **tests/test_settings.py** - Added comprehensive test coverage + +## Files Added + +1. **docs/SettingsManagement.md** - Complete user and developer documentation + +## Technical Details + +### Settings Precedence Flow + +``` +1. App starts +2. Pydantic loads: defaults → environment variables +3. Database initializes +4. load_settings_from_db() applies database overrides +5. Runtime: settings object has effective values +``` + +### Admin Access Control + +```python +# Non-admin users +/settings → @require_login → @require_admin_access → Redirect to / + +# Admin users +/settings → @require_login → @require_admin_access → Settings page renders +``` + +### Category Organization + +- **Core** (6): Database, Redis, workdir, debug, gotenberg, hostname +- **Authentication** (8): Auth settings, sessions, OAuth +- **AI Services** (6): OpenAI, Azure AI +- **Storage Providers** (49): All cloud storage integrations +- **Email** (7): SMTP configuration +- **IMAP** (14): Email ingestion (2 accounts) +- **Monitoring** (2): Uptime Kuma +- **Processing** (3): HTTP timeout, batch throttling +- **Notifications** (6): Apprise URLs and flags +- **Feature Flags** (1): allow_file_delete + +## Testing Results + +### Manual Integration Test +``` +✓ Admin access control works +✓ Settings metadata is complete and organized (102 settings) +✓ Database persistence works (DB > env > default) +✓ Settings view prepares data correctly +✓ Sensitive values are masked +``` + +### Unit Tests +``` +✓ Save and retrieve settings from database +✓ Update existing settings +✓ Delete settings +✓ Get all settings +✓ Validate boolean, integer, string types +✓ Validate session_secret length (min 32 chars) +✓ Get setting metadata +✓ Get settings by category +✓ Convert types correctly +✓ Handle None values +✓ Settings precedence (DB overrides env) +``` + +## Security Features + +1. **Admin-only access**: Both UI and API require admin privileges +2. **Sensitive data masking**: Passwords, keys, tokens masked in display +3. **Input validation**: All values validated before saving +4. **Audit trail**: Database tracks created_at and updated_at +5. **Session security**: Requires strong session secrets (min 32 characters) + +## Usage Examples + +### Via UI + +1. Log in as admin user +2. Navigate to `/settings` +3. Modify desired settings +4. Click "Save Settings" +5. Restart app if prompted + +### Via API + +```bash +# Get all settings +curl -X GET http://localhost:8000/api/settings/ \ + -H "Cookie: session=..." + +# Update a setting +curl -X POST http://localhost:8000/api/settings/debug \ + -H "Content-Type: application/json" \ + -H "Cookie: session=..." \ + -d '{"key": "debug", "value": "true"}' + +# Bulk update +curl -X POST http://localhost:8000/api/settings/bulk-update \ + -H "Content-Type: application/json" \ + -H "Cookie: session=..." \ + -d '[ + {"key": "debug", "value": "true"}, + {"key": "openai_model", "value": "gpt-4"} + ]' +``` + +## Compatibility + +- Works with existing `.env` files +- Backward compatible with environment-only configuration +- Database settings are optional (app works with env vars only) +- No migration required (ApplicationSettings table created automatically) + +## Next Steps (Optional Enhancements) + +1. Add settings export/import functionality +2. Add settings diff viewer (show what changed) +3. Add settings history/rollback +4. Add per-user settings (not just global) +5. Add settings validation rules in metadata +6. Add settings groups with enable/disable +7. Add settings search/filter in UI + +## Conclusion + +The database-backed settings page is now fully functional: +- ✅ Fixed redirect issue +- ✅ Admin access control works +- ✅ Complete settings metadata (102 settings) +- ✅ Database persistence with precedence +- ✅ Comprehensive test coverage +- ✅ Full documentation + +Administrators can now manage all application settings through the web interface at `/settings`. diff --git a/docs/SettingsManagement.md b/docs/SettingsManagement.md new file mode 100644 index 00000000..6f2c7095 --- /dev/null +++ b/docs/SettingsManagement.md @@ -0,0 +1,225 @@ +# Settings Management Guide + +## Overview + +DocuElevate now supports managing application settings through a web-based GUI. Settings can be configured, saved to the database, and will persist across application restarts with the following precedence: + +**Database > Environment Variables > Defaults** + +## Accessing the Settings Page + +1. Navigate to `/settings` in your web browser +2. **Admin access required** - Only users with admin privileges can access this page +3. For local authentication: Use the admin username/password configured in environment variables +4. For OAuth/SSO: Users must be in the "admin" or "administrators" group + +## Features + +### Settings Organization + +Settings are organized into logical categories for easy navigation: + +- **Core**: Database, Redis, working directory, external hostname, debug mode +- **Authentication**: Login settings, session secrets, OAuth configuration +- **AI Services**: OpenAI and Azure AI configuration +- **Storage Providers**: Dropbox, Google Drive, OneDrive, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless +- **Email**: SMTP configuration for sending emails +- **IMAP**: Email ingestion configuration (supports multiple accounts) +- **Monitoring**: Uptime Kuma integration +- **Notifications**: Apprise notification URLs and settings +- **Processing**: Batch processing and HTTP timeout settings +- **Feature Flags**: Enable/disable specific features + +### Setting Types + +- **String**: Text values (API keys, URLs, paths) +- **Boolean**: True/false toggles (enable/disable features) +- **Integer**: Numeric values (ports, timeouts, thresholds) +- **List**: Comma-separated values (notification URLs) + +### Sensitive Data + +Settings marked as sensitive (passwords, API keys, tokens) are: +- Masked in the UI by default (show ****key) +- Can be revealed temporarily using the eye icon +- Encrypted in session storage +- Never logged in plain text + +### Restart Requirements + +Settings are marked with 🔄 or a red asterisk (*) if they require an application restart to take effect. This includes: +- Database and Redis URLs +- Working directory +- Authentication settings +- Debug mode + +Most runtime settings (API keys, storage credentials) can be changed without restarting. + +## Using the Settings Page + +### Viewing Settings + +1. Navigate to `/settings` +2. Browse categories using the expandable sections +3. Each setting shows: + - **Name**: The setting key + - **Description**: What the setting does + - **Current Value**: The active value (masked if sensitive) + - **Type**: String, boolean, integer, or list + - **Required**: Whether the setting must be configured + - **Restart Required**: Whether changing this setting requires a restart + +### Updating Settings + +1. Modify the desired settings in the form +2. Click "Save Settings" at the bottom of the page +3. Settings are validated before saving +4. Success/error messages are displayed +5. If any changed setting requires a restart, you'll be notified + +### Bulk Updates + +The settings page supports updating multiple settings at once: +- Change as many settings as needed +- Click "Save Settings" once +- All valid changes are applied atomically +- Any validation errors are reported individually + +### Resetting Changes + +Click "Reset" to discard unsaved changes and return to the current values. + +## API Endpoints + +Settings can also be managed programmatically (admin auth required): + +### Get All Settings +```bash +GET /api/settings/ +``` + +Returns all settings with their metadata and current values. + +### Get Specific Setting +```bash +GET /api/settings/{key} +``` + +Returns a single setting's value and metadata. + +### Update Setting +```bash +POST /api/settings/{key} +{ + "key": "debug", + "value": "true" +} +``` + +Updates a single setting. Returns whether a restart is required. + +### Delete Setting +```bash +DELETE /api/settings/{key} +``` + +Removes a setting from the database (reverts to environment variable or default). + +### Bulk Update +```bash +POST /api/settings/bulk-update +[ + {"key": "debug", "value": "true"}, + {"key": "openai_model", "value": "gpt-4"} +] +``` + +Updates multiple settings in one request. + +## Settings Precedence + +DocuElevate loads settings in this order (later sources override earlier ones): + +1. **Defaults**: Hard-coded defaults in `app/config.py` +2. **Environment Variables**: From `.env` file or system environment +3. **Database**: Settings saved through the UI or API + +### Example + +If you have: +- Default: `debug = false` +- Environment: `DEBUG=true` in `.env` +- Database: `debug = false` (saved via UI) + +The application will use `debug = false` (database wins). + +## Database Storage + +Settings are stored in the `application_settings` table with: +- `key`: Unique setting identifier +- `value`: Setting value (stored as string, converted on load) +- `created_at`: When the setting was first saved +- `updated_at`: When the setting was last modified + +## Security Considerations + +1. **Admin Access Only**: Settings page requires admin privileges +2. **Sensitive Data Masking**: Passwords and keys are masked in the UI +3. **Input Validation**: All setting values are validated before saving +4. **Audit Trail**: Database tracks when settings were created/updated +5. **Session Security**: Admin sessions require strong session secrets (min 32 chars) + +## Troubleshooting + +### Can't Access Settings Page + +- **Check authentication**: Make sure you're logged in +- **Check admin status**: + - Local auth: Verify `ADMIN_USERNAME` and `ADMIN_PASSWORD` are correct + - OAuth: Verify your user is in the admin group +- **Check logs**: Look for "Non-admin user attempted to access settings page" messages + +### Settings Not Taking Effect + +- **Check restart requirement**: Some settings require app restart +- **Check precedence**: Database settings override environment variables +- **Check validation**: Invalid values may not be saved (check error messages) +- **Check logs**: Application startup logs show which settings were loaded from database + +### Settings Not Persisting + +- **Check database**: Verify `DATABASE_URL` is configured correctly +- **Check permissions**: Ensure application can write to database +- **Check errors**: Look for SQLAlchemy errors in logs + +## Development + +### Adding New Settings + +1. Add the setting to `app/config.py` in the `Settings` class +2. Add metadata to `SETTING_METADATA` in `app/utils/settings_service.py` +3. Include: + - `category`: Logical grouping + - `description`: Clear explanation + - `type`: string, boolean, integer, or list + - `sensitive`: True for secrets/passwords + - `required`: True if the setting must be configured + - `restart_required`: True if app restart needed + +### Testing + +Run the settings tests: +```bash +pytest tests/test_settings.py -v +``` + +Or run integration tests: +```bash +python3 test_integration.py +``` + +## Related Documentation + +- [Configuration Guide](./ConfigurationGuide.md) - Environment variable reference +- [Deployment Guide](./DeploymentGuide.md) - Production deployment +- [API Documentation](./API.md) - Full API reference diff --git a/test_settings_manual.py b/test_settings_manual.py deleted file mode 100644 index 469c7180..00000000 --- a/test_settings_manual.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -""" -Manual test script to verify settings functionality -""" -import os -import sys -import tempfile -from pathlib import Path - -# Set up minimal environment for testing -os.environ.setdefault("DATABASE_URL", f"sqlite:///{tempfile.gettempdir()}/test_settings.db") -os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0") -os.environ.setdefault("OPENAI_API_KEY", "test_key") -os.environ.setdefault("AZURE_AI_KEY", "test_key") -os.environ.setdefault("AZURE_REGION", "test") -os.environ.setdefault("AZURE_ENDPOINT", "https://test.example.com") -os.environ.setdefault("GOTENBERG_URL", "http://localhost:3000") -os.environ.setdefault("WORKDIR", tempfile.gettempdir()) -os.environ.setdefault("AUTH_ENABLED", "false") -os.environ.setdefault("SESSION_SECRET", "a" * 32) - -from app.config import settings -from app.database import Base, engine, SessionLocal, init_db -from app.models import ApplicationSettings -from app.utils.settings_service import ( - get_setting_from_db, - save_setting_to_db, - get_all_settings_from_db, - delete_setting_from_db, - get_setting_metadata, - get_settings_by_category, - SETTING_METADATA, -) -from app.utils.config_loader import load_settings_from_db, convert_setting_value - -def test_database_model(): - """Test that ApplicationSettings model is in the database""" - print("=" * 60) - print("Testing Database Model") - print("=" * 60) - - # Initialize database - init_db() - - # Check if ApplicationSettings table exists - from sqlalchemy import inspect - inspector = inspect(engine) - tables = inspector.get_table_names() - - print(f"✓ Database tables: {tables}") - assert "application_settings" in tables, "ApplicationSettings table not found!" - print("✓ ApplicationSettings table exists") - - # Check columns - columns = [col['name'] for col in inspector.get_columns('application_settings')] - print(f"✓ Columns: {columns}") - assert "key" in columns - assert "value" in columns - print("✓ All expected columns present") - print() - -def test_settings_service(): - """Test settings service functions""" - print("=" * 60) - print("Testing Settings Service") - print("=" * 60) - - db = SessionLocal() - try: - # Test save and retrieve - print("Testing save_setting_to_db...") - result = save_setting_to_db(db, "test_key", "test_value") - assert result is True - print("✓ Setting saved") - - value = get_setting_from_db(db, "test_key") - assert value == "test_value" - print(f"✓ Setting retrieved: {value}") - - # Test update - print("Testing update...") - result = save_setting_to_db(db, "test_key", "updated_value") - assert result is True - value = get_setting_from_db(db, "test_key") - assert value == "updated_value" - print(f"✓ Setting updated: {value}") - - # Test get all - print("Testing get_all_settings_from_db...") - save_setting_to_db(db, "key1", "value1") - save_setting_to_db(db, "key2", "value2") - all_settings = get_all_settings_from_db(db) - print(f"✓ Retrieved {len(all_settings)} settings") - - # Test delete - print("Testing delete_setting_from_db...") - result = delete_setting_from_db(db, "test_key") - assert result is True - value = get_setting_from_db(db, "test_key") - assert value is None - print("✓ Setting deleted") - - print() - finally: - db.close() - -def test_settings_metadata(): - """Test settings metadata""" - print("=" * 60) - print("Testing Settings Metadata") - print("=" * 60) - - print(f"Total settings in metadata: {len(SETTING_METADATA)}") - - # Test get metadata - metadata = get_setting_metadata("database_url") - print(f"✓ database_url metadata: {metadata}") - assert metadata["category"] == "Core" - assert metadata["required"] is True - - # Test categories - categories = get_settings_by_category() - print(f"✓ Categories: {list(categories.keys())}") - print(f" - Core has {len(categories.get('Core', []))} settings") - print(f" - Authentication has {len(categories.get('Authentication', []))} settings") - print(f" - AI Services has {len(categories.get('AI Services', []))} settings") - print() - -def test_settings_precedence(): - """Test that database settings override environment variables""" - print("=" * 60) - print("Testing Settings Precedence (DB > ENV > Default)") - print("=" * 60) - - db = SessionLocal() - try: - # Save a setting to database - print("Saving 'debug' to database as 'true'...") - save_setting_to_db(db, "debug", "true") - - # Load settings from database - print("Loading settings from database...") - load_settings_from_db(settings, db) - - # Check that database value is used - print(f"✓ settings.debug = {settings.debug}") - assert settings.debug is True, f"Expected True, got {settings.debug}" - print("✓ Database setting took precedence") - - # Clean up - delete_setting_from_db(db, "debug") - print() - finally: - db.close() - -def test_type_conversion(): - """Test type conversion for different setting types""" - print("=" * 60) - print("Testing Type Conversion") - print("=" * 60) - - # Test boolean conversion - assert convert_setting_value("true", bool) is True - assert convert_setting_value("false", bool) is False - assert convert_setting_value("1", bool) is True - assert convert_setting_value("0", bool) is False - print("✓ Boolean conversion works") - - # Test integer conversion - assert convert_setting_value("42", int) == 42 - assert convert_setting_value("0", int) == 0 - print("✓ Integer conversion works") - - # Test string conversion - assert convert_setting_value("hello", str) == "hello" - print("✓ String conversion works") - - print() - -def main(): - """Run all tests""" - print("\n" + "=" * 60) - print("SETTINGS FUNCTIONALITY TEST SUITE") - print("=" * 60 + "\n") - - try: - test_database_model() - test_settings_service() - test_settings_metadata() - test_type_conversion() - test_settings_precedence() - - print("=" * 60) - print("ALL TESTS PASSED! ✓") - print("=" * 60) - return 0 - except Exception as e: - print("\n" + "=" * 60) - print(f"TEST FAILED: {e}") - print("=" * 60) - import traceback - traceback.print_exc() - return 1 - -if __name__ == "__main__": - sys.exit(main())