feat(database): add database configuration wizard and migration tool
Add a guided database configuration wizard and a data migration tool that allows users to: - Build database connection strings through a step-by-step UI - Test database connections before applying - Preview and execute data migrations from SQLite to PostgreSQL/MySQL - Copy to clipboard for easy .env file updates New files: - app/utils/db_wizard.py — connection string builder, parser, and tester - app/utils/db_migrate.py — table-by-table data migration utility - app/api/database.py — REST API endpoints for wizard operations - app/views/db_wizard.py — view route for the wizard page - frontend/templates/db_wizard.html — multi-tab wizard UI - tests/test_db_wizard.py — unit tests for db_wizard utilities - tests/test_db_migrate.py — unit tests for db_migrate utilities - tests/test_db_wizard_api.py — integration tests for API and views Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -0,0 +1,149 @@
|
||||
"""Tests for app/utils/db_migrate.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.database import Base
|
||||
from app.utils.db_migrate import migrate_data, preview_migration
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestPreviewMigration:
|
||||
"""Tests for preview_migration function."""
|
||||
|
||||
def test_preview_in_memory_sqlite(self):
|
||||
"""Test previewing an in-memory SQLite database."""
|
||||
# Create a temporary source DB with some data
|
||||
src_engine = create_engine(
|
||||
"sqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=src_engine)
|
||||
|
||||
# Insert a test row
|
||||
Session = sessionmaker(bind=src_engine)
|
||||
session = Session()
|
||||
session.execute(text("INSERT INTO documents (filename) VALUES ('test.pdf')"))
|
||||
session.commit()
|
||||
session.close()
|
||||
|
||||
# Preview using the engine's URL won't work for :memory:,
|
||||
# but we can test the error path
|
||||
result = preview_migration("sqlite:///:memory:")
|
||||
# For :memory: this creates a new empty DB, so tables are empty
|
||||
assert result["success"] is True
|
||||
assert isinstance(result["tables"], list)
|
||||
|
||||
def test_preview_invalid_url(self):
|
||||
"""Test preview with invalid URL returns error."""
|
||||
result = preview_migration("invalid://not-a-db")
|
||||
assert result["success"] is False
|
||||
assert "error" in result
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMigrateData:
|
||||
"""Tests for migrate_data function."""
|
||||
|
||||
def test_migrate_empty_sqlite_to_sqlite(self):
|
||||
"""Test migrating an empty SQLite DB to another SQLite DB."""
|
||||
# Both are file-based temp databases for this test
|
||||
src_url = "sqlite:///:memory:"
|
||||
tgt_url = "sqlite://" # Another in-memory DB
|
||||
|
||||
# Create source schema
|
||||
src_engine = create_engine(src_url, connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(bind=src_engine)
|
||||
src_engine.dispose()
|
||||
|
||||
# Run migration from empty source
|
||||
with patch("app.utils.db_migrate._make_engine") as mock_make:
|
||||
# Create real engines for both
|
||||
real_src = create_engine(
|
||||
"sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool
|
||||
)
|
||||
Base.metadata.create_all(bind=real_src)
|
||||
real_tgt = create_engine(
|
||||
"sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool
|
||||
)
|
||||
mock_make.side_effect = [real_src, real_tgt]
|
||||
|
||||
with patch("app.utils.db_migrate._stamp_alembic_head"):
|
||||
result = migrate_data("sqlite:///:memory:", "sqlite:///:memory:")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["rows_copied"] == 0
|
||||
|
||||
def test_migrate_with_data(self):
|
||||
"""Test migrating a SQLite DB with actual data."""
|
||||
real_src = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(bind=real_src)
|
||||
|
||||
# Insert test data
|
||||
Session = sessionmaker(bind=real_src)
|
||||
session = Session()
|
||||
session.execute(text("INSERT INTO documents (filename) VALUES ('invoice.pdf')"))
|
||||
session.execute(text("INSERT INTO documents (filename) VALUES ('receipt.pdf')"))
|
||||
session.commit()
|
||||
session.close()
|
||||
|
||||
real_tgt = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
|
||||
with patch("app.utils.db_migrate._make_engine") as mock_make:
|
||||
mock_make.side_effect = [real_src, real_tgt]
|
||||
with patch("app.utils.db_migrate._stamp_alembic_head"):
|
||||
result = migrate_data("sqlite:///:memory:", "sqlite:///:memory:")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["rows_copied"] >= 2 # At least the 2 documents rows
|
||||
|
||||
def test_migrate_with_progress_callback(self):
|
||||
"""Test that progress callback is invoked during migration."""
|
||||
real_src = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(bind=real_src)
|
||||
|
||||
Session = sessionmaker(bind=real_src)
|
||||
session = Session()
|
||||
session.execute(text("INSERT INTO documents (filename) VALUES ('test.pdf')"))
|
||||
session.commit()
|
||||
session.close()
|
||||
|
||||
real_tgt = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
callback = MagicMock()
|
||||
|
||||
with patch("app.utils.db_migrate._make_engine") as mock_make:
|
||||
mock_make.side_effect = [real_src, real_tgt]
|
||||
with patch("app.utils.db_migrate._stamp_alembic_head"):
|
||||
result = migrate_data("sqlite:///:memory:", "sqlite:///:memory:", progress_callback=callback)
|
||||
|
||||
assert result["success"] is True
|
||||
# Callback should have been called at least once for the non-empty table
|
||||
if result["rows_copied"] > 0:
|
||||
assert callback.call_count > 0
|
||||
|
||||
def test_migrate_global_exception(self):
|
||||
"""Test that a global exception is caught gracefully."""
|
||||
with patch("app.utils.db_migrate._make_engine", side_effect=Exception("boom")):
|
||||
result = migrate_data("sqlite:///:memory:", "sqlite:///:memory:")
|
||||
assert result["success"] is False
|
||||
assert len(result["errors"]) > 0
|
||||
|
||||
def test_migrate_stamp_failure_is_recorded(self):
|
||||
"""Test that Alembic stamp failure is recorded as an error."""
|
||||
real_src = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(bind=real_src)
|
||||
real_tgt = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
|
||||
with patch("app.utils.db_migrate._make_engine") as mock_make:
|
||||
mock_make.side_effect = [real_src, real_tgt]
|
||||
with patch("app.utils.db_migrate._stamp_alembic_head", side_effect=Exception("stamp failed")):
|
||||
result = migrate_data("sqlite:///:memory:", "sqlite:///:memory:")
|
||||
|
||||
# Data copy succeeds but stamp fails — errors list non-empty
|
||||
assert len(result["errors"]) > 0
|
||||
assert any("stamp" in e.lower() for e in result["errors"])
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Tests for app/utils/db_wizard.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.db_wizard import (
|
||||
build_connection_string,
|
||||
get_supported_backends,
|
||||
parse_connection_string,
|
||||
test_connection as db_test_connection,
|
||||
validate_url_format,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetSupportedBackends:
|
||||
"""Tests for get_supported_backends function."""
|
||||
|
||||
def test_returns_list(self):
|
||||
"""Test that it returns a non-empty list."""
|
||||
result = get_supported_backends()
|
||||
assert isinstance(result, list)
|
||||
assert len(result) >= 3
|
||||
|
||||
def test_each_backend_has_required_keys(self):
|
||||
"""Test that each backend has expected keys."""
|
||||
required_keys = {"id", "label", "description", "requires_host"}
|
||||
for backend in get_supported_backends():
|
||||
assert required_keys.issubset(set(backend.keys())), f"Missing keys in {backend.get('id')}"
|
||||
|
||||
def test_includes_sqlite(self):
|
||||
"""Test that SQLite is included."""
|
||||
ids = [b["id"] for b in get_supported_backends()]
|
||||
assert "sqlite" in ids
|
||||
|
||||
def test_includes_postgresql(self):
|
||||
"""Test that PostgreSQL is included."""
|
||||
ids = [b["id"] for b in get_supported_backends()]
|
||||
assert "postgresql" in ids
|
||||
|
||||
def test_includes_mysql(self):
|
||||
"""Test that MySQL is included."""
|
||||
ids = [b["id"] for b in get_supported_backends()]
|
||||
assert "mysql" in ids
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestBuildConnectionString:
|
||||
"""Tests for build_connection_string function."""
|
||||
|
||||
def test_sqlite_default_path(self):
|
||||
"""Test building a SQLite URL with default path."""
|
||||
url = build_connection_string(backend="sqlite")
|
||||
assert url == "sqlite:///./app/database.db"
|
||||
|
||||
def test_sqlite_custom_path(self):
|
||||
"""Test building a SQLite URL with custom path."""
|
||||
url = build_connection_string(backend="sqlite", sqlite_path="/data/mydb.db")
|
||||
assert url == "sqlite:////data/mydb.db"
|
||||
|
||||
def test_postgresql_basic(self):
|
||||
"""Test building a basic PostgreSQL URL."""
|
||||
url = build_connection_string(
|
||||
backend="postgresql",
|
||||
host="localhost",
|
||||
database="docuelevate",
|
||||
username="user",
|
||||
password="pass",
|
||||
)
|
||||
assert url == "postgresql://user:pass@localhost:5432/docuelevate"
|
||||
|
||||
def test_postgresql_with_ssl(self):
|
||||
"""Test building a PostgreSQL URL with SSL."""
|
||||
url = build_connection_string(
|
||||
backend="postgresql",
|
||||
host="rds.amazonaws.com",
|
||||
database="docuelevate",
|
||||
username="admin",
|
||||
password="secret",
|
||||
ssl_mode="require",
|
||||
)
|
||||
assert "sslmode=require" in url
|
||||
assert "postgresql://admin:secret@rds.amazonaws.com:5432/docuelevate" in url
|
||||
|
||||
def test_postgresql_custom_port(self):
|
||||
"""Test building a PostgreSQL URL with custom port."""
|
||||
url = build_connection_string(
|
||||
backend="postgresql",
|
||||
host="localhost",
|
||||
port=5433,
|
||||
database="testdb",
|
||||
username="user",
|
||||
password="pass",
|
||||
)
|
||||
assert ":5433/" in url
|
||||
|
||||
def test_mysql_basic(self):
|
||||
"""Test building a MySQL URL."""
|
||||
url = build_connection_string(
|
||||
backend="mysql",
|
||||
host="localhost",
|
||||
database="docuelevate",
|
||||
username="root",
|
||||
password="password",
|
||||
)
|
||||
assert url.startswith("mysql+pymysql://")
|
||||
assert "charset=utf8mb4" in url
|
||||
|
||||
def test_mysql_no_duplicate_charset(self):
|
||||
"""Test that charset is not duplicated when passed in extra_options."""
|
||||
url = build_connection_string(
|
||||
backend="mysql",
|
||||
host="localhost",
|
||||
database="docuelevate",
|
||||
username="root",
|
||||
password="pass",
|
||||
extra_options="charset=utf8mb4",
|
||||
)
|
||||
assert url.count("charset=utf8mb4") == 1
|
||||
|
||||
def test_unsupported_backend_raises(self):
|
||||
"""Test that unsupported backend raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Unsupported backend"):
|
||||
build_connection_string(backend="oracle")
|
||||
|
||||
def test_missing_host_raises(self):
|
||||
"""Test that missing host for non-SQLite raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Host is required"):
|
||||
build_connection_string(backend="postgresql", database="db", username="u")
|
||||
|
||||
def test_missing_database_raises(self):
|
||||
"""Test that missing database name raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Database name is required"):
|
||||
build_connection_string(backend="postgresql", host="localhost", username="u")
|
||||
|
||||
def test_missing_username_raises(self):
|
||||
"""Test that missing username raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Username is required"):
|
||||
build_connection_string(backend="postgresql", host="localhost", database="db")
|
||||
|
||||
def test_no_password(self):
|
||||
"""Test building URL without password."""
|
||||
url = build_connection_string(
|
||||
backend="postgresql",
|
||||
host="localhost",
|
||||
database="db",
|
||||
username="user",
|
||||
)
|
||||
assert "user@localhost" in url
|
||||
assert ":@" not in url
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestParseConnectionString:
|
||||
"""Tests for parse_connection_string function."""
|
||||
|
||||
def test_parse_sqlite(self):
|
||||
"""Test parsing a SQLite URL."""
|
||||
result = parse_connection_string("sqlite:///./app/database.db")
|
||||
assert result["valid"] is True
|
||||
assert result["backend"] == "sqlite"
|
||||
assert result["is_sqlite"] is True
|
||||
|
||||
def test_parse_postgresql(self):
|
||||
"""Test parsing a PostgreSQL URL."""
|
||||
result = parse_connection_string("postgresql://user:pass@host:5432/mydb")
|
||||
assert result["valid"] is True
|
||||
assert result["backend"] == "postgresql"
|
||||
assert result["host"] == "host"
|
||||
assert result["port"] == 5432
|
||||
assert result["database"] == "mydb"
|
||||
assert result["username"] == "user"
|
||||
assert result["is_sqlite"] is False
|
||||
|
||||
def test_parse_mysql(self):
|
||||
"""Test parsing a MySQL URL."""
|
||||
result = parse_connection_string("mysql+pymysql://root:pass@localhost:3306/db")
|
||||
assert result["valid"] is True
|
||||
assert result["backend"] == "mysql"
|
||||
|
||||
def test_parse_invalid_url(self):
|
||||
"""Test parsing an invalid URL returns error."""
|
||||
result = parse_connection_string("not-a-valid-url://")
|
||||
# Should still return a dict (make_url may or may not raise)
|
||||
assert isinstance(result, dict)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestValidateUrlFormat:
|
||||
"""Tests for validate_url_format function."""
|
||||
|
||||
def test_valid_sqlite(self):
|
||||
"""Test valid SQLite URL."""
|
||||
result = validate_url_format("sqlite:///./db.sqlite")
|
||||
assert result["valid"] is True
|
||||
assert result["backend"] == "sqlite"
|
||||
|
||||
def test_valid_postgresql(self):
|
||||
"""Test valid PostgreSQL URL."""
|
||||
result = validate_url_format("postgresql://u:p@host/db")
|
||||
assert result["valid"] is True
|
||||
|
||||
def test_valid_mysql(self):
|
||||
"""Test valid MySQL URL."""
|
||||
result = validate_url_format("mysql+pymysql://u:p@host/db")
|
||||
assert result["valid"] is True
|
||||
|
||||
def test_unsupported_backend(self):
|
||||
"""Test that unsupported backends are flagged."""
|
||||
result = validate_url_format("mssql://u:p@host/db")
|
||||
assert result["valid"] is False
|
||||
assert "Unsupported" in result.get("error", "")
|
||||
|
||||
def test_invalid_format(self):
|
||||
"""Test that garbage input is invalid."""
|
||||
result = validate_url_format("")
|
||||
assert result["valid"] is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestTestConnection:
|
||||
"""Tests for test_connection function."""
|
||||
|
||||
def test_sqlite_memory_succeeds(self):
|
||||
"""Test connecting to an in-memory SQLite database."""
|
||||
result = db_test_connection("sqlite:///:memory:")
|
||||
assert result["success"] is True
|
||||
assert "SQLite" in result.get("server_version", "")
|
||||
|
||||
def test_unreachable_host_fails(self):
|
||||
"""Test that an unreachable host returns failure."""
|
||||
result = db_test_connection("postgresql://u:p@192.0.2.1:5432/db", timeout=2)
|
||||
assert result["success"] is False
|
||||
assert result["message"] # Should contain an error message
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests for app/api/database.py and app/views/db_wizard.py modules."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestDatabaseApiEndpoints:
|
||||
"""Tests for the database API endpoints."""
|
||||
|
||||
def test_list_backends(self, client):
|
||||
"""Test GET /api/database/backends returns supported backends."""
|
||||
response = client.get("/api/database/backends")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, list)
|
||||
assert len(data) >= 3
|
||||
ids = [b["id"] for b in data]
|
||||
assert "sqlite" in ids
|
||||
assert "postgresql" in ids
|
||||
|
||||
def test_build_url_requires_admin(self, client):
|
||||
"""Test POST /api/database/build-url requires admin."""
|
||||
response = client.post(
|
||||
"/api/database/build-url",
|
||||
json={"backend": "sqlite"},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
|
||||
def test_build_url_sqlite(self, client):
|
||||
"""Test building a SQLite URL as admin."""
|
||||
# Simulate admin session
|
||||
with client.session_transaction() if hasattr(client, "session_transaction") else _noop():
|
||||
pass
|
||||
# Use the session cookie approach
|
||||
client.cookies.set("session", "test")
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/build-url",
|
||||
json={"backend": "sqlite", "sqlite_path": "/data/test.db"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "sqlite:////data/test.db" in response.json().get("url", "")
|
||||
|
||||
def test_build_url_missing_host(self, client):
|
||||
"""Test building URL with missing host returns 400."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/build-url",
|
||||
json={"backend": "postgresql", "database": "db", "username": "u"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_test_connection_sqlite(self, client):
|
||||
"""Test connection to in-memory SQLite."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/test-connection",
|
||||
json={"url": "sqlite:///:memory:"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["success"] is True
|
||||
|
||||
def test_validate_url_valid(self, client):
|
||||
"""Test validate-url with valid SQLite URL."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/validate-url",
|
||||
json={"url": "sqlite:///test.db"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["valid"] is True
|
||||
|
||||
def test_validate_url_invalid(self, client):
|
||||
"""Test validate-url with unsupported backend."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/validate-url",
|
||||
json={"url": "mssql://u:p@h/d"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["valid"] is False
|
||||
|
||||
def test_parse_url(self, client):
|
||||
"""Test parse-url endpoint."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/parse-url",
|
||||
json={"url": "postgresql://user:pass@host:5432/db"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["backend"] == "postgresql"
|
||||
assert data["host"] == "host"
|
||||
|
||||
def test_preview_migration(self, client):
|
||||
"""Test preview-migration endpoint."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/preview-migration",
|
||||
json={"url": "sqlite:///:memory:"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "tables" in data
|
||||
|
||||
def test_migrate_invalid_source(self, client):
|
||||
"""Test migrate endpoint with invalid source URL."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/migrate",
|
||||
json={"source_url": "mssql://bad", "target_url": "sqlite:///:memory:"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_migrate_invalid_target(self, client):
|
||||
"""Test migrate endpoint with invalid target URL."""
|
||||
with patch("app.api.database._require_admin", return_value={"is_admin": True}):
|
||||
response = client.post(
|
||||
"/api/database/migrate",
|
||||
json={"source_url": "sqlite:///:memory:", "target_url": "mssql://bad"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestDatabaseWizardView:
|
||||
"""Tests for the database wizard view."""
|
||||
|
||||
def test_database_wizard_page_loads(self, client):
|
||||
"""Test GET /database-wizard returns 200."""
|
||||
response = client.get("/database-wizard")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_database_wizard_contains_title(self, client):
|
||||
"""Test that the wizard page contains expected content."""
|
||||
response = client.get("/database-wizard")
|
||||
assert response.status_code == 200
|
||||
assert "Database Configuration Wizard" in response.text
|
||||
|
||||
def test_database_wizard_contains_tabs(self, client):
|
||||
"""Test that the wizard page contains configure and migrate tabs."""
|
||||
response = client.get("/database-wizard")
|
||||
assert "Configure Database" in response.text
|
||||
assert "Migrate Data" in response.text
|
||||
|
||||
|
||||
# Context manager helper for tests that don't need session_transaction
|
||||
class _noop:
|
||||
def __enter__(self):
|
||||
return None
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
Reference in New Issue
Block a user