fix: ensure data persistence across reinstallations

- Add entrypoint.sh: creates /app/data, stamps legacy DBs, runs alembic upgrade head
- Update Dockerfile to use new entrypoint and pre-create /app/data
- Change default DATABASE_URL to sqlite:///./data/dmarq.db
- Add _ensure_sqlite_dir() to database.py for automatic directory creation
- Update docker-compose.yml with app_data named volume for /app/data
- Add 6 tests for _ensure_sqlite_dir and updated default URL

Agent-Logs-Url: https://github.com/christianlouis/dmarq/sessions/2bac317b-97a2-450b-84f1-3e316f7ef54c

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

backend/Dockerfile

@@ -19,7 +19,11 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copy application code including templates and static assets
 COPY . .
 
+# Make the entrypoint executable and create the default data directory so
+# SQLite has a place to write its file even without an explicit volume mount.
+RUN chmod +x /app/entrypoint.sh && mkdir -p /app/data
+
 # Expose application port
 EXPOSE 8080
 
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+ENTRYPOINT ["/app/entrypoint.sh"]

backend/app/core/config.py

@@ -23,7 +23,9 @@ class Settings(BaseSettings):
     API_V1_STR: str = "/api/v1"
 
     # Database
-    DATABASE_URL: str = "sqlite:///./dmarq.db"
+    # Default to a sub-directory so the SQLite file lives in a location that
+    # can be persisted via a Docker volume mount (e.g. /app/data).
+    DATABASE_URL: str = "sqlite:///./data/dmarq.db"
 
     # JWT Authentication
     SECRET_KEY: Optional[str] = None

backend/app/core/database.py

@@ -1,7 +1,9 @@
+import os
 from typing import Generator
 from urllib.parse import urlparse, urlunparse
 
 from sqlalchemy import create_engine
+from sqlalchemy.engine import make_url
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
@@ -30,10 +32,34 @@ def _make_sync_db_url(url: str) -> str:
     return urlunparse(parsed._replace(scheme=sync_scheme))
 
 
+def _ensure_sqlite_dir(url: str) -> None:
+    """Create the parent directory for a SQLite database file if needed.
+
+    For SQLite URLs (``sqlite:///relative/path`` or ``sqlite:////absolute/path``),
+    the parent directory must exist before SQLAlchemy tries to open (or create)
+    the file.  This is a no-op for in-memory databases (``sqlite://``) and for
+    non-SQLite URLs.
+    """
+    sa_url = make_url(url)
+    if not sa_url.drivername.startswith("sqlite"):
+        return
+    db_path = sa_url.database
+    if not db_path or db_path == ":memory:":
+        return  # in-memory – nothing to create
+    parent = os.path.dirname(db_path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+
+
 settings = get_settings()
 
+_sync_url = _make_sync_db_url(settings.DATABASE_URL)
+
+# Ensure the parent directory exists before SQLAlchemy tries to open the file
+_ensure_sqlite_dir(_sync_url)
+
 # Configure SQLAlchemy (normalise async driver schemes to their sync equivalents)
-engine = create_engine(_make_sync_db_url(settings.DATABASE_URL), pool_pre_ping=True)
+engine = create_engine(_sync_url, pool_pre_ping=True)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 # Create base class for SQLAlchemy models

backend/app/tests/test_config.py

@@ -7,7 +7,7 @@ could run (see: pydantic_settings sources/providers/env.py decode_complex_value)
 """
 
 from app.core.config import Settings
-from app.core.database import _make_sync_db_url
+from app.core.database import _ensure_sqlite_dir, _make_sync_db_url
 
 
 class TestBackendCorsOriginsValidator:
@@ -96,3 +96,45 @@ class TestMakeSyncDbUrl:
         assert settings.DATABASE_URL == "postgresql+asyncpg://user:pass@db:5432/mydb"
         # Normalised URL used by the engine must not contain asyncpg
         assert "asyncpg" not in _make_sync_db_url(settings.DATABASE_URL)
+
+
+class TestEnsureSqliteDir:
+    """Tests for the _ensure_sqlite_dir() helper."""
+
+    def test_relative_sqlite_path_creates_directory(self, tmp_path, monkeypatch):
+        """A relative SQLite URL creates its parent directory."""
+        monkeypatch.chdir(tmp_path)
+        _ensure_sqlite_dir("sqlite:///./subdir/dmarq.db")
+        assert (tmp_path / "subdir").is_dir()
+
+    def test_absolute_sqlite_path_creates_directory(self, tmp_path):
+        """An absolute SQLite URL creates its parent directory."""
+        db_path = tmp_path / "nested" / "dmarq.db"
+        _ensure_sqlite_dir(f"sqlite:///{db_path}")
+        assert db_path.parent.is_dir()
+
+    def test_in_memory_sqlite_no_directory_created(self, tmp_path, monkeypatch):
+        """An in-memory SQLite URL does not create any directory."""
+        monkeypatch.chdir(tmp_path)
+        _ensure_sqlite_dir("sqlite://")
+        _ensure_sqlite_dir("sqlite:///:memory:")
+        # tmp_path itself exists but no new subdirectories should appear
+        assert list(tmp_path.iterdir()) == []
+
+    def test_postgres_url_no_directory_created(self, tmp_path, monkeypatch):
+        """Non-SQLite URLs are ignored entirely."""
+        monkeypatch.chdir(tmp_path)
+        _ensure_sqlite_dir("postgresql://user:pass@db:5432/mydb")
+        assert list(tmp_path.iterdir()) == []
+
+    def test_existing_directory_is_noop(self, tmp_path):
+        """Calling _ensure_sqlite_dir when the directory already exists is a no-op."""
+        existing = tmp_path / "data"
+        existing.mkdir()
+        _ensure_sqlite_dir(f"sqlite:///{existing}/dmarq.db")  # should not raise
+        assert existing.is_dir()
+
+    def test_default_database_url_uses_data_subdir(self):
+        """Default DATABASE_URL places the SQLite file inside a data/ subdirectory."""
+        settings = Settings()
+        assert settings.DATABASE_URL.endswith("data/dmarq.db")

backend/entrypoint.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# entrypoint.sh – prepare the database and start the application.
+#
+# Responsibilities:
+#   1. Create the data directory so SQLite can write its file there.
+#   2. Ensure Alembic migration tracking is consistent (stamp existing
+#      databases that were created before Alembic was introduced).
+#   3. Apply all pending Alembic migrations (alembic upgrade head).
+#   4. Hand off to the real application process.
+
+set -e
+
+DATA_DIR="${DATA_DIR:-/app/data}"
+mkdir -p "$DATA_DIR"
+
+echo "==> Running database migrations …"
+
+# If the database already has tables but no alembic_version table (i.e. it
+# was bootstrapped via SQLAlchemy create_all before this entrypoint was
+# introduced), stamp it at the current head so that subsequent 'upgrade head'
+# calls are no-ops rather than errors.
+python - <<'PYEOF'
+import sys
+try:
+    from sqlalchemy import inspect, text
+    from app.core.database import engine
+
+    with engine.connect() as conn:
+        insp = inspect(conn)
+        tables = insp.get_table_names()
+
+    if "alembic_version" not in tables and tables:
+        import subprocess
+        print(
+            "WARNING: database has tables but no alembic_version – "
+            "stamping head to record current schema state."
+        )
+        result = subprocess.run(
+            ["alembic", "stamp", "head"],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            print("ERROR: alembic stamp failed:", result.stderr, file=sys.stderr)
+            sys.exit(1)
+        print(result.stdout)
+except Exception as exc:  # noqa: BLE001
+    # If we cannot connect at all (e.g. Postgres not ready yet), let
+    # 'alembic upgrade head' handle the error with a clear message.
+    print(f"WARNING: pre-migration inspection skipped: {exc}")
+PYEOF
+
+alembic upgrade head
+echo "==> Migrations complete."
+
+echo "==> Starting application …"
+exec uvicorn app.main:app --host 0.0.0.0 --port "${PORT:-8080}"

docker-compose.yml

@@ -26,7 +26,8 @@ services:
       context: ./backend
       dockerfile: Dockerfile
     volumes:
-      - ./backend:/app
+      - ./backend:/app          # development: mount source code for live reload
+      - app_data:/app/data      # persist SQLite database and other application data
     depends_on:
       db:
         condition: service_healthy
@@ -50,4 +51,6 @@ networks:
 # Define volumes
 volumes:
   postgres_data:
+    driver: local
+  app_data:
     driver: local