fix(db): use NullPool for SQLite and expose pool tuning settings

SQLite engines now use NullPool instead of QueuePool, eliminating the
"QueuePool limit of size 5 overflow 10 reached" TimeoutError under
concurrent load.  PostgreSQL/MySQL engines use a configurable QueuePool
with sensible defaults (pool_size=10, max_overflow=20) exposed via
DB_POOL_SIZE, DB_MAX_OVERFLOW, DB_POOL_TIMEOUT, DB_POOL_RECYCLE env
vars.  pool_pre_ping is enabled on all backends.

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

.env.demo

@@ -7,6 +7,12 @@ GOTENBERG_URL=http://gotenberg:3000
 ALLOW_FILE_DELETE=true  # Allow deletion of file records
 COMPLIANCE_ENABLED=true  # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2)
 
+# **Database Connection Pool** (PostgreSQL / MySQL only; ignored for SQLite)
+# DB_POOL_SIZE=10         # Persistent connections per worker (default: 10)
+# DB_MAX_OVERFLOW=20      # Extra connections under burst (default: 20)
+# DB_POOL_TIMEOUT=30      # Seconds to wait for a pool connection (default: 30)
+# DB_POOL_RECYCLE=1800    # Recycle connections after N seconds (default: 1800)
+
 # **System Reset / Factory Reset**
 # FACTORY_RESET_ON_STARTUP=false  # Wipe all user data on every startup (demo/testing only)
 # ENABLE_FACTORY_RESET=false      # Show the System Reset page in admin UI

app/config.py

@@ -13,6 +13,24 @@ class Settings(BaseSettings):
 
     database_url: str
     redis_url: str
+
+    # Database connection-pool tuning (ignored for SQLite, which uses NullPool).
+    db_pool_size: int = Field(
+        default=10,
+        description="Number of persistent connections kept in the pool per worker process.",
+    )
+    db_max_overflow: int = Field(
+        default=20,
+        description="Additional connections allowed beyond db_pool_size under burst load.",
+    )
+    db_pool_timeout: int = Field(
+        default=30,
+        description="Seconds to wait for a connection from the pool before raising a TimeoutError.",
+    )
+    db_pool_recycle: int = Field(
+        default=1800,
+        description="Recycle (close and reopen) connections after this many seconds to avoid stale connections.",
+    )
     openai_api_key: str
     openai_base_url: str = "https://api.openai.com/v1"  # Default to OpenAI's endpoint
     openai_model: str = "gpt-4o-mini"  # Default model

app/database.py

@@ -10,6 +10,7 @@ from typing import Any
 from sqlalchemy import create_engine, exc
 from sqlalchemy.engine.url import make_url
 from sqlalchemy.orm import Session, declarative_base, sessionmaker
+from sqlalchemy.pool import NullPool
 
 from app.config import settings
 
@@ -17,9 +18,36 @@ logger = logging.getLogger(__name__)
 
 Base = declarative_base()
 
-# Parse the DATABASE_URL
+# ---------------------------------------------------------------------------
+# Engine construction
+# ---------------------------------------------------------------------------
 DB_URL = settings.database_url
-engine = create_engine(DB_URL, connect_args={"check_same_thread": False})
+_parsed_url = make_url(DB_URL)
+
+_connect_args: dict[str, Any] = {}
+_engine_kwargs: dict[str, Any] = {
+    "pool_pre_ping": True,  # detect stale / dropped connections before use
+}
+
+if _parsed_url.get_backend_name() == "sqlite":
+    # SQLite does not benefit from connection pooling and is prone to
+    # QueuePool exhaustion under concurrent access.  NullPool opens a fresh
+    # connection for each request and closes it immediately afterwards,
+    # completely avoiding the "QueuePool limit reached" TimeoutError.
+    _connect_args["check_same_thread"] = False
+    _engine_kwargs["poolclass"] = NullPool
+else:
+    # PostgreSQL / MySQL — use a bounded QueuePool with configurable limits.
+    _engine_kwargs.update(
+        {
+            "pool_size": settings.db_pool_size,
+            "max_overflow": settings.db_max_overflow,
+            "pool_timeout": settings.db_pool_timeout,
+            "pool_recycle": settings.db_pool_recycle,
+        }
+    )
+
+engine = create_engine(DB_URL, connect_args=_connect_args, **_engine_kwargs)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 

docs/ConfigurationGuide.md

@@ -11,6 +11,10 @@ Configuration is primarily done through environment variables specified in a `.e
 | **Variable**           | **Description**                                          | **Example**                    |
 |------------------------|----------------------------------------------------------|--------------------------------|
 | `DATABASE_URL`         | Path/URL to the SQLite database (or other SQL backend). Use the [Database Wizard](/database-wizard) for guided setup. See [Database Configuration](DatabaseConfiguration.md). | `sqlite:///./app/database.db`  |
+| `DB_POOL_SIZE`         | Number of persistent connections in the pool per worker (PostgreSQL/MySQL only; ignored for SQLite). | `10` |
+| `DB_MAX_OVERFLOW`      | Additional connections beyond `DB_POOL_SIZE` under burst load (PostgreSQL/MySQL only). | `20` |
+| `DB_POOL_TIMEOUT`      | Seconds to wait for a pool connection before raising `TimeoutError` (PostgreSQL/MySQL only). | `30` |
+| `DB_POOL_RECYCLE`      | Recycle connections after this many seconds to avoid stale connections (PostgreSQL/MySQL only). | `1800` |
 | `REDIS_URL`            | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0`         |
 | `WORKDIR`              | Working directory for the application.                  | `/workdir`                     |
 | `GOTENBERG_URL`        | Gotenberg PDF processing URL.                           | `http://gotenberg:3000`        |

docs/DatabaseConfiguration.md

@@ -337,18 +337,26 @@ The Helm chart includes a pre-install and pre-upgrade Job hook that runs `alembi
 
 ## Connection Pooling
 
-SQLAlchemy manages a connection pool automatically.  The defaults are suitable for most deployments.  For high-concurrency or Kubernetes deployments you may want to tune:
+SQLAlchemy manages a connection pool automatically.  DocuElevate selects the pool
+strategy based on the database backend:
+
+- **SQLite** — uses `NullPool` (a fresh connection per request, closed immediately).
+  This avoids the `QueuePool limit reached` `TimeoutError` that can occur under
+  concurrent load because SQLite does not benefit from persistent connection pooling.
+- **PostgreSQL / MySQL** — uses a bounded `QueuePool` whose size is configurable
+  via environment variables.
 
 ```bash
-# Optional — these are set via environment variables if you extend app/database.py
+# Tune these for PostgreSQL / MySQL (ignored when using SQLite):
-# Typical production values:
+DB_POOL_SIZE=10         # Number of persistent connections per worker (default: 10)
-DB_POOL_SIZE=10         # Number of persistent connections per worker
+DB_MAX_OVERFLOW=20      # Additional connections allowed beyond pool_size (default: 20)
-DB_MAX_OVERFLOW=20      # Additional connections allowed beyond pool_size
+DB_POOL_TIMEOUT=30      # Seconds to wait for a connection from the pool (default: 30)
-DB_POOL_TIMEOUT=30      # Seconds to wait for a connection from the pool
+DB_POOL_RECYCLE=1800    # Recycle connections after 30 minutes (default: 1800)
-DB_POOL_RECYCLE=1800    # Recycle connections after 30 minutes (avoids stale connections)
 ```
 
-> **Note:** These environment variables are not exposed in the default `app/config.py`.  If you need to tune them, extend the database engine creation in `app/database.py`.
+All backends also enable `pool_pre_ping`, which sends a lightweight health-check
+before each connection is handed out.  This detects stale or dropped connections
+and transparently reconnects.
 
 For **PgBouncer** (external connection pooling), point `DATABASE_URL` at your PgBouncer instance and use transaction-mode pooling:
 
@@ -512,4 +520,13 @@ Then retry `alembic upgrade head`.
 
 Either increase `max_connections` in `postgresql.conf` or add PgBouncer in front of PostgreSQL.  The default PostgreSQL `max_connections` is `100`; reduce `DB_POOL_SIZE` per worker to stay within this limit.
 
+### "QueuePool limit reached" TimeoutError (SQLite)
+
+If you see `TimeoutError: QueuePool limit of size 5 overflow 10 reached`, your
+deployment is still running an older version of DocuElevate that used a bounded
+connection pool for SQLite.  Upgrade to the latest release — SQLite now uses
+`NullPool`, which eliminates this error entirely.  If you are already on the
+latest version and are still seeing pool exhaustion, ensure you are not
+overriding the engine creation manually.
+
 For more help, see the [Troubleshooting Guide](Troubleshooting.md).

tests/test_database.py

@@ -998,3 +998,49 @@ class TestAlembicUpgrade:
         # Verify head is reachable
         heads = script.get_heads()
         assert len(heads) == 1  # Should be a single linear chain
+
+
+@pytest.mark.unit
+class TestEnginePoolConfiguration:
+    """Tests for database engine pool configuration (pool class and options)."""
+
+    def test_sqlite_engine_uses_null_pool(self):
+        """SQLite engines must use NullPool to prevent QueuePool exhaustion."""
+        from sqlalchemy.pool import NullPool
+
+        from app.database import engine
+
+        # The test environment uses SQLite, so NullPool should be in effect.
+        assert isinstance(engine.pool, NullPool)
+
+    def test_create_engine_sqlite_null_pool(self):
+        """Explicitly create a SQLite engine to confirm NullPool is applied."""
+        from sqlalchemy import create_engine
+        from sqlalchemy.pool import NullPool
+
+        test_engine = create_engine(
+            "sqlite:///:memory:",
+            connect_args={"check_same_thread": False},
+            poolclass=NullPool,
+        )
+        assert isinstance(test_engine.pool, NullPool)
+        test_engine.dispose()
+
+    def test_pool_settings_exist_in_config(self):
+        """Verify that pool tuning settings are exposed through config."""
+        from app.config import settings
+
+        assert hasattr(settings, "db_pool_size")
+        assert hasattr(settings, "db_max_overflow")
+        assert hasattr(settings, "db_pool_timeout")
+        assert hasattr(settings, "db_pool_recycle")
+
+    def test_pool_settings_have_sensible_defaults(self):
+        """Default pool settings should be larger than SQLAlchemy's built-in defaults."""
+        from app.config import settings
+
+        # SQLAlchemy defaults: pool_size=5, max_overflow=10
+        assert settings.db_pool_size >= 10
+        assert settings.db_max_overflow >= 20
+        assert settings.db_pool_timeout >= 30
+        assert settings.db_pool_recycle >= 1800