From dfbb259fdff659b9786c29073ac7286d6c6d9bb1 Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Fri, 14 Aug 2026 17:09:56 +0200 Subject: [PATCH] better scale --- routstr/core/db.py | 22 +++++++++++++++++++--- routstr/core/settings.py | 10 +++++++--- tests/unit/test_db_pool_config.py | 14 +++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/routstr/core/db.py b/routstr/core/db.py index d7679933..0d0539ef 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -36,14 +36,18 @@ def create_db_engine(database_url: str = DATABASE_URL) -> AsyncEngine: is_sqlite = backend == "sqlite" is_memory_sqlite = is_sqlite and url.database in {None, "", ":memory:"} pool_pre_ping = settings.database_pool_pre_ping or not is_sqlite - options: dict[str, int | float | bool] = {"pool_pre_ping": pool_pre_ping} + options: dict[str, object] = {"pool_pre_ping": pool_pre_ping} if not is_memory_sqlite: options.update( - pool_size=settings.database_pool_size, - max_overflow=settings.database_max_overflow, + pool_size=1 if is_sqlite else settings.database_pool_size, + max_overflow=0 if is_sqlite else settings.database_max_overflow, pool_timeout=settings.database_pool_timeout, pool_recycle=settings.database_pool_recycle, ) + if is_sqlite: + options["connect_args"] = { + "timeout": settings.database_sqlite_busy_timeout, + } logger.info( "Database pool configured", @@ -80,8 +84,20 @@ def create_db_engine(database_url: str = DATABASE_URL) -> AsyncEngine: }, ) + def configure_sqlite_connection( + dbapi_connection: object, connection_record: object + ) -> None: + cursor = dbapi_connection.cursor() # type: ignore[attr-defined] + try: + timeout_ms = round(settings.database_sqlite_busy_timeout * 1000) + cursor.execute(f"PRAGMA busy_timeout={timeout_ms}") + finally: + cursor.close() + event.listen(created_engine.sync_engine, "checkout", record_pool_checkout) event.listen(created_engine.sync_engine, "checkin", record_pool_checkin) + if is_sqlite: + event.listen(created_engine.sync_engine, "connect", configure_sqlite_connection) return created_engine diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 6852eaa2..b947599b 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -127,9 +127,9 @@ class Settings(BaseSettings): ) # Database connection-pool controls (advanced). Capacity defaults provide - # headroom for Routstr's concurrent request and background-payment workload. - # Pre-ping is enabled by the engine factory for networked backends; SQLite - # can explicitly opt in. These fields are env-only below. + # headroom for networked backends. File-backed SQLite is forced to one + # connection because it permits only one writer. Pre-ping is automatic for + # networked backends; SQLite can explicitly opt in. These fields are env-only. database_pool_size: int = Field(default=10, ge=1, env="DATABASE_POOL_SIZE") database_max_overflow: int = Field(default=20, ge=0, env="DATABASE_MAX_OVERFLOW") database_pool_timeout: float = Field( @@ -140,6 +140,9 @@ class Settings(BaseSettings): database_pool_hold_warn_seconds: float = Field( default=10.0, gt=0, env="DATABASE_POOL_HOLD_WARN_SECONDS" ) + database_sqlite_busy_timeout: float = Field( + default=30.0, gt=0, env="DATABASE_SQLITE_BUSY_TIMEOUT" + ) # Logging log_level: str = Field(default="INFO", env="LOG_LEVEL") @@ -196,6 +199,7 @@ ENV_ONLY_FIELDS = frozenset( "database_pool_recycle", "database_pool_pre_ping", "database_pool_hold_warn_seconds", + "database_sqlite_busy_timeout", } ) diff --git a/tests/unit/test_db_pool_config.py b/tests/unit/test_db_pool_config.py index 8f5d0367..a0f74adb 100644 --- a/tests/unit/test_db_pool_config.py +++ b/tests/unit/test_db_pool_config.py @@ -9,7 +9,7 @@ from routstr.core.settings import settings @pytest.mark.asyncio -async def test_engine_uses_validated_bounded_pool_settings( +async def test_file_sqlite_serializes_connections_and_waits_for_writer( monkeypatch: pytest.MonkeyPatch, tmp_path: object ) -> None: monkeypatch.setattr(settings, "database_pool_size", 12) @@ -17,14 +17,18 @@ async def test_engine_uses_validated_bounded_pool_settings( monkeypatch.setattr(settings, "database_pool_timeout", 2.5) monkeypatch.setattr(settings, "database_pool_recycle", 900) monkeypatch.setattr(settings, "database_pool_pre_ping", False) + monkeypatch.setattr(settings, "database_sqlite_busy_timeout", 30.0) engine = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/pool.db") try: - assert engine.pool.size() == 12 # type: ignore[attr-defined] - assert engine.pool._max_overflow == 3 # type: ignore[attr-defined] + assert engine.pool.size() == 1 # type: ignore[attr-defined] + assert engine.pool._max_overflow == 0 # type: ignore[attr-defined] assert engine.pool._timeout == 2.5 # type: ignore[attr-defined] assert engine.pool._recycle == 900 assert engine.pool._pre_ping is False + async with engine.connect() as connection: + busy_timeout = await connection.exec_driver_sql("PRAGMA busy_timeout") + assert busy_timeout.scalar_one() == 30_000 finally: await engine.dispose() @@ -46,6 +50,8 @@ def test_non_sqlite_backend_enables_pre_ping_automatically( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "database_pool_pre_ping", False) + monkeypatch.setattr(settings, "database_pool_size", 12) + monkeypatch.setattr(settings, "database_max_overflow", 3) fake_engine = MagicMock() with ( @@ -56,6 +62,8 @@ def test_non_sqlite_backend_enables_pre_ping_automatically( assert created is fake_engine assert factory.call_args.kwargs["pool_pre_ping"] is True + assert factory.call_args.kwargs["pool_size"] == 12 + assert factory.call_args.kwargs["max_overflow"] == 3 assert listen.call_count == 2