diff --git a/.env.example b/.env.example index ab850738..5688f6c3 100644 --- a/.env.example +++ b/.env.example @@ -69,13 +69,8 @@ ROUTSTR_SECRET_KEY= # Upstream Connection Pools (one pool per upstream origin) # UPSTREAM_MAX_CONNECTIONS=200 -# UPSTREAM_MAX_KEEPALIVE_CONNECTIONS=50 -# UPSTREAM_KEEPALIVE_EXPIRY=60 # UPSTREAM_POOL_TIMEOUT=5 -# UPSTREAM_CONNECT_TIMEOUT=30 # UPSTREAM_READ_TIMEOUT=900 -# UPSTREAM_WRITE_TIMEOUT=30 -# UPSTREAM_CONNECT_RETRIES=1 # Logging # LOG_LEVEL=INFO diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 77a22963..da503a50 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -181,25 +181,10 @@ class Settings(BaseSettings): upstream_max_connections: int = Field( default=200, ge=1, env="UPSTREAM_MAX_CONNECTIONS" ) - upstream_max_keepalive_connections: int = Field( - default=50, ge=0, env="UPSTREAM_MAX_KEEPALIVE_CONNECTIONS" - ) - upstream_keepalive_expiry: float = Field( - default=60.0, gt=0, env="UPSTREAM_KEEPALIVE_EXPIRY" - ) upstream_pool_timeout: float = Field(default=5.0, gt=0, env="UPSTREAM_POOL_TIMEOUT") upstream_read_timeout: float = Field( default=900.0, gt=0, env="UPSTREAM_READ_TIMEOUT" ) - upstream_connect_timeout: float = Field( - default=30.0, gt=0, env="UPSTREAM_CONNECT_TIMEOUT" - ) - upstream_write_timeout: float = Field( - default=30.0, gt=0, env="UPSTREAM_WRITE_TIMEOUT" - ) - upstream_connect_retries: int = Field( - default=1, ge=0, env="UPSTREAM_CONNECT_RETRIES" - ) # Logging log_level: str = Field(default="INFO", env="LOG_LEVEL") @@ -264,13 +249,8 @@ ENV_ONLY_FIELDS = frozenset( "database_busy_timeout", # Reconfiguring a live pool would disrupt in-flight streams. "upstream_max_connections", - "upstream_max_keepalive_connections", - "upstream_keepalive_expiry", "upstream_pool_timeout", "upstream_read_timeout", - "upstream_connect_timeout", - "upstream_write_timeout", - "upstream_connect_retries", } ) diff --git a/routstr/upstream/base.py b/routstr/upstream/base.py index 122a8ddc..ebeb2e22 100644 --- a/routstr/upstream/base.py +++ b/routstr/upstream/base.py @@ -72,7 +72,12 @@ from .cache_breakpoints import ( is_explicit_cache_model, ) from .count_tokens import MissingUsageEstimator, count_tokens_locally -from .http_client import acquire_upstream_http_client +from .http_client import ( + UPSTREAM_CONNECT_RETRIES, + UPSTREAM_CONNECT_TIMEOUT, + UPSTREAM_WRITE_TIMEOUT, + acquire_upstream_http_client, +) from .litellm_routing import detect_litellm_prefix from .model_paths import public_provider_url from .rate_limit import UPSTREAM_RATE_LIMIT, classify_rate_limit @@ -280,12 +285,12 @@ def _build_x_cashu_client() -> httpx.AsyncClient: """ return httpx.AsyncClient( transport=httpx.AsyncHTTPTransport( - retries=settings.upstream_connect_retries, + retries=UPSTREAM_CONNECT_RETRIES, ), timeout=httpx.Timeout( - connect=settings.upstream_connect_timeout, + connect=UPSTREAM_CONNECT_TIMEOUT, read=settings.upstream_read_timeout, - write=settings.upstream_write_timeout, + write=UPSTREAM_WRITE_TIMEOUT, pool=settings.upstream_pool_timeout, ), ) diff --git a/routstr/upstream/http_client.py b/routstr/upstream/http_client.py index 0a09c8ca..253e80a0 100644 --- a/routstr/upstream/http_client.py +++ b/routstr/upstream/http_client.py @@ -31,6 +31,12 @@ _clients: dict[str, httpx.AsyncClient] = {} _client_loop: asyncio.AbstractEventLoop | None = None _closing = False +UPSTREAM_MAX_KEEPALIVE_CONNECTIONS = 50 +UPSTREAM_KEEPALIVE_EXPIRY = 60.0 +UPSTREAM_CONNECT_TIMEOUT = 30.0 +UPSTREAM_WRITE_TIMEOUT = 30.0 +UPSTREAM_CONNECT_RETRIES = 1 + @dataclass class _CloseSubmission: @@ -117,19 +123,19 @@ def _shared_ssl_context() -> ssl.SSLContext: def _build_client() -> httpx.AsyncClient: limits = httpx.Limits( max_connections=settings.upstream_max_connections, - max_keepalive_connections=settings.upstream_max_keepalive_connections, - keepalive_expiry=settings.upstream_keepalive_expiry, + max_keepalive_connections=UPSTREAM_MAX_KEEPALIVE_CONNECTIONS, + keepalive_expiry=UPSTREAM_KEEPALIVE_EXPIRY, ) client = httpx.AsyncClient( transport=httpx.AsyncHTTPTransport( verify=_shared_ssl_context(), limits=limits, - retries=settings.upstream_connect_retries, + retries=UPSTREAM_CONNECT_RETRIES, ), timeout=httpx.Timeout( - connect=settings.upstream_connect_timeout, + connect=UPSTREAM_CONNECT_TIMEOUT, read=settings.upstream_read_timeout, - write=settings.upstream_write_timeout, + write=UPSTREAM_WRITE_TIMEOUT, pool=settings.upstream_pool_timeout, ), ) @@ -441,7 +447,7 @@ def get_upstream_http_client(url: str) -> httpx.AsyncClient: extra={ "origin": key, "max_connections": settings.upstream_max_connections, - "max_keepalive_connections": settings.upstream_max_keepalive_connections, + "max_keepalive_connections": UPSTREAM_MAX_KEEPALIVE_CONNECTIONS, "pool_timeout": settings.upstream_pool_timeout, "read_timeout": settings.upstream_read_timeout, }, diff --git a/tests/unit/test_upstream_http_client.py b/tests/unit/test_upstream_http_client.py index 82b46057..bb8bb6b8 100644 --- a/tests/unit/test_upstream_http_client.py +++ b/tests/unit/test_upstream_http_client.py @@ -158,17 +158,17 @@ async def test_upstream_http_client_applies_configured_pool_bounds() -> None: try: assert client.timeout.pool == settings.upstream_pool_timeout assert client.timeout.read == settings.upstream_read_timeout - assert client.timeout.connect == settings.upstream_connect_timeout - assert client.timeout.write == settings.upstream_write_timeout + assert client.timeout.connect == http_client_module.UPSTREAM_CONNECT_TIMEOUT + assert client.timeout.write == http_client_module.UPSTREAM_WRITE_TIMEOUT build_limits.assert_called_once_with( max_connections=settings.upstream_max_connections, - max_keepalive_connections=settings.upstream_max_keepalive_connections, - keepalive_expiry=settings.upstream_keepalive_expiry, + max_keepalive_connections=http_client_module.UPSTREAM_MAX_KEEPALIVE_CONNECTIONS, + keepalive_expiry=http_client_module.UPSTREAM_KEEPALIVE_EXPIRY, ) build_transport.assert_called_once() assert ( build_transport.call_args.kwargs["retries"] - == settings.upstream_connect_retries + == http_client_module.UPSTREAM_CONNECT_RETRIES ) finally: await close_upstream_http_client()