laantungir/routstr-core
1
0
Fork 0
mirror of https://github.com/Routstr/routstr-core.git synced 2026-10-05 20:28:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: hardcode rarely-tuned upstream pool knobs

Browse Source
This commit is contained in:
9qeklajc
2026-09-26 11:58:49 +02:00
parent 10b105bc7b
commit eaa2496572
5 changed files with 26 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
-5
View File
@@ -69,13 +69,8 @@ ROUTSTR_SECRET_KEY=
# Upstream Connection Pools (one pool per upstream origin)
# UPSTREAM_MAX_CONNECTIONS=200
# UPSTREAM_MAX_KEEPALIVE_CONNECTIONS=50
# UPSTREAM_KEEPALIVE_EXPIRY=60
# UPSTREAM_POOL_TIMEOUT=5
# UPSTREAM_CONNECT_TIMEOUT=30
# UPSTREAM_READ_TIMEOUT=900
# UPSTREAM_WRITE_TIMEOUT=30
# UPSTREAM_CONNECT_RETRIES=1
# Logging
# LOG_LEVEL=INFO
routstr/core/settings.py
-20
View File
@@ -181,25 +181,10 @@ class Settings(BaseSettings):
upstream_max_connections: int = Field(
default=200, ge=1, env="UPSTREAM_MAX_CONNECTIONS"
)
upstream_max_keepalive_connections: int = Field(
default=50, ge=0, env="UPSTREAM_MAX_KEEPALIVE_CONNECTIONS"
)
upstream_keepalive_expiry: float = Field(
default=60.0, gt=0, env="UPSTREAM_KEEPALIVE_EXPIRY"
)
upstream_pool_timeout: float = Field(default=5.0, gt=0, env="UPSTREAM_POOL_TIMEOUT")
upstream_read_timeout: float = Field(
default=900.0, gt=0, env="UPSTREAM_READ_TIMEOUT"
)
upstream_connect_timeout: float = Field(
default=30.0, gt=0, env="UPSTREAM_CONNECT_TIMEOUT"
)
upstream_write_timeout: float = Field(
default=30.0, gt=0, env="UPSTREAM_WRITE_TIMEOUT"
)
upstream_connect_retries: int = Field(
default=1, ge=0, env="UPSTREAM_CONNECT_RETRIES"
)
# Logging
log_level: str = Field(default="INFO", env="LOG_LEVEL")
@@ -264,13 +249,8 @@ ENV_ONLY_FIELDS = frozenset(
"database_busy_timeout",
# Reconfiguring a live pool would disrupt in-flight streams.
"upstream_max_connections",
"upstream_max_keepalive_connections",
"upstream_keepalive_expiry",
"upstream_pool_timeout",
"upstream_read_timeout",
"upstream_connect_timeout",
"upstream_write_timeout",
"upstream_connect_retries",
}
)
routstr/upstream/base.py
+9 -4
View File
@@ -72,7 +72,12 @@ from .cache_breakpoints import (
is_explicit_cache_model,
)
from .count_tokens import MissingUsageEstimator, count_tokens_locally
from .http_client import acquire_upstream_http_client
from .http_client import (
UPSTREAM_CONNECT_RETRIES,
UPSTREAM_CONNECT_TIMEOUT,
UPSTREAM_WRITE_TIMEOUT,
acquire_upstream_http_client,
)
from .litellm_routing import detect_litellm_prefix
from .model_paths import public_provider_url
from .rate_limit import UPSTREAM_RATE_LIMIT, classify_rate_limit
@@ -280,12 +285,12 @@ def _build_x_cashu_client() -> httpx.AsyncClient:
"""
return httpx.AsyncClient(
transport=httpx.AsyncHTTPTransport(
retries=settings.upstream_connect_retries,
retries=UPSTREAM_CONNECT_RETRIES,
),
timeout=httpx.Timeout(
connect=settings.upstream_connect_timeout,
connect=UPSTREAM_CONNECT_TIMEOUT,
read=settings.upstream_read_timeout,
write=settings.upstream_write_timeout,
write=UPSTREAM_WRITE_TIMEOUT,
pool=settings.upstream_pool_timeout,
),
)
routstr/upstream/http_client.py
+12 -6
View File
@@ -31,6 +31,12 @@ _clients: dict[str, httpx.AsyncClient] = {}
_client_loop: asyncio.AbstractEventLoop | None = None
_closing = False
UPSTREAM_MAX_KEEPALIVE_CONNECTIONS = 50
UPSTREAM_KEEPALIVE_EXPIRY = 60.0
UPSTREAM_CONNECT_TIMEOUT = 30.0
UPSTREAM_WRITE_TIMEOUT = 30.0
UPSTREAM_CONNECT_RETRIES = 1
@dataclass
class _CloseSubmission:
@@ -117,19 +123,19 @@ def _shared_ssl_context() -> ssl.SSLContext:
def _build_client() -> httpx.AsyncClient:
limits = httpx.Limits(
max_connections=settings.upstream_max_connections,
max_keepalive_connections=settings.upstream_max_keepalive_connections,
keepalive_expiry=settings.upstream_keepalive_expiry,
max_keepalive_connections=UPSTREAM_MAX_KEEPALIVE_CONNECTIONS,
keepalive_expiry=UPSTREAM_KEEPALIVE_EXPIRY,
)
client = httpx.AsyncClient(
transport=httpx.AsyncHTTPTransport(
verify=_shared_ssl_context(),
limits=limits,
retries=settings.upstream_connect_retries,
retries=UPSTREAM_CONNECT_RETRIES,
),
timeout=httpx.Timeout(
connect=settings.upstream_connect_timeout,
connect=UPSTREAM_CONNECT_TIMEOUT,
read=settings.upstream_read_timeout,
write=settings.upstream_write_timeout,
write=UPSTREAM_WRITE_TIMEOUT,
pool=settings.upstream_pool_timeout,
),
)
@@ -441,7 +447,7 @@ def get_upstream_http_client(url: str) -> httpx.AsyncClient:
extra={
"origin": key,
"max_connections": settings.upstream_max_connections,
"max_keepalive_connections": settings.upstream_max_keepalive_connections,
"max_keepalive_connections": UPSTREAM_MAX_KEEPALIVE_CONNECTIONS,
"pool_timeout": settings.upstream_pool_timeout,
"read_timeout": settings.upstream_read_timeout,
},
tests/unit/test_upstream_http_client.py
+5 -5
View File
@@ -158,17 +158,17 @@ async def test_upstream_http_client_applies_configured_pool_bounds() -> None:
try:
assert client.timeout.pool == settings.upstream_pool_timeout
assert client.timeout.read == settings.upstream_read_timeout
assert client.timeout.connect == settings.upstream_connect_timeout
assert client.timeout.write == settings.upstream_write_timeout
assert client.timeout.connect == http_client_module.UPSTREAM_CONNECT_TIMEOUT
assert client.timeout.write == http_client_module.UPSTREAM_WRITE_TIMEOUT
build_limits.assert_called_once_with(
max_connections=settings.upstream_max_connections,
max_keepalive_connections=settings.upstream_max_keepalive_connections,
keepalive_expiry=settings.upstream_keepalive_expiry,
max_keepalive_connections=http_client_module.UPSTREAM_MAX_KEEPALIVE_CONNECTIONS,
keepalive_expiry=http_client_module.UPSTREAM_KEEPALIVE_EXPIRY,
)
build_transport.assert_called_once()
assert (
build_transport.call_args.kwargs["retries"]
== settings.upstream_connect_retries
== http_client_module.UPSTREAM_CONNECT_RETRIES
)
finally:
await close_upstream_http_client()