fix: disable upstream stream timeouts by default

This commit is contained in:
9qeklajc
2026-10-01 11:49:29 +02:00
parent 40bee57be5
commit 5833482240
3 changed files with 18 additions and 14 deletions
+6
View File
@@ -72,6 +72,12 @@ ROUTSTR_SECRET_KEY=
# UPSTREAM_POOL_TIMEOUT=5 # UPSTREAM_POOL_TIMEOUT=5
# UPSTREAM_READ_TIMEOUT=900 # UPSTREAM_READ_TIMEOUT=900
# Upstream Streaming Guards (0 disables; keep above reasoning models' think time)
# UPSTREAM_FIRST_TOKEN_TIMEOUT_SECONDS=0
# UPSTREAM_STREAM_IDLE_TIMEOUT_SECONDS=0
# UPSTREAM_ALLOWED_FAILS=3
# UPSTREAM_COOLDOWN_SECONDS=30
# Logging # Logging
# LOG_LEVEL=INFO # LOG_LEVEL=INFO
# ENABLE_CONSOLE_LOGGING=true # ENABLE_CONSOLE_LOGGING=true
+5 -4
View File
@@ -40,14 +40,15 @@ class Settings(BaseSettings):
upstream_5xx_retry_attempts: int = Field( upstream_5xx_retry_attempts: int = Field(
default=1, ge=0, env="UPSTREAM_5XX_RETRY_ATTEMPTS" default=1, ge=0, env="UPSTREAM_5XX_RETRY_ATTEMPTS"
) )
# Streaming guards, both disabled by 0. A stream that never produces a # Streaming guards, off by default (0). A stream that never produces a
# first chunk can still fail over; one that stalls later can only be # first chunk can still fail over; one that stalls later can only be
# aborted and billed for what it delivered. # aborted and billed for what it delivered. Reasoning models can stay
# silent for minutes, so set these above the longest expected think time.
upstream_first_token_timeout_seconds: float = Field( upstream_first_token_timeout_seconds: float = Field(
default=60.0, ge=0, env="UPSTREAM_FIRST_TOKEN_TIMEOUT_SECONDS" default=0.0, ge=0, env="UPSTREAM_FIRST_TOKEN_TIMEOUT_SECONDS"
) )
upstream_stream_idle_timeout_seconds: float = Field( upstream_stream_idle_timeout_seconds: float = Field(
default=120.0, ge=0, env="UPSTREAM_STREAM_IDLE_TIMEOUT_SECONDS" default=0.0, ge=0, env="UPSTREAM_STREAM_IDLE_TIMEOUT_SECONDS"
) )
# Circuit breaker: timeouts/5xx per (provider, model) within a minute that # Circuit breaker: timeouts/5xx per (provider, model) within a minute that
# take the pair out of candidate selection. 0 seconds disables it. # take the pair out of candidate selection. 0 seconds disables it.
+7 -10
View File
@@ -15,7 +15,7 @@ from routstr.core.error_scope import (
ERROR_SCOPE_UPSTREAM, ERROR_SCOPE_UPSTREAM,
) )
from routstr.core.exceptions import UpstreamError from routstr.core.exceptions import UpstreamError
from routstr.core.settings import settings from routstr.core.settings import Settings, settings
from routstr.upstream.base import BaseUpstreamProvider from routstr.upstream.base import BaseUpstreamProvider
from routstr.upstream.cooldown import is_cooling_down, record_failure from routstr.upstream.cooldown import is_cooling_down, record_failure
from routstr.upstream.stream_timeout import open_guarded_stream from routstr.upstream.stream_timeout import open_guarded_stream
@@ -149,15 +149,12 @@ async def test_zero_first_token_timeout_disables_the_guard(
assert [chunk async for chunk in stream] == [b"first"] assert [chunk async for chunk in stream] == [b"first"]
@pytest.mark.asyncio def test_stream_guards_are_off_by_default() -> None:
async def test_idle_timeout_ends_the_stream_without_raising( # Reasoning models can think silently for minutes; on by default, the
fast_timeouts: None, # guards would fail requests that succeed without them.
) -> None: fields = Settings.__fields__
stream = await open_guarded_stream(_response(_stalls_after_first()), "test") assert fields["upstream_first_token_timeout_seconds"].default == 0
assert fields["upstream_stream_idle_timeout_seconds"].default == 0
# The stalled stream ends after the delivered bytes; the caller's finalizer
# then settles actual usage instead of the request hanging.
assert [chunk async for chunk in stream] == [b"first"]
@pytest.mark.asyncio @pytest.mark.asyncio