[a-zA-Z_][a-zA-Z0-9_]*)[`'\"]\s+instead",
+ re.IGNORECASE | re.DOTALL,
+)
+
+
# A corrector inspects the parsed request body and the upstream error message
# and returns ``(new_body_dict, label)`` for a fix it can apply, or ``None`` to
# decline. ``label`` identifies the fix so it is applied at most once per request.
@@ -100,6 +112,51 @@ _SPEND_SHAPING_PARAMS = frozenset(
}
)
+# Output caps are interchangeable spellings of the same limit, so moving the
+# value from one to another keeps the priced bound intact.
+_OUTPUT_CAP_PARAMS = frozenset(
+ {
+ "max_tokens",
+ "max_completion_tokens",
+ "max_output_tokens",
+ "max_tokens_to_sample",
+ }
+)
+
+
+def rename_unsupported_param(body: dict, error_message: str) -> tuple[dict, str] | None:
+ """Move a rejected top-level param to the name the upstream asked for.
+
+ Returns ``(new_body, label)`` with the value carried over unchanged, or
+ ``None`` when the error names no replacement, the param is absent, or the
+ replacement is already set.
+
+ A spend-shaping field is only renamed to another output cap: that keeps the
+ reservation's bound, whereas renaming into or out of any other spend-shaping
+ field could uncap or fan out the retry.
+ """
+ match = _RENAMED_PARAM_RE.search(error_message)
+ if not match:
+ return None
+ param, replacement = match.group("param"), match.group("replacement")
+ if param == replacement or param not in body or replacement in body:
+ return None
+ param_spend = param.lower() in _SPEND_SHAPING_PARAMS
+ replacement_spend = replacement.lower() in _SPEND_SHAPING_PARAMS
+ if (param_spend or replacement_spend) and not (
+ param.lower() in _OUTPUT_CAP_PARAMS
+ and replacement.lower() in _OUTPUT_CAP_PARAMS
+ ):
+ logger.warning(
+ "Upstream asked to rename '%s' to '%s'; refusing because it would "
+ "change the request's spend bound — surfacing the error",
+ param,
+ replacement,
+ )
+ return None
+ new_body = {(replacement if k == param else k): v for k, v in body.items()}
+ return new_body, f"{param}->{replacement}"
+
def strip_unsupported_param(body: dict, error_message: str) -> tuple[dict, str] | None:
"""Drop a top-level param the upstream named as unsupported/deprecated.
@@ -130,7 +187,12 @@ def strip_unsupported_param(body: dict, error_message: str) -> tuple[dict, str]
# Ordered pipeline of correctors tried on each recoverable rejection.
-DEFAULT_CORRECTORS: tuple[Corrector, ...] = (strip_unsupported_param,)
+# Renaming runs first so a param with a named replacement keeps its value
+# instead of being dropped.
+DEFAULT_CORRECTORS: tuple[Corrector, ...] = (
+ rename_unsupported_param,
+ strip_unsupported_param,
+)
def correct_request(
diff --git a/routstr/upstream/stream_ownership.py b/routstr/upstream/stream_ownership.py
index e0e3e51d..ea06e400 100644
--- a/routstr/upstream/stream_ownership.py
+++ b/routstr/upstream/stream_ownership.py
@@ -10,6 +10,7 @@ from fastapi.responses import StreamingResponse
from starlette.types import Receive, Scope, Send
from ..core import get_logger
+from ..core.settings import settings
logger = get_logger(__name__)
@@ -74,10 +75,14 @@ class PersistentStreamFinalizer:
self._task: asyncio.Future[None] | None = None
self._lock = asyncio.Lock()
+ async def _bounded_finalize(self) -> None:
+ async with asyncio.timeout(settings.request_cleanup_timeout_seconds):
+ await self._finalize()
+
async def run(self) -> None:
async with self._lock:
if self._task is None:
- self._task = asyncio.ensure_future(self._finalize())
+ self._task = asyncio.ensure_future(self._bounded_finalize())
task = self._task
await asyncio.shield(task)
diff --git a/routstr/upstream/venice.py b/routstr/upstream/venice.py
index 379a4a66..15a476fa 100644
--- a/routstr/upstream/venice.py
+++ b/routstr/upstream/venice.py
@@ -1,13 +1,16 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any
+from collections.abc import AsyncGenerator, AsyncIterator
+from typing import TYPE_CHECKING, Any, cast
import httpx
from ..core.exceptions import UpstreamError
from ..core.logging import get_logger
from ..payment.models import Architecture, Model, Pricing, TopProvider
+from . import messages_dispatch
from .base import BaseUpstreamProvider
+from .stream_ownership import aclose_if_needed
if TYPE_CHECKING:
from ..core.db import UpstreamProviderRow
@@ -50,6 +53,73 @@ _UNENFORCEABLE_WEB_SEARCH_KEYS = frozenset(
{"allowed_domains", "blocked_domains", "user_location"}
)
+# Venice streams OpenAI reasoning models' encrypted reasoning as a trailing
+# ``reasoning_content`` delta carrying this marker. litellm turns it into a
+# plaintext ``thinking`` block after the answer, which clients render as
+# gibberish and which makes Claude Code report an empty final result.
+_ENCRYPTED_REASONING_MARKER = "__ENCRYPTED_REASONING__"
+
+
+async def _drop_encrypted_reasoning(
+ upstream: AsyncIterator[Any],
+) -> AsyncGenerator[bytes, None]:
+ """A thinking block's start carries no text, so it is held until its first
+ delta shows whether it is the encrypted payload; later indices shift down
+ to close the gap."""
+ encode = messages_dispatch.encode_sse
+ sse_buffer = b""
+ dropped: set[int] = set()
+ held: list[dict] | None = None
+ held_index: int | None = None
+
+ def shift(event: dict) -> dict:
+ index = event.get("index")
+ if not isinstance(index, int):
+ return event
+ gap = sum(1 for d in dropped if d < index)
+ return {**event, "index": index - gap} if gap else event
+
+ try:
+ async for chunk in upstream:
+ events, sse_buffer = messages_dispatch.events_from_chunk(chunk, sse_buffer)
+ for event in events:
+ etype = event.get("type")
+ index = event.get("index")
+ if held is not None:
+ delta = event.get("delta") or {}
+ is_own_delta = (
+ index == held_index and etype == "content_block_delta"
+ )
+ thinking = str(delta.get("thinking") or "")
+ if is_own_delta and thinking.startswith(
+ _ENCRYPTED_REASONING_MARKER
+ ):
+ dropped.add(cast(int, index))
+ held = None
+ continue
+ if is_own_delta and not thinking:
+ held.append(event)
+ continue
+ for pending in held:
+ yield encode(shift(pending))
+ held = None
+ if index in dropped:
+ continue
+ block = event.get("content_block") or {}
+ if (
+ etype == "content_block_start"
+ and block.get("type") == "thinking"
+ and not block.get("thinking")
+ ):
+ held, held_index = [event], index
+ continue
+ yield encode(shift(event))
+ if held is not None:
+ for pending in held:
+ yield encode(shift(pending))
+ finally:
+ await aclose_if_needed(upstream)
+
def _is_web_search_tool(tool: Any) -> bool:
"""An Anthropic server-side web-search tool, by either of its markers.
@@ -66,6 +136,35 @@ def _is_web_search_tool(tool: Any) -> bool:
) or tool.get("name") == "web_search"
+def _merge_cache_marked_system(body: dict) -> None:
+ """Venice rejects an OpenAI ``system`` message with two or more text parts
+ when any part carries ``cache_control`` (``400 system: text content blocks
+ must contain non-whitespace text``), even though every part is non-blank.
+ Claude Code always sends that shape. A single marked block is accepted and
+ still caches, so the prefix stays cacheable under the last marker.
+ """
+ system = body.get("system")
+ if not isinstance(system, list) or len(system) < 2:
+ return
+ if not all(
+ isinstance(block, dict)
+ and block.get("type") == "text"
+ and isinstance(block.get("text"), str)
+ for block in system
+ ):
+ return
+ markers = [block["cache_control"] for block in system if block.get("cache_control")]
+ if not markers:
+ return
+ body["system"] = [
+ {
+ "type": "text",
+ "text": "\n\n".join(block["text"] for block in system),
+ "cache_control": markers[-1],
+ }
+ ]
+
+
def _usd(entry: Any) -> float | None:
"""Read the USD leg of a Venice ``{usd, diem}`` price pair."""
if isinstance(entry, dict):
@@ -114,7 +213,16 @@ class VeniceUpstreamProvider(BaseUpstreamProvider):
def transform_model_name(self, model_id: str) -> str:
return model_id.removeprefix("venice/")
+ def transform_messages_stream(
+ self, stream: AsyncIterator[Any]
+ ) -> AsyncIterator[Any]:
+ return _drop_encrypted_reasoning(stream)
+
def adapt_messages_request(self, body: dict, model_obj: Model) -> str:
+ _merge_cache_marked_system(body)
+ return self._adapt_web_search(body)
+
+ def _adapt_web_search(self, body: dict) -> str:
"""Trade an Anthropic web-search tool for Venice's own search switch.
Left in the body, litellm's Anthropic adapter rewrites the tool into a
diff --git a/tests/unit/test_litellm_routing.py b/tests/unit/test_litellm_routing.py
index 264a142d..077a765b 100644
--- a/tests/unit/test_litellm_routing.py
+++ b/tests/unit/test_litellm_routing.py
@@ -91,3 +91,24 @@ def test_detect_litellm_prefix_custom_default() -> None:
assert detect_litellm_prefix("https://example.com", default="anthropic/") == (
"anthropic/"
)
+
+
+@pytest.mark.parametrize("model", ["gpt-6", "gpt-6-luna", "gpt-5.5"])
+def test_litellm_sends_max_completion_tokens_for_gpt_5_and_later(model: str) -> None:
+ """OpenAI rejects ``max_tokens`` on these models; litellm <1.101 only
+ rewrote it for names containing ``gpt-5``, so gpt-6 got a 400."""
+ import litellm
+ from litellm.utils import ProviderConfigManager
+
+ config = ProviderConfigManager.get_provider_chat_config(
+ model=model, provider=litellm.LlmProviders.OPENAI
+ )
+ assert config is not None
+ mapped = config.map_openai_params(
+ non_default_params={"max_tokens": 10},
+ optional_params={},
+ model=model,
+ drop_params=True,
+ )
+
+ assert mapped == {"max_completion_tokens": 10}
diff --git a/tests/unit/test_model_path_routing.py b/tests/unit/test_model_path_routing.py
index e2c3e52f..cb764289 100644
--- a/tests/unit/test_model_path_routing.py
+++ b/tests/unit/test_model_path_routing.py
@@ -706,6 +706,75 @@ async def test_pinned_recovery_preserves_routing_fields(
fallback.forward_request.assert_not_awaited()
+_OPENAI_MAX_TOKENS_ERROR = json.dumps(
+ {
+ "error": {
+ "message": "Unsupported parameter: 'max_tokens' is not supported "
+ "with this model. Use 'max_completion_tokens' instead.",
+ "type": "invalid_request_error",
+ "param": "max_tokens",
+ "code": "unsupported_parameter",
+ }
+ }
+).encode()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("pinned", [False, True])
+async def test_rejected_max_tokens_is_renamed_and_retried_on_same_upstream(
+ pinned: bool,
+) -> None:
+ selected, fallback = _make_upstream(1), _make_upstream(2)
+ selected.forward_request = AsyncMock(
+ side_effect=[
+ MagicMock(status_code=400, body=_OPENAI_MAX_TOKENS_ERROR),
+ MagicMock(status_code=200, body=b"{}"),
+ ]
+ )
+ headers = {"authorization": "Bearer key"}
+ if pinned:
+ headers["x-routstr-model-path"] = encode_model_path(selected.base_url, MODEL_ID)
+ request = _make_request(
+ headers,
+ json.dumps(
+ {"model": MODEL_ID, "max_tokens": 300, "messages": [], "stream": True}
+ ).encode(),
+ )
+
+ response = await _run_proxy(
+ request, [(MagicMock(), selected), (MagicMock(), fallback)]
+ )
+
+ assert response.status_code == 200
+ assert selected.forward_request.await_count == 2
+ before, after = [
+ json.loads(call.args[3]) for call in selected.forward_request.await_args_list
+ ]
+ assert before["max_tokens"] == 300 and "max_completion_tokens" not in before
+ assert after["max_completion_tokens"] == 300 and "max_tokens" not in after
+ assert {k: v for k, v in after.items() if k != "max_completion_tokens"} == {
+ k: v for k, v in before.items() if k != "max_tokens"
+ }
+ fallback.forward_request.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_rename_that_changes_spend_bound_is_not_retried() -> None:
+ selected = _make_upstream(1, 400)
+ selected.forward_request.return_value.body = json.dumps(
+ {"error": {"message": "'max_tokens' is not supported. Use 'n' instead."}}
+ ).encode()
+ request = _make_request(
+ {"authorization": "Bearer key"},
+ json.dumps({"model": MODEL_ID, "max_tokens": 300}).encode(),
+ )
+
+ response = await _run_proxy(request, [(MagicMock(), selected)])
+
+ assert response.status_code == 400
+ selected.forward_request.assert_awaited_once()
+
+
# --------------------------------------------------------------------------- #
# Upstream 5xx -> 424 + UPSTREAM_UNAVAILABLE + scope header; node faults stay 500.
# --------------------------------------------------------------------------- #
diff --git a/tests/unit/test_openai_output_cap.py b/tests/unit/test_openai_output_cap.py
new file mode 100644
index 00000000..84c7e234
--- /dev/null
+++ b/tests/unit/test_openai_output_cap.py
@@ -0,0 +1,90 @@
+"""OpenAI reasoning models get ``max_completion_tokens`` before the request is sent."""
+
+from __future__ import annotations
+
+import json
+import os
+
+os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
+os.environ.setdefault("UPSTREAM_API_KEY", "test")
+os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
+
+import pytest
+
+from routstr.payment.models import Architecture, Model, Pricing
+from routstr.upstream import GenericUpstreamProvider
+from routstr.upstream.openai import OpenAIUpstreamProvider
+
+
+def _model(model_id: str) -> Model:
+ return Model(
+ id=model_id,
+ name="test",
+ created=0,
+ description="",
+ context_length=128000,
+ architecture=Architecture(
+ modality="text->text",
+ input_modalities=["text"],
+ output_modalities=["text"],
+ tokenizer="x",
+ instruct_type=None,
+ ),
+ pricing=Pricing(prompt=0.0, completion=0.0),
+ )
+
+
+def _chat(model_id: str, **fields: object) -> bytes:
+ return json.dumps(
+ {"model": model_id, "messages": [{"role": "user", "content": "hi"}], **fields}
+ ).encode()
+
+
+def _prepare(provider: object, model_id: str, body: bytes) -> dict:
+ out = provider.prepare_request_body(body, _model(model_id)) # type: ignore[attr-defined]
+ assert out is not None
+ return json.loads(out)
+
+
+@pytest.mark.parametrize(
+ "model_id", ["gpt-5.6-sol", "openai/gpt-6-sol", "openai/gpt-5", "o3", "o4-mini"]
+)
+def test_reasoning_model_max_tokens_is_renamed(model_id: str) -> None:
+ provider = OpenAIUpstreamProvider(api_key="k")
+ data = _prepare(provider, model_id, _chat(model_id, max_tokens=300))
+ assert data["max_completion_tokens"] == 300
+ assert "max_tokens" not in data
+
+
+@pytest.mark.parametrize("model_id", ["gpt-4o", "openai/gpt-4.1"])
+def test_non_reasoning_model_keeps_max_tokens(model_id: str) -> None:
+ provider = OpenAIUpstreamProvider(api_key="k")
+ data = _prepare(provider, model_id, _chat(model_id, max_tokens=300))
+ assert data["max_tokens"] == 300
+ assert "max_completion_tokens" not in data
+
+
+def test_both_caps_set_is_left_for_upstream() -> None:
+ provider = OpenAIUpstreamProvider(api_key="k")
+ data = _prepare(
+ provider,
+ "gpt-5.6-sol",
+ _chat("gpt-5.6-sol", max_tokens=300, max_completion_tokens=200),
+ )
+ assert data["max_tokens"] == 300
+ assert data["max_completion_tokens"] == 200
+
+
+def test_non_chat_body_is_untouched() -> None:
+ provider = OpenAIUpstreamProvider(api_key="k")
+ body = json.dumps({"model": "gpt-5.6-sol", "input": "hi", "max_tokens": 5}).encode()
+ data = _prepare(provider, "gpt-5.6-sol", body)
+ assert data["max_tokens"] == 5
+ assert "max_completion_tokens" not in data
+
+
+def test_other_upstreams_keep_max_tokens() -> None:
+ provider = GenericUpstreamProvider(base_url="http://test", api_key="k")
+ data = _prepare(provider, "gpt-5.6-sol", _chat("gpt-5.6-sol", max_tokens=300))
+ assert data["max_tokens"] == 300
+ assert "max_completion_tokens" not in data
diff --git a/tests/unit/test_request_correction.py b/tests/unit/test_request_correction.py
index 3b1110a2..56a3423c 100644
--- a/tests/unit/test_request_correction.py
+++ b/tests/unit/test_request_correction.py
@@ -16,9 +16,15 @@ from routstr.upstream.request_correction import (
Correction,
correct_request,
extract_error_message,
+ rename_unsupported_param,
strip_unsupported_param,
)
+OPENAI_MAX_TOKENS_ERROR = (
+ "Unsupported parameter: 'max_tokens' is not supported with this model. "
+ "Use 'max_completion_tokens' instead."
+)
+
def _body(**kwargs: object) -> bytes:
return json.dumps(kwargs).encode()
@@ -139,6 +145,190 @@ class TestStripUnsupportedParam:
assert strip_unsupported_param(body, "`Max_Tokens` is deprecated") is None
+class TestRenameUnsupportedParam:
+ def test_renames_max_tokens_for_openai_reasoning_models(self) -> None:
+ body = {"model": "gpt-5.6-sol", "max_tokens": 256, "messages": []}
+ result = rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR)
+ assert result is not None
+ new_body, label = result
+ assert label == "max_tokens->max_completion_tokens"
+ assert new_body == {
+ "model": "gpt-5.6-sol",
+ "max_completion_tokens": 256,
+ "messages": [],
+ }
+
+ def test_preserves_key_order(self) -> None:
+ body = {"model": "m", "max_tokens": 1, "stream": True}
+ result = rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR)
+ assert result is not None
+ assert list(result[0]) == ["model", "max_completion_tokens", "stream"]
+
+ def test_does_not_mutate_input(self) -> None:
+ body = {"model": "m", "max_tokens": 8}
+ assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is not None
+ assert body == {"model": "m", "max_tokens": 8}
+
+ def test_renames_between_any_output_caps(self) -> None:
+ caps = (
+ "max_tokens",
+ "max_completion_tokens",
+ "max_output_tokens",
+ "max_tokens_to_sample",
+ )
+ for param in caps:
+ for replacement in caps:
+ if param == replacement:
+ continue
+ message = f"`{param}` is deprecated. Use `{replacement}` instead."
+ result = rename_unsupported_param({param: 7}, message)
+ assert result == ({replacement: 7}, f"{param}->{replacement}"), (
+ param,
+ replacement,
+ )
+
+ def test_renames_non_spend_param(self) -> None:
+ message = "'functions' is deprecated. Use 'tools' instead."
+ result = rename_unsupported_param({"functions": [{"name": "f"}]}, message)
+ assert result == ({"tools": [{"name": "f"}]}, "functions->tools")
+
+ def test_matches_across_quote_styles_case_and_newlines(self) -> None:
+ for message in (
+ 'Unsupported parameter: "max_tokens" is not supported.\nUse '
+ '"max_completion_tokens" instead.',
+ "`max_tokens` IS UNSUPPORTED here; please USE `max_completion_tokens`"
+ " INSTEAD",
+ "'max_tokens' is no longer supported, use 'max_completion_tokens' instead",
+ ):
+ result = rename_unsupported_param({"max_tokens": 3}, message)
+ assert result is not None, message
+ assert result[0] == {"max_completion_tokens": 3}
+
+ def test_refuses_renames_that_change_the_spend_bound(self) -> None:
+ for param, replacement in (
+ ("max_tokens", "n"),
+ ("n", "best_of"),
+ ("best_of", "n"),
+ ("temperature", "max_tokens"),
+ ("max_tokens", "temperature"),
+ ("n", "max_tokens"),
+ ):
+ message = f"'{param}' is not supported. Use '{replacement}' instead."
+ assert rename_unsupported_param({param: 2}, message) is None, (
+ param,
+ replacement,
+ )
+
+ def test_spend_guard_is_case_insensitive(self) -> None:
+ ok = "'Max_Tokens' is not supported. Use 'MAX_COMPLETION_TOKENS' instead."
+ assert rename_unsupported_param({"Max_Tokens": 4}, ok) == (
+ {"MAX_COMPLETION_TOKENS": 4},
+ "Max_Tokens->MAX_COMPLETION_TOKENS",
+ )
+ bad = "'Max_Tokens' is not supported. Use 'N' instead."
+ assert rename_unsupported_param({"Max_Tokens": 4}, bad) is None
+
+ def test_declines_when_replacement_already_present(self) -> None:
+ body = {"max_tokens": 4, "max_completion_tokens": 8}
+ assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is None
+
+ def test_declines_when_param_absent(self) -> None:
+ assert rename_unsupported_param({"model": "m"}, OPENAI_MAX_TOKENS_ERROR) is None
+
+ def test_declines_self_rename(self) -> None:
+ message = "'max_tokens' is deprecated. Use 'max_tokens' instead."
+ assert rename_unsupported_param({"max_tokens": 1}, message) is None
+
+ def test_declines_unquoted_or_missing_replacement(self) -> None:
+ for message in (
+ "`gpt-3` is deprecated, use gpt-4 instead",
+ "'max_tokens' is not supported, use max_completion_tokens instead",
+ "'max_tokens' is not supported with this model.",
+ "Use 'max_completion_tokens' instead.",
+ ):
+ assert rename_unsupported_param({"max_tokens": 1}, message) is None, message
+
+ def test_declines_nested_only_param(self) -> None:
+ body = {"reasoning": {"max_tokens": 5}}
+ assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is None
+
+
+class TestCorrectRequestRename:
+ def test_openai_max_tokens_error_is_renamed_not_refused(self) -> None:
+ body = _body(model="gpt-5.6-sol", max_tokens=512, messages=[])
+ result = correct_request(body, OPENAI_MAX_TOKENS_ERROR, set())
+ assert isinstance(result, Correction)
+ assert result.label == "max_tokens->max_completion_tokens"
+ decoded = json.loads(result.body)
+ assert "max_tokens" not in decoded
+ assert decoded["max_completion_tokens"] == 512
+
+ def test_rename_wins_over_strip_for_non_spend_param(self) -> None:
+ body = _body(model="m", functions=[1])
+ result = correct_request(
+ body, "'functions' is deprecated. Use 'tools' instead.", set()
+ )
+ assert result is not None
+ assert json.loads(result.body) == {"model": "m", "tools": [1]}
+
+ def test_unsafe_rename_of_cap_still_surfaces_error(self) -> None:
+ body = _body(model="m", max_tokens=5)
+ assert (
+ correct_request(
+ body, "'max_tokens' is not supported. Use 'n' instead.", set()
+ )
+ is None
+ )
+
+ def test_applied_rename_does_not_repeat_or_strip_cap(self) -> None:
+ body = _body(model="m", max_tokens=5)
+ applied = {"max_tokens->max_completion_tokens"}
+ assert correct_request(body, OPENAI_MAX_TOKENS_ERROR, applied) is None
+
+ def test_rename_ping_pong_terminates(self) -> None:
+ """An upstream that flip-flops between names cannot loop forever."""
+ forward = OPENAI_MAX_TOKENS_ERROR
+ backward = "'max_completion_tokens' is not supported. Use 'max_tokens' instead."
+ body = _body(model="m", max_tokens=5)
+ applied: set[str] = set()
+ for attempt in range(10):
+ message = forward if attempt % 2 == 0 else backward
+ result = correct_request(body, message, applied)
+ if result is None:
+ break
+ body, applied = result.body, applied | {result.label}
+ else:
+ raise AssertionError("correction loop did not terminate")
+ assert applied == {
+ "max_tokens->max_completion_tokens",
+ "max_completion_tokens->max_tokens",
+ }
+ assert json.loads(body) == {"model": "m", "max_tokens": 5}
+
+ def test_buffered_openai_error_response_is_renamed(self) -> None:
+ resp = Response(
+ content=json.dumps(
+ {
+ "error": {
+ "message": OPENAI_MAX_TOKENS_ERROR,
+ "type": "invalid_request_error",
+ "param": "max_tokens",
+ "code": "unsupported_parameter",
+ }
+ }
+ ).encode(),
+ status_code=400,
+ )
+ body = _body(model="gpt-5.6-sol", max_tokens=64, stream=True)
+ result = correct_request(body, extract_error_message(resp), set())
+ assert result is not None
+ assert json.loads(result.body) == {
+ "model": "gpt-5.6-sol",
+ "max_completion_tokens": 64,
+ "stream": True,
+ }
+
+
class TestExtractErrorMessage:
def test_extracts_nested_error_message(self) -> None:
resp = Response(
diff --git a/tests/unit/test_request_lifecycle.py b/tests/unit/test_request_lifecycle.py
new file mode 100644
index 00000000..1573f074
--- /dev/null
+++ b/tests/unit/test_request_lifecycle.py
@@ -0,0 +1,285 @@
+import asyncio
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+from sqlmodel import SQLModel
+from sqlmodel.ext.asyncio.session import AsyncSession
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+from starlette.routing import Route
+from starlette.types import Message, Receive, Scope, Send
+
+import routstr.core.db as db_module
+from routstr.auth import (
+ ReservationSnapshot,
+ _claim_reservation_for_charge,
+ _stop_reservation_heartbeat,
+ pay_for_request,
+)
+from routstr.core.db import ApiKey, ReservationRelease
+from routstr.core.lifecycle import RequestLifecycleMiddleware
+from routstr.core.middleware import LoggingMiddleware
+from routstr.core.settings import settings
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("reason", ["disconnect", "deadline", "send"])
+async def test_lifecycle_stops_live_work(reason: str) -> None:
+ closed = asyncio.Event()
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+ sent: list[Message] = []
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ try:
+ assert (await receive())["type"] == "http.request"
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ while True:
+ await send(
+ {"type": "http.response.body", "body": b"x", "more_body": True}
+ )
+ await asyncio.sleep(0.01)
+ finally:
+ closed.set()
+
+ async def send(message: Message) -> None:
+ sent.append(message)
+ if reason == "send" and message["type"] == "http.response.body":
+ await asyncio.sleep(100)
+
+ async def disconnect() -> None:
+ await asyncio.sleep(0.02)
+ await receive_queue.put({"type": "http.disconnect"})
+
+ task = asyncio.create_task(disconnect()) if reason == "disconnect" else None
+ with (
+ patch.object(settings, "max_request_lifetime_seconds", 0.08),
+ patch.object(settings, "downstream_send_timeout_seconds", 0.03),
+ patch.object(settings, "request_cleanup_timeout_seconds", 0.1),
+ ):
+ try:
+ await asyncio.wait_for(
+ RequestLifecycleMiddleware(app)(
+ {"type": "http"}, receive_queue.get, send
+ ),
+ 1,
+ )
+ except TimeoutError:
+ assert reason == "send"
+ if task:
+ await task
+ assert closed.is_set()
+ assert sent
+
+
+@pytest.mark.asyncio
+async def test_unrelated_oserror_still_propagates() -> None:
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ await receive()
+ raise OSError("Connection reset by peer")
+
+ async def send(message: Message) -> None:
+ pass
+
+ with pytest.raises(OSError, match="Connection reset by peer"):
+ await asyncio.wait_for(
+ RequestLifecycleMiddleware(app)({"type": "http"}, receive_queue.get, send),
+ 1,
+ )
+
+
+@pytest.mark.asyncio
+async def test_disconnect_before_headers_preserves_wallet_work() -> None:
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+ entered = asyncio.Event()
+ finish_wallet = asyncio.Event()
+ wallet_credited = asyncio.Event()
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ await receive()
+ entered.set()
+ await finish_wallet.wait() # The mint accepted the token; credit is still pending.
+ wallet_credited.set()
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+
+ run = asyncio.create_task(
+ RequestLifecycleMiddleware(app)(
+ {"type": "http"}, receive_queue.get, lambda message: asyncio.sleep(0)
+ )
+ )
+ await asyncio.wait_for(entered.wait(), 1)
+ await receive_queue.put({"type": "http.disconnect"})
+ await asyncio.sleep(0.02)
+ assert not run.done()
+ finish_wallet.set()
+ await asyncio.wait_for(run, 1) # No propagated exception for an expected disconnect.
+ assert wallet_credited.is_set()
+
+
+@pytest.mark.asyncio
+async def test_disconnect_before_headers_with_logging_middleware() -> None:
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+ entered = asyncio.Event()
+ finish_wallet = asyncio.Event()
+ wallet_credited = asyncio.Event()
+
+ async def wallet(request: Request) -> PlainTextResponse:
+ await request.body()
+ entered.set()
+ await finish_wallet.wait()
+ wallet_credited.set()
+ return PlainTextResponse("settled")
+
+ app = RequestLifecycleMiddleware(
+ LoggingMiddleware(Starlette(routes=[Route("/wallet", wallet, methods=["POST"])]))
+ )
+ scope: Scope = {
+ "type": "http",
+ "asgi": {"version": "3.0", "spec_version": "2.4"},
+ "http_version": "1.1",
+ "method": "POST",
+ "scheme": "http",
+ "path": "/wallet",
+ "raw_path": b"/wallet",
+ "root_path": "",
+ "query_string": b"",
+ "headers": [],
+ "client": ("test", 1234),
+ "server": ("test", 80),
+ }
+
+ async def send(message: Message) -> None:
+ pass
+
+ run = asyncio.create_task(app(scope, receive_queue.get, send))
+ try:
+ await asyncio.wait_for(entered.wait(), 1)
+ await receive_queue.put({"type": "http.disconnect"})
+ await asyncio.sleep(0.02)
+ assert not run.done()
+ finish_wallet.set()
+ await asyncio.wait_for(run, 1) # No propagated exception for an expected disconnect.
+ assert wallet_credited.is_set()
+ finally:
+ finish_wallet.set()
+ if not run.done():
+ run.cancel()
+ await asyncio.gather(run, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+async def test_deadline_cancels_app_before_sending_504() -> None:
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+ sent: list[Message] = []
+ app_stopped = asyncio.Event()
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ await receive()
+ try:
+ await asyncio.sleep(100)
+ finally:
+ with pytest.raises(OSError, match="Downstream request terminated"):
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ app_stopped.set()
+
+ async def send(message: Message) -> None:
+ assert app_stopped.is_set()
+ sent.append(message)
+
+ with (
+ patch.object(settings, "max_request_lifetime_seconds", 0.02),
+ patch.object(settings, "request_cleanup_timeout_seconds", 0.1),
+ ):
+ await asyncio.wait_for(
+ RequestLifecycleMiddleware(app)({"type": "http"}, receive_queue.get, send),
+ 1,
+ )
+ assert [message["type"] for message in sent] == [
+ "http.response.start",
+ "http.response.body",
+ ]
+ assert sent[0]["status"] == 504
+
+
+@pytest.mark.asyncio
+async def test_disconnect_does_not_release_before_stream_settles(tmp_path: Path) -> None:
+ engine = create_async_engine(
+ f"sqlite+aiosqlite:///{tmp_path / 'reservations.db'}", poolclass=NullPool
+ )
+ async with engine.begin() as conn:
+ await conn.run_sync(SQLModel.metadata.create_all)
+
+ @asynccontextmanager
+ async def session() -> AsyncGenerator[AsyncSession, None]:
+ async with AsyncSession(engine, expire_on_commit=False) as db:
+ yield db
+
+ with patch.object(db_module, "create_session", session):
+ async with session() as db:
+ db.add(ApiKey(hashed_key="stream-key", balance=10_000))
+ await db.commit()
+ started = asyncio.Event()
+ finalizer_started = asyncio.Event()
+ settle = asyncio.Event()
+ result: asyncio.Future[bool] = asyncio.get_running_loop().create_future()
+ snapshot: ReservationSnapshot | None = None
+ receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+ await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ nonlocal snapshot
+ async with session() as db:
+ key = await db.get(ApiKey, "stream-key")
+ assert key is not None
+ snapshot = await pay_for_request(key, 1000, db)
+ await receive()
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ started.set()
+ try:
+ await asyncio.sleep(100)
+ finally:
+ async def finalize() -> None:
+ assert snapshot is not None
+ finalizer_started.set()
+ await settle.wait()
+ async with session() as db:
+ claimed = await _claim_reservation_for_charge(snapshot, db)
+ await db.commit()
+ await _stop_reservation_heartbeat(snapshot.release_id)
+ result.set_result(claimed)
+
+ asyncio.create_task(finalize())
+
+ run = asyncio.create_task(
+ RequestLifecycleMiddleware(app)(
+ {"type": "http"}, receive_queue.get, lambda message: asyncio.sleep(0)
+ )
+ )
+ try:
+ await asyncio.wait_for(started.wait(), 1)
+ await receive_queue.put({"type": "http.disconnect"})
+ await asyncio.wait_for(finalizer_started.wait(), 1)
+ await asyncio.wait_for(run, 1)
+ settle.set()
+ assert await asyncio.wait_for(result, 1)
+ assert snapshot is not None
+ async with session() as db:
+ row = await db.get(ReservationRelease, snapshot.release_id)
+ assert row is not None and row.status == "charged"
+ finally:
+ settle.set()
+ if snapshot is not None:
+ await _stop_reservation_heartbeat(snapshot.release_id)
+ await engine.dispose()
diff --git a/tests/unit/test_stale_reservations.py b/tests/unit/test_stale_reservations.py
index 558fdda5..61a3da5a 100644
--- a/tests/unit/test_stale_reservations.py
+++ b/tests/unit/test_stale_reservations.py
@@ -9,6 +9,7 @@ Covers:
"""
import asyncio
+import math
import time
from typing import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock, patch
@@ -69,7 +70,6 @@ async def test_pay_for_request_sets_reserved_at(
payments_info = MagicMock()
monkeypatch.setattr(auth_module.logger, "info", logger_info)
monkeypatch.setattr(auth_module.payments_logger, "info", payments_info)
-
before = int(time.time())
await pay_for_request(key, 1_000, session)
@@ -87,6 +87,34 @@ async def test_pay_for_request_sets_reserved_at(
assert payments_info.call_args.args == ("RESERVE",)
+@pytest.mark.asyncio
+async def test_pay_for_request_expires_at_has_floor_margin(
+ session: AsyncSession, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """reserved_at_now floors to the second; expires_at must add 1s so a
+ finalizer finishing exactly at the nominal deadline isn't fenced out."""
+ key = ApiKey(hashed_key="floorkey", balance=10_000)
+ session.add(key)
+ await session.commit()
+
+ fixed_time = 1_700_000_000.9 # fractional second, floors when int()'d
+ monkeypatch.setattr(auth_module.time, "time", lambda: fixed_time)
+
+ snapshot = await pay_for_request(key, 1_000, session)
+
+ row = await session.get(ReservationRelease, snapshot.release_id)
+ assert row is not None
+ expected = (
+ int(fixed_time)
+ + math.ceil(
+ auth_module.settings.max_request_lifetime_seconds
+ + auth_module.settings.request_cleanup_timeout_seconds
+ )
+ + 1
+ )
+ assert row.expires_at == expected
+
+
@pytest.mark.asyncio
@pytest.mark.asyncio
async def test_pay_for_request_releases_reservation_when_validation_fails(
@@ -428,3 +456,62 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
await proxy_module.proxy(request, "v1/chat/completions")
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
+
+
+@pytest.mark.asyncio
+async def test_absolute_expiry_releases_fresh_lease(session: AsyncSession) -> None:
+ now = int(time.time())
+ key = ApiKey(
+ hashed_key="expired-deadline",
+ balance=5000,
+ reserved_balance=1000,
+ reserved_at=now,
+ )
+ session.add(key)
+ session.add(
+ ReservationRelease(
+ id="expired",
+ key_hash=key.hashed_key,
+ billing_key_hash=key.hashed_key,
+ reserved_msats=1000,
+ created_at=now,
+ started_at=now - 100,
+ expires_at=now - 1,
+ )
+ )
+ await session.commit()
+ assert await release_stale_reservations(session, 300) == 1
+ await session.refresh(key)
+ assert key.reserved_balance == 0
+ assert key.balance == 5000
+
+
+@pytest.mark.asyncio
+async def test_expired_reservation_cannot_renew_or_claim_charge(
+ session: AsyncSession,
+) -> None:
+ from routstr.auth import (
+ ReservationSnapshot,
+ _claim_reservation_for_charge,
+ renew_reservation,
+ )
+
+ snapshot = ReservationSnapshot(
+ release_id="fenced",
+ key_hash="fenced-key",
+ billing_key_hash="fenced-key",
+ reserved_msats=1000,
+ )
+ session.add(ApiKey(hashed_key="fenced-key", balance=5000, reserved_balance=1000))
+ session.add(
+ ReservationRelease(
+ id="fenced",
+ key_hash="fenced-key",
+ billing_key_hash="fenced-key",
+ reserved_msats=1000,
+ expires_at=int(time.time()) - 1,
+ )
+ )
+ await session.commit()
+ assert not await renew_reservation(snapshot, session)
+ assert not await _claim_reservation_for_charge(snapshot, session)
diff --git a/tests/unit/test_venice_encrypted_reasoning.py b/tests/unit/test_venice_encrypted_reasoning.py
new file mode 100644
index 00000000..43af7614
--- /dev/null
+++ b/tests/unit/test_venice_encrypted_reasoning.py
@@ -0,0 +1,210 @@
+import json
+from collections.abc import AsyncIterator
+from typing import Any
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from routstr.upstream import messages_dispatch
+from routstr.upstream.base import BaseUpstreamProvider
+from routstr.upstream.venice import VeniceUpstreamProvider, _drop_encrypted_reasoning
+
+from .test_venice_web_search import _model
+
+ENCRYPTED = "__ENCRYPTED_REASONING__id=rs_0b04\ngAAAAABqvDJD"
+
+
+def _block(index: int, block: dict, deltas: list[dict]) -> list[dict]:
+ return [
+ {"type": "content_block_start", "index": index, "content_block": block},
+ *({"type": "content_block_delta", "index": index, "delta": d} for d in deltas),
+ {"type": "content_block_stop", "index": index},
+ ]
+
+
+def _thinking(index: int, text: str) -> list[dict]:
+ return _block(
+ index,
+ {"type": "thinking", "thinking": "", "signature": ""},
+ [{"type": "thinking_delta", "thinking": text}],
+ )
+
+
+def _text(index: int, text: str) -> list[dict]:
+ return _block(
+ index,
+ {"type": "text", "text": ""},
+ [{"type": "text_delta", "text": text}],
+ )
+
+
+def _tool(index: int) -> list[dict]:
+ return _block(
+ index,
+ {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}},
+ [{"type": "input_json_delta", "partial_json": '{"command":"ls"}'}],
+ )
+
+
+def _message(blocks: list[dict], stop_reason: str = "end_turn") -> list[dict]:
+ return [
+ {
+ "type": "message_start",
+ "message": {"id": "msg_1", "role": "assistant", "content": []},
+ },
+ *blocks,
+ {"type": "message_delta", "delta": {"stop_reason": stop_reason}},
+ {"type": "message_stop"},
+ ]
+
+
+async def _upstream(events: list[dict], *, split: bool = False) -> AsyncIterator[Any]:
+ payload = b"".join(messages_dispatch.encode_sse(e) for e in events)
+ if split:
+ for i in range(0, len(payload), 7):
+ yield payload[i : i + 7]
+ else:
+ yield payload
+
+
+async def _filtered(events: list[dict], **kwargs: Any) -> list[dict]:
+ buffer = b""
+ out: list[dict] = []
+ async for chunk in _drop_encrypted_reasoning(_upstream(events, **kwargs)):
+ parsed, buffer = messages_dispatch.events_from_chunk(chunk, buffer)
+ out.extend(parsed)
+ return out
+
+
+def _starts(events: list[dict]) -> list[tuple[int, str]]:
+ return [
+ (e["index"], e["content_block"]["type"])
+ for e in events
+ if e["type"] == "content_block_start"
+ ]
+
+
+@pytest.mark.asyncio
+async def test_trailing_encrypted_reasoning_is_dropped() -> None:
+ events = _message([*_text(0, "a.txt contains: hello"), *_thinking(1, ENCRYPTED)])
+
+ out = await _filtered(events)
+
+ assert _starts(out) == [(0, "text")]
+ assert all(ENCRYPTED not in json.dumps(e) for e in out)
+ assert out[-2]["delta"]["stop_reason"] == "end_turn"
+
+
+@pytest.mark.asyncio
+async def test_leading_encrypted_reasoning_closes_index_gap() -> None:
+ events = _message(
+ [*_thinking(0, ENCRYPTED), *_text(1, "hi"), *_tool(2)], "tool_use"
+ )
+
+ out = await _filtered(events, split=True)
+
+ assert _starts(out) == [(0, "text"), (1, "tool_use")]
+ assert {e["index"] for e in out if "index" in e} == {0, 1}
+
+
+@pytest.mark.asyncio
+async def test_plaintext_thinking_is_kept_in_order() -> None:
+ events = _message([*_thinking(0, "Let me list files."), *_tool(1)], "tool_use")
+
+ out = await _filtered(events)
+
+ assert out == events
+
+
+@pytest.mark.asyncio
+async def test_thinking_start_without_delta_is_flushed() -> None:
+ events = _message(
+ [
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {"type": "thinking", "thinking": "", "signature": ""},
+ },
+ {"type": "content_block_stop", "index": 0},
+ *_text(1, "ok"),
+ ]
+ )
+
+ out = await _filtered(events)
+
+ assert out == events
+
+
+@pytest.mark.asyncio
+async def test_aggregated_message_ends_with_answer_text() -> None:
+ events = _message([*_text(0, "hello"), *_thinking(1, ENCRYPTED)])
+
+ message = await messages_dispatch.aggregate_anthropic_events_to_message(
+ _drop_encrypted_reasoning(_upstream(events))
+ )
+
+ assert [b["type"] for b in message["content"]] == ["text"]
+ assert message["content"][0]["text"] == "hello"
+
+
+async def _dispatched_blocks(
+ provider: BaseUpstreamProvider, *, stream: bool
+) -> list[str]:
+ events = _message([*_text(0, "hello"), *_thinking(1, ENCRYPTED)])
+ with patch(
+ "litellm.anthropic.messages.acreate",
+ new=AsyncMock(return_value=_upstream(events)),
+ ):
+ _, result, _ = await provider._dispatch_anthropic_messages(
+ request_body=json.dumps(
+ {
+ "model": "x",
+ "stream": stream,
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "hi"}],
+ }
+ ).encode(),
+ model_obj=_model(),
+ )
+ if not stream:
+ return [b["type"] for b in result["content"]]
+ buffer = b""
+ out: list[dict] = []
+ async for chunk in result:
+ parsed, buffer = messages_dispatch.events_from_chunk(chunk, buffer)
+ out.extend(parsed)
+ return [t for _, t in _starts(out)]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [True, False])
+async def test_venice_dispatch_drops_encrypted_reasoning(stream: bool) -> None:
+ provider = VeniceUpstreamProvider(api_key="sk-test")
+
+ assert await _dispatched_blocks(provider, stream=stream) == ["text"]
+
+
+@pytest.mark.asyncio
+async def test_other_providers_keep_thinking_blocks() -> None:
+ provider = BaseUpstreamProvider(base_url="https://example.com/v1", api_key="k")
+
+ assert await _dispatched_blocks(provider, stream=True) == ["text", "thinking"]
+
+
+@pytest.mark.asyncio
+async def test_closing_the_filter_closes_upstream() -> None:
+ closed = False
+
+ async def upstream() -> AsyncIterator[bytes]:
+ nonlocal closed
+ try:
+ for event in _message(_text(0, "hello")):
+ yield messages_dispatch.encode_sse(event)
+ finally:
+ closed = True
+
+ filtered = _drop_encrypted_reasoning(upstream())
+ await filtered.__anext__()
+ await filtered.aclose()
+
+ assert closed
diff --git a/tests/unit/test_venice_system_cache.py b/tests/unit/test_venice_system_cache.py
new file mode 100644
index 00000000..6b8cc67f
--- /dev/null
+++ b/tests/unit/test_venice_system_cache.py
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import pytest
+
+from routstr.upstream.venice import VeniceUpstreamProvider
+
+from .test_venice_web_search import _body, _dispatch
+
+EPHEMERAL = {"type": "ephemeral"}
+
+CLAUDE_CODE_SYSTEM = [
+ {
+ "type": "text",
+ "text": "x-anthropic-billing-header: cc_version=2.1.281; cc_entrypoint=cli;",
+ },
+ {"type": "text", "text": "You are a Claude agent.", "cache_control": EPHEMERAL},
+ {
+ "type": "text",
+ "text": "\nYou are an interactive agent.",
+ "cache_control": {"type": "ephemeral", "ttl": "1h"},
+ },
+]
+
+
+@pytest.mark.asyncio
+async def test_cache_marked_multi_block_system_is_merged_into_one_block() -> None:
+ provider = VeniceUpstreamProvider(api_key="sk-test")
+
+ kwargs = await _dispatch(provider, _body(system=CLAUDE_CODE_SYSTEM))
+
+ assert kwargs["system"] == [
+ {
+ "type": "text",
+ "text": (
+ "x-anthropic-billing-header: cc_version=2.1.281; cc_entrypoint=cli;"
+ "\n\nYou are a Claude agent.\n\n\nYou are an interactive agent."
+ ),
+ "cache_control": {"type": "ephemeral", "ttl": "1h"},
+ }
+ ]
+
+
+@pytest.mark.asyncio
+async def test_unmarked_multi_block_system_is_untouched() -> None:
+ provider = VeniceUpstreamProvider(api_key="sk-test")
+ system = [{"type": "text", "text": "A."}, {"type": "text", "text": "B."}]
+
+ kwargs = await _dispatch(provider, _body(system=system))
+
+ assert kwargs["system"] == system
+
+
+@pytest.mark.asyncio
+async def test_single_marked_block_and_string_system_are_untouched() -> None:
+ provider = VeniceUpstreamProvider(api_key="sk-test")
+ single = [{"type": "text", "text": "A.", "cache_control": EPHEMERAL}]
+
+ assert (await _dispatch(provider, _body(system=single)))["system"] == single
+ assert (await _dispatch(provider, _body(system="A.")))["system"] == "A."
+
+
+@pytest.mark.asyncio
+async def test_message_and_tool_cache_markers_are_kept() -> None:
+ provider = VeniceUpstreamProvider(api_key="sk-test")
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "hi", "cache_control": EPHEMERAL}],
+ }
+ ]
+ tools = [
+ {
+ "name": "Bash",
+ "description": "Run a command",
+ "input_schema": {"type": "object", "properties": {}},
+ "cache_control": EPHEMERAL,
+ }
+ ]
+
+ kwargs = await _dispatch(
+ provider,
+ _body(system=CLAUDE_CODE_SYSTEM, messages=messages, tools=tools),
+ )
+
+ assert kwargs["messages"] == messages
+ assert kwargs["tools"] == tools
diff --git a/ui/app/model/loading.tsx b/ui/app/model/loading.tsx
new file mode 100644
index 00000000..390be4d8
--- /dev/null
+++ b/ui/app/model/loading.tsx
@@ -0,0 +1,23 @@
+import { AppPageShell } from '@/components/app-page-shell';
+import { PageHeader } from '@/components/page-header';
+import { Skeleton } from '@/components/ui/skeleton';
+
+/**
+ * Route-level fallback so clicking "Models" lands on the page immediately
+ * instead of holding the previous route until this one's chunk is parsed.
+ */
+export default function ModelPageLoading() {
+ return (
+
+
+
+ );
+}
diff --git a/ui/components/model-provider-section.tsx b/ui/components/model-provider-section.tsx
index 9f441678..3acd132e 100644
--- a/ui/components/model-provider-section.tsx
+++ b/ui/components/model-provider-section.tsx
@@ -1,5 +1,6 @@
import { useMemo } from 'react';
import type { Model } from '@/lib/api/schemas/models';
+import { useProgressiveList } from '@/lib/hooks/use-progressive-list';
import type { AdminModelGroup } from '@/lib/api/services/admin';
import type { DisplayUnit } from '@/lib/types/units';
import { ModelItemCard } from '@/components/model-item-card';
@@ -24,6 +25,7 @@ import {
Edit3,
Globe,
Key,
+ Loader2,
MoreVertical,
RefreshCw,
} from 'lucide-react';
@@ -103,10 +105,21 @@ export function ModelProviderSection({
});
}, [provider, providerModels]);
+ const { visibleItems: visibleProviderModels, hiddenCount } =
+ useProgressiveList(keyedProviderModels);
+
+ const pendingRowsNotice =
+ hiddenCount > 0 ? (
+
+
+ Rendering {hiddenCount} more model{hiddenCount === 1 ? '' : 's'}…
+
+ ) : null;
+
if (filterProvider) {
return (
- {keyedProviderModels.map(({ model, renderKey }) => (
+ {visibleProviderModels.map(({ model, renderKey }) => (
onDeleteModel(model.id)}
/>
))}
+ {pendingRowsNotice}
);
}
@@ -217,7 +231,7 @@ export function ModelProviderSection({
- {keyedProviderModels.map(({ model, renderKey }) => (
+ {visibleProviderModels.map(({ model, renderKey }) => (
onDeleteModel(model.id)}
/>
))}
+ {pendingRowsNotice}
diff --git a/ui/components/model-selector.tsx b/ui/components/model-selector.tsx
index 8636fb36..54a345b8 100644
--- a/ui/components/model-selector.tsx
+++ b/ui/components/model-selector.tsx
@@ -1,7 +1,7 @@
'use client';
import React, { useState, useMemo } from 'react';
-import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
+import { useMutation, useQueryClient } from '@tanstack/react-query';
import { type Model, type GroupSettings } from '@/lib/api/schemas/models';
import {
AdminService,
@@ -13,6 +13,7 @@ import { AddProviderModelDialog } from '@/components/add-provider-model-dialog';
import { EditGroupForm } from '@/components/edit-group-form';
import { ModelProviderSection } from '@/components/model-provider-section';
import { useDisplayCurrency } from '@/lib/hooks/use-display-currency';
+import { useModelsWithProviders } from '@/lib/hooks/use-models-with-providers';
import { Button } from '@/components/ui/button';
import { Checkbox } from '@/components/ui/checkbox';
import { Skeleton } from '@/components/ui/skeleton';
@@ -131,19 +132,14 @@ export function ModelSelector({
const queryClient = useQueryClient();
- // Fetch models and groups
+ // Shared with the page shell, so mounting this panel costs no extra fetch.
const {
- data: modelsData,
+ models,
+ groups,
isLoading: isLoadingModels,
error: modelsError,
refetch: refetchModels,
- } = useQuery({
- queryKey: ['models-with-providers'],
- queryFn: () => AdminService.getModelsWithProviders(),
- refetchOnWindowFocus: false,
- });
-
- const { models = [], groups = [] } = modelsData || {};
+ } = useModelsWithProviders();
const allOverrideModels = useMemo(
() => models.filter(isOverrideModel),
[models]
diff --git a/ui/components/models-page.tsx b/ui/components/models-page.tsx
index bff3a379..2bfc2149 100644
--- a/ui/components/models-page.tsx
+++ b/ui/components/models-page.tsx
@@ -1,16 +1,14 @@
'use client';
import { useMemo, useState } from 'react';
-import { useQuery } from '@tanstack/react-query';
+import dynamic from 'next/dynamic';
import { AlertCircle } from 'lucide-react';
import type { Model } from '@/lib/api/schemas/models';
-import { AdminService } from '@/lib/api/services/admin';
+import { useModelsWithProviders } from '@/lib/hooks/use-models-with-providers';
import { groupAndSortModelsByProvider } from '@/lib/utils/model-sort';
import { AppPageShell } from '@/components/app-page-shell';
import { PageHeader } from '@/components/page-header';
import { ModelSelector } from '@/components/model-selector';
-import { ModelTester } from '@/components/model-tester';
-import { ApiEndpointTester } from '@/components/api-endpoint-tester';
import { ModelSearchFilter } from '@/components/model-search-filter';
import { Alert, AlertDescription } from '@/components/ui/alert';
import {
@@ -23,6 +21,19 @@ import {
import { Skeleton } from '@/components/ui/skeleton';
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
+// The testing tabs are never the landing view, so keeping them out of this
+// route's chunk is what lets the navigation itself resolve quickly.
+const ModelTester = dynamic(
+ () => import('@/components/model-tester').then((m) => m.ModelTester),
+ { loading: () => , ssr: false }
+);
+
+const ApiEndpointTester = dynamic(
+ () =>
+ import('@/components/api-endpoint-tester').then((m) => m.ApiEndpointTester),
+ { loading: () => , ssr: false }
+);
+
export function ModelsPage() {
const [filteredModels, setFilteredModels] = useState(
undefined
@@ -31,16 +42,11 @@ export function ModelsPage() {
useState('all');
const {
- data: modelsData,
+ models,
+ groups,
isLoading: isLoadingModels,
error: modelsError,
- } = useQuery({
- queryKey: ['admin-models-with-providers'],
- queryFn: () => AdminService.getModelsWithProviders(),
- refetchOnWindowFocus: false,
- });
-
- const { models = [], groups = [] } = modelsData || {};
+ } = useModelsWithProviders();
const groupedModels = useMemo(
() => groupAndSortModelsByProvider(models),
diff --git a/ui/lib/api/services/admin.ts b/ui/lib/api/services/admin.ts
index e510a571..15daa38d 100644
--- a/ui/lib/api/services/admin.ts
+++ b/ui/lib/api/services/admin.ts
@@ -446,14 +446,50 @@ export class AdminService {
const allModels: AdminModelAsModel[] = [];
const seenModelIds = new Set();
- for (const provider of providers) {
- try {
- const providerModels = await this.getProviderModels(provider.id);
+ // One provider's catalog never depends on another's, and each miss costs an
+ // upstream round trip, so the whole fan-out happens in a single wave.
+ const providerResults = await Promise.all(
+ providers.map(async (provider) => {
+ try {
+ return {
+ provider,
+ models: await this.getProviderModels(provider.id),
+ };
+ } catch (error) {
+ console.error(
+ `Failed to fetch models for provider ${provider.id}:`,
+ error
+ );
+ return null;
+ }
+ })
+ );
- providerModels.db_models.forEach((dbModel) => {
- seenModelIds.add(dbModel.id);
+ for (const result of providerResults) {
+ if (!result) {
+ continue;
+ }
+ const { provider, models: providerModels } = result;
+ providerModels.db_models.forEach((dbModel) => {
+ seenModelIds.add(dbModel.id);
+ const modelWithProvider = {
+ ...dbModel,
+ upstream_provider_id: provider.id,
+ };
+ allModels.push({
+ ...this.transformAdminModelToModel(
+ modelWithProvider,
+ provider.provider_type
+ ),
+ has_own_api_key: false,
+ api_key_type: 'group',
+ });
+ });
+
+ providerModels.remote_models.forEach((remoteModel) => {
+ if (!seenModelIds.has(remoteModel.id)) {
const modelWithProvider = {
- ...dbModel,
+ ...remoteModel,
upstream_provider_id: provider.id,
};
allModels.push({
@@ -462,33 +498,11 @@ export class AdminService {
provider.provider_type
),
has_own_api_key: false,
- api_key_type: 'group',
+ api_key_type: 'remote',
+ soft_deleted: false,
});
- });
-
- providerModels.remote_models.forEach((remoteModel) => {
- if (!seenModelIds.has(remoteModel.id)) {
- const modelWithProvider = {
- ...remoteModel,
- upstream_provider_id: provider.id,
- };
- allModels.push({
- ...this.transformAdminModelToModel(
- modelWithProvider,
- provider.provider_type
- ),
- has_own_api_key: false,
- api_key_type: 'remote',
- soft_deleted: false,
- });
- }
- });
- } catch (error) {
- console.error(
- `Failed to fetch models for provider ${provider.id}:`,
- error
- );
- }
+ }
+ });
}
return { models: allModels, groups };
diff --git a/ui/lib/hooks/use-models-with-providers.ts b/ui/lib/hooks/use-models-with-providers.ts
new file mode 100644
index 00000000..6aeec5cf
--- /dev/null
+++ b/ui/lib/hooks/use-models-with-providers.ts
@@ -0,0 +1,26 @@
+'use client';
+
+import { useQuery } from '@tanstack/react-query';
+import { AdminService } from '@/lib/api/services/admin';
+
+export const modelsWithProvidersQueryKey = ['models-with-providers'] as const;
+
+/**
+ * Shared catalog read for every models view, so the page shell and the
+ * selector panel share one request instead of each fanning out to providers.
+ */
+export function useModelsWithProviders() {
+ const query = useQuery({
+ queryKey: modelsWithProvidersQueryKey,
+ queryFn: () => AdminService.getModelsWithProviders(),
+ refetchOnWindowFocus: false,
+ });
+
+ return {
+ models: query.data?.models ?? [],
+ groups: query.data?.groups ?? [],
+ isLoading: query.isLoading,
+ error: query.error,
+ refetch: query.refetch,
+ };
+}
diff --git a/ui/lib/hooks/use-progressive-list.ts b/ui/lib/hooks/use-progressive-list.ts
new file mode 100644
index 00000000..8b55ffc4
--- /dev/null
+++ b/ui/lib/hooks/use-progressive-list.ts
@@ -0,0 +1,46 @@
+'use client';
+
+import { useEffect, useState } from 'react';
+
+/**
+ * Reveal a long list in frame-sized batches.
+ *
+ * A provider catalog can hold thousands of rows, and mounting them in one
+ * commit blocks the main thread long enough that the page looks frozen right
+ * after navigation. Each batch yields back to the browser, so the first rows
+ * paint immediately and the rest fill in without freezing input.
+ */
+export function useProgressiveList(
+ items: T[],
+ initialCount = 40,
+ step = 80
+): { visibleItems: T[]; hiddenCount: number } {
+ const [count, setCount] = useState(initialCount);
+ const [trackedItems, setTrackedItems] = useState(items);
+
+ // Reset during render, not in an effect: an effect would first commit the new
+ // list at the old (possibly full) count, which is the freeze this avoids.
+ if (trackedItems !== items) {
+ setTrackedItems(items);
+ setCount(initialCount);
+ }
+
+ useEffect(() => {
+ if (count >= items.length) {
+ return;
+ }
+
+ const frame = requestAnimationFrame(() => {
+ setCount((current) => Math.min(items.length, current + step));
+ });
+
+ return () => cancelAnimationFrame(frame);
+ }, [count, items.length, step]);
+
+ const visibleCount = Math.min(count, items.length);
+
+ return {
+ visibleItems: items.slice(0, visibleCount),
+ hiddenCount: items.length - visibleCount,
+ };
+}
diff --git a/uv.lock b/uv.lock
index fad67b51..86d2be5d 100644
--- a/uv.lock
+++ b/uv.lock
@@ -337,6 +337,34 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/9e/78e59887cbf94116bdc890af7726ae264d55df14f1c777724c656e8a35fe/bolt11-2.1.1-py3-none-any.whl", hash = "sha256:fd4edb9e73e27bf5e017f47c97f7c6827b523fcf9cab152b123961ca78323e2d", size = 17102, upload-time = "2025-03-12T13:33:08.142Z" },
]
+[[package]]
+name = "boto3"
+version = "1.43.105"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "botocore" },
+ { name = "jmespath" },
+ { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/75/46/d8c87ada70a7647fb3d206c7f19eafca3580a0ae4c06d62da539a1ee1207/boto3-1.43.105.tar.gz", hash = "sha256:e51260aed9cc1474778b5488bc6f97ad28f27a0a7002f4bbaaf8191aff1422ea", size = 112682, upload-time = "2026-09-29T19:37:40.784Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bc/8e/0310a37ff609529dab9153cbc9fd0b66c685364d741bb6d1ae31134b728e/boto3-1.43.105-py3-none-any.whl", hash = "sha256:b8b6236ae7fe2724eee608c9b0649afbb86f0ec98158f39f67e64e678ec47499", size = 140042, upload-time = "2026-09-29T19:37:39.415Z" },
+]
+
+[[package]]
+name = "botocore"
+version = "1.43.105"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "jmespath" },
+ { name = "python-dateutil" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2b/30/668f3c0533a440787e212cf56404cb6ec234ae8e6baf97fe17329d512d88/botocore-1.43.105.tar.gz", hash = "sha256:afb3e7706b123ab069d1c34571ca1fdf82528a48425574fe4693df3d039d503f", size = 16263910, upload-time = "2026-09-29T19:37:36.456Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f3/94/50923cd46840e4d2b56cad1dcf5008fb20c099b04b2d93d32231f2d0bfaa/botocore-1.43.105-py3-none-any.whl", hash = "sha256:7abd19e1ef2c5e4a0314ca493fa7cebefabe33e559d7dd570fe2432a5431e6ec", size = 15958067, upload-time = "2026-09-29T19:37:33.373Z" },
+]
+
[[package]]
name = "brotli"
version = "1.2.0"
@@ -1509,6 +1537,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
]
+[[package]]
+name = "jmespath"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
+]
+
[[package]]
name = "jsonschema"
version = "4.26.0"
@@ -1552,10 +1589,11 @@ wheels = [
[[package]]
name = "litellm"
-version = "1.93.2"
+version = "1.101.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
+ { name = "boto3" },
{ name = "click" },
{ name = "fastuuid" },
{ name = "httpx", extra = ["socks"] },
@@ -1564,40 +1602,20 @@ dependencies = [
{ name = "jsonschema" },
{ name = "openai" },
{ name = "pydantic" },
+ { name = "pydantic-settings" },
{ name = "python-dotenv" },
{ name = "tiktoken" },
{ name = "tokenizers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/97/dd/28024c0e4cf2dc6ab1bad59b8357af7f460e952c69526eae28f12ac4ee5e/litellm-1.93.2.tar.gz", hash = "sha256:c5d5223ef07f36e0886397fb45cc9db4150f86a0c6f6835cee1d5524cab69dfd", size = 15955441, upload-time = "2026-08-09T02:17:49.646Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/26/c9/cb2730c6c763233e322fe7c5b2f53783eb10893cea9304e5474f1f20c306/litellm-1.101.2.tar.gz", hash = "sha256:790adf4ce19116d7bf4342492b1be5a90dd56e08d89795979bf6c1c3446a9670", size = 17493188, upload-time = "2026-09-24T00:04:22.712Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/64/c7/cb3f49dc60d57dda7fe368310fd5da2a94ec9b6a746bcf343a61e10bdeda/litellm-1.93.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:1bd0690efc94357e559de97927fd98437555cd5b5dd832544cfcca87297ccb80", size = 19938326, upload-time = "2026-08-09T02:16:38.041Z" },
- { url = "https://files.pythonhosted.org/packages/0c/bd/d77184fdaaf57d67d65da91dcfc61c7f656703e7ce4f950e07523e7de4e3/litellm-1.93.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:845ececc628737909b1422d1af18bd19ae453727a66244aa9da3ca37a3773111", size = 19862606, upload-time = "2026-08-09T02:16:40.653Z" },
- { url = "https://files.pythonhosted.org/packages/53/99/d8dd58b6840754a13cc2e1111b283aa28cbfc0ccc653a8725050916bb08e/litellm-1.93.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:498f9878ea773305e0638b6159d7e1ef27bb0b9a4292538d6634312d18a4e781", size = 20168532, upload-time = "2026-08-09T02:16:42.997Z" },
- { url = "https://files.pythonhosted.org/packages/d7/ca/559ca0f5e0b99b9f641086ae924c782f8d521d09384fbe9abbe0bddb6e61/litellm-1.93.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1e5618ef495b2e02299b376ca84ffb2647837aafee478cf3a1be17d47a8f0f73", size = 20162696, upload-time = "2026-08-09T02:16:45.283Z" },
- { url = "https://files.pythonhosted.org/packages/92/3e/18c31b27c7d1271b43bdc8ffbef01bfba68d90248bbe60bb2130dd17e43c/litellm-1.93.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2da463d70c9fffbea9532fd000e035328f5266b399a2fb4c6c76b3470478337", size = 20233518, upload-time = "2026-08-09T02:16:47.87Z" },
- { url = "https://files.pythonhosted.org/packages/d9/98/a6bae7c52f09cd03487a040f98eeedb899b3cf3fc541b87c6d051ee92e0d/litellm-1.93.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2cf122399f84f8f04621ed6ef8f276dd6d61f4fab108932ce0e30368de34dd42", size = 20291180, upload-time = "2026-08-09T02:16:50.549Z" },
- { url = "https://files.pythonhosted.org/packages/77/2d/81d974f2533cf039afda7e3e0f769dc73dc692c75ec867cf29ec6f41c06f/litellm-1.93.2-cp311-cp311-win_amd64.whl", hash = "sha256:8eaaf780fab9a19234735ef94225172179d15bc28b67ddbec125194249a504b7", size = 19775654, upload-time = "2026-08-09T02:16:53.162Z" },
- { url = "https://files.pythonhosted.org/packages/d0/05/72fd8051f0f2f3c84b90986e6f4551db7c8b190ba3300f111461b7701689/litellm-1.93.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3bf532c164ad7cb1b76f2c62afefdcc656b9b296374d075a4150e2ce10bb74c3", size = 19937403, upload-time = "2026-08-09T02:16:55.545Z" },
- { url = "https://files.pythonhosted.org/packages/9e/4d/5081b39bdb73cab04f8a86294a4534a029cf0434ac6932c7ae8049d55723/litellm-1.93.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:526b7afc037f79dfdd5c607f5085ac597c7fd301a6dedabea40baae899b27f19", size = 19853652, upload-time = "2026-08-09T02:16:57.977Z" },
- { url = "https://files.pythonhosted.org/packages/70/3f/fb70691266a7fd08c202406abea0153e82fa17f134cd9d58e4029cc741db/litellm-1.93.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:294ad19f356f821ce97a5428d09439be5f38d22b218c73008d8a49e3e42eb145", size = 20165680, upload-time = "2026-08-09T02:17:00.65Z" },
- { url = "https://files.pythonhosted.org/packages/81/91/84424ce2a25595463e5d24e9cf8949877cd4ce93c0fcbf6486ecd685094f/litellm-1.93.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6f6a5e3907f0a1c9d8ff8d71a6cbac8a592e47a40da3f97167074947b5ba7d11", size = 20157772, upload-time = "2026-08-09T02:17:03.027Z" },
- { url = "https://files.pythonhosted.org/packages/8f/8d/b0eac7ee6d174564f820565c8c9a726ae83dbb8c4d3522daf175b95da002/litellm-1.93.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8541f1b7fd5c437ad249ad68d0a11f68e5e2866b0649da5fa7d63b595e9b8b22", size = 20229256, upload-time = "2026-08-09T02:17:05.271Z" },
- { url = "https://files.pythonhosted.org/packages/ee/6d/03e931c1cb2d1e1b7a968de21aa9e4db853928200da856c35c940ee6faa9/litellm-1.93.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:712c9387419d7b06a10df59973f5e530592d61b2314102b0fa3142f3743f9a9e", size = 20287257, upload-time = "2026-08-09T02:17:08.175Z" },
- { url = "https://files.pythonhosted.org/packages/16/05/6c0fe2fcf31c260474c55fabe4ecb0e9e1343c9b9132e28589391b2ad33e/litellm-1.93.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc0d58ccabd22ef7ef44a9e6f7247deb54ae42f5e126e6f00360c2b28b41bc2b", size = 19772580, upload-time = "2026-08-09T02:17:11.254Z" },
- { url = "https://files.pythonhosted.org/packages/70/74/e9046cffa69b32b710452480598e418b26a29896ece680c80ec23997fd16/litellm-1.93.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f4071bef03e4c2942cd2ddc752727345b85447d6a7fee1ff5a4f8b92187966b0", size = 19938095, upload-time = "2026-08-09T02:17:13.929Z" },
- { url = "https://files.pythonhosted.org/packages/fa/db/6ef38a7a2f73d5cc507423954fa535a8546ead375c4c71265c093bdb4e9e/litellm-1.93.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a99ac7c0c1b78acd6bfd1959e9f203dca71fdbceb5f0c8691c2ad8eee450d7d", size = 19854187, upload-time = "2026-08-09T02:17:16.588Z" },
- { url = "https://files.pythonhosted.org/packages/cb/b3/80ee0143b88e2921f8c8f24c7331478258a8bf25a3d4d4450bd96043403e/litellm-1.93.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a81ceff44c58ef504ab8bd787d03b82618765b9cfd530942386ae6d23c58be94", size = 20166307, upload-time = "2026-08-09T02:17:19.078Z" },
- { url = "https://files.pythonhosted.org/packages/98/60/cb326e1094f7042f28f9e21543d9f367a8aa25af6915bf4253b77da5c2a2/litellm-1.93.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:dee1b02b7f52a5a408bf7c8d499f0834e49194651743758a511dcdd926c0b692", size = 20158336, upload-time = "2026-08-09T02:17:21.507Z" },
- { url = "https://files.pythonhosted.org/packages/b1/87/bad75146863531172c9dbae189486c7f4425b56a6641b55ab20745316048/litellm-1.93.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d2edfa14b99bce706b35981703692e3ee631f9b87bf6dc28fb53b574f6480b20", size = 20229711, upload-time = "2026-08-09T02:17:24.073Z" },
- { url = "https://files.pythonhosted.org/packages/df/28/040b1853021ed8fd57be19eb2affb024d168951fe7e7abdbad91da3f6f3f/litellm-1.93.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ae75a61c9abc827aa3131b7e640c952367a450830bb7c531b426b4ec2bb45f85", size = 20287584, upload-time = "2026-08-09T02:17:26.542Z" },
- { url = "https://files.pythonhosted.org/packages/d9/0b/4208815b0d666636cbf7afbd571eec3004d3a15d3150a23a9009fc2ce930/litellm-1.93.2-cp313-cp313-win_amd64.whl", hash = "sha256:c54a09ab20f94120a9d60a30d9970439dcefa00d2565d190505ff006a80c7a69", size = 19772641, upload-time = "2026-08-09T02:17:29.308Z" },
- { url = "https://files.pythonhosted.org/packages/09/4a/ff7a9c000519d2bab362318bf744a24c2500228e5fceaa6ac23acab96fa0/litellm-1.93.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:204cb0763fff9285bc87eb2dc0fc59b591999e5d94863d0964f806424d3c0cd6", size = 19943639, upload-time = "2026-08-09T02:17:31.811Z" },
- { url = "https://files.pythonhosted.org/packages/c4/26/29e9276ce4aa8ed133d9fd5ecc07375017d2228215547c6bbb17ccbc59b4/litellm-1.93.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3126c84361606b9fb07fde7d57eccd8a1747304d64c4143e2e5e40ae6e7693fb", size = 19855435, upload-time = "2026-08-09T02:17:34.376Z" },
- { url = "https://files.pythonhosted.org/packages/f8/20/2c9c818248ae019b2d496ca41900a9a5651ab05e2400794cd8dc8b89b6d2/litellm-1.93.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:1c84f7c4acb4e926a79b93145ab23231b300fc687bde7172ef884fc52d6011e0", size = 20166947, upload-time = "2026-08-09T02:17:36.828Z" },
- { url = "https://files.pythonhosted.org/packages/50/af/4016682be48350407837941ad1a1ae8185cca65b102e04e89eee2a2abccb/litellm-1.93.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:cacf35cf703b12c54516fc6464a3e08c6dbb1dcfb97239e1f629294fe36a1cba", size = 20160055, upload-time = "2026-08-09T02:17:39.674Z" },
- { url = "https://files.pythonhosted.org/packages/5b/b5/c25d7fbe08490d8211bd6b69af23f3a922b68ad8c87c776480b0de64a505/litellm-1.93.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0a7f3e5138e307e429bd8fa29cc0c48bb1e2b827792e8f7799ca4c8cff736103", size = 20230910, upload-time = "2026-08-09T02:17:42.159Z" },
- { url = "https://files.pythonhosted.org/packages/21/27/341b18a40d4d98a2ac09025c248a3a7edddaf15ce4096ac4a783ff2f70db/litellm-1.93.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8684629be3f7b5f8e2b6e5fe5ea27ff957c63a8d525d81c1460d8436e2e1857", size = 20288903, upload-time = "2026-08-09T02:17:44.433Z" },
- { url = "https://files.pythonhosted.org/packages/8d/45/dd9ef72075a83854f852b1bf9a97ec7029a2be9fb4e338fc6623eb09fc90/litellm-1.93.2-cp314-cp314-win_amd64.whl", hash = "sha256:a783b8b18ed68cb6a3b79d2b00273ec21aef92442e9b2712a50036cb84bfe583", size = 19772974, upload-time = "2026-08-09T02:17:46.972Z" },
+ { url = "https://files.pythonhosted.org/packages/44/a7/4bccec0ac9cb1b2e94e391b666458d07480d342039c66383ac191819e8e7/litellm-1.101.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:48c42c2c2cf9d4b0d75f4e1670b1b64b9e0513488d0b737fe057fda0bc716551", size = 23827328, upload-time = "2026-09-24T00:04:03.196Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/3d/faf394e5ac5a1469de5cbd3939c9e330f744e648351934b981078ccc40d9/litellm-1.101.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:77195c8ed502c052bb31d4c3887356a308ac2e8c3b0b30c97e2b04b0be10dd44", size = 23484770, upload-time = "2026-09-24T00:04:06.248Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/84/60f70aa2683666626c4abe7aa44b52acca52cad911ee870ea244fd3b0796/litellm-1.101.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:abb7b3ac04f56ced46e53cca2369a5dd29539cab9fdcd6cc9a94987aa55a38d1", size = 23618000, upload-time = "2026-09-24T00:04:08.827Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/8e/c57a4e157f97b1bcef9b410d51e17507047bbb11c676f5c81b22e7190c7c/litellm-1.101.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:210c89194225778759aa6649f5c0d605572bf14708ec85712462ace479f47f04", size = 23994795, upload-time = "2026-09-24T00:04:12.292Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/49/8737aee5a5a15cac7eb8a972b800529923e2837bbbadf0617f56111344ab/litellm-1.101.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:144dd8d1ead7174a718d1748deffcda7438cf7dfe8dc7c20761b72c35117e9a3", size = 23693332, upload-time = "2026-09-24T00:04:14.886Z" },
+ { url = "https://files.pythonhosted.org/packages/04/50/4e711caa0374309d6aaf5696549449c2078f0225dd36a22ee0ca44dd068f/litellm-1.101.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ae95e7ef15e109472f2cec69da6028a874b416e56a704cd9b797362993fc63cf", size = 24092655, upload-time = "2026-09-24T00:04:17.677Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/7d/32d391ddcb30d4d5d08fddd0abe918e836f9b3f753237c2b12ecb3d7425a/litellm-1.101.2-cp310-abi3-win_amd64.whl", hash = "sha256:0f5ee6daf9082b7efca1dc851c10c0d4884a2f1e0d7ea410bd508961b5a2cdae", size = 23894930, upload-time = "2026-09-24T00:04:20.432Z" },
]
[[package]]
@@ -2439,6 +2457,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" },
]
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
[[package]]
name = "python-dotenv"
version = "1.2.2"
@@ -2722,7 +2752,7 @@ requires-dist = [
{ name = "greenlet", specifier = ">=3.2.1" },
{ name = "h11", specifier = ">=0.16" },
{ name = "httpx", extras = ["socks"], specifier = ">=0.28.1" },
- { name = "litellm", specifier = ">=1.93.0,<1.94" },
+ { name = "litellm", specifier = ">=1.101.2,<1.102" },
{ name = "marshmallow", specifier = ">=3.13,<4.0" },
{ name = "mdurl", specifier = "==0.1.2" },
{ name = "nostr-sdk", specifier = ">=0.45.1,<0.46" },
@@ -2882,6 +2912,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" },
]
+[[package]]
+name = "s3transfer"
+version = "0.19.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" },
+]
+
[[package]]
name = "setuptools"
version = "84.0.0"