chore: merge main

.env.example

@@ -72,6 +72,12 @@ ROUTSTR_SECRET_KEY=
 # UPSTREAM_POOL_TIMEOUT=5
 # UPSTREAM_READ_TIMEOUT=900
 
+# Request and reservation lifetime limits (seconds)
+# STALE_RESERVATION_TIMEOUT_SECONDS=300
+# MAX_REQUEST_LIFETIME_SECONDS=1800
+# DOWNSTREAM_SEND_TIMEOUT_SECONDS=60
+# REQUEST_CLEANUP_TIMEOUT_SECONDS=30
+
 # Logging
 # LOG_LEVEL=INFO
 # ENABLE_CONSOLE_LOGGING=true

migrations/versions/a73d19b6c204_reservation_deadlines.py

@@ -0,0 +1,42 @@
+"""Immutable reservation start and absolute recovery deadline.
+
+Revision ID: a73d19b6c204
+Revises: e4c7a1b9d520
+"""
+
+import time
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "a73d19b6c204"
+down_revision = "e4c7a1b9d520"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "reservation_releases", sa.Column("started_at", sa.Integer(), nullable=True)
+    )
+    op.add_column(
+        "reservation_releases", sa.Column("expires_at", sa.Integer(), nullable=True)
+    )
+    op.create_index(
+        "ix_reservation_releases_expires_at", "reservation_releases", ["expires_at"]
+    )
+    # Original ages are unknowable for renewed legacy rows. Give them a finite
+    # migration grace period; deploy only after draining old workers.
+    op.execute(
+        sa.text(
+            "UPDATE reservation_releases SET expires_at = :expiry WHERE status = 'active'"
+        ).bindparams(expiry=int(time.time()) + 1830)
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_reservation_releases_expires_at", table_name="reservation_releases"
+    )
+    op.drop_column("reservation_releases", "expires_at")
+    op.drop_column("reservation_releases", "started_at")

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "mdurl==0.1.2",
     "pillow>=10",
     "openai>=1.98.0",
-    "litellm>=1.93.0,<1.94",  # 1.93 is the first line supporting Python 3.14
+    "litellm>=1.101.2,<1.102",
     "orjson>=3.10",
 ]
 
@@ -72,10 +72,12 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools]
 packages = ["routstr"]
 
+[tool.ruff]
+extend-exclude = ["examples"]
+
 [tool.ruff.lint]
 select = ["E", "F", "I"]
 ignore = ["E501"]
-exclude = ["examples"]
 
 [tool.mypy]
 python_version = "3.11"

routstr/auth.py

@@ -49,9 +49,7 @@ payments_logger = get_logger("routstr.payments")
 
 # Routstr platform fee constants
 ROUTSTR_FEE_PERCENT: float = 2.1
-ROUTSTR_LN_ADDRESS: str = (
-    "npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
-)
+ROUTSTR_LN_ADDRESS: str = "routstr-fees@rizful.com"
 ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
 ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
 
@@ -637,6 +635,14 @@ async def pay_for_request(
     )
 
     # Charge the base cost for the request atomically to avoid race conditions
+    from .core.lifecycle import request_lifetime
+
+    lifetime = request_lifetime.get()
+    remaining_lifetime = (
+        max(0, lifetime.deadline - asyncio.get_running_loop().time())
+        if lifetime is not None
+        else settings.max_request_lifetime_seconds
+    )
     reserved_at_now = int(time.time())
     stmt = (
         update(ApiKey)
@@ -686,6 +692,13 @@ async def pay_for_request(
             billing_key_hash=reservation.billing_key_hash,
             reserved_msats=reservation.reserved_msats,
             status="active",
+            started_at=reserved_at_now,
+            # reserved_at_now floors to the second; add 1s margin so a
+            # finalizer finishing right at the nominal deadline isn't fenced
+            # out by truncation.
+            expires_at=reserved_at_now
+            + math.ceil(remaining_lifetime + settings.request_cleanup_timeout_seconds)
+            + 1,
         )
     )
     # Publish the identity before commit. If the commit succeeds but its
@@ -875,6 +888,10 @@ async def renew_reservation(
         update(ReservationRelease)
         .where(col(ReservationRelease.id) == snapshot.release_id)
         .where(col(ReservationRelease.status) == "active")
+        .where(
+            (col(ReservationRelease.expires_at).is_(None))
+            | (col(ReservationRelease.expires_at) > int(time.time()))
+        )
         .values(created_at=int(time.time()))
     )
     await session.commit()
@@ -902,12 +919,21 @@ def _start_reservation_heartbeat(snapshot: ReservationSnapshot) -> None:
     """
     interval = max(1, settings.stale_reservation_timeout_seconds // 3)
     owner = asyncio.current_task()
+    from .core.lifecycle import request_lifetime
+
+    lifetime = request_lifetime.get()
+    deadline = asyncio.get_running_loop().time() + settings.max_request_lifetime_seconds
 
     async def beat() -> None:
         try:
             while True:
                 await asyncio.sleep(interval)
-                if owner is None or owner.done():
+                if (
+                    owner is None
+                    or owner.done()
+                    or (lifetime is not None and lifetime.stopped)
+                    or asyncio.get_running_loop().time() >= deadline
+                ):
                     # Request control is gone; let the lease expire so the
                     # sweeper can release the reservation if no terminal
                     # transition ever ran.
@@ -1091,6 +1117,10 @@ async def _claim_reservation_for_charge(
         update(ReservationRelease)
         .where(col(ReservationRelease.id) == snapshot.release_id)
         .where(col(ReservationRelease.status) == "active")
+        .where(
+            col(ReservationRelease.expires_at).is_(None)
+            | (col(ReservationRelease.expires_at) > int(time.time()))
+        )
         .where(col(ReservationRelease.key_hash) == snapshot.key_hash)
         .where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
         .where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)

routstr/core/db.py

@@ -174,7 +174,10 @@ async def _transition_stale_reservation(
         update(ReservationRelease)
         .where(col(ReservationRelease.id) == reservation_id)
         .where(col(ReservationRelease.status) == "active")
-        .where(col(ReservationRelease.created_at) < cutoff)
+        .where(
+            (col(ReservationRelease.created_at) < cutoff)
+            | (col(ReservationRelease.expires_at) <= int(time.time()))
+        )
         .values(status="released")
     )
     return bool(transition.rowcount == 1)
@@ -221,7 +224,10 @@ async def release_stale_reservations(
     query = (
         select(ReservationRelease)
         .where(col(ReservationRelease.status) == "active")
-        .where(col(ReservationRelease.created_at) < cutoff)
+        .where(
+            (col(ReservationRelease.created_at) < cutoff)
+            | (col(ReservationRelease.expires_at) <= int(time.time()))
+        )
     )
     if key_hash is not None:
         query = query.where(
@@ -784,6 +790,8 @@ class ReservationRelease(SQLModel, table=True):  # type: ignore
     key_hash: str = Field(index=True)
     billing_key_hash: str = Field(index=True)
     reserved_msats: int
+    started_at: int | None = Field(default=None)
+    expires_at: int | None = Field(default=None, index=True)
     status: str = Field(default="active")
     created_at: int = Field(default_factory=lambda: int(time.time()))
 

routstr/core/lifecycle.py

@@ -0,0 +1,134 @@
+"""Supervise the real downstream connection, outside HTTP middleware wrappers."""
+
+from __future__ import annotations
+
+import asyncio
+from contextvars import ContextVar
+from dataclasses import dataclass
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from . import get_logger
+from .settings import settings
+
+logger = get_logger(__name__)
+
+
+class DownstreamTerminated(OSError):
+    """Raised by downstream_send after disconnect; expected, not a server error."""
+
+
+@dataclass
+class RequestLifetime:
+    deadline: float = 0
+    stopped: bool = False
+
+
+request_lifetime: ContextVar[RequestLifetime | None] = ContextVar(
+    "request_lifetime", default=None
+)
+
+
+class RequestLifecycleMiddleware:
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        lifetime = RequestLifetime(
+            deadline=asyncio.get_running_loop().time()
+            + settings.max_request_lifetime_seconds
+        )
+        token = request_lifetime.set(lifetime)
+        disconnected = asyncio.Event()
+        # One receive consumer. Backpressure uploads until consumed; after the
+        # final body message, continue listening independently of the app.
+        messages: asyncio.Queue[Message] = asyncio.Queue(maxsize=1)
+        response_started = False
+
+        async def pump() -> None:
+            while True:
+                message = await receive()
+                if message["type"] == "http.disconnect":
+                    disconnected.set()
+                    return
+                await messages.put(message)
+
+        async def downstream_receive() -> Message:
+            if disconnected.is_set():
+                return {"type": "http.disconnect"}
+            get = asyncio.create_task(messages.get())
+            gone = asyncio.create_task(disconnected.wait())
+            try:
+                await asyncio.wait((get, gone), return_when=asyncio.FIRST_COMPLETED)
+                if disconnected.is_set():
+                    return {"type": "http.disconnect"}
+                return get.result()
+            finally:
+                for task in (get, gone):
+                    task.cancel()
+                await asyncio.gather(get, gone, return_exceptions=True)
+
+        async def downstream_send(message: Message) -> None:
+            nonlocal response_started
+            if disconnected.is_set() or lifetime.stopped:
+                raise DownstreamTerminated("Downstream request terminated")
+            async with asyncio.timeout(settings.downstream_send_timeout_seconds):
+                await send(message)
+            if message["type"] == "http.response.start":
+                response_started = True
+
+        receiver = asyncio.create_task(pump())
+        work: asyncio.Future[None] = asyncio.ensure_future(
+            self.app(scope, downstream_receive, downstream_send)
+        )
+        gone = asyncio.create_task(disconnected.wait())
+        timed_out = False
+        try:
+            done, _ = await asyncio.wait(
+                (work, gone),
+                timeout=settings.max_request_lifetime_seconds,
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+            if gone in done and not response_started and work not in done:
+                # A pre-response wallet or billing operation may have accepted
+                # funds already. Let it reach its own settlement before closing.
+                done, _ = await asyncio.wait(
+                    (work,),
+                    timeout=max(0, lifetime.deadline - asyncio.get_running_loop().time()),
+                )
+            if work in done:
+                try:
+                    await work
+                except DownstreamTerminated:
+                    if not disconnected.is_set():
+                        raise
+                    logger.debug("Client disconnected before response completed")
+            elif not disconnected.is_set():
+                timed_out = True
+        finally:
+            lifetime.stopped = True
+            for task in (receiver, gone, work):
+                task.cancel()
+            # Detached stream finalizers own settlement. The heartbeat stops
+            # with the request; durable expiry recovers any abandoned row.
+            done, pending = await asyncio.wait(
+                (receiver, gone, work), timeout=settings.request_cleanup_timeout_seconds
+            )
+            for task in done:
+                if not task.cancelled():
+                    task.exception()
+            for task in pending:
+                task.cancel()
+                task.add_done_callback(
+                    lambda t: t.exception() if not t.cancelled() else None
+                )
+            request_lifetime.reset(token)
+        if timed_out and work.done() and not disconnected.is_set() and not response_started:
+            async with asyncio.timeout(settings.downstream_send_timeout_seconds):
+                await send({"type": "http.response.start", "status": 504, "headers": []})
+                await send(
+                    {"type": "http.response.body", "body": b"Request deadline exceeded"}
+                )

routstr/core/main.py

@@ -45,6 +45,7 @@ from .exceptions import (
     http_exception_handler,
     validation_exception_handler,
 )
+from .lifecycle import RequestLifecycleMiddleware
 from .logging import get_logger, setup_logging
 from .middleware import LoggingMiddleware
 from .not_found import _NOT_FOUND_HTML, not_found_catch_all  # noqa: F401
@@ -315,6 +316,10 @@ app.add_middleware(
 # Add logging middleware
 app.add_middleware(LoggingMiddleware)
 
+# Outermost: observe the actual downstream connection, not middleware streams.
+
+app.add_middleware(RequestLifecycleMiddleware)
+
 # Add exception handlers
 app.add_exception_handler(HTTPException, http_exception_handler)  # type: ignore
 app.add_exception_handler(RequestValidationError, validation_exception_handler)

routstr/core/settings.py

@@ -117,6 +117,16 @@ class Settings(BaseSettings):
         default=604_800, env="DEAD_KEY_MIN_AGE_SECONDS"
     )
 
+    max_request_lifetime_seconds: float = Field(
+        default=1800, gt=0, env="MAX_REQUEST_LIFETIME_SECONDS"
+    )
+    downstream_send_timeout_seconds: float = Field(
+        default=60, gt=0, env="DOWNSTREAM_SEND_TIMEOUT_SECONDS"
+    )
+    request_cleanup_timeout_seconds: float = Field(
+        default=30, gt=0, env="REQUEST_CLEANUP_TIMEOUT_SECONDS"
+    )
+
     # Network
     cors_origins: list[str] = Field(default_factory=lambda: ["*"], env="CORS_ORIGINS")
     # Comma-separated METHOD:path pairs adding to the proxy's canonical

routstr/proxy.py

@@ -1030,7 +1030,7 @@ async def _proxy(
                         already_stripped.add(bad_param)
                         logger.warning(
                             "Upstream %s rejected param '%s' for model=%s; "
-                            "stripping and retrying same upstream",
+                            "correcting and retrying same upstream",
                             upstream.provider_type,
                             bad_param,
                             model_id,

routstr/upstream/base.py

@@ -2613,6 +2613,11 @@ class BaseUpstreamProvider:
     ) -> dict:
         return await messages_dispatch.aggregate_anthropic_events_to_message(iterator)
 
+    def transform_messages_stream(
+        self, stream: AsyncIterator[Any]
+    ) -> AsyncIterator[Any]:
+        return stream
+
     def adapt_messages_request(self, body: dict, model_obj: Model) -> str:
         """Rewrite an allowlisted /v1/messages body for this upstream.
 
@@ -2638,6 +2643,7 @@ class BaseUpstreamProvider:
             provider_prefix=self.get_litellm_provider_prefix(),
             transform_model_name=self.transform_model_name,
             adapt_request=lambda body: self.adapt_messages_request(body, model_obj),
+            transform_stream=self.transform_messages_stream,
             log_extra=log_extra,
         )
 

routstr/upstream/messages_dispatch.py

@@ -406,16 +406,9 @@ def annotate_event(event: dict, requested_model: str | None) -> AnnotatedEvent:
             _coerce_float(root_cost_details.get("output_cost")),
         )
 
-    event_type = str(event.get("type") or "")
-    payload = json.dumps(event)
-    if event_type:
-        sse_bytes = f"event: {event_type}\ndata: {payload}\n\n".encode()
-    else:
-        sse_bytes = f"data: {payload}\n\n".encode()
-
     return AnnotatedEvent(
         event,
-        sse_bytes,
+        encode_sse(event),
         in_tokens,
         out_tokens,
         cache_read_tokens,
@@ -427,6 +420,14 @@ def annotate_event(event: dict, requested_model: str | None) -> AnnotatedEvent:
     )
 
 
+def encode_sse(event: dict) -> bytes:
+    event_type = str(event.get("type") or "")
+    payload = json.dumps(event)
+    if event_type:
+        return f"event: {event_type}\ndata: {payload}\n\n".encode()
+    return f"data: {payload}\n\n".encode()
+
+
 async def stream_annotated_events(
     iterator: AsyncIterator[Any],
     requested_model: str | None,
@@ -493,6 +494,7 @@ async def dispatch_anthropic_messages(
     provider_prefix: str,
     transform_model_name: Callable[[str], str],
     adapt_request: Callable[[dict], str] | None = None,
+    transform_stream: Callable[[AsyncIterator[Any]], AsyncIterator[Any]] | None = None,
     log_extra: dict[str, Any] | None = None,
 ) -> tuple[bool, Any, str | None]:
     """Call ``litellm.anthropic.messages.acreate`` and return
@@ -505,6 +507,10 @@ async def dispatch_anthropic_messages(
     may rewrite it in place and returns a suffix for the upstream model name,
     which is how a provider expresses a feature litellm would otherwise
     translate into a parameter the upstream rejects.
+
+    ``transform_stream`` rewrites the upstream event stream before it is
+    aggregated or handed to the client, so a provider can repair events
+    litellm translates faithfully but clients cannot use.
     """
     if not request_body:
         raise UpstreamError("Missing request body for /v1/messages", status_code=400)
@@ -640,6 +646,9 @@ async def dispatch_anthropic_messages(
             from_upstream_response=True,
         ) from exc
 
+    if transform_stream is not None and hasattr(result, "__aiter__"):
+        result = transform_stream(cast(AsyncIterator[Any], result))
+
     if not client_stream and hasattr(result, "__aiter__"):
         # Client asked for a non-streaming response but we always stream
         # from upstream — drain the events into a single Anthropic Message

routstr/upstream/openai.py

@@ -1,11 +1,23 @@
+import json
 from typing import TYPE_CHECKING
 
+from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
+from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig
+
 from ..payment.models import Model, async_fetch_openrouter_models
 from .base import BaseUpstreamProvider
 
 if TYPE_CHECKING:
     from ..core.db import UpstreamProviderRow
 
+_O_SERIES = OpenAIOSeriesConfig()
+
+
+def _rejects_max_tokens(model: str) -> bool:
+    return OpenAIGPT5Config.is_model_gpt_5_model(
+        model
+    ) or _O_SERIES.is_model_o_series_model(model)
+
 
 class OpenAIUpstreamProvider(BaseUpstreamProvider):
     """Upstream provider specifically configured for OpenAI API."""
@@ -43,6 +55,33 @@ class OpenAIUpstreamProvider(BaseUpstreamProvider):
         """Strip 'openai/' prefix for OpenAI API compatibility."""
         return model_id.removeprefix("openai/")
 
+    def prepare_request_body(
+        self,
+        body: bytes | None,
+        model_obj: Model,
+        include_stream_usage: bool = False,
+    ) -> bytes | None:
+        body = super().prepare_request_body(body, model_obj, include_stream_usage)
+        if not body:
+            return body
+        try:
+            data = json.loads(body)
+        except ValueError:
+            return body
+        # Reasoning models 400 on max_tokens; renaming up front saves the
+        # reject-and-retry round trip. Names litellm doesn't know yet still
+        # fall through to request_correction's reactive rename.
+        if (
+            isinstance(data, dict)
+            and "messages" in data
+            and "max_tokens" in data
+            and "max_completion_tokens" not in data
+            and _rejects_max_tokens(self.transform_model_name(model_obj.id))
+        ):
+            data["max_completion_tokens"] = data.pop("max_tokens")
+            return json.dumps(data).encode()
+        return body
+
     async def fetch_models(self) -> list[Model]:
         """Fetch OpenAI models from OpenRouter API filtered by openai source."""
         models_data = await async_fetch_openrouter_models(source_filter="openai")

routstr/upstream/request_correction.py

@@ -43,6 +43,18 @@ _UNSUPPORTED_PARAM_RE = re.compile(
 )
 
 
+# Matches upstream error text that rejects a param and names its replacement,
+# e.g. OpenAI's "Unsupported parameter: 'max_tokens' is not supported with this
+# model. Use 'max_completion_tokens' instead." Both names must be quoted so a
+# free-form hint like "use gpt-4 instead" never reads as a rename.
+_RENAMED_PARAM_RE = re.compile(
+    r"[`'\"](?P<param>[a-zA-Z_][a-zA-Z0-9_]*)[`'\"]\s+is\s+"
+    r"(?:deprecated|not\s+supported|unsupported|no\s+longer\s+supported)\b"
+    r".*?\buse\s+[`'\"](?P<replacement>[a-zA-Z_][a-zA-Z0-9_]*)[`'\"]\s+instead",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
 # A corrector inspects the parsed request body and the upstream error message
 # and returns ``(new_body_dict, label)`` for a fix it can apply, or ``None`` to
 # decline. ``label`` identifies the fix so it is applied at most once per request.
@@ -100,6 +112,51 @@ _SPEND_SHAPING_PARAMS = frozenset(
     }
 )
 
+# Output caps are interchangeable spellings of the same limit, so moving the
+# value from one to another keeps the priced bound intact.
+_OUTPUT_CAP_PARAMS = frozenset(
+    {
+        "max_tokens",
+        "max_completion_tokens",
+        "max_output_tokens",
+        "max_tokens_to_sample",
+    }
+)
+
+
+def rename_unsupported_param(body: dict, error_message: str) -> tuple[dict, str] | None:
+    """Move a rejected top-level param to the name the upstream asked for.
+
+    Returns ``(new_body, label)`` with the value carried over unchanged, or
+    ``None`` when the error names no replacement, the param is absent, or the
+    replacement is already set.
+
+    A spend-shaping field is only renamed to another output cap: that keeps the
+    reservation's bound, whereas renaming into or out of any other spend-shaping
+    field could uncap or fan out the retry.
+    """
+    match = _RENAMED_PARAM_RE.search(error_message)
+    if not match:
+        return None
+    param, replacement = match.group("param"), match.group("replacement")
+    if param == replacement or param not in body or replacement in body:
+        return None
+    param_spend = param.lower() in _SPEND_SHAPING_PARAMS
+    replacement_spend = replacement.lower() in _SPEND_SHAPING_PARAMS
+    if (param_spend or replacement_spend) and not (
+        param.lower() in _OUTPUT_CAP_PARAMS
+        and replacement.lower() in _OUTPUT_CAP_PARAMS
+    ):
+        logger.warning(
+            "Upstream asked to rename '%s' to '%s'; refusing because it would "
+            "change the request's spend bound — surfacing the error",
+            param,
+            replacement,
+        )
+        return None
+    new_body = {(replacement if k == param else k): v for k, v in body.items()}
+    return new_body, f"{param}->{replacement}"
+
 
 def strip_unsupported_param(body: dict, error_message: str) -> tuple[dict, str] | None:
     """Drop a top-level param the upstream named as unsupported/deprecated.
@@ -130,7 +187,12 @@ def strip_unsupported_param(body: dict, error_message: str) -> tuple[dict, str]
 
 
 # Ordered pipeline of correctors tried on each recoverable rejection.
-DEFAULT_CORRECTORS: tuple[Corrector, ...] = (strip_unsupported_param,)
+# Renaming runs first so a param with a named replacement keeps its value
+# instead of being dropped.
+DEFAULT_CORRECTORS: tuple[Corrector, ...] = (
+    rename_unsupported_param,
+    strip_unsupported_param,
+)
 
 
 def correct_request(

routstr/upstream/stream_ownership.py

@@ -10,6 +10,7 @@ from fastapi.responses import StreamingResponse
 from starlette.types import Receive, Scope, Send
 
 from ..core import get_logger
+from ..core.settings import settings
 
 logger = get_logger(__name__)
 
@@ -74,10 +75,14 @@ class PersistentStreamFinalizer:
         self._task: asyncio.Future[None] | None = None
         self._lock = asyncio.Lock()
 
+    async def _bounded_finalize(self) -> None:
+        async with asyncio.timeout(settings.request_cleanup_timeout_seconds):
+            await self._finalize()
+
     async def run(self) -> None:
         async with self._lock:
             if self._task is None:
-                self._task = asyncio.ensure_future(self._finalize())
+                self._task = asyncio.ensure_future(self._bounded_finalize())
             task = self._task
         await asyncio.shield(task)
 

routstr/upstream/venice.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from collections.abc import AsyncGenerator, AsyncIterator
+from typing import TYPE_CHECKING, Any, cast
 
 import httpx
 
 from ..core.exceptions import UpstreamError
 from ..core.logging import get_logger
 from ..payment.models import Architecture, Model, Pricing, TopProvider
+from . import messages_dispatch
 from .base import BaseUpstreamProvider
+from .stream_ownership import aclose_if_needed
 
 if TYPE_CHECKING:
     from ..core.db import UpstreamProviderRow
@@ -50,6 +53,73 @@ _UNENFORCEABLE_WEB_SEARCH_KEYS = frozenset(
     {"allowed_domains", "blocked_domains", "user_location"}
 )
 
+# Venice streams OpenAI reasoning models' encrypted reasoning as a trailing
+# ``reasoning_content`` delta carrying this marker. litellm turns it into a
+# plaintext ``thinking`` block after the answer, which clients render as
+# gibberish and which makes Claude Code report an empty final result.
+_ENCRYPTED_REASONING_MARKER = "__ENCRYPTED_REASONING__"
+
+
+async def _drop_encrypted_reasoning(
+    upstream: AsyncIterator[Any],
+) -> AsyncGenerator[bytes, None]:
+    """A thinking block's start carries no text, so it is held until its first
+    delta shows whether it is the encrypted payload; later indices shift down
+    to close the gap."""
+    encode = messages_dispatch.encode_sse
+    sse_buffer = b""
+    dropped: set[int] = set()
+    held: list[dict] | None = None
+    held_index: int | None = None
+
+    def shift(event: dict) -> dict:
+        index = event.get("index")
+        if not isinstance(index, int):
+            return event
+        gap = sum(1 for d in dropped if d < index)
+        return {**event, "index": index - gap} if gap else event
+
+    try:
+        async for chunk in upstream:
+            events, sse_buffer = messages_dispatch.events_from_chunk(chunk, sse_buffer)
+            for event in events:
+                etype = event.get("type")
+                index = event.get("index")
+                if held is not None:
+                    delta = event.get("delta") or {}
+                    is_own_delta = (
+                        index == held_index and etype == "content_block_delta"
+                    )
+                    thinking = str(delta.get("thinking") or "")
+                    if is_own_delta and thinking.startswith(
+                        _ENCRYPTED_REASONING_MARKER
+                    ):
+                        dropped.add(cast(int, index))
+                        held = None
+                        continue
+                    if is_own_delta and not thinking:
+                        held.append(event)
+                        continue
+                    for pending in held:
+                        yield encode(shift(pending))
+                    held = None
+                if index in dropped:
+                    continue
+                block = event.get("content_block") or {}
+                if (
+                    etype == "content_block_start"
+                    and block.get("type") == "thinking"
+                    and not block.get("thinking")
+                ):
+                    held, held_index = [event], index
+                    continue
+                yield encode(shift(event))
+        if held is not None:
+            for pending in held:
+                yield encode(shift(pending))
+    finally:
+        await aclose_if_needed(upstream)
+
 
 def _is_web_search_tool(tool: Any) -> bool:
     """An Anthropic server-side web-search tool, by either of its markers.
@@ -66,6 +136,35 @@ def _is_web_search_tool(tool: Any) -> bool:
     ) or tool.get("name") == "web_search"
 
 
+def _merge_cache_marked_system(body: dict) -> None:
+    """Venice rejects an OpenAI ``system`` message with two or more text parts
+    when any part carries ``cache_control`` (``400 system: text content blocks
+    must contain non-whitespace text``), even though every part is non-blank.
+    Claude Code always sends that shape. A single marked block is accepted and
+    still caches, so the prefix stays cacheable under the last marker.
+    """
+    system = body.get("system")
+    if not isinstance(system, list) or len(system) < 2:
+        return
+    if not all(
+        isinstance(block, dict)
+        and block.get("type") == "text"
+        and isinstance(block.get("text"), str)
+        for block in system
+    ):
+        return
+    markers = [block["cache_control"] for block in system if block.get("cache_control")]
+    if not markers:
+        return
+    body["system"] = [
+        {
+            "type": "text",
+            "text": "\n\n".join(block["text"] for block in system),
+            "cache_control": markers[-1],
+        }
+    ]
+
+
 def _usd(entry: Any) -> float | None:
     """Read the USD leg of a Venice ``{usd, diem}`` price pair."""
     if isinstance(entry, dict):
@@ -114,7 +213,16 @@ class VeniceUpstreamProvider(BaseUpstreamProvider):
     def transform_model_name(self, model_id: str) -> str:
         return model_id.removeprefix("venice/")
 
+    def transform_messages_stream(
+        self, stream: AsyncIterator[Any]
+    ) -> AsyncIterator[Any]:
+        return _drop_encrypted_reasoning(stream)
+
     def adapt_messages_request(self, body: dict, model_obj: Model) -> str:
+        _merge_cache_marked_system(body)
+        return self._adapt_web_search(body)
+
+    def _adapt_web_search(self, body: dict) -> str:
         """Trade an Anthropic web-search tool for Venice's own search switch.
 
         Left in the body, litellm's Anthropic adapter rewrites the tool into a

tests/unit/test_litellm_routing.py

@@ -91,3 +91,24 @@ def test_detect_litellm_prefix_custom_default() -> None:
     assert detect_litellm_prefix("https://example.com", default="anthropic/") == (
         "anthropic/"
     )
+
+
+@pytest.mark.parametrize("model", ["gpt-6", "gpt-6-luna", "gpt-5.5"])
+def test_litellm_sends_max_completion_tokens_for_gpt_5_and_later(model: str) -> None:
+    """OpenAI rejects ``max_tokens`` on these models; litellm <1.101 only
+    rewrote it for names containing ``gpt-5``, so gpt-6 got a 400."""
+    import litellm
+    from litellm.utils import ProviderConfigManager
+
+    config = ProviderConfigManager.get_provider_chat_config(
+        model=model, provider=litellm.LlmProviders.OPENAI
+    )
+    assert config is not None
+    mapped = config.map_openai_params(
+        non_default_params={"max_tokens": 10},
+        optional_params={},
+        model=model,
+        drop_params=True,
+    )
+
+    assert mapped == {"max_completion_tokens": 10}

tests/unit/test_model_path_routing.py

@@ -706,6 +706,75 @@ async def test_pinned_recovery_preserves_routing_fields(
     fallback.forward_request.assert_not_awaited()
 
 
+_OPENAI_MAX_TOKENS_ERROR = json.dumps(
+    {
+        "error": {
+            "message": "Unsupported parameter: 'max_tokens' is not supported "
+            "with this model. Use 'max_completion_tokens' instead.",
+            "type": "invalid_request_error",
+            "param": "max_tokens",
+            "code": "unsupported_parameter",
+        }
+    }
+).encode()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("pinned", [False, True])
+async def test_rejected_max_tokens_is_renamed_and_retried_on_same_upstream(
+    pinned: bool,
+) -> None:
+    selected, fallback = _make_upstream(1), _make_upstream(2)
+    selected.forward_request = AsyncMock(
+        side_effect=[
+            MagicMock(status_code=400, body=_OPENAI_MAX_TOKENS_ERROR),
+            MagicMock(status_code=200, body=b"{}"),
+        ]
+    )
+    headers = {"authorization": "Bearer key"}
+    if pinned:
+        headers["x-routstr-model-path"] = encode_model_path(selected.base_url, MODEL_ID)
+    request = _make_request(
+        headers,
+        json.dumps(
+            {"model": MODEL_ID, "max_tokens": 300, "messages": [], "stream": True}
+        ).encode(),
+    )
+
+    response = await _run_proxy(
+        request, [(MagicMock(), selected), (MagicMock(), fallback)]
+    )
+
+    assert response.status_code == 200
+    assert selected.forward_request.await_count == 2
+    before, after = [
+        json.loads(call.args[3]) for call in selected.forward_request.await_args_list
+    ]
+    assert before["max_tokens"] == 300 and "max_completion_tokens" not in before
+    assert after["max_completion_tokens"] == 300 and "max_tokens" not in after
+    assert {k: v for k, v in after.items() if k != "max_completion_tokens"} == {
+        k: v for k, v in before.items() if k != "max_tokens"
+    }
+    fallback.forward_request.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_rename_that_changes_spend_bound_is_not_retried() -> None:
+    selected = _make_upstream(1, 400)
+    selected.forward_request.return_value.body = json.dumps(
+        {"error": {"message": "'max_tokens' is not supported. Use 'n' instead."}}
+    ).encode()
+    request = _make_request(
+        {"authorization": "Bearer key"},
+        json.dumps({"model": MODEL_ID, "max_tokens": 300}).encode(),
+    )
+
+    response = await _run_proxy(request, [(MagicMock(), selected)])
+
+    assert response.status_code == 400
+    selected.forward_request.assert_awaited_once()
+
+
 # --------------------------------------------------------------------------- #
 # Upstream 5xx -> 424 + UPSTREAM_UNAVAILABLE + scope header; node faults stay 500.
 # --------------------------------------------------------------------------- #

tests/unit/test_openai_output_cap.py

@@ -0,0 +1,90 @@
+"""OpenAI reasoning models get ``max_completion_tokens`` before the request is sent."""
+
+from __future__ import annotations
+
+import json
+import os
+
+os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
+os.environ.setdefault("UPSTREAM_API_KEY", "test")
+os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
+
+import pytest
+
+from routstr.payment.models import Architecture, Model, Pricing
+from routstr.upstream import GenericUpstreamProvider
+from routstr.upstream.openai import OpenAIUpstreamProvider
+
+
+def _model(model_id: str) -> Model:
+    return Model(
+        id=model_id,
+        name="test",
+        created=0,
+        description="",
+        context_length=128000,
+        architecture=Architecture(
+            modality="text->text",
+            input_modalities=["text"],
+            output_modalities=["text"],
+            tokenizer="x",
+            instruct_type=None,
+        ),
+        pricing=Pricing(prompt=0.0, completion=0.0),
+    )
+
+
+def _chat(model_id: str, **fields: object) -> bytes:
+    return json.dumps(
+        {"model": model_id, "messages": [{"role": "user", "content": "hi"}], **fields}
+    ).encode()
+
+
+def _prepare(provider: object, model_id: str, body: bytes) -> dict:
+    out = provider.prepare_request_body(body, _model(model_id))  # type: ignore[attr-defined]
+    assert out is not None
+    return json.loads(out)
+
+
+@pytest.mark.parametrize(
+    "model_id", ["gpt-5.6-sol", "openai/gpt-6-sol", "openai/gpt-5", "o3", "o4-mini"]
+)
+def test_reasoning_model_max_tokens_is_renamed(model_id: str) -> None:
+    provider = OpenAIUpstreamProvider(api_key="k")
+    data = _prepare(provider, model_id, _chat(model_id, max_tokens=300))
+    assert data["max_completion_tokens"] == 300
+    assert "max_tokens" not in data
+
+
+@pytest.mark.parametrize("model_id", ["gpt-4o", "openai/gpt-4.1"])
+def test_non_reasoning_model_keeps_max_tokens(model_id: str) -> None:
+    provider = OpenAIUpstreamProvider(api_key="k")
+    data = _prepare(provider, model_id, _chat(model_id, max_tokens=300))
+    assert data["max_tokens"] == 300
+    assert "max_completion_tokens" not in data
+
+
+def test_both_caps_set_is_left_for_upstream() -> None:
+    provider = OpenAIUpstreamProvider(api_key="k")
+    data = _prepare(
+        provider,
+        "gpt-5.6-sol",
+        _chat("gpt-5.6-sol", max_tokens=300, max_completion_tokens=200),
+    )
+    assert data["max_tokens"] == 300
+    assert data["max_completion_tokens"] == 200
+
+
+def test_non_chat_body_is_untouched() -> None:
+    provider = OpenAIUpstreamProvider(api_key="k")
+    body = json.dumps({"model": "gpt-5.6-sol", "input": "hi", "max_tokens": 5}).encode()
+    data = _prepare(provider, "gpt-5.6-sol", body)
+    assert data["max_tokens"] == 5
+    assert "max_completion_tokens" not in data
+
+
+def test_other_upstreams_keep_max_tokens() -> None:
+    provider = GenericUpstreamProvider(base_url="http://test", api_key="k")
+    data = _prepare(provider, "gpt-5.6-sol", _chat("gpt-5.6-sol", max_tokens=300))
+    assert data["max_tokens"] == 300
+    assert "max_completion_tokens" not in data

tests/unit/test_request_correction.py

@@ -16,9 +16,15 @@ from routstr.upstream.request_correction import (
     Correction,
     correct_request,
     extract_error_message,
+    rename_unsupported_param,
     strip_unsupported_param,
 )
 
+OPENAI_MAX_TOKENS_ERROR = (
+    "Unsupported parameter: 'max_tokens' is not supported with this model. "
+    "Use 'max_completion_tokens' instead."
+)
+
 
 def _body(**kwargs: object) -> bytes:
     return json.dumps(kwargs).encode()
@@ -139,6 +145,190 @@ class TestStripUnsupportedParam:
         assert strip_unsupported_param(body, "`Max_Tokens` is deprecated") is None
 
 
+class TestRenameUnsupportedParam:
+    def test_renames_max_tokens_for_openai_reasoning_models(self) -> None:
+        body = {"model": "gpt-5.6-sol", "max_tokens": 256, "messages": []}
+        result = rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR)
+        assert result is not None
+        new_body, label = result
+        assert label == "max_tokens->max_completion_tokens"
+        assert new_body == {
+            "model": "gpt-5.6-sol",
+            "max_completion_tokens": 256,
+            "messages": [],
+        }
+
+    def test_preserves_key_order(self) -> None:
+        body = {"model": "m", "max_tokens": 1, "stream": True}
+        result = rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR)
+        assert result is not None
+        assert list(result[0]) == ["model", "max_completion_tokens", "stream"]
+
+    def test_does_not_mutate_input(self) -> None:
+        body = {"model": "m", "max_tokens": 8}
+        assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is not None
+        assert body == {"model": "m", "max_tokens": 8}
+
+    def test_renames_between_any_output_caps(self) -> None:
+        caps = (
+            "max_tokens",
+            "max_completion_tokens",
+            "max_output_tokens",
+            "max_tokens_to_sample",
+        )
+        for param in caps:
+            for replacement in caps:
+                if param == replacement:
+                    continue
+                message = f"`{param}` is deprecated. Use `{replacement}` instead."
+                result = rename_unsupported_param({param: 7}, message)
+                assert result == ({replacement: 7}, f"{param}->{replacement}"), (
+                    param,
+                    replacement,
+                )
+
+    def test_renames_non_spend_param(self) -> None:
+        message = "'functions' is deprecated. Use 'tools' instead."
+        result = rename_unsupported_param({"functions": [{"name": "f"}]}, message)
+        assert result == ({"tools": [{"name": "f"}]}, "functions->tools")
+
+    def test_matches_across_quote_styles_case_and_newlines(self) -> None:
+        for message in (
+            'Unsupported parameter: "max_tokens" is not supported.\nUse '
+            '"max_completion_tokens" instead.',
+            "`max_tokens` IS UNSUPPORTED here; please USE `max_completion_tokens`"
+            " INSTEAD",
+            "'max_tokens' is no longer supported, use 'max_completion_tokens' instead",
+        ):
+            result = rename_unsupported_param({"max_tokens": 3}, message)
+            assert result is not None, message
+            assert result[0] == {"max_completion_tokens": 3}
+
+    def test_refuses_renames_that_change_the_spend_bound(self) -> None:
+        for param, replacement in (
+            ("max_tokens", "n"),
+            ("n", "best_of"),
+            ("best_of", "n"),
+            ("temperature", "max_tokens"),
+            ("max_tokens", "temperature"),
+            ("n", "max_tokens"),
+        ):
+            message = f"'{param}' is not supported. Use '{replacement}' instead."
+            assert rename_unsupported_param({param: 2}, message) is None, (
+                param,
+                replacement,
+            )
+
+    def test_spend_guard_is_case_insensitive(self) -> None:
+        ok = "'Max_Tokens' is not supported. Use 'MAX_COMPLETION_TOKENS' instead."
+        assert rename_unsupported_param({"Max_Tokens": 4}, ok) == (
+            {"MAX_COMPLETION_TOKENS": 4},
+            "Max_Tokens->MAX_COMPLETION_TOKENS",
+        )
+        bad = "'Max_Tokens' is not supported. Use 'N' instead."
+        assert rename_unsupported_param({"Max_Tokens": 4}, bad) is None
+
+    def test_declines_when_replacement_already_present(self) -> None:
+        body = {"max_tokens": 4, "max_completion_tokens": 8}
+        assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is None
+
+    def test_declines_when_param_absent(self) -> None:
+        assert rename_unsupported_param({"model": "m"}, OPENAI_MAX_TOKENS_ERROR) is None
+
+    def test_declines_self_rename(self) -> None:
+        message = "'max_tokens' is deprecated. Use 'max_tokens' instead."
+        assert rename_unsupported_param({"max_tokens": 1}, message) is None
+
+    def test_declines_unquoted_or_missing_replacement(self) -> None:
+        for message in (
+            "`gpt-3` is deprecated, use gpt-4 instead",
+            "'max_tokens' is not supported, use max_completion_tokens instead",
+            "'max_tokens' is not supported with this model.",
+            "Use 'max_completion_tokens' instead.",
+        ):
+            assert rename_unsupported_param({"max_tokens": 1}, message) is None, message
+
+    def test_declines_nested_only_param(self) -> None:
+        body = {"reasoning": {"max_tokens": 5}}
+        assert rename_unsupported_param(body, OPENAI_MAX_TOKENS_ERROR) is None
+
+
+class TestCorrectRequestRename:
+    def test_openai_max_tokens_error_is_renamed_not_refused(self) -> None:
+        body = _body(model="gpt-5.6-sol", max_tokens=512, messages=[])
+        result = correct_request(body, OPENAI_MAX_TOKENS_ERROR, set())
+        assert isinstance(result, Correction)
+        assert result.label == "max_tokens->max_completion_tokens"
+        decoded = json.loads(result.body)
+        assert "max_tokens" not in decoded
+        assert decoded["max_completion_tokens"] == 512
+
+    def test_rename_wins_over_strip_for_non_spend_param(self) -> None:
+        body = _body(model="m", functions=[1])
+        result = correct_request(
+            body, "'functions' is deprecated. Use 'tools' instead.", set()
+        )
+        assert result is not None
+        assert json.loads(result.body) == {"model": "m", "tools": [1]}
+
+    def test_unsafe_rename_of_cap_still_surfaces_error(self) -> None:
+        body = _body(model="m", max_tokens=5)
+        assert (
+            correct_request(
+                body, "'max_tokens' is not supported. Use 'n' instead.", set()
+            )
+            is None
+        )
+
+    def test_applied_rename_does_not_repeat_or_strip_cap(self) -> None:
+        body = _body(model="m", max_tokens=5)
+        applied = {"max_tokens->max_completion_tokens"}
+        assert correct_request(body, OPENAI_MAX_TOKENS_ERROR, applied) is None
+
+    def test_rename_ping_pong_terminates(self) -> None:
+        """An upstream that flip-flops between names cannot loop forever."""
+        forward = OPENAI_MAX_TOKENS_ERROR
+        backward = "'max_completion_tokens' is not supported. Use 'max_tokens' instead."
+        body = _body(model="m", max_tokens=5)
+        applied: set[str] = set()
+        for attempt in range(10):
+            message = forward if attempt % 2 == 0 else backward
+            result = correct_request(body, message, applied)
+            if result is None:
+                break
+            body, applied = result.body, applied | {result.label}
+        else:
+            raise AssertionError("correction loop did not terminate")
+        assert applied == {
+            "max_tokens->max_completion_tokens",
+            "max_completion_tokens->max_tokens",
+        }
+        assert json.loads(body) == {"model": "m", "max_tokens": 5}
+
+    def test_buffered_openai_error_response_is_renamed(self) -> None:
+        resp = Response(
+            content=json.dumps(
+                {
+                    "error": {
+                        "message": OPENAI_MAX_TOKENS_ERROR,
+                        "type": "invalid_request_error",
+                        "param": "max_tokens",
+                        "code": "unsupported_parameter",
+                    }
+                }
+            ).encode(),
+            status_code=400,
+        )
+        body = _body(model="gpt-5.6-sol", max_tokens=64, stream=True)
+        result = correct_request(body, extract_error_message(resp), set())
+        assert result is not None
+        assert json.loads(result.body) == {
+            "model": "gpt-5.6-sol",
+            "max_completion_tokens": 64,
+            "stream": True,
+        }
+
+
 class TestExtractErrorMessage:
     def test_extracts_nested_error_message(self) -> None:
         resp = Response(

tests/unit/test_request_lifecycle.py

@@ -0,0 +1,285 @@
+import asyncio
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+from sqlmodel import SQLModel
+from sqlmodel.ext.asyncio.session import AsyncSession
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+from starlette.routing import Route
+from starlette.types import Message, Receive, Scope, Send
+
+import routstr.core.db as db_module
+from routstr.auth import (
+    ReservationSnapshot,
+    _claim_reservation_for_charge,
+    _stop_reservation_heartbeat,
+    pay_for_request,
+)
+from routstr.core.db import ApiKey, ReservationRelease
+from routstr.core.lifecycle import RequestLifecycleMiddleware
+from routstr.core.middleware import LoggingMiddleware
+from routstr.core.settings import settings
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("reason", ["disconnect", "deadline", "send"])
+async def test_lifecycle_stops_live_work(reason: str) -> None:
+    closed = asyncio.Event()
+    receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+    await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+    sent: list[Message] = []
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        try:
+            assert (await receive())["type"] == "http.request"
+            await send({"type": "http.response.start", "status": 200, "headers": []})
+            while True:
+                await send(
+                    {"type": "http.response.body", "body": b"x", "more_body": True}
+                )
+                await asyncio.sleep(0.01)
+        finally:
+            closed.set()
+
+    async def send(message: Message) -> None:
+        sent.append(message)
+        if reason == "send" and message["type"] == "http.response.body":
+            await asyncio.sleep(100)
+
+    async def disconnect() -> None:
+        await asyncio.sleep(0.02)
+        await receive_queue.put({"type": "http.disconnect"})
+
+    task = asyncio.create_task(disconnect()) if reason == "disconnect" else None
+    with (
+        patch.object(settings, "max_request_lifetime_seconds", 0.08),
+        patch.object(settings, "downstream_send_timeout_seconds", 0.03),
+        patch.object(settings, "request_cleanup_timeout_seconds", 0.1),
+    ):
+        try:
+            await asyncio.wait_for(
+                RequestLifecycleMiddleware(app)(
+                    {"type": "http"}, receive_queue.get, send
+                ),
+                1,
+            )
+        except TimeoutError:
+            assert reason == "send"
+    if task:
+        await task
+    assert closed.is_set()
+    assert sent
+
+
+@pytest.mark.asyncio
+async def test_unrelated_oserror_still_propagates() -> None:
+    receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+    await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        await receive()
+        raise OSError("Connection reset by peer")
+
+    async def send(message: Message) -> None:
+        pass
+
+    with pytest.raises(OSError, match="Connection reset by peer"):
+        await asyncio.wait_for(
+            RequestLifecycleMiddleware(app)({"type": "http"}, receive_queue.get, send),
+            1,
+        )
+
+
+@pytest.mark.asyncio
+async def test_disconnect_before_headers_preserves_wallet_work() -> None:
+    receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+    await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+    entered = asyncio.Event()
+    finish_wallet = asyncio.Event()
+    wallet_credited = asyncio.Event()
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        await receive()
+        entered.set()
+        await finish_wallet.wait()  # The mint accepted the token; credit is still pending.
+        wallet_credited.set()
+        await send({"type": "http.response.start", "status": 200, "headers": []})
+
+    run = asyncio.create_task(
+        RequestLifecycleMiddleware(app)(
+            {"type": "http"}, receive_queue.get, lambda message: asyncio.sleep(0)
+        )
+    )
+    await asyncio.wait_for(entered.wait(), 1)
+    await receive_queue.put({"type": "http.disconnect"})
+    await asyncio.sleep(0.02)
+    assert not run.done()
+    finish_wallet.set()
+    await asyncio.wait_for(run, 1)  # No propagated exception for an expected disconnect.
+    assert wallet_credited.is_set()
+
+
+@pytest.mark.asyncio
+async def test_disconnect_before_headers_with_logging_middleware() -> None:
+    receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+    await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+    entered = asyncio.Event()
+    finish_wallet = asyncio.Event()
+    wallet_credited = asyncio.Event()
+
+    async def wallet(request: Request) -> PlainTextResponse:
+        await request.body()
+        entered.set()
+        await finish_wallet.wait()
+        wallet_credited.set()
+        return PlainTextResponse("settled")
+
+    app = RequestLifecycleMiddleware(
+        LoggingMiddleware(Starlette(routes=[Route("/wallet", wallet, methods=["POST"])]))
+    )
+    scope: Scope = {
+        "type": "http",
+        "asgi": {"version": "3.0", "spec_version": "2.4"},
+        "http_version": "1.1",
+        "method": "POST",
+        "scheme": "http",
+        "path": "/wallet",
+        "raw_path": b"/wallet",
+        "root_path": "",
+        "query_string": b"",
+        "headers": [],
+        "client": ("test", 1234),
+        "server": ("test", 80),
+    }
+
+    async def send(message: Message) -> None:
+        pass
+
+    run = asyncio.create_task(app(scope, receive_queue.get, send))
+    try:
+        await asyncio.wait_for(entered.wait(), 1)
+        await receive_queue.put({"type": "http.disconnect"})
+        await asyncio.sleep(0.02)
+        assert not run.done()
+        finish_wallet.set()
+        await asyncio.wait_for(run, 1)  # No propagated exception for an expected disconnect.
+        assert wallet_credited.is_set()
+    finally:
+        finish_wallet.set()
+        if not run.done():
+            run.cancel()
+            await asyncio.gather(run, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+async def test_deadline_cancels_app_before_sending_504() -> None:
+    receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+    await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+    sent: list[Message] = []
+    app_stopped = asyncio.Event()
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        await receive()
+        try:
+            await asyncio.sleep(100)
+        finally:
+            with pytest.raises(OSError, match="Downstream request terminated"):
+                await send({"type": "http.response.start", "status": 200, "headers": []})
+            app_stopped.set()
+
+    async def send(message: Message) -> None:
+        assert app_stopped.is_set()
+        sent.append(message)
+
+    with (
+        patch.object(settings, "max_request_lifetime_seconds", 0.02),
+        patch.object(settings, "request_cleanup_timeout_seconds", 0.1),
+    ):
+        await asyncio.wait_for(
+            RequestLifecycleMiddleware(app)({"type": "http"}, receive_queue.get, send),
+            1,
+        )
+    assert [message["type"] for message in sent] == [
+        "http.response.start",
+        "http.response.body",
+    ]
+    assert sent[0]["status"] == 504
+
+
+@pytest.mark.asyncio
+async def test_disconnect_does_not_release_before_stream_settles(tmp_path: Path) -> None:
+    engine = create_async_engine(
+        f"sqlite+aiosqlite:///{tmp_path / 'reservations.db'}", poolclass=NullPool
+    )
+    async with engine.begin() as conn:
+        await conn.run_sync(SQLModel.metadata.create_all)
+
+    @asynccontextmanager
+    async def session() -> AsyncGenerator[AsyncSession, None]:
+        async with AsyncSession(engine, expire_on_commit=False) as db:
+            yield db
+
+    with patch.object(db_module, "create_session", session):
+        async with session() as db:
+            db.add(ApiKey(hashed_key="stream-key", balance=10_000))
+            await db.commit()
+        started = asyncio.Event()
+        finalizer_started = asyncio.Event()
+        settle = asyncio.Event()
+        result: asyncio.Future[bool] = asyncio.get_running_loop().create_future()
+        snapshot: ReservationSnapshot | None = None
+        receive_queue: asyncio.Queue[Message] = asyncio.Queue()
+        await receive_queue.put({"type": "http.request", "body": b"", "more_body": False})
+
+        async def app(scope: Scope, receive: Receive, send: Send) -> None:
+            nonlocal snapshot
+            async with session() as db:
+                key = await db.get(ApiKey, "stream-key")
+                assert key is not None
+                snapshot = await pay_for_request(key, 1000, db)
+            await receive()
+            await send({"type": "http.response.start", "status": 200, "headers": []})
+            started.set()
+            try:
+                await asyncio.sleep(100)
+            finally:
+                async def finalize() -> None:
+                    assert snapshot is not None
+                    finalizer_started.set()
+                    await settle.wait()
+                    async with session() as db:
+                        claimed = await _claim_reservation_for_charge(snapshot, db)
+                        await db.commit()
+                    await _stop_reservation_heartbeat(snapshot.release_id)
+                    result.set_result(claimed)
+
+                asyncio.create_task(finalize())
+
+        run = asyncio.create_task(
+            RequestLifecycleMiddleware(app)(
+                {"type": "http"}, receive_queue.get, lambda message: asyncio.sleep(0)
+            )
+        )
+        try:
+            await asyncio.wait_for(started.wait(), 1)
+            await receive_queue.put({"type": "http.disconnect"})
+            await asyncio.wait_for(finalizer_started.wait(), 1)
+            await asyncio.wait_for(run, 1)
+            settle.set()
+            assert await asyncio.wait_for(result, 1)
+            assert snapshot is not None
+            async with session() as db:
+                row = await db.get(ReservationRelease, snapshot.release_id)
+                assert row is not None and row.status == "charged"
+        finally:
+            settle.set()
+            if snapshot is not None:
+                await _stop_reservation_heartbeat(snapshot.release_id)
+            await engine.dispose()

tests/unit/test_stale_reservations.py

@@ -9,6 +9,7 @@ Covers:
 """
 
 import asyncio
+import math
 import time
 from typing import AsyncGenerator
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -69,7 +70,6 @@ async def test_pay_for_request_sets_reserved_at(
     payments_info = MagicMock()
     monkeypatch.setattr(auth_module.logger, "info", logger_info)
     monkeypatch.setattr(auth_module.payments_logger, "info", payments_info)
-
     before = int(time.time())
     await pay_for_request(key, 1_000, session)
 
@@ -87,6 +87,34 @@ async def test_pay_for_request_sets_reserved_at(
     assert payments_info.call_args.args == ("RESERVE",)
 
 
+@pytest.mark.asyncio
+async def test_pay_for_request_expires_at_has_floor_margin(
+    session: AsyncSession, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """reserved_at_now floors to the second; expires_at must add 1s so a
+    finalizer finishing exactly at the nominal deadline isn't fenced out."""
+    key = ApiKey(hashed_key="floorkey", balance=10_000)
+    session.add(key)
+    await session.commit()
+
+    fixed_time = 1_700_000_000.9  # fractional second, floors when int()'d
+    monkeypatch.setattr(auth_module.time, "time", lambda: fixed_time)
+
+    snapshot = await pay_for_request(key, 1_000, session)
+
+    row = await session.get(ReservationRelease, snapshot.release_id)
+    assert row is not None
+    expected = (
+        int(fixed_time)
+        + math.ceil(
+            auth_module.settings.max_request_lifetime_seconds
+            + auth_module.settings.request_cleanup_timeout_seconds
+        )
+        + 1
+    )
+    assert row.expires_at == expected
+
+
 @pytest.mark.asyncio
 @pytest.mark.asyncio
 async def test_pay_for_request_releases_reservation_when_validation_fails(
@@ -428,3 +456,62 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
             await proxy_module.proxy(request, "v1/chat/completions")
 
     revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
+
+
+@pytest.mark.asyncio
+async def test_absolute_expiry_releases_fresh_lease(session: AsyncSession) -> None:
+    now = int(time.time())
+    key = ApiKey(
+        hashed_key="expired-deadline",
+        balance=5000,
+        reserved_balance=1000,
+        reserved_at=now,
+    )
+    session.add(key)
+    session.add(
+        ReservationRelease(
+            id="expired",
+            key_hash=key.hashed_key,
+            billing_key_hash=key.hashed_key,
+            reserved_msats=1000,
+            created_at=now,
+            started_at=now - 100,
+            expires_at=now - 1,
+        )
+    )
+    await session.commit()
+    assert await release_stale_reservations(session, 300) == 1
+    await session.refresh(key)
+    assert key.reserved_balance == 0
+    assert key.balance == 5000
+
+
+@pytest.mark.asyncio
+async def test_expired_reservation_cannot_renew_or_claim_charge(
+    session: AsyncSession,
+) -> None:
+    from routstr.auth import (
+        ReservationSnapshot,
+        _claim_reservation_for_charge,
+        renew_reservation,
+    )
+
+    snapshot = ReservationSnapshot(
+        release_id="fenced",
+        key_hash="fenced-key",
+        billing_key_hash="fenced-key",
+        reserved_msats=1000,
+    )
+    session.add(ApiKey(hashed_key="fenced-key", balance=5000, reserved_balance=1000))
+    session.add(
+        ReservationRelease(
+            id="fenced",
+            key_hash="fenced-key",
+            billing_key_hash="fenced-key",
+            reserved_msats=1000,
+            expires_at=int(time.time()) - 1,
+        )
+    )
+    await session.commit()
+    assert not await renew_reservation(snapshot, session)
+    assert not await _claim_reservation_for_charge(snapshot, session)

tests/unit/test_venice_encrypted_reasoning.py

@@ -0,0 +1,210 @@
+import json
+from collections.abc import AsyncIterator
+from typing import Any
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from routstr.upstream import messages_dispatch
+from routstr.upstream.base import BaseUpstreamProvider
+from routstr.upstream.venice import VeniceUpstreamProvider, _drop_encrypted_reasoning
+
+from .test_venice_web_search import _model
+
+ENCRYPTED = "__ENCRYPTED_REASONING__id=rs_0b04\ngAAAAABqvDJD"
+
+
+def _block(index: int, block: dict, deltas: list[dict]) -> list[dict]:
+    return [
+        {"type": "content_block_start", "index": index, "content_block": block},
+        *({"type": "content_block_delta", "index": index, "delta": d} for d in deltas),
+        {"type": "content_block_stop", "index": index},
+    ]
+
+
+def _thinking(index: int, text: str) -> list[dict]:
+    return _block(
+        index,
+        {"type": "thinking", "thinking": "", "signature": ""},
+        [{"type": "thinking_delta", "thinking": text}],
+    )
+
+
+def _text(index: int, text: str) -> list[dict]:
+    return _block(
+        index,
+        {"type": "text", "text": ""},
+        [{"type": "text_delta", "text": text}],
+    )
+
+
+def _tool(index: int) -> list[dict]:
+    return _block(
+        index,
+        {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}},
+        [{"type": "input_json_delta", "partial_json": '{"command":"ls"}'}],
+    )
+
+
+def _message(blocks: list[dict], stop_reason: str = "end_turn") -> list[dict]:
+    return [
+        {
+            "type": "message_start",
+            "message": {"id": "msg_1", "role": "assistant", "content": []},
+        },
+        *blocks,
+        {"type": "message_delta", "delta": {"stop_reason": stop_reason}},
+        {"type": "message_stop"},
+    ]
+
+
+async def _upstream(events: list[dict], *, split: bool = False) -> AsyncIterator[Any]:
+    payload = b"".join(messages_dispatch.encode_sse(e) for e in events)
+    if split:
+        for i in range(0, len(payload), 7):
+            yield payload[i : i + 7]
+    else:
+        yield payload
+
+
+async def _filtered(events: list[dict], **kwargs: Any) -> list[dict]:
+    buffer = b""
+    out: list[dict] = []
+    async for chunk in _drop_encrypted_reasoning(_upstream(events, **kwargs)):
+        parsed, buffer = messages_dispatch.events_from_chunk(chunk, buffer)
+        out.extend(parsed)
+    return out
+
+
+def _starts(events: list[dict]) -> list[tuple[int, str]]:
+    return [
+        (e["index"], e["content_block"]["type"])
+        for e in events
+        if e["type"] == "content_block_start"
+    ]
+
+
+@pytest.mark.asyncio
+async def test_trailing_encrypted_reasoning_is_dropped() -> None:
+    events = _message([*_text(0, "a.txt contains: hello"), *_thinking(1, ENCRYPTED)])
+
+    out = await _filtered(events)
+
+    assert _starts(out) == [(0, "text")]
+    assert all(ENCRYPTED not in json.dumps(e) for e in out)
+    assert out[-2]["delta"]["stop_reason"] == "end_turn"
+
+
+@pytest.mark.asyncio
+async def test_leading_encrypted_reasoning_closes_index_gap() -> None:
+    events = _message(
+        [*_thinking(0, ENCRYPTED), *_text(1, "hi"), *_tool(2)], "tool_use"
+    )
+
+    out = await _filtered(events, split=True)
+
+    assert _starts(out) == [(0, "text"), (1, "tool_use")]
+    assert {e["index"] for e in out if "index" in e} == {0, 1}
+
+
+@pytest.mark.asyncio
+async def test_plaintext_thinking_is_kept_in_order() -> None:
+    events = _message([*_thinking(0, "Let me list files."), *_tool(1)], "tool_use")
+
+    out = await _filtered(events)
+
+    assert out == events
+
+
+@pytest.mark.asyncio
+async def test_thinking_start_without_delta_is_flushed() -> None:
+    events = _message(
+        [
+            {
+                "type": "content_block_start",
+                "index": 0,
+                "content_block": {"type": "thinking", "thinking": "", "signature": ""},
+            },
+            {"type": "content_block_stop", "index": 0},
+            *_text(1, "ok"),
+        ]
+    )
+
+    out = await _filtered(events)
+
+    assert out == events
+
+
+@pytest.mark.asyncio
+async def test_aggregated_message_ends_with_answer_text() -> None:
+    events = _message([*_text(0, "hello"), *_thinking(1, ENCRYPTED)])
+
+    message = await messages_dispatch.aggregate_anthropic_events_to_message(
+        _drop_encrypted_reasoning(_upstream(events))
+    )
+
+    assert [b["type"] for b in message["content"]] == ["text"]
+    assert message["content"][0]["text"] == "hello"
+
+
+async def _dispatched_blocks(
+    provider: BaseUpstreamProvider, *, stream: bool
+) -> list[str]:
+    events = _message([*_text(0, "hello"), *_thinking(1, ENCRYPTED)])
+    with patch(
+        "litellm.anthropic.messages.acreate",
+        new=AsyncMock(return_value=_upstream(events)),
+    ):
+        _, result, _ = await provider._dispatch_anthropic_messages(
+            request_body=json.dumps(
+                {
+                    "model": "x",
+                    "stream": stream,
+                    "max_tokens": 64,
+                    "messages": [{"role": "user", "content": "hi"}],
+                }
+            ).encode(),
+            model_obj=_model(),
+        )
+    if not stream:
+        return [b["type"] for b in result["content"]]
+    buffer = b""
+    out: list[dict] = []
+    async for chunk in result:
+        parsed, buffer = messages_dispatch.events_from_chunk(chunk, buffer)
+        out.extend(parsed)
+    return [t for _, t in _starts(out)]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [True, False])
+async def test_venice_dispatch_drops_encrypted_reasoning(stream: bool) -> None:
+    provider = VeniceUpstreamProvider(api_key="sk-test")
+
+    assert await _dispatched_blocks(provider, stream=stream) == ["text"]
+
+
+@pytest.mark.asyncio
+async def test_other_providers_keep_thinking_blocks() -> None:
+    provider = BaseUpstreamProvider(base_url="https://example.com/v1", api_key="k")
+
+    assert await _dispatched_blocks(provider, stream=True) == ["text", "thinking"]
+
+
+@pytest.mark.asyncio
+async def test_closing_the_filter_closes_upstream() -> None:
+    closed = False
+
+    async def upstream() -> AsyncIterator[bytes]:
+        nonlocal closed
+        try:
+            for event in _message(_text(0, "hello")):
+                yield messages_dispatch.encode_sse(event)
+        finally:
+            closed = True
+
+    filtered = _drop_encrypted_reasoning(upstream())
+    await filtered.__anext__()
+    await filtered.aclose()
+
+    assert closed

tests/unit/test_venice_system_cache.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import pytest
+
+from routstr.upstream.venice import VeniceUpstreamProvider
+
+from .test_venice_web_search import _body, _dispatch
+
+EPHEMERAL = {"type": "ephemeral"}
+
+CLAUDE_CODE_SYSTEM = [
+    {
+        "type": "text",
+        "text": "x-anthropic-billing-header: cc_version=2.1.281; cc_entrypoint=cli;",
+    },
+    {"type": "text", "text": "You are a Claude agent.", "cache_control": EPHEMERAL},
+    {
+        "type": "text",
+        "text": "\nYou are an interactive agent.",
+        "cache_control": {"type": "ephemeral", "ttl": "1h"},
+    },
+]
+
+
+@pytest.mark.asyncio
+async def test_cache_marked_multi_block_system_is_merged_into_one_block() -> None:
+    provider = VeniceUpstreamProvider(api_key="sk-test")
+
+    kwargs = await _dispatch(provider, _body(system=CLAUDE_CODE_SYSTEM))
+
+    assert kwargs["system"] == [
+        {
+            "type": "text",
+            "text": (
+                "x-anthropic-billing-header: cc_version=2.1.281; cc_entrypoint=cli;"
+                "\n\nYou are a Claude agent.\n\n\nYou are an interactive agent."
+            ),
+            "cache_control": {"type": "ephemeral", "ttl": "1h"},
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_unmarked_multi_block_system_is_untouched() -> None:
+    provider = VeniceUpstreamProvider(api_key="sk-test")
+    system = [{"type": "text", "text": "A."}, {"type": "text", "text": "B."}]
+
+    kwargs = await _dispatch(provider, _body(system=system))
+
+    assert kwargs["system"] == system
+
+
+@pytest.mark.asyncio
+async def test_single_marked_block_and_string_system_are_untouched() -> None:
+    provider = VeniceUpstreamProvider(api_key="sk-test")
+    single = [{"type": "text", "text": "A.", "cache_control": EPHEMERAL}]
+
+    assert (await _dispatch(provider, _body(system=single)))["system"] == single
+    assert (await _dispatch(provider, _body(system="A.")))["system"] == "A."
+
+
+@pytest.mark.asyncio
+async def test_message_and_tool_cache_markers_are_kept() -> None:
+    provider = VeniceUpstreamProvider(api_key="sk-test")
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "text", "text": "hi", "cache_control": EPHEMERAL}],
+        }
+    ]
+    tools = [
+        {
+            "name": "Bash",
+            "description": "Run a command",
+            "input_schema": {"type": "object", "properties": {}},
+            "cache_control": EPHEMERAL,
+        }
+    ]
+
+    kwargs = await _dispatch(
+        provider,
+        _body(system=CLAUDE_CODE_SYSTEM, messages=messages, tools=tools),
+    )
+
+    assert kwargs["messages"] == messages
+    assert kwargs["tools"] == tools

ui/app/model/loading.tsx

@@ -0,0 +1,23 @@
+import { AppPageShell } from '@/components/app-page-shell';
+import { PageHeader } from '@/components/page-header';
+import { Skeleton } from '@/components/ui/skeleton';
+
+/**
+ * Route-level fallback so clicking "Models" lands on the page immediately
+ * instead of holding the previous route until this one's chunk is parsed.
+ */
+export default function ModelPageLoading() {
+  return (
+    <AppPageShell contentClassName='mx-auto w-full max-w-5xl'>
+      <div className='space-y-3 sm:space-y-4'>
+        <PageHeader
+          title='Model Management'
+          description='Manage provider model catalogs and validate endpoints from one place.'
+        />
+        <Skeleton className='h-10 w-full' />
+        <Skeleton className='h-16 w-full' />
+        <Skeleton className='h-[420px] w-full' />
+      </div>
+    </AppPageShell>
+  );
+}

ui/components/model-provider-section.tsx

@@ -1,5 +1,6 @@
 import { useMemo } from 'react';
 import type { Model } from '@/lib/api/schemas/models';
+import { useProgressiveList } from '@/lib/hooks/use-progressive-list';
 import type { AdminModelGroup } from '@/lib/api/services/admin';
 import type { DisplayUnit } from '@/lib/types/units';
 import { ModelItemCard } from '@/components/model-item-card';
@@ -24,6 +25,7 @@ import {
   Edit3,
   Globe,
   Key,
+  Loader2,
   MoreVertical,
   RefreshCw,
 } from 'lucide-react';
@@ -103,10 +105,21 @@ export function ModelProviderSection({
     });
   }, [provider, providerModels]);
 
+  const { visibleItems: visibleProviderModels, hiddenCount } =
+    useProgressiveList(keyedProviderModels);
+
+  const pendingRowsNotice =
+    hiddenCount > 0 ? (
+      <div className='text-muted-foreground flex items-center justify-center gap-2 p-3 text-xs sm:text-sm'>
+        <Loader2 className='h-3.5 w-3.5 animate-spin' />
+        Rendering {hiddenCount} more model{hiddenCount === 1 ? '' : 's'}…
+      </div>
+    ) : null;
+
   if (filterProvider) {
     return (
       <div className='bg-card/35 border-border/70 md:divide-border/75 overflow-hidden rounded-lg border md:divide-y'>
-        {keyedProviderModels.map(({ model, renderKey }) => (
+        {visibleProviderModels.map(({ model, renderKey }) => (
           <ModelItemCard
             key={renderKey}
             model={model}
@@ -125,6 +138,7 @@ export function ModelProviderSection({
             onDelete={() => onDeleteModel(model.id)}
           />
         ))}
+        {pendingRowsNotice}
       </div>
     );
   }
@@ -217,7 +231,7 @@ export function ModelProviderSection({
 
       <CardContent className='px-3 pt-0 pb-3 sm:px-6 sm:pb-6'>
         <div className='bg-card/35 border-border/70 md:divide-border/75 overflow-hidden rounded-lg border md:divide-y'>
-          {keyedProviderModels.map(({ model, renderKey }) => (
+          {visibleProviderModels.map(({ model, renderKey }) => (
             <ModelItemCard
               key={renderKey}
               model={model}
@@ -236,6 +250,7 @@ export function ModelProviderSection({
               onDelete={() => onDeleteModel(model.id)}
             />
           ))}
+          {pendingRowsNotice}
         </div>
       </CardContent>
     </Card>

ui/components/model-selector.tsx

@@ -1,7 +1,7 @@
 'use client';
 
 import React, { useState, useMemo } from 'react';
-import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
+import { useMutation, useQueryClient } from '@tanstack/react-query';
 import { type Model, type GroupSettings } from '@/lib/api/schemas/models';
 import {
   AdminService,
@@ -13,6 +13,7 @@ import { AddProviderModelDialog } from '@/components/add-provider-model-dialog';
 import { EditGroupForm } from '@/components/edit-group-form';
 import { ModelProviderSection } from '@/components/model-provider-section';
 import { useDisplayCurrency } from '@/lib/hooks/use-display-currency';
+import { useModelsWithProviders } from '@/lib/hooks/use-models-with-providers';
 import { Button } from '@/components/ui/button';
 import { Checkbox } from '@/components/ui/checkbox';
 import { Skeleton } from '@/components/ui/skeleton';
@@ -131,19 +132,14 @@ export function ModelSelector({
 
   const queryClient = useQueryClient();
 
-  // Fetch models and groups
+  // Shared with the page shell, so mounting this panel costs no extra fetch.
   const {
-    data: modelsData,
+    models,
+    groups,
     isLoading: isLoadingModels,
     error: modelsError,
     refetch: refetchModels,
-  } = useQuery({
-    queryKey: ['models-with-providers'],
-    queryFn: () => AdminService.getModelsWithProviders(),
-    refetchOnWindowFocus: false,
-  });
-
-  const { models = [], groups = [] } = modelsData || {};
+  } = useModelsWithProviders();
   const allOverrideModels = useMemo(
     () => models.filter(isOverrideModel),
     [models]

ui/components/models-page.tsx

@@ -1,16 +1,14 @@
 'use client';
 
 import { useMemo, useState } from 'react';
-import { useQuery } from '@tanstack/react-query';
+import dynamic from 'next/dynamic';
 import { AlertCircle } from 'lucide-react';
 import type { Model } from '@/lib/api/schemas/models';
-import { AdminService } from '@/lib/api/services/admin';
+import { useModelsWithProviders } from '@/lib/hooks/use-models-with-providers';
 import { groupAndSortModelsByProvider } from '@/lib/utils/model-sort';
 import { AppPageShell } from '@/components/app-page-shell';
 import { PageHeader } from '@/components/page-header';
 import { ModelSelector } from '@/components/model-selector';
-import { ModelTester } from '@/components/model-tester';
-import { ApiEndpointTester } from '@/components/api-endpoint-tester';
 import { ModelSearchFilter } from '@/components/model-search-filter';
 import { Alert, AlertDescription } from '@/components/ui/alert';
 import {
@@ -23,6 +21,19 @@ import {
 import { Skeleton } from '@/components/ui/skeleton';
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
 
+// The testing tabs are never the landing view, so keeping them out of this
+// route's chunk is what lets the navigation itself resolve quickly.
+const ModelTester = dynamic(
+  () => import('@/components/model-tester').then((m) => m.ModelTester),
+  { loading: () => <Skeleton className='h-[420px] w-full' />, ssr: false }
+);
+
+const ApiEndpointTester = dynamic(
+  () =>
+    import('@/components/api-endpoint-tester').then((m) => m.ApiEndpointTester),
+  { loading: () => <Skeleton className='h-[420px] w-full' />, ssr: false }
+);
+
 export function ModelsPage() {
   const [filteredModels, setFilteredModels] = useState<Model[] | undefined>(
     undefined
@@ -31,16 +42,11 @@ export function ModelsPage() {
     useState<string>('all');
 
   const {
-    data: modelsData,
+    models,
+    groups,
     isLoading: isLoadingModels,
     error: modelsError,
-  } = useQuery({
-    queryKey: ['admin-models-with-providers'],
-    queryFn: () => AdminService.getModelsWithProviders(),
-    refetchOnWindowFocus: false,
-  });
-
-  const { models = [], groups = [] } = modelsData || {};
+  } = useModelsWithProviders();
 
   const groupedModels = useMemo(
     () => groupAndSortModelsByProvider(models),

ui/lib/api/services/admin.ts

@@ -446,10 +446,30 @@ export class AdminService {
     const allModels: AdminModelAsModel[] = [];
     const seenModelIds = new Set<string>();
 
-    for (const provider of providers) {
+    // One provider's catalog never depends on another's, and each miss costs an
+    // upstream round trip, so the whole fan-out happens in a single wave.
+    const providerResults = await Promise.all(
+      providers.map(async (provider) => {
         try {
-        const providerModels = await this.getProviderModels(provider.id);
+          return {
+            provider,
+            models: await this.getProviderModels(provider.id),
+          };
+        } catch (error) {
+          console.error(
+            `Failed to fetch models for provider ${provider.id}:`,
+            error
+          );
+          return null;
+        }
+      })
+    );
 
+    for (const result of providerResults) {
+      if (!result) {
+        continue;
+      }
+      const { provider, models: providerModels } = result;
       providerModels.db_models.forEach((dbModel) => {
         seenModelIds.add(dbModel.id);
         const modelWithProvider = {
@@ -483,12 +503,6 @@ export class AdminService {
           });
         }
       });
-      } catch (error) {
-        console.error(
-          `Failed to fetch models for provider ${provider.id}:`,
-          error
-        );
-      }
     }
 
     return { models: allModels, groups };

ui/lib/hooks/use-models-with-providers.ts

@@ -0,0 +1,26 @@
+'use client';
+
+import { useQuery } from '@tanstack/react-query';
+import { AdminService } from '@/lib/api/services/admin';
+
+export const modelsWithProvidersQueryKey = ['models-with-providers'] as const;
+
+/**
+ * Shared catalog read for every models view, so the page shell and the
+ * selector panel share one request instead of each fanning out to providers.
+ */
+export function useModelsWithProviders() {
+  const query = useQuery({
+    queryKey: modelsWithProvidersQueryKey,
+    queryFn: () => AdminService.getModelsWithProviders(),
+    refetchOnWindowFocus: false,
+  });
+
+  return {
+    models: query.data?.models ?? [],
+    groups: query.data?.groups ?? [],
+    isLoading: query.isLoading,
+    error: query.error,
+    refetch: query.refetch,
+  };
+}

ui/lib/hooks/use-progressive-list.ts

@@ -0,0 +1,46 @@
+'use client';
+
+import { useEffect, useState } from 'react';
+
+/**
+ * Reveal a long list in frame-sized batches.
+ *
+ * A provider catalog can hold thousands of rows, and mounting them in one
+ * commit blocks the main thread long enough that the page looks frozen right
+ * after navigation. Each batch yields back to the browser, so the first rows
+ * paint immediately and the rest fill in without freezing input.
+ */
+export function useProgressiveList<T>(
+  items: T[],
+  initialCount = 40,
+  step = 80
+): { visibleItems: T[]; hiddenCount: number } {
+  const [count, setCount] = useState(initialCount);
+  const [trackedItems, setTrackedItems] = useState(items);
+
+  // Reset during render, not in an effect: an effect would first commit the new
+  // list at the old (possibly full) count, which is the freeze this avoids.
+  if (trackedItems !== items) {
+    setTrackedItems(items);
+    setCount(initialCount);
+  }
+
+  useEffect(() => {
+    if (count >= items.length) {
+      return;
+    }
+
+    const frame = requestAnimationFrame(() => {
+      setCount((current) => Math.min(items.length, current + step));
+    });
+
+    return () => cancelAnimationFrame(frame);
+  }, [count, items.length, step]);
+
+  const visibleCount = Math.min(count, items.length);
+
+  return {
+    visibleItems: items.slice(0, visibleCount),
+    hiddenCount: items.length - visibleCount,
+  };
+}

uv.lock

@@ -337,6 +337,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9d/9e/78e59887cbf94116bdc890af7726ae264d55df14f1c777724c656e8a35fe/bolt11-2.1.1-py3-none-any.whl", hash = "sha256:fd4edb9e73e27bf5e017f47c97f7c6827b523fcf9cab152b123961ca78323e2d", size = 17102, upload-time = "2025-03-12T13:33:08.142Z" },
 ]
 
+[[package]]
+name = "boto3"
+version = "1.43.105"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+    { name = "jmespath" },
+    { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/75/46/d8c87ada70a7647fb3d206c7f19eafca3580a0ae4c06d62da539a1ee1207/boto3-1.43.105.tar.gz", hash = "sha256:e51260aed9cc1474778b5488bc6f97ad28f27a0a7002f4bbaaf8191aff1422ea", size = 112682, upload-time = "2026-09-29T19:37:40.784Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/8e/0310a37ff609529dab9153cbc9fd0b66c685364d741bb6d1ae31134b728e/boto3-1.43.105-py3-none-any.whl", hash = "sha256:b8b6236ae7fe2724eee608c9b0649afbb86f0ec98158f39f67e64e678ec47499", size = 140042, upload-time = "2026-09-29T19:37:39.415Z" },
+]
+
+[[package]]
+name = "botocore"
+version = "1.43.105"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "jmespath" },
+    { name = "python-dateutil" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2b/30/668f3c0533a440787e212cf56404cb6ec234ae8e6baf97fe17329d512d88/botocore-1.43.105.tar.gz", hash = "sha256:afb3e7706b123ab069d1c34571ca1fdf82528a48425574fe4693df3d039d503f", size = 16263910, upload-time = "2026-09-29T19:37:36.456Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f3/94/50923cd46840e4d2b56cad1dcf5008fb20c099b04b2d93d32231f2d0bfaa/botocore-1.43.105-py3-none-any.whl", hash = "sha256:7abd19e1ef2c5e4a0314ca493fa7cebefabe33e559d7dd570fe2432a5431e6ec", size = 15958067, upload-time = "2026-09-29T19:37:33.373Z" },
+]
+
 [[package]]
 name = "brotli"
 version = "1.2.0"
@@ -1509,6 +1537,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
 ]
 
+[[package]]
+name = "jmespath"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
+]
+
 [[package]]
 name = "jsonschema"
 version = "4.26.0"
@@ -1552,10 +1589,11 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.93.2"
+version = "1.101.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiohttp" },
+    { name = "boto3" },
     { name = "click" },
     { name = "fastuuid" },
     { name = "httpx", extra = ["socks"] },
@@ -1564,40 +1602,20 @@ dependencies = [
     { name = "jsonschema" },
     { name = "openai" },
     { name = "pydantic" },
+    { name = "pydantic-settings" },
     { name = "python-dotenv" },
     { name = "tiktoken" },
     { name = "tokenizers" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/97/dd/28024c0e4cf2dc6ab1bad59b8357af7f460e952c69526eae28f12ac4ee5e/litellm-1.93.2.tar.gz", hash = "sha256:c5d5223ef07f36e0886397fb45cc9db4150f86a0c6f6835cee1d5524cab69dfd", size = 15955441, upload-time = "2026-08-09T02:17:49.646Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/26/c9/cb2730c6c763233e322fe7c5b2f53783eb10893cea9304e5474f1f20c306/litellm-1.101.2.tar.gz", hash = "sha256:790adf4ce19116d7bf4342492b1be5a90dd56e08d89795979bf6c1c3446a9670", size = 17493188, upload-time = "2026-09-24T00:04:22.712Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/64/c7/cb3f49dc60d57dda7fe368310fd5da2a94ec9b6a746bcf343a61e10bdeda/litellm-1.93.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:1bd0690efc94357e559de97927fd98437555cd5b5dd832544cfcca87297ccb80", size = 19938326, upload-time = "2026-08-09T02:16:38.041Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/bd/d77184fdaaf57d67d65da91dcfc61c7f656703e7ce4f950e07523e7de4e3/litellm-1.93.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:845ececc628737909b1422d1af18bd19ae453727a66244aa9da3ca37a3773111", size = 19862606, upload-time = "2026-08-09T02:16:40.653Z" },
-    { url = "https://files.pythonhosted.org/packages/53/99/d8dd58b6840754a13cc2e1111b283aa28cbfc0ccc653a8725050916bb08e/litellm-1.93.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:498f9878ea773305e0638b6159d7e1ef27bb0b9a4292538d6634312d18a4e781", size = 20168532, upload-time = "2026-08-09T02:16:42.997Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/ca/559ca0f5e0b99b9f641086ae924c782f8d521d09384fbe9abbe0bddb6e61/litellm-1.93.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1e5618ef495b2e02299b376ca84ffb2647837aafee478cf3a1be17d47a8f0f73", size = 20162696, upload-time = "2026-08-09T02:16:45.283Z" },
-    { url = "https://files.pythonhosted.org/packages/92/3e/18c31b27c7d1271b43bdc8ffbef01bfba68d90248bbe60bb2130dd17e43c/litellm-1.93.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2da463d70c9fffbea9532fd000e035328f5266b399a2fb4c6c76b3470478337", size = 20233518, upload-time = "2026-08-09T02:16:47.87Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/98/a6bae7c52f09cd03487a040f98eeedb899b3cf3fc541b87c6d051ee92e0d/litellm-1.93.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2cf122399f84f8f04621ed6ef8f276dd6d61f4fab108932ce0e30368de34dd42", size = 20291180, upload-time = "2026-08-09T02:16:50.549Z" },
-    { url = "https://files.pythonhosted.org/packages/77/2d/81d974f2533cf039afda7e3e0f769dc73dc692c75ec867cf29ec6f41c06f/litellm-1.93.2-cp311-cp311-win_amd64.whl", hash = "sha256:8eaaf780fab9a19234735ef94225172179d15bc28b67ddbec125194249a504b7", size = 19775654, upload-time = "2026-08-09T02:16:53.162Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/05/72fd8051f0f2f3c84b90986e6f4551db7c8b190ba3300f111461b7701689/litellm-1.93.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3bf532c164ad7cb1b76f2c62afefdcc656b9b296374d075a4150e2ce10bb74c3", size = 19937403, upload-time = "2026-08-09T02:16:55.545Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/4d/5081b39bdb73cab04f8a86294a4534a029cf0434ac6932c7ae8049d55723/litellm-1.93.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:526b7afc037f79dfdd5c607f5085ac597c7fd301a6dedabea40baae899b27f19", size = 19853652, upload-time = "2026-08-09T02:16:57.977Z" },
-    { url = "https://files.pythonhosted.org/packages/70/3f/fb70691266a7fd08c202406abea0153e82fa17f134cd9d58e4029cc741db/litellm-1.93.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:294ad19f356f821ce97a5428d09439be5f38d22b218c73008d8a49e3e42eb145", size = 20165680, upload-time = "2026-08-09T02:17:00.65Z" },
-    { url = "https://files.pythonhosted.org/packages/81/91/84424ce2a25595463e5d24e9cf8949877cd4ce93c0fcbf6486ecd685094f/litellm-1.93.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6f6a5e3907f0a1c9d8ff8d71a6cbac8a592e47a40da3f97167074947b5ba7d11", size = 20157772, upload-time = "2026-08-09T02:17:03.027Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/8d/b0eac7ee6d174564f820565c8c9a726ae83dbb8c4d3522daf175b95da002/litellm-1.93.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8541f1b7fd5c437ad249ad68d0a11f68e5e2866b0649da5fa7d63b595e9b8b22", size = 20229256, upload-time = "2026-08-09T02:17:05.271Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/6d/03e931c1cb2d1e1b7a968de21aa9e4db853928200da856c35c940ee6faa9/litellm-1.93.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:712c9387419d7b06a10df59973f5e530592d61b2314102b0fa3142f3743f9a9e", size = 20287257, upload-time = "2026-08-09T02:17:08.175Z" },
-    { url = "https://files.pythonhosted.org/packages/16/05/6c0fe2fcf31c260474c55fabe4ecb0e9e1343c9b9132e28589391b2ad33e/litellm-1.93.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc0d58ccabd22ef7ef44a9e6f7247deb54ae42f5e126e6f00360c2b28b41bc2b", size = 19772580, upload-time = "2026-08-09T02:17:11.254Z" },
-    { url = "https://files.pythonhosted.org/packages/70/74/e9046cffa69b32b710452480598e418b26a29896ece680c80ec23997fd16/litellm-1.93.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f4071bef03e4c2942cd2ddc752727345b85447d6a7fee1ff5a4f8b92187966b0", size = 19938095, upload-time = "2026-08-09T02:17:13.929Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/db/6ef38a7a2f73d5cc507423954fa535a8546ead375c4c71265c093bdb4e9e/litellm-1.93.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a99ac7c0c1b78acd6bfd1959e9f203dca71fdbceb5f0c8691c2ad8eee450d7d", size = 19854187, upload-time = "2026-08-09T02:17:16.588Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/b3/80ee0143b88e2921f8c8f24c7331478258a8bf25a3d4d4450bd96043403e/litellm-1.93.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a81ceff44c58ef504ab8bd787d03b82618765b9cfd530942386ae6d23c58be94", size = 20166307, upload-time = "2026-08-09T02:17:19.078Z" },
-    { url = "https://files.pythonhosted.org/packages/98/60/cb326e1094f7042f28f9e21543d9f367a8aa25af6915bf4253b77da5c2a2/litellm-1.93.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:dee1b02b7f52a5a408bf7c8d499f0834e49194651743758a511dcdd926c0b692", size = 20158336, upload-time = "2026-08-09T02:17:21.507Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/87/bad75146863531172c9dbae189486c7f4425b56a6641b55ab20745316048/litellm-1.93.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d2edfa14b99bce706b35981703692e3ee631f9b87bf6dc28fb53b574f6480b20", size = 20229711, upload-time = "2026-08-09T02:17:24.073Z" },
-    { url = "https://files.pythonhosted.org/packages/df/28/040b1853021ed8fd57be19eb2affb024d168951fe7e7abdbad91da3f6f3f/litellm-1.93.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ae75a61c9abc827aa3131b7e640c952367a450830bb7c531b426b4ec2bb45f85", size = 20287584, upload-time = "2026-08-09T02:17:26.542Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/0b/4208815b0d666636cbf7afbd571eec3004d3a15d3150a23a9009fc2ce930/litellm-1.93.2-cp313-cp313-win_amd64.whl", hash = "sha256:c54a09ab20f94120a9d60a30d9970439dcefa00d2565d190505ff006a80c7a69", size = 19772641, upload-time = "2026-08-09T02:17:29.308Z" },
-    { url = "https://files.pythonhosted.org/packages/09/4a/ff7a9c000519d2bab362318bf744a24c2500228e5fceaa6ac23acab96fa0/litellm-1.93.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:204cb0763fff9285bc87eb2dc0fc59b591999e5d94863d0964f806424d3c0cd6", size = 19943639, upload-time = "2026-08-09T02:17:31.811Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/26/29e9276ce4aa8ed133d9fd5ecc07375017d2228215547c6bbb17ccbc59b4/litellm-1.93.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3126c84361606b9fb07fde7d57eccd8a1747304d64c4143e2e5e40ae6e7693fb", size = 19855435, upload-time = "2026-08-09T02:17:34.376Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/20/2c9c818248ae019b2d496ca41900a9a5651ab05e2400794cd8dc8b89b6d2/litellm-1.93.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:1c84f7c4acb4e926a79b93145ab23231b300fc687bde7172ef884fc52d6011e0", size = 20166947, upload-time = "2026-08-09T02:17:36.828Z" },
-    { url = "https://files.pythonhosted.org/packages/50/af/4016682be48350407837941ad1a1ae8185cca65b102e04e89eee2a2abccb/litellm-1.93.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:cacf35cf703b12c54516fc6464a3e08c6dbb1dcfb97239e1f629294fe36a1cba", size = 20160055, upload-time = "2026-08-09T02:17:39.674Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/b5/c25d7fbe08490d8211bd6b69af23f3a922b68ad8c87c776480b0de64a505/litellm-1.93.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0a7f3e5138e307e429bd8fa29cc0c48bb1e2b827792e8f7799ca4c8cff736103", size = 20230910, upload-time = "2026-08-09T02:17:42.159Z" },
-    { url = "https://files.pythonhosted.org/packages/21/27/341b18a40d4d98a2ac09025c248a3a7edddaf15ce4096ac4a783ff2f70db/litellm-1.93.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8684629be3f7b5f8e2b6e5fe5ea27ff957c63a8d525d81c1460d8436e2e1857", size = 20288903, upload-time = "2026-08-09T02:17:44.433Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/45/dd9ef72075a83854f852b1bf9a97ec7029a2be9fb4e338fc6623eb09fc90/litellm-1.93.2-cp314-cp314-win_amd64.whl", hash = "sha256:a783b8b18ed68cb6a3b79d2b00273ec21aef92442e9b2712a50036cb84bfe583", size = 19772974, upload-time = "2026-08-09T02:17:46.972Z" },
+    { url = "https://files.pythonhosted.org/packages/44/a7/4bccec0ac9cb1b2e94e391b666458d07480d342039c66383ac191819e8e7/litellm-1.101.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:48c42c2c2cf9d4b0d75f4e1670b1b64b9e0513488d0b737fe057fda0bc716551", size = 23827328, upload-time = "2026-09-24T00:04:03.196Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/3d/faf394e5ac5a1469de5cbd3939c9e330f744e648351934b981078ccc40d9/litellm-1.101.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:77195c8ed502c052bb31d4c3887356a308ac2e8c3b0b30c97e2b04b0be10dd44", size = 23484770, upload-time = "2026-09-24T00:04:06.248Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/84/60f70aa2683666626c4abe7aa44b52acca52cad911ee870ea244fd3b0796/litellm-1.101.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:abb7b3ac04f56ced46e53cca2369a5dd29539cab9fdcd6cc9a94987aa55a38d1", size = 23618000, upload-time = "2026-09-24T00:04:08.827Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/8e/c57a4e157f97b1bcef9b410d51e17507047bbb11c676f5c81b22e7190c7c/litellm-1.101.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:210c89194225778759aa6649f5c0d605572bf14708ec85712462ace479f47f04", size = 23994795, upload-time = "2026-09-24T00:04:12.292Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/49/8737aee5a5a15cac7eb8a972b800529923e2837bbbadf0617f56111344ab/litellm-1.101.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:144dd8d1ead7174a718d1748deffcda7438cf7dfe8dc7c20761b72c35117e9a3", size = 23693332, upload-time = "2026-09-24T00:04:14.886Z" },
+    { url = "https://files.pythonhosted.org/packages/04/50/4e711caa0374309d6aaf5696549449c2078f0225dd36a22ee0ca44dd068f/litellm-1.101.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ae95e7ef15e109472f2cec69da6028a874b416e56a704cd9b797362993fc63cf", size = 24092655, upload-time = "2026-09-24T00:04:17.677Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/7d/32d391ddcb30d4d5d08fddd0abe918e836f9b3f753237c2b12ecb3d7425a/litellm-1.101.2-cp310-abi3-win_amd64.whl", hash = "sha256:0f5ee6daf9082b7efca1dc851c10c0d4884a2f1e0d7ea410bd508961b5a2cdae", size = 23894930, upload-time = "2026-09-24T00:04:20.432Z" },
 ]
 
 [[package]]
@@ -2439,6 +2457,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" },
 ]
 
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
 [[package]]
 name = "python-dotenv"
 version = "1.2.2"
@@ -2722,7 +2752,7 @@ requires-dist = [
     { name = "greenlet", specifier = ">=3.2.1" },
     { name = "h11", specifier = ">=0.16" },
     { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" },
-    { name = "litellm", specifier = ">=1.93.0,<1.94" },
+    { name = "litellm", specifier = ">=1.101.2,<1.102" },
     { name = "marshmallow", specifier = ">=3.13,<4.0" },
     { name = "mdurl", specifier = "==0.1.2" },
     { name = "nostr-sdk", specifier = ">=0.45.1,<0.46" },
@@ -2882,6 +2912,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" },
 ]
 
+[[package]]
+name = "s3transfer"
+version = "0.19.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" },
+]
+
 [[package]]
 name = "setuptools"
 version = "84.0.0"