make openrouter default to provider supporting tool use

consolidate streaming

routstr/core/main.py

@@ -24,6 +24,7 @@ from ..payment.models import models_router, update_sats_pricing
 from ..payment.price import update_prices_periodically
 from ..proxy import initialize_upstreams, proxy_router, refresh_model_maps_periodically
 from ..upstream.auto_topup import periodic_auto_topup
+from ..upstream.deepseek_v4_pricing_shim import register_deepseek_v4_pricing
 from ..upstream.litellm_routing import configure_litellm
 from ..wallet import periodic_payout, periodic_refund_sweep, periodic_routstr_fee_payout
 from .admin import admin_router
@@ -65,6 +66,11 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
         # debug logging) before any upstream provider dispatches a request.
         configure_litellm()
 
+        # TEMPORARY: backfill DeepSeek V4 pricing missing from litellm's cost
+        # map (BerriAI/litellm#30430). Remove this call and
+        # deepseek_v4_pricing_shim.py once litellm ships these models.
+        register_deepseek_v4_pricing()
+
         # Run database migrations on startup
         run_migrations()
 

routstr/payment/cost_calculation.py

@@ -418,10 +418,21 @@ def _calculate_from_tokens(
         },
     )
 
+    # Fold the cache-read/write cost into the visible ``input_msats`` so a
+    # dashboard that renders I / O / T sees ``input + output == total``
+    # exactly. This mirrors ``_fold_cache_into_input_tokens`` (which rolls the
+    # cache token counts into the visible prompt total). The standalone
+    # ``cache_read_msats`` / ``cache_creation_msats`` fields stay populated for
+    # clients that want the breakdown; nothing sums the components to derive
+    # ``total_msats`` (it is computed independently above), so this is
+    # display-only and does not change what is billed.
+    visible_output_msats = int(calc_output_msats)
+    visible_input_msats = token_based_cost - visible_output_msats
+
     return CostData(
         base_msats=0,
-        input_msats=int(calc_input_msats),
-        output_msats=int(calc_output_msats),
+        input_msats=visible_input_msats,
+        output_msats=visible_output_msats,
         total_msats=token_based_cost,
         total_usd=total_usd,
         input_tokens=input_tokens,

routstr/upstream/deepseek_v4_pricing_shim.py

@@ -0,0 +1,66 @@
+"""TEMPORARY: local DeepSeek V4 pricing shim.
+
+litellm's bundled cost map does not yet ship ``deepseek-v4-flash`` /
+``deepseek-v4-pro``. Without an entry, ``backfill_cache_pricing`` cannot find a
+``cache_read_input_token_cost`` and cache reads fall back to the full input
+rate — a ~60% overcharge on cache hits (DeepSeek hits are ~0.2x input).
+
+This module injects the missing entries into ``litellm.model_cost`` at startup
+so the existing backfill path resolves them. Rates mirror the open upstream PR
+https://github.com/BerriAI/litellm/pull/26380 (issue
+https://github.com/BerriAI/litellm/issues/30430).
+
+=== REMOVAL (once litellm ships these models) ===
+Delete this file and the single ``register_deepseek_v4_pricing()`` call in
+``routstr/core/main.py``. Nothing else depends on it. Entries are only added
+when absent, so a stale shim is harmless after upstream lands — but remove it.
+"""
+
+import litellm
+
+from ..core import get_logger
+
+logger = get_logger(__name__)
+
+# USD per token. Source: BerriAI/litellm PR #26380.
+_DEEPSEEK_V4_RATES: dict[str, dict[str, float]] = {
+    "deepseek-v4-flash": {
+        "input_cost_per_token": 1.4e-07,
+        "output_cost_per_token": 2.8e-07,
+        "cache_read_input_token_cost": 2.8e-08,
+        "cache_creation_input_token_cost": 0.0,
+        "input_cost_per_token_cache_hit": 2.8e-08,
+    },
+    "deepseek-v4-pro": {
+        "input_cost_per_token": 1.74e-06,
+        "output_cost_per_token": 3.48e-06,
+        "cache_read_input_token_cost": 1.4e-07,
+        "cache_creation_input_token_cost": 0.0,
+        "input_cost_per_token_cache_hit": 1.4e-07,
+    },
+}
+
+
+def register_deepseek_v4_pricing() -> None:
+    """Inject DeepSeek V4 pricing into ``litellm.model_cost`` if absent.
+
+    Idempotent and non-destructive: a key already present in the cost map
+    (e.g. once litellm ships it) is left untouched. Registers both the bare
+    (``deepseek-v4-flash``) and prefixed (``deepseek/deepseek-v4-flash``)
+    spellings since ``backfill_cache_pricing`` tries both.
+    """
+    added = []
+    for bare, rates in _DEEPSEEK_V4_RATES.items():
+        for key in (bare, f"deepseek/{bare}"):
+            if key in litellm.model_cost:
+                continue
+            entry: dict[str, object] = dict(rates)
+            entry["litellm_provider"] = "deepseek"
+            entry["mode"] = "chat"
+            litellm.model_cost[key] = entry
+            added.append(key)
+    if added:
+        logger.info(
+            "Registered temporary DeepSeek V4 pricing shim",
+            extra={"models": added},
+        )

routstr/upstream/openrouter.py

@@ -1,3 +1,4 @@
+import json
 from typing import TYPE_CHECKING
 
 import httpx
@@ -18,6 +19,38 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider):
     supports_anthropic_messages = True
     litellm_provider_prefix = "openrouter/"
 
+    def prepare_request_body(
+        self, body: bytes | None, model_obj: Model
+    ) -> bytes | None:
+        """Set provider.require_parameters on tool-use requests.
+
+        Without it OpenRouter can route a tool call to an endpoint that doesn't
+        support function calling and 404 with "No endpoints found that support
+        tool use". We leave a client-supplied value untouched.
+        """
+        body = super().prepare_request_body(body, model_obj)
+        if not body:
+            return body
+
+        try:
+            data = json.loads(body)
+        except json.JSONDecodeError:
+            return body
+
+        if not isinstance(data, dict) or not data.get("tools"):
+            return body
+
+        provider = data.get("provider")
+        if not isinstance(provider, dict):
+            provider = {}
+
+        if "require_parameters" in provider:
+            return body
+
+        provider["require_parameters"] = True
+        data["provider"] = provider
+        return json.dumps(data).encode()
+
     def _apply_provider_field(self, response_json: object) -> None:
         """Stamp the ``provider`` field for OpenRouter responses.
 

tests/unit/test_cache_pricing.py

@@ -181,10 +181,14 @@ async def test_deepseek_cache_hits_billed_at_cache_rate(model_pricing: Mock) ->
         result = await calculate_cost(response, max_cost=100000)
 
     assert isinstance(result, CostData)
-    # 1000 input @ 1 msat + 9000 cache reads @ 0.1 msat + 500 output @ 2 msat
-    assert result.input_msats == 1000
+    # 1000 input @ 1 msat + 9000 cache reads @ 0.1 msat + 500 output @ 2 msat.
+    # input_msats folds the cache-read cost in (1000 + 900) so a dashboard
+    # rendering I/O/T sees input + output == total; the cache portion stays
+    # visible in cache_read_msats.
     assert result.cache_read_msats == 900
     assert result.output_msats == 1000
+    assert result.input_msats == 1900
+    assert result.input_msats + result.output_msats == result.total_msats
     assert result.total_msats == 2900
 
 

tests/unit/test_require_parameters.py

@@ -0,0 +1,95 @@
+import json
+import os
+
+os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
+os.environ.setdefault("UPSTREAM_API_KEY", "test")
+os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
+
+from routstr.upstream import GenericUpstreamProvider
+from routstr.upstream.openrouter import OpenRouterUpstreamProvider
+
+
+def _model(model_id: str = "openai/gpt-4o"):  # type: ignore[no-untyped-def]
+    from routstr.payment.models import Architecture, Model, Pricing
+
+    return Model(
+        id=model_id,
+        name=model_id,
+        created=0,
+        description="",
+        context_length=128000,
+        architecture=Architecture(
+            modality="text->text",
+            input_modalities=["text"],
+            output_modalities=["text"],
+            tokenizer="GPT",
+            instruct_type=None,
+        ),
+        pricing=Pricing(prompt=0.0, completion=0.0),
+    )
+
+
+def _tool_body() -> dict:
+    return {
+        "model": "openai/gpt-4o",
+        "messages": [{"role": "user", "content": "What's the weather?"}],
+        "tools": [
+            {
+                "type": "function",
+                "function": {"name": "get_weather", "parameters": {}},
+            }
+        ],
+    }
+
+
+def _prepare(provider, body: dict) -> dict:  # type: ignore[no-untyped-def]
+    out = provider.prepare_request_body(json.dumps(body).encode(), _model())
+    assert out is not None
+    return json.loads(out)
+
+
+def test_injects_require_parameters_for_tool_request() -> None:
+    data = _prepare(OpenRouterUpstreamProvider(api_key="test"), _tool_body())
+    assert data["provider"]["require_parameters"] is True
+
+
+def test_generic_provider_on_openrouter_url_is_left_alone() -> None:
+    # Only OpenRouterUpstreamProvider injects; a generic provider pointed at the
+    # same base URL doesn't.
+    provider = GenericUpstreamProvider(base_url="https://openrouter.ai/api/v1")
+    data = _prepare(provider, _tool_body())
+    assert "provider" not in data
+
+
+def test_no_injection_without_tools() -> None:
+    body = {"model": "openai/gpt-4o", "messages": [{"role": "user", "content": "hi"}]}
+    data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
+    assert "provider" not in data
+
+
+def test_empty_tools_list_does_not_inject() -> None:
+    body = _tool_body()
+    body["tools"] = []
+    data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
+    assert "provider" not in data
+
+
+def test_direct_provider_does_not_inject() -> None:
+    provider = GenericUpstreamProvider(base_url="https://api.openai.com/v1")
+    data = _prepare(provider, _tool_body())
+    assert "provider" not in data
+
+
+def test_keeps_client_set_require_parameters() -> None:
+    body = _tool_body()
+    body["provider"] = {"require_parameters": False}
+    data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
+    assert data["provider"]["require_parameters"] is False
+
+
+def test_preserves_other_provider_fields() -> None:
+    body = _tool_body()
+    body["provider"] = {"order": ["openai", "azure"]}
+    data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
+    assert data["provider"]["order"] == ["openai", "azure"]
+    assert data["provider"]["require_parameters"] is True