diff --git a/routstr/payment/models.py b/routstr/payment/models.py index e62f7875..db477b93 100644 --- a/routstr/payment/models.py +++ b/routstr/payment/models.py @@ -94,7 +94,10 @@ def backfill_cache_pricing(model_id: str, pricing: Pricing) -> Pricing: cache reads (DeepSeek hits are 10x cheaper) and undercharges Anthropic cache writes (1.25x). litellm ships per-model USD rates keyed by the exact OpenRouter id (deepseek/deepseek-chat) or by the bare model name - (gpt-4o, claude-sonnet-4-5), so both spellings are tried. + (gpt-4o, claude-sonnet-4-5), so both spellings are tried. litellm keys are + lowercase, but a generic upstream may report a mixed-case id + (``deepseek-ai/DeepSeek-V4-Flash``); an exact match is attempted first, then + a case-insensitive fallback so such ids still resolve. Rates already present (e.g. provided by OpenRouter) are authoritative and never overwritten. Unknown models are returned unchanged. @@ -106,12 +109,26 @@ def backfill_cache_pricing(model_id: str, pricing: Pricing) -> Pricing: import litellm + candidates = (model_id, model_id.split("/", 1)[-1]) info: dict | None = None - for key in (model_id, model_id.split("/", 1)[-1]): + for key in candidates: candidate = litellm.model_cost.get(key) if isinstance(candidate, dict): info = candidate break + if info is None: + # Case-insensitive fallback: a mixed-case upstream id (e.g. + # ``deepseek-ai/DeepSeek-V4-Flash``) won't match litellm's lowercase + # keys exactly. Build a lowercased index once and retry. + lowered = {c.lower() for c in candidates} + for key, candidate in litellm.model_cost.items(): + if ( + isinstance(key, str) + and key.lower() in lowered + and isinstance(candidate, dict) + ): + info = candidate + break if info is None: return pricing @@ -215,13 +232,23 @@ def _row_to_model( ) top_provider_dict = json.loads(row.top_provider) if row.top_provider else None - if apply_provider_fee and isinstance(pricing, dict): - pricing = {k: float(v) * provider_fee for k, v in pricing.items()} - if isinstance(pricing, dict) and float(pricing.get("request", 0.0)) <= 0.0: pricing["request"] = max(pricing.get("request", 0.0), 0.0) parsed_pricing = Pricing.parse_obj(pricing) + + # Fill missing cache-read/write rates from litellm's cost map BEFORE applying + # the provider fee, so they carry the same markup as every other component. + # DB-stored override pricing (e.g. generic providers) omits cache rates; + # without this, ``_row_to_model`` bills cache reads at the full input rate — + # the ``_apply_provider_fee_to_model`` path backfills, but the override path + # used for admin-configured providers did not. + parsed_pricing = backfill_cache_pricing(row.id, parsed_pricing) + + if apply_provider_fee: + parsed_pricing = Pricing.parse_obj( + {k: float(v) * provider_fee for k, v in parsed_pricing.dict().items()} + ) model = Model( id=row.id, name=row.name, diff --git a/routstr/upstream/deepseek_v4_pricing_shim.py b/routstr/upstream/deepseek_v4_pricing_shim.py index 8720fb40..ba0c392d 100644 --- a/routstr/upstream/deepseek_v4_pricing_shim.py +++ b/routstr/upstream/deepseek_v4_pricing_shim.py @@ -3,10 +3,15 @@ litellm's bundled cost map does not yet ship ``deepseek-v4-flash`` / ``deepseek-v4-pro``. Without an entry, ``backfill_cache_pricing`` cannot find a ``cache_read_input_token_cost`` and cache reads fall back to the full input -rate — a ~60% overcharge on cache hits (DeepSeek hits are ~0.2x input). +rate — a large overcharge on cache hits (DeepSeek V4 hits are ~0.008-0.02x +input, i.e. cached tokens cost 50-120x less than regular input). This module injects the missing entries into ``litellm.model_cost`` at startup -so the existing backfill path resolves them. Rates mirror the open upstream PR +so the existing backfill path resolves them. Rates mirror the canonical +``deepseek`` provider entries now in litellm's ``model_prices`` map +(``input_cost_per_token`` is the cache-*miss* rate; +``cache_read_input_token_cost`` is the cache-*hit* rate), sourced from +https://api-docs.deepseek.com/quick_start/pricing via https://github.com/BerriAI/litellm/pull/26380 (issue https://github.com/BerriAI/litellm/issues/30430). @@ -22,21 +27,23 @@ from ..core import get_logger logger = get_logger(__name__) -# USD per token. Source: BerriAI/litellm PR #26380. +# USD per token. Mirrors the canonical ``deepseek`` provider entries in +# litellm's model_prices map (source: DeepSeek API pricing docs). Keep these in +# sync with ``litellm.model_cost["deepseek/deepseek-v4-*"]``. _DEEPSEEK_V4_RATES: dict[str, dict[str, float]] = { "deepseek-v4-flash": { "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 2.8e-09, "cache_creation_input_token_cost": 0.0, - "input_cost_per_token_cache_hit": 2.8e-08, + "input_cost_per_token_cache_hit": 2.8e-09, }, "deepseek-v4-pro": { - "input_cost_per_token": 1.74e-06, - "output_cost_per_token": 3.48e-06, - "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 4.35e-07, + "output_cost_per_token": 8.7e-07, + "cache_read_input_token_cost": 3.625e-09, "cache_creation_input_token_cost": 0.0, - "input_cost_per_token_cache_hit": 1.4e-07, + "input_cost_per_token_cache_hit": 3.625e-09, }, } diff --git a/tests/unit/test_cache_pricing.py b/tests/unit/test_cache_pricing.py index c9af6e2d..14bb884c 100644 --- a/tests/unit/test_cache_pricing.py +++ b/tests/unit/test_cache_pricing.py @@ -81,6 +81,21 @@ def test_backfill_strips_vendor_prefix_for_litellm_lookup() -> None: assert result.input_cache_read == expected +def test_backfill_case_insensitive_lookup() -> None: + """A generic upstream may report a mixed-case id + (deepseek-ai/DeepSeek-V4-Flash); litellm keys are lowercase. The + case-insensitive fallback still resolves the cache rate.""" + pricing = Pricing(prompt=1.4e-07, completion=2.8e-07) + + result = backfill_cache_pricing("deepseek-ai/DeepSeek-V4-Flash", pricing) + + expected = litellm.model_cost["deepseek-v4-flash"][ + "cache_read_input_token_cost" + ] + assert result.input_cache_read == expected + assert result.input_cache_read < pricing.prompt # sanity: it's a discount + + def test_backfill_fills_cache_write_rate() -> None: """Anthropic cache writes cost more than input (1.25x); billing them at the input rate undercharges. litellm carries the write rate.""" @@ -136,6 +151,51 @@ def test_provider_fee_applies_to_backfilled_cache_rates() -> None: assert adjusted.pricing.prompt == pytest.approx(2.8e-07 * 2.0) +def test_row_to_model_backfills_cache_rate() -> None: + """The DB-override path (admin-configured providers, e.g. a generic + upstream) stores pricing without cache rates. ``_row_to_model`` must + backfill them from litellm just like ``_apply_provider_fee_to_model``, + otherwise cache reads bill at the full input rate.""" + import json + + from routstr.core.db import ModelRow + from routstr.payment.models import _row_to_model + + row = ModelRow( + id="deepseek-v4-flash", + name="deepseek-v4-flash", + created=0, + description="", + context_length=1000000, + architecture=json.dumps( + { + "modality": "text", + "input_modalities": ["text"], + "output_modalities": ["text"], + "tokenizer": "unknown", + "instruct_type": None, + } + ), + # Stored pricing omits input_cache_read (generic provider never sets it). + pricing=json.dumps({"prompt": 1.4e-07, "completion": 2.8e-07}), + enabled=True, + upstream_provider_id=1, + ) + + with patch( + "routstr.payment.models.sats_usd_price", return_value=5.0e-5 + ): + model = _row_to_model(row, apply_provider_fee=True, provider_fee=1.0) + + litellm_read = litellm.model_cost["deepseek-v4-flash"][ + "cache_read_input_token_cost" + ] + assert model.pricing.input_cache_read == pytest.approx(litellm_read) + assert model.pricing.input_cache_read < model.pricing.prompt # a discount + assert model.sats_pricing is not None + assert model.sats_pricing.input_cache_read > 0 + + # ============================================================================ # calculate_cost — cached tokens billed at cache rates # ============================================================================