mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-10-05 20:28:23 +00:00
fix(ehbp): resolve the served model within the tinfoil namespace before pricing
The cache discount never applied in production even though everything
downstream of it worked: the enclave reported cache splits
(cached_prompt_tokens=160512 of 161652) and the parse put them into
cache_read_input_tokens, but the applied cache-read rate was the full input
rate.
Root cause: the SDK strips the routstr "tinfoil-" namespace prefix for the
encrypted body (getTinfoilUpstreamModelId), so the enclave always reports the
bare upstream id ("deepseek-v4-1-flash") in X-Tinfoil-Usage-Metrics, while
the catalog registers the model as "tinfoil-deepseek-v4-1-flash" and
forwarded_model_id carries the prefix too. _normalize_upstream_model_id only
lowercases, so every request took the "served model differs" path and
re-derived pricing via get_model_instance on the bare id — which resolves
globally to a cheaper cross-provider model whose pricing has no cache rate
(input_cache_read=0), falling back to the full input price.
Billing was therefore doubly wrong: no cache discount, and E2EE requests
undercharged at the cross-provider rate instead of the Tinfoil rate.
Fix: when the requested model is namespaced "tinfoil-" and the served id is
not, resolve the served id within the same namespace first (with a bare-id
fallback). A same-model report then maps back onto the requested Tinfoil
model (keeping its pricing), and a genuine failover lands on the
actually-served Tinfoil model — while the bare-id lookup that previously
hijacked pricing is only used as a last resort.
This commit is contained in:
@@ -127,6 +127,13 @@ _PROXY_ONLY_HEADERS = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# Namespace prefix the routstr catalog applies to Tinfoil models
|
||||
# (e.g. ``tinfoil-deepseek-v4-1-flash``). The SDK strips this prefix for the
|
||||
# encrypted body (``getTinfoilUpstreamModelId`` in client/TinfoilSecure.ts), so
|
||||
# the enclave always reports the *bare* upstream model id in the usage-metrics
|
||||
# header even though the routstr model id and ``forwarded_model_id`` carry it.
|
||||
TINFOIL_MODEL_PREFIX = "tinfoil-"
|
||||
|
||||
|
||||
def parse_tinfoil_usage_metrics(header_value: str | None) -> dict | None:
|
||||
"""Parse ``X-Tinfoil-Usage-Metrics`` into an OpenAI-style usage dict.
|
||||
@@ -451,7 +458,24 @@ async def _compute_ehbp_actual_cost(
|
||||
# the global model map. The resolved object can belong to a different
|
||||
# provider and therefore have a different client-facing ``id`` while
|
||||
# still representing the same upstream model.
|
||||
actual_model_obj = get_model_instance(actual_model)
|
||||
#
|
||||
# The enclave reports the *bare* upstream id, but the routstr model is
|
||||
# namespaced ``tinfoil-`` (and the SDK strips that prefix for the
|
||||
# encrypted body). Resolve the served id within the same namespace
|
||||
# first: a same-model report then maps back onto the requested Tinfoil
|
||||
# model, and a genuine failover lands on the actually-served Tinfoil
|
||||
# model — instead of the cheaper cross-provider model the bare id
|
||||
# would resolve to in the global map.
|
||||
namespaced_served = actual_model
|
||||
if (
|
||||
expected_upstream_model.startswith(TINFOIL_MODEL_PREFIX)
|
||||
and not actual_model.startswith(TINFOIL_MODEL_PREFIX)
|
||||
):
|
||||
namespaced_served = TINFOIL_MODEL_PREFIX + actual_model
|
||||
|
||||
actual_model_obj = get_model_instance(namespaced_served)
|
||||
if actual_model_obj is None and namespaced_served != actual_model:
|
||||
actual_model_obj = get_model_instance(actual_model)
|
||||
if actual_model_obj is None:
|
||||
logger.warning(
|
||||
"EHBP served model not found in registry, falling back "
|
||||
|
||||
@@ -488,6 +488,126 @@ class TestComputeEhbpActualCost:
|
||||
call_args = mock_calc.call_args
|
||||
assert call_args[0][0]["model"] == "tinfoil-llama3-3-70b"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_namespaced_prefix_served_bare_keeps_requested_pricing(
|
||||
self,
|
||||
) -> None:
|
||||
"""Production shape: the catalog model is ``tinfoil-X`` and its
|
||||
``forwarded_model_id`` carries the prefix, the SDK strips the prefix
|
||||
for the encrypted body, and the enclave reports bare ``X``. Pricing
|
||||
must stay on the requested Tinfoil model (correct rate + cache
|
||||
discount), not the cheaper cross-provider model the bare id resolves
|
||||
to in the global map."""
|
||||
model_obj = MagicMock()
|
||||
model_obj.id = "tinfoil-deepseek-v4-1-flash"
|
||||
model_obj.forwarded_model_id = "tinfoil-deepseek-v4-1-flash"
|
||||
|
||||
tinfoil_model = MagicMock()
|
||||
tinfoil_model.id = "tinfoil-deepseek-v4-1-flash"
|
||||
tinfoil_model.forwarded_model_id = "tinfoil-deepseek-v4-1-flash"
|
||||
|
||||
# The cheaper cross-provider model the bare id resolves to globally.
|
||||
cross_provider_model = MagicMock()
|
||||
cross_provider_model.id = "deepseek-v4-1-flash"
|
||||
cross_provider_model.forwarded_model_id = "deepseek-v4-1-flash"
|
||||
|
||||
registry = {
|
||||
"tinfoil-deepseek-v4-1-flash": tinfoil_model,
|
||||
"deepseek-v4-1-flash": cross_provider_model,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.proxy.get_model_instance",
|
||||
side_effect=lambda name: registry.get(name),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.ehbp.calculate_cost",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_calc,
|
||||
):
|
||||
from routstr.payment.cost_calculation import CostData
|
||||
|
||||
mock_calc.return_value = CostData(
|
||||
base_msats=0,
|
||||
input_msats=5,
|
||||
output_msats=10,
|
||||
total_msats=15,
|
||||
total_usd=0.0,
|
||||
input_tokens=5,
|
||||
output_tokens=10,
|
||||
cache_read_input_tokens=64,
|
||||
cache_creation_input_tokens=0,
|
||||
cache_read_msats=1,
|
||||
cache_creation_msats=0,
|
||||
)
|
||||
result = await _compute_ehbp_actual_cost(
|
||||
"prompt=69,completion=10,total=79,"
|
||||
"cached_prompt_tokens=64,uncached_prompt_tokens=5,"
|
||||
"model=deepseek-v4-1-flash",
|
||||
model_obj,
|
||||
100_000,
|
||||
)
|
||||
# No mismatch: pricing stays on the requested Tinfoil model.
|
||||
assert "actual_model" not in result
|
||||
call_args = mock_calc.call_args
|
||||
assert call_args[0][0]["model"] == "tinfoil-deepseek-v4-1-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_namespaced_prefix_failover_uses_served_tinfoil_model(
|
||||
self,
|
||||
) -> None:
|
||||
"""A genuine failover (asked ``tinfoil-glm-5-3``, enclave served
|
||||
``glm-5-3-flash``) must bill the served *Tinfoil* model, not the
|
||||
cheaper cross-provider alias the bare id resolves to."""
|
||||
model_obj = MagicMock()
|
||||
model_obj.id = "tinfoil-glm-5-3"
|
||||
model_obj.forwarded_model_id = "tinfoil-glm-5-3"
|
||||
|
||||
served_tinfoil = MagicMock()
|
||||
served_tinfoil.id = "tinfoil-glm-5-3-flash"
|
||||
served_tinfoil.forwarded_model_id = "tinfoil-glm-5-3-flash"
|
||||
|
||||
cross_provider = MagicMock()
|
||||
cross_provider.id = "glm-5-3-flash"
|
||||
cross_provider.forwarded_model_id = "glm-5-3-flash"
|
||||
|
||||
registry = {
|
||||
"tinfoil-glm-5-3-flash": served_tinfoil,
|
||||
"glm-5-3-flash": cross_provider,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.proxy.get_model_instance",
|
||||
side_effect=lambda name: registry.get(name),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.ehbp.calculate_cost",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_calc,
|
||||
):
|
||||
from routstr.payment.cost_calculation import CostData
|
||||
|
||||
mock_calc.return_value = CostData(
|
||||
base_msats=0,
|
||||
input_msats=20,
|
||||
output_msats=40,
|
||||
total_msats=60,
|
||||
total_usd=0.0,
|
||||
input_tokens=42,
|
||||
output_tokens=10,
|
||||
)
|
||||
result = await _compute_ehbp_actual_cost(
|
||||
"prompt=42,completion=10,total=52,model=glm-5-3-flash",
|
||||
model_obj,
|
||||
100_000,
|
||||
)
|
||||
assert result["actual_model"] == "glm-5-3-flash"
|
||||
# Billed on the served *Tinfoil* model, not the bare-id alias.
|
||||
call_args = mock_calc.call_args
|
||||
assert call_args[0][0]["model"] == "tinfoil-glm-5-3-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_mismatch_unknown_model_falls_back(self) -> None:
|
||||
"""When the served model is not in the registry, use requested model."""
|
||||
|
||||
Reference in New Issue
Block a user