fix(ehbp): resolve the served model within the tinfoil namespace before pricing

The cache discount never applied in production even though everything
downstream of it worked: the enclave reported cache splits
(cached_prompt_tokens=160512 of 161652) and the parse put them into
cache_read_input_tokens, but the applied cache-read rate was the full input
rate.

Root cause: the SDK strips the routstr "tinfoil-" namespace prefix for the
encrypted body (getTinfoilUpstreamModelId), so the enclave always reports the
bare upstream id ("deepseek-v4-1-flash") in X-Tinfoil-Usage-Metrics, while
the catalog registers the model as "tinfoil-deepseek-v4-1-flash" and
forwarded_model_id carries the prefix too. _normalize_upstream_model_id only
lowercases, so every request took the "served model differs" path and
re-derived pricing via get_model_instance on the bare id — which resolves
globally to a cheaper cross-provider model whose pricing has no cache rate
(input_cache_read=0), falling back to the full input price.

Billing was therefore doubly wrong: no cache discount, and E2EE requests
undercharged at the cross-provider rate instead of the Tinfoil rate.

Fix: when the requested model is namespaced "tinfoil-" and the served id is
not, resolve the served id within the same namespace first (with a bare-id
fallback). A same-model report then maps back onto the requested Tinfoil
model (keeping its pricing), and a genuine failover lands on the
actually-served Tinfoil model — while the bare-id lookup that previously
hijacked pricing is only used as a last resort.
This commit is contained in:
redshift
2026-09-17 22:20:25 +02:00
parent 6246028eb6
commit 3c813daedc
2 changed files with 145 additions and 1 deletions
+25 -1
View File
@@ -127,6 +127,13 @@ _PROXY_ONLY_HEADERS = frozenset(
}
)
# Namespace prefix the routstr catalog applies to Tinfoil models
# (e.g. ``tinfoil-deepseek-v4-1-flash``). The SDK strips this prefix for the
# encrypted body (``getTinfoilUpstreamModelId`` in client/TinfoilSecure.ts), so
# the enclave always reports the *bare* upstream model id in the usage-metrics
# header even though the routstr model id and ``forwarded_model_id`` carry it.
TINFOIL_MODEL_PREFIX = "tinfoil-"
def parse_tinfoil_usage_metrics(header_value: str | None) -> dict | None:
"""Parse ``X-Tinfoil-Usage-Metrics`` into an OpenAI-style usage dict.
@@ -451,7 +458,24 @@ async def _compute_ehbp_actual_cost(
# the global model map. The resolved object can belong to a different
# provider and therefore have a different client-facing ``id`` while
# still representing the same upstream model.
actual_model_obj = get_model_instance(actual_model)
#
# The enclave reports the *bare* upstream id, but the routstr model is
# namespaced ``tinfoil-`` (and the SDK strips that prefix for the
# encrypted body). Resolve the served id within the same namespace
# first: a same-model report then maps back onto the requested Tinfoil
# model, and a genuine failover lands on the actually-served Tinfoil
# model — instead of the cheaper cross-provider model the bare id
# would resolve to in the global map.
namespaced_served = actual_model
if (
expected_upstream_model.startswith(TINFOIL_MODEL_PREFIX)
and not actual_model.startswith(TINFOIL_MODEL_PREFIX)
):
namespaced_served = TINFOIL_MODEL_PREFIX + actual_model
actual_model_obj = get_model_instance(namespaced_served)
if actual_model_obj is None and namespaced_served != actual_model:
actual_model_obj = get_model_instance(actual_model)
if actual_model_obj is None:
logger.warning(
"EHBP served model not found in registry, falling back "
+120
View File
@@ -488,6 +488,126 @@ class TestComputeEhbpActualCost:
call_args = mock_calc.call_args
assert call_args[0][0]["model"] == "tinfoil-llama3-3-70b"
@pytest.mark.asyncio
async def test_namespaced_prefix_served_bare_keeps_requested_pricing(
self,
) -> None:
"""Production shape: the catalog model is ``tinfoil-X`` and its
``forwarded_model_id`` carries the prefix, the SDK strips the prefix
for the encrypted body, and the enclave reports bare ``X``. Pricing
must stay on the requested Tinfoil model (correct rate + cache
discount), not the cheaper cross-provider model the bare id resolves
to in the global map."""
model_obj = MagicMock()
model_obj.id = "tinfoil-deepseek-v4-1-flash"
model_obj.forwarded_model_id = "tinfoil-deepseek-v4-1-flash"
tinfoil_model = MagicMock()
tinfoil_model.id = "tinfoil-deepseek-v4-1-flash"
tinfoil_model.forwarded_model_id = "tinfoil-deepseek-v4-1-flash"
# The cheaper cross-provider model the bare id resolves to globally.
cross_provider_model = MagicMock()
cross_provider_model.id = "deepseek-v4-1-flash"
cross_provider_model.forwarded_model_id = "deepseek-v4-1-flash"
registry = {
"tinfoil-deepseek-v4-1-flash": tinfoil_model,
"deepseek-v4-1-flash": cross_provider_model,
}
with (
patch(
"routstr.proxy.get_model_instance",
side_effect=lambda name: registry.get(name),
),
patch(
"routstr.upstream.ehbp.calculate_cost",
new_callable=AsyncMock,
) as mock_calc,
):
from routstr.payment.cost_calculation import CostData
mock_calc.return_value = CostData(
base_msats=0,
input_msats=5,
output_msats=10,
total_msats=15,
total_usd=0.0,
input_tokens=5,
output_tokens=10,
cache_read_input_tokens=64,
cache_creation_input_tokens=0,
cache_read_msats=1,
cache_creation_msats=0,
)
result = await _compute_ehbp_actual_cost(
"prompt=69,completion=10,total=79,"
"cached_prompt_tokens=64,uncached_prompt_tokens=5,"
"model=deepseek-v4-1-flash",
model_obj,
100_000,
)
# No mismatch: pricing stays on the requested Tinfoil model.
assert "actual_model" not in result
call_args = mock_calc.call_args
assert call_args[0][0]["model"] == "tinfoil-deepseek-v4-1-flash"
@pytest.mark.asyncio
async def test_namespaced_prefix_failover_uses_served_tinfoil_model(
self,
) -> None:
"""A genuine failover (asked ``tinfoil-glm-5-3``, enclave served
``glm-5-3-flash``) must bill the served *Tinfoil* model, not the
cheaper cross-provider alias the bare id resolves to."""
model_obj = MagicMock()
model_obj.id = "tinfoil-glm-5-3"
model_obj.forwarded_model_id = "tinfoil-glm-5-3"
served_tinfoil = MagicMock()
served_tinfoil.id = "tinfoil-glm-5-3-flash"
served_tinfoil.forwarded_model_id = "tinfoil-glm-5-3-flash"
cross_provider = MagicMock()
cross_provider.id = "glm-5-3-flash"
cross_provider.forwarded_model_id = "glm-5-3-flash"
registry = {
"tinfoil-glm-5-3-flash": served_tinfoil,
"glm-5-3-flash": cross_provider,
}
with (
patch(
"routstr.proxy.get_model_instance",
side_effect=lambda name: registry.get(name),
),
patch(
"routstr.upstream.ehbp.calculate_cost",
new_callable=AsyncMock,
) as mock_calc,
):
from routstr.payment.cost_calculation import CostData
mock_calc.return_value = CostData(
base_msats=0,
input_msats=20,
output_msats=40,
total_msats=60,
total_usd=0.0,
input_tokens=42,
output_tokens=10,
)
result = await _compute_ehbp_actual_cost(
"prompt=42,completion=10,total=52,model=glm-5-3-flash",
model_obj,
100_000,
)
assert result["actual_model"] == "glm-5-3-flash"
# Billed on the served *Tinfoil* model, not the bare-id alias.
call_args = mock_calc.call_args
assert call_args[0][0]["model"] == "tinfoil-glm-5-3-flash"
@pytest.mark.asyncio
async def test_model_mismatch_unknown_model_falls_back(self) -> None:
"""When the served model is not in the registry, use requested model."""