mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-10-05 12:28:22 +00:00
routstr-devandredshift
f1d4b3c175
The proxy accepts an API endpoint with or without a leading `v1/`
(`_canonical_api_path`) and forwards the caller's path verbatim, so a
client that spells the endpoint `chat/completions` reaches the upstream
as `chat/completions`. That is fine for providers whose `default_base_url`
carries the version prefix (openai, groq, fireworks, ...) — the client's
`v1/` is stripped and the base URL re-adds its own — but Tinfoil's base URL
is unversioned and its router serves only `/v1/...`. Every Tinfoil request
from such a client therefore got
404 {"error":{"message":"Not found.","type":"invalid_request_error"}}
from `https://inference.tinfoil.sh/chat/completions`, while the same
request with the prefix succeeded. Tinfoil's own error text blamed the
model id, which sent the search in the wrong direction.
Give the EHBP path builders the same hooks the non-EHBP forwarding path
uses: `normalize_request_path` strips the client's optional `v1/`, and a
new `ehbp_path_prefix` re-adds the prefix the provider's enclave actually
serves (`v1` for Tinfoil, `private/v1` for PPQ.AI, whose target had the
same latent bug). Both spellings now reach the same upstream URL.
`_resolve_ehbp_target_url` re-appended the caller's raw path to the
client-supplied enclave URL, re-introducing the spelling the provider had
just normalized away; it now takes the path from the target URL the
provider built, so the override swaps the host only.
1427 lines
54 KiB
Python
1427 lines
54 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import time
|
|
import traceback
|
|
from dataclasses import dataclass, field
|
|
from typing import AsyncIterator, Awaitable, Mapping
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
from fastapi import Request
|
|
from fastapi.responses import Response, StreamingResponse
|
|
|
|
from ..auth import (
|
|
ROUTSTR_FEE_PERCENT,
|
|
ReservationSnapshot,
|
|
_charge_reservation_rows,
|
|
_claim_reservation_for_charge,
|
|
_stop_reservation_heartbeat,
|
|
_validate_reservation_snapshot,
|
|
get_reservation_snapshot,
|
|
payments_logger,
|
|
release_reservation,
|
|
)
|
|
from ..core import get_logger
|
|
from ..core.db import (
|
|
ApiKey,
|
|
AsyncSession,
|
|
accumulate_routstr_fee,
|
|
)
|
|
from ..core.db import (
|
|
store_cashu_transaction_with_retry as store_cashu_transaction,
|
|
)
|
|
from ..core.error_scope import (
|
|
ERROR_SCOPE_HEADER,
|
|
ERROR_SCOPE_NODE,
|
|
ERROR_SCOPE_UPSTREAM,
|
|
UPSTREAM_ERROR_STATUS,
|
|
client_code_for_upstream_error,
|
|
client_status_for_upstream_error,
|
|
)
|
|
from ..core.exceptions import EhbpTimeoutError, UpstreamError
|
|
from ..core.settings import settings
|
|
from ..payment.cost_calculation import (
|
|
CostData,
|
|
MaxCostData,
|
|
calculate_cost,
|
|
)
|
|
from ..payment.helpers import create_error_response
|
|
from ..payment.models import Model
|
|
from ..wallet import (
|
|
SPENT_TOKEN_CODES,
|
|
classify_redemption_error,
|
|
recieve_token,
|
|
send_token,
|
|
)
|
|
from .tinfoil_trailer import TrailerResponse, forward_with_trailer
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
# Provider-neutral confidential-inference defaults. Tinfoil is the first
|
|
# EHBP implementation, but provider-specific routing, usage extraction and
|
|
# header policy belong in a profile so future TEE providers do not inherit
|
|
# Tinfoil-only assumptions.
|
|
_ENCLAVE_URL_HEADER = "X-Tinfoil-Enclave-Url"
|
|
_REQUEST_USAGE_HEADER = "X-Tinfoil-Request-Usage-Metrics"
|
|
_RESPONSE_USAGE_HEADER = "X-Tinfoil-Usage-Metrics"
|
|
_TINFOIL_PROVIDER_TYPE = "tinfoil"
|
|
_TINFOIL_ALLOWED_ENCLAVE_HOST_SUFFIX = ".tinfoil.sh"
|
|
_TINFOIL_ALLOWED_ENCLAVE_HOSTS = frozenset({"tinfoil.sh"})
|
|
|
|
|
|
_KEY_CONFIG_PROBLEM_TYPE = "urn:ietf:params:ehbp:error:key-config"
|
|
|
|
|
|
def _is_ehbp_key_config_response(resp: TrailerResponse) -> bool:
|
|
"""Check whether an upstream EHBP response is a key-config mismatch.
|
|
|
|
The enclave returns ``422 application/problem+json`` with
|
|
``type=urn:ietf:params:ehbp:error:key-config`` when it cannot decrypt the
|
|
request body — meaning the client's HPKE key is stale (the enclave rotated
|
|
keys). The proxy must pass this response through with its original
|
|
content type so EHBP clients can detect it and trigger re-attestation.
|
|
"""
|
|
if resp.status_code != 422:
|
|
return False
|
|
ct = ""
|
|
for k, v in resp.headers:
|
|
if k.lower() == "content-type":
|
|
ct = v.lower()
|
|
break
|
|
media_type = ct.split(";", 1)[0].strip()
|
|
if media_type != "application/problem+json":
|
|
return False
|
|
try:
|
|
body = json.loads(resp.body)
|
|
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
return False
|
|
return isinstance(body, dict) and body.get("type") == _KEY_CONFIG_PROBLEM_TYPE
|
|
|
|
|
|
def _passthrough_key_config_response(resp: TrailerResponse) -> Response:
|
|
"""Return the enclave's key-config 422 with its original body and content
|
|
type so the EHBP client's ``KeyConfigMismatchError`` detection fires.
|
|
|
|
Only the content type is forwarded. ``Ehbp-Response-Nonce`` must be
|
|
dropped: a nonce only carries meaning for an *encrypted* response body,
|
|
and the stock ``ehbp`` client (``shouldDecryptResponse``) checks for the
|
|
nonce *before* checking for a key-config mismatch — forwarding it would
|
|
send that client down the decrypt path on this plaintext error body, so
|
|
the re-attestation loop would never fire. Content-length is recomputed
|
|
from the body, and upstream-internal headers are filtered out.
|
|
"""
|
|
return Response(
|
|
content=resp.body,
|
|
status_code=422,
|
|
headers={"content-type": "application/problem+json"},
|
|
media_type="application/problem+json",
|
|
)
|
|
|
|
|
|
def _normalize_upstream_model_id(model_id: str | None) -> str:
|
|
"""Normalize casing and whitespace for upstream identity comparisons."""
|
|
if not model_id:
|
|
return ""
|
|
return model_id.strip().lower()
|
|
|
|
|
|
# Headers that must not be forwarded to the upstream enclave.
|
|
_PROXY_ONLY_HEADERS = frozenset(
|
|
{
|
|
"x-routstr-model",
|
|
"x-tinfoil-enclave-url",
|
|
"x-tinfoil-request-usage-metrics",
|
|
}
|
|
)
|
|
|
|
# Namespace prefix the routstr catalog applies to Tinfoil models
|
|
# (e.g. ``tinfoil-deepseek-v4-1-flash``). The SDK strips this prefix for the
|
|
# encrypted body (``getTinfoilUpstreamModelId`` in client/TinfoilSecure.ts), so
|
|
# the enclave always reports the *bare* upstream model id in the usage-metrics
|
|
# header even though the routstr model id and ``forwarded_model_id`` carry it.
|
|
TINFOIL_MODEL_PREFIX = "tinfoil-"
|
|
|
|
|
|
def parse_tinfoil_usage_metrics(header_value: str | None) -> dict | None:
|
|
"""Parse ``X-Tinfoil-Usage-Metrics`` into an OpenAI-style usage dict.
|
|
|
|
The header format is::
|
|
|
|
prompt=<n>,completion=<n>,total=<n>[,cached_prompt_tokens=<n>,
|
|
uncached_prompt_tokens=<n>][,model=<name>][,cost_usd=<usd>]
|
|
|
|
``prompt`` is the inclusive prompt total and ``cached_prompt_tokens`` is
|
|
the cache-read portion included within it. Routstr maps these to
|
|
``prompt_tokens`` and ``cache_read_input_tokens`` so ``normalize_usage``
|
|
can subtract the cached read from the prompt total (OpenAI-family
|
|
semantics). ``cost_usd`` is parsed as a float and kept for logging/
|
|
cross-checking only — billing uses the token path.
|
|
|
|
The ``model`` field (added in tinfoilsh/confidential-model-router PR #385)
|
|
is extracted as a string so callers can compare the served model against
|
|
the requested one and adjust pricing.
|
|
|
|
Returns a dict suitable for :func:`calculate_cost`, or ``None`` when the
|
|
header is absent or malformed.
|
|
"""
|
|
if not header_value:
|
|
return None
|
|
|
|
int_parts: dict[str, int] = {}
|
|
model: str | None = None
|
|
cost_usd: float | None = None
|
|
|
|
for item in header_value.split(","):
|
|
key, sep, value = item.partition("=")
|
|
if not sep:
|
|
continue
|
|
key = key.strip()
|
|
value = value.strip()
|
|
if key == "model":
|
|
model = value
|
|
continue
|
|
if key == "cost_usd":
|
|
try:
|
|
cost_usd = float(value)
|
|
except (ValueError, TypeError):
|
|
cost_usd = None
|
|
continue
|
|
try:
|
|
int_parts[key] = int(value)
|
|
except (ValueError, TypeError):
|
|
continue
|
|
|
|
prompt = int_parts.get("prompt")
|
|
completion = int_parts.get("completion")
|
|
if prompt is None or completion is None:
|
|
logger.warning(
|
|
"Failed to parse X-Tinfoil-Usage-Metrics header",
|
|
extra={
|
|
"header_value": header_value,
|
|
"parsed_parts": int_parts,
|
|
},
|
|
)
|
|
return None
|
|
|
|
result: dict[str, int | float | str] = {
|
|
"prompt_tokens": prompt,
|
|
"completion_tokens": completion,
|
|
}
|
|
if "total" in int_parts:
|
|
result["total_tokens"] = int_parts["total"]
|
|
if "cached_prompt_tokens" in int_parts:
|
|
result["cache_read_input_tokens"] = int_parts["cached_prompt_tokens"]
|
|
if "uncached_prompt_tokens" in int_parts:
|
|
result["uncached_prompt_tokens"] = int_parts["uncached_prompt_tokens"]
|
|
if cost_usd is not None:
|
|
result["cost_usd"] = cost_usd
|
|
if model:
|
|
result["model"] = model
|
|
return result
|
|
|
|
|
|
def _get_header_case_insensitive(
|
|
headers: Mapping[str, str], header_name: str
|
|
) -> str | None:
|
|
header_name_lower = header_name.lower()
|
|
for key, value in headers.items():
|
|
if key.lower() == header_name_lower:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _validated_tinfoil_enclave_base_url(enclave_url: str) -> str | None:
|
|
"""Validate and normalize a Tinfoil enclave base URL.
|
|
|
|
``X-Tinfoil-Enclave-Url`` is client supplied. Treating it as an arbitrary
|
|
forwarding destination would let callers turn Routstr into an SSRF proxy and
|
|
exfiltrate upstream Authorization headers. Only HTTPS URLs on Tinfoil-owned
|
|
hostnames are accepted.
|
|
"""
|
|
try:
|
|
parsed = urlsplit(enclave_url.strip())
|
|
port = parsed.port
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
hostname = parsed.hostname
|
|
if not hostname:
|
|
return None
|
|
|
|
host = hostname.rstrip(".").lower()
|
|
if parsed.scheme.lower() != "https":
|
|
return None
|
|
if parsed.username or parsed.password:
|
|
return None
|
|
if port not in (None, 443):
|
|
return None
|
|
if host not in _TINFOIL_ALLOWED_ENCLAVE_HOSTS and not host.endswith(
|
|
_TINFOIL_ALLOWED_ENCLAVE_HOST_SUFFIX
|
|
):
|
|
return None
|
|
|
|
# Preserve an optional base path but discard query/fragment. The request
|
|
# query string is forwarded separately via ``prepare_params``.
|
|
netloc = host if port is None else f"{host}:{port}"
|
|
return urlunsplit(("https", netloc, parsed.path.rstrip("/"), "", ""))
|
|
|
|
|
|
def _resolve_ehbp_target_url(
|
|
target_url: str,
|
|
headers: Mapping[str, str],
|
|
provider_type: str | None = None,
|
|
profile: "ConfidentialInferenceProfile | None" = None,
|
|
) -> str:
|
|
"""Resolve the provider-approved destination for an EHBP request.
|
|
|
|
Tinfoil can send the actual enclave URL in ``X-Tinfoil-Enclave-Url`` when
|
|
the SDK is pointed at a Routstr proxy. A provider profile must explicitly
|
|
opt in to client-supplied target overrides and constrain the destination;
|
|
otherwise the header is ignored so callers cannot redirect other providers
|
|
or leak upstream API keys.
|
|
|
|
Only the *host* is taken from the override: the path comes from
|
|
``target_url``, which the provider built through
|
|
:meth:`~routstr.upstream.base.BaseUpstreamProvider.build_ehbp_request_path`.
|
|
Appending the caller's raw path here instead would re-introduce the
|
|
spelling the provider just normalized away, so a client that posts to
|
|
``/chat/completions`` would reach the enclave's unversioned (404) route
|
|
even though the default target was built correctly.
|
|
"""
|
|
override_header = (
|
|
profile.client_target_url_header if profile else _ENCLAVE_URL_HEADER
|
|
)
|
|
if not override_header:
|
|
return target_url
|
|
enclave_url = _get_header_case_insensitive(headers, override_header)
|
|
if not enclave_url:
|
|
return target_url
|
|
|
|
if profile is None:
|
|
if provider_type != _TINFOIL_PROVIDER_TYPE:
|
|
logger.warning(
|
|
"Ignoring EHBP target override for provider without profile",
|
|
extra={"provider": provider_type or "unknown"},
|
|
)
|
|
return target_url
|
|
validated_base_url = _validated_tinfoil_enclave_base_url(enclave_url)
|
|
elif not profile.allow_client_target_override:
|
|
logger.warning(
|
|
"Ignoring EHBP target override for provider profile",
|
|
extra={"provider": provider_type or "unknown"},
|
|
)
|
|
return target_url
|
|
else:
|
|
validated_base_url = _validated_confidential_target_url(enclave_url, profile)
|
|
|
|
if validated_base_url is None:
|
|
logger.warning(
|
|
"Rejected invalid EHBP target override",
|
|
extra={"provider": provider_type or "unknown"},
|
|
)
|
|
raise UpstreamError(
|
|
f"Invalid {override_header}: target is not allowed for this provider",
|
|
status_code=400,
|
|
)
|
|
|
|
return f"{validated_base_url}{urlsplit(target_url).path}"
|
|
|
|
|
|
def _validated_confidential_target_url(
|
|
enclave_url: str, profile: "ConfidentialInferenceProfile"
|
|
) -> str | None:
|
|
# Client target overrides are Tinfoil-only for now. Future confidential
|
|
# inference providers must add their own constrained validator here before
|
|
# opting into ``allow_client_target_override``.
|
|
if profile.client_target_url_header == _ENCLAVE_URL_HEADER:
|
|
return _validated_tinfoil_enclave_base_url(enclave_url)
|
|
return None
|
|
|
|
|
|
def _strip_proxy_headers(
|
|
headers: dict[str, str],
|
|
profile: "ConfidentialInferenceProfile | None" = None,
|
|
) -> dict[str, str]:
|
|
"""Remove proxy-routing headers that must not reach the upstream enclave."""
|
|
proxy_only_headers = profile.proxy_only_headers if profile else _PROXY_ONLY_HEADERS
|
|
clean = {}
|
|
for key, value in headers.items():
|
|
if key.lower() not in proxy_only_headers:
|
|
clean[key] = value
|
|
return clean
|
|
|
|
|
|
def _prepare_ehbp_upstream_headers(
|
|
headers: dict[str, str],
|
|
target_headers: Mapping[str, str],
|
|
profile: "ConfidentialInferenceProfile | None" = None,
|
|
) -> dict[str, str]:
|
|
"""Merge safe request headers with provider-controlled EHBP target headers.
|
|
|
|
Client-supplied proxy control headers must be stripped, but provider-added
|
|
target headers such as ``X-Tinfoil-Request-Usage-Metrics: true`` must still
|
|
reach the upstream enclave. Strip first, then merge target headers so
|
|
callers cannot spoof proxy controls while providers can opt into protocol
|
|
features.
|
|
"""
|
|
return {**_strip_proxy_headers(headers, profile), **dict(target_headers)}
|
|
|
|
|
|
def _build_cost_info(
|
|
total_msats: int,
|
|
input_tokens: int = 0,
|
|
output_tokens: int = 0,
|
|
input_msats: int = 0,
|
|
output_msats: int = 0,
|
|
cache_read_input_tokens: int = 0,
|
|
cache_creation_input_tokens: int = 0,
|
|
cache_read_msats: int = 0,
|
|
cache_creation_msats: int = 0,
|
|
total_usd: float = 0.0,
|
|
actual_model: str | None = None,
|
|
) -> dict:
|
|
"""Build a cost-info dict with token counts and per-token-type costs.
|
|
|
|
When ``actual_model`` is set (the served model differs from the requested
|
|
one), it is included in the returned dict so callers can use it for billing
|
|
finalization and logging.
|
|
"""
|
|
result: dict[str, int | float | str | None] = {
|
|
"total_msats": total_msats,
|
|
"input_tokens": input_tokens,
|
|
"output_tokens": output_tokens,
|
|
"total_tokens": input_tokens + output_tokens,
|
|
"input_msats": input_msats,
|
|
"output_msats": output_msats,
|
|
"cache_read_input_tokens": cache_read_input_tokens,
|
|
"cache_creation_input_tokens": cache_creation_input_tokens,
|
|
"cache_read_msats": cache_read_msats,
|
|
"cache_creation_msats": cache_creation_msats,
|
|
"total_usd": total_usd,
|
|
}
|
|
if actual_model:
|
|
result["actual_model"] = actual_model
|
|
return result
|
|
|
|
|
|
def _inject_cost_response_headers(headers: dict[str, str], cost_info: dict) -> None:
|
|
"""Add per-request cost headers to an EHBP response.
|
|
|
|
Since EHBP response bodies are opaque encrypted blobs, cost cannot be
|
|
injected into the JSON body. Instead, it goes into response headers that
|
|
the client/Tinfoil SDK can read without decrypting.
|
|
"""
|
|
headers["X-Routstr-Cost-Msats"] = str(cost_info["total_msats"])
|
|
if "computed_msats" in cost_info:
|
|
headers["X-Routstr-Computed-Cost-Msats"] = str(cost_info["computed_msats"])
|
|
headers["X-Routstr-Input-Cost-Msats"] = str(cost_info["input_msats"])
|
|
headers["X-Routstr-Output-Cost-Msats"] = str(cost_info["output_msats"])
|
|
headers["X-Routstr-Cache-Read-Msats"] = str(cost_info.get("cache_read_msats", 0))
|
|
headers["X-Routstr-Cache-Creation-Msats"] = str(
|
|
cost_info.get("cache_creation_msats", 0)
|
|
)
|
|
|
|
|
|
async def _compute_ehbp_actual_cost(
|
|
usage_header: str | None,
|
|
model_obj: Model,
|
|
max_cost_for_model: int,
|
|
) -> dict:
|
|
"""Compute the actual cost in msats from Tinfoil usage metrics.
|
|
|
|
When usage is present, the result is clamped to ``[min_request_msat,
|
|
max_cost_for_model]``. Missing or unpriceable usage returns zero: encrypted
|
|
EHBP bodies cannot be estimated locally, and the authorization ceiling is
|
|
not evidence of consumption.
|
|
|
|
When the usage-metrics header includes ``model=<name>`` and it differs
|
|
from ``model_obj.id``, the actual served model's pricing is used for the
|
|
cost calculation. The returned dict includes an ``"actual_model"`` key
|
|
in that case so callers can use it for billing finalization.
|
|
|
|
Returns a dict with ``total_msats``, ``input_tokens``, ``output_tokens``,
|
|
``total_tokens``, ``input_msats``, and ``output_msats`` (and optionally
|
|
``actual_model``).
|
|
"""
|
|
usage_dict = parse_tinfoil_usage_metrics(usage_header)
|
|
if usage_dict is None:
|
|
return _build_cost_info(0)
|
|
|
|
# The enclave may serve a different model than the one requested (e.g.
|
|
# due to failover). The usage-metrics header's ``model=<name>`` carries
|
|
# the actual upstream model ID (e.g. ``glm-5-2``), which may differ from
|
|
# the client-facing ``model_obj.id`` (e.g. ``tinfoil-glm-5-2``) even when
|
|
# the correct model was served — the alias is resolved through
|
|
# ``model_obj.forwarded_model_id``. Only when the served model differs
|
|
# from the expected upstream ID do we treat it as a real mismatch and
|
|
# look up the actual model's pricing.
|
|
actual_model: str | None = usage_dict.pop("model", None) # type: ignore[arg-type]
|
|
pricing_model_id = model_obj.id
|
|
# Bill the model we actually routed to. Passing only the model *string*
|
|
# to calculate_cost makes it re-derive pricing from the global alias map,
|
|
# which resolves the id to the best-ranked candidate — not the serving
|
|
# one. Tinfoil's catalog id (e.g. ``deepseek-v4-1-flash``) is also a
|
|
# cross-provider alias, and that cheaper candidate has no cache rate, so
|
|
# the cache discount silently disappeared (and the request was
|
|
# undercharged). Hand calculate_cost the identity it cannot reconstruct.
|
|
pricing_model_obj: Model = model_obj
|
|
expected_upstream_model = model_obj.forwarded_model_id or model_obj.id
|
|
expected_identity = _normalize_upstream_model_id(expected_upstream_model)
|
|
served_identity = _normalize_upstream_model_id(actual_model)
|
|
|
|
# Ignore casing and surrounding whitespace when comparing the model
|
|
# reported by the enclave with the expected upstream model. Version
|
|
# suffixes remain part of the identity because a configured
|
|
# ``forwarded_model_id`` may intentionally include one.
|
|
if actual_model and served_identity != expected_identity:
|
|
from ..proxy import get_model_instance
|
|
|
|
# ``forwarded_model_id`` values are registered as routable aliases in
|
|
# the global model map. The resolved object can belong to a different
|
|
# provider and therefore have a different client-facing ``id`` while
|
|
# still representing the same upstream model.
|
|
#
|
|
# The enclave reports the *bare* upstream id, but the routstr model is
|
|
# namespaced ``tinfoil-`` (and the SDK strips that prefix for the
|
|
# encrypted body). Resolve the served id within the same namespace
|
|
# first: a same-model report then maps back onto the requested Tinfoil
|
|
# model, and a genuine failover lands on the actually-served Tinfoil
|
|
# model — instead of the cheaper cross-provider model the bare id
|
|
# would resolve to in the global map.
|
|
namespaced_served = actual_model
|
|
if (
|
|
expected_upstream_model.startswith(TINFOIL_MODEL_PREFIX)
|
|
and not actual_model.startswith(TINFOIL_MODEL_PREFIX)
|
|
):
|
|
namespaced_served = TINFOIL_MODEL_PREFIX + actual_model
|
|
|
|
actual_model_obj = get_model_instance(namespaced_served)
|
|
if actual_model_obj is None and namespaced_served != actual_model:
|
|
actual_model_obj = get_model_instance(actual_model)
|
|
if actual_model_obj is None:
|
|
logger.warning(
|
|
"EHBP served model not found in registry, falling back "
|
|
"to requested model for pricing",
|
|
extra={
|
|
"requested_model": model_obj.id,
|
|
"expected_upstream_model": expected_upstream_model,
|
|
"actual_model": actual_model,
|
|
},
|
|
)
|
|
actual_model = None
|
|
else:
|
|
resolved_upstream_model = (
|
|
actual_model_obj.forwarded_model_id or actual_model_obj.id
|
|
)
|
|
resolved_identity = _normalize_upstream_model_id(resolved_upstream_model)
|
|
if resolved_identity != expected_identity:
|
|
logger.info(
|
|
"EHBP served model differs from requested, using actual "
|
|
"model for pricing",
|
|
extra={
|
|
"requested_model": model_obj.id,
|
|
"expected_upstream_model": expected_upstream_model,
|
|
"actual_model": actual_model,
|
|
"resolved_upstream_model": resolved_upstream_model,
|
|
},
|
|
)
|
|
pricing_model_id = actual_model_obj.id
|
|
pricing_model_obj = actual_model_obj
|
|
else:
|
|
# A different registry/client alias resolved to the same
|
|
# upstream model; retain the requested model's pricing.
|
|
actual_model = None
|
|
else:
|
|
# Models match or no model in header — use requested model's pricing.
|
|
actual_model = None
|
|
|
|
try:
|
|
cost = await calculate_cost(
|
|
{"model": pricing_model_id, "usage": usage_dict},
|
|
max_cost_for_model,
|
|
pricing_model_obj,
|
|
)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"EHBP usage cost calculation failed; releasing instead of charging max cost",
|
|
extra={
|
|
"model": pricing_model_id,
|
|
"error": str(e),
|
|
"usage": usage_dict,
|
|
},
|
|
)
|
|
return _build_cost_info(0, actual_model=actual_model)
|
|
|
|
if isinstance(cost, MaxCostData):
|
|
logger.warning(
|
|
"EHBP calculate_cost returned MaxCostData (no usable pricing); "
|
|
"releasing instead of charging max cost",
|
|
extra={
|
|
"model": pricing_model_id,
|
|
"max_cost_for_model": max_cost_for_model,
|
|
"usage": usage_dict,
|
|
"cost_total_msats": cost.total_msats,
|
|
},
|
|
)
|
|
return _build_cost_info(0, actual_model=actual_model)
|
|
if isinstance(cost, CostData):
|
|
actual = max(int(cost.total_msats), int(settings.min_request_msat))
|
|
clamped = min(actual, max_cost_for_model)
|
|
logger.info(
|
|
"EHBP actual cost computed from usage metrics",
|
|
extra={
|
|
"model": pricing_model_id,
|
|
"usage": usage_dict,
|
|
"cost_total_msats": cost.total_msats,
|
|
"clamped_msats": clamped,
|
|
"max_cost_for_model": max_cost_for_model,
|
|
},
|
|
)
|
|
return _build_cost_info(
|
|
total_msats=clamped,
|
|
input_tokens=cost.input_tokens,
|
|
output_tokens=cost.output_tokens,
|
|
input_msats=cost.input_msats,
|
|
output_msats=cost.output_msats,
|
|
cache_read_input_tokens=cost.cache_read_input_tokens,
|
|
cache_creation_input_tokens=cost.cache_creation_input_tokens,
|
|
cache_read_msats=cost.cache_read_msats,
|
|
cache_creation_msats=cost.cache_creation_msats,
|
|
total_usd=cost.total_usd,
|
|
actual_model=actual_model,
|
|
)
|
|
# CostDataError
|
|
logger.warning(
|
|
"EHBP usage cost calculation error; releasing instead of charging max cost",
|
|
extra={
|
|
"model": pricing_model_id,
|
|
"error": getattr(cost, "message", str(cost)),
|
|
},
|
|
)
|
|
return _build_cost_info(0, actual_model=actual_model)
|
|
|
|
|
|
def _extract_usage_from_response(
|
|
resp_headers: list[tuple[str, str]],
|
|
trailers: list[tuple[str, str]],
|
|
usage_header_name: str | None = _RESPONSE_USAGE_HEADER,
|
|
) -> str | None:
|
|
"""Find provider usage metrics in response headers or trailers.
|
|
|
|
Non-streaming responses put usage in a response header. Streaming responses
|
|
put it in an HTTP trailer. httpx/httpcore silently discard trailers, so we
|
|
use h11 directly when forwarding EHBP requests.
|
|
"""
|
|
if not usage_header_name:
|
|
return None
|
|
usage_header_name_lower = usage_header_name.lower()
|
|
for k, v in resp_headers:
|
|
if k.lower() == usage_header_name_lower:
|
|
return v
|
|
for k, v in trailers:
|
|
if k.lower() == usage_header_name_lower:
|
|
return v
|
|
return None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ConfidentialInferenceProfile:
|
|
"""Provider-neutral policy for encrypted/confidential inference forwarding."""
|
|
|
|
usage_response_header: str | None = None
|
|
client_target_url_header: str | None = None
|
|
allow_client_target_override: bool = False
|
|
proxy_only_headers: frozenset[str] = _PROXY_ONLY_HEADERS
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EHBPForwardingTarget:
|
|
"""Provider-specific destination for an EHBP opaque request."""
|
|
|
|
url: str
|
|
headers: Mapping[str, str] = field(default_factory=dict)
|
|
profile: ConfidentialInferenceProfile | None = None
|
|
|
|
|
|
async def _release_failed_ehbp_charge(
|
|
reservation: ReservationSnapshot, session: AsyncSession
|
|
) -> None:
|
|
if await release_reservation(reservation, session, reservation.reserved_msats):
|
|
return
|
|
await _stop_reservation_heartbeat(reservation.release_id)
|
|
logger.critical(
|
|
"Failed to release EHBP reservation after rejected charge",
|
|
extra={"reservation_id": reservation.release_id},
|
|
)
|
|
|
|
|
|
async def _record_ehbp_settlement(
|
|
operation: Awaitable[int],
|
|
*,
|
|
key: ApiKey,
|
|
model_id: str,
|
|
settlement_type: str,
|
|
) -> int:
|
|
"""Expose EHBP settlement latency alongside normal request settlement."""
|
|
started = time.perf_counter()
|
|
# A rollback can expire the ORM instance, so capture this before the operation.
|
|
key_log_hash = key.hashed_key[:8] + "..."
|
|
succeeded = False
|
|
try:
|
|
result = await operation
|
|
succeeded = True
|
|
return result
|
|
finally:
|
|
logger.info(
|
|
"Payment settlement finished",
|
|
extra={
|
|
"key_hash": key_log_hash,
|
|
"model": model_id,
|
|
"settlement_type": settlement_type,
|
|
"settlement_duration_ms": round(
|
|
(time.perf_counter() - started) * 1000, 2
|
|
),
|
|
"settlement_succeeded": succeeded,
|
|
},
|
|
)
|
|
|
|
|
|
async def finalize_ehbp_actual_cost_payment(
|
|
key: ApiKey,
|
|
session: AsyncSession,
|
|
reserved_cost_for_model: int,
|
|
model_id: str,
|
|
cost_info: dict,
|
|
reservation_snapshot: ReservationSnapshot | None = None,
|
|
) -> int:
|
|
"""Finalize an EHBP bearer request using clamped provider usage metrics."""
|
|
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
|
await _validate_reservation_snapshot(key, reservation, session)
|
|
if not await _claim_reservation_for_charge(reservation, session):
|
|
return 0
|
|
reserved_cost_for_model = reservation.reserved_msats
|
|
key_hash = key.hashed_key
|
|
billing_key_hash = key_hash
|
|
total_cost_msats = max(
|
|
0, int(cost_info.get("total_msats", reserved_cost_for_model))
|
|
)
|
|
now = int(time.time())
|
|
|
|
charged = await _charge_reservation_rows(
|
|
session,
|
|
billing_key_hash=billing_key_hash,
|
|
reserved_msats=reserved_cost_for_model,
|
|
charge_msats=total_cost_msats,
|
|
)
|
|
if not charged:
|
|
logger.error(
|
|
"Failed to finalize EHBP usage-based payment",
|
|
extra={
|
|
"key_hash": key_hash[:8] + "...",
|
|
"billing_key_hash": billing_key_hash[:8] + "...",
|
|
"model": model_id,
|
|
"reserved_cost_for_model": reserved_cost_for_model,
|
|
"total_cost_msats": total_cost_msats,
|
|
},
|
|
)
|
|
await _release_failed_ehbp_charge(reservation, session)
|
|
return 0
|
|
|
|
await session.commit()
|
|
await _stop_reservation_heartbeat(reservation.release_id)
|
|
await session.refresh(key)
|
|
|
|
if total_cost_msats > 0 and ROUTSTR_FEE_PERCENT > 0:
|
|
fee_msats = math.ceil(total_cost_msats * ROUTSTR_FEE_PERCENT / 100)
|
|
try:
|
|
await accumulate_routstr_fee(session, fee_msats)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"Failed to accumulate Routstr fee for EHBP request",
|
|
extra={"error": str(e), "fee_msats": fee_msats},
|
|
)
|
|
|
|
payments_logger.info(
|
|
"FINALIZE",
|
|
extra={
|
|
"event": "finalize",
|
|
"key_hash": key.hashed_key[:8] + "...",
|
|
"billing_key_hash": key.hashed_key[:8] + "...",
|
|
"model": model_id,
|
|
"cost_reserved": reserved_cost_for_model,
|
|
"cost_charged": total_cost_msats,
|
|
"input_tokens": cost_info.get("input_tokens", 0),
|
|
"output_tokens": cost_info.get("output_tokens", 0),
|
|
# Cache splits are only knowable when the enclave reports
|
|
# ``cached_prompt_tokens``; absent that they are a measured zero on
|
|
# the token counts the provider did report (not an unknown), so the
|
|
# event key set stays stable for usage-analytics consumers.
|
|
"cache_read_input_tokens": cost_info.get("cache_read_input_tokens", 0),
|
|
"cache_creation_input_tokens": cost_info.get(
|
|
"cache_creation_input_tokens", 0
|
|
),
|
|
"cache_read_msats": cost_info.get("cache_read_msats", 0),
|
|
"cache_creation_msats": cost_info.get("cache_creation_msats", 0),
|
|
"balance": key.balance,
|
|
"reserved_balance": key.reserved_balance,
|
|
"total_spent": key.total_spent,
|
|
"finalize_type": "ehbp_usage",
|
|
"finalized_at": now,
|
|
},
|
|
)
|
|
return total_cost_msats
|
|
|
|
|
|
async def finalize_ehbp_max_cost_payment(
|
|
key: ApiKey,
|
|
session: AsyncSession,
|
|
max_cost_for_model: int,
|
|
model_id: str,
|
|
reservation_snapshot: ReservationSnapshot | None = None,
|
|
) -> int:
|
|
"""Release an unmeasured EHBP request without charging its reservation.
|
|
|
|
The legacy name is retained for compatibility with internal callers. EHBP
|
|
responses are encrypted, so no local estimate is possible when the trusted
|
|
usage header/trailer is absent.
|
|
"""
|
|
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
|
await _validate_reservation_snapshot(key, reservation, session)
|
|
key_log_hash = key.hashed_key[:8] + "..."
|
|
await release_reservation(reservation, session, reservation.reserved_msats)
|
|
logger.warning(
|
|
"Released unmeasured EHBP reservation without charging max cost",
|
|
extra={
|
|
"key_hash": key_log_hash,
|
|
"model": model_id,
|
|
"max_cost_for_model": max_cost_for_model,
|
|
},
|
|
)
|
|
return 0
|
|
|
|
|
|
async def send_cashu_refund(
|
|
amount: int,
|
|
unit: str,
|
|
mint: str | None = None,
|
|
request_id: str | None = None,
|
|
) -> str:
|
|
"""Create a Cashu refund token and record the outgoing transaction."""
|
|
refund_token = await send_token(amount, unit=unit, mint_url=mint)
|
|
await store_cashu_transaction(
|
|
token=refund_token,
|
|
amount=amount,
|
|
unit=unit,
|
|
mint_url=mint,
|
|
typ="out",
|
|
request_id=request_id,
|
|
)
|
|
return refund_token
|
|
|
|
|
|
def _msats_to_unit_amount(msats: int, unit: str) -> int:
|
|
if unit == "msat":
|
|
return msats
|
|
if unit == "sat":
|
|
return (msats + 999) // 1000
|
|
raise ValueError(f"Invalid unit: {unit}")
|
|
|
|
|
|
async def forward_ehbp_request(
|
|
*,
|
|
request: Request,
|
|
path: str,
|
|
headers: dict,
|
|
request_body: bytes | None,
|
|
upstream: object,
|
|
key: ApiKey,
|
|
max_cost_for_model: int,
|
|
session: AsyncSession,
|
|
model_obj: Model,
|
|
reservation_snapshot: ReservationSnapshot | None = None,
|
|
) -> Response | StreamingResponse:
|
|
"""Forward an EHBP bearer-auth request and finalize billing.
|
|
|
|
Sends ``X-Tinfoil-Request-Usage-Metrics: true`` so the enclave returns token
|
|
counts in the ``X-Tinfoil-Usage-Metrics`` response header (non-streaming) or
|
|
trailer (streaming). Usage is captured from both response headers and HTTP
|
|
trailers via an h11-based client (httpx silently discards trailers).
|
|
"""
|
|
target = upstream.get_ehbp_forwarding_target(path, model_obj) # type: ignore[attr-defined]
|
|
|
|
provider_type = getattr(upstream, "provider_type", "unknown")
|
|
profile = target.profile or upstream.get_confidential_inference_profile() # type: ignore[attr-defined]
|
|
target_url = _resolve_ehbp_target_url(target.url, headers, provider_type, profile)
|
|
upstream_headers = _prepare_ehbp_upstream_headers(headers, target.headers, profile)
|
|
|
|
# Merge query params into the target URL since forward_with_trailer
|
|
# doesn't have a separate params argument.
|
|
query_params = upstream.prepare_params(path, request.query_params) # type: ignore[attr-defined]
|
|
if query_params:
|
|
from urllib.parse import urlencode
|
|
|
|
target_url = f"{target_url}?{urlencode(query_params)}"
|
|
|
|
logger.debug(
|
|
"Forwarding EHBP request to upstream",
|
|
extra={
|
|
"url": target_url,
|
|
"method": request.method,
|
|
"path": path,
|
|
"model": model_obj.id,
|
|
"provider": provider_type,
|
|
"key_hash": key.hashed_key[:8] + "...",
|
|
},
|
|
)
|
|
|
|
try:
|
|
resp = await forward_with_trailer(
|
|
method=request.method,
|
|
url=target_url,
|
|
headers=upstream_headers,
|
|
body=request_body or b"",
|
|
)
|
|
|
|
if resp.status_code != 200:
|
|
body_preview = resp.body.decode("utf-8", errors="ignore").strip()[:500]
|
|
logger.error(
|
|
"EHBP upstream %s returned %s for model=%s path=%s: %s",
|
|
provider_type,
|
|
resp.status_code,
|
|
model_obj.id,
|
|
path,
|
|
body_preview or "<empty>",
|
|
extra={
|
|
"provider": provider_type,
|
|
"model": model_obj.id,
|
|
"status_code": resp.status_code,
|
|
"path": path,
|
|
"body_preview": body_preview,
|
|
},
|
|
)
|
|
# Key-config mismatch (stale client HPKE key): return the
|
|
# enclave's 422 problem+json directly so the SDK's
|
|
# KeyConfigMismatchError detection fires and triggers
|
|
# re-attestation. Wrapping it as application/json would
|
|
# destroy the signal and cause permanent failure.
|
|
if _is_ehbp_key_config_response(resp):
|
|
logger.warning(
|
|
"EHBP upstream %s returned key-config mismatch for model=%s, "
|
|
"passing through for client re-attestation",
|
|
provider_type,
|
|
model_obj.id,
|
|
extra={
|
|
"provider": provider_type,
|
|
"model": model_obj.id,
|
|
"path": path,
|
|
},
|
|
)
|
|
return _passthrough_key_config_response(resp)
|
|
|
|
raise UpstreamError(
|
|
f"EHBP upstream {provider_type} returned {resp.status_code} "
|
|
f"for model {model_obj.id}: {body_preview[:200] or '<empty>'}",
|
|
status_code=resp.status_code,
|
|
from_upstream_response=True,
|
|
)
|
|
|
|
# Check for usage metrics in response headers (non-streaming) or
|
|
# trailers (streaming). h11 captures both.
|
|
usage_header_name = (
|
|
profile.usage_response_header if profile else _RESPONSE_USAGE_HEADER
|
|
)
|
|
usage_header = _extract_usage_from_response(
|
|
resp.headers, resp.trailers, usage_header_name
|
|
)
|
|
usage_dict = parse_tinfoil_usage_metrics(usage_header)
|
|
usage_source = (
|
|
"header"
|
|
if usage_header_name
|
|
and any(k.lower() == usage_header_name.lower() for k, _ in resp.headers)
|
|
else ("trailer" if usage_header else "none")
|
|
)
|
|
|
|
logger.info(
|
|
"EHBP upstream response received",
|
|
extra={
|
|
"model": model_obj.id,
|
|
"provider": provider_type,
|
|
"target_url": target_url,
|
|
"status_code": resp.status_code,
|
|
"usage_header_raw": usage_header,
|
|
"usage_source": usage_source,
|
|
"has_trailers": bool(resp.trailers),
|
|
"body_length": len(resp.body),
|
|
"key_hash": key.hashed_key[:8] + "...",
|
|
},
|
|
)
|
|
|
|
if usage_dict is not None:
|
|
logger.info(
|
|
"EHBP usage metrics received, finalizing with actual token cost",
|
|
extra={
|
|
"model": model_obj.id,
|
|
"provider": provider_type,
|
|
"usage": usage_dict,
|
|
"usage_source": usage_source,
|
|
"key_hash": key.hashed_key[:8] + "...",
|
|
},
|
|
)
|
|
cost_info = await _compute_ehbp_actual_cost(
|
|
usage_header, model_obj, max_cost_for_model
|
|
)
|
|
billing_model = cost_info.pop("actual_model", None) or model_obj.id
|
|
computed_msats = int(cost_info["total_msats"])
|
|
charged_msats = await _record_ehbp_settlement(
|
|
finalize_ehbp_actual_cost_payment(
|
|
key,
|
|
session,
|
|
max_cost_for_model,
|
|
billing_model,
|
|
cost_info,
|
|
reservation_snapshot,
|
|
),
|
|
key=key,
|
|
model_id=billing_model,
|
|
settlement_type="ehbp_usage",
|
|
)
|
|
cost_data = {
|
|
**cost_info,
|
|
"total_msats": charged_msats,
|
|
"charged_msats": charged_msats,
|
|
"total_usd": cost_info.get("total_usd", 0.0),
|
|
}
|
|
if computed_msats != charged_msats:
|
|
cost_data["computed_msats"] = computed_msats
|
|
else:
|
|
logger.warning(
|
|
"EHBP usage metrics not found in headers or trailers; "
|
|
"releasing instead of charging the authorization ceiling",
|
|
extra={
|
|
"model": model_obj.id,
|
|
"provider": provider_type,
|
|
"key_hash": key.hashed_key[:8] + "...",
|
|
},
|
|
)
|
|
charged_msats = await _record_ehbp_settlement(
|
|
finalize_ehbp_max_cost_payment(
|
|
key,
|
|
session,
|
|
max_cost_for_model,
|
|
model_obj.id,
|
|
reservation_snapshot,
|
|
),
|
|
key=key,
|
|
model_id=model_obj.id,
|
|
settlement_type="ehbp_unmeasured_release",
|
|
)
|
|
cost_data = {
|
|
"total_msats": charged_msats,
|
|
"charged_msats": charged_msats,
|
|
"total_usd": 0.0,
|
|
"input_tokens": 0,
|
|
"output_tokens": 0,
|
|
}
|
|
|
|
# Build the cost_info dict from measured usage or the unmeasured-release
|
|
# fallback. Fields match CostData/MaxCostData.dict().
|
|
cost_info = {
|
|
"total_msats": cost_data.get("total_msats", max_cost_for_model),
|
|
"input_tokens": cost_data.get("input_tokens", 0),
|
|
"output_tokens": cost_data.get("output_tokens", 0),
|
|
"total_tokens": cost_data.get("input_tokens", 0)
|
|
+ cost_data.get("output_tokens", 0),
|
|
"input_msats": cost_data.get("input_msats", 0),
|
|
"output_msats": cost_data.get("output_msats", 0),
|
|
"cache_read_input_tokens": cost_data.get("cache_read_input_tokens", 0),
|
|
"cache_creation_input_tokens": cost_data.get(
|
|
"cache_creation_input_tokens", 0
|
|
),
|
|
"cache_read_msats": cost_data.get("cache_read_msats", 0),
|
|
"cache_creation_msats": cost_data.get("cache_creation_msats", 0),
|
|
}
|
|
if "computed_msats" in cost_data:
|
|
cost_info["computed_msats"] = cost_data["computed_msats"]
|
|
cost_usd = cost_data.get("total_usd", 0.0)
|
|
|
|
# Build response headers, filtering out hop-by-hop headers
|
|
response_headers: dict[str, str] = {}
|
|
hop_by_hop = {
|
|
"connection",
|
|
"keep-alive",
|
|
"transfer-encoding",
|
|
"trailer",
|
|
"content-length",
|
|
}
|
|
for k, v in resp.headers:
|
|
if k.lower() not in hop_by_hop:
|
|
response_headers[k] = v
|
|
|
|
# Surface per-request cost to the client. Since EHBP bodies are
|
|
# opaque, cost info can only go into response headers.
|
|
_inject_cost_response_headers(response_headers, cost_info)
|
|
response_headers["X-Routstr-Cost-Usd"] = str(cost_usd)
|
|
|
|
async def _stream_body() -> AsyncIterator[bytes]:
|
|
yield resp.body
|
|
|
|
return StreamingResponse(
|
|
_stream_body(),
|
|
status_code=resp.status_code,
|
|
headers=response_headers,
|
|
)
|
|
except UpstreamError:
|
|
raise
|
|
except Exception as exc:
|
|
tb = traceback.format_exc()
|
|
logger.error(
|
|
"Unexpected error in EHBP upstream forwarding",
|
|
extra={
|
|
"error": str(exc),
|
|
"error_type": type(exc).__name__,
|
|
"method": request.method,
|
|
"url": target_url,
|
|
"path": path,
|
|
"traceback": tb,
|
|
},
|
|
)
|
|
raise UpstreamError(
|
|
"An unexpected server error occurred",
|
|
status_code=500,
|
|
scope=ERROR_SCOPE_NODE,
|
|
)
|
|
|
|
|
|
async def forward_ehbp_x_cashu_request(
|
|
*,
|
|
request: Request,
|
|
x_cashu_token: str,
|
|
path: str,
|
|
max_cost_for_model: int,
|
|
model_obj: Model,
|
|
upstream: object,
|
|
) -> Response | StreamingResponse:
|
|
"""Redeem X-Cashu, forward EHBP opaquely, and refund unspent value.
|
|
|
|
When the upstream returns ``X-Tinfoil-Usage-Metrics`` in the response
|
|
header (non-streaming) or as an HTTP trailer (streaming), the refund is
|
|
computed from the actual token cost. Trailers are captured via an h11-based
|
|
client because httpx silently discards them.
|
|
"""
|
|
request_id = getattr(request.state, "request_id", None)
|
|
amount = 0
|
|
unit = "msat"
|
|
mint: str | None = None
|
|
redeemed = False
|
|
|
|
try:
|
|
amount, unit, mint = await recieve_token(x_cashu_token)
|
|
redeemed = True
|
|
await store_cashu_transaction(
|
|
token=x_cashu_token,
|
|
amount=amount,
|
|
unit=unit,
|
|
mint_url=mint,
|
|
typ="in",
|
|
request_id=request_id,
|
|
collected=True,
|
|
)
|
|
|
|
headers = upstream.prepare_headers(dict(request.headers)) # type: ignore[attr-defined]
|
|
target = upstream.get_ehbp_forwarding_target(path, model_obj) # type: ignore[attr-defined]
|
|
provider_type = getattr(upstream, "provider_type", "unknown")
|
|
profile = target.profile or upstream.get_confidential_inference_profile() # type: ignore[attr-defined]
|
|
target_url = _resolve_ehbp_target_url(
|
|
target.url, headers, provider_type, profile
|
|
)
|
|
upstream_headers = _prepare_ehbp_upstream_headers(
|
|
headers, target.headers, profile
|
|
)
|
|
request_body = await request.body()
|
|
|
|
# Merge query params into the target URL
|
|
query_params = upstream.prepare_params(path, request.query_params) # type: ignore[attr-defined]
|
|
if query_params:
|
|
from urllib.parse import urlencode
|
|
|
|
target_url = f"{target_url}?{urlencode(query_params)}"
|
|
|
|
try:
|
|
resp = await forward_with_trailer(
|
|
method=request.method,
|
|
url=target_url,
|
|
headers=upstream_headers,
|
|
body=request_body,
|
|
)
|
|
|
|
if resp.status_code != 200:
|
|
# Key-config mismatch (stale client HPKE key): refund the
|
|
# full token and pass the enclave's 422 problem+json through
|
|
# so the SDK's KeyConfigMismatchError detection fires.
|
|
if _is_ehbp_key_config_response(resp):
|
|
logger.warning(
|
|
"EHBP upstream %s returned key-config mismatch for "
|
|
"model=%s, refunding and passing through",
|
|
provider_type,
|
|
model_obj.id,
|
|
extra={
|
|
"provider": provider_type,
|
|
"model": model_obj.id,
|
|
"path": path,
|
|
"refunded_amount": amount,
|
|
},
|
|
)
|
|
refund_token = await send_cashu_refund(
|
|
amount, unit, mint, request_id
|
|
)
|
|
passthrough = _passthrough_key_config_response(resp)
|
|
passthrough.headers["X-Cashu"] = refund_token
|
|
return passthrough
|
|
|
|
refund_token = await send_cashu_refund(amount, unit, mint, request_id)
|
|
error_response = Response(
|
|
content=json.dumps(
|
|
{
|
|
"error": {
|
|
"message": "Error forwarding EHBP request to upstream",
|
|
"type": "upstream_error",
|
|
# Pass the status as the code so a provider 4xx
|
|
# keeps the legacy numeric ``code``.
|
|
"code": client_code_for_upstream_error(
|
|
resp.status_code, resp.status_code
|
|
),
|
|
"upstream_status": resp.status_code,
|
|
"refund_token": refund_token,
|
|
}
|
|
}
|
|
),
|
|
status_code=client_status_for_upstream_error(resp.status_code),
|
|
media_type="application/json",
|
|
)
|
|
error_response.headers["X-Cashu"] = refund_token
|
|
error_response.headers[ERROR_SCOPE_HEADER] = ERROR_SCOPE_UPSTREAM
|
|
return error_response
|
|
|
|
# Compute refund from actual usage when available — check both
|
|
# response headers (non-streaming) and trailers (streaming).
|
|
usage_header_name = (
|
|
profile.usage_response_header if profile else _RESPONSE_USAGE_HEADER
|
|
)
|
|
usage_header = _extract_usage_from_response(
|
|
resp.headers, resp.trailers, usage_header_name
|
|
)
|
|
usage_source = (
|
|
"header"
|
|
if usage_header_name
|
|
and any(k.lower() == usage_header_name.lower() for k, _ in resp.headers)
|
|
else ("trailer" if usage_header else "none")
|
|
)
|
|
|
|
logger.info(
|
|
"EHBP X-Cashu upstream response received",
|
|
extra={
|
|
"model": model_obj.id,
|
|
"provider": provider_type,
|
|
"target_url": target_url,
|
|
"status_code": resp.status_code,
|
|
"usage_header_raw": usage_header,
|
|
"usage_source": usage_source,
|
|
"has_trailers": bool(resp.trailers),
|
|
"body_length": len(resp.body),
|
|
"redeemed_amount": amount,
|
|
"unit": unit,
|
|
"max_cost_for_model": max_cost_for_model,
|
|
},
|
|
)
|
|
|
|
cost_info = await _compute_ehbp_actual_cost(
|
|
usage_header, model_obj, max_cost_for_model
|
|
)
|
|
actual_cost_msats = cost_info["total_msats"]
|
|
actual_model = cost_info.get("actual_model")
|
|
billing_model = actual_model or model_obj.id
|
|
refund_amount = amount - _msats_to_unit_amount(actual_cost_msats, unit)
|
|
logger.info(
|
|
"EHBP X-Cashu refund computed",
|
|
extra={
|
|
"model": billing_model,
|
|
"requested_model": model_obj.id,
|
|
"actual_model": actual_model,
|
|
"redeemed_amount": amount,
|
|
"actual_cost_msats": actual_cost_msats,
|
|
"refund_amount": refund_amount,
|
|
"unit": unit,
|
|
"usage_source": usage_source,
|
|
},
|
|
)
|
|
|
|
# Build response headers, filtering out hop-by-hop headers
|
|
response_headers: dict[str, str] = {}
|
|
hop_by_hop = {
|
|
"connection",
|
|
"keep-alive",
|
|
"transfer-encoding",
|
|
"trailer",
|
|
"content-length",
|
|
}
|
|
for k, v in resp.headers:
|
|
if k.lower() not in hop_by_hop:
|
|
response_headers[k] = v
|
|
|
|
# Surface per-request cost to the client. Since EHBP bodies are
|
|
# opaque encrypted blobs, cost can only go into response headers.
|
|
_inject_cost_response_headers(response_headers, cost_info)
|
|
|
|
if refund_amount > 0:
|
|
response_headers["X-Cashu"] = await send_cashu_refund(
|
|
refund_amount, unit, mint, request_id
|
|
)
|
|
|
|
async def _stream_body_xcashu() -> AsyncIterator[bytes]:
|
|
yield resp.body
|
|
|
|
return StreamingResponse(
|
|
_stream_body_xcashu(),
|
|
status_code=resp.status_code,
|
|
headers=response_headers,
|
|
)
|
|
except Exception:
|
|
raise
|
|
|
|
except EhbpTimeoutError as e:
|
|
logger.warning(
|
|
"EHBP X-Cashu upstream timed out",
|
|
extra={
|
|
"error": str(e),
|
|
"path": path,
|
|
"method": request.method,
|
|
"redeemed": redeemed,
|
|
},
|
|
)
|
|
|
|
if redeemed and amount > 0:
|
|
try:
|
|
refund_token = await send_cashu_refund(amount, unit, mint, request_id)
|
|
error_response = create_error_response(
|
|
"upstream_timeout",
|
|
str(e),
|
|
UPSTREAM_ERROR_STATUS,
|
|
request=request,
|
|
code="UPSTREAM_TIMEOUT",
|
|
error_scope=ERROR_SCOPE_UPSTREAM,
|
|
)
|
|
error_response.headers["X-Cashu"] = refund_token
|
|
return error_response
|
|
except Exception as refund_error:
|
|
logger.error(
|
|
"Failed to refund EHBP X-Cashu token after timeout",
|
|
extra={
|
|
"error": str(refund_error),
|
|
"original_error": str(e),
|
|
},
|
|
)
|
|
|
|
return create_error_response(
|
|
"upstream_timeout",
|
|
str(e),
|
|
UPSTREAM_ERROR_STATUS,
|
|
request=request,
|
|
code="UPSTREAM_TIMEOUT",
|
|
error_scope=ERROR_SCOPE_UPSTREAM,
|
|
)
|
|
|
|
except Exception as e:
|
|
error_message = str(e)
|
|
logger.error(
|
|
"EHBP X-Cashu request failed",
|
|
extra={
|
|
"error": error_message,
|
|
"error_type": type(e).__name__,
|
|
"path": path,
|
|
"method": request.method,
|
|
"redeemed": redeemed,
|
|
},
|
|
)
|
|
|
|
if redeemed and amount > 0:
|
|
try:
|
|
refund_token = await send_cashu_refund(amount, unit, mint, request_id)
|
|
error_response = create_error_response(
|
|
"upstream_error",
|
|
"EHBP request failed after token redemption; refunded token",
|
|
UPSTREAM_ERROR_STATUS,
|
|
request=request,
|
|
error_scope=ERROR_SCOPE_UPSTREAM,
|
|
)
|
|
error_response.headers["X-Cashu"] = refund_token
|
|
return error_response
|
|
except Exception as refund_error:
|
|
logger.error(
|
|
"Failed to refund EHBP X-Cashu token after error",
|
|
extra={
|
|
"error": str(refund_error),
|
|
"original_error": error_message,
|
|
},
|
|
)
|
|
|
|
if not redeemed:
|
|
classified = classify_redemption_error(e)
|
|
if classified is not None:
|
|
error_type, status_code, message, error_code = classified
|
|
# Never re-offer a spent/consumed token.
|
|
echo_token = None if error_code in SPENT_TOKEN_CODES else x_cashu_token
|
|
return create_error_response(
|
|
error_type,
|
|
message,
|
|
status_code,
|
|
request=request,
|
|
token=echo_token,
|
|
code=error_code,
|
|
)
|
|
# Raw exception text may contain the attacker-supplied mint URL.
|
|
return create_error_response(
|
|
"api_error",
|
|
"Internal error during token redemption",
|
|
500,
|
|
request=request,
|
|
token=x_cashu_token,
|
|
code="internal_error",
|
|
)
|
|
|
|
if "already spent" in error_message.lower():
|
|
return create_error_response(
|
|
"token_already_spent",
|
|
"The provided CASHU token has already been spent",
|
|
400,
|
|
request=request,
|
|
token=x_cashu_token,
|
|
)
|
|
|
|
if "invalid token" in error_message.lower():
|
|
return create_error_response(
|
|
"invalid_token",
|
|
"The provided CASHU token is invalid",
|
|
400,
|
|
request=request,
|
|
token=x_cashu_token,
|
|
)
|
|
|
|
if "mint error" in error_message.lower():
|
|
return create_error_response(
|
|
"mint_error",
|
|
f"CASHU mint error: {error_message}",
|
|
422,
|
|
request=request,
|
|
token=x_cashu_token,
|
|
)
|
|
|
|
return create_error_response(
|
|
"cashu_error" if not redeemed else "upstream_error",
|
|
f"EHBP X-Cashu request failed: {error_message}",
|
|
400 if not redeemed else UPSTREAM_ERROR_STATUS,
|
|
request=request,
|
|
token=x_cashu_token if not redeemed else None,
|
|
error_scope=None if not redeemed else ERROR_SCOPE_UPSTREAM,
|
|
)
|