From f0ae282b3faea598d06808fa089ba766417f0555 Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Sun, 23 Aug 2026 22:11:11 +0200 Subject: [PATCH] fix modal routing for cheapest first --- routstr/algorithm.py | 33 +++- tests/unit/test_ranking_maxcost_mismatch.py | 169 ++++++++++++++++++++ 2 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_ranking_maxcost_mismatch.py diff --git a/routstr/algorithm.py b/routstr/algorithm.py index 084972a8..1e338768 100644 --- a/routstr/algorithm.py +++ b/routstr/algorithm.py @@ -59,6 +59,23 @@ def calculate_model_cost_score(model: "Model") -> float: return total_cost +def calculate_model_reservation_score(model: "Model") -> float: + """Context-based reservation ceiling for ranking same-model candidates. + + The balance gate reserves on ``sats_pricing.max_cost``, which scales with + ``context_length``. Ranking on the same ceiling keeps the advertised, + routed, and reserved candidate consistent. Lower is better. + """ + from .payment.models import _calculate_usd_max_costs + + try: + _, _, max_cost = _calculate_usd_max_costs(model) + return float(max_cost) + except Exception: + # Pricing shape missing/invalid; per-token score keeps order deterministic + return calculate_model_cost_score(model) + + def get_provider_penalty(provider: "BaseUpstreamProvider") -> float: """Calculate a penalty multiplier for certain providers. @@ -345,15 +362,19 @@ def create_model_mappings( return 1 for alias, items in candidates.items(): - # Sort key: (priority DESC, cost ASC) - # Using negative cost for DESC sort overall to keep high priority first - def sort_key(item: tuple["Model", "BaseUpstreamProvider"]) -> tuple[int, float]: + # Sort key: (priority DESC, reservation ASC, cost ASC) + # Using negative costs for DESC sort overall to keep high priority first + def sort_key( + item: tuple["Model", "BaseUpstreamProvider"], + ) -> tuple[int, float, float]: model, provider = item priority = alias_priority(model, alias) - cost = calculate_model_cost_score(model) penalty = get_provider_penalty(provider) - adjusted_cost = cost * penalty - return (priority, -adjusted_cost) + # Rank on the reservation ceiling the balance gate enforces, with + # per-token typical-usage cost as tiebreaker + adjusted_reservation = calculate_model_reservation_score(model) * penalty + adjusted_cost = calculate_model_cost_score(model) * penalty + return (priority, -adjusted_reservation, -adjusted_cost) items.sort(key=sort_key, reverse=True) diff --git a/tests/unit/test_ranking_maxcost_mismatch.py b/tests/unit/test_ranking_maxcost_mismatch.py new file mode 100644 index 00000000..438dcf0d --- /dev/null +++ b/tests/unit/test_ranking_maxcost_mismatch.py @@ -0,0 +1,169 @@ +"""Ranking-vs-reservation mismatch for same-model multi-provider setups. + +``calculate_model_cost_score`` weights a typical request and ignores +``context_length``, while the balance gate reserves on the context-based +``sats_pricing.max_cost``. When two providers serve the same model these can +disagree, so the "cheapest" advertised provider may demand a far larger +reservation and surprise the client with a 402. Ranking must therefore use the +same context-based ceiling as the gate. +""" + +import os +from unittest.mock import Mock + +import pytest + +os.environ["UPSTREAM_BASE_URL"] = "http://test" +os.environ["UPSTREAM_API_KEY"] = "test" + +from routstr.algorithm import ( # noqa: E402 + calculate_model_cost_score, + create_model_mappings, +) +from routstr.payment.helpers import get_max_cost_for_model # noqa: E402 +from routstr.payment.models import Architecture, Model, Pricing # noqa: E402 +from routstr.upstream.base import BaseUpstreamProvider # noqa: E402 + + +def _arch() -> Architecture: + return Architecture( + modality="text", + input_modalities=["text"], + output_modalities=["text"], + tokenizer="gpt", + instruct_type=None, + ) + + +def _model( + model_id: str, + prompt: float, + completion: float, + context_length: int, + max_cost_sats: float, +) -> Model: + """Build a Model whose sats_pricing.max_cost mirrors the context-based gate.""" + pricing = Pricing( + prompt=prompt, + completion=completion, + request=0.0, + image=0.0, + web_search=0.0, + internal_reasoning=0.0, + ) + model = Model( + id=model_id, + name=model_id, + created=1, + description="", + context_length=context_length, + architecture=_arch(), + pricing=pricing, + ) + model.sats_pricing = Pricing( + prompt=prompt, + completion=completion, + request=0.0, + image=0.0, + web_search=0.0, + internal_reasoning=0.0, + max_prompt_cost=context_length * prompt, + max_completion_cost=context_length * completion, + max_cost=max_cost_sats, + ) + return model + + +def _provider(name: str, db_id: int, models: list[Model]) -> Mock: + provider = Mock() + provider.provider_type = name + provider.base_url = f"https://{name}.example/v1" + provider.db_id = db_id + provider.upstream_name = name + provider.provider_fee = 1.0 + provider.get_cached_models.return_value = models + return provider + + +# Provider LOW-SCORE: cheaper per typical token, but a huge context window makes +# its context-based max_cost enormous. +_LOW_SCORE = _model( + "shared-model", + prompt=0.001, + completion=0.001, + context_length=1_000_000, + max_cost_sats=1000.0, +) +# Provider LOW-RESERVE: pricier per typical token, but a small context window +# makes its reservation ceiling tiny. +_LOW_RESERVE = _model( + "shared-model", + prompt=0.002, + completion=0.002, + context_length=8_000, + max_cost_sats=16.0, +) + + +def test_ranking_metric_and_reservation_metric_disagree() -> None: + """The two cost metrics rank the same two providers in opposite order.""" + assert calculate_model_cost_score(_LOW_SCORE) < calculate_model_cost_score( + _LOW_RESERVE + ) + assert _max_cost(_LOW_SCORE) > _max_cost(_LOW_RESERVE) + + +@pytest.mark.asyncio +async def test_get_max_cost_uses_context_based_max_cost_not_score() -> None: + """The balance gate reserves on max_cost, so the low-score model costs more.""" + session = Mock() + low_score_reserve = await get_max_cost_for_model( + "shared-model", session=session, model_obj=_LOW_SCORE + ) + low_reserve_reserve = await get_max_cost_for_model( + "shared-model", session=session, model_obj=_LOW_RESERVE + ) + # The "cheapest" model (by ranking score) demands the LARGER reservation. + assert low_score_reserve > low_reserve_reserve + + +def _max_cost(model: Model) -> float: + assert model.sats_pricing is not None + assert model.sats_pricing.max_cost is not None + return model.sats_pricing.max_cost + + +def _both_orderings() -> list[list[BaseUpstreamProvider]]: + low_score = _provider("low-score", 1, [_LOW_SCORE]) + low_reserve = _provider("low-reserve", 2, [_LOW_RESERVE]) + return [[low_score, low_reserve], [low_reserve, low_score]] + + +def test_catalog_advertises_the_lower_reservation_provider() -> None: + """Catalog and routing pick the provider with the smaller reservation, + regardless of upstream iteration order.""" + for providers in _both_orderings(): + _, provider_map, unique_models = create_model_mappings( + upstreams=providers, + overrides_by_key={}, + disabled_model_keys=set(), + ) + selected_model, selected_provider = provider_map["shared-model"][0] + assert selected_provider.provider_type == "low-reserve" + assert _max_cost(selected_model) == 16.0 + assert _max_cost(unique_models["shared-model"]) == 16.0 + + +def test_selected_candidate_has_minimal_reservation() -> None: + """The first-tried candidate never demands a larger reservation than + another available candidate for the same model.""" + for providers in _both_orderings(): + _, provider_map, _ = create_model_mappings( + upstreams=providers, + overrides_by_key={}, + disabled_model_keys=set(), + ) + candidates = provider_map["shared-model"] + selected_max_cost = _max_cost(candidates[0][0]) + min_max_cost = min(_max_cost(m) for m, _ in candidates) + assert selected_max_cost == min_max_cost