From f5ec383bee691ef26628dea96a20233d223d99bd Mon Sep 17 00:00:00 2001 From: thefux Date: Fri, 4 Sep 2026 07:30:49 +0000 Subject: [PATCH 1/5] chore: upgrade litellm to 1.84.10 - pin litellm>=1.84.0,<1.85 in pyproject.toml - uv override-dependencies for importlib-metadata>=8 (<9) and httpx>=0.28, required because cashu 0.20.3 pins importlib-metadata<7 and httpx<0.26 - litellm 1.84 now ships deepseek-v4 pricing natively in model_cost - full unit suite: 1339 passed on litellm 1.84.10 --- pyproject.toml | 7 +++++++ uv.lock | 37 ++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e9abb3cc..b471118e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,3 +87,10 @@ disallow_untyped_decorators = true [tool.uv.sources] routstr = { workspace = true } + +[tool.uv] +override-dependencies = [ + "litellm>=1.84.0,<1.85", + "importlib-metadata>=8.0.0,<9.0", + "httpx[socks]>=0.28.0", +] diff --git a/uv.lock b/uv.lock index 462fd619..ff3b5141 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,13 @@ resolution-markers = [ "python_full_version < '3.14'", ] +[manifest] +overrides = [ + { name = "httpx", extras = ["socks"], specifier = ">=0.28.0" }, + { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, + { name = "litellm", specifier = ">=1.84.0,<1.85" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -834,7 +841,7 @@ wheels = [ standard = [ { name = "email-validator" }, { name = "fastapi-cli", extra = ["standard"] }, - { name = "httpx" }, + { name = "httpx", extra = ["socks"] }, { name = "jinja2" }, { name = "python-multipart" }, { name = "uvicorn", extra = ["standard"] }, @@ -1234,18 +1241,17 @@ wheels = [ [[package]] name = "httpx" -version = "0.25.2" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "certifi" }, { name = "httpcore" }, { name = "idna" }, - { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8c/23/911d93a022979d3ea295f659fbe7edb07b3f4561a477e83b3a6d0e0c914e/httpx-0.25.2.tar.gz", hash = "sha256:8b8fcaa0c8ea7b05edd69a094e63a2094c4efcb48129fb757361bc423c0ad9e8", size = 123889, upload-time = "2023-11-24T12:36:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/65/6940eeb21dcb2953778a6895281c179efd9100463ff08cb6232bb6480da7/httpx-0.25.2-py3-none-any.whl", hash = "sha256:a05d3d052d9b2dfce0e3896636467f8a5342fb2b902c819428e1ac65413ca118", size = 74980, upload-time = "2023-11-24T12:36:31.403Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [package.optional-dependencies] @@ -1261,7 +1267,7 @@ dependencies = [ { name = "filelock" }, { name = "fsspec" }, { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx" }, + { name = "httpx", extra = ["socks"] }, { name = "packaging" }, { name = "pyyaml" }, { name = "tqdm" }, @@ -1284,14 +1290,14 @@ wheels = [ [[package]] name = "importlib-metadata" -version = "6.11.0" +version = "8.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/eb/58c2ab27ee628ad801f56d4017fe62afab0293116f6d0b08f1d5bd46e06f/importlib_metadata-6.11.0.tar.gz", hash = "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443", size = 54593, upload-time = "2023-12-03T17:33:10.693Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/72/c600ae4f68c28fc19f9c31b9403053e5dbb8cace2e6842c7b7c3e4d42fe9/importlib_metadata-8.9.0.tar.gz", hash = "sha256:58850626cef4bd2df100378b0f2aea9724a7b92f10770d547725b047078f99ee", size = 56140, upload-time = "2026-03-20T16:56:26.362Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/9b/ecce94952ab5ea74c31dcf9ccf78ccd484eebebef06019bf8cb579ab4519/importlib_metadata-6.11.0-py3-none-any.whl", hash = "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b", size = 23427, upload-time = "2023-12-03T17:33:08.965Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, ] [[package]] @@ -1418,13 +1424,13 @@ wheels = [ [[package]] name = "litellm" -version = "1.83.0" +version = "1.84.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "click" }, { name = "fastuuid" }, - { name = "httpx" }, + { name = "httpx", extra = ["socks"] }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -1434,9 +1440,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/22/92/6ce9737554994ca8e536e5f4f6a87cc7c4774b656c9eb9add071caf7d54b/litellm-1.83.0.tar.gz", hash = "sha256:860bebc76c4bb27b4cf90b4a77acd66dba25aced37e3db98750de8a1766bfb7a", size = 17333062, upload-time = "2026-03-31T05:08:25.331Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c4/512c8cb204450b585bb7bee2cef9466c8b79b90cf774766f319de5c444ed/litellm-1.84.10.tar.gz", hash = "sha256:5ccb6aec803c35f463a7ea1a446030fe99f7c556388b435dc2fb7ad91aa48a24", size = 15123874, upload-time = "2026-06-24T03:57:19.791Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/2c/a670cc050fcd6f45c6199eb99e259c73aea92edba8d5c2fc1b3686d36217/litellm-1.83.0-py3-none-any.whl", hash = "sha256:88c536d339248f3987571493015784671ba3f193a328e1ea6780dbebaa2094a8", size = 15610306, upload-time = "2026-03-31T05:08:21.987Z" }, + { url = "https://files.pythonhosted.org/packages/5b/88/e45bcdefc7a85bbef8eb852111dd4e02b92ea25727b6009c78893a768deb/litellm-1.84.10-py3-none-any.whl", hash = "sha256:7e175ebec04aa92149794adc83e4dd82b60d2b833c1ec265d68c08e8f56edde5", size = 16753091, upload-time = "2026-06-24T03:57:16.759Z" }, ] [[package]] @@ -1718,7 +1724,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "distro" }, - { name = "httpx" }, + { name = "httpx", extra = ["socks"] }, { name = "jiter" }, { name = "pydantic" }, { name = "sniffio" }, @@ -2459,7 +2465,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "aiohttp" }, - { name = "httpx" }, + { name = "httpx", extra = ["socks"] }, { name = "mypy" }, { name = "openai" }, { name = "psutil" }, @@ -2722,6 +2728,7 @@ version = "2.0.42" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "(python_full_version < '3.14' and platform_machine == 'AMD64') or (python_full_version < '3.14' and platform_machine == 'WIN32') or (python_full_version < '3.14' and platform_machine == 'aarch64') or (python_full_version < '3.14' and platform_machine == 'amd64') or (python_full_version < '3.14' and platform_machine == 'ppc64le') or (python_full_version < '3.14' and platform_machine == 'win32') or (python_full_version < '3.14' and platform_machine == 'x86_64')" }, + { name = "importlib-metadata" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/03/a0af991e3a43174d6b83fca4fb399745abceddd1171bdabae48ce877ff47/sqlalchemy-2.0.42.tar.gz", hash = "sha256:160bedd8a5c28765bd5be4dec2d881e109e33b34922e50a3b881a7681773ac5f", size = 9749972, upload-time = "2025-07-29T12:48:09.323Z" } From 8ec5d409d520ee9fa5455b8b7a1da5c9009592d2 Mon Sep 17 00:00:00 2001 From: thefux Date: Tue, 29 Sep 2026 08:08:59 +0000 Subject: [PATCH 2/5] fix: stop reporting OpenRouter provider as unknown on stream and envelope payloads The OpenRouter stamper wrote "unknown" whenever a payload lacked a top-level provider. That hit every Anthropic /messages event, every Responses event, and the usage/cost payloads routstr synthesizes at the end of a stream. - Read the provider from the Anthropic `message` and Responses `response` envelopes as well as the top level. - Carry the provider reported earlier in a stream to later events and to the synthesized usage/cost payloads. --- routstr/upstream/base.py | 55 +++++++++++++++++---- routstr/upstream/generic.py | 5 +- routstr/upstream/openrouter.py | 5 +- tests/unit/test_provider_field_injection.py | 27 ++++++++++ tests/unit/test_x_cashu_provider_path.py | 47 ++++++++++++++++++ 5 files changed, 124 insertions(+), 15 deletions(-) diff --git a/routstr/upstream/base.py b/routstr/upstream/base.py index 5b0e5c0f..8f46d90b 100644 --- a/routstr/upstream/base.py +++ b/routstr/upstream/base.py @@ -214,6 +214,20 @@ def _responses_usage_payload(data_json: dict) -> dict: return nested if isinstance(nested, dict) else data_json +def _reported_provider(payload: dict) -> str | None: + """Provider named by an upstream payload, if any. + + Checked at top level first, then inside the Anthropic ``message`` and + Responses ``response`` envelopes, which is where those dialects nest it. + """ + for obj in (payload, payload.get("message"), payload.get("response")): + if isinstance(obj, dict): + value = obj.get("provider") + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def _render_sse_event(field_lines: list[str], data: str) -> str: """Re-frame one parsed event, re-prefixing every line of a multi-line data.""" body = "".join(f"{line}\n" for line in field_lines) @@ -485,8 +499,7 @@ class BaseUpstreamProvider: return response_json["provider_url"] = public_provider_url(self.base_url) provider_type = (self.provider_type or "").strip() - existing = response_json.get("provider") - existing_str = existing.strip() if isinstance(existing, str) else "" + existing_str = _reported_provider(response_json) or "" if not existing_str: response_json["provider"] = provider_type return @@ -498,6 +511,17 @@ class BaseUpstreamProvider: return response_json["provider"] = f"{provider_type}:{existing_str}" + def _stamp_streamed_provider( + self, payload: dict, carried: str | None + ) -> str | None: + """Stamp a streamed payload, falling back to a provider an earlier event + reported. Returns the provider to carry forward to later payloads.""" + reported = _reported_provider(payload) + if reported is None and carried is not None: + payload["provider"] = carried + self._apply_provider_field(payload) + return reported or carried + def _log_full_refund( self, *, @@ -1169,6 +1193,7 @@ class BaseUpstreamProvider: usage_finalized = False last_model_seen: str | None = None + provider_seen: str | None = None async def finalize_db_only() -> None: nonlocal usage_finalized @@ -1243,6 +1268,7 @@ class BaseUpstreamProvider: end of stream. """ nonlocal last_model_seen, usage_chunk_data, done_seen, stream_id + nonlocal provider_seen event = raw_event.strip(b"\r\n") if not event: @@ -1282,7 +1308,7 @@ class BaseUpstreamProvider: if isinstance(obj, dict): usage_estimator.observe(obj) - self._apply_provider_field(obj) + provider_seen = self._stamp_streamed_provider(obj, provider_seen) if obj.get("model"): last_model_seen = str(obj.get("model")) if requested_model: @@ -1408,6 +1434,7 @@ class BaseUpstreamProvider: if legacy_completion else "chat.completion.chunk", "model": last_model_seen or "unknown", + "provider": provider_seen, "choices": [], "usage": { "prompt_tokens": cost_data.get("input_tokens", 0), @@ -1652,6 +1679,7 @@ class BaseUpstreamProvider: usage_finalized = False last_model_seen: str | None = None + provider_seen: str | None = None async def finalize_db_only() -> None: nonlocal usage_finalized @@ -1715,7 +1743,7 @@ class BaseUpstreamProvider: and preserves ``event:``/``id:`` fields attached to their data line so Responses API event framing stays intact. """ - nonlocal last_model_seen, usage_chunk_data, done_seen + nonlocal last_model_seen, usage_chunk_data, done_seen, provider_seen nonlocal reasoning_tokens event = raw_event.strip(b"\r\n") @@ -1751,7 +1779,7 @@ class BaseUpstreamProvider: obj = json_codec.loads(data) if isinstance(obj, dict): - self._apply_provider_field(obj) + provider_seen = self._stamp_streamed_provider(obj, provider_seen) if obj.get("model"): last_model_seen = str(obj.get("model")) if requested_model: @@ -1840,6 +1868,7 @@ class BaseUpstreamProvider: if usage_chunk_data is None: usage_chunk_data = { "type": "response.completed", + "provider": provider_seen, "response": { "model": last_model_seen or "unknown", "usage": { @@ -2195,6 +2224,7 @@ class BaseUpstreamProvider: usage_estimator = MissingUsageEstimator(request_body, model_obj) usage_finalized = False last_model_seen: str | None = None + provider_seen: str | None = None async def finalize_without_usage() -> bytes | None: nonlocal usage_finalized @@ -2244,7 +2274,7 @@ class BaseUpstreamProvider: async def stream_with_cost( max_cost_for_model: int, ) -> AsyncGenerator[bytes, None]: - nonlocal usage_finalized, last_model_seen + nonlocal usage_finalized, last_model_seen, provider_seen stored_chunks: list[bytes] = [] input_tokens: int = 0 output_tokens: int = 0 @@ -2301,7 +2331,9 @@ class BaseUpstreamProvider: last_model_seen = str(msg.get("model")) provider_added = "provider" not in data - self._apply_provider_field(data) + provider_seen = self._stamp_streamed_provider( + data, provider_seen + ) if requested_model: # Apply requested_model override @@ -2419,6 +2451,7 @@ class BaseUpstreamProvider: try: combined_data = { "model": last_model_seen or "unknown", + "provider": provider_seen, "usage": usage_data, } cost_data = await adjust_payment_for_tokens( @@ -4197,6 +4230,7 @@ class BaseUpstreamProvider: }, ) + provider_seen: str | None = None for i, line in enumerate(lines): if line.startswith("data: "): try: @@ -4204,7 +4238,9 @@ class BaseUpstreamProvider: if not isinstance(data_json, dict): continue provider_before = data_json.get("provider") - self._apply_provider_field(data_json) + provider_seen = self._stamp_streamed_provider( + data_json, provider_seen + ) changed = data_json.get("provider") != provider_before if cost_data and "usage" in data_json and data_json["usage"]: _inject_cost_into_usage(data_json, cost_data) @@ -5265,6 +5301,7 @@ class BaseUpstreamProvider: }, ) + provider_seen: str | None = None for i, (fields, data) in enumerate(events): if data.strip() == "[DONE]": continue @@ -5275,7 +5312,7 @@ class BaseUpstreamProvider: if not isinstance(data_json, dict): continue provider_before = data_json.get("provider") - self._apply_provider_field(data_json) + provider_seen = self._stamp_streamed_provider(data_json, provider_seen) changed = data_json.get("provider") != provider_before payload = _responses_usage_payload(data_json) if cost_data and isinstance(payload.get("usage"), dict): diff --git a/routstr/upstream/generic.py b/routstr/upstream/generic.py index 03bfa015..c9edf109 100644 --- a/routstr/upstream/generic.py +++ b/routstr/upstream/generic.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx -from .base import BaseUpstreamProvider +from .base import BaseUpstreamProvider, _reported_provider from .model_paths import public_provider_url from .pricing_resolver import ( FallbackPricingResolver, @@ -60,8 +60,7 @@ class GenericUpstreamProvider(BaseUpstreamProvider): """ if not isinstance(response_json, dict): return - existing = response_json.get("provider") - if not (isinstance(existing, str) and existing.strip()): + if _reported_provider(response_json) is None: response_json["provider"] = ( urlparse(public_provider_url(self.base_url)).hostname or self.upstream_name diff --git a/routstr/upstream/openrouter.py b/routstr/upstream/openrouter.py index 9ca190ce..34394335 100644 --- a/routstr/upstream/openrouter.py +++ b/routstr/upstream/openrouter.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING import httpx from ..payment.models import Model, async_fetch_openrouter_models -from .base import BaseUpstreamProvider +from .base import BaseUpstreamProvider, _reported_provider from .model_paths import public_provider_url if TYPE_CHECKING: @@ -35,8 +35,7 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider): return response_json["provider_url"] = public_provider_url(self.base_url) provider_type = (self.provider_type or "").strip() - existing = response_json.get("provider") - sub = existing.strip() if isinstance(existing, str) else "" + sub = _reported_provider(response_json) or "" # Strip any already-applied "openrouter:" prefixes (idempotency). prefix = f"{provider_type}:" while sub.lower().startswith(prefix.lower()): diff --git a/tests/unit/test_provider_field_injection.py b/tests/unit/test_provider_field_injection.py index 86942ae6..d620e054 100644 --- a/tests/unit/test_provider_field_injection.py +++ b/tests/unit/test_provider_field_injection.py @@ -89,6 +89,33 @@ def test_apply_provider_field_non_string_upstream_treated_as_missing() -> None: assert data["provider"] == "unknown" +def test_apply_provider_field_openrouter_reads_nested_envelopes() -> None: + """Anthropic ``message`` and Responses ``response`` envelopes nest the + upstream provider; it must not be reported as unknown.""" + p = _make_provider(OpenRouterUpstreamProvider, "openrouter") + message_start: dict = { + "type": "message_start", + "message": {"provider": "Anthropic"}, + } + p._apply_provider_field(message_start) + assert message_start["provider"] == "openrouter:Anthropic" + + created: dict = {"type": "response.created", "response": {"provider": "OpenAI"}} + p._apply_provider_field(created) + assert created["provider"] == "openrouter:OpenAI" + + +def test_stamp_streamed_provider_carries_earlier_provider() -> None: + """Events without their own provider inherit the one reported earlier in + the stream instead of becoming ``unknown``.""" + p = _make_provider(OpenRouterUpstreamProvider, "openrouter") + first: dict = {"provider": "Fireworks"} + carried = p._stamp_streamed_provider(first, None) + delta: dict = {"type": "content_block_delta"} + assert p._stamp_streamed_provider(delta, carried) == "Fireworks" + assert first["provider"] == delta["provider"] == "openrouter:Fireworks" + + def test_apply_provider_field_idempotent_for_direct_upstream() -> None: """Calling twice on a direct upstream payload keeps the same value and never nests the prefix (no ``anthropic:anthropic``).""" diff --git a/tests/unit/test_x_cashu_provider_path.py b/tests/unit/test_x_cashu_provider_path.py index 11b47d6f..6b4fdec2 100644 --- a/tests/unit/test_x_cashu_provider_path.py +++ b/tests/unit/test_x_cashu_provider_path.py @@ -52,3 +52,50 @@ async def test_x_cashu_responses_stream_reports_complete_provider_path() -> None payload = json.loads((await _body(response)).decode().removeprefix("data: ")) assert payload["provider"] == "openrouter:z.ai" + + +@pytest.mark.asyncio +async def test_x_cashu_messages_stream_carries_provider_to_later_events() -> None: + provider = OpenRouterUpstreamProvider(api_key="test-key") + events = [ + {"type": "message_start", "message": {"provider": "Anthropic"}}, + {"type": "content_block_delta", "delta": {"text": "hi"}}, + ] + content = "".join(f"data: {json.dumps(e)}\n" for e in events) + + response = await provider.handle_x_cashu_streaming_response( + content, + httpx.Response(200, headers={"content-type": "text/event-stream"}), + amount=1, + unit="sat", + max_cost_for_model=1, + ) + + lines = (await _body(response)).decode().splitlines() + stamped = [json.loads(line.removeprefix("data: ")) for line in lines if line] + assert [e["provider"] for e in stamped] == ["openrouter:Anthropic"] * 2 + + +@pytest.mark.asyncio +async def test_x_cashu_responses_stream_carries_nested_provider() -> None: + provider = OpenRouterUpstreamProvider(api_key="test-key") + events = [ + {"type": "response.created", "response": {"provider": "OpenAI"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + ] + content = "".join(f"data: {json.dumps(e)}\n\n" for e in events) + + with patch.object( + provider, "get_x_cashu_cost", new=AsyncMock(return_value=None) + ): + response = await provider.handle_x_cashu_streaming_responses_response( + content, + httpx.Response(200, headers={"content-type": "text/event-stream"}), + amount=1, + unit="sat", + max_cost_for_model=1, + ) + + lines = (await _body(response)).decode().splitlines() + stamped = [json.loads(line.removeprefix("data: ")) for line in lines if line] + assert [e["provider"] for e in stamped] == ["openrouter:OpenAI"] * 2 From 3a601ee00e887e5d7771d5a78d3ab9dce4d253ec Mon Sep 17 00:00:00 2001 From: thefux Date: Tue, 29 Sep 2026 08:23:20 +0000 Subject: [PATCH 3/5] fix: send placeholder key to litellm for keyless upstreams on /v1/messages LiteLLM treats a blank api_key as missing and falls back to OPENAI_API_KEY, so /v1/messages against a keyless OpenAI-compatible upstream failed with "The api_key client option must be set", while /v1/chat/completions on the same upstream worked (it omits auth). LiteLLM has no option to skip the key, so pass a placeholder instead. --- routstr/upstream/messages_dispatch.py | 8 +++++- tests/unit/test_messages_litellm_dispatch.py | 28 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/routstr/upstream/messages_dispatch.py b/routstr/upstream/messages_dispatch.py index 488129a8..4ba0c720 100644 --- a/routstr/upstream/messages_dispatch.py +++ b/routstr/upstream/messages_dispatch.py @@ -36,6 +36,12 @@ from .reasoning_effort import adapt_messages_body_for_litellm logger = get_logger(__name__) +# Sent in place of a blank upstream key. LiteLLM treats ``""`` as missing and +# falls back to the provider's env var (e.g. ``OPENAI_API_KEY``), failing with +# an AuthenticationError for keyless upstreams such as self-hosted +# OpenAI-compatible servers, which the chat path reaches without auth. +KEYLESS_UPSTREAM_API_KEY = "no-key" + # Anthropic-Messages-only fields that don't translate to OpenAI # Chat Completions. ``litellm.drop_params`` only filters *known* # unsupported params; these newer/extension fields get passed through @@ -519,7 +525,7 @@ async def dispatch_anthropic_messages( kwargs: dict = { "model": litellm_model, "api_base": base_url, - "api_key": api_key, + "api_key": api_key or KEYLESS_UPSTREAM_API_KEY, "stream": upstream_stream, **body, } diff --git a/tests/unit/test_messages_litellm_dispatch.py b/tests/unit/test_messages_litellm_dispatch.py index c9b13bd2..7e2da476 100644 --- a/tests/unit/test_messages_litellm_dispatch.py +++ b/tests/unit/test_messages_litellm_dispatch.py @@ -1755,3 +1755,31 @@ async def test_x_cashu_zero_value_rejected_not_forwarded( assert body["error"]["code"] == "cashu_token_zero_value" # Spent-to-zero token must not be echoed back for retry. assert "X-Cashu" not in response.headers + + +@pytest.mark.asyncio +async def test_dispatch_passes_placeholder_key_for_keyless_upstream() -> None: + """A blank upstream key must not reach litellm, which would fall back to + OPENAI_API_KEY and fail with an AuthenticationError.""" + provider = BaseUpstreamProvider(base_url="http://localhost:8000/v1", api_key="") + captured_kwargs: dict[str, Any] = {} + + async def fake_acreate(**kwargs: Any) -> AsyncIterator[dict]: + captured_kwargs.update(kwargs) + + async def no_events() -> AsyncIterator[dict]: + return + yield + + return no_events() + + with patch( + "litellm.anthropic.messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ): + await provider._dispatch_anthropic_messages( + request_body=_anthropic_request_body(stream=True), + model_obj=_make_model(), + ) + + assert captured_kwargs["api_key"] == "no-key" From d58e7b9ab04e535dc0a929b0121f92f28e88a0ed Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:48:19 +0800 Subject: [PATCH 4/5] fix: reject unsupported web search before LiteLLM translation --- routstr/upstream/messages_dispatch.py | 26 +++++++++++++++++++++++++ tests/unit/test_venice_web_search.py | 28 ++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/routstr/upstream/messages_dispatch.py b/routstr/upstream/messages_dispatch.py index 488129a8..e7429150 100644 --- a/routstr/upstream/messages_dispatch.py +++ b/routstr/upstream/messages_dispatch.py @@ -507,6 +507,32 @@ async def dispatch_anthropic_messages( model_suffix = adapt_request(body) if adapt_request else "" + # LiteLLM turns Anthropic's server-side web_search tool into the OpenAI + # `web_search_options` parameter. Generic OpenAI-compatible chat endpoints + # (including those serving Claude through a proxy) may reject that field. + # Only a provider with an explicit adaptation (e.g. Venice's model suffix) + # can preserve search semantics; do not silently remove the tool and return + # an answer that never searched. Native /v1/messages providers bypass this + # dispatcher and receive the original tool unchanged. + tools = body.get("tools") + if provider_prefix == "openai/" and isinstance(tools, list) and any( + isinstance(tool, dict) + and ( + ( + isinstance(tool.get("type"), str) + and tool["type"].startswith("web_search") + ) + or tool.get("name") == "web_search" + ) + for tool in tools + ): + raise UpstreamError( + "This upstream does not support Anthropic web search through " + "OpenAI-compatible /v1/messages translation", + status_code=400, + code="UNSUPPORTED_WEB_SEARCH", + ) + # Convention: `model.id` is the canonical upstream model name; # `forwarded_model_id` is the public alias the internal API exposes # and echoes back to the client. diff --git a/tests/unit/test_venice_web_search.py b/tests/unit/test_venice_web_search.py index a836e956..380ccc33 100644 --- a/tests/unit/test_venice_web_search.py +++ b/tests/unit/test_venice_web_search.py @@ -123,13 +123,35 @@ async def test_requests_without_web_search_are_untouched() -> None: @pytest.mark.asyncio -async def test_other_providers_keep_their_existing_behaviour() -> None: - """The base hook is a no-op, so no non-Venice upstream changes shape.""" +async def test_generic_openai_upstream_rejects_untranslatable_web_search() -> None: + """Do not let LiteLLM send unsupported web_search_options to a generic API.""" provider = BaseUpstreamProvider(base_url="http://test", api_key="k") + with pytest.raises(UpstreamError) as excinfo: + await _dispatch(provider, _body(tools=[WEB_SEARCH_TOOL, FUNCTION_TOOL])) + + assert excinfo.value.status_code == 400 + assert excinfo.value.code == "UNSUPPORTED_WEB_SEARCH" + + +@pytest.mark.asyncio +async def test_generic_openai_upstream_still_accepts_function_tools() -> None: + provider = BaseUpstreamProvider(base_url="http://test", api_key="k") + + kwargs = await _dispatch(provider, _body(tools=[FUNCTION_TOOL])) + + assert kwargs["model"] == "openai/deepseek-v4-flash-0731" + assert kwargs["tools"] == [FUNCTION_TOOL] + + +@pytest.mark.asyncio +async def test_non_openai_adapter_can_still_handle_search_tool() -> None: + provider = BaseUpstreamProvider( + base_url="https://openrouter.ai/api/v1", api_key="k" + ) + kwargs = await _dispatch(provider, _body(tools=[WEB_SEARCH_TOOL])) - assert kwargs["model"] == "openai/deepseek-v4-flash-0731" assert kwargs["tools"] == [WEB_SEARCH_TOOL] From 785fbc519007ec2c43af121316cc6aa0f4d2b034 Mon Sep 17 00:00:00 2001 From: thefux Date: Tue, 29 Sep 2026 09:43:47 +0000 Subject: [PATCH 5/5] fix: report openrouter:unknown when OpenRouter omits the serving provider --- routstr/upstream/openrouter.py | 33 +++++++++++++++- tests/unit/test_provider_field_injection.py | 42 ++++++++++++++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/routstr/upstream/openrouter.py b/routstr/upstream/openrouter.py index 34394335..3ff13d90 100644 --- a/routstr/upstream/openrouter.py +++ b/routstr/upstream/openrouter.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING import httpx +from ..core.logging import get_logger from ..payment.models import Model, async_fetch_openrouter_models from .base import BaseUpstreamProvider, _reported_provider from .model_paths import public_provider_url @@ -9,6 +10,19 @@ from .model_paths import public_provider_url if TYPE_CHECKING: from ..core.db import UpstreamProviderRow +logger = get_logger(__name__) + +_UNKNOWN_SUB_PROVIDER = "unknown" + + +def _carries_usage(payload: dict) -> bool: + """Whether a payload holds usage, at top level or in the Anthropic + ``message`` / Responses ``response`` envelope.""" + return any( + isinstance(obj, dict) and isinstance(obj.get("usage"), dict) + for obj in (payload, payload.get("message"), payload.get("response")) + ) + class OpenRouterUpstreamProvider(BaseUpstreamProvider): """Upstream provider specifically configured for OpenRouter API.""" @@ -27,7 +41,8 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider): - Real upstream sub-provider (e.g. ``"GMICloud"``) -> ``"openrouter:GMICloud"``. - Missing sub-provider, or one that merely echoes ``"openrouter"`` -> - ``"unknown"``. + ``"openrouter:unknown"``: the router is still known even when the + serving provider is not (e.g. the Responses API never reports it). - Idempotent: re-stamping never produces ``"openrouter:openrouter:..."``; the ``openrouter:`` prefix appears at most once. """ @@ -40,9 +55,23 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider): prefix = f"{provider_type}:" while sub.lower().startswith(prefix.lower()): sub = sub[len(prefix) :].strip() + # Already stamped as unknown on an earlier pass; keep it without + # warning again. + if sub.lower() == _UNKNOWN_SUB_PROVIDER: + response_json["provider"] = f"{provider_type}:{_UNKNOWN_SUB_PROVIDER}" + return # No real sub-provider, or it just echoes our own router name. if not sub or sub.lower() == provider_type.lower(): - response_json["provider"] = "unknown" + # Warn only on the billed payload, not on every stream chunk. + if _carries_usage(response_json): + logger.warning( + "OpenRouter did not report the serving provider", + extra={ + "model": response_json.get("model"), + "response_id": response_json.get("id"), + }, + ) + response_json["provider"] = f"{provider_type}:{_UNKNOWN_SUB_PROVIDER}" return response_json["provider"] = f"{provider_type}:{sub}" diff --git a/tests/unit/test_provider_field_injection.py b/tests/unit/test_provider_field_injection.py index d620e054..6caea063 100644 --- a/tests/unit/test_provider_field_injection.py +++ b/tests/unit/test_provider_field_injection.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from routstr.upstream.anthropic import AnthropicUpstreamProvider from routstr.upstream.base import BaseUpstreamProvider from routstr.upstream.generic import GenericUpstreamProvider @@ -33,12 +35,12 @@ def test_apply_provider_field_openrouter_passthrough() -> None: def test_apply_provider_field_openrouter_no_upstream_provider() -> None: - """If OpenRouter omits the provider field, the real serving provider is - unknown — a bare ``openrouter`` value carries no information.""" + """If OpenRouter omits the provider field, the serving provider is + unknown but the router is not.""" p = _make_provider(OpenRouterUpstreamProvider, "openrouter") data: dict = {"id": "gen-abc"} p._apply_provider_field(data) - assert data["provider"] == "unknown" + assert data["provider"] == "openrouter:unknown" def test_apply_provider_field_openrouter_echoes_router_name() -> None: @@ -46,7 +48,35 @@ def test_apply_provider_field_openrouter_echoes_router_name() -> None: p = _make_provider(OpenRouterUpstreamProvider, "openrouter") data: dict = {"provider": "openrouter"} p._apply_provider_field(data) - assert data["provider"] == "unknown" + assert data["provider"] == "openrouter:unknown" + + +def test_apply_provider_field_openrouter_unknown_is_idempotent() -> None: + """Re-stamping an unknown payload (e.g. in inject_cost_metadata) keeps + ``openrouter:unknown`` instead of reading ``unknown`` as a sub-provider.""" + p = _make_provider(OpenRouterUpstreamProvider, "openrouter") + data: dict = {"id": "gen-abc"} + p._apply_provider_field(data) + p._apply_provider_field(data) + assert data["provider"] == "openrouter:unknown" + + +def test_apply_provider_field_openrouter_warns_once_on_billed_payload() -> None: + """A missing provider is logged on the payload carrying usage, not on + every stream chunk or on a re-stamp.""" + p = _make_provider(OpenRouterUpstreamProvider, "openrouter") + chunk: dict = {"type": "response.output_text.delta", "delta": "hi"} + completed: dict = { + "type": "response.completed", + "response": {"id": "gen-abc", "usage": {"input_tokens": 1}}, + } + with patch("routstr.upstream.openrouter.logger.warning") as warning: + p._apply_provider_field(chunk) + p._apply_provider_field(completed) + p._apply_provider_field(completed) + + warning.assert_called_once() + assert chunk["provider"] == completed["provider"] == "openrouter:unknown" def test_apply_provider_field_openrouter_idempotent_no_double_prefix() -> None: @@ -79,14 +109,14 @@ def test_apply_provider_field_blank_upstream_treated_as_missing() -> None: p = _make_provider(OpenRouterUpstreamProvider, "openrouter") data: dict = {"provider": " "} p._apply_provider_field(data) - assert data["provider"] == "unknown" + assert data["provider"] == "openrouter:unknown" def test_apply_provider_field_non_string_upstream_treated_as_missing() -> None: p = _make_provider(OpenRouterUpstreamProvider, "openrouter") data: dict = {"provider": 42} p._apply_provider_field(data) - assert data["provider"] == "unknown" + assert data["provider"] == "openrouter:unknown" def test_apply_provider_field_openrouter_reads_nested_envelopes() -> None: