From 5d1004d3ce3cf13264993c1d830bd9014dbc9fd9 Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Fri, 25 Sep 2026 00:48:02 +0200 Subject: [PATCH] feat: translate anthropic web search to venice search on /v1/messages --- routstr/upstream/base.py | 11 ++ routstr/upstream/messages_dispatch.py | 10 +- routstr/upstream/venice.py | 89 +++++++++++ tests/unit/test_venice_web_search.py | 221 ++++++++++++++++++++++++++ 4 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_venice_web_search.py diff --git a/routstr/upstream/base.py b/routstr/upstream/base.py index bc7244e1..3f9d7bde 100644 --- a/routstr/upstream/base.py +++ b/routstr/upstream/base.py @@ -2506,6 +2506,16 @@ class BaseUpstreamProvider: ) -> dict: return await messages_dispatch.aggregate_anthropic_events_to_message(iterator) + def adapt_messages_request(self, body: dict, model_obj: Model) -> str: + """Rewrite an allowlisted /v1/messages body for this upstream. + + Returns a suffix appended to the upstream model name, empty when the + provider needs none. Subclasses override this to express an Anthropic + feature the upstream spells differently; the base forwards the body + untouched. + """ + return "" + async def _dispatch_anthropic_messages( self, request_body: bytes | None, @@ -2520,6 +2530,7 @@ class BaseUpstreamProvider: api_key=self.api_key, provider_prefix=self.get_litellm_provider_prefix(), transform_model_name=self.transform_model_name, + adapt_request=lambda body: self.adapt_messages_request(body, model_obj), log_extra=log_extra, ) diff --git a/routstr/upstream/messages_dispatch.py b/routstr/upstream/messages_dispatch.py index 2856256c..488129a8 100644 --- a/routstr/upstream/messages_dispatch.py +++ b/routstr/upstream/messages_dispatch.py @@ -458,6 +458,7 @@ async def dispatch_anthropic_messages( api_key: str, provider_prefix: str, transform_model_name: Callable[[str], str], + adapt_request: Callable[[dict], str] | None = None, log_extra: dict[str, Any] | None = None, ) -> tuple[bool, Any, str | None]: """Call ``litellm.anthropic.messages.acreate`` and return @@ -465,6 +466,11 @@ async def dispatch_anthropic_messages( Shared by the bearer-key and x-cashu paths. Raises :class:`UpstreamError` on bad input or upstream failure. + + ``adapt_request`` is the provider's last word on the allowlisted body: it + may rewrite it in place and returns a suffix for the upstream model name, + which is how a provider expresses a feature litellm would otherwise + translate into a parameter the upstream rejects. """ if not request_body: raise UpstreamError("Missing request body for /v1/messages", status_code=400) @@ -499,13 +505,15 @@ async def dispatch_anthropic_messages( ) body = {k: v for k, v in body.items() if k in ALLOWED_MESSAGES_REQUEST_FIELDS} + model_suffix = adapt_request(body) if adapt_request else "" + # Convention: `model.id` is the canonical upstream model name; # `forwarded_model_id` is the public alias the internal API exposes # and echoes back to the client. requested_model = ( (model_obj.forwarded_model_id or model_obj.id) if model_obj else None ) - upstream_model = transform_model_name(model_obj.id) + upstream_model = f"{transform_model_name(model_obj.id)}{model_suffix}" litellm_model = f"{provider_prefix}{upstream_model}" kwargs: dict = { diff --git a/routstr/upstream/venice.py b/routstr/upstream/venice.py index 5949c205..30404335 100644 --- a/routstr/upstream/venice.py +++ b/routstr/upstream/venice.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any import httpx +from ..core.exceptions import UpstreamError from ..core.logging import get_logger from ..payment.models import Architecture, Model, Pricing, TopProvider from .base import BaseUpstreamProvider @@ -30,6 +31,37 @@ _ARCHITECTURES: dict[str, tuple[str, list[str], list[str]]] = { "embedding": ("text->embedding", ["text"], ["embedding"]), } +# Venice runs search itself and reports it back through ``venice_parameters``; +# it has no Anthropic-shaped server tool and rejects the ``web_search_options`` +# that litellm's Anthropic adapter derives from one. ``auto`` matches Anthropic +# semantics, where declaring the tool leaves the decision to the model. +# Citations are asked for because litellm's Anthropic response translation +# carries no ``venice_parameters``, so inline ``[REF]n[/REF]`` markers in the +# text are the only way a caller sees which sources were used. +_WEB_SEARCH_SUFFIX = ":enable_web_search=auto&enable_web_citations=true" + +# Anthropic web-search constraints with no Venice equivalent. Honouring the +# request means enforcing them, so a request that sets one is refused rather +# than answered by a search that ignored it. +_UNENFORCEABLE_WEB_SEARCH_KEYS = frozenset( + {"max_uses", "allowed_domains", "blocked_domains", "user_location"} +) + + +def _is_web_search_tool(tool: Any) -> bool: + """An Anthropic server-side web-search tool, by either of its markers. + + Matches litellm's own detection (``litellm/llms/anthropic/ + experimental_pass_through/adapters/transformation.py``), so every tool it + would turn into ``web_search_options`` is caught here first. + """ + if not isinstance(tool, dict): + return False + tool_type = tool.get("type") + return ( + isinstance(tool_type, str) and tool_type.startswith("web_search") + ) or tool.get("name") == "web_search" + def _usd(entry: Any) -> float | None: """Read the USD leg of a Venice ``{usd, diem}`` price pair.""" @@ -79,6 +111,63 @@ class VeniceUpstreamProvider(BaseUpstreamProvider): def transform_model_name(self, model_id: str) -> str: return model_id.removeprefix("venice/") + def adapt_messages_request(self, body: dict, model_obj: Model) -> str: + """Trade an Anthropic web-search tool for Venice's own search switch. + + Left in the body, litellm's Anthropic adapter rewrites the tool into a + top-level ``web_search_options``, which Venice answers with a 400. The + tool is lifted out here and the same intent re-expressed as a model + feature suffix, the one form of ``venice_parameters`` that survives + that adapter. + """ + tools = body.get("tools") + if not isinstance(tools, list): + return "" + search_tools = [tool for tool in tools if _is_web_search_tool(tool)] + if not search_tools: + return "" + + # A key carrying null or an empty list states no constraint, so it is + # read as absent rather than refused. + unenforceable = sorted( + { + key + for tool in search_tools + for key, value in tool.items() + if key in _UNENFORCEABLE_WEB_SEARCH_KEYS + and value is not None + and value != [] + } + ) + if unenforceable: + raise UpstreamError( + "Venice web search cannot honour these Anthropic web_search " + f"options: {', '.join(unenforceable)}", + status_code=400, + code="UNSUPPORTED_WEB_SEARCH_OPTION", + details={"unsupported_options": unenforceable}, + ) + + tool_choice = body.get("tool_choice") + if isinstance(tool_choice, dict) and tool_choice.get("name") == "web_search": + raise UpstreamError( + "Venice web search cannot be forced through tool_choice; it is " + "decided by the model", + status_code=400, + code="UNSUPPORTED_WEB_SEARCH_OPTION", + details={"unsupported_options": ["tool_choice"]}, + ) + + remaining = [tool for tool in tools if not _is_web_search_tool(tool)] + if remaining: + body["tools"] = remaining + else: + body.pop("tools", None) + # tool_choice without tools is rejected by OpenAI-shaped upstreams. + body.pop("tool_choice", None) + + return _WEB_SEARCH_SUFFIX + async def _fetch_provider_models(self) -> dict: url = f"{self.base_url.rstrip('/')}/models" headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else None diff --git a/tests/unit/test_venice_web_search.py b/tests/unit/test_venice_web_search.py new file mode 100644 index 00000000..df2ac903 --- /dev/null +++ b/tests/unit/test_venice_web_search.py @@ -0,0 +1,221 @@ +"""Venice web search over ``/v1/messages``. + +litellm's Anthropic adapter rewrites an Anthropic server-side web-search tool +into a top-level ``web_search_options``, which Venice rejects with +``400 Unrecognized key(s) in object: 'web_search_options'``. These tests pin +the trade: the tool is lifted out of the body and the same intent re-expressed +as a Venice model feature suffix. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator +from unittest.mock import AsyncMock, patch + +import pytest + +from routstr.core.exceptions import UpstreamError +from routstr.payment.models import Architecture, Model, Pricing +from routstr.upstream.base import BaseUpstreamProvider +from routstr.upstream.venice import VeniceUpstreamProvider + +WEB_SEARCH_TOOL = {"type": "web_search_20250305", "name": "web_search"} +FUNCTION_TOOL = { + "name": "lookup", + "description": "Look something up", + "input_schema": {"type": "object", "properties": {}}, +} + + +def _model(model_id: str = "deepseek-v4-flash-0731") -> Model: + return Model( + id=model_id, + name=model_id, + created=0, + description="", + context_length=8192, + architecture=Architecture( + modality="text->text", + input_modalities=["text"], + output_modalities=["text"], + tokenizer="Unknown", + instruct_type=None, + ), + pricing=Pricing(prompt=0.0, completion=0.0), + ) + + +def _body(**extra: Any) -> dict[str, Any]: + return { + "messages": [{"role": "user", "content": "what shipped today?"}], + "max_tokens": 64, + **extra, + } + + +async def _dispatch(provider: BaseUpstreamProvider, body: dict[str, Any]) -> dict: + """Run the real dispatcher, capturing the kwargs litellm would receive.""" + captured: dict[str, Any] = {} + + async def empty_iter() -> AsyncIterator[dict]: + if False: + yield {} + + async def fake_acreate(**kwargs: Any) -> AsyncIterator[dict]: + captured.update(kwargs) + return empty_iter() + + with patch( + "litellm.anthropic.messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ): + await provider._dispatch_anthropic_messages( + request_body=json.dumps( + {"model": "venice/x", "stream": True, **body} + ).encode(), + model_obj=_model(), + ) + return captured + + +@pytest.mark.asyncio +async def test_web_search_tool_never_reaches_venice_as_web_search_options() -> None: + """The reported 400: the derived parameter must not be sent at all.""" + provider = VeniceUpstreamProvider(api_key="sk-test") + + kwargs = await _dispatch(provider, _body(tools=[WEB_SEARCH_TOOL])) + + assert "web_search_options" not in kwargs + assert "tools" not in kwargs + assert kwargs["model"] == ( + "openai/deepseek-v4-flash-0731:enable_web_search=auto&enable_web_citations=true" + ) + assert kwargs["api_base"] == "https://api.venice.ai/api/v1" + + +@pytest.mark.asyncio +async def test_function_tools_survive_alongside_web_search() -> None: + provider = VeniceUpstreamProvider(api_key="sk-test") + + kwargs = await _dispatch( + provider, + _body( + tools=[WEB_SEARCH_TOOL, FUNCTION_TOOL], + tool_choice={"type": "tool", "name": "lookup"}, + ), + ) + + assert kwargs["tools"] == [FUNCTION_TOOL] + assert kwargs["tool_choice"] == {"type": "tool", "name": "lookup"} + assert "web_search_options" not in kwargs + assert kwargs["model"].endswith(":enable_web_search=auto&enable_web_citations=true") + + +@pytest.mark.asyncio +async def test_requests_without_web_search_are_untouched() -> None: + provider = VeniceUpstreamProvider(api_key="sk-test") + + kwargs = await _dispatch(provider, _body(tools=[FUNCTION_TOOL])) + + assert kwargs["model"] == "openai/deepseek-v4-flash-0731" + assert kwargs["tools"] == [FUNCTION_TOOL] + + +@pytest.mark.asyncio +async def test_other_providers_keep_their_existing_behaviour() -> None: + """The base hook is a no-op, so no non-Venice upstream changes shape.""" + provider = BaseUpstreamProvider(base_url="http://test", api_key="k") + + kwargs = await _dispatch(provider, _body(tools=[WEB_SEARCH_TOOL])) + + assert kwargs["model"] == "openai/deepseek-v4-flash-0731" + assert kwargs["tools"] == [WEB_SEARCH_TOOL] + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}, + { + "type": "web_search_20250305", + "name": "web_search", + "allowed_domains": ["example.com"], + }, + {"type": "web_search_20250305", "name": "web_search", "blocked_domains": ["x"]}, + { + "type": "web_search_20250305", + "name": "web_search", + "user_location": {"type": "approximate", "country": "DE"}, + }, + ], +) +def test_constraints_venice_cannot_enforce_are_refused(tool: dict[str, Any]) -> None: + """Better an explicit 400 than a search that quietly ignored the limit.""" + provider = VeniceUpstreamProvider(api_key="sk-test") + + with pytest.raises(UpstreamError) as excinfo: + provider.adapt_messages_request(_body(tools=[tool]), _model()) + + assert excinfo.value.status_code == 400 + assert excinfo.value.code == "UNSUPPORTED_WEB_SEARCH_OPTION" + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": None}, + {"type": "web_search_20250305", "name": "web_search", "allowed_domains": []}, + ], +) +def test_constraint_keys_stating_nothing_are_read_as_absent( + tool: dict[str, Any], +) -> None: + provider = VeniceUpstreamProvider(api_key="sk-test") + + assert provider.adapt_messages_request(_body(tools=[tool]), _model()) != "" + + +def test_forcing_web_search_through_tool_choice_is_refused() -> None: + provider = VeniceUpstreamProvider(api_key="sk-test") + body = _body( + tools=[WEB_SEARCH_TOOL], + tool_choice={"type": "tool", "name": "web_search"}, + ) + + with pytest.raises(UpstreamError) as excinfo: + provider.adapt_messages_request(body, _model()) + + assert excinfo.value.status_code == 400 + + +def test_tool_named_web_search_without_the_type_marker_is_caught() -> None: + """litellm matches on either marker, so this one would also be rewritten.""" + provider = VeniceUpstreamProvider(api_key="sk-test") + body = _body(tools=[{"name": "web_search"}]) + + assert provider.adapt_messages_request(body, _model()) != "" + assert "tools" not in body + + +def test_litellm_adapter_derives_no_web_search_options_from_the_adapted_body() -> None: + """The fix at its cause: run the real litellm translation over the body + this provider produces and assert the rejected key is never derived.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( # noqa: E501 + LiteLLMAnthropicMessagesAdapter, + ) + + provider = VeniceUpstreamProvider(api_key="sk-test") + adapter = LiteLLMAnthropicMessagesAdapter() + body = _body(tools=[WEB_SEARCH_TOOL, FUNCTION_TOOL]) + + # Unadapted, litellm derives the parameter Venice rejects. + before, _ = adapter.translate_anthropic_to_openai( + {"model": "m", **_body(tools=[WEB_SEARCH_TOOL])} + ) + assert "web_search_options" in before + + provider.adapt_messages_request(body, _model()) + after, _ = adapter.translate_anthropic_to_openai({"model": "m", **body}) + + assert "web_search_options" not in after