From 985e76528572b377d4e9eb5b09b57d3ce30e14ef Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Sat, 2 May 2026 21:21:58 +0200 Subject: [PATCH] fix gemini api --- routstr/upstream/base.py | 76 ++++++++++- tests/unit/test_messages_litellm_dispatch.py | 126 ++++++++++++++++++- 2 files changed, 198 insertions(+), 4 deletions(-) diff --git a/routstr/upstream/base.py b/routstr/upstream/base.py index 7ab96738..18731695 100644 --- a/routstr/upstream/base.py +++ b/routstr/upstream/base.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import hashlib import json +import os import re import traceback import uuid @@ -11,6 +12,34 @@ from typing import Any, Mapping, cast import httpx import litellm + +if os.getenv("LITELLM_DEBUG") == "1": + try: + litellm._turn_on_debug() # type: ignore[no-untyped-call] + except Exception: + pass + +# Force litellm's Anthropic-messages adapter to use OpenAI Chat Completions +# (POST /chat/completions) instead of OpenAI Responses API (POST /responses) +# for openai-prefixed providers. OpenAI-compatible upstreams like Google's +# generativelanguage compat endpoint expose /chat/completions but not +# /responses, which produces a 404. Override with +# `LITELLM_USE_RESPONSES_API_FOR_ANTHROPIC_MESSAGES=1` if a future upstream +# requires the Responses API. +if os.getenv("LITELLM_USE_RESPONSES_API_FOR_ANTHROPIC_MESSAGES") != "1": + try: + litellm.use_chat_completions_url_for_anthropic_messages = True + except Exception: + pass + +# Silently drop Anthropic-Messages-only parameters (e.g. `context_management`, +# `cache_control`, `thinking`) when translating to providers that don't +# accept them. Without this, litellm raises UnsupportedParamsError for any +# unrecognized field and rejects the whole request. Override with +# `LITELLM_STRICT_PARAMS=1` if an integration depends on the strict +# behavior. +if os.getenv("LITELLM_STRICT_PARAMS") != "1": + litellm.drop_params = True from fastapi import BackgroundTasks, HTTPException, Request from fastapi.responses import Response, StreamingResponse from pydantic.v1 import BaseModel @@ -1577,6 +1606,34 @@ class BaseUpstreamProvider: body.pop("model", None) stream = bool(body.pop("stream", False)) + # Anthropic-Messages-only fields that don't translate to OpenAI + # Chat Completions. litellm.drop_params only filters *known* + # unsupported params; these newer/extension fields get passed + # through verbatim and the upstream rejects them with a 400. + # Pop them here so the request reaches the upstream cleanly. + anthropic_only_fields = ( + "thinking", + "cache_control", + "context_management", + "output_config", + "mcp_servers", + "service_tier", + "anthropic_version", + "anthropic_beta", + ) + dropped: dict[str, Any] = {} + for field in anthropic_only_fields: + if field in body: + dropped[field] = body.pop(field) + if dropped: + logger.debug( + "Dropped anthropic-only fields before litellm dispatch", + extra={"dropped_keys": sorted(dropped.keys())}, + ) + + # Convention: `model.id` is the canonical upstream model name; + # `forwarded_model_id` is the public alias the internal API + # exposes and echoes back to the client. requested_model = ( (model_obj.forwarded_model_id or model_obj.id) if model_obj else None ) @@ -1603,16 +1660,31 @@ class BaseUpstreamProvider: try: result = await litellm.anthropic.messages.acreate(**kwargs) except Exception as exc: + exc_message = getattr(exc, "message", None) or str(exc) or repr(exc) + exc_status = getattr(exc, "status_code", None) + exc_response = getattr(exc, "response", None) + response_text = None + if exc_response is not None: + try: + response_text = getattr(exc_response, "text", str(exc_response)) + except Exception: + response_text = "" logger.error( "litellm dispatch failed", extra={ - "error": str(exc), + "error": exc_message, "error_type": type(exc).__name__, + "status_code": exc_status, + "llm_provider": getattr(exc, "llm_provider", None), + "body": getattr(exc, "body", None), + "response_text": response_text, "model": litellm_model, + "api_base": self.base_url, }, ) raise UpstreamError( - f"Upstream error via litellm: {exc}", status_code=502 + f"Upstream error via litellm: {exc_message}", + status_code=exc_status if isinstance(exc_status, int) else 502, ) from exc return stream, result, requested_model diff --git a/tests/unit/test_messages_litellm_dispatch.py b/tests/unit/test_messages_litellm_dispatch.py index 0759c4a9..4403afb8 100644 --- a/tests/unit/test_messages_litellm_dispatch.py +++ b/tests/unit/test_messages_litellm_dispatch.py @@ -38,11 +38,14 @@ def _make_key() -> ApiKey: return ApiKey(hashed_key="abcdef0123" * 4, balance=1_000_000) -def _make_model(model_id: str = "openai/gpt-4o-mini") -> Model: +def _make_model( + model_id: str = "openai/gpt-4o-mini", + forwarded_model_id: str | None = None, +) -> Model: return Model( id=model_id, name=model_id, - forwarded_model_id=model_id, + forwarded_model_id=forwarded_model_id if forwarded_model_id is not None else model_id, created=0, description="", context_length=8192, @@ -233,6 +236,125 @@ def test_provider_prefix_overrides() -> None: # --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_dispatch_strips_anthropic_only_fields_before_litellm() -> None: + """Regression: Anthropic-Messages-only fields like `output_config`, + `thinking`, `context_management`, and `cache_control` must be removed + from the request body before litellm dispatches to non-Anthropic + upstreams. Otherwise upstream returns 400 (unknown field). + """ + provider = _make_provider() + key = _make_key() + model = _make_model() + session = _make_session() + + body = json.dumps( + { + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 64, + "thinking": {"type": "adaptive"}, + "context_management": {"edits": []}, + "output_config": {"effort": "medium"}, + "cache_control": {"type": "ephemeral"}, + "mcp_servers": [], + "service_tier": "auto", + "anthropic_beta": "abc", + "anthropic_version": "2023-06-01", + } + ).encode() + + captured: dict[str, Any] = {} + + async def fake_acreate(**kwargs: Any) -> dict: + captured["kwargs"] = kwargs + return _anthropic_non_stream_response() + + with ( + patch( + "litellm.anthropic.messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ), + patch( + "routstr.upstream.base.adjust_payment_for_tokens", + new=AsyncMock(return_value={"total_msats": 0, "total_usd": 0.0}), + ), + ): + await provider._forward_messages_via_litellm( + request_body=body, + key=key, + session=session, + max_cost_for_model=10_000, + model_obj=model, + ) + + forwarded = captured["kwargs"] + for stripped in ( + "thinking", + "context_management", + "output_config", + "cache_control", + "mcp_servers", + "service_tier", + "anthropic_beta", + "anthropic_version", + ): + assert stripped not in forwarded, ( + f"Anthropic-only field {stripped!r} leaked through to litellm" + ) + # Core fields preserved + assert forwarded["max_tokens"] == 64 + assert forwarded["messages"] == [{"role": "user", "content": "hi"}] + + +@pytest.mark.asyncio +async def test_dispatch_uses_model_id_for_upstream_and_forwarded_for_client() -> None: + """Convention: `model.id` is the canonical upstream model name; + `forwarded_model_id` is the public alias echoed back to the client. + """ + provider = _make_provider() + key = _make_key() + # Upstream knows "gpt-4o-mini"; clients see public alias "gpt-5.4-test". + model = _make_model( + model_id="gpt-4o-mini", + forwarded_model_id="gpt-5.4-test", + ) + session = _make_session() + + captured: dict[str, Any] = {} + + async def fake_acreate(**kwargs: Any) -> dict: + captured["model"] = kwargs["model"] + resp = _anthropic_non_stream_response() + resp["model"] = "gpt-4o-mini" # upstream echoes its own id + return resp + + with ( + patch( + "litellm.anthropic.messages.acreate", + new=AsyncMock(side_effect=fake_acreate), + ), + patch( + "routstr.upstream.base.adjust_payment_for_tokens", + new=AsyncMock(return_value={"total_msats": 0, "total_usd": 0.0}), + ), + ): + result = await provider._forward_messages_via_litellm( + request_body=_anthropic_request_body(stream=False), + key=key, + session=session, + max_cost_for_model=10_000, + model_obj=model, + ) + + # Upstream call uses `model.id` with provider prefix + assert captured["model"] == "openai/gpt-4o-mini" + # Response to client echoes the public `forwarded_model_id` + assert isinstance(result, Response) + body = json.loads(result.body) + assert body["model"] == "gpt-5.4-test" + + @pytest.mark.asyncio async def test_non_streaming_dispatches_via_litellm_and_returns_anthropic_response() -> ( None