mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-10-05 12:28:22 +00:00
1003 lines
33 KiB
Python
1003 lines
33 KiB
Python
"""Integration tests for POST /admin/api/upstream-providers/{id}/certify.
|
|
|
|
Exercises the endpoint against a mocked upstream (no real network, no
|
|
real spend). The test fixtures create a provider + model row in the
|
|
integration DB, then mock the two HTTP calls the probe makes (GET /models
|
|
and POST /chat/completions) with ``respx`` so every verdict — ok, warn,
|
|
fail — is reachable deterministically.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import respx
|
|
from httpx import AsyncClient, Response
|
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
|
|
|
from routstr.core.admin import admin_sessions
|
|
from routstr.core.db import ModelPathRow, ModelRow, UpstreamProviderRow
|
|
from routstr.proxy import reinitialize_upstreams
|
|
from routstr.upstream.model_paths import encode_model_path
|
|
|
|
|
|
# The conftest patches ``routstr.payment.price.sats_usd_price``, but
|
|
# ``cost_calculation.py`` and ``models.py`` import it as a local binding
|
|
# the conftest-level patch cannot reach. Pin it here so every test that
|
|
# goes through ``_row_to_model`` or ``calculate_cost`` gets a real sats
|
|
# price — same pattern as ``test_model_price_propagation.py``.
|
|
@pytest.fixture(autouse=True)
|
|
def _pin_sats_usd() -> Any:
|
|
with patch("routstr.payment.models.sats_usd_price", return_value=0.0005):
|
|
with patch(
|
|
"routstr.payment.cost_calculation.sats_usd_price", return_value=0.0005
|
|
):
|
|
with patch("routstr.payment.price.SATS_USD_PRICE", 0.0005):
|
|
yield
|
|
|
|
|
|
ARCHITECTURE = {
|
|
"modality": "text",
|
|
"input_modalities": ["text"],
|
|
"output_modalities": ["text"],
|
|
"tokenizer": "unknown",
|
|
"instruct_type": None,
|
|
}
|
|
|
|
|
|
def _pricing(**overrides: float) -> dict[str, Any]:
|
|
pricing: dict[str, Any] = {
|
|
"prompt": 1.4e-7,
|
|
"completion": 2.8e-7,
|
|
"request": 0.0,
|
|
"image": 0.0,
|
|
"web_search": 0.0,
|
|
"internal_reasoning": 0.0,
|
|
"input_cache_read": 0.0,
|
|
"input_cache_write": 0.0,
|
|
}
|
|
pricing.update(overrides)
|
|
return pricing
|
|
|
|
|
|
def _admin_headers() -> dict[str, str]:
|
|
token = "test-certify-token"
|
|
admin_sessions[token] = int(
|
|
(datetime.now(timezone.utc) + timedelta(minutes=5)).timestamp()
|
|
)
|
|
return {"Authorization": f"Bearer {token}"}
|
|
|
|
|
|
async def _make_provider(
|
|
session: AsyncSession,
|
|
*,
|
|
slug: str | None = None,
|
|
provider_fee: float = 1.0,
|
|
base_url: str = "https://certify-upstream.example/v1",
|
|
api_key: str = "test-key",
|
|
) -> UpstreamProviderRow:
|
|
provider = UpstreamProviderRow(
|
|
provider_type="generic",
|
|
base_url=base_url,
|
|
api_key=api_key,
|
|
provider_fee=provider_fee,
|
|
slug=slug,
|
|
)
|
|
session.add(provider)
|
|
await session.commit()
|
|
await session.refresh(provider)
|
|
assert provider.id is not None
|
|
return provider
|
|
|
|
|
|
def _model_row(provider_id: int, **overrides: Any) -> ModelRow:
|
|
model_id = overrides.pop("model_id", "cert-test-model")
|
|
return ModelRow(
|
|
id=model_id,
|
|
name=model_id,
|
|
description="d",
|
|
created=0,
|
|
context_length=8192,
|
|
architecture=json.dumps(ARCHITECTURE),
|
|
pricing=json.dumps(_pricing(**overrides.pop("pricing_overrides", {}))),
|
|
upstream_provider_id=provider_id,
|
|
enabled=True,
|
|
forwarded_model_id=model_id,
|
|
)
|
|
|
|
|
|
async def _seed_and_init(
|
|
session: AsyncSession,
|
|
client: AsyncClient,
|
|
*,
|
|
provider_fee: float = 1.0,
|
|
model_id: str = "cert-test-model",
|
|
pricing_overrides: dict[str, Any] | None = None,
|
|
base_url: str = "https://certify-upstream.example/v1",
|
|
) -> int:
|
|
provider = await _make_provider(
|
|
session, provider_fee=provider_fee, base_url=base_url
|
|
)
|
|
assert provider.id is not None
|
|
session.add(
|
|
_model_row(
|
|
provider.id,
|
|
model_id=model_id,
|
|
pricing_overrides=pricing_overrides or {},
|
|
)
|
|
)
|
|
await session.commit()
|
|
with patch("routstr.payment.models.sats_usd_price", return_value=0.0005):
|
|
await reinitialize_upstreams()
|
|
return provider.id
|
|
|
|
|
|
def _find_row(rows: list[dict[str, Any]], row_id: str) -> dict[str, Any]:
|
|
for row in rows:
|
|
if row["id"] == row_id:
|
|
return row
|
|
raise AssertionError(f"row {row_id!r} not found")
|
|
|
|
|
|
def _mock_models_response(
|
|
base_url: str = "https://certify-upstream.example/v1",
|
|
models: list[dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
if models is None:
|
|
models = [{"id": "cert-test-model"}]
|
|
return {"data": models}
|
|
|
|
|
|
def _mock_chat_response(
|
|
model: str = "cert-test-model",
|
|
prompt_tokens: int = 5,
|
|
completion_tokens: int = 1,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": "chatcmpl-test",
|
|
"object": "chat.completion",
|
|
"model": model,
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": "ok"},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"usage": {
|
|
"prompt_tokens": prompt_tokens,
|
|
"completion_tokens": completion_tokens,
|
|
},
|
|
}
|
|
|
|
|
|
def _caching_upstream(*, cached_tokens: int = 2900, report_cost: bool = True) -> Any:
|
|
"""Side effect that answers the one-token probe and then two long
|
|
prompts, reporting a cache hit (and optionally a USD cost) on the
|
|
repeated one — the shape an OpenAI-compatible caching upstream returns."""
|
|
long_calls = {"n": 0}
|
|
|
|
def _respond(request: Any) -> Response:
|
|
body = json.loads(request.content)
|
|
is_long = body["messages"][0]["role"] == "system"
|
|
if not is_long:
|
|
usage: dict[str, Any] = {"prompt_tokens": 5, "completion_tokens": 1}
|
|
if report_cost:
|
|
usage["cost"] = 9e-7
|
|
else:
|
|
long_calls["n"] += 1
|
|
usage = {"prompt_tokens": 3000, "completion_tokens": 1}
|
|
if long_calls["n"] > 1 and cached_tokens:
|
|
usage["prompt_tokens_details"] = {"cached_tokens": cached_tokens}
|
|
if report_cost:
|
|
usage["cost"] = 5e-5 if long_calls["n"] > 1 else 4e-4
|
|
payload = _mock_chat_response()
|
|
payload["usage"] = usage
|
|
return Response(200, json=payload)
|
|
|
|
return _respond
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_certify_requires_admin_auth(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider = await _make_provider(integration_session)
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider.id}/certify", json={}
|
|
)
|
|
assert resp.status_code == 403
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_certify_unknown_provider_returns_404(
|
|
integration_client: AsyncClient,
|
|
) -> None:
|
|
resp = await integration_client.post(
|
|
"/admin/api/upstream-providers/999999999/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_all_ok(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
pricing_overrides={"input_cache_read": 1.4e-8},
|
|
)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
side_effect=_caching_upstream()
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
body = resp.json()
|
|
assert "rows" in body
|
|
assert "checklist" in body
|
|
|
|
live_row_ids = [
|
|
"endpoint.validity",
|
|
"endpoint.reachable",
|
|
"endpoint.models_payload",
|
|
"usage.capture",
|
|
"cost.prompt_completion",
|
|
"cache.reported",
|
|
"cache.billing",
|
|
"cost.margin",
|
|
]
|
|
for row_id in live_row_ids:
|
|
row = _find_row(body["rows"], row_id)
|
|
assert row["status"] == "ok", f"{row_id}: {row}"
|
|
|
|
for item in body["checklist"]:
|
|
assert item["status"] == "ok", f"{item['goal']}: {item}"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_model_path_pins_every_completion(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
base_url = "https://openrouter.ai/api/v1"
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
pricing_overrides={"input_cache_read": 1.4e-8},
|
|
base_url=base_url,
|
|
)
|
|
model_path = encode_model_path(base_url, "cert-test-model", "azure")
|
|
integration_session.add(
|
|
ModelPathRow(
|
|
model_id="cert-test-model",
|
|
path=model_path,
|
|
provider_slug="openrouter",
|
|
provider_type="openrouter",
|
|
endpoint_tag="azure",
|
|
endpoint_name="Azure",
|
|
model_metadata="{}",
|
|
upstream_provider_id=provider_id,
|
|
)
|
|
)
|
|
await integration_session.commit()
|
|
|
|
respx.get(f"{base_url}/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
chat = respx.post(f"{base_url}/chat/completions").mock(
|
|
side_effect=_caching_upstream()
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={"model_id": "cert-test-model", "model_path": model_path},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
assert chat.call_count == 3
|
|
for call in chat.calls:
|
|
body = json.loads(call.request.content)
|
|
assert body["provider"] == {
|
|
"order": ["azure"],
|
|
"allow_fallbacks": False,
|
|
}
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_uses_selected_path_pricing_for_margin(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
base_url = "https://openrouter.ai/api/v1"
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
provider_fee=0.4,
|
|
pricing_overrides={
|
|
"prompt": 1e-7,
|
|
"completion": 5e-7,
|
|
"input_cache_read": 1e-8,
|
|
},
|
|
base_url=base_url,
|
|
)
|
|
model_path = encode_model_path(
|
|
base_url, "cert-test-model", "deepinfra/fp8"
|
|
)
|
|
integration_session.add(
|
|
ModelPathRow(
|
|
model_id="cert-test-model",
|
|
path=model_path,
|
|
provider_slug="openrouter",
|
|
provider_type="openrouter",
|
|
endpoint_tag="deepinfra/fp8",
|
|
endpoint_name="DeepInfra",
|
|
model_metadata=json.dumps(
|
|
{
|
|
"id": "cert-test-model",
|
|
"pricing": {
|
|
"prompt": 1.4e-7,
|
|
"completion": 4.2e-7,
|
|
"input_cache_read": 4.2e-9,
|
|
},
|
|
}
|
|
),
|
|
upstream_provider_id=provider_id,
|
|
)
|
|
)
|
|
await integration_session.commit()
|
|
|
|
respx.get(f"{base_url}/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
usages = iter(
|
|
[
|
|
{
|
|
"prompt_tokens": 31,
|
|
"completion_tokens": 1,
|
|
"cost": 0.00000459,
|
|
},
|
|
{
|
|
"prompt_tokens": 4442,
|
|
"completion_tokens": 1,
|
|
"cost": 0.00057802,
|
|
},
|
|
{
|
|
"prompt_tokens": 4442,
|
|
"completion_tokens": 1,
|
|
"prompt_tokens_details": {"cached_tokens": 4352},
|
|
"cost": 0.00005578,
|
|
},
|
|
]
|
|
)
|
|
|
|
def _respond(_request: Any) -> Response:
|
|
payload = _mock_chat_response()
|
|
payload["usage"] = next(usages)
|
|
return Response(200, json=payload)
|
|
|
|
respx.post(f"{base_url}/chat/completions").mock(side_effect=_respond)
|
|
sats_usd = 0.0008616302499999999
|
|
with patch("routstr.payment.price.sats_usd_price", return_value=sats_usd):
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={"model_id": "cert-test-model", "model_path": model_path},
|
|
)
|
|
|
|
assert resp.status_code == 200, resp.text
|
|
margin = _find_row(resp.json()["rows"], "cost.margin")
|
|
assert [
|
|
(sample["upstream_msats_with_fee"], sample["configured_msats"])
|
|
for sample in margin["evidence"]["samples"]
|
|
] == [(3, 3), (269, 289), (26, 15)]
|
|
assert "289 < 269" not in margin["detail"]
|
|
assert "15 < 26" in margin["detail"]
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_provider_models_includes_certification_paths(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
base_url = "https://openrouter.ai/api/v1"
|
|
provider_id = await _seed_and_init(
|
|
integration_session, integration_client, base_url=base_url
|
|
)
|
|
model_path = encode_model_path(base_url, "cert-test-model", "azure")
|
|
integration_session.add(
|
|
ModelPathRow(
|
|
model_id="cert-test-model",
|
|
path=model_path,
|
|
provider_slug="openrouter",
|
|
provider_type="openrouter",
|
|
endpoint_tag="azure",
|
|
endpoint_name="Azure",
|
|
model_metadata="{}",
|
|
upstream_provider_id=provider_id,
|
|
)
|
|
)
|
|
await integration_session.commit()
|
|
respx.get(f"{base_url}/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
|
|
resp = await integration_client.get(
|
|
f"/admin/api/upstream-providers/{provider_id}/models",
|
|
headers=_admin_headers(),
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
assert resp.json()["certification_paths"]["cert-test-model"] == [
|
|
{
|
|
"path": model_path,
|
|
"endpoint_tag": "azure",
|
|
"endpoint_name": "Azure",
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_heartbeat_fail_on_500(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(500, json={"error": "internal"})
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "endpoint.reachable")
|
|
assert row["status"] == "fail"
|
|
assert row["evidence"]["status_code"] == 500
|
|
|
|
heartbeat_goal = next(
|
|
item for item in body["checklist"] if item["goal"] == "heartbeat"
|
|
)
|
|
assert heartbeat_goal["status"] == "fail"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_heartbeat_fail_on_transport_error(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
side_effect=__import__("httpx").ConnectError("connection refused")
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "endpoint.reachable")
|
|
assert row["status"] == "fail"
|
|
assert row["evidence"]["error"] is not None
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_models_payload_fail_on_malformed(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json={"error": "no data field"})
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "endpoint.models_payload")
|
|
assert row["status"] == "fail"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_usage_warn_when_no_usage(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(
|
|
200,
|
|
json={
|
|
"id": "x",
|
|
"model": "cert-test-model",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": "ok"},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
},
|
|
)
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
usage_row = _find_row(body["rows"], "usage.capture")
|
|
assert usage_row["status"] == "warn"
|
|
|
|
cost_row = _find_row(body["rows"], "cost.prompt_completion")
|
|
assert cost_row["status"] == "warn"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_usage_fail_on_non_2xx(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(401, json={"error": {"message": "invalid api key"}})
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "usage.capture")
|
|
assert row["status"] == "fail"
|
|
assert "401" in row["detail"]
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_cost_ok_with_token_pricing(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
provider_fee=1.0,
|
|
pricing_overrides={"prompt": 1e-7, "completion": 2e-7},
|
|
)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(
|
|
200, json=_mock_chat_response(prompt_tokens=10, completion_tokens=5)
|
|
)
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "cost.prompt_completion")
|
|
assert row["status"] == "ok", row
|
|
assert row["evidence"]["basis"] == "configured_token_pricing"
|
|
assert row["evidence"]["input_tokens"] == 10
|
|
assert row["evidence"]["output_tokens"] == 5
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_cost_ok_with_usd_reported(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
provider_fee=1.05,
|
|
)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
chat_payload = _mock_chat_response()
|
|
chat_payload["usage"]["cost_details"] = {"total_cost": 0.0001}
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=chat_payload)
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row = _find_row(body["rows"], "cost.prompt_completion")
|
|
assert row["status"] == "ok", row
|
|
assert row["evidence"]["basis"] == "upstream_reported_usd"
|
|
assert row["evidence"]["reported_usd"] == 0.0001
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_certify_with_no_served_model(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
"""When the provider has no model that is being served (e.g. all have
|
|
unusable pricing), the live checks should be skipped as warn, not
|
|
crash."""
|
|
provider = await _make_provider(integration_session)
|
|
assert provider.id is not None
|
|
# A negative prompt price makes has_usable_pricing() return False,
|
|
# withholding the model from the served map.
|
|
session_add = _model_row(
|
|
provider.id, model_id="bad-model", pricing_overrides={"prompt": -1.0}
|
|
)
|
|
integration_session.add(session_add)
|
|
await integration_session.commit()
|
|
with patch("routstr.payment.models.sats_usd_price", return_value=0.0005):
|
|
await reinitialize_upstreams()
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider.id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
body = resp.json()
|
|
for row_id in ["endpoint.reachable", "usage.capture", "cost.prompt_completion"]:
|
|
row = _find_row(body["rows"], row_id)
|
|
assert row["status"] == "warn", f"{row_id}: {row}"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_with_explicit_model_id(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(
|
|
integration_session,
|
|
integration_client,
|
|
model_id="explicit-model",
|
|
)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(
|
|
200, json=_mock_models_response(models=[{"id": "explicit-model"}])
|
|
)
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response(model="explicit-model"))
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={"model_id": "explicit-model"},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
body = resp.json()
|
|
usage_row = _find_row(body["rows"], "usage.capture")
|
|
assert usage_row["status"] == "ok"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_explicit_discovered_model_without_override(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
"""An operator can probe a discovered model before creating an override."""
|
|
from routstr.payment.models import (
|
|
Architecture,
|
|
Model,
|
|
Pricing,
|
|
_update_model_sats_pricing,
|
|
)
|
|
|
|
provider = await _make_provider(integration_session)
|
|
assert provider.id is not None
|
|
remote_model = _update_model_sats_pricing(
|
|
Model(
|
|
id="remote-model",
|
|
name="Remote model",
|
|
description="",
|
|
created=0,
|
|
context_length=8192,
|
|
architecture=Architecture(
|
|
modality="text",
|
|
input_modalities=["text"],
|
|
output_modalities=["text"],
|
|
tokenizer="unknown",
|
|
instruct_type=None,
|
|
),
|
|
pricing=Pricing(prompt=1e-7, completion=2e-7),
|
|
sats_pricing=None,
|
|
per_request_limits=None,
|
|
top_provider=None,
|
|
enabled=True,
|
|
upstream_provider_id=provider.id,
|
|
canonical_slug=None,
|
|
),
|
|
0.0005,
|
|
)
|
|
|
|
class FakeUpstream:
|
|
db_id = provider.id
|
|
|
|
def get_cached_models(self) -> list[Model]:
|
|
return [remote_model]
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(
|
|
200, json=_mock_models_response(models=[{"id": "remote-model"}])
|
|
)
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response(model="remote-model"))
|
|
)
|
|
|
|
with patch("routstr.proxy.get_upstreams", return_value=[FakeUpstream()]):
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider.id}/certify",
|
|
headers=_admin_headers(),
|
|
json={"model_id": "remote-model", "check_cache": False},
|
|
)
|
|
|
|
assert resp.status_code == 200, resp.text
|
|
rows = resp.json()["rows"]
|
|
assert _find_row(rows, "endpoint.reachable")["status"] == "ok"
|
|
assert _find_row(rows, "usage.capture")["status"] == "ok"
|
|
assert _find_row(rows, "cost.prompt_completion")["status"] == "ok"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_includes_pricing_rows(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
"""The certify response should also carry the four pricing.* rows from
|
|
the read-only report, so the certification is self-contained."""
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
row_ids = [row["id"] for row in body["rows"]]
|
|
assert "pricing.served_matches_configured" in row_ids
|
|
assert "pricing.sats_pricing_present" in row_ids
|
|
assert "pricing.enabled_models_served" in row_ids
|
|
assert "pricing.cache_rate" in row_ids
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_row_contract_shape(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
"""Every row in the response has the required keys and a valid status."""
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
|
|
assert "provider_id" in body
|
|
assert "generated_at" in body
|
|
assert "rows" in body
|
|
assert "checklist" in body
|
|
|
|
for row in body["rows"]:
|
|
assert set(row) >= {"id", "status", "title", "detail", "evidence"}
|
|
assert row["status"] in {"ok", "warn", "fail"}
|
|
|
|
for item in body["checklist"]:
|
|
assert set(item) >= {"goal", "label", "status", "tick", "rows"}
|
|
assert item["status"] in {"ok", "warn", "fail"}
|
|
assert item["tick"] in {"☑️", "⚠️", "❌"}
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_cache_warn_when_upstream_never_hits(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
side_effect=_caching_upstream(cached_tokens=0)
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
rows = resp.json()["rows"]
|
|
assert _find_row(rows, "cache.reported")["status"] == "warn"
|
|
assert _find_row(rows, "cache.billing")["status"] == "warn"
|
|
goals = {item["goal"]: item["status"] for item in resp.json()["checklist"]}
|
|
assert goals["caching"] == "warn"
|
|
assert goals["margin"] == "ok"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_cache_billing_warns_without_cache_rate(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
side_effect=_caching_upstream(report_cost=False)
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
rows = resp.json()["rows"]
|
|
assert _find_row(rows, "cache.reported")["status"] == "ok"
|
|
billing = _find_row(rows, "cache.billing")
|
|
assert billing["status"] == "warn"
|
|
assert billing["evidence"]["actual_total_msats"] == 841
|
|
assert _find_row(rows, "cost.margin")["status"] == "warn"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_margin_fails_when_upstream_costs_more(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
|
|
def _expensive(request: Any) -> Response:
|
|
payload = _mock_chat_response()
|
|
payload["usage"] = {"prompt_tokens": 5, "completion_tokens": 1, "cost": 1e-3}
|
|
return Response(200, json=payload)
|
|
|
|
respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
side_effect=_expensive
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
margin = _find_row(resp.json()["rows"], "cost.margin")
|
|
assert margin["status"] == "fail"
|
|
goals = {item["goal"]: item["status"] for item in resp.json()["checklist"]}
|
|
assert goals["margin"] == "fail"
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@respx.mock
|
|
async def test_certify_check_cache_false_skips_probe(
|
|
integration_client: AsyncClient, integration_session: AsyncSession
|
|
) -> None:
|
|
provider_id = await _seed_and_init(integration_session, integration_client)
|
|
respx.get("https://certify-upstream.example/v1/models").mock(
|
|
return_value=Response(200, json=_mock_models_response())
|
|
)
|
|
chat = respx.post("https://certify-upstream.example/v1/chat/completions").mock(
|
|
return_value=Response(200, json=_mock_chat_response())
|
|
)
|
|
|
|
resp = await integration_client.post(
|
|
f"/admin/api/upstream-providers/{provider_id}/certify",
|
|
headers=_admin_headers(),
|
|
json={"check_cache": False},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
assert chat.call_count == 1
|
|
rows = resp.json()["rows"]
|
|
for row_id in ("cache.reported", "cache.billing", "cost.margin"):
|
|
row = _find_row(rows, row_id)
|
|
assert row["status"] == "warn"
|
|
assert "disabled" in row["detail"]
|