mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-08-01 00:06:14 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 9qeklajc
|
8c0ac499ef | ||
 9qeklajcandGitHub
|
28d91227af | ||
|
9qeklajcandGitHub
|
e43ceb2e43 | ||
|
9qeklajc
|
689a07f562 |
@@ -103,8 +103,11 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
btc_price_task = asyncio.create_task(update_prices_periodically())
|
||||
pricing_task = asyncio.create_task(update_sats_pricing())
|
||||
if global_settings.models_refresh_interval_seconds > 0:
|
||||
# Pass the accessor (not its current value) so the loop sees providers
|
||||
# added/changed via reinitialize_upstreams() instead of staying pinned
|
||||
# to the startup snapshot.
|
||||
models_refresh_task = asyncio.create_task(
|
||||
refresh_upstreams_models_periodically(get_upstreams())
|
||||
refresh_upstreams_models_periodically(get_upstreams)
|
||||
)
|
||||
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
|
||||
payout_task = asyncio.create_task(periodic_payout())
|
||||
|
||||
+177
-107
@@ -604,57 +604,83 @@ class BaseUpstreamProvider:
|
||||
)
|
||||
yield prefix + part
|
||||
|
||||
# Stream finished, process usage if found
|
||||
if usage_chunk_data:
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(key.__class__, key.hashed_key)
|
||||
if fresh_key:
|
||||
try:
|
||||
cost_data = await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
usage_chunk_data,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
)
|
||||
remaining_balance_msats = fresh_key.balance
|
||||
# Merge cost into usage
|
||||
usage_chunk_data["usage"]["cost"] = cost_data.get(
|
||||
"total_usd", 0.0
|
||||
)
|
||||
usage_chunk_data["usage"]["cost_sats"] = (
|
||||
cost_data.get("total_msats", 0) // 1000
|
||||
)
|
||||
usage_chunk_data["usage"]["remaining_balance_msats"] = (
|
||||
remaining_balance_msats
|
||||
)
|
||||
# Keep detailed cost in metadata
|
||||
usage_chunk_data["metadata"] = usage_chunk_data.get(
|
||||
"metadata", {}
|
||||
)
|
||||
usage_chunk_data["metadata"]["routstr"] = {
|
||||
"cost": cost_data
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(key.__class__, key.hashed_key)
|
||||
if fresh_key:
|
||||
cost_data: dict
|
||||
try:
|
||||
adjustment_input = (
|
||||
usage_chunk_data
|
||||
if usage_chunk_data is not None
|
||||
else {
|
||||
"model": last_model_seen or "unknown",
|
||||
"usage": None,
|
||||
}
|
||||
usage_chunk_data["metadata"]["routstr"]["cost"][
|
||||
"sats_cost"
|
||||
] = cost_data.get("total_msats", 0) // 1000
|
||||
usage_chunk_data["metadata"]["routstr"]["cost"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
usage_finalized = True
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error during usage finalization",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
# Fallback: yield original usage chunk if adjustment fails
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
)
|
||||
cost_data = await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
adjustment_input,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error during usage finalization",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
|
||||
if not usage_finalized:
|
||||
await finalize_db_only()
|
||||
# Fall back so we still emit a non-zero sats cost downstream.
|
||||
cost_data = {
|
||||
"base_msats": 0,
|
||||
"input_msats": 0,
|
||||
"output_msats": 0,
|
||||
"total_msats": 0,
|
||||
"total_usd": 0.0,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
}
|
||||
|
||||
if usage_chunk_data is None:
|
||||
if not hasattr(self, "_current_stream_id"):
|
||||
self._current_stream_id = (
|
||||
f"chatcmpl-{uuid.uuid4()}"
|
||||
)
|
||||
usage_chunk_data = {
|
||||
"id": self._current_stream_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": last_model_seen or "unknown",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
),
|
||||
"completion_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
self.inject_cost_metadata(
|
||||
usage_chunk_data, cost_data, fresh_key
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to inject cost metadata into streaming chunk",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
|
||||
if done_seen:
|
||||
yield b"data: [DONE]\n\n"
|
||||
@@ -926,65 +952,108 @@ class BaseUpstreamProvider:
|
||||
)
|
||||
yield prefix + part
|
||||
|
||||
# Stream finished, process usage if found
|
||||
if usage_chunk_data:
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(key.__class__, key.hashed_key)
|
||||
if fresh_key:
|
||||
try:
|
||||
cost_data = await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
usage_chunk_data,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
)
|
||||
remaining_balance_msats = fresh_key.balance
|
||||
# Merge cost into usage chunk
|
||||
if (
|
||||
"response" in usage_chunk_data
|
||||
and "usage" in usage_chunk_data["response"]
|
||||
):
|
||||
usage_chunk_data["response"]["usage"]["cost"] = (
|
||||
cost_data.get("total_usd", 0.0)
|
||||
)
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"cost_sats"
|
||||
] = cost_data.get("total_msats", 0) // 1000
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
elif "usage" in usage_chunk_data:
|
||||
usage_chunk_data["usage"]["cost"] = cost_data.get(
|
||||
"total_usd", 0.0
|
||||
)
|
||||
usage_chunk_data["usage"]["cost_sats"] = (
|
||||
cost_data.get("total_msats", 0) // 1000
|
||||
)
|
||||
usage_chunk_data["usage"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
|
||||
# Keep detailed cost in metadata
|
||||
usage_chunk_data["metadata"] = usage_chunk_data.get(
|
||||
"metadata", {}
|
||||
)
|
||||
usage_chunk_data["metadata"]["routstr"] = {
|
||||
"cost": cost_data
|
||||
# Always emit a cost-bearing data chunk
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(key.__class__, key.hashed_key)
|
||||
if fresh_key:
|
||||
cost_data: dict
|
||||
try:
|
||||
adjustment_input = (
|
||||
usage_chunk_data
|
||||
if usage_chunk_data is not None
|
||||
else {
|
||||
"model": last_model_seen or "unknown",
|
||||
"usage": None,
|
||||
}
|
||||
usage_chunk_data["metadata"]["routstr"]["cost"][
|
||||
"sats_cost"
|
||||
] = cost_data.get("total_msats", 0) // 1000
|
||||
usage_chunk_data["metadata"]["routstr"]["cost"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
usage_finalized = True
|
||||
except Exception:
|
||||
# Fallback: yield original usage chunk if adjustment fails
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
)
|
||||
cost_data = await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
adjustment_input,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error during Responses API usage finalization",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
cost_data = {
|
||||
"base_msats": 0,
|
||||
"input_msats": 0,
|
||||
"output_msats": 0,
|
||||
"total_msats": 0,
|
||||
"total_usd": 0.0,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
}
|
||||
|
||||
if not usage_finalized:
|
||||
await finalize_db_only()
|
||||
if usage_chunk_data is None:
|
||||
usage_chunk_data = {
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"model": last_model_seen or "unknown",
|
||||
"usage": {
|
||||
"input_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
),
|
||||
"output_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
},
|
||||
"usage": {
|
||||
"input_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
),
|
||||
"output_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
}
|
||||
|
||||
remaining_balance_msats = fresh_key.balance
|
||||
sats_cost = cost_data.get("total_msats", 0) // 1000
|
||||
|
||||
if (
|
||||
"response" in usage_chunk_data
|
||||
and isinstance(usage_chunk_data["response"], dict)
|
||||
and "usage" in usage_chunk_data["response"]
|
||||
):
|
||||
usage_chunk_data["response"]["usage"]["cost"] = (
|
||||
cost_data.get("total_usd", 0.0)
|
||||
)
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"cost_sats"
|
||||
] = sats_cost
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
|
||||
try:
|
||||
self.inject_cost_metadata(
|
||||
usage_chunk_data, cost_data, fresh_key
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to inject cost metadata into Responses streaming chunk",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
|
||||
yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
|
||||
|
||||
if done_seen:
|
||||
yield b"data: [DONE]\n\n"
|
||||
@@ -1308,7 +1377,8 @@ class BaseUpstreamProvider:
|
||||
)
|
||||
|
||||
usage_finalized = True
|
||||
yield f"event: cost\ndata: {json.dumps({'cost': cost_data})}\n\n".encode()
|
||||
# Emit the full combined_data as the cost
|
||||
yield f"event: cost\ndata: {json.dumps(combined_data)}\n\n".encode()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Callable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..core.settings import Settings
|
||||
@@ -122,12 +122,16 @@ async def get_all_models_with_overrides(
|
||||
|
||||
|
||||
async def refresh_upstreams_models_periodically(
|
||||
upstreams: list[BaseUpstreamProvider],
|
||||
upstreams_provider: (
|
||||
Callable[[], list[BaseUpstreamProvider]] | list[BaseUpstreamProvider]
|
||||
),
|
||||
) -> None:
|
||||
"""Background task to periodically refresh models cache for all providers.
|
||||
|
||||
Args:
|
||||
upstreams: List of upstream provider instances
|
||||
upstreams_provider: Either a callable returning the live upstream list
|
||||
(preferred — picks up providers added/changed via reinitialize_upstreams),
|
||||
or a static list (legacy, will go stale after reinitialize_upstreams).
|
||||
"""
|
||||
import asyncio
|
||||
import random
|
||||
@@ -139,9 +143,14 @@ async def refresh_upstreams_models_periodically(
|
||||
logger.info("Provider models refresh disabled (interval <= 0)")
|
||||
return
|
||||
|
||||
def _resolve_upstreams() -> list[BaseUpstreamProvider]:
|
||||
if callable(upstreams_provider):
|
||||
return upstreams_provider()
|
||||
return upstreams_provider
|
||||
|
||||
while True:
|
||||
try:
|
||||
for upstream in upstreams:
|
||||
for upstream in _resolve_upstreams():
|
||||
try:
|
||||
await upstream.refresh_models_cache()
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Regression tests for the periodic upstream models refresh loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
|
||||
from routstr.upstream.base import BaseUpstreamProvider # noqa: E402
|
||||
|
||||
|
||||
class _FakeUpstream:
|
||||
"""Minimal stand-in for BaseUpstreamProvider used by the refresh loop.
|
||||
|
||||
Only ``base_url`` (for error logging) and ``refresh_models_cache`` (the call
|
||||
under test) are exercised; everything else stays unused.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.base_url = f"http://{name}"
|
||||
self.refresh_models_cache = AsyncMock()
|
||||
|
||||
|
||||
def _make_fake_upstream(name: str) -> BaseUpstreamProvider:
|
||||
# The loop only uses duck-typed attributes — cast keeps the test type-clean
|
||||
# without dragging in BaseUpstreamProvider's full constructor.
|
||||
return cast(BaseUpstreamProvider, _FakeUpstream(name))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_loop_picks_up_providers_added_after_startup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""If a provider is added after the loop starts (e.g. via reinitialize_upstreams),
|
||||
the next loop iteration must refresh it. Previously the loop captured the upstream
|
||||
list at startup and missed any later additions."""
|
||||
from routstr.core.settings import settings as global_settings
|
||||
from routstr.upstream.helpers import refresh_upstreams_models_periodically
|
||||
|
||||
# Tight interval so the test finishes quickly.
|
||||
monkeypatch.setattr(
|
||||
global_settings, "models_refresh_interval_seconds", 1, raising=False
|
||||
)
|
||||
|
||||
initial_upstream = _make_fake_upstream("initial")
|
||||
live_list: list[BaseUpstreamProvider] = [initial_upstream]
|
||||
|
||||
# Stub out the post-iteration sats-pricing refresh so the loop body has no DB deps.
|
||||
async def _noop_pricing_refresh() -> None: # pragma: no cover - trivial stub
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"routstr.payment.models._update_sats_pricing_once",
|
||||
_noop_pricing_refresh,
|
||||
)
|
||||
|
||||
task = asyncio.create_task(
|
||||
refresh_upstreams_models_periodically(lambda: live_list)
|
||||
)
|
||||
|
||||
try:
|
||||
# Wait for the first iteration to refresh the initial upstream.
|
||||
for _ in range(40):
|
||||
if initial_upstream.refresh_models_cache.await_count >= 1: # type: ignore[attr-defined]
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
assert initial_upstream.refresh_models_cache.await_count >= 1, ( # type: ignore[attr-defined]
|
||||
"loop did not refresh the initial upstream within the timeout"
|
||||
)
|
||||
|
||||
# Simulate reinitialize_upstreams: replace the live list contents with new
|
||||
# provider instances. The loop must observe the swap on its next tick.
|
||||
new_upstream = _make_fake_upstream("added-after-startup")
|
||||
live_list[:] = [new_upstream]
|
||||
|
||||
for _ in range(60):
|
||||
if new_upstream.refresh_models_cache.await_count >= 1: # type: ignore[attr-defined]
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
assert new_upstream.refresh_models_cache.await_count >= 1, ( # type: ignore[attr-defined]
|
||||
"loop did not refresh the upstream added after startup — "
|
||||
"regression: list snapshot captured at startup"
|
||||
)
|
||||
finally:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_loop_disabled_when_interval_non_positive(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from routstr.core.settings import settings as global_settings
|
||||
from routstr.upstream.helpers import refresh_upstreams_models_periodically
|
||||
|
||||
monkeypatch.setattr(
|
||||
global_settings, "models_refresh_interval_seconds", 0, raising=False
|
||||
)
|
||||
|
||||
upstream = _make_fake_upstream("never-refreshed")
|
||||
|
||||
# Loop must return immediately without ever touching the upstream.
|
||||
await asyncio.wait_for(
|
||||
refresh_upstreams_models_periodically(lambda: [upstream]),
|
||||
timeout=1.0,
|
||||
)
|
||||
upstream.refresh_models_cache.assert_not_awaited() # type: ignore[attr-defined]
|
||||
@@ -51,7 +51,15 @@ async def test_stream_with_id_injection() -> None:
|
||||
base.adjust_payment_for_tokens = AsyncMock(
|
||||
return_value={"total_usd": 0.1, "total_msats": 100}
|
||||
)
|
||||
base.create_session = MagicMock()
|
||||
# create_session() is used as an async context manager whose entered
|
||||
# value exposes an awaitable .get(). Build a mock that behaves that
|
||||
# way so the post-stream cost-chunk emission can run.
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = AsyncMock(return_value=key)
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_ctx.__aexit__ = AsyncMock(return_value=None)
|
||||
base.create_session = MagicMock(return_value=mock_ctx)
|
||||
|
||||
streaming_response = await provider.handle_streaming_chat_completion(
|
||||
response=mock_response,
|
||||
|
||||
Reference in New Issue
Block a user