Compare commits

..
Author SHA1 Message Date
9qeklajc ec80f39e64 Merge branch 'fix-x-cashu-misscalculation' into refresh-models
# Conflicts:
#	routstr/core/admin.py
2025-10-15 22:10:42 +02:00
9qeklajc 4327f542a0 test 2025-10-15 20:23:00 +02:00
9qeklajc b4eac33542 clean up 2025-10-14 00:16:16 +02:00
9qeklajc 9911ce9dcd fix different max cost calculation 2025-10-14 00:02:04 +02:00
18 changed files with 427 additions and 949 deletions
-2
View File
@@ -12,8 +12,6 @@ dist/
# Development
.notes
.*keys.db
*.db-shm
*.db-wal
.*wallet.sqlite3
*models.json
.cashu
-1
View File
@@ -1 +0,0 @@
3.11
+114 -1
View File
@@ -9,6 +9,7 @@ from pydantic import BaseModel
from sqlmodel import select
from ..payment.models import Model, get_model_by_id, list_models
from ..payment.models import sync_models_with_api
from ..wallet import (
fetch_all_balances,
get_proofs_per_mint_and_unit,
@@ -501,6 +502,64 @@ async def dashboard(request: Request) -> str:
window.location.href = `/admin/logs/${requestId}`;
}
function openSyncModelsModal() {
const modal = document.getElementById('sync-models-modal');
modal.style.display = 'block';
}
function closeSyncModelsModal() {
const modal = document.getElementById('sync-models-modal');
modal.style.display = 'none';
}
async function syncModels() {
const deleteRemoved = document.getElementById('delete-removed-models').checked;
const button = document.getElementById('sync-models-btn');
const resultDiv = document.getElementById('sync-models-result');
button.disabled = true;
button.textContent = 'Syncing...';
resultDiv.style.display = 'none';
try {
const response = await fetch('/admin/api/sync_models', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'same-origin',
body: JSON.stringify({
delete_removed: deleteRemoved
})
});
if (response.ok) {
const data = await response.json();
resultDiv.innerHTML = `<strong>✅ Success!</strong><br>${data.message}`;
resultDiv.style.display = 'block';
resultDiv.style.backgroundColor = '#d4edda';
resultDiv.style.borderColor = '#c3e6cb';
resultDiv.style.color = '#155724';
} else {
const errorData = await response.json();
resultDiv.innerHTML = `<strong>❌ Error:</strong><br>${errorData.detail || 'Unknown error'}`;
resultDiv.style.display = 'block';
resultDiv.style.backgroundColor = '#f8d7da';
resultDiv.style.borderColor = '#f5c6cb';
resultDiv.style.color = '#721c24';
}
} catch (error) {
resultDiv.innerHTML = `<strong>❌ Error:</strong><br>${error.message}`;
resultDiv.style.display = 'block';
resultDiv.style.backgroundColor = '#f8d7da';
resultDiv.style.borderColor = '#f5c6cb';
resultDiv.style.color = '#721c24';
} finally {
button.disabled = false;
button.textContent = 'Sync Models';
}
}
async function openSettingsModal() {
const modal = document.getElementById('settings-modal');
const textarea = document.getElementById('settings-json');
@@ -582,12 +641,15 @@ async def dashboard(request: Request) -> str:
const withdrawModal = document.getElementById('withdraw-modal');
const investigateModal = document.getElementById('investigate-modal');
const settingsModal = document.getElementById('settings-modal');
const syncModelsModal = document.getElementById('sync-models-modal');
if (event.target == withdrawModal) {
closeWithdrawModal();
} else if (event.target == investigateModal) {
closeInvestigateModal();
} else if (event.target == settingsModal) {
closeSettingsModal();
} else if (event.target == syncModelsModal) {
closeSyncModelsModal();
}
}
</script>
@@ -619,6 +681,29 @@ async def dashboard(request: Request) -> str:
<button onclick="openSettingsModal()">
⚙️ Settings
</button>
<button onclick="openSyncModelsModal()">
🔄 Sync Models
</button>
<div id="sync-models-modal" class="modal">
<div class="modal-content">
<span class="close" onclick="closeSyncModelsModal()">&times;</span>
<h3>Sync Models from API</h3>
<p>This will fetch the latest models from OpenRouter and update the database.</p>
<div style="margin: 15px 0;">
<label style="display: flex; align-items: center; gap: 10px; cursor: pointer;">
<input type="checkbox" id="delete-removed-models" style="width: auto; margin: 0;">
<span>Delete models that are no longer available</span>
</label>
<p style="font-size: 0.85rem; color: #718096; margin-top: 5px; margin-left: 30px;">
⚠️ Warning: This will permanently remove models that are no longer in the API.
</p>
</div>
<div id="sync-models-result" style="display: none; padding: 12px; border-radius: 6px; margin: 15px 0; border: 1px solid;"></div>
<button id="sync-models-btn" onclick="syncModels()">🔄 Sync Models</button>
<button onclick="closeSyncModelsModal()" style="background-color: #718096;">Cancel</button>
</div>
</div>
<div id="withdraw-modal" class="modal">
<div class="modal-content">
@@ -808,7 +893,6 @@ async def view_logs(request: Request, request_id: str) -> str:
async def withdraw(
request: Request, withdraw_request: WithdrawRequest
) -> dict[str, str]:
# Get wallet and check balance
from .settings import settings as global_settings
wallet = await get_wallet(
@@ -836,6 +920,35 @@ async def withdraw(
)
return {"token": token}
class SyncModelsRequest(BaseModel):
delete_removed: bool = False
@admin_router.post("/api/sync_models", dependencies=[Depends(require_admin_api)])
async def sync_models(request: Request, sync_request: SyncModelsRequest) -> dict:
try:
src = settings.source or None
source_filter = src if src and src.strip() else None
except Exception:
source_filter = None
logger.info(
"Manual models sync triggered",
extra={"delete_removed": sync_request.delete_removed},
)
counts = await sync_models_with_api(
source_filter=source_filter, delete_removed=sync_request.delete_removed
)
return {
"success": True,
"inserted": counts["inserted"],
"updated": counts["updated"],
"deleted": counts["deleted"],
"message": f"Synced: {counts['inserted']} inserted, {counts['updated']} updated, {counts['deleted']} deleted",
}
DASHBOARD_MODELS_JS: str = """<!--html-->
<script>
-2
View File
@@ -79,8 +79,6 @@ async def balances_for_mint_and_unit(
async def init_db() -> None:
"""Initializes the database and creates tables if they don't exist."""
async with engine.begin() as conn:
if DATABASE_URL.startswith("sqlite"):
await conn.exec_driver_sql("PRAGMA journal_mode=WAL")
await conn.run_sync(SQLModel.metadata.create_all)
+2 -7
View File
@@ -1,5 +1,4 @@
import asyncio
import os
from contextlib import asynccontextmanager
from typing import AsyncGenerator
@@ -31,10 +30,7 @@ from .settings import settings as global_settings
setup_logging()
logger = get_logger(__name__)
if os.getenv("VERSION_SUFFIX") is not None:
__version__ = f"0.1.4-{os.getenv('VERSION_SUFFIX')}"
else:
__version__ = "0.1.4"
__version__ = "0.1.4-dev"
@asynccontextmanager
@@ -71,8 +67,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
await ensure_models_bootstrapped()
pricing_task = asyncio.create_task(update_sats_pricing())
if global_settings.models_refresh_interval_seconds > 0:
models_refresh_task = asyncio.create_task(refresh_models_periodically())
models_refresh_task = asyncio.create_task(refresh_models_periodically())
payout_task = asyncio.create_task(periodic_payout())
nip91_task = asyncio.create_task(announce_provider())
providers_task = asyncio.create_task(providers_cache_refresher())
+3 -2
View File
@@ -64,10 +64,11 @@ class Settings(BaseSettings):
default=120, env="PRICING_REFRESH_INTERVAL_SECONDS"
)
models_refresh_interval_seconds: int = Field(
default=360, env="MODELS_REFRESH_INTERVAL_SECONDS"
default=30, env="MODELS_REFRESH_INTERVAL_SECONDS"
)
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
delete_removed_models: bool = Field(default=False, env="DELETE_REMOVED_MODELS")
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
# Logging
@@ -234,7 +235,7 @@ class SettingsService:
merged_dict: dict[str, Any] = dict(env_resolved.dict())
merged_dict.update(
{k: v for k, v in db_json.items() if v not in (None, "", []) and v}
{k: v for k, v in db_json.items() if v not in (None, "")}
)
# Ensure primary_mint is consistent with cashu_mints if not explicitly set
+15 -11
View File
@@ -11,7 +11,7 @@ from ..core import get_logger
from ..core.db import ModelRow
from ..core.settings import settings
from ..wallet import deserialize_token_from_string
from .models import Pricing
from .models import Pricing, compute_effective_max_cost_msats
logger = get_logger(__name__)
@@ -72,7 +72,8 @@ def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> N
token_obj.amount if token_obj.unit == "msat" else token_obj.amount * 1000
)
if max_cost_for_model > amount_msat:
fee_buffer = 60
if max_cost_for_model > amount_msat + fee_buffer:
raise HTTPException(
status_code=413,
detail={
@@ -133,16 +134,17 @@ async def get_max_cost_for_model(
row = await session.get(ModelRow, model)
if row and row.sats_pricing:
try:
sats = Pricing(**json.loads(row.sats_pricing)) # type: ignore
max_cost = sats.max_cost * 1000 * (1 - settings.tolerance_percentage / 100)
logger.debug(
"Found model-specific max cost",
extra={"model": model, "max_cost_msats": max_cost},
)
calculated_msats = int(max_cost)
return max(settings.min_request_msat, calculated_msats)
sats_dict = json.loads(row.sats_pricing)
except Exception:
pass
sats_dict = None
if isinstance(sats_dict, dict):
effective_msats = compute_effective_max_cost_msats(sats_dict)
if effective_msats > 0:
logger.debug(
"Found model-specific max cost",
extra={"model": model, "max_cost_msats": effective_msats},
)
return effective_msats
logger.warning(
"Model pricing not found, using fixed cost",
@@ -201,6 +203,8 @@ async def calculate_discounted_max_cost(
else:
adjusted = adjusted + math.ceil(-estimated_completion_delta_sats * 1000)
adjusted = min(max_cost_for_model, adjusted)
logger.debug(
"Discounted max cost computed",
extra={
+122 -52
View File
@@ -58,6 +58,39 @@ class Model(BaseModel):
top_provider: TopProvider | None = None
def compute_effective_max_cost_msats(
pricing: Pricing | dict | None,
) -> int:
if pricing is None:
try:
return max(1, int(settings.min_request_msat))
except Exception:
return 1
pricing_obj = (
pricing if isinstance(pricing, Pricing) else Pricing.parse_obj(pricing) # type: ignore[arg-type]
)
try:
tolerance = float(getattr(settings, "tolerance_percentage", 0.0))
except Exception:
tolerance = 0.0
tolerance_factor = max(0.0, 1.0 - tolerance / 100.0)
try:
min_request_msat = max(1, int(settings.min_request_msat))
except Exception:
min_request_msat = 1
base_msats = int(float(pricing_obj.max_cost or 0.0) * 1000.0 * tolerance_factor)
if base_msats <= 0:
return min_request_msat
return max(min_request_msat, base_msats)
def fetch_openrouter_models(source_filter: str | None = None) -> list[dict]:
"""Fetches model information from OpenRouter API."""
base_url = "https://openrouter.ai/api/v1"
@@ -177,6 +210,10 @@ def _row_to_model(row: ModelRow) -> Model:
except Exception:
pass
if isinstance(sats_pricing, dict):
effective_msats = compute_effective_max_cost_msats(sats_pricing)
sats_pricing["max_cost"] = effective_msats / 1000.0
return Model(
id=row.id,
name=row.name,
@@ -192,29 +229,6 @@ def _row_to_model(row: ModelRow) -> Model:
def _model_to_row_payload(model: Model) -> dict[str, str | int | None]:
# Apply fees to USD pricing when storing in database
exchange_fee = settings.exchange_fee
upstream_provider_fee = settings.upstream_provider_fee
total_fee_multiplier = exchange_fee * upstream_provider_fee
# Create adjusted pricing with fees applied
adjusted_pricing = model.pricing.dict()
for key in [
"prompt",
"completion",
"request",
"image",
"web_search",
"internal_reasoning",
]:
if key in adjusted_pricing:
adjusted_pricing[key] = adjusted_pricing[key] * total_fee_multiplier
# Also adjust max costs if present
for key in ["max_prompt_cost", "max_completion_cost", "max_cost"]:
if key in adjusted_pricing:
adjusted_pricing[key] = adjusted_pricing[key] * total_fee_multiplier
return {
"id": model.id,
"name": model.name,
@@ -222,7 +236,7 @@ def _model_to_row_payload(model: Model) -> dict[str, str | int | None]:
"description": model.description,
"context_length": model.context_length,
"architecture": json.dumps(model.architecture.dict()),
"pricing": json.dumps(adjusted_pricing),
"pricing": json.dumps(model.pricing.dict()),
"sats_pricing": json.dumps(model.sats_pricing.dict())
if model.sats_pricing
else None,
@@ -423,11 +437,76 @@ async def update_sats_pricing() -> None:
break
async def sync_models_with_api(
source_filter: str | None = None, delete_removed: bool = False
) -> dict[str, int]:
"""Fetch models from OpenRouter and sync with database.
Args:
source_filter: Optional source filter (e.g., 'anthropic')
delete_removed: If True, delete models that no longer exist in API
Returns:
Dict with counts: inserted, updated, deleted
"""
models = fetch_openrouter_models(source_filter=source_filter)
if not models:
return {"inserted": 0, "updated": 0, "deleted": 0}
async with create_session() as s:
result = await s.exec(select(ModelRow)) # type: ignore
existing_rows = {row.id: row for row in result.all()}
fetched_ids = set()
inserted = 0
updated = 0
for m in models:
try:
model = Model(**m) # type: ignore
except Exception:
continue
fetched_ids.add(model.id)
payload = _model_to_row_payload(model)
if model.id not in existing_rows:
try:
s.add(ModelRow(**payload)) # type: ignore
inserted += 1
except Exception:
pass
else:
existing_row = existing_rows[model.id]
changed = False
for key, value in payload.items():
if getattr(existing_row, key) != value:
setattr(existing_row, key, value)
changed = True
if changed:
s.add(existing_row)
updated += 1
deleted = 0
if delete_removed:
for existing_id in existing_rows:
if existing_id not in fetched_ids:
row_to_delete = existing_rows[existing_id]
await s.delete(row_to_delete)
deleted += 1
if inserted or updated or deleted:
await s.commit()
return {"inserted": inserted, "updated": updated, "deleted": deleted}
async def refresh_models_periodically() -> None:
"""Background task: periodically fetch OpenRouter models and insert new ones.
"""Background task: periodically fetch OpenRouter models and sync with database.
- Respects optional SOURCE filter from settings
- Does not overwrite existing rows
- Updates existing models with new information
- Optionally deletes models no longer in API (if settings.delete_removed_models)
- Sleeps according to settings.models_refresh_interval_seconds; disabled when 0
"""
interval = getattr(settings, "models_refresh_interval_seconds", 0)
@@ -452,33 +531,24 @@ async def refresh_models_periodically() -> None:
except Exception:
source_filter = None
models = fetch_openrouter_models(source_filter=source_filter)
if not models:
await asyncio.sleep(interval)
continue
try:
delete_removed = getattr(settings, "delete_removed_models", False)
except Exception:
delete_removed = False
async with create_session() as s:
result = await s.exec(select(ModelRow.id)) # type: ignore
existing_ids = {
row[0] if isinstance(row, tuple) else row for row in result.all()
}
inserted = 0
for m in models:
try:
model = Model(**m) # type: ignore
except Exception:
continue
if model.id in existing_ids:
continue
payload = _model_to_row_payload(model)
try:
s.add(ModelRow(**payload)) # type: ignore
except Exception:
pass
inserted += 1
if inserted:
await s.commit()
logger.info(f"Inserted {inserted} new models from OpenRouter")
counts = await sync_models_with_api(
source_filter=source_filter, delete_removed=delete_removed
)
if counts["inserted"] or counts["updated"] or counts["deleted"]:
logger.info(
"Models synced",
extra={
"inserted": counts["inserted"],
"updated": counts["updated"],
"deleted": counts["deleted"],
},
)
except asyncio.CancelledError:
break
except Exception as e:
+8 -11
View File
@@ -46,7 +46,7 @@ async def x_cashu_handler(
)
return await forward_to_upstream(
request, path, headers, amount, unit, max_cost_for_model, mint
request, path, headers, amount, unit, max_cost_for_model
)
except Exception as e:
error_message = str(e)
@@ -105,7 +105,6 @@ async def forward_to_upstream(
amount: int,
unit: str,
max_cost_for_model: int,
mint: str,
) -> Response | StreamingResponse:
"""Forward request to upstream and handle the response."""
if path.startswith("v1/"):
@@ -160,7 +159,7 @@ async def forward_to_upstream(
},
)
refund_token = await send_refund(amount - 60, unit, mint)
refund_token = await send_refund(amount - 60, unit)
logger.info(
"Refund processed for failed upstream request",
@@ -198,7 +197,7 @@ async def forward_to_upstream(
)
result = await handle_x_cashu_chat_completion(
response, amount, unit, max_cost_for_model, mint
response, amount, unit, max_cost_for_model
)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
@@ -243,7 +242,7 @@ async def forward_to_upstream(
async def handle_x_cashu_chat_completion(
response: httpx.Response, amount: int, unit: str, max_cost_for_model: int, mint: str
response: httpx.Response, amount: int, unit: str, max_cost_for_model: int
) -> StreamingResponse | Response:
"""Handle both streaming and non-streaming chat completion responses with token-based pricing."""
logger.debug(
@@ -268,11 +267,11 @@ async def handle_x_cashu_chat_completion(
if is_streaming:
return await handle_streaming_response(
content_str, response, amount, unit, max_cost_for_model, mint
content_str, response, amount, unit, max_cost_for_model
)
else:
return await handle_non_streaming_response(
content_str, response, amount, unit, max_cost_for_model, mint
content_str, response, amount, unit, max_cost_for_model
)
except Exception as e:
@@ -299,7 +298,6 @@ async def handle_streaming_response(
amount: int,
unit: str,
max_cost_for_model: int,
mint: str,
) -> StreamingResponse:
"""Handle Server-Sent Events (SSE) streaming response."""
logger.debug(
@@ -374,7 +372,7 @@ async def handle_streaming_response(
},
)
refund_token = await send_refund(refund_amount, unit, mint)
refund_token = await send_refund(refund_amount, unit)
response_headers["X-Cashu"] = refund_token
logger.info(
@@ -426,7 +424,6 @@ async def handle_non_streaming_response(
amount: int,
unit: str,
max_cost_for_model: int,
mint: str,
) -> Response:
"""Handle regular JSON response."""
logger.debug(
@@ -487,7 +484,7 @@ async def handle_non_streaming_response(
)
if refund_amount > 0:
refund_token = await send_refund(refund_amount, unit, mint)
refund_token = await send_refund(refund_amount, unit)
response_headers["X-Cashu"] = refund_token
logger.info(
-6
View File
@@ -636,12 +636,6 @@ async def proxy(
if request_body:
try:
request_body_dict = json.loads(request_body)
if "max_tokens" in request_body_dict:
raise HTTPException(
status_code=400,
detail={"error": "max_tokens must be an integer (without quotes)"},
)
logger.debug(
"Request body parsed",
extra={
@@ -549,9 +549,9 @@ class TestPerformance:
max_time = max(times)
# Average should be well under 100ms
assert (
avg_time < 100
), f"{op_type} average time {avg_time}ms exceeds 100ms"
assert avg_time < 100, (
f"{op_type} average time {avg_time}ms exceeds 100ms"
)
# No single operation should exceed 200ms
assert max_time < 200, f"{op_type} max time {max_time}ms exceeds 200ms"
@@ -359,15 +359,15 @@ async def test_all_info_endpoints_no_database_changes(
# Check no database changes after each request
diff = await db_snapshot.diff()
assert (
len(diff["api_keys"]["added"]) == 0
), f"Endpoint {endpoint} added API keys"
assert (
len(diff["api_keys"]["removed"]) == 0
), f"Endpoint {endpoint} removed API keys"
assert (
len(diff["api_keys"]["modified"]) == 0
), f"Endpoint {endpoint} modified API keys"
assert len(diff["api_keys"]["added"]) == 0, (
f"Endpoint {endpoint} added API keys"
)
assert len(diff["api_keys"]["removed"]) == 0, (
f"Endpoint {endpoint} removed API keys"
)
assert len(diff["api_keys"]["modified"]) == 0, (
f"Endpoint {endpoint} modified API keys"
)
# Final verification - database state should be identical
final_state = await db_snapshot.capture()
+27 -27
View File
@@ -131,9 +131,9 @@ class TestPerformanceBaseline:
# Verify 95th percentile < 500ms
p95 = sorted(response_times)[int(len(response_times) * 0.95)]
assert (
p95 < 500
), f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
assert p95 < 500, (
f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
)
print(f"\n{method} {path}:")
print(f" Mean: {statistics.mean(response_times):.2f}ms")
@@ -175,9 +175,9 @@ class TestPerformanceBaseline:
query_times.append(duration)
# All queries should complete < 100ms
assert (
max(query_times) < 100
), f"Max query time {max(query_times)}ms exceeds 100ms limit"
assert max(query_times) < 100, (
f"Max query time {max(query_times)}ms exceeds 100ms limit"
)
print("\nDatabase query performance:")
print(f" Mean: {statistics.mean(query_times):.2f}ms")
print(f" Max: {max(query_times):.2f}ms")
@@ -272,9 +272,9 @@ class TestLoadScenarios:
# Performance requirements
assert summary["error_rate"] < 0.05, "Error rate exceeds 5%"
assert (
summary["response_times"]["p95"] < 2.0
), "P95 response time exceeds 2 seconds"
assert summary["response_times"]["p95"] < 2.0, (
"P95 response time exceeds 2 seconds"
)
@pytest.mark.asyncio
async def test_sustained_load_1000_rpm(
@@ -354,13 +354,13 @@ class TestLoadScenarios:
)
# Verify performance
assert (
actual_rps >= target_rps * 0.95
), f"Could not sustain target rate (achieved {actual_rps:.2f} req/s)"
assert actual_rps >= target_rps * 0.95, (
f"Could not sustain target rate (achieved {actual_rps:.2f} req/s)"
)
assert summary["error_rate"] < 0.01, "Error rate exceeds 1%"
assert (
summary["response_times"]["p95"] < 1.0
), "P95 response time exceeds 1 second"
assert summary["response_times"]["p95"] < 1.0, (
"P95 response time exceeds 1 second"
)
@pytest.mark.integration
@@ -418,12 +418,12 @@ class TestMemoryLeaks:
# Check for significant memory leaks
# Allow some growth but not more than 20% or 50MB total
assert (
memory_growth < 50
), f"Memory grew by {memory_growth} MB, indicating a potential leak"
assert (
memory_samples[-1] < memory_samples[0] * 1.2
), "Memory grew by more than 20%"
assert memory_growth < 50, (
f"Memory grew by {memory_growth} MB, indicating a potential leak"
)
assert memory_samples[-1] < memory_samples[0] * 1.2, (
"Memory grew by more than 20%"
)
@pytest.mark.integration
@@ -471,9 +471,9 @@ class TestPerformanceRegression:
print(f" Current: {mean_time * 1000:.1f}ms")
print(f" Difference: {((mean_time / baseline - 1) * 100):.1f}%")
assert (
mean_time <= max_allowed
), f"{endpoint} performance degraded by more than 20% (baseline: {baseline}s, current: {mean_time}s)"
assert mean_time <= max_allowed, (
f"{endpoint} performance degraded by more than 20% (baseline: {baseline}s, current: {mean_time}s)"
)
# Get overall validation results
results = {}
@@ -482,9 +482,9 @@ class TestPerformanceRegression:
endpoint, max_duration=baselines[endpoint] * 1.2, percentile=0.95
)
results[endpoint] = result
assert result[
"valid"
], f"Performance validation failed for {endpoint}: {result}"
assert result["valid"], (
f"Performance validation failed for {endpoint}: {result}"
)
# Performance test utilities
@@ -42,12 +42,12 @@ async def test_reserved_balance_never_negative(integration_client: AsyncClient)
async with create_session() as session:
key = await session.get(ApiKey, "test_reserved_balance_key")
assert key is not None
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative: {key.reserved_balance}"
assert (
key.balance == 1000
), "Balance should remain unchanged after failed request"
assert key.reserved_balance >= 0, (
f"Reserved balance went negative: {key.reserved_balance}"
)
assert key.balance == 1000, (
"Balance should remain unchanged after failed request"
)
# Test 2: Simulate concurrent failed requests
# This tests the race condition protection
@@ -71,9 +71,9 @@ async def test_reserved_balance_never_negative(integration_client: AsyncClient)
async with create_session() as session:
key = await session.get(ApiKey, "test_reserved_balance_key")
assert key is not None
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative after concurrent requests: {key.reserved_balance}"
assert key.reserved_balance >= 0, (
f"Reserved balance went negative after concurrent requests: {key.reserved_balance}"
)
print(f"Final state - Balance: {key.balance}, Reserved: {key.reserved_balance}")
@@ -113,20 +113,20 @@ async def test_reserved_balance_with_successful_requests(
async with create_session() as session:
key = await session.get(ApiKey, unique_key)
assert key is not None
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative: {key.reserved_balance}"
assert key.reserved_balance >= 0, (
f"Reserved balance went negative: {key.reserved_balance}"
)
# Check if the request was processed (might fail due to model pricing in test env)
# The important part is that reserved_balance doesn't go negative
if key.total_spent > 0:
assert (
key.balance < 100000
), "Balance should decrease after successful request"
assert key.balance < 100000, (
"Balance should decrease after successful request"
)
else:
# Request failed, but reserved balance should still be non-negative
assert (
key.balance == 100000
), "Balance should remain unchanged if request failed"
assert key.balance == 100000, (
"Balance should remain unchanged if request failed"
)
print(
f"After successful request - Balance: {key.balance}, Reserved: {key.reserved_balance}, Spent: {key.total_spent}"
)
@@ -157,9 +157,9 @@ async def test_insufficient_reserved_balance_for_revert(
await integration_session.refresh(test_key)
# Current implementation allows negative reserved balance
assert (
test_key.reserved_balance == -100
), f"Expected reserved_balance to be -100, got: {test_key.reserved_balance}"
assert (
test_key.total_requests == -1
), f"Expected total_requests to be -1, got: {test_key.total_requests}"
assert test_key.reserved_balance == -100, (
f"Expected reserved_balance to be -100, got: {test_key.reserved_balance}"
)
assert test_key.total_requests == -1, (
f"Expected total_requests to be -1, got: {test_key.total_requests}"
)
@@ -94,9 +94,9 @@ async def test_api_key_generation_invalid_token(
response = await integration_client.get("/v1/wallet/info")
# Should fail with 401
assert (
response.status_code == 401
), f"Token {invalid_token[:20]}... should be invalid"
assert response.status_code == 401, (
f"Token {invalid_token[:20]}... should be invalid"
)
# Validate error response
validator = ResponseValidator()
@@ -198,9 +198,9 @@ async def test_authorization_header_validation(
# Make request to protected endpoint
response = await integration_client.get("/v1/wallet/")
assert (
response.status_code == expected_status
), f"{description}: Expected {expected_status}, got {response.status_code}"
assert response.status_code == expected_status, (
f"{description}: Expected {expected_status}, got {response.status_code}"
)
if expected_status == 401:
assert "detail" in response.json()
+3 -3
View File
@@ -133,9 +133,9 @@ async def test_topup_with_invalid_token(
)
# Should fail with 400
assert (
response.status_code == 400
), f"Token {invalid_token[:20]}... should be invalid"
assert response.status_code == 400, (
f"Token {invalid_token[:20]}... should be invalid"
)
# Validate error response
validator = ResponseValidator()
-781
View File
@@ -1,781 +0,0 @@
"""Unit tests for upstream provider fee application to USD pricing.
This module tests the fix for issue #188: "Upstream provider fee not being applied
to USD pricing (only sats)". The fix ensures that both exchange_fee and
upstream_provider_fee are correctly applied to USD pricing when models are stored
in the database via the _model_to_row_payload function.
Key behaviors tested:
1. exchange_fee is applied to all USD pricing fields
2. upstream_provider_fee is applied to all USD pricing fields
3. Both fees are compounded correctly (multiplied together)
4. Fees apply to all pricing attributes (prompt, completion, request, etc.)
5. Fees apply to max cost fields (max_prompt_cost, max_completion_cost, max_cost)
6. sats_pricing remains unaffected (fees not double-applied)
7. Zero-value pricing fields are handled correctly
8. Edge cases and boundary conditions are properly handled
"""
import json
import os
from unittest.mock import patch
import pytest
# Set required env vars before importing
os.environ["UPSTREAM_BASE_URL"] = "http://test"
os.environ["UPSTREAM_API_KEY"] = "test"
from routstr.core.settings import settings # noqa: E402
from routstr.payment.models import ( # noqa: E402
Architecture,
Model,
Pricing,
_model_to_row_payload,
)
@pytest.fixture
def base_architecture() -> Architecture:
"""Provide standard architecture for test models."""
return Architecture(
modality="text",
input_modalities=["text"],
output_modalities=["text"],
tokenizer="gpt",
instruct_type="chat",
)
@pytest.fixture
def standard_pricing() -> Pricing:
"""Provide standard USD pricing with known values for testing."""
return Pricing(
prompt=0.001, # $0.001 per prompt token
completion=0.002, # $0.002 per completion token
request=0.01, # $0.01 per request
image=0.05, # $0.05 per image
web_search=0.03, # $0.03 per web search
internal_reasoning=0.015, # $0.015 per internal reasoning token
max_prompt_cost=10.0, # $10 max prompt cost
max_completion_cost=20.0, # $20 max completion cost
max_cost=30.0, # $30 max total cost
)
@pytest.fixture
def standard_model(base_architecture: Architecture, standard_pricing: Pricing) -> Model:
"""Create a standard test model with known pricing."""
return Model(
id="test-model-standard",
name="Test Model Standard",
created=1234567890,
description="A standard test model for fee application",
context_length=8192,
architecture=base_architecture,
pricing=standard_pricing,
)
# =============================================================================
# Individual Fee Application Tests
# =============================================================================
def test_exchange_fee_applied_to_usd_pricing(standard_model: Model) -> None:
"""Verify exchange_fee is applied to all USD pricing fields."""
exchange_fee = 1.005 # 0.5% fee
upstream_fee = 1.0 # No upstream fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Verify all pricing fields have exchange fee applied
assert pricing["prompt"] == pytest.approx(0.001 * exchange_fee, rel=1e-9)
assert pricing["completion"] == pytest.approx(
0.002 * exchange_fee, rel=1e-9
)
assert pricing["request"] == pytest.approx(0.01 * exchange_fee, rel=1e-9)
assert pricing["image"] == pytest.approx(0.05 * exchange_fee, rel=1e-9)
assert pricing["web_search"] == pytest.approx(0.03 * exchange_fee, rel=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(
0.015 * exchange_fee, rel=1e-9
)
# Verify max cost fields have exchange fee applied
assert pricing["max_prompt_cost"] == pytest.approx(
10.0 * exchange_fee, rel=1e-9
)
assert pricing["max_completion_cost"] == pytest.approx(
20.0 * exchange_fee, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(30.0 * exchange_fee, rel=1e-9)
def test_upstream_provider_fee_applied_to_usd_pricing(standard_model: Model) -> None:
"""Verify upstream_provider_fee is applied to all USD pricing fields."""
exchange_fee = 1.0 # No exchange fee
upstream_fee = 1.05 # 5% upstream provider fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Verify all pricing fields have upstream fee applied
assert pricing["prompt"] == pytest.approx(0.001 * upstream_fee, rel=1e-9)
assert pricing["completion"] == pytest.approx(
0.002 * upstream_fee, rel=1e-9
)
assert pricing["request"] == pytest.approx(0.01 * upstream_fee, rel=1e-9)
assert pricing["image"] == pytest.approx(0.05 * upstream_fee, rel=1e-9)
assert pricing["web_search"] == pytest.approx(0.03 * upstream_fee, rel=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(
0.015 * upstream_fee, rel=1e-9
)
# Verify max cost fields have upstream fee applied
assert pricing["max_prompt_cost"] == pytest.approx(
10.0 * upstream_fee, rel=1e-9
)
assert pricing["max_completion_cost"] == pytest.approx(
20.0 * upstream_fee, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(30.0 * upstream_fee, rel=1e-9)
# =============================================================================
# Combined Fee Application Tests (Core Issue #188 Fix)
# =============================================================================
def test_both_fees_compounded_correctly(standard_model: Model) -> None:
"""Test that exchange_fee and upstream_provider_fee are compounded (multiplied).
This is the PRIMARY test for issue #188. Before the fix, only exchange_fee was
applied to USD pricing. The fix ensures both fees are compounded correctly.
"""
exchange_fee = 1.005 # 0.5% exchange fee
upstream_fee = 1.05 # 5% upstream provider fee
expected_multiplier = exchange_fee * upstream_fee # 1.05525
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# All pricing fields should be multiplied by the combined fee
assert pricing["prompt"] == pytest.approx(
0.001 * expected_multiplier, rel=1e-9
)
assert pricing["completion"] == pytest.approx(
0.002 * expected_multiplier, rel=1e-9
)
assert pricing["request"] == pytest.approx(
0.01 * expected_multiplier, rel=1e-9
)
assert pricing["image"] == pytest.approx(
0.05 * expected_multiplier, rel=1e-9
)
assert pricing["web_search"] == pytest.approx(
0.03 * expected_multiplier, rel=1e-9
)
assert pricing["internal_reasoning"] == pytest.approx(
0.015 * expected_multiplier, rel=1e-9
)
# Max cost fields should also have combined fee applied
assert pricing["max_prompt_cost"] == pytest.approx(
10.0 * expected_multiplier, rel=1e-9
)
assert pricing["max_completion_cost"] == pytest.approx(
20.0 * expected_multiplier, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(
30.0 * expected_multiplier, rel=1e-9
)
def test_default_fee_values_from_settings(standard_model: Model) -> None:
"""Test with actual default fee values from settings.
Default values per routstr/core/settings.py:
- exchange_fee: 1.005 (0.5%)
- upstream_provider_fee: 1.05 (5%)
Combined: 1.05525 (5.525% total fee)
"""
# Use actual default values (don't mock)
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Calculate expected multiplier with production defaults
default_exchange_fee = 1.005
default_upstream_fee = 1.05
expected_multiplier = default_exchange_fee * default_upstream_fee # 1.05525
# Verify pricing is higher than original due to fees
assert pricing["prompt"] > standard_model.pricing.prompt
assert pricing["completion"] > standard_model.pricing.completion
assert pricing["request"] > standard_model.pricing.request
# Verify exact values with default fees
assert pricing["prompt"] == pytest.approx(0.001 * expected_multiplier, rel=1e-9)
assert pricing["completion"] == pytest.approx(0.002 * expected_multiplier, rel=1e-9)
assert pricing["request"] == pytest.approx(0.01 * expected_multiplier, rel=1e-9)
assert pricing["image"] == pytest.approx(0.05 * expected_multiplier, rel=1e-9)
assert pricing["web_search"] == pytest.approx(0.03 * expected_multiplier, rel=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(
0.015 * expected_multiplier, rel=1e-9
)
# =============================================================================
# Varied Fee Scenarios
# =============================================================================
def test_higher_fee_values(standard_model: Model) -> None:
"""Test with significantly higher fee values to ensure scalability."""
exchange_fee = 1.02 # 2% exchange fee
upstream_fee = 1.15 # 15% upstream provider fee
expected_multiplier = exchange_fee * upstream_fee # 1.173
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
assert pricing["prompt"] == pytest.approx(
0.001 * expected_multiplier, rel=1e-9
)
assert pricing["completion"] == pytest.approx(
0.002 * expected_multiplier, rel=1e-9
)
assert pricing["request"] == pytest.approx(
0.01 * expected_multiplier, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(
30.0 * expected_multiplier, rel=1e-9
)
def test_minimal_fee_values(standard_model: Model) -> None:
"""Test with fees very close to 1.0 (minimal markup)."""
exchange_fee = 1.001 # 0.1% exchange fee
upstream_fee = 1.001 # 0.1% upstream provider fee
expected_multiplier = exchange_fee * upstream_fee # 1.002001
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Verify precise calculation even with small fees
assert pricing["prompt"] == pytest.approx(
0.001 * expected_multiplier, rel=1e-9
)
assert pricing["completion"] == pytest.approx(
0.002 * expected_multiplier, rel=1e-9
)
def test_no_fees_applied(standard_model: Model) -> None:
"""Test with both fees set to 1.0 (no markup)."""
exchange_fee = 1.0 # No fee
upstream_fee = 1.0 # No fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Prices should remain unchanged
assert pricing["prompt"] == pytest.approx(0.001, rel=1e-9)
assert pricing["completion"] == pytest.approx(0.002, rel=1e-9)
assert pricing["request"] == pytest.approx(0.01, rel=1e-9)
assert pricing["max_cost"] == pytest.approx(30.0, rel=1e-9)
# =============================================================================
# Zero and Edge Case Pricing Tests
# =============================================================================
def test_zero_value_pricing_fields(base_architecture: Architecture) -> None:
"""Verify that zero-value pricing fields are handled correctly.
Zero values should remain zero after fee application (0 * multiplier = 0).
"""
zero_pricing = Pricing(
prompt=0.0,
completion=0.0,
request=0.0,
image=0.0,
web_search=0.0,
internal_reasoning=0.0,
max_prompt_cost=0.0,
max_completion_cost=0.0,
max_cost=0.0,
)
model = Model(
id="test-zero-pricing",
name="Zero Pricing Model",
created=1234567890,
description="Model with all zero pricing",
context_length=8192,
architecture=base_architecture,
pricing=zero_pricing,
)
exchange_fee = 1.005
upstream_fee = 1.05
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# All values should remain zero
assert pricing["prompt"] == pytest.approx(0.0, abs=1e-9)
assert pricing["completion"] == pytest.approx(0.0, abs=1e-9)
assert pricing["request"] == pytest.approx(0.0, abs=1e-9)
assert pricing["image"] == pytest.approx(0.0, abs=1e-9)
assert pricing["web_search"] == pytest.approx(0.0, abs=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(0.0, abs=1e-9)
assert pricing["max_prompt_cost"] == pytest.approx(0.0, abs=1e-9)
assert pricing["max_completion_cost"] == pytest.approx(0.0, abs=1e-9)
assert pricing["max_cost"] == pytest.approx(0.0, abs=1e-9)
def test_mixed_zero_and_nonzero_pricing(base_architecture: Architecture) -> None:
"""Test models with some zero and some non-zero pricing fields."""
mixed_pricing = Pricing(
prompt=0.001, # Non-zero
completion=0.002, # Non-zero
request=0.0, # Zero
image=0.0, # Zero
web_search=0.03, # Non-zero
internal_reasoning=0.0, # Zero
max_prompt_cost=10.0, # Non-zero
max_completion_cost=0.0, # Zero
max_cost=15.0, # Non-zero
)
model = Model(
id="test-mixed-pricing",
name="Mixed Pricing Model",
created=1234567890,
description="Model with mixed zero/non-zero pricing",
context_length=8192,
architecture=base_architecture,
pricing=mixed_pricing,
)
exchange_fee = 1.005
upstream_fee = 1.05
expected_multiplier = exchange_fee * upstream_fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Non-zero values should have fees applied
assert pricing["prompt"] == pytest.approx(
0.001 * expected_multiplier, rel=1e-9
)
assert pricing["completion"] == pytest.approx(
0.002 * expected_multiplier, rel=1e-9
)
assert pricing["web_search"] == pytest.approx(
0.03 * expected_multiplier, rel=1e-9
)
assert pricing["max_prompt_cost"] == pytest.approx(
10.0 * expected_multiplier, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(
15.0 * expected_multiplier, rel=1e-9
)
# Zero values should remain zero
assert pricing["request"] == pytest.approx(0.0, abs=1e-9)
assert pricing["image"] == pytest.approx(0.0, abs=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(0.0, abs=1e-9)
assert pricing["max_completion_cost"] == pytest.approx(0.0, abs=1e-9)
def test_very_small_pricing_values(base_architecture: Architecture) -> None:
"""Test with very small pricing values to verify precision."""
tiny_pricing = Pricing(
prompt=0.000001, # $0.000001 per token
completion=0.000002,
request=0.00001,
image=0.0001,
web_search=0.0001,
internal_reasoning=0.000001,
max_prompt_cost=0.01,
max_completion_cost=0.02,
max_cost=0.03,
)
model = Model(
id="test-tiny-pricing",
name="Tiny Pricing Model",
created=1234567890,
description="Model with very small pricing values",
context_length=8192,
architecture=base_architecture,
pricing=tiny_pricing,
)
exchange_fee = 1.005
upstream_fee = 1.05
expected_multiplier = exchange_fee * upstream_fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Verify precision is maintained for very small values
assert pricing["prompt"] == pytest.approx(
0.000001 * expected_multiplier, rel=1e-6
)
assert pricing["completion"] == pytest.approx(
0.000002 * expected_multiplier, rel=1e-6
)
assert pricing["request"] == pytest.approx(
0.00001 * expected_multiplier, rel=1e-6
)
def test_very_large_pricing_values(base_architecture: Architecture) -> None:
"""Test with very large pricing values to ensure no overflow."""
large_pricing = Pricing(
prompt=100.0,
completion=200.0,
request=500.0,
image=1000.0,
web_search=750.0,
internal_reasoning=150.0,
max_prompt_cost=100000.0,
max_completion_cost=200000.0,
max_cost=500000.0,
)
model = Model(
id="test-large-pricing",
name="Large Pricing Model",
created=1234567890,
description="Model with very large pricing values",
context_length=8192,
architecture=base_architecture,
pricing=large_pricing,
)
exchange_fee = 1.005
upstream_fee = 1.05
expected_multiplier = exchange_fee * upstream_fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Verify large values are handled correctly
assert pricing["prompt"] == pytest.approx(
100.0 * expected_multiplier, rel=1e-9
)
assert pricing["max_cost"] == pytest.approx(
500000.0 * expected_multiplier, rel=1e-9
)
# =============================================================================
# Sats Pricing Isolation Tests
# =============================================================================
def test_sats_pricing_not_modified(
base_architecture: Architecture, standard_pricing: Pricing
) -> None:
"""Verify that sats_pricing is NOT affected by USD fee application.
This ensures the fix doesn't break existing sats pricing behavior.
The fees should only be applied to USD pricing, not sats pricing.
"""
sats_pricing = Pricing(
prompt=10.0,
completion=20.0,
request=100.0,
image=500.0,
web_search=300.0,
internal_reasoning=150.0,
max_prompt_cost=10000.0,
max_completion_cost=20000.0,
max_cost=30000.0,
)
model = Model(
id="test-with-sats",
name="Model With Sats Pricing",
created=1234567890,
description="Model with both USD and sats pricing",
context_length=8192,
architecture=base_architecture,
pricing=standard_pricing,
sats_pricing=sats_pricing,
)
exchange_fee = 1.005
upstream_fee = 1.05
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(model)
sats_pricing_str = payload["sats_pricing"]
assert isinstance(sats_pricing_str, str)
sats_pricing_result = json.loads(sats_pricing_str)
# Sats pricing should be completely unchanged
assert sats_pricing_result["prompt"] == pytest.approx(10.0, rel=1e-9)
assert sats_pricing_result["completion"] == pytest.approx(20.0, rel=1e-9)
assert sats_pricing_result["request"] == pytest.approx(100.0, rel=1e-9)
assert sats_pricing_result["image"] == pytest.approx(500.0, rel=1e-9)
assert sats_pricing_result["web_search"] == pytest.approx(300.0, rel=1e-9)
assert sats_pricing_result["internal_reasoning"] == pytest.approx(
150.0, rel=1e-9
)
assert sats_pricing_result["max_prompt_cost"] == pytest.approx(
10000.0, rel=1e-9
)
assert sats_pricing_result["max_completion_cost"] == pytest.approx(
20000.0, rel=1e-9
)
assert sats_pricing_result["max_cost"] == pytest.approx(30000.0, rel=1e-9)
def test_model_without_sats_pricing(standard_model: Model) -> None:
"""Test models that don't have sats_pricing (None value)."""
assert standard_model.sats_pricing is None
exchange_fee = 1.005
upstream_fee = 1.05
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
# sats_pricing should remain None
assert payload["sats_pricing"] is None
# =============================================================================
# Payload Structure Tests
# =============================================================================
def test_payload_structure_unchanged(standard_model: Model) -> None:
"""Verify the database row payload structure is not corrupted by the fix."""
with patch.object(settings, "exchange_fee", 1.005):
with patch.object(settings, "upstream_provider_fee", 1.05):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
# Verify all expected keys exist
assert "id" in payload
assert "name" in payload
assert "created" in payload
assert "description" in payload
assert "context_length" in payload
assert "architecture" in payload
assert "pricing" in payload
assert "sats_pricing" in payload
assert "per_request_limits" in payload
assert "top_provider" in payload
# Verify types
assert isinstance(payload["id"], str)
assert isinstance(payload["name"], str)
assert isinstance(payload["created"], int)
assert isinstance(payload["description"], str)
assert isinstance(payload["context_length"], int)
assert isinstance(payload["architecture"], str) # JSON string
assert isinstance(payload["pricing"], str) # JSON string
assert payload["sats_pricing"] is None # None for this test model
# Verify JSON fields can be parsed
architecture_str = payload["architecture"]
assert isinstance(architecture_str, str)
architecture = json.loads(architecture_str)
pricing = json.loads(pricing_str)
assert isinstance(architecture, dict)
assert isinstance(pricing, dict)
# Verify pricing has all expected fields
expected_pricing_keys = {
"prompt",
"completion",
"request",
"image",
"web_search",
"internal_reasoning",
"max_prompt_cost",
"max_completion_cost",
"max_cost",
}
assert set(pricing.keys()) == expected_pricing_keys
def test_all_pricing_fields_present_after_fee_application(
standard_model: Model,
) -> None:
"""Ensure no pricing fields are accidentally dropped during fee application."""
with patch.object(settings, "exchange_fee", 1.005):
with patch.object(settings, "upstream_provider_fee", 1.05):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# All original pricing fields must be present
assert "prompt" in pricing
assert "completion" in pricing
assert "request" in pricing
assert "image" in pricing
assert "web_search" in pricing
assert "internal_reasoning" in pricing
assert "max_prompt_cost" in pricing
assert "max_completion_cost" in pricing
assert "max_cost" in pricing
# No extra fields should be added
assert len(pricing) == 9
# =============================================================================
# Consistency and Regression Tests
# =============================================================================
def test_fee_consistency_across_all_fields(standard_model: Model) -> None:
"""Verify the same fee multiplier is applied consistently to all fields."""
exchange_fee = 1.005
upstream_fee = 1.05
expected_multiplier = exchange_fee * upstream_fee
with patch.object(settings, "exchange_fee", exchange_fee):
with patch.object(settings, "upstream_provider_fee", upstream_fee):
payload = _model_to_row_payload(standard_model)
pricing_str = payload["pricing"]
assert isinstance(pricing_str, str)
pricing = json.loads(pricing_str)
# Calculate actual multipliers for each field
prompt_multiplier = pricing["prompt"] / standard_model.pricing.prompt
completion_multiplier = (
pricing["completion"] / standard_model.pricing.completion
)
request_multiplier = pricing["request"] / standard_model.pricing.request
image_multiplier = pricing["image"] / standard_model.pricing.image
web_search_multiplier = (
pricing["web_search"] / standard_model.pricing.web_search
)
internal_reasoning_multiplier = (
pricing["internal_reasoning"]
/ standard_model.pricing.internal_reasoning
)
max_prompt_multiplier = (
pricing["max_prompt_cost"] / standard_model.pricing.max_prompt_cost
)
max_completion_multiplier = (
pricing["max_completion_cost"]
/ standard_model.pricing.max_completion_cost
)
max_cost_multiplier = pricing["max_cost"] / standard_model.pricing.max_cost
# All multipliers should be identical and equal to expected multiplier
assert prompt_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert completion_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert request_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert image_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert web_search_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert internal_reasoning_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert max_prompt_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert max_completion_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert max_cost_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
def test_multiple_calls_produce_consistent_results(standard_model: Model) -> None:
"""Verify that calling _model_to_row_payload multiple times is idempotent."""
with patch.object(settings, "exchange_fee", 1.005):
with patch.object(settings, "upstream_provider_fee", 1.05):
# Call multiple times
payload1 = _model_to_row_payload(standard_model)
payload2 = _model_to_row_payload(standard_model)
payload3 = _model_to_row_payload(standard_model)
pricing1_str = payload1["pricing"]
pricing2_str = payload2["pricing"]
pricing3_str = payload3["pricing"]
assert isinstance(pricing1_str, str)
assert isinstance(pricing2_str, str)
assert isinstance(pricing3_str, str)
pricing1 = json.loads(pricing1_str)
pricing2 = json.loads(pricing2_str)
pricing3 = json.loads(pricing3_str)
# All results should be identical
assert pricing1 == pricing2
assert pricing2 == pricing3
# Original model should not be mutated
assert standard_model.pricing.prompt == 0.001
assert standard_model.pricing.completion == 0.002
def test_original_model_not_mutated(standard_model: Model) -> None:
"""Ensure the original model object is not modified by fee application."""
original_prompt = standard_model.pricing.prompt
original_completion = standard_model.pricing.completion
original_max_cost = standard_model.pricing.max_cost
with patch.object(settings, "exchange_fee", 1.005):
with patch.object(settings, "upstream_provider_fee", 1.05):
_ = _model_to_row_payload(standard_model)
# Original model should be unchanged
assert standard_model.pricing.prompt == original_prompt
assert standard_model.pricing.completion == original_completion
assert standard_model.pricing.max_cost == original_max_cost
+91 -1
View File
@@ -1,12 +1,20 @@
import os
from unittest.mock import AsyncMock, Mock, patch
import pytest
from fastapi import HTTPException
# Set required env vars before importing
os.environ["UPSTREAM_BASE_URL"] = "http://test"
os.environ["UPSTREAM_API_KEY"] = "test"
from routstr.core.settings import settings # noqa: E402
from routstr.payment.helpers import get_max_cost_for_model # noqa: E402
from routstr.payment.helpers import ( # noqa: E402
calculate_discounted_max_cost,
check_token_balance,
get_max_cost_for_model,
)
from routstr.payment.models import Pricing # noqa: E402
async def test_get_max_cost_for_model_known() -> None:
@@ -73,3 +81,85 @@ async def test_get_max_cost_for_model_tolerance() -> None:
with patch.object(settings, "tolerance_percentage", 10):
cost = await get_max_cost_for_model("gpt-4", session=mock_session)
assert cost == 450000 # 500 sats * 1000 * 0.9 = 450000
async def test_calculate_discounted_max_cost_no_session() -> None:
with patch.object(settings, "fixed_pricing", False):
result = await calculate_discounted_max_cost(123, {}, session=None)
assert result == 123
async def test_calculate_discounted_max_cost_clamped(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mock_session = AsyncMock()
mock_exec_result = Mock()
mock_exec_result.all = Mock(return_value=[("gpt-4",)])
mock_session.exec.return_value = mock_exec_result
pricing = {
"prompt": 0.0,
"completion": 0.0,
"request": 0.0,
"image": 0.0,
"web_search": 0.0,
"internal_reasoning": 0.0,
"max_prompt_cost": 100.0,
"max_completion_cost": 200.0,
"max_cost": 300.0,
}
async def mock_get_model_cost_info(*args, **kwargs): # type: ignore
return Pricing(**pricing)
monkeypatch.setattr(
"routstr.payment.helpers.get_model_cost_info", mock_get_model_cost_info
)
with patch.object(settings, "fixed_pricing", False):
with patch.object(settings, "tolerance_percentage", 0):
result = await calculate_discounted_max_cost(
320000,
{"model": "gpt-4", "max_tokens": 10, "messages": ["one"]},
session=mock_session,
)
assert 0 <= result <= 320000
async def test_check_token_balance_with_fee_buffer(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Token:
unit = "msat"
amount = 320400
def mock_deserialize(_value: str) -> Token:
return Token()
monkeypatch.setattr(
"routstr.payment.helpers.deserialize_token_from_string", mock_deserialize
)
headers = {"x-cashu": "token"}
body = {"model": "gpt-4"}
check_token_balance(headers, body, max_cost_for_model=320450)
def test_check_token_balance_insufficient(monkeypatch: pytest.MonkeyPatch) -> None:
class Token:
unit = "msat"
amount = 320200
def mock_deserialize(_value: str) -> Token:
return Token()
monkeypatch.setattr(
"routstr.payment.helpers.deserialize_token_from_string", mock_deserialize
)
headers = {"x-cashu": "token"}
body = {"model": "gpt-4"}
with pytest.raises(HTTPException) as exc:
check_token_balance(headers, body, max_cost_for_model=320450)
assert exc.value.status_code == 413
detail = exc.value.detail # type: ignore[assignment]
assert isinstance(detail, dict)
assert detail.get("type") == "minimum_balance_required"