Compare commits

...
Author SHA1 Message Date
9qeklajc 7b56dcb902 resolve review comments 2026-06-24 22:54:07 +02:00
9qeklajc 1e50afbd82 feat: prune dead zero-balance API keys with background janitor 2026-06-21 15:04:16 +02:00
9qeklajcandGitHub 90f1ba89e5 Merge pull request #563 from Routstr/fix-streaming-issue
consolidate streaming
2026-06-20 19:46:22 +02:00
9qeklajcandGitHub e29c9241ce Merge pull request #560 from Routstr/fix-deepseek-cache-pricing
Fix deepseek cache pricing
2026-06-20 19:46:07 +02:00
9qeklajc b57f2d408e display cr as i 2026-06-20 00:57:11 +02:00
9qeklajc 70d9e1a1ce do correct deepseek cache price calc. 2026-06-20 00:26:34 +02:00
10 changed files with 519 additions and 7 deletions
+29
View File
@@ -1305,6 +1305,35 @@ async def periodic_key_reset() -> None:
logger.error(f"Error in periodic_key_reset: {e}")
async def periodic_dead_key_prune() -> None:
"""Periodically prune dead API keys. Interval <= 0 disables it.
See ``prune_dead_api_keys`` for eligibility.
"""
from .core.db import create_session, prune_dead_api_keys
interval = settings.dead_key_prune_interval_seconds
if interval <= 0:
logger.info("Dead-key pruning disabled (interval <= 0)")
return
while True:
try:
await asyncio.sleep(interval)
except asyncio.CancelledError:
break
try:
async with create_session() as session:
await prune_dead_api_keys(
session, settings.dead_key_min_age_seconds
)
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Error in periodic_dead_key_prune: {e}")
STALE_RESERVATION_SWEEP_INTERVAL_SECONDS: int = 60
+59 -1
View File
@@ -9,9 +9,10 @@ from typing import AsyncGenerator
from alembic import command
from alembic.config import Config
from alembic.util.exc import CommandError
from sqlalchemy import UniqueConstraint
from sqlalchemy import UniqueConstraint, delete
from sqlalchemy.exc import OperationalError
from sqlalchemy.ext.asyncio.engine import create_async_engine
from sqlalchemy.orm import aliased
from sqlmodel import Field, Relationship, SQLModel, col, func, select, update
from sqlmodel.ext.asyncio.session import AsyncSession
@@ -125,6 +126,63 @@ async def release_stale_reservations(
return released
async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> int:
"""Delete dead parentless API keys; return the count removed.
Dead = 0 balance/reservation/spend/requests, older than the grace period,
no parent, no children, no pending invoice. Cashu rows are unlinked (not
deleted) first to keep the audit trail.
"""
cutoff = int(time.time()) - min_age_seconds
child = aliased(ApiKey)
has_children = (
select(child.hashed_key).where(
col(child.parent_key_hash) == col(ApiKey.hashed_key)
)
).exists()
pending_invoice = (
select(LightningInvoice.id)
.where(col(LightningInvoice.api_key_hash) == col(ApiKey.hashed_key))
.where(col(LightningInvoice.status) == "pending")
).exists()
eligible_hashes = (
select(ApiKey.hashed_key)
.where(col(ApiKey.balance) == 0)
.where(col(ApiKey.reserved_balance) == 0)
.where(col(ApiKey.total_spent) == 0)
.where(col(ApiKey.total_requests) == 0)
.where(col(ApiKey.parent_key_hash).is_(None))
.where(
(col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff)
)
.where(~pending_invoice)
.where(~has_children)
)
# Unlink transactions rather than cascade-deleting them, so the financial
# audit trail survives. The eligibility predicate is re-evaluated inside both
# statements so a key that gained balance mid-run is left untouched.
await session.exec( # type: ignore[call-overload]
update(CashuTransaction)
.where(col(CashuTransaction.api_key_hashed_key).in_(eligible_hashes))
.values(api_key_hashed_key=None)
)
result = await session.exec( # type: ignore[call-overload]
delete(ApiKey).where(col(ApiKey.hashed_key).in_(eligible_hashes))
)
await session.commit()
pruned = int(result.rowcount or 0)
logger.info(
"Pruned dead API keys",
extra={"pruned_keys": pruned, "min_age_seconds": min_age_seconds},
)
return pruned
class ModelRow(SQLModel, table=True): # type: ignore
__tablename__ = "models"
id: str = Field(primary_key=True)
+17 -1
View File
@@ -11,7 +11,11 @@ from starlette.exceptions import HTTPException
from starlette.responses import Response as StarletteResponse
from starlette.types import Scope
from ..auth import periodic_key_reset, periodic_stale_reservation_sweep
from ..auth import (
periodic_dead_key_prune,
periodic_key_reset,
periodic_stale_reservation_sweep,
)
from ..balance import balance_router, deprecated_wallet_router
from ..lightning import lightning_router, periodic_invoice_watcher
from ..nostr import (
@@ -24,6 +28,7 @@ from ..payment.models import models_router, update_sats_pricing
from ..payment.price import update_prices_periodically
from ..proxy import initialize_upstreams, proxy_router, refresh_model_maps_periodically
from ..upstream.auto_topup import periodic_auto_topup
from ..upstream.deepseek_v4_pricing_shim import register_deepseek_v4_pricing
from ..upstream.litellm_routing import configure_litellm
from ..wallet import periodic_payout, periodic_refund_sweep, periodic_routstr_fee_payout
from .admin import admin_router
@@ -55,6 +60,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
model_maps_refresh_task = None
key_reset_task = None
stale_reservation_task = None
dead_key_prune_task = None
auto_topup_task = None
refund_sweep_task = None
routstr_fee_task = None
@@ -65,6 +71,11 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
# debug logging) before any upstream provider dispatches a request.
configure_litellm()
# TEMPORARY: backfill DeepSeek V4 pricing missing from litellm's cost
# map (BerriAI/litellm#30430). Remove this call and
# deepseek_v4_pricing_shim.py once litellm ships these models.
register_deepseek_v4_pricing()
# Run database migrations on startup
run_migrations()
@@ -126,6 +137,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
stale_reservation_task = asyncio.create_task(
periodic_stale_reservation_sweep()
)
dead_key_prune_task = asyncio.create_task(periodic_dead_key_prune())
auto_topup_task = asyncio.create_task(periodic_auto_topup())
refund_sweep_task = asyncio.create_task(periodic_refund_sweep())
routstr_fee_task = asyncio.create_task(periodic_routstr_fee_payout())
@@ -165,6 +177,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
key_reset_task.cancel()
if stale_reservation_task is not None:
stale_reservation_task.cancel()
if dead_key_prune_task is not None:
dead_key_prune_task.cancel()
if auto_topup_task is not None:
auto_topup_task.cancel()
if refund_sweep_task is not None:
@@ -196,6 +210,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
tasks_to_wait.append(key_reset_task)
if stale_reservation_task is not None:
tasks_to_wait.append(stale_reservation_task)
if dead_key_prune_task is not None:
tasks_to_wait.append(dead_key_prune_task)
if auto_topup_task is not None:
tasks_to_wait.append(auto_topup_task)
if refund_sweep_task is not None:
+8
View File
@@ -73,6 +73,14 @@ class Settings(BaseSettings):
stale_reservation_timeout_seconds: int = Field(
default=300, env="STALE_RESERVATION_TIMEOUT_SECONDS"
)
# Background prune of dead (zero balance, never used) API keys.
# Interval 0 disables it; min-age is a grace period (default 1 week).
dead_key_prune_interval_seconds: int = Field(
default=3600, env="DEAD_KEY_PRUNE_INTERVAL_SECONDS"
)
dead_key_min_age_seconds: int = Field(
default=604_800, env="DEAD_KEY_MIN_AGE_SECONDS"
)
# Network
cors_origins: list[str] = Field(default_factory=lambda: ["*"], env="CORS_ORIGINS")
+13 -2
View File
@@ -418,10 +418,21 @@ def _calculate_from_tokens(
},
)
# Fold the cache-read/write cost into the visible ``input_msats`` so a
# dashboard that renders I / O / T sees ``input + output == total``
# exactly. This mirrors ``_fold_cache_into_input_tokens`` (which rolls the
# cache token counts into the visible prompt total). The standalone
# ``cache_read_msats`` / ``cache_creation_msats`` fields stay populated for
# clients that want the breakdown; nothing sums the components to derive
# ``total_msats`` (it is computed independently above), so this is
# display-only and does not change what is billed.
visible_output_msats = int(calc_output_msats)
visible_input_msats = token_based_cost - visible_output_msats
return CostData(
base_msats=0,
input_msats=int(calc_input_msats),
output_msats=int(calc_output_msats),
input_msats=visible_input_msats,
output_msats=visible_output_msats,
total_msats=token_based_cost,
total_usd=total_usd,
input_tokens=input_tokens,
@@ -0,0 +1,66 @@
"""TEMPORARY: local DeepSeek V4 pricing shim.
litellm's bundled cost map does not yet ship ``deepseek-v4-flash`` /
``deepseek-v4-pro``. Without an entry, ``backfill_cache_pricing`` cannot find a
``cache_read_input_token_cost`` and cache reads fall back to the full input
rate — a ~60% overcharge on cache hits (DeepSeek hits are ~0.2x input).
This module injects the missing entries into ``litellm.model_cost`` at startup
so the existing backfill path resolves them. Rates mirror the open upstream PR
https://github.com/BerriAI/litellm/pull/26380 (issue
https://github.com/BerriAI/litellm/issues/30430).
=== REMOVAL (once litellm ships these models) ===
Delete this file and the single ``register_deepseek_v4_pricing()`` call in
``routstr/core/main.py``. Nothing else depends on it. Entries are only added
when absent, so a stale shim is harmless after upstream lands — but remove it.
"""
import litellm
from ..core import get_logger
logger = get_logger(__name__)
# USD per token. Source: BerriAI/litellm PR #26380.
_DEEPSEEK_V4_RATES: dict[str, dict[str, float]] = {
"deepseek-v4-flash": {
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 2.8e-08,
"cache_creation_input_token_cost": 0.0,
"input_cost_per_token_cache_hit": 2.8e-08,
},
"deepseek-v4-pro": {
"input_cost_per_token": 1.74e-06,
"output_cost_per_token": 3.48e-06,
"cache_read_input_token_cost": 1.4e-07,
"cache_creation_input_token_cost": 0.0,
"input_cost_per_token_cache_hit": 1.4e-07,
},
}
def register_deepseek_v4_pricing() -> None:
"""Inject DeepSeek V4 pricing into ``litellm.model_cost`` if absent.
Idempotent and non-destructive: a key already present in the cost map
(e.g. once litellm ships it) is left untouched. Registers both the bare
(``deepseek-v4-flash``) and prefixed (``deepseek/deepseek-v4-flash``)
spellings since ``backfill_cache_pricing`` tries both.
"""
added = []
for bare, rates in _DEEPSEEK_V4_RATES.items():
for key in (bare, f"deepseek/{bare}"):
if key in litellm.model_cost:
continue
entry: dict[str, object] = dict(rates)
entry["litellm_provider"] = "deepseek"
entry["mode"] = "chat"
litellm.model_cost[key] = entry
added.append(key)
if added:
logger.info(
"Registered temporary DeepSeek V4 pricing shim",
extra={"models": added},
)
+5 -1
View File
@@ -441,7 +441,11 @@ async def credit_balance(
.where(col(db.ApiKey.hashed_key) == key.hashed_key)
.values(balance=(db.ApiKey.balance) + amount)
)
await session.exec(stmt) # type: ignore[call-overload]
result = await session.exec(stmt) # type: ignore[call-overload]
# If pruning removed this key after redemption, do not commit a no-op
# balance update and pretend the top-up succeeded.
if (getattr(result, "rowcount", 0) or 0) == 0:
raise ValueError("API key disappeared before credit could be recorded")
await session.commit()
await session.refresh(key)
@@ -0,0 +1,283 @@
"""
Tests for prune_dead_api_keys — the janitor that removes provably-dead 0/0/0
API keys (funded keys fully refunded/expired without ever being used, plus bare
orphans), while protecting keys that are still meaningful.
"""
import time
import uuid
from collections.abc import Awaitable, Callable
from typing import Any, cast
import pytest
from sqlalchemy.sql.dml import Update
from sqlmodel import col, update
from routstr.core.db import (
ApiKey,
CashuTransaction,
LightningInvoice,
create_session,
prune_dead_api_keys,
)
OLD = 100 # min_age_seconds used by the tests
NOW = int(time.time())
LONG_AGO = NOW - 10_000 # well past the grace period
async def _exists(key_hash: str) -> bool:
async with create_session() as session:
return (await session.get(ApiKey, key_hash)) is not None
def _dead_key(created_at: int | None) -> ApiKey:
return ApiKey(
hashed_key=f"dead_{uuid.uuid4().hex}",
balance=0,
reserved_balance=0,
total_spent=0,
total_requests=0,
created_at=created_at,
)
@pytest.mark.asyncio
async def test_prunes_old_refunded_zero_key(patched_db_engine: None) -> None:
"""A funded-then-refunded key (0/0/0, NULL parent, old) is pruned."""
key = _dead_key(LONG_AGO)
async with create_session() as session:
session.add(key)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 1
assert not await _exists(key.hashed_key)
@pytest.mark.asyncio
async def test_grace_period_protects_fresh_key(patched_db_engine: None) -> None:
"""A dead-looking but recently created key is protected by the grace period."""
key = _dead_key(int(time.time()))
async with create_session() as session:
session.add(key)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 0
assert await _exists(key.hashed_key)
@pytest.mark.asyncio
async def test_used_key_never_pruned(patched_db_engine: None) -> None:
"""Keys with any spend/requests or live balance are never pruned."""
spent = _dead_key(LONG_AGO)
spent.total_spent = 1
requested = _dead_key(LONG_AGO)
requested.total_requests = 1
funded = _dead_key(LONG_AGO)
funded.balance = 1000
reserved = _dead_key(LONG_AGO)
reserved.reserved_balance = 500
async with create_session() as session:
for k in (spent, requested, funded, reserved):
session.add(k)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 0
for k in (spent, requested, funded, reserved):
assert await _exists(k.hashed_key)
@pytest.mark.asyncio
async def test_parent_and_child_keys_are_not_pruned(
patched_db_engine: None,
) -> None:
"""Pruning must not orphan child keys or delete valid children."""
parent = _dead_key(LONG_AGO)
child = ApiKey(
hashed_key=f"child_{uuid.uuid4().hex}",
balance=0,
reserved_balance=0,
total_spent=0,
total_requests=0,
created_at=LONG_AGO,
parent_key_hash=parent.hashed_key,
)
async with create_session() as session:
session.add(parent)
session.add(child)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 0
assert await _exists(parent.hashed_key)
assert await _exists(child.hashed_key)
@pytest.mark.asyncio
async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
"""A key referenced by a pending topup invoice is never pruned mid-topup."""
key = _dead_key(LONG_AGO)
invoice = LightningInvoice(
id=f"inv_{uuid.uuid4().hex}",
bolt11=f"lnbc_{uuid.uuid4().hex}",
amount_sats=10,
description="topup",
payment_hash=uuid.uuid4().hex,
status="pending",
api_key_hash=key.hashed_key,
purpose="topup",
expires_at=NOW + 10_000,
)
async with create_session() as session:
session.add(key)
session.add(invoice)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 0
assert await _exists(key.hashed_key)
@pytest.mark.asyncio
async def test_paid_invoice_does_not_protect_key(patched_db_engine: None) -> None:
"""A settled (non-pending) invoice does not keep a dead key alive."""
key = _dead_key(LONG_AGO)
invoice = LightningInvoice(
id=f"inv_{uuid.uuid4().hex}",
bolt11=f"lnbc_{uuid.uuid4().hex}",
amount_sats=10,
description="topup",
payment_hash=uuid.uuid4().hex,
status="paid",
api_key_hash=key.hashed_key,
purpose="topup",
expires_at=NOW - 1,
)
async with create_session() as session:
session.add(key)
session.add(invoice)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 1
assert not await _exists(key.hashed_key)
@pytest.mark.asyncio
async def test_key_that_becomes_meaningful_during_prune_survives(
patched_db_engine: None, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Revalidate before unlink/delete so a late top-up cannot be pruned."""
key = _dead_key(LONG_AGO)
txn = CashuTransaction(
id=uuid.uuid4().hex,
token="cashuABC",
amount=21,
unit="sat",
type="in",
source="apikey",
api_key_hashed_key=key.hashed_key,
)
async with create_session() as session:
session.add(key)
session.add(txn)
await session.commit()
async with create_session() as session:
original_exec = cast(Callable[..., Awaitable[Any]], session.exec)
topped_up = False
async def exec_with_late_topup(
statement: Any, *args: Any, **kwargs: Any
) -> Any:
nonlocal topped_up
if (
not topped_up
and isinstance(statement, Update)
and getattr(statement.table, "name", None) == "cashu_transactions"
):
topped_up = True
async with create_session() as topup_session:
await topup_session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.values(balance=42)
)
await topup_session.commit()
return await original_exec(statement, *args, **kwargs)
monkeypatch.setattr(session, "exec", exec_with_late_topup)
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 0
assert await _exists(key.hashed_key)
async with create_session() as session:
surviving = await session.get(CashuTransaction, txn.id)
assert surviving is not None
assert surviving.api_key_hashed_key == key.hashed_key
@pytest.mark.asyncio
async def test_transaction_audit_trail_preserved(patched_db_engine: None) -> None:
"""Pruning a refunded key keeps its cashu_transactions, unlinked from the key."""
key = _dead_key(LONG_AGO)
txn = CashuTransaction(
id=uuid.uuid4().hex,
token="cashuABC",
amount=21,
unit="sat",
type="in",
source="apikey",
api_key_hashed_key=key.hashed_key,
)
async with create_session() as session:
session.add(key)
session.add(txn)
await session.commit()
async with create_session() as session:
pruned = await prune_dead_api_keys(session, OLD)
assert pruned == 1
assert not await _exists(key.hashed_key)
async with create_session() as session:
surviving = await session.get(CashuTransaction, txn.id)
assert surviving is not None, "Financial audit row must survive key deletion"
assert surviving.api_key_hashed_key is None, "Link must be nulled, not dangling"
assert surviving.amount == 21
@pytest.mark.asyncio
async def test_periodic_prune_disabled_returns_immediately(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Non-positive intervals disable the janitor."""
from unittest.mock import AsyncMock
from routstr import auth
from routstr.core.settings import settings
monkeypatch.setattr(settings, "dead_key_prune_interval_seconds", 0)
sleep_mock = AsyncMock()
monkeypatch.setattr(auth.asyncio, "sleep", sleep_mock)
await auth.periodic_dead_key_prune()
sleep_mock.assert_not_called()
+6 -2
View File
@@ -181,10 +181,14 @@ async def test_deepseek_cache_hits_billed_at_cache_rate(model_pricing: Mock) ->
result = await calculate_cost(response, max_cost=100000)
assert isinstance(result, CostData)
# 1000 input @ 1 msat + 9000 cache reads @ 0.1 msat + 500 output @ 2 msat
assert result.input_msats == 1000
# 1000 input @ 1 msat + 9000 cache reads @ 0.1 msat + 500 output @ 2 msat.
# input_msats folds the cache-read cost in (1000 + 900) so a dashboard
# rendering I/O/T sees input + output == total; the cache portion stays
# visible in cache_read_msats.
assert result.cache_read_msats == 900
assert result.output_msats == 1000
assert result.input_msats == 1900
assert result.input_msats + result.output_msats == result.total_msats
assert result.total_msats == 2900
+33
View File
@@ -150,6 +150,7 @@ async def test_credit_balance() -> None:
mock_key.balance = 5000000
mock_key.hashed_key = "test_hash"
mock_session = AsyncMock()
mock_session.exec.return_value.rowcount = 1
# Mock session.refresh to update the balance (simulates DB reload)
async def mock_refresh(key: ApiKey) -> None:
@@ -206,6 +207,38 @@ async def test_credit_balance_rejects_zero_amount() -> None:
assert not mock_session.commit.called
@pytest.mark.asyncio
async def test_credit_balance_rejects_missing_key() -> None:
"""A top-up must fail if the key was pruned after redemption."""
token_data = {
"token": [{"mint": "http://mint:3338", "proofs": [{"amount": 1000}]}],
"unit": "sat",
}
token_json = json.dumps(token_data)
token_b64 = base64.urlsafe_b64encode(token_json.encode()).decode()
token_str = f"cashuA{token_b64}"
mock_key = Mock()
mock_key.balance = 0
mock_key.hashed_key = "test_hash"
mock_session = AsyncMock()
mock_session.exec.return_value.rowcount = 0
from routstr.core.settings import settings
with patch.object(settings, "cashu_mints", ["http://mint:3338"]):
with patch(
"routstr.wallet.recieve_token",
return_value=(1000, "sat", "http://mint:3338"),
):
with pytest.raises(ValueError, match="disappeared"):
await credit_balance(token_str, mock_key, mock_session)
# UPDATE matched nothing; committing would hide the failed credit.
assert mock_session.exec.called
assert not mock_session.commit.called
@pytest.mark.asyncio
async def test_swap_to_primary_mint_insufficient_for_fees() -> None:
"""Token amount is less than melt_quote.amount + melt_quote.fee_reserve."""