Compare commits

...
35 Commits
Author SHA1 Message Date
9qeklajc 443c910b9e Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-30 08:38:27 +02:00
9qeklajc 3befe063f4 fix: annotate lightning settlement test session 2026-07-30 01:19:35 +02:00
9qeklajc f8adaee362 revert: restore default compose port 2026-07-30 01:14:36 +02:00
9qeklajc 895ea90bfa Merge branch 'main' into fix/mint-rate-limit-and-fallback
# Conflicts:
#	routstr/core/settings.py
#	routstr/lightning.py
#	routstr/wallet.py
#	tests/integration/test_lightning_invoice_constraints.py
#	tests/unit/test_fee_payout_migration.py
#	tests/unit/test_fetch_all_balances.py
2026-07-30 01:09:32 +02:00
thefux c75dee147a fix: set routstr-core port to 8011 to avoid Portainer conflict on 8000 2026-07-28 00:08:47 +00:00
9qeklajc 48c11eb7bc fix Lightning settlement test typing 2026-07-27 00:07:36 +02:00
9qeklajc c829685f80 fix Cashu fallback and Lightning settlement 2026-07-26 23:23:32 +02:00
9qeklajc 1138cdd4ef Merge main and recreate mint URL migration 2026-07-26 00:16:10 +02:00
9qeklajc 92246b78d0 add import 2026-07-23 01:08:34 +02:00
9qeklajc 6023c03959 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-23 00:22:38 +02:00
9qeklajc a9a6381614 fix: recreate mint URL migration from latest head 2026-07-22 22:27:50 +02:00
9qeklajc 040799a4d7 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-22 21:27:46 +02:00
thefux 586af15a1b chore: fix ruff lint errors (E402, F401, I001) 2026-07-18 14:41:32 +00:00
thefux 3e906605a0 fix: strict rate-limit detection, probe non-escalation, distinct error codes
Fixes three issues that caused per-mint rate-limit state to never recover:

1. _is_mint_rate_limited: remove substring matching on 'rate limit' /
   'too many requests' in exception messages.  Only HTTP 429
   (httpx.HTTPStatusError) is now classified as a rate limit, preventing
   false positives (e.g. a 503 with 'database rate exceeded' in its body).

2. _run_probe: use apply_cooldown() instead of apply_rate_limit_cooldown()
   when a probe fails due to a rate limit.  The probe is a recovery check,
   not a new request, so it should not escalate the exponential backoff
   counter (_consecutive_rate_limits).  This prevents the cooldown from
   ratcheting 60s → 120s → 240s → ... → 7h on repeated probe failures.

3. classify_redemption_error: split the combined _is_mint_rate_limited ||
   is_mint_connection_error check into two separate classifications:
   - mint_rate_limited / cashu_mint_rate_limited (503, retryable)
   - mint_unreachable / cashu_mint_unreachable (503, retryable)
   Callers (routstrd) can now distinguish temporary rate limits from
   permanent connection failures when deciding fallback strategy.

Tests: 20 new tests covering strict 429 detection, classification
priority, probe non-escalation, and cooldown reset behaviour.
2026-07-18 14:34:42 +00:00
9qeklajc 1957e716a3 clean up keyset unit recog. 2026-07-15 01:45:19 +02:00
9qeklajc 69f19ff991 defensive cooldown 2026-07-15 01:43:00 +02:00
9qeklajc 8b942f3c14 show mint status correclty 2026-07-15 01:00:32 +02:00
9qeklajc 09e1c7bf2d better cooldown 2026-07-15 00:10:17 +02:00
9qeklajc cc2a96e2ef make trusted mint available for lightning topup 2026-07-14 21:59:43 +02:00
9qeklajc 93ab1d927b mint cooldown 2026-07-14 01:55:35 +02:00
9qeklajc 39970d8bee clean up 2026-07-14 01:23:14 +02:00
9qeklajc d7c401d204 primary mint fallback 2026-07-14 01:10:23 +02:00
9qeklajc d44b98fd0d fix fallback 2026-07-14 00:54:44 +02:00
9qeklajc 6fa3610423 fix: account for mint fees in balance checks 2026-07-14 00:11:27 +02:00
9qeklajc 65702171e4 fix: harden mint fallback and refund recovery 2026-07-13 23:37:43 +02:00
9qeklajc 65abcbce92 fix: harden mint fallback and refund recovery 2026-07-13 23:33:56 +02:00
9qeklajc 40153d4c36 fix: report cashu transaction persistence 2026-07-12 15:07:31 +02:00
9qeklajc 40bf976fbc fix: recreate mint URL migration 2026-07-12 15:04:43 +02:00
9qeklajc eae20f04a7 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-12 15:02:07 +02:00
9qeklajc acb630f6cf refactor: adapt mint throttling to 429 responses 2026-07-10 23:54:05 +02:00
9qeklajc 1230d528de fix: avoid rate limiting balance proof checks 2026-07-10 23:46:09 +02:00
9qeklajc d23c90b939 fix: type wallet test fixture 2026-07-10 21:50:50 +02:00
9qeklajc d8db2a3051 fix: harden mint rate limiting and fallback 2026-07-10 21:46:56 +02:00
9qeklajc 0bbbf902cd Merge origin/main into fix/mint-rate-limit-and-fallback 2026-07-10 21:12:54 +02:00
9qeklajc 7ed18a9d02 fix: per-mint rate limiting, trusted-mint fallback, and retry factory fix 2026-07-10 20:43:07 +02:00
29 changed files with 3621 additions and 415 deletions
@@ -0,0 +1,25 @@
"""add mint url to lightning invoices
Revision ID: bf76270b66c4
Revises: aa50fde387a2
Create Date: 2026-07-30 00:54:30.306876
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "bf76270b66c4"
down_revision = "aa50fde387a2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
)
def downgrade() -> None:
op.drop_column("lightning_invoices", "mint_url")
+22 -14
View File
@@ -51,6 +51,24 @@ ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
def _format_msat_amount(amount: int) -> str:
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
return f"{sats} sats ({amount} msats)"
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
return {
"error": {
"message": (
f"Insufficient balance: {_format_msat_amount(required)} required "
f"for this model; {_format_msat_amount(available)} available."
),
"type": "insufficient_quota",
"code": "insufficient_balance",
}
}
@dataclass(frozen=True)
class ReservationSnapshot:
release_id: str
@@ -264,13 +282,7 @@ async def _validate_bearer_key_locked(
)
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(min_cost, billing_key.total_balance),
)
# Early check: Spending limit check (Child key limit)
@@ -360,13 +372,9 @@ async def _validate_bearer_key_locked(
if min_cost > 0 and existing_key.total_balance < min_cost:
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(
min_cost, existing_key.total_balance
),
)
return existing_key
+84 -13
View File
@@ -30,6 +30,7 @@ from .wallet import (
recieve_token,
send_to_lnurl,
send_token,
token_mint_url,
)
router = APIRouter()
@@ -184,6 +185,17 @@ class TopupRequest(BaseModel):
cashu_token: str
def _error_chain(error: BaseException) -> list[dict[str, str]]:
chain: list[dict[str, str]] = []
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
chain.append({"type": type(current).__name__, "message": str(current)})
current = current.__cause__ or current.__context__
return chain
@router.post("/topup")
async def topup_wallet_endpoint(
cashu_token: str | None = None,
@@ -201,6 +213,18 @@ async def topup_wallet_endpoint(
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
if len(cashu_token) < 10 or "cashu" not in cashu_token:
raise HTTPException(status_code=400, detail="Invalid token format")
source_mint = token_mint_url(cashu_token, "unknown")
logger.warning(
"Cashu wallet top-up started",
extra={
"event": "cashu_topup_started",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"key_hash": billing_key.hashed_key[:8],
},
)
try:
amount_msats = await credit_balance(cashu_token, billing_key, session)
except Exception as e:
@@ -209,12 +233,41 @@ async def topup_wallet_endpoint(
classified = classify_redemption_error(e)
if classified is None:
logger.error(
"topup_wallet_endpoint: unhandled error",
extra={"error": str(e), "error_type": type(e).__name__},
"Cashu wallet top-up failed with an unhandled error",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=500, detail="Internal server error")
_type, status_code, message, _code = classified
error_type, status_code, message, error_code = classified
logger.warning(
"Cashu wallet top-up failed",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"status_code": status_code,
"error_type": error_type,
"error_code": error_code,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=status_code, detail=message)
logger.warning(
"Cashu wallet top-up completed",
extra={
"event": "cashu_topup_completed",
"source_mint": source_mint,
"credited_msats": amount_msats,
"key_hash": billing_key.hashed_key[:8],
},
)
return {"msats": amount_msats}
@@ -260,7 +313,11 @@ async def _lookup_key_no_create(
async def _restore_balance(
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
session: AsyncSession,
hashed_key: str,
balance: int,
reserved_balance: int,
mint_url: str,
) -> None:
"""Restore balance after a failed refund mint attempt."""
restore_stmt = (
@@ -275,7 +332,11 @@ async def _restore_balance(
await session.commit()
logger.info(
"refund_wallet_endpoint: balance restored after mint failure",
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
extra={
"hashed_key": hashed_key,
"restored_balance": balance,
"mint_url": mint_url,
},
)
@@ -418,15 +479,14 @@ async def refund_wallet_endpoint(
detail="Balance changed concurrently. Please retry the refund.",
)
# --- MINT: balance is locked at zero, safe to create the refund token ---
# Proofs from untrusted mints are swapped to primary_mint on receive.
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
# The balance is locked at zero, so it is safe to create the refund token.
effective_refund_mint = (
key.refund_mint_url
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
else settings.primary_mint
)
try:
refund_currency = key.refund_currency or "sat"
if key.refund_address:
await send_to_lnurl(
remaining_balance,
@@ -436,10 +496,10 @@ async def refund_wallet_endpoint(
)
result = {"recipient": key.refund_address}
else:
refund_currency = key.refund_currency or "sat"
token = await send_token(
remaining_balance, refund_currency, effective_refund_mint
)
effective_refund_mint = token_mint_url(token, effective_refund_mint)
result = {"token": token}
if key.refund_currency == "sat":
@@ -460,11 +520,23 @@ async def refund_wallet_endpoint(
except HTTPException:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
raise
except Exception as e:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
error_msg = str(e)
logger.error(
"refund_wallet_endpoint: mint/send failed",
@@ -491,7 +563,7 @@ async def refund_wallet_endpoint(
token=result["token"],
amount=remaining_balance,
unit=key.refund_currency or "sat",
mint_url=key.refund_mint_url,
mint_url=effective_refund_mint,
typ="out",
collected=False,
source="apikey",
@@ -685,7 +757,6 @@ async def reset_child_key_spent(
return {"success": True, "message": "Child key balance reset successfully."}
@router.api_route(
"/{path:path}",
methods=["GET", "POST", "PUT", "DELETE"],
+3
View File
@@ -389,6 +389,9 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
default=None, description="Associated API key hash for topup operations"
)
purpose: str = Field(description="create or topup")
mint_url: str | None = Field(
default=None, description="Mint URL where the quote was created (fallback tracking)"
)
created_at: int = Field(
default_factory=lambda: int(time.time()), description="Unix timestamp"
)
+17 -4
View File
@@ -53,6 +53,18 @@ class Settings(BaseSettings):
payout_interval_seconds: int = Field(
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
)
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
# checkstate). When a mint is slow or rate-limiting, operations are
# cancelled after this delay instead of hanging indefinitely.
mint_operation_timeout_seconds: int = Field(
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
)
# Maximum concurrent API operations per mint. Actual mint quotas vary by
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
# pacing. 0 = unlimited concurrency.
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
# Max retries when a mint returns 429 or times out (exponential backoff).
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
# Pricing
# Default behavior: derive pricing from MODELS
@@ -101,7 +113,9 @@ class Settings(BaseSettings):
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
refund_sweep_ttl_seconds: int = Field(default=604800, env="REFUND_SWEEP_TTL_SECONDS")
refund_sweep_ttl_seconds: int = Field(
default=604800, env="REFUND_SWEEP_TTL_SECONDS"
)
refund_sweep_claim_timeout_seconds: int = Field(
default=900, gt=0, env="REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS"
)
@@ -138,9 +152,8 @@ class Settings(BaseSettings):
# Discovery
relays: list[str] = Field(default_factory=list, env="RELAYS")
enable_analytics_sharing: bool = Field(
default=True, env="ENABLE_ANALYTICS_SHARING"
)
enable_analytics_sharing: bool = Field(default=True, env="ENABLE_ANALYTICS_SHARING")
def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
"""Discard unknown keys from persisted settings."""
+363 -156
View File
@@ -1,7 +1,10 @@
import asyncio
import hashlib
import re
import secrets
import time
from dataclasses import dataclass
from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException
from pydantic import BaseModel, Field
@@ -12,12 +15,59 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from .core.db import ApiKey, LightningInvoice, create_session, get_session
from .core.logging import get_logger
from .core.settings import settings
from .wallet import get_wallet, wallet_operation_guard
from .wallet import (
MintConnectionError,
_is_mint_rate_limited,
_mint_cooldown_remaining,
_mint_operation,
get_wallet,
is_mint_connection_error,
wallet_operation_guard,
)
logger = get_logger(__name__)
lightning_router = APIRouter(prefix="/lightning")
# Avoid duplicate work within one process. Cross-process credit fencing is done
# by the conditional pending -> paid update in _finalize_invoice_settlement().
_invoice_settlement_locks: dict[str, asyncio.Lock] = {}
@dataclass(frozen=True)
class _InvoiceSettlement:
id: str
payment_hash: str
amount_sats: int
purpose: str
api_key_hash: str | None
mint_url: str | None
balance_limit: int | None
balance_limit_reset: str | None
validity_date: int | None
@classmethod
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
return cls(
id=invoice.id,
payment_hash=invoice.payment_hash,
amount_sats=invoice.amount_sats,
purpose=invoice.purpose,
api_key_hash=invoice.api_key_hash,
mint_url=invoice.mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
)
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
"""Update a caller view without marking a mapped object dirty."""
try:
set_committed_value(invoice, key, value)
except AttributeError:
setattr(invoice, key, value)
class InvoiceCreateRequest(BaseModel):
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
@@ -65,12 +115,80 @@ class InvoiceRecoverRequest(BaseModel):
bolt11: str = Field(description="BOLT11 invoice string")
def _trusted_mint_candidates() -> list[str]:
return [
mint
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
if mint
]
async def _request_mint_with_fallback(
amount_sats: int,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
"""Request a quote, falling back only among the allowed trusted mints.
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
cryptic validation error deep in the stack. Fail fast with context.
"""
if amount_sats <= 0:
raise ValueError(
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
)
tried: list[str] = []
configured = allowed_mints or [settings.primary_mint, *settings.cashu_mints]
candidates = list(dict.fromkeys(configured))
for mint_url in candidates:
cooldown = _mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.info(
"Skipping rate-limited mint",
extra={
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"op_name": "request_mint_invoice",
},
)
continue
try:
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
quote = await _mint_operation(
lambda: wallet.request_mint(amount_sats),
op_name="request_mint_invoice",
mint_url=mint_url,
retry_on_rate_limit=False,
)
return quote.request, quote.quote, mint_url
except Exception as e:
tried.append(f"{mint_url}: {type(e).__name__}")
if not is_mint_connection_error(e) and not _is_mint_rate_limited(e):
raise
logger.warning(
"request_mint failed, trying fallback mint",
extra={
"failed_mint": mint_url,
"error": str(e),
"tried": tried,
},
)
continue
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
async def generate_lightning_invoice(
amount_sats: int, description: str
) -> tuple[str, str]:
wallet = await get_wallet(settings.primary_mint, "sat")
quote = await wallet.request_mint(amount_sats)
return quote.request, quote.quote
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
amount_sats, allowed_mints=allowed_mints
)
return bolt11, payment_hash, mint_url
def generate_invoice_id() -> str:
@@ -84,6 +202,7 @@ async def create_invoice(
session: AsyncSession = Depends(get_session),
) -> InvoiceCreateResponse:
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
topup_api_key: ApiKey | None = None
if request.purpose == "topup":
if not api_key_token:
@@ -94,14 +213,23 @@ async def create_invoice(
if not api_key_token.startswith("sk-"):
raise HTTPException(status_code=400, detail="Invalid API key format")
api_key = await session.get(ApiKey, api_key_token[3:])
if not api_key:
topup_api_key = await session.get(ApiKey, api_key_token[3:])
if not topup_api_key:
raise HTTPException(status_code=404, detail="API key not found")
try:
description = f"Routstr {request.purpose} {request.amount_sats} sats"
bolt11, payment_hash = await generate_lightning_invoice(
request.amount_sats, description
allowed_mints = None
if request.purpose == "topup":
assert topup_api_key is not None
# A key's liabilities are attributed to a single refund mint. Keep
# top-up collateral on that same mint so balances and payouts cannot
# misclassify funds held by another mint as owner profit.
allowed_mints = [
topup_api_key.refund_mint_url or settings.primary_mint
]
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
request.amount_sats, description, allowed_mints=allowed_mints
)
invoice_id = generate_invoice_id()
@@ -116,6 +244,7 @@ async def create_invoice(
status="pending",
api_key_hash=api_key_token[3:] if api_key_token else None,
purpose=request.purpose,
mint_url=mint_url,
balance_limit=request.balance_limit,
balance_limit_reset=request.balance_limit_reset,
validity_date=request.validity_date,
@@ -223,158 +352,180 @@ async def recover_invoice(
async def check_invoice_payment(
invoice: LightningInvoice, session: AsyncSession
) -> None:
# Minting makes proofs visible before database finalization. Share the
# cross-process wallet guard with owner payout so that visibility and the
# corresponding liability commit are observed atomically by the payout loop.
async with wallet_operation_guard():
await _check_invoice_payment_locked(invoice, session)
async def _check_invoice_payment_locked(
invoice: LightningInvoice, session: AsyncSession
) -> None:
minted = False
invoice_id = invoice.id
invoice_purpose = invoice.purpose
invoice_amount_sats = invoice.amount_sats
invoice_payment_hash = invoice.payment_hash
finalized_api_key_hash = invoice.api_key_hash
try:
# A preceding invoice lookup starts a transaction. End it before the
# potentially slow mint request so it cannot pin a pool connection.
await session.commit()
wallet = await get_wallet(settings.primary_mint, "sat")
mint_status = await wallet.get_mint_quote(invoice_payment_hash)
if not mint_status.paid:
return
# Do not redeem a paid top-up quote if its target has already been
# pruned. This validation owns a short-lived session and releases its
# connection before mint redemption starts.
if invoice_purpose == "topup":
if not finalized_api_key_hash:
raise ValueError("No API key associated with topup invoice")
async with create_session() as validation_session:
target = await validation_session.get(ApiKey, finalized_api_key_hash)
if target is None:
terminal = await validation_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice_id,
col(LightningInvoice.status) == "pending",
)
.values(status="reconciliation_required")
)
await validation_session.commit()
if terminal.rowcount == 1:
set_committed_value(
invoice, "status", "reconciliation_required"
)
else:
committed_invoice = await validation_session.get(
LightningInvoice, invoice_id
)
if committed_invoice is not None:
set_committed_value(
invoice, "status", committed_invoice.status
)
logger.critical(
"Paid topup invoice target API key was not found; reconciliation required",
extra={"invoice_id": invoice_id},
)
return
# The mint enforces single-use quotes, so a concurrent checker that
# races us here fails inside wallet.mint rather than double-minting.
await wallet.mint(invoice_amount_sats, quote_id=invoice_payment_hash)
minted = True
# Paid finalization owns a fresh session. The API/watcher session is
# never rolled back by this function, so its invoice and sibling ORM
# objects remain usable after a DB failure or lost CAS race.
async with create_session() as finalization_session:
if invoice_purpose == "create":
api_key = await _create_api_key_record(invoice, finalization_session)
finalized_api_key_hash = api_key.hashed_key
elif invoice_purpose == "topup":
await _credit_topup_record(invoice, finalization_session)
# Conditional transition guards against double-credit: the credit
# above and this status flip commit atomically, and a lost race
# rolls both back in the owned finalization session.
paid_at = int(time.time())
finalized = await finalization_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice_id,
col(LightningInvoice.status) == "pending",
)
.values(
status="paid",
paid_at=paid_at,
api_key_hash=finalized_api_key_hash,
)
)
if finalized.rowcount != 1:
await finalization_session.rollback()
committed_invoice = await finalization_session.get(
LightningInvoice, invoice_id
)
await finalization_session.commit()
if committed_invoice is not None:
# A concurrent finalizer won the CAS. Publish only the
# state observed from the database after ending the owned
# read transaction; never refresh the caller's session.
set_committed_value(
invoice, "api_key_hash", committed_invoice.api_key_hash
)
set_committed_value(invoice, "status", committed_invoice.status)
set_committed_value(invoice, "paid_at", committed_invoice.paid_at)
lock = _invoice_settlement_locks.setdefault(invoice.id, asyncio.Lock())
async with lock, wallet_operation_guard():
minted = False
try:
# Snapshot the row and end the caller's read transaction before any
# potentially slow mint I/O. All final DB mutations use owned,
# short-lived sessions below.
await session.refresh(invoice)
if invoice.status != "pending":
await session.commit()
return
await finalization_session.commit()
settlement = _InvoiceSettlement.from_invoice(invoice)
await session.commit()
# Only publish finalized values to the caller-owned object after the
# owned transaction has committed successfully.
set_committed_value(invoice, "api_key_hash", finalized_api_key_hash)
set_committed_value(invoice, "status", "paid")
set_committed_value(invoice, "paid_at", paid_at)
logger.info(
"Lightning invoice paid",
extra={
"invoice_id": invoice_id,
"amount_sats": invoice_amount_sats,
"purpose": invoice_purpose,
"api_key_hash": finalized_api_key_hash[:8] + "..."
if finalized_api_key_hash
else None,
},
)
except BaseException as e:
# BaseException so task cancellation (e.g. client disconnect) after a
# successful mint still triggers the reconciliation alert. Any rollback
# belongs to create_session(), never to the caller-owned session.
if minted:
logger.critical(
"Invoice mint succeeded but DB finalization failed; reconciliation required",
extra={"invoice_id": invoice_id, "purpose": invoice_purpose},
mint_url = settlement.mint_url or settings.primary_mint
wallet = await get_wallet(mint_url, "sat")
mint_status = await _mint_operation(
lambda: wallet.get_mint_quote(settlement.payment_hash),
op_name="get_mint_quote",
mint_url=mint_url,
)
if not isinstance(e, Exception):
if not mint_status.paid:
return
# Reject a paid top-up whose target was pruned before redeeming its
# single-use quote. The validation session is closed before mint I/O.
if settlement.purpose == "topup":
if not settlement.api_key_hash:
raise ValueError("No API key associated with topup invoice")
async with create_session() as validation_session:
target = await validation_session.get(
ApiKey, settlement.api_key_hash
)
if target is None:
terminal = await validation_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == settlement.id,
col(LightningInvoice.status) == "pending",
)
.values(status="reconciliation_required")
)
await validation_session.commit()
if terminal.rowcount == 1:
_publish_invoice_value(
invoice, "status", "reconciliation_required"
)
else:
await _reload_invoice_view(invoice, session)
logger.critical(
"Paid topup invoice target API key was not found; reconciliation required",
extra={"invoice_id": settlement.id},
)
return
# Quote-linked proof verification makes an ambiguous mint response
# retryable without crediting unrelated wallet balance growth.
await _mint_invoice_quote(wallet, settlement)
minted = True
paid_at = int(time.time())
async with create_session() as finalization_session:
settled, api_key_hash = await _finalize_invoice_settlement(
settlement, finalization_session, paid_at
)
if not settled:
await _reload_invoice_view(invoice, session)
return
_publish_invoice_value(invoice, "status", "paid")
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
logger.info(
"Lightning invoice paid",
extra={
"invoice_id": settlement.id,
"amount_sats": settlement.amount_sats,
"purpose": settlement.purpose,
"api_key_hash": api_key_hash[:8] + "..."
if api_key_hash
else None,
},
)
except BaseException as error:
# Never roll back the caller-owned session: doing so expires invoice
# and sibling ORM objects. Owned sessions roll themselves back.
if minted:
logger.critical(
"Invoice mint succeeded but DB finalization failed; reconciliation required",
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
)
try:
await _reload_invoice_view(invoice, session)
except Exception:
pass
if not isinstance(error, Exception):
raise
logger.error(f"Failed to check invoice payment: {error}")
def _is_outputs_already_signed(error: BaseException) -> bool:
message = str(error)
return bool(
re.search(
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
message,
re.IGNORECASE,
)
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
)
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
"""Return spendable wallet value minted by one Lightning quote."""
return sum(
proof.amount
for proof in wallet.proofs
if proof.mint_id == quote_id and not proof.reserved
)
async def _mint_invoice_quote(
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
) -> None:
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
mint_url = invoice.mint_url or settings.primary_mint
await wallet.load_proofs(reload=True)
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
return
try:
await _mint_operation(
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
op_name=f"invoice_mint_{invoice.purpose}",
mint_url=mint_url,
retry_timeouts=False,
)
except Exception as error:
if not _is_outputs_already_signed(error):
raise
logger.error(f"Failed to check invoice payment: {e}")
for keyset_id in wallet.keysets:
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
await wallet.load_proofs(reload=True)
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if recovered < invoice.amount_sats:
raise RuntimeError(
"Invoice outputs were already signed but quote-linked recovery returned "
f"{recovered} sats; expected at least {invoice.amount_sats}"
) from error
else:
await wallet.load_proofs(reload=True)
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if minted_amount < invoice.amount_sats:
raise RuntimeError(
"Invoice mint succeeded but quote-linked proofs total "
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
)
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
return hashlib.sha256(dummy_token.encode()).hexdigest()
async def _create_api_key_record(
invoice: LightningInvoice, session: AsyncSession
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> ApiKey:
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
mint_url = invoice.mint_url or settings.primary_mint
api_key = ApiKey(
hashed_key=hashed_key,
hashed_key=_invoice_api_key_hash(invoice),
balance=invoice.amount_sats * 1000,
refund_currency="sat",
refund_mint_url=settings.primary_mint,
refund_mint_url=mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
@@ -384,21 +535,77 @@ async def _create_api_key_record(
return api_key
async def _credit_topup_record(
invoice: LightningInvoice, session: AsyncSession
async def _topup_api_key_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
if not invoice.api_key_hash:
raise ValueError("No API key associated with topup invoice")
credited = await session.exec( # type: ignore[call-overload]
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
.execution_options(synchronize_session=False)
)
if credited.rowcount != 1:
if result.rowcount != 1:
raise ValueError("Associated API key not found")
INVOICE_WATCH_INTERVAL_SECONDS = 5
async def _finalize_invoice_settlement(
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
) -> tuple[bool, str | None]:
"""Atomically fence and apply one invoice credit in the provided owned session."""
api_key_hash = (
_invoice_api_key_hash(invoice)
if invoice.purpose == "create"
else invoice.api_key_hash
)
claim = await session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(col(LightningInvoice.id) == invoice.id)
.where(col(LightningInvoice.status) == "pending")
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
.execution_options(synchronize_session=False)
)
if claim.rowcount != 1:
await session.rollback()
return False, None
if invoice.purpose == "create":
await _create_api_key_record(invoice, session)
elif invoice.purpose == "topup":
await _topup_api_key_record(invoice, session)
else:
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
await session.commit()
return True, api_key_hash
async def _reload_invoice_view(
invoice: LightningInvoice, _caller_session: AsyncSession
) -> None:
"""Publish committed invoice state without touching the caller transaction."""
async with create_session() as reload_session:
stored = await reload_session.get(LightningInvoice, invoice.id)
if stored is None:
return
status = stored.status
paid_at = stored.paid_at
api_key_hash = stored.api_key_hash
await reload_session.commit()
_publish_invoice_value(invoice, "status", status)
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
async def _credit_topup_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
await _topup_api_key_record(invoice, session)
# Nutshell mints throttle Lightning backend lookups to once per 10s per
# quote, so polling faster just burns the global request budget for nothing.
INVOICE_WATCH_INTERVAL_SECONDS = 10
INVOICE_WATCH_BATCH_LIMIT = 100
+11
View File
@@ -18,6 +18,17 @@ from ..wallet import deserialize_token_from_string
logger = get_logger(__name__)
# Interim policy: when Routstr must move value to another trusted mint, the
# cross-mint Lightning round trip can consume fees that are not visible to the
# client. Reserve 5% headroom until the fee-payer policy is made explicit.
_MINT_FEE_ALLOWANCE = 0.05
def apply_mint_fee_allowance(cost_msat: int) -> int:
"""Reserve headroom for possible trusted-mint fallback fees."""
adjusted = math.ceil(cost_msat * (1 - _MINT_FEE_ALLOWANCE))
return max(settings.min_request_msat, adjusted)
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
if x_cashu := headers.get("x-cashu", None):
+20 -8
View File
@@ -9,7 +9,8 @@ from cashu.wallet.wallet import Proof, Wallet
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
# very slow mint can block a melt (and any caller, e.g. the payout loop)
# indefinitely. Bound it here so callers fail instead of hanging forever.
# indefinitely. _mint_operation (imported lazily in raw_send_to_lnurl to avoid
# a circular import with wallet.py) bounds it via MINT_OPERATION_TIMEOUT_SECONDS.
MELT_TIMEOUT_SECONDS = 60
try:
@@ -221,22 +222,33 @@ async def raw_send_to_lnurl(
lnurl_data["callback_url"], final_amount
)
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
from ..wallet import _mint_operation
melt_quote_resp = await _mint_operation(
lambda: wallet.melt_quote(invoice=bolt11_invoice),
op_name="lnurl_melt_quote",
mint_url=str(wallet.url),
)
if amount:
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
try:
_ = await asyncio.wait_for(
wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
_mint_operation(
lambda: wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
),
op_name="lnurl_melt",
mint_url=str(wallet.url),
retry_timeouts=False,
),
timeout=MELT_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError as e:
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
raise LNURLError(
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
) from e
+25 -12
View File
@@ -26,8 +26,8 @@ from .core.db import (
)
from .core.exceptions import UpstreamError
from .core.not_found import build_not_found_response
from .core.settings import settings
from .payment.helpers import (
apply_mint_fee_allowance,
calculate_discounted_max_cost,
check_token_balance,
create_error_response,
@@ -354,8 +354,7 @@ async def _proxy(
max_cost_for_model = await calculate_discounted_max_cost(
_max_cost_for_model, request_body_dict, model_obj=model_obj
)
# Ensure max_cost_for_model is at least the minimum allowed request cost
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
max_cost_for_model = apply_mint_fee_allowance(max_cost_for_model)
check_token_balance(headers, request_body_dict, max_cost_for_model)
@@ -494,7 +493,9 @@ async def _proxy(
candidate_max = await calculate_discounted_max_cost(
candidate_max, request_body_dict, model_obj=model_obj
)
candidate_max = max(candidate_max, settings.min_request_msat)
# Apply the same interim 5% trusted-mint fee headroom used for the
# first candidate; failover must not silently change admission.
candidate_max = apply_mint_fee_allowance(candidate_max)
if candidate_max > max_cost_for_model:
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
@@ -794,17 +795,29 @@ async def get_bearer_token_key(
},
)
return key
except Exception as e:
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
logger.error(
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
except HTTPException as error:
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
raw_error = detail.get("error")
error_info = raw_error if isinstance(raw_error, dict) else {}
logger.warning(
"Bearer token rejected",
extra={
"error": str(e),
"error_type": type(e).__name__,
"status_code": error.status_code,
"error_code": error_info.get("code"),
"path": path,
"model_id": model_id,
"min_cost_msat": min_cost,
"bearer_key_preview": key_preview,
"required_msat": min_cost,
},
)
raise
except Exception as error:
logger.exception(
"Bearer token validation failed",
extra={
"error_type": type(error).__name__,
"path": path,
"model_id": model_id,
"required_msat": min_cost,
},
)
raise
+20 -3
View File
@@ -12,7 +12,7 @@ from ..core.db import (
from ..core.db import (
store_cashu_transaction_with_retry as store_cashu_transaction,
)
from ..wallet import send_token
from ..wallet import release_token_reservation, send_token, token_mint_url
from .routstr import RoutstrUpstreamProvider
logger = get_logger(__name__)
@@ -144,12 +144,13 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
)
return
actual_mint_url = token_mint_url(token, mint_url)
try:
await store_cashu_transaction(
token=token,
amount=amount,
unit="sat",
mint_url=mint_url,
mint_url=actual_mint_url,
typ="out",
collected=False,
source="auto_topup",
@@ -157,8 +158,24 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
except Exception:
logger.critical(
"Aborting auto top-up because its cashu token could not be persisted",
extra={"provider_id": row.id, "mint_url": mint_url},
extra={"provider_id": row.id, "mint_url": actual_mint_url},
)
try:
await release_token_reservation(token)
except Exception as error:
logger.critical(
"Failed to release untracked auto-topup token",
extra={
"provider_id": row.id,
"mint_url": actual_mint_url,
"error": str(error),
},
)
else:
logger.warning(
"Auto-topup token was released after persistence failed",
extra={"provider_id": row.id, "mint_url": actual_mint_url},
)
return
result = await provider.topup(token)
+4 -3
View File
@@ -53,6 +53,7 @@ from ..wallet import (
classify_redemption_error,
recieve_token,
send_token,
token_mint_url,
)
from . import messages_dispatch
from .cache_breakpoints import (
@@ -3520,7 +3521,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=amount,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
@@ -3873,7 +3874,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=emergency_refund,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
@@ -4843,7 +4844,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=emergency_refund,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
+1076 -128
View File
File diff suppressed because it is too large Load Diff
+25 -3
View File
@@ -207,8 +207,30 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
assert key.balance == model_cost # balance unchanged, only reserved goes up
@pytest.mark.asyncio
async def test_five_percent_mint_fallback_headroom_is_admitted_and_reserved(
integration_session: AsyncSession,
) -> None:
from routstr.auth import pay_for_request, validate_bearer_key
from routstr.payment.helpers import apply_mint_fee_allowance
key = _key(balance=95_000)
integration_session.add(key)
await integration_session.commit()
admission_cost = apply_mint_fee_allowance(100_000)
validated = await validate_bearer_key(
f"sk-{key.hashed_key}", integration_session, min_cost=admission_cost
)
await pay_for_request(validated, admission_cost, integration_session)
await integration_session.refresh(key)
assert admission_cost == 95_000
assert key.reserved_balance == 95_000
# ---------------------------------------------------------------------------
# Test 6 — HTTP layer returns 402 JSON with the right shape
# HTTP layer returns 402 JSON with the right shape
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@@ -266,8 +288,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
error = body["detail"]["error"]
assert error["code"] == "insufficient_balance"
assert error["type"] == "insufficient_quota"
assert str(model_cost) in error["message"]
assert str(user_balance) in error["message"]
assert "591.744 sats (591744 msats) required" in error["message"]
assert "20.32 sats (20320 msats) available" in error["message"]
# Balance must be completely untouched
await integration_session.refresh(key)
@@ -14,6 +14,7 @@ import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy import inspect
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel.ext.asyncio.session import AsyncSession
@@ -22,6 +23,12 @@ from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import _create_api_key_record
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
wallet.proofs = []
wallet.keysets = {}
wallet.load_proofs = AsyncMock()
def _make_invoice(**kwargs: object) -> LightningInvoice:
base = dict(
id="inv_test_001",
@@ -42,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
def mock_wallet_mint() -> object:
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
wallet = AsyncMock()
wallet.mint = AsyncMock(return_value=[])
wallet.proofs = []
wallet.load_proofs = AsyncMock()
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
mock_get_wallet.return_value = wallet
yield mock_get_wallet
@@ -183,6 +198,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
mint_calls = 0
@@ -196,7 +212,9 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
await asyncio.sleep(0.05)
if call_number > 1:
raise Exception("quote already issued")
return []
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=single_use_mint)
@@ -238,6 +256,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
@@ -349,8 +368,15 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock(return_value=[])
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=successful_mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
stored_sibling = await session.get(LightningInvoice, sibling.id)
@@ -428,11 +454,14 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[object]:
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
await asyncio.sleep(0.05)
return []
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
@@ -472,7 +501,7 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
assert first_invoice not in first.dirty
assert second_invoice not in second.dirty
assert wallet.mint.await_count == 2
assert wallet.mint.await_count == 1
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
assert stored_invoice is not None
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
counter = {"n": 0}
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
async def fake_generate(
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
counter["n"] += 1
return (
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
f"payment_hash_{counter['n']}",
"http://localhost:3338",
)
with patch(
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
body = resp.json()
assert body["amount_sats"] == 500
assert body["bolt11"].startswith("lnbc")
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
assert allowed_mints == ["http://localhost:3338"]
@pytest.mark.integration
@@ -0,0 +1,273 @@
import asyncio
import time
import uuid
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel import col, update
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import (
_finalize_invoice_settlement,
_InvoiceSettlement,
check_invoice_payment,
)
def _lightning_invoice(**overrides: object) -> LightningInvoice:
suffix = uuid.uuid4().hex
values = {
"id": f"invoice-{suffix}",
"bolt11": f"lnbc-{suffix}",
"amount_sats": 100,
"description": "settlement test",
"payment_hash": f"quote-{suffix}",
"status": "pending",
"purpose": "create",
"mint_url": "http://mint:3338",
"expires_at": int(time.time()) + 3600,
}
values.update(overrides)
return LightningInvoice(**values) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_read_transaction_closes_before_external_mint_io(
integration_session: AsyncSession,
) -> None:
invoice = _lightning_invoice()
integration_session.add(invoice)
await integration_session.commit()
stored = await integration_session.get(LightningInvoice, invoice.id)
assert stored is not None
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
async def get_wallet_without_open_db_transaction(
*args: object, **kwargs: object
) -> Mock:
assert not integration_session.in_transaction()
return wallet
with patch(
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
):
await check_invoice_payment(stored, integration_session)
assert not integration_session.in_transaction()
@pytest.mark.asyncio
async def test_separate_sessions_cas_topup_credit_exactly_once(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
):
results = await asyncio.gather(
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
)
assert sorted(settled for settled, _ in results) == [False, True]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
stored_key = await verify.get(ApiKey, key_hash)
assert stored_invoice is not None
assert stored_invoice.status == "paid"
assert stored_key is not None
assert stored_key.balance == 200_000
@pytest.mark.asyncio
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
async def debit_balance(session: AsyncSession) -> None:
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == key_hash)
.values(balance=col(ApiKey.balance) - 10_000)
.execution_options(synchronize_session=False)
)
assert result.rowcount == 1
await session.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
AsyncSession(integration_engine, expire_on_commit=False) as debit,
):
settlement_result, _ = await asyncio.gather(
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
debit_balance(debit),
)
assert settlement_result[0]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_key = await verify.get(ApiKey, key_hash)
assert stored_key is not None
assert stored_key.balance == 190_000
@pytest.mark.asyncio
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
with patch.object(
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
):
with pytest.raises(Exception, match="db unavailable"):
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
settled, _ = await _finalize_invoice_settlement(
snapshot, retry, 1_700_000_001
)
assert settled
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
@pytest.mark.asyncio
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
wallet = Mock(
proofs=[],
keysets={"keyset-1": Mock()},
load_proofs=AsyncMock(),
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
restore_tokens_for_keyset=AsyncMock(),
)
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
stored = await failed.get(LightningInvoice, invoice.id)
assert stored is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(side_effect=Exception("db unavailable")),
),
):
await check_invoice_payment(stored, failed)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
stored = await retry.get(LightningInvoice, invoice.id)
assert stored is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
await check_invoice_payment(stored, retry)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
wallet.restore_tokens_for_keyset.assert_not_awaited()
+6 -1
View File
@@ -89,7 +89,12 @@ def _make_swap_mocks(
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
def fake_get_wallet(
mint_url: str,
unit: str = "sat",
load: bool = True,
**kwargs: object,
) -> Mock:
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
return fake_get_wallet
+11 -1
View File
@@ -66,6 +66,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(return_value=True),
) as store,
patch(
"routstr.upstream.auto_topup.token_mint_url",
return_value="https://fallback-mint.test",
),
patch("routstr.upstream.auto_topup.create_session", return_value=session),
):
await _check_and_topup(_row())
@@ -74,7 +78,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
token="cashu-token",
amount=50,
unit="sat",
mint_url="https://mint.test",
mint_url="https://fallback-mint.test",
typ="out",
collected=False,
source="auto_topup",
@@ -138,6 +142,12 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(side_effect=RuntimeError("database unavailable")),
),
patch(
"routstr.upstream.auto_topup.release_token_reservation",
AsyncMock(),
) as reclaim,
):
await _check_and_topup(_row())
reclaim.assert_awaited_once_with("cashu-token")
provider.topup.assert_not_awaited()
+23
View File
@@ -534,6 +534,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
assert exc_info.value.detail == "Cashu mint is unreachable"
@pytest.mark.asyncio
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
from fastapi import HTTPException
from routstr.wallet import SourceMintConnectionError
key = _make_api_key(balance=1000)
session = MagicMock()
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
with (
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
):
with pytest.raises(HTTPException) as exc_info:
await topup_wallet_endpoint(
cashu_token="cashuAtoken", key=key, session=session
)
assert exc_info.value.status_code == 503
assert "cannot be redeemed at another mint" in exc_info.value.detail
@pytest.mark.asyncio
async def test_topup_already_spent_still_returns_400() -> None:
"""Regression: the mint-unreachable short-circuit must not swallow the
+1 -1
View File
@@ -37,7 +37,7 @@ def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
).fetchone()
assert version == ("aa50fde387a2",)
assert version == ("bf76270b66c4",)
assert {
"id",
"accumulated_msats",
+173 -2
View File
@@ -1,9 +1,10 @@
import asyncio
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Generator
from contextlib import asynccontextmanager
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel import SQLModel
@@ -12,6 +13,21 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.wallet import fetch_all_balances
@pytest.fixture(autouse=True)
def clear_balance_fetch_state() -> Generator[None, None, None]:
from routstr import wallet
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
yield
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
@asynccontextmanager
async def _fake_session(): # type: ignore[no-untyped-def]
yield MagicMock()
@@ -29,7 +45,7 @@ def _patches( # type: ignore[no-untyped-def]
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
@@ -63,6 +79,161 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
assert total_wallet == 1000
@pytest.mark.asyncio
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(return_value=["sat"]),
) as supported_units,
):
for p in _patches(proof_amount=1000):
p.start()
try:
details, *_ = await fetch_all_balances()
finally:
patch.stopall()
supported_units.assert_awaited_once_with("http://mint:3338")
assert [detail["unit"] for detail in details] == ["sat"]
@pytest.mark.asyncio
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances()
assert details[0]["unit"] == settings.primary_mint_unit
assert details[0]["error_code"] == "unreachable"
assert details[0]["retry_after_seconds"] > 0
get_wallet.assert_not_awaited()
@pytest.mark.asyncio
async def test_supported_mint_units_come_from_active_keysets() -> None:
from routstr.core.settings import settings
from routstr.wallet import _get_supported_mint_units
# Cashu versions/mints may deserialize keyset units as either strings or
# Unit enum-like objects. Both representations must be accepted.
sat = MagicMock(active=True, unit="sat")
msat = MagicMock(active=False, unit="msat")
usd = MagicMock(active=True)
usd.unit.name = "usd"
wallet = MagicMock()
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
with (
patch.object(settings, "primary_mint_unit", "sat"),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
):
units = await _get_supported_mint_units("http://mint:3338")
cached_units = await _get_supported_mint_units("http://mint:3338")
assert units == ["sat", "usd"]
assert cached_units == units
wallet._get_keysets.assert_awaited_once()
@pytest.mark.asyncio
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
first = await fetch_all_balances(units=["sat"])
second = await fetch_all_balances(units=["sat"])
assert first[0][0]["error"] == "mint unavailable"
assert first[0][0]["error_code"] == "unreachable"
assert first[0][0]["retry_after_seconds"] == 60
assert second[0][0]["error"] == "mint unavailable"
assert second[0][0]["error_code"] == "unreachable"
assert get_wallet.await_count == 1
warning.assert_called_once()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=71),
patch("routstr.wallet.logger.warning"),
):
await fetch_all_balances(units=["sat"])
assert get_wallet.await_count == 2
@pytest.mark.asyncio
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
from routstr.core.settings import settings
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances(units=["sat"])
assert details[0]["error_code"] == "rate_limited"
assert details[0]["retry_after_seconds"] == 60
@pytest.mark.asyncio
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
from routstr.core.settings import settings
from routstr.wallet import _mint_cooldown_remaining
mint = "http://mint:3338"
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", [mint]),
patch.object(settings, "primary_mint", mint),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
details, *_ = await fetch_all_balances(units=["sat", "msat"])
cooldown = _mint_cooldown_remaining(mint)
assert get_wallet.await_count == 1
assert warning.call_count == 1
assert cooldown == 60
assert details[0]["error"] == "mint unavailable"
assert details[0]["error_code"] == "unreachable"
assert details[1]["error"] == "Mint is unreachable"
assert details[1]["error_code"] == "unreachable"
@pytest.mark.asyncio
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
"""Slow mint checks must never run while the balance DB session is open."""
+214
View File
@@ -0,0 +1,214 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import Proof
from routstr.lightning import (
_invoice_settlement_locks,
_is_outputs_already_signed,
_mint_invoice_quote,
check_invoice_payment,
)
from routstr.wallet import Wallet
def _invoice(**overrides: object) -> SimpleNamespace:
values = {
"id": "invoice-1",
"payment_hash": "quote-1",
"amount_sats": 100,
"purpose": "create",
"status": "pending",
"paid_at": None,
"api_key_hash": None,
"mint_url": "http://mint:3338",
"balance_limit": None,
"balance_limit_reset": None,
"validity_date": None,
}
values.update(overrides)
return SimpleNamespace(**values)
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
def _recovery_wallet(
error: Exception,
*,
proofs_before: list[Proof] | None = None,
proofs_after: list[Proof] | None = None,
) -> Mock:
async def load_proofs(*, reload: bool) -> None:
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
wallet.proofs = list(proofs_after)
wallet = Mock(
mint=AsyncMock(side_effect=error),
keysets={"keyset-1": Mock()},
restore_tokens_for_keyset=AsyncMock(),
load_proofs=AsyncMock(side_effect=load_proofs),
proofs=list(proofs_before or []),
)
return wallet
@pytest.mark.asyncio
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
proofs_after=[_proof(100, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
"keyset-1", to=1, batch=25
)
assert wallet.load_proofs.await_count == 2
@pytest.mark.asyncio
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("must not mint"),
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.mint.assert_not_awaited()
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
invoice = _invoice()
error = Exception("backend request 11003 failed")
wallet = _recovery_wallet(error)
with pytest.raises(Exception) as caught:
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
assert caught.value is error
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
response = httpx.Response(
400,
request=request,
json={"detail": "outputs have already been signed before", "code": 11003},
)
with pytest.raises(Exception) as caught:
Wallet.raise_on_error_request(response)
assert _is_outputs_already_signed(caught.value)
@pytest.mark.asyncio
@pytest.mark.parametrize("recovered", [0, 99])
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
recovered: int,
) -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
)
with pytest.raises(RuntimeError, match="expected at least 100"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(10_000, "different-quote")],
)
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_non_pending_invoice_is_not_minted() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="expired")
session = AsyncMock()
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
get_wallet.assert_not_awaited()
session.commit.assert_awaited_once()
@pytest.mark.asyncio
async def test_ambiguous_invoice_mint_timeout_does_not_expose_paid() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert invoice.status == "pending"
session.rollback.assert_not_awaited()
# One commit closes the initial read transaction before external I/O.
session.commit.assert_awaited_once()
@pytest.mark.asyncio
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
async def refresh(obj: SimpleNamespace) -> None:
return None
session.refresh = AsyncMock(side_effect=refresh)
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
yield AsyncMock()
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(return_value=(True, "b" * 64)),
) as finalize,
):
await asyncio.gather(
check_invoice_payment(invoice, session), # type: ignore[arg-type]
check_invoice_payment(invoice, session), # type: ignore[arg-type]
)
assert invoice.status == "paid"
finalize.assert_awaited_once()
+47
View File
@@ -0,0 +1,47 @@
import os
import sqlite3
import subprocess
import sys
from pathlib import Path
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
env = os.environ.copy()
env["DATABASE_URL"] = database_url
subprocess.run(
[sys.executable, "-m", "alembic", command, revision],
cwd=root,
env=env,
check=True,
capture_output=True,
text=True,
)
def _lightning_invoice_columns(database_path: Path) -> set[str]:
with sqlite3.connect(database_path) as connection:
return {
row[1]
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
}
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "mint-url-migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
previous_head = "aa50fde387a2"
_run_alembic(root, database_url, "upgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "bf76270b66c4")
assert "mint_url" in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "downgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "head")
assert "mint_url" in _lightning_invoice_columns(database_path)
+15 -1
View File
@@ -7,7 +7,21 @@ os.environ["UPSTREAM_BASE_URL"] = "http://test"
os.environ["UPSTREAM_API_KEY"] = "test"
from routstr.core.settings import settings # noqa: E402
from routstr.payment.helpers import get_max_cost_for_model # noqa: E402
from routstr.payment.helpers import ( # noqa: E402
apply_mint_fee_allowance,
get_max_cost_for_model,
)
def test_mint_fee_allowance_reserves_five_percent_fallback_headroom() -> None:
# Interim policy: Routstr may pay hidden cross-mint Lightning fees when a
# trusted-mint fallback is required.
assert apply_mint_fee_allowance(124_886) == 118_642
def test_mint_fee_allowance_never_drops_below_minimum() -> None:
with patch.object(settings, "min_request_msat", 100):
assert apply_mint_fee_allowance(50) == 100
async def test_get_max_cost_for_model_known() -> None:
+11 -9
View File
@@ -71,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
@pytest.mark.asyncio
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
async def test_pay_for_request_sets_reserved_at_on_child_key(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="parentkey", balance=10_000)
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
session.add(parent)
@@ -204,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
@pytest.mark.asyncio
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
async def test_release_stale_reservations_skips_null_reserved_at(
session: AsyncSession,
) -> None:
# Reservations without a timestamp may belong to instances running older
# code (rolling deploy) — the background sweeper must not touch them.
key = ApiKey(
@@ -224,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
@pytest.mark.asyncio
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
async def test_reset_all_reserved_balances_clears_reserved_at(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="resetkey",
balance=5_000,
@@ -404,9 +410,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
@@ -418,6 +422,4 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
with pytest.raises(asyncio.CancelledError):
await proxy_module.proxy(request, "v1/chat/completions", session=session)
revert_mock.assert_awaited_once_with(
key, session, 1_000, reservation_snapshot
)
revert_mock.assert_awaited_once_with(key, session, 950, reservation_snapshot)
+2 -4
View File
@@ -383,9 +383,7 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
@@ -408,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
assert RAW_ORG_ID not in serialized
assert "org-[REDACTED]" in serialized
# Single upstream failed -> reservation reverted exactly once (no double-charge).
revert_mock.assert_awaited_once_with(key, session, 1_000, reservation)
revert_mock.assert_awaited_once_with(key, session, 950, reservation)
+1077 -41
View File
File diff suppressed because it is too large Load Diff
+27 -4
View File
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
const formatMintLabel = (detail: BalanceDetail) =>
`${detail.mint_url.replace('https://', '').replace('http://', '')}${detail.unit.toUpperCase()}`;
const formatBalanceError = (detail: BalanceDetail) => {
const labels: Record<string, string> = {
rate_limited: 'rate limited',
unreachable: 'unreachable',
cooldown: 'cooling down',
mint_error: 'mint error',
};
const label =
(detail.error_code ? labels[detail.error_code] : undefined) ??
detail.error ??
'error';
const retryAfter = detail.retry_after_seconds;
return retryAfter && retryAfter > 0
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
: label;
};
return (
<>
<Card>
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
{formatMintLabel(detail)}
</TableCell>
<TableCell className='text-right font-mono'>
<TableCell
className='text-right font-mono'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</TableCell>
<TableCell className='text-right font-mono'>
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
<p className='text-muted-foreground text-xs'>
Wallet
</p>
<p className='font-mono text-sm'>
<p
className='font-mono text-sm'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</p>
</div>
+2
View File
@@ -36,6 +36,8 @@ export interface BalanceDetail {
user_balance: number;
owner_balance: number;
error?: string;
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
retry_after_seconds?: number;
}
export interface WithdrawResponse {