Merge pull request #597 from Routstr/fix/mint-rate-limit-and-fallback

fix: per-mint rate limiting, trusted-mint fallback, and retry factory…
This commit is contained in:
9qeklajc
2026-08-04 00:11:08 +02:00
committed by GitHub
44 changed files with 5504 additions and 671 deletions
+3
View File
@@ -45,6 +45,9 @@ ROUTSTR_SECRET_KEY=
# ENABLE_ANALYTICS_SHARING=true
# CASHU_MINTS="https://mint.minibits.cash/Bitcoin,https://mint.cubabitcoin.org,https://ecashmint.otrta.me"
# MINT_OPERATION_CONCURRENCY=4
# MINT_OPERATION_TIMEOUT_SECONDS=30
# MINT_MAX_CONCURRENCY=4
# MINT_RETRY_MAX_ATTEMPTS=3
# RECEIVE_LN_ADDRESS=
# REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS=900
+9
View File
@@ -136,6 +136,10 @@ Use environment variables for:
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
| `MINT_OPERATION_CONCURRENCY` | Concurrent mint/unit balance reads | `4` |
| `MINT_OPERATION_TIMEOUT_SECONDS` | Per-attempt timeout for mint network calls | `30` |
| `MINT_MAX_CONCURRENCY` | Concurrent operations allowed per mint (`0` disables the limit) | `4` |
| `MINT_RETRY_MAX_ATTEMPTS` | Retries after a timeout or HTTP 429 (`0` disables retries) | `3` |
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
| `MIN_PAYOUT_SAT` | Min payout balance in sats (applies to all mints) | `210` |
| `PAYOUT_INTERVAL_SECONDS` | Payout loop interval (seconds) | `900` |
@@ -145,6 +149,11 @@ Use environment variables for:
| `MODEL_PATHS_REFRESH_INTERVAL_SECONDS` | How often to refresh `/v1/models/paths` discovery data; set `0` to pause the refresh (previously discovered paths keep being served) | `600` |
| `ENABLE_MODEL_PATHS_REFRESH` | Kill switch for the background model-path refresh (OpenRouter endpoint fan-out) | `true` |
Mint HTTP 429 responses create a per-mint cooldown. Operations that already hold
Routstr's wallet mutation lock fail fast during that cooldown instead of waiting
while blocking every other wallet mutation. Callers receive an error and may retry
later; the current response does not include the cooldown duration.
### Priority
Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting.
@@ -0,0 +1,67 @@
"""add mint url to lightning invoices
Revision ID: ecfa0d6e2a36
Revises: 64ed5594df1f
Create Date: 2026-08-02 23:53:00.037456
"""
import json
import os
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "ecfa0d6e2a36"
down_revision = "64ed5594df1f"
branch_labels = None
depends_on = None
def _resolve_backfill_mint_url(bind: sa.engine.Connection) -> str | None:
"""Best-effort resolution of the mint that issued pre-existing invoices.
Order: persisted settings JSON -> PRIMARY_MINT_URL env -> first CASHU_MINTS entry.
"""
try:
row = bind.execute(
sa.text("SELECT data FROM settings ORDER BY id LIMIT 1")
).fetchone()
if row and row[0]:
data = json.loads(row[0])
mint = data.get("primary_mint") or next(
iter(data.get("cashu_mints") or []), None
)
if mint:
return str(mint)
except Exception:
pass
env_mint = os.environ.get("PRIMARY_MINT_URL", "").strip()
if env_mint:
return env_mint
cashu_mints = os.environ.get("CASHU_MINTS", "").strip()
if cashu_mints:
return cashu_mints.split(",")[0].strip() or None
return None
def upgrade() -> None:
op.add_column(
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
)
bind = op.get_bind()
backfill_mint = _resolve_backfill_mint_url(bind)
if backfill_mint:
bind.execute(
sa.text(
"UPDATE lightning_invoices SET mint_url = :mint WHERE mint_url IS NULL"
),
{"mint": backfill_mint},
)
def downgrade() -> None:
op.drop_column("lightning_invoices", "mint_url")
+36 -16
View File
@@ -51,6 +51,24 @@ ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
def _format_msat_amount(amount: int) -> str:
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
return f"{sats} sats ({amount} msats)"
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
return {
"error": {
"message": (
f"Insufficient balance: {_format_msat_amount(required)} required "
f"for this model; {_format_msat_amount(available)} available."
),
"type": "insufficient_quota",
"code": "insufficient_balance",
}
}
@dataclass(frozen=True)
class ReservationSnapshot:
release_id: str
@@ -264,13 +282,7 @@ async def _validate_bearer_key_locked(
)
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(min_cost, billing_key.total_balance),
)
# Early check: Spending limit check (Child key limit)
@@ -360,13 +372,9 @@ async def _validate_bearer_key_locked(
if min_cost > 0 and existing_key.total_balance < min_cost:
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(
min_cost, existing_key.total_balance
),
)
return existing_key
@@ -379,11 +387,23 @@ async def _validate_bearer_key_locked(
"has_expiry_time": bool(key_expiry_time),
},
)
if token_obj.mint in settings.cashu_mints:
if token_obj.mint == settings.primary_mint:
if token_obj.unit != settings.primary_mint_unit:
raise redemption_error_to_http_exception(
ValueError(
"Cashu token unit does not match the configured primary "
f"mint unit: expected {settings.primary_mint_unit}, "
f"got {token_obj.unit}"
)
)
refund_currency = token_obj.unit
refund_mint_url = settings.primary_mint
elif token_obj.mint in settings.cashu_mints:
refund_currency = token_obj.unit
refund_mint_url = token_obj.mint
else:
refund_currency = "sat"
# Foreign tokens are swapped into the configured primary mint.
refund_currency = settings.primary_mint_unit
refund_mint_url = settings.primary_mint
new_key = ApiKey(
+107 -13
View File
@@ -23,6 +23,7 @@ from .core.db import (
from .core.logging import get_logger
from .core.settings import settings
from .lightning import lightning_router
from .payment.lnurl import MeltOutcomeAmbiguousError
from .wallet import (
classify_redemption_error,
credit_balance,
@@ -30,6 +31,7 @@ from .wallet import (
recieve_token,
send_to_lnurl,
send_token,
token_mint_url,
)
router = APIRouter()
@@ -184,6 +186,17 @@ class TopupRequest(BaseModel):
cashu_token: str
def _error_chain(error: BaseException) -> list[dict[str, str]]:
chain: list[dict[str, str]] = []
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
chain.append({"type": type(current).__name__, "message": str(current)})
current = current.__cause__ or current.__context__
return chain
@router.post("/topup")
async def topup_wallet_endpoint(
cashu_token: str | None = None,
@@ -201,6 +214,18 @@ async def topup_wallet_endpoint(
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
if len(cashu_token) < 10 or "cashu" not in cashu_token:
raise HTTPException(status_code=400, detail="Invalid token format")
source_mint = token_mint_url(cashu_token, "unknown")
logger.info(
"Cashu wallet top-up started",
extra={
"event": "cashu_topup_started",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"key_hash": billing_key.hashed_key[:8],
},
)
try:
amount_msats = await credit_balance(cashu_token, billing_key, session)
except Exception as e:
@@ -209,12 +234,41 @@ async def topup_wallet_endpoint(
classified = classify_redemption_error(e)
if classified is None:
logger.error(
"topup_wallet_endpoint: unhandled error",
extra={"error": str(e), "error_type": type(e).__name__},
"Cashu wallet top-up failed with an unhandled error",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=500, detail="Internal server error")
_type, status_code, message, _code = classified
error_type, status_code, message, error_code = classified
logger.warning(
"Cashu wallet top-up failed",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"status_code": status_code,
"error_type": error_type,
"error_code": error_code,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=status_code, detail=message)
logger.info(
"Cashu wallet top-up completed",
extra={
"event": "cashu_topup_completed",
"source_mint": source_mint,
"credited_msats": amount_msats,
"key_hash": billing_key.hashed_key[:8],
},
)
return {"msats": amount_msats}
@@ -290,7 +344,11 @@ async def _get_persisted_api_key_refund(
async def _restore_balance(
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
session: AsyncSession,
hashed_key: str,
balance: int,
reserved_balance: int,
mint_url: str,
) -> None:
"""Restore balance after a failed refund mint attempt."""
restore_stmt = (
@@ -305,7 +363,11 @@ async def _restore_balance(
await session.commit()
logger.info(
"refund_wallet_endpoint: balance restored after mint failure",
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
extra={
"hashed_key": hashed_key,
"restored_balance": balance,
"mint_url": mint_url,
},
)
@@ -450,15 +512,14 @@ async def refund_wallet_endpoint(
detail="Balance changed concurrently. Please retry the refund.",
)
# --- MINT: balance is locked at zero, safe to create the refund token ---
# Proofs from untrusted mints are swapped to primary_mint on receive.
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
# The balance is locked at zero, so it is safe to create the refund token.
effective_refund_mint = (
key.refund_mint_url
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
else settings.primary_mint
)
try:
refund_currency = key.refund_currency or "sat"
if key.refund_address:
await send_to_lnurl(
remaining_balance,
@@ -468,10 +529,10 @@ async def refund_wallet_endpoint(
)
result = {"recipient": key.refund_address}
else:
refund_currency = key.refund_currency or "sat"
token = await send_token(
remaining_balance, refund_currency, effective_refund_mint
)
effective_refund_mint = token_mint_url(token, effective_refund_mint)
result = {"token": token}
if key.refund_currency == "sat":
@@ -490,13 +551,47 @@ async def refund_wallet_endpoint(
},
)
except MeltOutcomeAmbiguousError as e:
# The melt was dispatched and may still settle. Restoring the balance
# here would let the same debit be paid out twice; keep the debit and
# leave the outcome to reconciliation.
logger.error(
"refund_wallet_endpoint: melt outcome ambiguous; balance withheld "
"pending reconciliation",
extra={
"error": str(e),
"hashed_key": key.hashed_key,
"remaining_balance": remaining_balance,
"refund_currency": key.refund_currency,
"refund_mint_url": key.refund_mint_url,
},
)
raise HTTPException(
status_code=502,
detail=(
"Refund was dispatched but its outcome is unconfirmed; the "
"balance is withheld until reconciliation completes"
),
)
except HTTPException:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
raise
except Exception as e:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
error_msg = str(e)
logger.error(
"refund_wallet_endpoint: mint/send failed",
@@ -523,7 +618,7 @@ async def refund_wallet_endpoint(
token=result["token"],
amount=remaining_balance,
unit=key.refund_currency or "sat",
mint_url=key.refund_mint_url,
mint_url=effective_refund_mint,
typ="out",
collected=False,
source="apikey",
@@ -717,7 +812,6 @@ async def reset_child_key_spent(
return {"success": True, "message": "Child key balance reset successfully."}
@router.api_route(
"/{path:path}",
methods=["GET", "POST", "PUT", "DELETE"],
+15 -27
View File
@@ -13,13 +13,7 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from ..payment.models import _row_to_model, list_models
from ..proxy import refresh_model_maps, reinitialize_upstreams
from ..wallet import (
fetch_all_balances,
get_proofs_per_mint_and_unit,
get_wallet,
send_token,
slow_filter_spend_proofs,
)
from ..wallet import fetch_all_balances, send_token, token_mint_url
from . import vault
from .db import (
ApiKey,
@@ -442,37 +436,31 @@ class WithdrawRequest(BaseModel):
async def withdraw(
request: Request, withdraw_request: WithdrawRequest
) -> dict[str, str]:
# Get wallet and check balance
from .settings import settings as global_settings
effective_mint = withdraw_request.mint_url or global_settings.primary_mint
wallet = await get_wallet(effective_mint, withdraw_request.unit)
proofs = get_proofs_per_mint_and_unit(
wallet,
effective_mint,
withdraw_request.unit,
not_reserved=True,
)
proofs = await slow_filter_spend_proofs(proofs, wallet)
current_balance = sum(proof.amount for proof in proofs)
if withdraw_request.amount <= 0:
raise HTTPException(
status_code=400, detail="Withdrawal amount must be positive"
)
if withdraw_request.amount > current_balance:
raise HTTPException(status_code=400, detail="Insufficient wallet balance")
token = await send_token(
withdraw_request.amount, withdraw_request.unit, effective_mint
)
try:
token = await send_token(
withdraw_request.amount, withdraw_request.unit, effective_mint
)
except ValueError as error:
if not str(error).startswith("No trusted mint has "):
raise
raise HTTPException(
status_code=400, detail="Insufficient wallet balance"
) from error
actual_mint = token_mint_url(token, effective_mint)
try:
await store_cashu_transaction(
token=token,
amount=withdraw_request.amount,
unit=withdraw_request.unit,
mint_url=effective_mint,
mint_url=actual_mint,
typ="out",
collected=False,
source="admin",
@@ -483,10 +471,10 @@ async def withdraw(
extra={
"amount": withdraw_request.amount,
"unit": withdraw_request.unit,
"mint_url": effective_mint,
"mint_url": actual_mint,
},
)
return {"token": token}
return {"token": token, "mint_url": actual_mint}
class ModelCreate(BaseModel):
+11 -3
View File
@@ -293,7 +293,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
"""Delete dead parentless API keys; return the count removed.
Dead = 0 balance/reservation/spend/requests, older than the grace period,
no parent, no children, no pending invoice. Cashu rows are unlinked (not
no parent, no children, no retryable invoice. Cashu rows are unlinked (not
deleted) first to keep the audit trail.
"""
cutoff = int(time.time()) - min_age_seconds
@@ -307,7 +307,9 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
pending_invoice = (
select(LightningInvoice.id)
.where(col(LightningInvoice.api_key_hash) == col(ApiKey.hashed_key))
.where(col(LightningInvoice.status) == "pending")
.where(
col(LightningInvoice.status).in_(("pending", "settlement_pending"))
)
).exists()
eligible_hashes = (
@@ -435,12 +437,18 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
payment_hash: str = Field(description="Payment hash for tracking", unique=True)
status: str = Field(
default="pending",
description="pending, paid, expired, cancelled, reconciliation_required",
description=(
"pending, settlement_pending, paid, expired, cancelled, "
"reconciliation_required"
),
)
api_key_hash: str | None = Field(
default=None, description="Associated API key hash for topup operations"
)
purpose: str = Field(description="create or topup")
mint_url: str | None = Field(
default=None, description="Mint URL where the quote was created (fallback tracking)"
)
created_at: int = Field(
default_factory=lambda: int(time.time()), description="Unix timestamp"
)
+12
View File
@@ -53,6 +53,18 @@ class Settings(BaseSettings):
payout_interval_seconds: int = Field(
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
)
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
# checkstate). When a mint is slow or rate-limiting, operations are
# cancelled after this delay instead of hanging indefinitely.
mint_operation_timeout_seconds: int = Field(
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
)
# Maximum concurrent API operations per mint. Actual mint quotas vary by
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
# pacing. 0 = unlimited concurrency.
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
# Max retries when a mint returns 429 or times out (exponential backoff).
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
# Pricing
# Default behavior: derive pricing from MODELS
+541 -175
View File
@@ -1,8 +1,13 @@
import asyncio
import hashlib
import re
import secrets
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import Any, AsyncGenerator
from cashu.core.base import MintQuoteState
from fastapi import APIRouter, Depends, Header, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.orm.attributes import set_committed_value
@@ -12,12 +17,85 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from .core.db import ApiKey, LightningInvoice, create_session, get_session
from .core.logging import get_logger
from .core.settings import settings
from .wallet import get_wallet, wallet_operation_guard
from .mint import (
is_mint_rate_limited,
mint_cooldown_remaining,
run_mint_operation,
)
from .wallet import (
MintConnectionError,
get_wallet,
is_mint_connection_error,
wallet_operation_guard,
)
logger = get_logger(__name__)
lightning_router = APIRouter(prefix="/lightning")
# Avoid duplicate work within one process. Cross-process settlement is fenced
# by claiming a paid quote before minting and by the final conditional update.
@dataclass
class _InvoiceLockEntry:
lock: asyncio.Lock
users: int = 0
_invoice_settlement_locks: dict[str, _InvoiceLockEntry] = {}
@asynccontextmanager
async def _invoice_settlement_lock(invoice_id: str) -> AsyncGenerator[None, None]:
"""Serialize one invoice and remove its lock after the last waiter leaves."""
entry = _invoice_settlement_locks.get(invoice_id)
if entry is None:
entry = _InvoiceLockEntry(asyncio.Lock())
_invoice_settlement_locks[invoice_id] = entry
entry.users += 1
try:
async with entry.lock:
yield
finally:
entry.users -= 1
if entry.users == 0 and _invoice_settlement_locks.get(invoice_id) is entry:
del _invoice_settlement_locks[invoice_id]
@dataclass(frozen=True)
class _InvoiceSettlement:
id: str
payment_hash: str
amount_sats: int
purpose: str
api_key_hash: str | None
mint_url: str | None
balance_limit: int | None
balance_limit_reset: str | None
validity_date: int | None
@classmethod
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
return cls(
id=invoice.id,
payment_hash=invoice.payment_hash,
amount_sats=invoice.amount_sats,
purpose=invoice.purpose,
api_key_hash=invoice.api_key_hash,
mint_url=invoice.mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
)
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
"""Update a caller view without marking a mapped object dirty."""
try:
set_committed_value(invoice, key, value)
except AttributeError:
setattr(invoice, key, value)
class InvoiceCreateRequest(BaseModel):
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
@@ -61,16 +139,102 @@ class InvoiceStatusResponse(BaseModel):
expires_at: int
_RETRYABLE_INVOICE_STATUSES = ("pending", "settlement_pending")
class InvoiceRecoverRequest(BaseModel):
bolt11: str = Field(description="BOLT11 invoice string")
def _trusted_mint_candidates() -> list[str]:
return [
mint
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
if mint
]
async def _request_mint_with_fallback(
amount_sats: int,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
"""Request a quote, falling back only among the allowed trusted mints.
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
cryptic validation error deep in the stack. Fail fast with context.
"""
if amount_sats <= 0:
raise ValueError(
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
)
tried: list[str] = []
trusted = _trusted_mint_candidates()
if allowed_mints:
# Persisted mint preferences (e.g. an API key's refund_mint_url) must
# not outlive the operator's trusted-mint configuration.
candidates = [m for m in dict.fromkeys(allowed_mints) if m in trusted]
if not candidates:
logger.warning(
"Requested mints are no longer trusted; falling back to "
"configured mints",
extra={
"requested_mints": list(dict.fromkeys(allowed_mints)),
"op_name": "request_mint_invoice",
},
)
candidates = trusted
else:
candidates = trusted
for mint_url in candidates:
cooldown = mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.info(
"Skipping mint during cooldown",
extra={
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"op_name": "request_mint_invoice",
},
)
continue
try:
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
quote = await run_mint_operation(
lambda: wallet.request_mint(amount_sats),
op_name="request_mint_invoice",
mint_url=mint_url,
retry_on_rate_limit=False,
)
return quote.request, quote.quote, mint_url
except Exception as e:
tried.append(f"{mint_url}: {type(e).__name__}")
if not is_mint_connection_error(e) and not is_mint_rate_limited(e):
raise
logger.warning(
"request_mint failed, trying fallback mint",
extra={
"failed_mint": mint_url,
"error": str(e),
"tried": tried,
},
)
continue
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
async def generate_lightning_invoice(
amount_sats: int, description: str
) -> tuple[str, str]:
wallet = await get_wallet(settings.primary_mint, "sat")
quote = await wallet.request_mint(amount_sats)
return quote.request, quote.quote
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
amount_sats, allowed_mints=allowed_mints
)
return bolt11, payment_hash, mint_url
def generate_invoice_id() -> str:
@@ -84,6 +248,7 @@ async def create_invoice(
session: AsyncSession = Depends(get_session),
) -> InvoiceCreateResponse:
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
topup_api_key: ApiKey | None = None
if request.purpose == "topup":
if not api_key_token:
@@ -94,14 +259,23 @@ async def create_invoice(
if not api_key_token.startswith("sk-"):
raise HTTPException(status_code=400, detail="Invalid API key format")
api_key = await session.get(ApiKey, api_key_token[3:])
if not api_key:
topup_api_key = await session.get(ApiKey, api_key_token[3:])
if not topup_api_key:
raise HTTPException(status_code=404, detail="API key not found")
try:
description = f"Routstr {request.purpose} {request.amount_sats} sats"
bolt11, payment_hash = await generate_lightning_invoice(
request.amount_sats, description
allowed_mints = None
if request.purpose == "topup":
assert topup_api_key is not None
# A key's liabilities are attributed to a single refund mint. Keep
# top-up collateral on that same mint so balances and payouts cannot
# misclassify funds held by another mint as owner profit.
allowed_mints = [
topup_api_key.refund_mint_url or settings.primary_mint
]
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
request.amount_sats, description, allowed_mints=allowed_mints
)
invoice_id = generate_invoice_id()
@@ -116,6 +290,7 @@ async def create_invoice(
status="pending",
api_key_hash=api_key_token[3:] if api_key_token else None,
purpose=request.purpose,
mint_url=mint_url,
balance_limit=request.balance_limit,
balance_limit_reset=request.balance_limit_reset,
validity_date=request.validity_date,
@@ -161,12 +336,12 @@ async def get_invoice_status(
if not invoice:
raise HTTPException(status_code=404, detail="Invoice not found")
if invoice.status == "pending":
await check_invoice_payment(invoice, session)
if invoice.status == "pending" and int(time.time()) > invoice.expires_at:
invoice.status = "expired"
await session.commit()
definitively_unpaid = False
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
api_key = None
if invoice.status == "paid" and invoice.purpose == "create":
@@ -200,8 +375,12 @@ async def recover_invoice(
if not invoice:
raise HTTPException(status_code=404, detail="Invoice not found")
if invoice.status == "pending":
await check_invoice_payment(invoice, session)
definitively_unpaid = False
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
api_key = None
if invoice.status == "paid":
@@ -220,161 +399,256 @@ async def recover_invoice(
)
async def _claim_paid_invoice_for_settlement(
invoice: LightningInvoice,
caller_session: AsyncSession,
observed_status: str,
) -> bool:
"""Claim an authoritative paid quote before consuming it at the mint."""
if observed_status == "settlement_pending":
return True
if observed_status != "pending":
await _reload_invoice_view(invoice, caller_session)
return False
async with create_session() as claim_session:
claim = await claim_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status) == "pending",
)
.values(status="settlement_pending")
.execution_options(synchronize_session=False)
)
await claim_session.commit()
if claim.rowcount != 1:
await _reload_invoice_view(invoice, caller_session)
return False
_publish_invoice_value(invoice, "status", "settlement_pending")
return True
async def check_invoice_payment(
invoice: LightningInvoice, session: AsyncSession
) -> None:
# Minting makes proofs visible before database finalization. Share the
# cross-process wallet guard with owner payout so that visibility and the
# corresponding liability commit are observed atomically by the payout loop.
async with wallet_operation_guard():
await _check_invoice_payment_locked(invoice, session)
) -> bool:
"""Settle an invoice and report whether its quote is definitively unpaid.
False covers paid, pending, and ambiguous transport/DB outcomes so callers
never expire a quote merely because reconciliation could not complete.
"""
async with _invoice_settlement_lock(invoice.id), wallet_operation_guard():
minted = False
payment_confirmed = False
try:
# Snapshot the row and end the caller's read transaction before any
# potentially slow mint I/O. All final DB mutations use owned,
# short-lived sessions below.
await session.refresh(invoice)
if invoice.status not in _RETRYABLE_INVOICE_STATUSES:
await session.commit()
return False
observed_status = invoice.status
settlement = _InvoiceSettlement.from_invoice(invoice)
await session.commit()
async def _check_invoice_payment_locked(
invoice: LightningInvoice, session: AsyncSession
) -> None:
minted = False
invoice_id = invoice.id
invoice_purpose = invoice.purpose
invoice_amount_sats = invoice.amount_sats
invoice_payment_hash = invoice.payment_hash
finalized_api_key_hash = invoice.api_key_hash
try:
# A preceding invoice lookup starts a transaction. End it before the
# potentially slow mint request so it cannot pin a pool connection.
await session.commit()
mint_url = settlement.mint_url or settings.primary_mint
wallet = await get_wallet(mint_url, "sat")
mint_status = await run_mint_operation(
lambda: wallet.get_mint_quote(settlement.payment_hash),
op_name="get_mint_quote",
mint_url=mint_url,
)
if not mint_status.paid:
return getattr(mint_status, "state", None) == MintQuoteState.unpaid
payment_confirmed = True
wallet = await get_wallet(settings.primary_mint, "sat")
mint_status = await wallet.get_mint_quote(invoice_payment_hash)
if not mint_status.paid:
return
# Fence expiry and other workers before consuming the paid quote.
# If a concurrent expiry/finalization won, this worker must not mint.
if not await _claim_paid_invoice_for_settlement(
invoice, session, observed_status
):
return False
# Do not redeem a paid top-up quote if its target has already been
# pruned. This validation owns a short-lived session and releases its
# connection before mint redemption starts.
if invoice_purpose == "topup":
if not finalized_api_key_hash:
raise ValueError("No API key associated with topup invoice")
async with create_session() as validation_session:
target = await validation_session.get(ApiKey, finalized_api_key_hash)
if target is None:
terminal = await validation_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice_id,
col(LightningInvoice.status) == "pending",
)
.values(status="reconciliation_required")
# Reject a paid top-up whose target was pruned before redeeming its
# single-use quote. The validation session is closed before mint I/O.
if settlement.purpose == "topup":
if not settlement.api_key_hash:
raise ValueError("No API key associated with topup invoice")
async with create_session() as validation_session:
target = await validation_session.get(
ApiKey, settlement.api_key_hash
)
await validation_session.commit()
if terminal.rowcount == 1:
set_committed_value(
invoice, "status", "reconciliation_required"
)
else:
committed_invoice = await validation_session.get(
LightningInvoice, invoice_id
)
if committed_invoice is not None:
set_committed_value(
invoice, "status", committed_invoice.status
if target is None:
terminal = await validation_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == settlement.id,
col(LightningInvoice.status).in_(
_RETRYABLE_INVOICE_STATUSES
),
)
logger.critical(
"Paid topup invoice target API key was not found; reconciliation required",
extra={"invoice_id": invoice_id},
)
return
.values(status="reconciliation_required")
)
await validation_session.commit()
if terminal.rowcount == 1:
_publish_invoice_value(
invoice, "status", "reconciliation_required"
)
else:
await _reload_invoice_view(invoice, session)
logger.critical(
"Paid topup invoice target API key was not found; reconciliation required",
extra={"invoice_id": settlement.id},
)
return False
# The mint enforces single-use quotes, so a concurrent checker that
# races us here fails inside wallet.mint rather than double-minting.
await wallet.mint(invoice_amount_sats, quote_id=invoice_payment_hash)
minted = True
# Quote-linked proof verification makes an ambiguous mint response
# retryable without crediting unrelated wallet balance growth.
await _mint_invoice_quote(wallet, settlement)
minted = True
# Paid finalization owns a fresh session. The API/watcher session is
# never rolled back by this function, so its invoice and sibling ORM
# objects remain usable after a DB failure or lost CAS race.
async with create_session() as finalization_session:
if invoice_purpose == "create":
api_key = await _create_api_key_record(invoice, finalization_session)
finalized_api_key_hash = api_key.hashed_key
elif invoice_purpose == "topup":
await _credit_topup_record(invoice, finalization_session)
# Conditional transition guards against double-credit: the credit
# above and this status flip commit atomically, and a lost race
# rolls both back in the owned finalization session.
paid_at = int(time.time())
finalized = await finalization_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice_id,
col(LightningInvoice.status) == "pending",
)
.values(
status="paid",
paid_at=paid_at,
api_key_hash=finalized_api_key_hash,
async with create_session() as finalization_session:
settled, api_key_hash = await _finalize_invoice_settlement(
settlement, finalization_session, paid_at
)
if not settled:
await _reload_invoice_view(invoice, session)
return False
_publish_invoice_value(invoice, "status", "paid")
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
logger.info(
"Lightning invoice paid",
extra={
"invoice_id": settlement.id,
"amount_sats": settlement.amount_sats,
"purpose": settlement.purpose,
"api_key_hash": api_key_hash[:8] + "..."
if api_key_hash
else None,
},
)
if finalized.rowcount != 1:
await finalization_session.rollback()
committed_invoice = await finalization_session.get(
LightningInvoice, invoice_id
)
await finalization_session.commit()
if committed_invoice is not None:
# A concurrent finalizer won the CAS. Publish only the
# state observed from the database after ending the owned
# read transaction; never refresh the caller's session.
set_committed_value(
invoice, "api_key_hash", committed_invoice.api_key_hash
return False
except BaseException as error:
# Never roll back the caller-owned session: doing so expires invoice
# and sibling ORM objects. Owned sessions roll themselves back.
if payment_confirmed and invoice.status != "settlement_pending":
try:
async with create_session() as state_session:
pending = await state_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status).in_(
_RETRYABLE_INVOICE_STATUSES
),
)
.values(status="settlement_pending")
)
await state_session.commit()
if pending.rowcount == 1:
_publish_invoice_value(
invoice, "status", "settlement_pending"
)
except Exception as state_error:
logger.critical(
"Paid invoice reconciliation state could not be persisted",
extra={"invoice_id": invoice.id, "error": str(state_error)},
)
set_committed_value(invoice, "status", committed_invoice.status)
set_committed_value(invoice, "paid_at", committed_invoice.paid_at)
return
await finalization_session.commit()
if minted:
logger.critical(
"Invoice mint succeeded but DB finalization failed; reconciliation required",
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
)
try:
await _reload_invoice_view(invoice, session)
except Exception:
pass
if not isinstance(error, Exception):
raise
logger.error(f"Failed to check invoice payment: {error}")
return False
# Only publish finalized values to the caller-owned object after the
# owned transaction has committed successfully.
set_committed_value(invoice, "api_key_hash", finalized_api_key_hash)
set_committed_value(invoice, "status", "paid")
set_committed_value(invoice, "paid_at", paid_at)
logger.info(
"Lightning invoice paid",
extra={
"invoice_id": invoice_id,
"amount_sats": invoice_amount_sats,
"purpose": invoice_purpose,
"api_key_hash": finalized_api_key_hash[:8] + "..."
if finalized_api_key_hash
else None,
},
def _is_outputs_already_signed(error: BaseException) -> bool:
message = str(error)
return bool(
re.search(
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
message,
re.IGNORECASE,
)
except BaseException as e:
# BaseException so task cancellation (e.g. client disconnect) after a
# successful mint still triggers the reconciliation alert. Any rollback
# belongs to create_session(), never to the caller-owned session.
if minted:
logger.critical(
"Invoice mint succeeded but DB finalization failed; reconciliation required",
extra={"invoice_id": invoice_id, "purpose": invoice_purpose},
)
if not isinstance(e, Exception):
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
)
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
"""Return spendable wallet value minted by one Lightning quote."""
return sum(
proof.amount
for proof in wallet.proofs
if proof.mint_id == quote_id and not proof.reserved
)
async def _mint_invoice_quote(
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
) -> None:
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
mint_url = invoice.mint_url or settings.primary_mint
await wallet.load_proofs(reload=True)
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
return
try:
await run_mint_operation(
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
op_name=f"invoice_mint_{invoice.purpose}",
mint_url=mint_url,
retry_timeouts=False,
)
except Exception as error:
if not _is_outputs_already_signed(error):
raise
logger.error(f"Failed to check invoice payment: {e}")
for keyset_id in wallet.keysets:
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
await wallet.load_proofs(reload=True)
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if recovered < invoice.amount_sats:
raise RuntimeError(
"Invoice outputs were already signed but quote-linked recovery returned "
f"{recovered} sats; expected at least {invoice.amount_sats}"
) from error
else:
await wallet.load_proofs(reload=True)
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if minted_amount < invoice.amount_sats:
raise RuntimeError(
"Invoice mint succeeded but quote-linked proofs total "
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
)
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
return hashlib.sha256(dummy_token.encode()).hexdigest()
async def _create_api_key_record(
invoice: LightningInvoice, session: AsyncSession
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> ApiKey:
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
mint_url = invoice.mint_url or settings.primary_mint
api_key = ApiKey(
hashed_key=hashed_key,
hashed_key=_invoice_api_key_hash(invoice),
balance=invoice.amount_sats * 1000,
refund_currency="sat",
refund_mint_url=settings.primary_mint,
refund_mint_url=mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
@@ -384,50 +658,142 @@ async def _create_api_key_record(
return api_key
async def _credit_topup_record(
invoice: LightningInvoice, session: AsyncSession
async def _topup_api_key_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
if not invoice.api_key_hash:
raise ValueError("No API key associated with topup invoice")
credited = await session.exec( # type: ignore[call-overload]
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
.execution_options(synchronize_session=False)
)
if credited.rowcount != 1:
if result.rowcount != 1:
raise ValueError("Associated API key not found")
INVOICE_WATCH_INTERVAL_SECONDS = 5
async def _finalize_invoice_settlement(
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
) -> tuple[bool, str | None]:
"""Atomically fence and apply one invoice credit in the provided owned session."""
api_key_hash = (
_invoice_api_key_hash(invoice)
if invoice.purpose == "create"
else invoice.api_key_hash
)
claim = await session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(col(LightningInvoice.id) == invoice.id)
.where(
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
)
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
.execution_options(synchronize_session=False)
)
if claim.rowcount != 1:
await session.rollback()
return False, None
if invoice.purpose == "create":
await _create_api_key_record(invoice, session)
elif invoice.purpose == "topup":
await _topup_api_key_record(invoice, session)
else:
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
await session.commit()
return True, api_key_hash
async def _reload_invoice_view(
invoice: LightningInvoice, _caller_session: AsyncSession
) -> None:
"""Publish committed invoice state without touching the caller transaction."""
async with create_session() as reload_session:
stored = await reload_session.get(LightningInvoice, invoice.id)
if stored is None:
return
status = stored.status
paid_at = stored.paid_at
api_key_hash = stored.api_key_hash
await reload_session.commit()
_publish_invoice_value(invoice, "status", status)
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
async def _expire_invoice_if_authoritatively_unpaid(
invoice: LightningInvoice,
caller_session: AsyncSession,
definitively_unpaid: bool,
) -> bool:
"""Expire one overdue unpaid invoice without overwriting concurrent settlement."""
if (
not definitively_unpaid
or invoice.status != "pending"
or int(time.time()) <= invoice.expires_at
):
return False
async with create_session() as expiry_session:
expired = await expiry_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status) == "pending",
)
.values(status="expired")
.execution_options(synchronize_session=False)
)
await expiry_session.commit()
if expired.rowcount == 1:
_publish_invoice_value(invoice, "status", "expired")
return True
await _reload_invoice_view(invoice, caller_session)
return False
async def _credit_topup_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
await _topup_api_key_record(invoice, session)
# Nutshell mints throttle Lightning backend lookups to once per 10s per
# quote, so polling faster just burns the global request budget for nothing.
INVOICE_WATCH_INTERVAL_SECONDS = 10
INVOICE_WATCH_BATCH_LIMIT = 100
async def periodic_invoice_watcher() -> None:
"""Background task: detect paid Lightning invoices and credit balances.
async def _process_invoice_watch_batch(session: AsyncSession) -> None:
result = await session.exec(
select(LightningInvoice)
.where(
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
)
.limit(INVOICE_WATCH_BATCH_LIMIT)
)
for invoice in result.all():
try:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
except Exception as e:
logger.error(
"Invoice watcher failed for invoice",
extra={"invoice_id": invoice.id, "error": str(e)},
)
Removes the need for clients to poll the status endpoint after paying.
"""
async def periodic_invoice_watcher() -> None:
"""Background task: detect paid Lightning invoices and credit balances."""
while True:
try:
async with create_session() as session:
now = int(time.time())
result = await session.exec(
select(LightningInvoice)
.where(
LightningInvoice.status == "pending",
col(LightningInvoice.expires_at) > now,
)
.limit(INVOICE_WATCH_BATCH_LIMIT)
)
pending = result.all()
for invoice in pending:
try:
await check_invoice_payment(invoice, session)
except Exception as e:
logger.error(
"Invoice watcher failed for invoice",
extra={"invoice_id": invoice.id, "error": str(e)},
)
await _process_invoice_watch_batch(session)
except asyncio.CancelledError:
raise
except Exception as e:
+343
View File
@@ -0,0 +1,343 @@
"""Shared policy for bounded, rate-aware Cashu mint API operations."""
from __future__ import annotations
import asyncio
import socket
import time
from contextlib import asynccontextmanager
from contextvars import ContextVar
from typing import Any, AsyncGenerator, Awaitable, Callable
import httpx
from .core.logging import get_logger
from .core.settings import settings
logger = get_logger(__name__)
MINT_TRANSPORT_EXCEPTIONS: tuple[type[BaseException], ...] = (
httpx.NetworkError,
httpx.TimeoutException,
ConnectionError,
socket.gaierror,
asyncio.TimeoutError,
)
MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
_fail_fast_depth: ContextVar[int] = ContextVar("mint_fail_fast_depth", default=0)
class MintRateLimitedError(httpx.HTTPStatusError):
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
class MintCooldownError(Exception):
"""A mint is cooling down and this operation must not wait."""
def __init__(self, mint_url: str, retry_after_seconds: float):
self.mint_url = mint_url
self.retry_after_seconds = max(0.0, retry_after_seconds)
super().__init__(
f"Mint {mint_url} is cooling down; retry after "
f"{self.retry_after_seconds:.2f}s"
)
@asynccontextmanager
async def fail_fast_mint_operations() -> AsyncGenerator[None, None]:
"""Make mint cooldown/probe waits fail fast in the current task.
Wallet mutation code holds a process-wide file lock. It enters this scope so
an existing mint cooldown can never turn that lock into a multi-hour wait.
"""
token = _fail_fast_depth.set(_fail_fast_depth.get() + 1)
try:
yield
finally:
_fail_fast_depth.reset(token)
class MintRateGuard:
"""Limit concurrency and remember per-mint cooldown/probe state."""
_guards: dict[str, "MintRateGuard"] = {}
@classmethod
def get(cls, mint_url: str) -> "MintRateGuard":
concurrency = settings.mint_max_concurrency
guard = cls._guards.get(mint_url)
if guard is None or guard._max_concurrency != concurrency:
previous = guard
guard = cls(mint_url, concurrency)
if previous is not None:
# Concurrency changed at runtime: keep the live cooldown/backoff
# state so an active 429 cooldown is not silently discarded.
guard._cooldown_until = previous._cooldown_until
guard._cooldown_reason = previous._cooldown_reason
guard._consecutive_rate_limits = previous._consecutive_rate_limits
guard._needs_probe = previous._needs_probe
cls._guards[mint_url] = guard
return guard
def __init__(self, mint_url: str, max_concurrency: int):
self._mint_url = mint_url
self._max_concurrency = max_concurrency
self._semaphore = (
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
)
self._cooldown_until = 0.0
self._cooldown_reason: str | None = None
self._consecutive_rate_limits = 0
self._needs_probe = False
self._probe_lock = asyncio.Lock()
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
deadline = time.monotonic() + max(0.0, delay)
if deadline >= self._cooldown_until:
self._cooldown_until = deadline
if reason is not None:
self._cooldown_reason = reason
elif self._cooldown_reason is None and reason is not None:
self._cooldown_reason = reason
self._needs_probe = True
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
remaining = self.cooldown_remaining()
if remaining > 0 and self._cooldown_reason == "rate_limited":
minimum = min(
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
)
if minimum > remaining:
self.apply_cooldown(minimum, reason="rate_limited")
return minimum
return remaining
self._consecutive_rate_limits += 1
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
self.apply_cooldown(delay, reason="rate_limited")
return delay
def cooldown_remaining(self) -> float:
return max(0.0, self._cooldown_until - time.monotonic())
def cooldown_reason(self) -> str | None:
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
def _raise_if_wait_forbidden(self) -> None:
remaining = self.cooldown_remaining()
if _fail_fast_depth.get() and remaining > 0:
raise MintCooldownError(self._mint_url, remaining)
async def _wait_for_cooldown(self) -> None:
while True:
self._raise_if_wait_forbidden()
deadline = self._cooldown_until
wait = max(0.0, deadline - time.monotonic())
if wait <= 0:
return
logger.debug(
"Mint rate guard: cooling down",
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
)
await asyncio.sleep(wait)
if self._cooldown_until <= deadline:
return
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
await self._wait_for_cooldown()
logger.info(
"Mint cooldown ended; sending one probe request",
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
)
try:
result = await factory()
except Exception as error:
if is_mint_rate_limited(error):
retry_after = None
if isinstance(error, httpx.HTTPStatusError):
retry_after = parse_retry_after(error.response.headers)
self.apply_rate_limit_cooldown(retry_after)
else:
self.apply_cooldown(1.0)
logger.warning(
"Mint cooldown probe failed",
extra={
"event": "mint_cooldown_probe_failed",
"mint_url": self._mint_url,
"error": str(error),
"error_type": type(error).__name__,
"cooldown_seconds": round(self.cooldown_remaining(), 2),
"consecutive_rate_limits": self._consecutive_rate_limits,
},
)
raise
self._needs_probe = False
self._cooldown_until = 0.0
self._cooldown_reason = None
self._consecutive_rate_limits = 0
logger.info(
"Mint cooldown probe succeeded; restoring normal concurrency",
extra={
"event": "mint_cooldown_probe_succeeded",
"mint_url": self._mint_url,
},
)
return result
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
while True:
self._raise_if_wait_forbidden()
if self._needs_probe or self.cooldown_remaining() > 0:
if _fail_fast_depth.get() and self._probe_lock.locked():
raise MintCooldownError(self._mint_url, self.cooldown_remaining())
async with self._probe_lock:
self._raise_if_wait_forbidden()
if self.cooldown_remaining() > 0:
self._needs_probe = True
if self._needs_probe:
return await self._run_probe(factory)
continue
if self._semaphore is None:
return await factory()
async with self._semaphore:
self._raise_if_wait_forbidden()
if self._needs_probe:
continue
return await factory()
def mint_cooldown_remaining(mint_url: str) -> float:
return MintRateGuard.get(mint_url).cooldown_remaining()
def mint_cooldown_reason(mint_url: str) -> str | None:
return MintRateGuard.get(mint_url).cooldown_reason()
def is_mint_rate_limited(error: BaseException) -> bool:
"""Return whether an exception chain represents HTTP 429/cooldown."""
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, MintCooldownError):
return True
if isinstance(current, httpx.HTTPStatusError):
if current.response.status_code == 429:
return True
current = current.__cause__ or current.__context__
return False
def parse_retry_after(headers: Any) -> float | None:
raw = headers.get("retry-after") or headers.get("Retry-After")
if raw is None:
return None
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return None
async def run_mint_operation(
factory: Callable[[], Awaitable[Any]],
*,
op_name: str = "mint_operation",
mint_url: str = "",
retry_timeouts: bool = True,
retry_on_rate_limit: bool = True,
) -> Any:
"""Run one mint operation with bounded concurrency and adaptive cooldown."""
guard = MintRateGuard.get(mint_url) if mint_url else None
timeout = settings.mint_operation_timeout_seconds
max_attempts = settings.mint_retry_max_attempts + 1
async def timed_factory() -> Any:
if timeout > 0:
return await asyncio.wait_for(factory(), timeout=timeout)
return await factory()
async def invoke() -> Any:
if guard is not None:
return await guard.run(timed_factory)
return await timed_factory()
for attempt in range(max_attempts):
try:
return await invoke()
except MintCooldownError:
raise
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
if retry_timeouts and attempt < max_attempts - 1:
backoff = (2**attempt) + (time.monotonic() % 1.0)
logger.warning(
"Mint operation timed out, retrying",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"backoff_seconds": round(backoff, 2),
},
)
await asyncio.sleep(backoff)
continue
raise httpx.TimeoutException(
f"{op_name} timed out (attempts: {attempt + 1})"
) from exc
except Exception as exc:
if not is_mint_rate_limited(exc):
raise
backoff = (2**attempt) + (time.monotonic() % 1.0)
if isinstance(exc, httpx.HTTPStatusError):
retry_after = parse_retry_after(exc.response.headers)
if retry_after is not None:
backoff = max(retry_after, backoff)
cooldown = backoff
if guard is not None:
cooldown = guard.apply_rate_limit_cooldown(backoff)
if not retry_on_rate_limit:
logger.warning(
"Mint rate-limited, skipping retries for fallback",
extra={
"op_name": op_name,
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
raise
if attempt >= max_attempts - 1:
raise
logger.warning(
"Mint rate-limited, applying cooldown",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
if guard is None:
await asyncio.sleep(cooldown)
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
+1 -2
View File
@@ -18,7 +18,6 @@ from ..wallet import deserialize_token_from_string
logger = get_logger(__name__)
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
if x_cashu := headers.get("x-cashu", None):
cashu_token = x_cashu
@@ -243,7 +242,7 @@ async def calculate_discounted_max_cost(
},
)
return max(0, adjusted)
return max(settings.min_request_msat, adjusted)
def estimate_tokens(messages: list) -> int:
+63 -14
View File
@@ -1,16 +1,17 @@
from __future__ import annotations
import asyncio
import math
from typing import TypedDict
import httpx
from cashu.core.base import MeltQuoteState
from cashu.wallet.wallet import Proof, Wallet
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
# very slow mint can block a melt (and any caller, e.g. the payout loop)
# indefinitely. Bound it here so callers fail instead of hanging forever.
MELT_TIMEOUT_SECONDS = 60
from ..mint import (
MINT_TRANSPORT_EXCEPTIONS,
is_mint_rate_limited,
run_mint_operation,
)
try:
from bech32 import bech32_decode, convertbits # type: ignore
@@ -31,6 +32,15 @@ class LNURLError(Exception):
"""LNURL related errors."""
class MeltOutcomeAmbiguousError(LNURLError):
"""A melt was dispatched but its final outcome could not be confirmed.
Callers must NOT treat this as a clean failure: the payment may still
settle, so debits backing it must be kept until reconciliation confirms
the true outcome.
"""
async def decode_lnurl(lnurl: str) -> str:
"""Decode LNURL to get the actual URL.
@@ -221,23 +231,62 @@ async def raw_send_to_lnurl(
lnurl_data["callback_url"], final_amount
)
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
melt_quote_resp = await run_mint_operation(
lambda: wallet.melt_quote(invoice=bolt11_invoice),
op_name="lnurl_melt_quote",
mint_url=str(wallet.url),
)
if amount:
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
try:
_ = await asyncio.wait_for(
wallet.melt(
melt_response = await run_mint_operation(
lambda: wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
),
timeout=MELT_TIMEOUT_SECONDS,
op_name="lnurl_melt",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except asyncio.TimeoutError as e:
raise LNURLError(
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
) from e
return final_amount
except Exception as error:
if is_mint_rate_limited(error):
# Cooldown failures happen before dispatch, and HTTP 429 means the
# mint rejected the request. Neither outcome may keep proofs
# reserved as though a Lightning payment could still settle.
await wallet.set_reserved_for_send(proofs, reserved=False)
raise
if not isinstance(error, MINT_TRANSPORT_EXCEPTIONS):
raise
melt_response = None
melt_error: BaseException | None = error
else:
melt_error = None
if getattr(melt_response, "state", None) == MeltQuoteState.paid:
return final_amount
try:
quote = await run_mint_operation(
lambda: wallet.get_melt_quote(melt_quote_resp.quote),
op_name="reconcile_lnurl_melt_quote",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except Exception as reconciliation_error:
raise MeltOutcomeAmbiguousError(
"Melt outcome is ambiguous; quote reconciliation failed and proofs "
"must not be retried"
) from reconciliation_error
if quote is not None and quote.state == MeltQuoteState.paid:
return final_amount
state = getattr(getattr(quote, "state", None), "value", "unknown")
raise MeltOutcomeAmbiguousError(
"Melt outcome is ambiguous; proofs must not be retried "
f"(quote_state={state})"
) from melt_error
+20 -12
View File
@@ -26,7 +26,6 @@ from .core.db import (
)
from .core.exceptions import UpstreamError
from .core.not_found import build_not_found_response
from .core.settings import settings
from .payment.helpers import (
calculate_discounted_max_cost,
check_token_balance,
@@ -367,8 +366,6 @@ async def _proxy(
max_cost_for_model = await calculate_discounted_max_cost(
_max_cost_for_model, request_body_dict, model_obj=model_obj
)
# Ensure max_cost_for_model is at least the minimum allowed request cost
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
check_token_balance(headers, request_body_dict, max_cost_for_model)
@@ -507,7 +504,6 @@ async def _proxy(
candidate_max = await calculate_discounted_max_cost(
candidate_max, request_body_dict, model_obj=model_obj
)
candidate_max = max(candidate_max, settings.min_request_msat)
if candidate_max > max_cost_for_model:
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
@@ -807,17 +803,29 @@ async def get_bearer_token_key(
},
)
return key
except Exception as e:
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
logger.error(
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
except HTTPException as error:
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
raw_error = detail.get("error")
error_info = raw_error if isinstance(raw_error, dict) else {}
logger.warning(
"Bearer token rejected",
extra={
"error": str(e),
"error_type": type(e).__name__,
"status_code": error.status_code,
"error_code": error_info.get("code"),
"path": path,
"model_id": model_id,
"min_cost_msat": min_cost,
"bearer_key_preview": key_preview,
"required_msat": min_cost,
},
)
raise
except Exception as error:
logger.exception(
"Bearer token validation failed",
extra={
"error_type": type(error).__name__,
"path": path,
"model_id": model_id,
"required_msat": min_cost,
},
)
raise
+20 -3
View File
@@ -12,7 +12,7 @@ from ..core.db import (
from ..core.db import (
store_cashu_transaction_with_retry as store_cashu_transaction,
)
from ..wallet import send_token
from ..wallet import release_token_reservation, send_token, token_mint_url
from .routstr import RoutstrUpstreamProvider
logger = get_logger(__name__)
@@ -144,12 +144,13 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
)
return
actual_mint_url = token_mint_url(token, mint_url)
try:
await store_cashu_transaction(
token=token,
amount=amount,
unit="sat",
mint_url=mint_url,
mint_url=actual_mint_url,
typ="out",
collected=False,
source="auto_topup",
@@ -157,8 +158,24 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
except Exception:
logger.critical(
"Aborting auto top-up because its cashu token could not be persisted",
extra={"provider_id": row.id, "mint_url": mint_url},
extra={"provider_id": row.id, "mint_url": actual_mint_url},
)
try:
await release_token_reservation(token)
except Exception as error:
logger.critical(
"Failed to release untracked auto-topup token",
extra={
"provider_id": row.id,
"mint_url": actual_mint_url,
"error": str(error),
},
)
else:
logger.warning(
"Auto-topup token was released after persistence failed",
extra={"provider_id": row.id, "mint_url": actual_mint_url},
)
return
result = await provider.topup(token)
+4 -3
View File
@@ -53,6 +53,7 @@ from ..wallet import (
classify_redemption_error,
recieve_token,
send_token,
token_mint_url,
)
from . import messages_dispatch
from .cache_breakpoints import (
@@ -3520,7 +3521,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=amount,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
@@ -3873,7 +3874,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=emergency_refund,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
@@ -4843,7 +4844,7 @@ class BaseUpstreamProvider:
token=refund_token,
amount=emergency_refund,
unit=unit,
mint_url=mint,
mint_url=token_mint_url(refund_token, mint),
typ="out",
request_id=request_id,
)
+1081 -228
View File
File diff suppressed because it is too large Load Diff
+7 -2
View File
@@ -203,8 +203,13 @@ class TestmintWallet:
token_base64 = base64.urlsafe_b64encode(token_json.encode()).decode()
return f"cashuA{token_base64}"
async def redeem_token(self, token: str) -> Tuple[int, str, str]:
"""Redeem a Cashu token - compatible with wallet.recieve_token"""
async def redeem_token(
self,
token: str,
destination_mint: str | None = None,
destination_unit: str | None = None,
) -> Tuple[int, str, str]:
"""Redeem a Cashu token - compatible with wallet.recieve_token."""
if not self.wallet:
await self.init()
+30 -3
View File
@@ -207,8 +207,35 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
assert key.balance == model_cost # balance unchanged, only reserved goes up
@pytest.mark.asyncio
async def test_full_model_maximum_is_required_and_reserved(
integration_session: AsyncSession,
) -> None:
from routstr.auth import pay_for_request, validate_bearer_key
short_key = _key(balance=95_000)
exact_key = _key(balance=100_000)
integration_session.add(short_key)
integration_session.add(exact_key)
await integration_session.commit()
with pytest.raises(HTTPException) as insufficient:
await validate_bearer_key(
f"sk-{short_key.hashed_key}", integration_session, min_cost=100_000
)
assert insufficient.value.status_code == 402
validated = await validate_bearer_key(
f"sk-{exact_key.hashed_key}", integration_session, min_cost=100_000
)
await pay_for_request(validated, 100_000, integration_session)
await integration_session.refresh(exact_key)
assert exact_key.reserved_balance == 100_000
# ---------------------------------------------------------------------------
# Test 6 — HTTP layer returns 402 JSON with the right shape
# HTTP layer returns 402 JSON with the right shape
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@@ -266,8 +293,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
error = body["detail"]["error"]
assert error["code"] == "insufficient_balance"
assert error["type"] == "insufficient_quota"
assert str(model_cost) in error["message"]
assert str(user_balance) in error["message"]
assert "622.888 sats (622888 msats) required" in error["message"]
assert "20.32 sats (20320 msats) available" in error["message"]
# Balance must be completely untouched
await integration_session.refresh(key)
@@ -14,6 +14,7 @@ import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy import inspect
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel.ext.asyncio.session import AsyncSession
@@ -22,6 +23,12 @@ from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import _create_api_key_record
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
wallet.proofs = []
wallet.keysets = {}
wallet.load_proofs = AsyncMock()
def _make_invoice(**kwargs: object) -> LightningInvoice:
base = dict(
id="inv_test_001",
@@ -42,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
def mock_wallet_mint() -> object:
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
wallet = AsyncMock()
wallet.mint = AsyncMock(return_value=[])
wallet.proofs = []
wallet.load_proofs = AsyncMock()
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
mock_get_wallet.return_value = wallet
yield mock_get_wallet
@@ -183,6 +198,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
mint_calls = 0
@@ -196,7 +212,9 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
await asyncio.sleep(0.05)
if call_number > 1:
raise Exception("quote already issued")
return []
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=single_use_mint)
@@ -228,7 +246,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
@pytest.mark.asyncio
async def test_failed_mint_keeps_invoice_pending_for_retry(
async def test_failed_mint_marks_invoice_for_settlement_retry(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
@@ -238,6 +256,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
@@ -251,7 +270,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "pending"
assert stored.status == "settlement_pending"
@pytest.mark.asyncio
@@ -349,8 +368,15 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock(return_value=[])
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=successful_mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
stored_sibling = await session.get(LightningInvoice, sibling.id)
@@ -373,14 +399,14 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
assert sibling_state is not None
assert stored_state.expired is False
assert sibling_state.expired is False
assert stored.status == "pending"
assert stored.status == "settlement_pending"
assert stored_sibling.id == sibling.id
assert wallet.mint.await_count == 1
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "pending"
assert stored.status == "settlement_pending"
@pytest.mark.asyncio
@@ -428,11 +454,14 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[object]:
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
await asyncio.sleep(0.05)
return []
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
@@ -472,7 +501,7 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
assert first_invoice not in first.dirty
assert second_invoice not in second.dirty
assert wallet.mint.await_count == 2
assert wallet.mint.await_count == 1
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
assert stored_invoice is not None
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
counter = {"n": 0}
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
async def fake_generate(
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
counter["n"] += 1
return (
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
f"payment_hash_{counter['n']}",
"http://localhost:3338",
)
with patch(
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
body = resp.json()
assert body["amount_sats"] == 500
assert body["bolt11"].startswith("lnbc")
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
assert allowed_mints == ["http://localhost:3338"]
@pytest.mark.integration
@@ -0,0 +1,365 @@
import asyncio
import time
import uuid
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel import col, update
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import (
_expire_invoice_if_authoritatively_unpaid,
_finalize_invoice_settlement,
_InvoiceSettlement,
check_invoice_payment,
)
def _lightning_invoice(**overrides: object) -> LightningInvoice:
suffix = uuid.uuid4().hex
values = {
"id": f"invoice-{suffix}",
"bolt11": f"lnbc-{suffix}",
"amount_sats": 100,
"description": "settlement test",
"payment_hash": f"quote-{suffix}",
"status": "pending",
"purpose": "create",
"mint_url": "http://mint:3338",
"expires_at": int(time.time()) + 3600,
}
values.update(overrides)
return LightningInvoice(**values) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_read_transaction_closes_before_external_mint_io(
integration_session: AsyncSession,
) -> None:
invoice = _lightning_invoice()
integration_session.add(invoice)
await integration_session.commit()
stored = await integration_session.get(LightningInvoice, invoice.id)
assert stored is not None
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
async def get_wallet_without_open_db_transaction(
*args: object, **kwargs: object
) -> Mock:
assert not integration_session.in_transaction()
return wallet
with patch(
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
):
await check_invoice_payment(stored, integration_session)
assert not integration_session.in_transaction()
@pytest.mark.asyncio
async def test_separate_sessions_cas_topup_credit_exactly_once(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
):
results = await asyncio.gather(
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
)
assert sorted(settled for settled, _ in results) == [False, True]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
stored_key = await verify.get(ApiKey, key_hash)
assert stored_invoice is not None
assert stored_invoice.status == "paid"
assert stored_key is not None
assert stored_key.balance == 200_000
@pytest.mark.asyncio
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
async def debit_balance(session: AsyncSession) -> None:
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == key_hash)
.values(balance=col(ApiKey.balance) - 10_000)
.execution_options(synchronize_session=False)
)
assert result.rowcount == 1
await session.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
AsyncSession(integration_engine, expire_on_commit=False) as debit,
):
settlement_result, _ = await asyncio.gather(
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
debit_balance(debit),
)
assert settlement_result[0]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_key = await verify.get(ApiKey, key_hash)
assert stored_key is not None
assert stored_key.balance == 190_000
@pytest.mark.asyncio
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
with patch.object(
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
):
with pytest.raises(Exception, match="db unavailable"):
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
settled, _ = await _finalize_invoice_settlement(
snapshot, retry, 1_700_000_001
)
assert settled
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
@pytest.mark.asyncio
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
wallet = Mock(
proofs=[],
keysets={"keyset-1": Mock()},
load_proofs=AsyncMock(),
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
restore_tokens_for_keyset=AsyncMock(),
)
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
stored = await failed.get(LightningInvoice, invoice.id)
assert stored is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(side_effect=Exception("db unavailable")),
),
):
await check_invoice_payment(stored, failed)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "settlement_pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
stored = await retry.get(LightningInvoice, invoice.id)
assert stored is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
await check_invoice_payment(stored, retry)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_expiry_cas_cannot_overwrite_concurrent_paid_invoice(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _lightning_invoice(expires_at=0)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(invoice)
await seed.commit()
async with AsyncSession(integration_engine, expire_on_commit=False) as caller:
stale = await caller.get(LightningInvoice, invoice.id)
assert stale is not None
await caller.commit()
async with AsyncSession(integration_engine, expire_on_commit=False) as paid:
result = await paid.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(col(LightningInvoice.id) == invoice.id)
.values(status="paid", paid_at=123)
)
assert result.rowcount == 1
await paid.commit()
expired = await _expire_invoice_if_authoritatively_unpaid(
stale, caller, True
)
assert expired is False
assert stale.status == "paid"
assert stale.paid_at == 123
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "paid"
assert stored.paid_at == 123
@pytest.mark.asyncio
async def test_paid_quote_worker_does_not_mint_after_expiry_claim_wins(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _lightning_invoice(expires_at=0)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(invoice)
await seed.commit()
quote_started = asyncio.Event()
release_quote = asyncio.Event()
async def paid_quote_after_expiry(*_args: object, **_kwargs: object) -> Mock:
quote_started.set()
await release_quote.wait()
return Mock(paid=True)
wallet = Mock(
get_mint_quote=AsyncMock(side_effect=paid_quote_after_expiry),
mint=AsyncMock(),
)
async with AsyncSession(integration_engine, expire_on_commit=False) as worker:
observed_pending = await worker.get(LightningInvoice, invoice.id)
assert observed_pending is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
settlement_task = asyncio.create_task(
check_invoice_payment(observed_pending, worker)
)
await quote_started.wait()
async with AsyncSession(
integration_engine, expire_on_commit=False
) as expirer:
expiry_view = await expirer.get(LightningInvoice, invoice.id)
assert expiry_view is not None
await expirer.commit()
assert await _expire_invoice_if_authoritatively_unpaid(
expiry_view, expirer, True
)
release_quote.set()
assert await settlement_task is False
wallet.mint.assert_not_awaited()
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "expired"
@@ -110,7 +110,11 @@ async def test_payout_does_not_send_proofs_whose_liability_commit_is_in_flight(
finish_redemption = asyncio.Event()
liability_read = asyncio.Event()
async def redeem_token(token: str) -> tuple[int, str, str]:
async def redeem_token(
token: str,
destination_mint: str | None = None,
destination_unit: str | None = None,
) -> tuple[int, str, str]:
proofs.append(MagicMock(amount=200))
proof_visible.set()
await finish_redemption.wait()
@@ -126,8 +126,11 @@ async def test_parent_and_child_keys_are_not_pruned(
@pytest.mark.asyncio
async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
"""A key referenced by a pending topup invoice is never pruned mid-topup."""
@pytest.mark.parametrize("status", ["pending", "settlement_pending"])
async def test_retryable_invoice_protects_key(
patched_db_engine: None, status: str
) -> None:
"""A key referenced by a retryable topup invoice is never pruned mid-topup."""
key = _dead_key(LONG_AGO)
invoice = LightningInvoice(
id=f"inv_{uuid.uuid4().hex}",
@@ -135,7 +138,7 @@ async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
amount_sats=10,
description="topup",
payment_hash=uuid.uuid4().hex,
status="pending",
status=status,
api_key_hash=key.hashed_key,
purpose="topup",
expires_at=NOW + 10_000,
+14 -4
View File
@@ -20,6 +20,7 @@ from collections.abc import Callable
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import MeltQuoteState
from httpx import AsyncClient, Response
from routstr.core.settings import settings
@@ -28,7 +29,9 @@ from routstr.core.settings import settings
# with the testmint stub that bypasses swapping (see conftest.py).
from routstr.wallet import recieve_token as _real_recieve_token
PRIMARY_MINT = "http://primary:3338"
# Match the authenticated fixture's persisted refund mint: existing-key topups
# are intentionally constrained to that mint for collateral provenance.
PRIMARY_MINT = "http://localhost:3338"
def _make_swap_mocks(
@@ -81,7 +84,9 @@ def _make_swap_mocks(
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
)
)
mock_token_wallet.melt = AsyncMock(return_value=Mock())
mock_token_wallet.melt = AsyncMock(
return_value=Mock(state=MeltQuoteState.paid)
)
return mock_token, mock_token_wallet, mock_primary_wallet
@@ -89,7 +94,12 @@ def _make_swap_mocks(
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
def fake_get_wallet(
mint_url: str,
unit: str = "sat",
load: bool = True,
**kwargs: object,
) -> Mock:
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
return fake_get_wallet
@@ -139,7 +149,7 @@ async def test_topup_retries_when_melt_demands_more_than_quoted(
"Mint Error: not enough inputs provided for melt. "
"Provided: 179, needed: 180 (Code: 11000)"
),
Mock(),
Mock(state=MeltQuoteState.paid),
]
response = await _post_topup(
+92 -22
View File
@@ -1,8 +1,12 @@
import base64
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from fastapi import HTTPException
import routstr.wallet as wallet_module
from routstr.core import admin
@@ -13,20 +17,12 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
) -> None:
primary_mint = "https://primary.example"
effective_mint = requested_mint or primary_mint
wallet = object()
proofs = [SimpleNamespace(amount=40), SimpleNamespace(amount=60)]
token = "cashuBoutgoing"
get_wallet = AsyncMock(return_value=wallet)
get_proofs = Mock(return_value=proofs)
filter_proofs = AsyncMock(return_value=proofs)
send_token = AsyncMock(return_value=token)
store_transaction = AsyncMock(return_value=True)
monkeypatch.setattr(admin, "get_wallet", get_wallet)
monkeypatch.setattr(admin, "get_proofs_per_mint_and_unit", get_proofs)
monkeypatch.setattr(admin, "slow_filter_spend_proofs", filter_proofs)
monkeypatch.setattr(admin, "send_token", send_token)
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=effective_mint))
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
monkeypatch.setattr(admin.settings, "primary_mint", primary_mint)
@@ -35,10 +31,7 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
admin.WithdrawRequest(amount=75, mint_url=requested_mint, unit="sat"),
)
assert result == {"token": token}
get_wallet.assert_awaited_once_with(effective_mint, "sat")
get_proofs.assert_called_once_with(wallet, effective_mint, "sat", not_reserved=True)
filter_proofs.assert_awaited_once_with(proofs, wallet)
assert result == {"token": token, "mint_url": effective_mint}
send_token.assert_awaited_once_with(75, "sat", effective_mint)
store_transaction.assert_awaited_once_with(
token=token,
@@ -56,17 +49,10 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mint = "https://primary.example"
proofs = [SimpleNamespace(amount=100)]
token = "cashuBrecoverable"
monkeypatch.setattr(admin, "get_wallet", AsyncMock(return_value=object()))
monkeypatch.setattr(
admin, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
)
monkeypatch.setattr(
admin, "slow_filter_spend_proofs", AsyncMock(return_value=proofs)
)
monkeypatch.setattr(admin, "send_token", AsyncMock(return_value=token))
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=mint))
monkeypatch.setattr(
admin,
"store_cashu_transaction",
@@ -78,5 +64,89 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
result = await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
assert result == {"token": token}
assert result == {"token": token, "mint_url": mint}
critical.assert_called_once()
@pytest.mark.asyncio
async def test_withdraw_falls_back_from_insufficient_preferred_mint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
requested_mint = "https://primary.example"
actual_mint = "https://secondary.example"
proofs = [SimpleNamespace(amount=100, reserved=False, id="00")]
token_payload = {
"token": [
{
"mint": actual_mint,
"proofs": [
{
"id": "00",
"amount": 75,
"secret": "secret",
"C": "02" + "00" * 32,
}
],
}
],
"unit": "sat",
}
token = "cashuA" + base64.urlsafe_b64encode(
json.dumps(token_payload).encode()
).decode()
wallet = SimpleNamespace(
keysets={},
proofs=proofs,
select_to_send=AsyncMock(return_value=(proofs, 0)),
serialize_proofs=AsyncMock(return_value=token),
set_reserved_for_send=AsyncMock(),
)
find_funded = AsyncMock(return_value=actual_mint)
store_transaction = AsyncMock(return_value=True)
monkeypatch.setattr(wallet_module, "find_trusted_mint_with_funds", find_funded)
monkeypatch.setattr(wallet_module, "get_wallet", AsyncMock(return_value=wallet))
monkeypatch.setattr(
wallet_module, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
)
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
result = await admin.withdraw(
Mock(), admin.WithdrawRequest(amount=75, mint_url=requested_mint)
)
assert result == {"token": token, "mint_url": actual_mint}
find_funded.assert_awaited_once_with(
75, "sat", requested_mint, force_reload=True
)
wallet.select_to_send.assert_awaited_once()
store_transaction.assert_awaited_once_with(
token=token,
amount=75,
unit="sat",
mint_url=actual_mint,
typ="out",
collected=False,
source="admin",
)
@pytest.mark.asyncio
async def test_withdraw_maps_true_aggregate_insufficient_funds_to_400(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
admin,
"send_token",
AsyncMock(
side_effect=ValueError(
"No trusted mint has 75 sat available; balances={'mint': 0}"
)
),
)
with pytest.raises(HTTPException) as exc_info:
await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
assert exc_info.value.status_code == 400
assert exc_info.value.detail == "Insufficient wallet balance"
+48
View File
@@ -270,6 +270,54 @@ async def test_internal_error_with_invalid_keyword_does_not_masquerade(
assert await session.get(ApiKey, hashed_key) is None
@pytest.mark.asyncio
async def test_primary_msat_token_sets_provenance_without_cashu_mint_duplicate(
session: AsyncSession,
) -> None:
token = "cashuAprimary_msat_token"
token_obj = SimpleNamespace(mint="http://primary:3338", unit="msat")
credit = AsyncMock(return_value=1_000)
from routstr.core.settings import settings
with (
patch.object(settings, "primary_mint", token_obj.mint),
patch.object(settings, "primary_mint_unit", "msat"),
patch.object(settings, "cashu_mints", []),
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
patch("routstr.auth.credit_balance", new=credit),
):
key = await validate_bearer_key(token, session)
assert key.refund_mint_url == token_obj.mint
assert key.refund_currency == "msat"
credit.assert_awaited_once()
@pytest.mark.asyncio
async def test_primary_token_unit_mismatch_is_rejected_before_redemption(
session: AsyncSession,
) -> None:
token = "cashuAprimary_wrong_unit"
token_obj = SimpleNamespace(mint="http://primary:3338", unit="sat")
credit = AsyncMock(return_value=1_000)
from routstr.core.settings import settings
with (
patch.object(settings, "primary_mint", token_obj.mint),
patch.object(settings, "primary_mint_unit", "msat"),
patch.object(settings, "cashu_mints", []),
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
patch("routstr.auth.credit_balance", new=credit),
):
with pytest.raises(HTTPException) as exc_info:
await validate_bearer_key(token, session)
assert exc_info.value.status_code == 400
credit.assert_not_awaited()
@pytest.mark.asyncio
async def test_malformed_cashu_token_returns_400_invalid_token(
session: AsyncSession,
+11 -1
View File
@@ -66,6 +66,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(return_value=True),
) as store,
patch(
"routstr.upstream.auto_topup.token_mint_url",
return_value="https://fallback-mint.test",
),
patch("routstr.upstream.auto_topup.create_session", return_value=session),
):
await _check_and_topup(_row())
@@ -74,7 +78,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
token="cashu-token",
amount=50,
unit="sat",
mint_url="https://mint.test",
mint_url="https://fallback-mint.test",
typ="out",
collected=False,
source="auto_topup",
@@ -138,6 +142,12 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(side_effect=RuntimeError("database unavailable")),
),
patch(
"routstr.upstream.auto_topup.release_token_reservation",
AsyncMock(),
) as reclaim,
):
await _check_and_topup(_row())
reclaim.assert_awaited_once_with("cashu-token")
provider.topup.assert_not_awaited()
+88
View File
@@ -606,6 +606,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
assert exc_info.value.detail == "Cashu mint is unreachable"
@pytest.mark.asyncio
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
from fastapi import HTTPException
from routstr.wallet import SourceMintConnectionError
key = _make_api_key(balance=1000)
session = MagicMock()
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
with (
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
):
with pytest.raises(HTTPException) as exc_info:
await topup_wallet_endpoint(
cashu_token="cashuAtoken", key=key, session=session
)
assert exc_info.value.status_code == 503
assert "cannot be redeemed at another mint" in exc_info.value.detail
@pytest.mark.asyncio
async def test_topup_already_spent_still_returns_400() -> None:
"""Regression: the mint-unreachable short-circuit must not swallow the
@@ -758,3 +781,68 @@ async def test_topup_unexpected_non_valueerror_returns_500() -> None:
assert exc_info.value.status_code == 500
assert exc_info.value.detail == "Internal server error"
@pytest.mark.asyncio
async def test_apikey_refund_ambiguous_melt_does_not_restore_balance() -> None:
"""An ambiguous LNURL melt may still settle: the debit must be kept."""
from fastapi import HTTPException
from routstr.payment.lnurl import MeltOutcomeAmbiguousError
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
session.commit = AsyncMock()
with (
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
patch("routstr.balance._refund_cache_set", AsyncMock()),
patch(
"routstr.balance.send_to_lnurl",
AsyncMock(side_effect=MeltOutcomeAmbiguousError("outcome is ambiguous")),
),
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
):
with pytest.raises(HTTPException) as exc_info:
await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
assert exc_info.value.status_code == 502
mock_restore.assert_not_awaited()
@pytest.mark.asyncio
async def test_apikey_refund_clean_failure_still_restores_balance() -> None:
"""A definitively failed melt must keep restoring the debited balance."""
from fastapi import HTTPException
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
session.commit = AsyncMock()
with (
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
patch("routstr.balance._refund_cache_set", AsyncMock()),
patch(
"routstr.balance.send_to_lnurl",
AsyncMock(side_effect=RuntimeError("mint rejected melt")),
),
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
):
with pytest.raises(HTTPException):
await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
mock_restore.assert_awaited_once()
+11 -14
View File
@@ -4,7 +4,7 @@ Tests admin endpoints that are testable without full app setup:
withdraw validation, authentication guards, and slug validation.
"""
from unittest.mock import Mock, patch
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException, Request
@@ -46,22 +46,19 @@ async def test_withdraw_rejects_insufficient_balance() -> None:
request = Request(scope={"type": "http", "method": "POST"})
with patch("routstr.core.admin.get_wallet") as mock_wallet, \
patch("routstr.core.admin.get_proofs_per_mint_and_unit") as mock_proofs, \
patch("routstr.core.admin.slow_filter_spend_proofs") as mock_filter:
mock_w = Mock()
mock_w.keysets = {}
mock_w.proofs = []
mock_wallet.return_value = mock_w
mock_proofs.return_value = []
mock_filter.return_value = []
with patch(
"routstr.core.admin.send_token",
new=AsyncMock(
side_effect=ValueError(
"No trusted mint has 1000000 sat available; balances={}"
)
),
):
with pytest.raises(HTTPException) as exc_info:
await withdraw(request, WithdrawRequest(amount=1000000, unit="sat"))
assert exc_info.value.status_code == 400
assert "Insufficient" in str(exc_info.value.detail)
assert exc_info.value.status_code == 400
assert "Insufficient" in str(exc_info.value.detail)
# ===========================================================================
+1 -1
View File
@@ -67,7 +67,7 @@ async def test_fee_payout_prepares_wallet_then_checkpoints_before_sending() -> N
payout_wallet = Mock()
events: list[str] = []
async def prepare(*_args: object) -> Mock:
async def prepare(*_args: object, **_kwargs: object) -> Mock:
events.append("prepare")
return payout_wallet
+173 -2
View File
@@ -1,9 +1,10 @@
import asyncio
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Generator
from contextlib import asynccontextmanager
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel import SQLModel
@@ -12,6 +13,21 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.wallet import fetch_all_balances
@pytest.fixture(autouse=True)
def clear_balance_fetch_state() -> Generator[None, None, None]:
from routstr import wallet
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
yield
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
@asynccontextmanager
async def _fake_session(): # type: ignore[no-untyped-def]
yield MagicMock()
@@ -29,7 +45,7 @@ def _patches( # type: ignore[no-untyped-def]
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
@@ -63,6 +79,161 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
assert total_wallet == 1000
@pytest.mark.asyncio
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(return_value=["sat"]),
) as supported_units,
):
for p in _patches(proof_amount=1000):
p.start()
try:
details, *_ = await fetch_all_balances()
finally:
patch.stopall()
supported_units.assert_awaited_once_with("http://mint:3338")
assert [detail["unit"] for detail in details] == ["sat"]
@pytest.mark.asyncio
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances()
assert details[0]["unit"] == settings.primary_mint_unit
assert details[0]["error_code"] == "unreachable"
assert details[0]["retry_after_seconds"] > 0
get_wallet.assert_not_awaited()
@pytest.mark.asyncio
async def test_supported_mint_units_come_from_active_keysets() -> None:
from routstr.core.settings import settings
from routstr.wallet import _get_supported_mint_units
# Cashu versions/mints may deserialize keyset units as either strings or
# Unit enum-like objects. Both representations must be accepted.
sat = MagicMock(active=True, unit="sat")
msat = MagicMock(active=False, unit="msat")
usd = MagicMock(active=True)
usd.unit.name = "usd"
wallet = MagicMock()
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
with (
patch.object(settings, "primary_mint_unit", "sat"),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
):
units = await _get_supported_mint_units("http://mint:3338")
cached_units = await _get_supported_mint_units("http://mint:3338")
assert units == ["sat", "usd"]
assert cached_units == units
wallet._get_keysets.assert_awaited_once()
@pytest.mark.asyncio
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
first = await fetch_all_balances(units=["sat"])
second = await fetch_all_balances(units=["sat"])
assert first[0][0]["error"] == "mint unavailable"
assert first[0][0]["error_code"] == "unreachable"
assert first[0][0]["retry_after_seconds"] == 60
assert second[0][0]["error"] == "mint unavailable"
assert second[0][0]["error_code"] == "unreachable"
assert get_wallet.await_count == 1
warning.assert_called_once()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=71),
patch("routstr.wallet.logger.warning"),
):
await fetch_all_balances(units=["sat"])
assert get_wallet.await_count == 2
@pytest.mark.asyncio
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
from routstr.core.settings import settings
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances(units=["sat"])
assert details[0]["error_code"] == "rate_limited"
assert details[0]["retry_after_seconds"] == 60
@pytest.mark.asyncio
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
from routstr.core.settings import settings
from routstr.wallet import _mint_cooldown_remaining
mint = "http://mint:3338"
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", [mint]),
patch.object(settings, "primary_mint", mint),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
details, *_ = await fetch_all_balances(units=["sat", "msat"])
cooldown = _mint_cooldown_remaining(mint)
assert get_wallet.await_count == 1
assert warning.call_count == 1
assert cooldown == 60
assert details[0]["error"] == "mint unavailable"
assert details[0]["error_code"] == "unreachable"
assert details[1]["error"] == "Mint is unreachable"
assert details[1]["error_code"] == "unreachable"
@pytest.mark.asyncio
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
"""Slow mint checks must never run while the balance DB session is open."""
+370
View File
@@ -0,0 +1,370 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import MintQuoteState, Proof
from routstr.lightning import (
InvoiceRecoverRequest,
_invoice_settlement_locks,
_is_outputs_already_signed,
_mint_invoice_quote,
check_invoice_payment,
get_invoice_status,
recover_invoice,
)
from routstr.wallet import Wallet
def _invoice(**overrides: object) -> SimpleNamespace:
values = {
"id": "invoice-1",
"payment_hash": "quote-1",
"amount_sats": 100,
"purpose": "create",
"status": "pending",
"paid_at": None,
"api_key_hash": None,
"mint_url": "http://mint:3338",
"balance_limit": None,
"balance_limit_reset": None,
"validity_date": None,
"created_at": 1,
"expires_at": 2,
}
values.update(overrides)
return SimpleNamespace(**values)
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
def _recovery_wallet(
error: Exception,
*,
proofs_before: list[Proof] | None = None,
proofs_after: list[Proof] | None = None,
) -> Mock:
async def load_proofs(*, reload: bool) -> None:
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
wallet.proofs = list(proofs_after)
wallet = Mock(
mint=AsyncMock(side_effect=error),
keysets={"keyset-1": Mock()},
restore_tokens_for_keyset=AsyncMock(),
load_proofs=AsyncMock(side_effect=load_proofs),
proofs=list(proofs_before or []),
)
return wallet
@pytest.mark.asyncio
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
proofs_after=[_proof(100, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
"keyset-1", to=1, batch=25
)
assert wallet.load_proofs.await_count == 2
@pytest.mark.asyncio
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("must not mint"),
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.mint.assert_not_awaited()
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
invoice = _invoice()
error = Exception("backend request 11003 failed")
wallet = _recovery_wallet(error)
with pytest.raises(Exception) as caught:
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
assert caught.value is error
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
response = httpx.Response(
400,
request=request,
json={"detail": "outputs have already been signed before", "code": 11003},
)
with pytest.raises(Exception) as caught:
Wallet.raise_on_error_request(response)
assert _is_outputs_already_signed(caught.value)
@pytest.mark.asyncio
@pytest.mark.parametrize("recovered", [0, 99])
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
recovered: int,
) -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
)
with pytest.raises(RuntimeError, match="expected at least 100"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(10_000, "different-quote")],
)
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_non_pending_invoice_is_not_minted() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="expired")
session = AsyncMock()
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
get_wallet.assert_not_awaited()
session.commit.assert_awaited_once()
assert _invoice_settlement_locks == {}
@pytest.mark.asyncio
async def test_ambiguous_invoice_mint_timeout_remains_recoverable() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
state_session = AsyncMock()
state_session.exec.return_value.rowcount = 1
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
yield state_session
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert invoice.status == "settlement_pending"
state_session.commit.assert_awaited_once()
session.rollback.assert_not_awaited()
# One commit closes the initial read transaction before external I/O.
session.commit.assert_awaited_once()
@pytest.mark.asyncio
async def test_quote_lookup_timeout_is_not_definitively_unpaid() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(side_effect=httpx.TimeoutException("quote timeout"))
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is False
@pytest.mark.asyncio
async def test_overdue_invoice_does_not_expire_after_ambiguous_quote_lookup() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock(return_value=False)
with patch("routstr.lightning.check_invoice_payment", check):
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
assert response.status == "pending"
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_overdue_invoice_expires_only_after_definitive_unpaid_quote() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock(return_value=True)
async def expire(
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
) -> bool:
assert definitive is True
candidate.status = "expired"
return True
with (
patch("routstr.lightning.check_invoice_payment", check),
patch(
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
side_effect=expire,
) as expire_invoice,
):
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
assert response.status == "expired"
expire_invoice.assert_awaited_once_with(invoice, session, True)
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_recover_applies_authoritative_expiry_helper() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
result = Mock()
result.first.return_value = invoice
session.exec.return_value = result
check = AsyncMock(return_value=True)
async def expire(
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
) -> bool:
assert definitive is True
candidate.status = "expired"
return True
with (
patch("routstr.lightning.check_invoice_payment", check),
patch(
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
side_effect=expire,
) as expire_invoice,
):
response = await recover_invoice(
InvoiceRecoverRequest(bolt11="lnbc-test"), session # type: ignore[arg-type]
)
assert response.status == "expired"
expire_invoice.assert_awaited_once_with(invoice, session, True)
@pytest.mark.asyncio
async def test_paid_state_write_failure_still_reports_non_expirable_outcome() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(
return_value=Mock(paid=True, state=MintQuoteState.paid)
)
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
),
patch(
"routstr.lightning.create_session",
side_effect=RuntimeError("database unavailable"),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
definitively_unpaid = await check_invoice_payment(
invoice, session # type: ignore[arg-type]
)
assert definitively_unpaid is False
assert invoice.status == "pending"
@pytest.mark.asyncio
async def test_settlement_pending_invoice_does_not_expire() -> None:
invoice = _invoice(status="settlement_pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock()
with patch("routstr.lightning.check_invoice_payment", check):
response = await get_invoice_status(
invoice.id, session # type: ignore[arg-type]
)
check.assert_awaited_once_with(invoice, session)
assert response.status == "settlement_pending"
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
async def refresh(obj: SimpleNamespace) -> None:
return None
session.refresh = AsyncMock(side_effect=refresh)
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
owned = AsyncMock()
owned.exec.return_value.rowcount = 1
yield owned
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(return_value=(True, "b" * 64)),
) as finalize,
):
await asyncio.gather(
check_invoice_payment(invoice, session), # type: ignore[arg-type]
check_invoice_payment(invoice, session), # type: ignore[arg-type]
)
assert invoice.status == "paid"
finalize.assert_awaited_once()
assert _invoice_settlement_locks == {}
+179 -46
View File
@@ -1,70 +1,202 @@
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
timeout as LNURLError instead of hanging.
"""
"""LNURL melt attempts must not misclassify ambiguous payment outcomes."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from cashu.core.base import MeltQuoteState
from routstr.payment import lnurl
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
from routstr.core.settings import settings
from routstr.mint import MintCooldownError, MintRateGuard
from routstr.payment.lnurl import (
MeltOutcomeAmbiguousError,
raw_send_to_lnurl,
)
LNURL_DATA = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
def _wallet() -> tuple[MagicMock, list[MagicMock]]:
proofs = [MagicMock(amount=1000)]
wallet = MagicMock(url="https://mint.test")
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
return wallet, proofs
def _lnurl_patches() -> tuple[Any, Any]:
return (
patch(
"routstr.payment.lnurl.get_lnurl_data",
AsyncMock(return_value=LNURL_DATA),
),
patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
),
)
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
proofs = [MagicMock(amount=1000)]
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
async def test_raw_send_to_lnurl_timeout_keeps_unpaid_outcome_ambiguous() -> None:
wallet, proofs = _wallet()
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5) # far longer than the patched timeout
await asyncio.sleep(5)
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.unpaid)
)
data_patch, invoice_patch = _lnurl_patches()
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
):
with pytest.raises(LNURLError, match="Melt timed out"):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
wallet.set_reserved_for_melt.assert_not_called()
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
"""A prompt melt still returns the net amount, unaffected by the guard."""
proofs = [MagicMock(amount=1000)]
async def test_raw_send_to_lnurl_timeout_reconciled_paid_is_success() -> None:
wallet, proofs = _wallet()
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
wallet.melt = AsyncMock(return_value=MagicMock())
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5)
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.paid)
)
data_patch, invoice_patch = _lnurl_patches()
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
assert paid > 0
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_pending_response_stays_ambiguous() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.pending))
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.pending)
)
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
@pytest.mark.parametrize("rate_error", ["cooldown", "http_429"])
async def test_raw_send_to_lnurl_rate_rejection_unreserves_proofs(
rate_error: str,
) -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock()
wallet.set_reserved_for_send = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
async def run_operation(factory: Any, *, op_name: str, **_: object) -> Any:
if op_name == "lnurl_melt":
if rate_error == "cooldown":
raise MintCooldownError(str(wallet.url), 60)
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
response = httpx.Response(429, request=request)
raise httpx.HTTPStatusError(
"rate limited", request=request, response=response
)
return await factory()
with (
data_patch,
invoice_patch,
patch(
"routstr.payment.lnurl.run_mint_operation",
side_effect=run_operation,
),
pytest.raises((MintCooldownError, httpx.HTTPStatusError)),
):
await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
wallet.melt.assert_not_awaited()
wallet.set_reserved_for_send.assert_awaited_once_with(
proofs, reserved=False
)
@pytest.mark.asyncio
async def test_real_mint_wrapper_http_429_unreserves_proofs() -> None:
wallet, proofs = _wallet()
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
response = httpx.Response(429, request=request)
wallet.melt = AsyncMock(
side_effect=httpx.HTTPStatusError(
"rate limited", request=request, response=response
)
)
wallet.set_reserved_for_send = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
pytest.raises(httpx.HTTPStatusError),
):
await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
wallet.melt.assert_awaited_once()
wallet.set_reserved_for_send.assert_awaited_once_with(
proofs, reserved=False
)
MintRateGuard._guards.pop(str(wallet.url), None)
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_on_explicit_paid_response() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.paid))
wallet.get_melt_quote = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
@@ -72,3 +204,4 @@ async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
assert paid > 0
wallet.melt.assert_awaited_once()
wallet.get_melt_quote.assert_not_awaited()
+91
View File
@@ -0,0 +1,91 @@
from unittest.mock import AsyncMock, Mock
import pytest
from cashu.core.base import MeltQuoteState, ProofSpentState
from routstr.wallet import (
TokenConsumedError,
_confirm_melt_paid,
_reconcile_ambiguous_melt,
)
@pytest.mark.asyncio
async def test_paid_quote_is_authoritative_when_proof_lookup_would_fail() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
check_proof_state=AsyncMock(side_effect=RuntimeError("proof API unavailable")),
)
assert await _reconcile_ambiguous_melt(wallet, "quote-1", [Mock()]) is True
wallet.check_proof_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_timeout_snapshot_unpaid_unspent_remains_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.unpaid)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.unspent)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _reconcile_ambiguous_melt(wallet, "quote-2", [Mock()])
@pytest.mark.asyncio
async def test_successful_pending_melt_response_requires_reconciliation() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.pending)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.pending)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _confirm_melt_paid(
wallet,
"quote-pending",
[Mock()],
Mock(state=MeltQuoteState.pending),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("quote_state", "proof_state"),
[
(MeltQuoteState.pending, ProofSpentState.pending),
(MeltQuoteState.unpaid, ProofSpentState.spent),
(MeltQuoteState.unpaid, ProofSpentState.pending),
],
)
async def test_ambiguous_or_consumed_melt_is_never_reported_unspent(
quote_state: MeltQuoteState, proof_state: ProofSpentState
) -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=quote_state)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=proof_state)])
),
)
with pytest.raises(TokenConsumedError, match="reconciliation required"):
await _reconcile_ambiguous_melt(wallet, "quote-3", [Mock()])
@pytest.mark.asyncio
async def test_failed_melt_reconciliation_is_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(side_effect=RuntimeError("mint unavailable")),
check_proof_state=AsyncMock(),
)
with pytest.raises(TokenConsumedError, match="outcome is unknown"):
await _reconcile_ambiguous_melt(wallet, "quote-4", [Mock()])
+121
View File
@@ -0,0 +1,121 @@
import asyncio
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import Unit
from routstr.mint import (
MintCooldownError,
MintRateGuard,
MintRateLimitedError,
fail_fast_mint_operations,
)
from routstr.wallet import Wallet
@pytest.mark.asyncio
async def test_cooldown_fails_fast_while_wallet_mutation_scope_is_held() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(3600, reason="rate_limited")
operation = AsyncMock(return_value="should not run")
with (
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
pytest.raises(MintCooldownError) as caught,
):
async with fail_fast_mint_operations():
await guard.run(operation)
assert caught.value.retry_after_seconds > 0
operation.assert_not_awaited()
sleep.assert_not_awaited()
@pytest.mark.asyncio
async def test_expired_cooldown_allows_probe_in_wallet_mutation_scope() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(0, reason="rate_limited")
operation = AsyncMock(return_value="recovered")
async with fail_fast_mint_operations():
result = await guard.run(operation)
assert result == "recovered"
operation.assert_awaited_once()
assert guard._needs_probe is False
@pytest.mark.asyncio
async def test_fail_fast_does_not_wait_behind_existing_probe() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(0, reason="rate_limited")
probe_started = asyncio.Event()
release_probe = asyncio.Event()
async def probe() -> str:
probe_started.set()
await release_probe.wait()
return "recovered"
first = asyncio.create_task(guard.run(probe))
await probe_started.wait()
try:
async with fail_fast_mint_operations():
with pytest.raises(MintCooldownError):
await asyncio.wait_for(guard.run(AsyncMock()), timeout=0.05)
finally:
release_probe.set()
assert await first == "recovered"
@pytest.mark.asyncio
async def test_cashu_429_dispatches_through_wallet_override() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
429,
request=request,
json={"detail": "too many requests", "code": 42900},
)
wallet = object.__new__(Wallet)
wallet.url = "http://mint:3338"
wallet.db = Mock()
wallet.keysets = {"loaded": Mock()}
wallet.mint_info = Mock()
wallet.mint_info.requires_blind_auth_path.return_value = False
wallet.mint_info.requires_clear_auth_path.return_value = False
wallet.auth_db = None
wallet.auth_keyset_id = None
real_client = httpx.AsyncClient
def client_factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
return real_client(
transport=httpx.MockTransport(handler),
base_url=str(kwargs["base_url"]),
)
with (
patch("cashu.wallet.v1_api.httpx.AsyncClient", side_effect=client_factory),
pytest.raises(MintRateLimitedError),
):
await wallet.mint_quote(1, Unit.sat)
async def test_guard_concurrency_change_preserves_cooldown_state() -> None:
from routstr.core.settings import settings
mint_url = "https://mint.test-concurrency-carryover"
with patch.object(settings, "mint_max_concurrency", 2):
guard = MintRateGuard.get(mint_url)
guard.apply_cooldown(120.0, reason="rate_limited")
guard._consecutive_rate_limits = 3
with patch.object(settings, "mint_max_concurrency", 5):
rebuilt = MintRateGuard.get(mint_url)
assert rebuilt is not guard
assert rebuilt.cooldown_remaining() > 0
assert rebuilt._cooldown_reason == "rate_limited"
assert rebuilt._consecutive_rate_limits == 3
+50
View File
@@ -0,0 +1,50 @@
"""Persisted mint preferences must not bypass the configured trusted set."""
from unittest.mock import AsyncMock, patch
import pytest
from routstr.core.settings import settings
from routstr.lightning import _request_mint_with_fallback
TRUSTED = "https://good-mint.example.com"
UNTRUSTED = "https://removed-mint.example.com"
async def test_untrusted_allowed_mints_fall_back_to_trusted_set() -> None:
attempted: list[str] = []
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
attempted.append(mint_url)
raise ConnectionError("unreachable in test")
with (
patch.object(settings, "primary_mint", TRUSTED),
patch.object(settings, "cashu_mints", [TRUSTED]),
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
):
with pytest.raises(Exception):
await _request_mint_with_fallback(10, allowed_mints=[UNTRUSTED])
assert UNTRUSTED not in attempted
assert attempted == [TRUSTED]
async def test_trusted_allowed_mints_are_used_verbatim() -> None:
attempted: list[str] = []
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
attempted.append(mint_url)
raise ConnectionError("unreachable in test")
with (
patch.object(settings, "primary_mint", TRUSTED),
patch.object(settings, "cashu_mints", [TRUSTED, "https://other.example.com"]),
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
):
with pytest.raises(Exception):
await _request_mint_with_fallback(10, allowed_mints=[TRUSTED])
assert attempted == [TRUSTED]
+47
View File
@@ -0,0 +1,47 @@
import os
import sqlite3
import subprocess
import sys
from pathlib import Path
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
env = os.environ.copy()
env["DATABASE_URL"] = database_url
subprocess.run(
[sys.executable, "-m", "alembic", command, revision],
cwd=root,
env=env,
check=True,
capture_output=True,
text=True,
)
def _lightning_invoice_columns(database_path: Path) -> set[str]:
with sqlite3.connect(database_path) as connection:
return {
row[1]
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
}
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "mint-url-migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
previous_head = "64ed5594df1f"
_run_alembic(root, database_url, "upgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "ecfa0d6e2a36")
assert "mint_url" in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "downgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "head")
assert "mint_url" in _lightning_invoice_columns(database_path)
+30
View File
@@ -125,3 +125,33 @@ async def test_get_max_cost_for_model_tolerance() -> None:
"gpt-4", session=mock_session, model_obj=mock_model
)
assert cost == 450000 # 500 sats * 1000 * 0.9 = 450000
async def test_discounted_max_cost_floors_at_min_request_msat() -> None:
from routstr.payment.helpers import calculate_discounted_max_cost
pricing = Mock()
pricing.prompt = 0.001
pricing.completion = 0.001
pricing.max_prompt_cost = 100.0
pricing.max_completion_cost = 100.0
model_obj = Mock()
model_obj.sats_pricing = pricing
model_obj.top_provider = None
model_obj.context_length = None
body = {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 1,
}
with (
patch.object(settings, "fixed_pricing", False),
patch.object(settings, "tolerance_percentage", 0),
patch.object(settings, "min_request_msat", 1000),
):
cost = await calculate_discounted_max_cost(150_000, body, model_obj)
assert cost == 1000
+3 -1
View File
@@ -147,7 +147,9 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
"""A failing mint does not prevent payout for the other mints."""
from routstr.core.settings import settings
async def _get_wallet(mint_url: str, unit: str) -> MagicMock:
async def _get_wallet(
mint_url: str, unit: str, force_reload: bool = False
) -> MagicMock:
if mint_url == "http://bad:3338":
raise RuntimeError("mint unreachable")
return MagicMock()
+11 -9
View File
@@ -71,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
@pytest.mark.asyncio
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
async def test_pay_for_request_sets_reserved_at_on_child_key(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="parentkey", balance=10_000)
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
session.add(parent)
@@ -204,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
@pytest.mark.asyncio
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
async def test_release_stale_reservations_skips_null_reserved_at(
session: AsyncSession,
) -> None:
# Reservations without a timestamp may belong to instances running older
# code (rolling deploy) — the background sweeper must not touch them.
key = ApiKey(
@@ -224,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
@pytest.mark.asyncio
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
async def test_reset_all_reserved_balances_clears_reserved_at(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="resetkey",
balance=5_000,
@@ -404,9 +410,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
@@ -418,6 +422,4 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
with pytest.raises(asyncio.CancelledError):
await proxy_module.proxy(request, "v1/chat/completions", session=session)
revert_mock.assert_awaited_once_with(
key, session, 1_000, reservation_snapshot
)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
+2 -4
View File
@@ -383,9 +383,7 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
@@ -408,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
assert RAW_ORG_ID not in serialized
assert "org-[REDACTED]" in serialized
# Single upstream failed -> reservation reverted exactly once (no double-charge).
revert_mock.assert_awaited_once_with(key, session, 1_000, reservation)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation)
+1338 -47
View File
File diff suppressed because it is too large Load Diff
+27 -4
View File
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
const formatMintLabel = (detail: BalanceDetail) =>
`${detail.mint_url.replace('https://', '').replace('http://', '')} • ${detail.unit.toUpperCase()}`;
const formatBalanceError = (detail: BalanceDetail) => {
const labels: Record<string, string> = {
rate_limited: 'rate limited',
unreachable: 'unreachable',
cooldown: 'cooling down',
mint_error: 'mint error',
};
const label =
(detail.error_code ? labels[detail.error_code] : undefined) ??
detail.error ??
'error';
const retryAfter = detail.retry_after_seconds;
return retryAfter && retryAfter > 0
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
: label;
};
return (
<>
<Card>
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
{formatMintLabel(detail)}
</TableCell>
<TableCell className='text-right font-mono'>
<TableCell
className='text-right font-mono'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</TableCell>
<TableCell className='text-right font-mono'>
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
<p className='text-muted-foreground text-xs'>
Wallet
</p>
<p className='font-mono text-sm'>
<p
className='font-mono text-sm'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</p>
</div>
+3
View File
@@ -36,10 +36,13 @@ export interface BalanceDetail {
user_balance: number;
owner_balance: number;
error?: string;
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
retry_after_seconds?: number;
}
export interface WithdrawResponse {
token: string;
mint_url: string;
}
export interface CreateChildKeyResponse {