fix: resolve mint fallback review comments

This commit is contained in:
9qeklajc
2026-07-31 02:10:00 +02:00
parent 443c910b9e
commit 2ec6b27200
20 changed files with 957 additions and 612 deletions
+3
View File
@@ -45,6 +45,9 @@ ROUTSTR_SECRET_KEY=
# ENABLE_ANALYTICS_SHARING=true
# CASHU_MINTS="https://mint.minibits.cash/Bitcoin,https://mint.cubabitcoin.org,https://ecashmint.otrta.me"
# MINT_OPERATION_CONCURRENCY=4
# MINT_OPERATION_TIMEOUT_SECONDS=30
# MINT_MAX_CONCURRENCY=4
# MINT_RETRY_MAX_ATTEMPTS=3
# RECEIVE_LN_ADDRESS=
# REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS=900
+9
View File
@@ -136,6 +136,10 @@ Use environment variables for:
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
| `MINT_OPERATION_CONCURRENCY` | Concurrent mint/unit balance reads | `4` |
| `MINT_OPERATION_TIMEOUT_SECONDS` | Per-attempt timeout for mint network calls | `30` |
| `MINT_MAX_CONCURRENCY` | Concurrent operations allowed per mint (`0` disables the limit) | `4` |
| `MINT_RETRY_MAX_ATTEMPTS` | Retries after a timeout or HTTP 429 (`0` disables retries) | `3` |
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
| `MIN_PAYOUT_SAT` | Min payout balance in sats (applies to all mints) | `210` |
| `PAYOUT_INTERVAL_SECONDS` | Payout loop interval (seconds) | `900` |
@@ -143,6 +147,11 @@ Use environment variables for:
| `CORS_ORIGINS` | Allowed CORS origins | `*` |
| `RELAYS` | Nostr relays (comma-separated) | (default set) |
Mint HTTP 429 responses create a per-mint cooldown. Operations that already hold
Routstr's wallet mutation lock fail fast during that cooldown instead of waiting
while blocking every other wallet mutation. Callers receive an error and may retry
later; the current response does not include the cooldown duration.
### Priority
Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting.
+2 -2
View File
@@ -215,7 +215,7 @@ async def topup_wallet_endpoint(
raise HTTPException(status_code=400, detail="Invalid token format")
source_mint = token_mint_url(cashu_token, "unknown")
logger.warning(
logger.info(
"Cashu wallet top-up started",
extra={
"event": "cashu_topup_started",
@@ -259,7 +259,7 @@ async def topup_wallet_endpoint(
)
raise HTTPException(status_code=status_code, detail=message)
logger.warning(
logger.info(
"Cashu wallet top-up completed",
extra={
"event": "cashu_topup_completed",
+39 -13
View File
@@ -3,8 +3,9 @@ import hashlib
import re
import secrets
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import Any
from typing import Any, AsyncGenerator
from fastapi import APIRouter, Depends, Header, HTTPException
from pydantic import BaseModel, Field
@@ -15,11 +16,13 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from .core.db import ApiKey, LightningInvoice, create_session, get_session
from .core.logging import get_logger
from .core.settings import settings
from .mint import (
is_mint_rate_limited,
mint_cooldown_remaining,
run_mint_operation,
)
from .wallet import (
MintConnectionError,
_is_mint_rate_limited,
_mint_cooldown_remaining,
_mint_operation,
get_wallet,
is_mint_connection_error,
wallet_operation_guard,
@@ -31,7 +34,31 @@ lightning_router = APIRouter(prefix="/lightning")
# Avoid duplicate work within one process. Cross-process credit fencing is done
# by the conditional pending -> paid update in _finalize_invoice_settlement().
_invoice_settlement_locks: dict[str, asyncio.Lock] = {}
@dataclass
class _InvoiceLockEntry:
lock: asyncio.Lock
users: int = 0
_invoice_settlement_locks: dict[str, _InvoiceLockEntry] = {}
@asynccontextmanager
async def _invoice_settlement_lock(invoice_id: str) -> AsyncGenerator[None, None]:
"""Serialize one invoice and remove its lock after the last waiter leaves."""
entry = _invoice_settlement_locks.get(invoice_id)
if entry is None:
entry = _InvoiceLockEntry(asyncio.Lock())
_invoice_settlement_locks[invoice_id] = entry
entry.users += 1
try:
async with entry.lock:
yield
finally:
entry.users -= 1
if entry.users == 0 and _invoice_settlement_locks.get(invoice_id) is entry:
del _invoice_settlement_locks[invoice_id]
@dataclass(frozen=True)
@@ -142,11 +169,11 @@ async def _request_mint_with_fallback(
configured = allowed_mints or [settings.primary_mint, *settings.cashu_mints]
candidates = list(dict.fromkeys(configured))
for mint_url in candidates:
cooldown = _mint_cooldown_remaining(mint_url)
cooldown = mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.info(
"Skipping rate-limited mint",
"Skipping mint during cooldown",
extra={
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
@@ -156,7 +183,7 @@ async def _request_mint_with_fallback(
continue
try:
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
quote = await _mint_operation(
quote = await run_mint_operation(
lambda: wallet.request_mint(amount_sats),
op_name="request_mint_invoice",
mint_url=mint_url,
@@ -165,7 +192,7 @@ async def _request_mint_with_fallback(
return quote.request, quote.quote, mint_url
except Exception as e:
tried.append(f"{mint_url}: {type(e).__name__}")
if not is_mint_connection_error(e) and not _is_mint_rate_limited(e):
if not is_mint_connection_error(e) and not is_mint_rate_limited(e):
raise
logger.warning(
"request_mint failed, trying fallback mint",
@@ -352,8 +379,7 @@ async def recover_invoice(
async def check_invoice_payment(
invoice: LightningInvoice, session: AsyncSession
) -> None:
lock = _invoice_settlement_locks.setdefault(invoice.id, asyncio.Lock())
async with lock, wallet_operation_guard():
async with _invoice_settlement_lock(invoice.id), wallet_operation_guard():
minted = False
try:
# Snapshot the row and end the caller's read transaction before any
@@ -368,7 +394,7 @@ async def check_invoice_payment(
mint_url = settlement.mint_url or settings.primary_mint
wallet = await get_wallet(mint_url, "sat")
mint_status = await _mint_operation(
mint_status = await run_mint_operation(
lambda: wallet.get_mint_quote(settlement.payment_hash),
op_name="get_mint_quote",
mint_url=mint_url,
@@ -483,7 +509,7 @@ async def _mint_invoice_quote(
return
try:
await _mint_operation(
await run_mint_operation(
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
op_name=f"invoice_mint_{invoice.purpose}",
mint_url=mint_url,
+338
View File
@@ -0,0 +1,338 @@
"""Shared policy for bounded, rate-aware Cashu mint API operations."""
from __future__ import annotations
import asyncio
import socket
import time
from contextlib import asynccontextmanager
from contextvars import ContextVar
from typing import Any, AsyncGenerator, Awaitable, Callable
import httpx
from .core.logging import get_logger
from .core.settings import settings
logger = get_logger(__name__)
MINT_TRANSPORT_EXCEPTIONS: tuple[type[BaseException], ...] = (
httpx.NetworkError,
httpx.TimeoutException,
ConnectionError,
socket.gaierror,
asyncio.TimeoutError,
)
MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
_fail_fast_depth: ContextVar[int] = ContextVar("mint_fail_fast_depth", default=0)
class MintRateLimitedError(httpx.HTTPStatusError):
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
class MintCooldownError(Exception):
"""A mint is cooling down and this operation must not wait."""
def __init__(self, mint_url: str, retry_after_seconds: float):
self.mint_url = mint_url
self.retry_after_seconds = max(0.0, retry_after_seconds)
super().__init__(
f"Mint {mint_url} is cooling down; retry after "
f"{self.retry_after_seconds:.2f}s"
)
@asynccontextmanager
async def fail_fast_mint_operations() -> AsyncGenerator[None, None]:
"""Make mint cooldown/probe waits fail fast in the current task.
Wallet mutation code holds a process-wide file lock. It enters this scope so
an existing mint cooldown can never turn that lock into a multi-hour wait.
"""
token = _fail_fast_depth.set(_fail_fast_depth.get() + 1)
try:
yield
finally:
_fail_fast_depth.reset(token)
class MintRateGuard:
"""Limit concurrency and remember per-mint cooldown/probe state."""
_guards: dict[str, "MintRateGuard"] = {}
@classmethod
def get(cls, mint_url: str) -> "MintRateGuard":
concurrency = settings.mint_max_concurrency
guard = cls._guards.get(mint_url)
if guard is None or guard._max_concurrency != concurrency:
guard = cls(mint_url, concurrency)
cls._guards[mint_url] = guard
return guard
def __init__(self, mint_url: str, max_concurrency: int):
self._mint_url = mint_url
self._max_concurrency = max_concurrency
self._semaphore = (
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
)
self._cooldown_until = 0.0
self._cooldown_reason: str | None = None
self._consecutive_rate_limits = 0
self._needs_probe = False
self._probe_lock = asyncio.Lock()
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
deadline = time.monotonic() + max(0.0, delay)
if deadline >= self._cooldown_until:
self._cooldown_until = deadline
if reason is not None:
self._cooldown_reason = reason
elif self._cooldown_reason is None and reason is not None:
self._cooldown_reason = reason
self._needs_probe = True
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
remaining = self.cooldown_remaining()
if remaining > 0 and self._cooldown_reason == "rate_limited":
minimum = min(
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
)
if minimum > remaining:
self.apply_cooldown(minimum, reason="rate_limited")
return minimum
return remaining
self._consecutive_rate_limits += 1
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
self.apply_cooldown(delay, reason="rate_limited")
return delay
def cooldown_remaining(self) -> float:
return max(0.0, self._cooldown_until - time.monotonic())
def cooldown_reason(self) -> str | None:
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
def _raise_if_wait_forbidden(self) -> None:
if _fail_fast_depth.get() and (
self._needs_probe or self.cooldown_remaining() > 0
):
raise MintCooldownError(self._mint_url, self.cooldown_remaining())
async def _wait_for_cooldown(self) -> None:
while True:
self._raise_if_wait_forbidden()
deadline = self._cooldown_until
wait = max(0.0, deadline - time.monotonic())
if wait <= 0:
return
logger.debug(
"Mint rate guard: cooling down",
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
)
await asyncio.sleep(wait)
if self._cooldown_until <= deadline:
return
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
await self._wait_for_cooldown()
logger.info(
"Mint cooldown ended; sending one probe request",
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
)
try:
result = await factory()
except Exception as error:
if is_mint_rate_limited(error):
retry_after = None
if isinstance(error, httpx.HTTPStatusError):
retry_after = parse_retry_after(error.response.headers)
delay = max(
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS,
retry_after or 0.0,
)
self.apply_cooldown(delay, reason="rate_limited")
else:
self.apply_cooldown(1.0)
logger.warning(
"Mint cooldown probe failed",
extra={
"event": "mint_cooldown_probe_failed",
"mint_url": self._mint_url,
"error": str(error),
"error_type": type(error).__name__,
"cooldown_seconds": round(self.cooldown_remaining(), 2),
"consecutive_rate_limits": self._consecutive_rate_limits,
},
)
raise
self._needs_probe = False
self._cooldown_until = 0.0
self._cooldown_reason = None
self._consecutive_rate_limits = 0
logger.info(
"Mint cooldown probe succeeded; restoring normal concurrency",
extra={
"event": "mint_cooldown_probe_succeeded",
"mint_url": self._mint_url,
},
)
return result
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
while True:
self._raise_if_wait_forbidden()
if self._needs_probe or self.cooldown_remaining() > 0:
async with self._probe_lock:
self._raise_if_wait_forbidden()
if self.cooldown_remaining() > 0:
self._needs_probe = True
if self._needs_probe:
return await self._run_probe(factory)
continue
if self._semaphore is None:
return await factory()
async with self._semaphore:
self._raise_if_wait_forbidden()
if self._needs_probe:
continue
return await factory()
def mint_cooldown_remaining(mint_url: str) -> float:
return MintRateGuard.get(mint_url).cooldown_remaining()
def mint_cooldown_reason(mint_url: str) -> str | None:
return MintRateGuard.get(mint_url).cooldown_reason()
def is_mint_rate_limited(error: BaseException) -> bool:
"""Return whether an exception chain represents HTTP 429/cooldown."""
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, MintCooldownError):
return True
if isinstance(current, httpx.HTTPStatusError):
if current.response.status_code == 429:
return True
current = current.__cause__ or current.__context__
return False
def parse_retry_after(headers: Any) -> float | None:
raw = headers.get("retry-after") or headers.get("Retry-After")
if raw is None:
return None
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return None
async def run_mint_operation(
factory: Callable[[], Awaitable[Any]],
*,
op_name: str = "mint_operation",
mint_url: str = "",
retry_timeouts: bool = True,
retry_on_rate_limit: bool = True,
) -> Any:
"""Run one mint operation with bounded concurrency and adaptive cooldown."""
guard = MintRateGuard.get(mint_url) if mint_url else None
timeout = settings.mint_operation_timeout_seconds
max_attempts = settings.mint_retry_max_attempts + 1
async def timed_factory() -> Any:
if timeout > 0:
return await asyncio.wait_for(factory(), timeout=timeout)
return await factory()
async def invoke() -> Any:
if guard is not None:
return await guard.run(timed_factory)
return await timed_factory()
for attempt in range(max_attempts):
try:
return await invoke()
except MintCooldownError:
raise
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
if retry_timeouts and attempt < max_attempts - 1:
backoff = (2**attempt) + (time.monotonic() % 1.0)
logger.warning(
"Mint operation timed out, retrying",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"backoff_seconds": round(backoff, 2),
},
)
await asyncio.sleep(backoff)
continue
raise httpx.TimeoutException(
f"{op_name} timed out (attempts: {attempt + 1})"
) from exc
except Exception as exc:
if not is_mint_rate_limited(exc):
raise
backoff = (2**attempt) + (time.monotonic() % 1.0)
if isinstance(exc, httpx.HTTPStatusError):
retry_after = parse_retry_after(exc.response.headers)
if retry_after is not None:
backoff = max(retry_after, backoff)
cooldown = backoff
if guard is not None:
cooldown = guard.apply_rate_limit_cooldown(backoff)
if not retry_on_rate_limit:
logger.warning(
"Mint rate-limited, skipping retries for fallback",
extra={
"op_name": op_name,
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
raise
if attempt >= max_attempts - 1:
raise
logger.warning(
"Mint rate-limited, applying cooldown",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
if guard is None:
await asyncio.sleep(cooldown)
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
-12
View File
@@ -18,18 +18,6 @@ from ..wallet import deserialize_token_from_string
logger = get_logger(__name__)
# Interim policy: when Routstr must move value to another trusted mint, the
# cross-mint Lightning round trip can consume fees that are not visible to the
# client. Reserve 5% headroom until the fee-payer policy is made explicit.
_MINT_FEE_ALLOWANCE = 0.05
def apply_mint_fee_allowance(cost_msat: int) -> int:
"""Reserve headroom for possible trusted-mint fallback fees."""
adjusted = math.ceil(cost_msat * (1 - _MINT_FEE_ALLOWANCE))
return max(settings.min_request_msat, adjusted)
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
if x_cashu := headers.get("x-cashu", None):
cashu_token = x_cashu
+41 -25
View File
@@ -1,17 +1,13 @@
from __future__ import annotations
import asyncio
import math
from typing import TypedDict
import httpx
from cashu.core.base import MeltQuoteState
from cashu.wallet.wallet import Proof, Wallet
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
# very slow mint can block a melt (and any caller, e.g. the payout loop)
# indefinitely. _mint_operation (imported lazily in raw_send_to_lnurl to avoid
# a circular import with wallet.py) bounds it via MINT_OPERATION_TIMEOUT_SECONDS.
MELT_TIMEOUT_SECONDS = 60
from ..mint import MINT_TRANSPORT_EXCEPTIONS, run_mint_operation
try:
from bech32 import bech32_decode, convertbits # type: ignore
@@ -222,9 +218,7 @@ async def raw_send_to_lnurl(
lnurl_data["callback_url"], final_amount
)
from ..wallet import _mint_operation
melt_quote_resp = await _mint_operation(
melt_quote_resp = await run_mint_operation(
lambda: wallet.melt_quote(invoice=bolt11_invoice),
op_name="lnurl_melt_quote",
mint_url=str(wallet.url),
@@ -234,22 +228,44 @@ async def raw_send_to_lnurl(
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
try:
_ = await asyncio.wait_for(
_mint_operation(
lambda: wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
),
op_name="lnurl_melt",
mint_url=str(wallet.url),
retry_timeouts=False,
melt_response = await run_mint_operation(
lambda: wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
),
timeout=MELT_TIMEOUT_SECONDS,
op_name="lnurl_melt",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
except MINT_TRANSPORT_EXCEPTIONS as error:
melt_response = None
melt_error: BaseException | None = error
else:
melt_error = None
if getattr(melt_response, "state", None) == MeltQuoteState.paid:
return final_amount
try:
quote = await run_mint_operation(
lambda: wallet.get_melt_quote(melt_quote_resp.quote),
op_name="reconcile_lnurl_melt_quote",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except Exception as reconciliation_error:
raise LNURLError(
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
) from e
return final_amount
"Melt outcome is ambiguous; quote reconciliation failed and proofs "
"must not be retried"
) from reconciliation_error
if quote is not None and quote.state == MeltQuoteState.paid:
return final_amount
state = getattr(getattr(quote, "state", None), "value", "unknown")
raise LNURLError(
"Melt outcome is ambiguous; proofs must not be retried "
f"(quote_state={state})"
) from melt_error
-5
View File
@@ -27,7 +27,6 @@ from .core.db import (
from .core.exceptions import UpstreamError
from .core.not_found import build_not_found_response
from .payment.helpers import (
apply_mint_fee_allowance,
calculate_discounted_max_cost,
check_token_balance,
create_error_response,
@@ -354,7 +353,6 @@ async def _proxy(
max_cost_for_model = await calculate_discounted_max_cost(
_max_cost_for_model, request_body_dict, model_obj=model_obj
)
max_cost_for_model = apply_mint_fee_allowance(max_cost_for_model)
check_token_balance(headers, request_body_dict, max_cost_for_model)
@@ -493,9 +491,6 @@ async def _proxy(
candidate_max = await calculate_discounted_max_cost(
candidate_max, request_body_dict, model_obj=model_obj
)
# Apply the same interim 5% trusted-mint fee headroom used for the
# first candidate; failover must not silently change admission.
candidate_max = apply_mint_fee_allowance(candidate_max)
if candidate_max > max_cost_for_model:
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
+190 -423
View File
@@ -2,16 +2,15 @@ import asyncio
import fcntl
import os
import re
import socket
import time
import typing
from contextlib import asynccontextmanager
from contextvars import ContextVar
from pathlib import Path
from typing import Any, AsyncGenerator, Awaitable, Callable, TypedDict
from typing import AsyncGenerator, TypedDict
import httpx
from cashu.core.base import MintQuote, Proof, Token
from cashu.core.base import MeltQuoteState, MintQuote, Proof, Token
from cashu.core.mint_info import MintInfo as _CashuMintInfo
from cashu.wallet.helpers import deserialize_token_from_string
from cashu.wallet.wallet import Wallet as _CashuWallet
@@ -21,8 +20,27 @@ from sqlmodel import col, select, update
from .core import db, get_logger
from .core.db import store_cashu_transaction_with_retry as store_cashu_transaction
from .core.settings import settings
from .mint import (
MINT_TRANSPORT_COOLDOWN_SECONDS,
MINT_TRANSPORT_EXCEPTIONS,
MintRateGuard,
MintRateLimitedError,
fail_fast_mint_operations,
is_mint_rate_limited,
mint_cooldown_reason,
mint_cooldown_remaining,
run_mint_operation,
)
from .payment.lnurl import raw_send_to_lnurl
# Backwards-compatible aliases for callers/tests that imported the former
# wallet-local policy. Production modules use the public routstr.mint API.
_MintRateGuard = MintRateGuard
_mint_operation = run_mint_operation
_mint_cooldown_remaining = mint_cooldown_remaining
_mint_cooldown_reason = mint_cooldown_reason
_is_mint_rate_limited = is_mint_rate_limited
# cashu still declares Optional[X] without explicit defaults on MintInfo.
# Under pydantic v2 those are required, but real mints omit many of them.
# Default Optional fields to None at import time so balance fetches don't 422.
@@ -69,7 +87,8 @@ async def wallet_operation_guard() -> AsyncGenerator[None, None]:
except BlockingIOError:
await _scheduler_sleep(0.05)
depth_token = _wallet_operation_depth.set(1)
yield
async with fail_fast_mint_operations():
yield
finally:
if depth_token is not None:
_wallet_operation_depth.reset(depth_token)
@@ -94,6 +113,20 @@ def _mints_to_inspect() -> list[str]:
return mint_urls
class Wallet(_CashuWallet):
"""Cashu adapter that preserves HTTP 429 for Routstr's mint policy."""
@staticmethod
def raise_on_error_request(resp: httpx.Response) -> None:
if resp.status_code == 429:
raise MintRateLimitedError(
"Cashu mint rate limited",
request=resp.request,
response=resp,
)
_CashuWallet.raise_on_error_request(resp)
class MintConnectionError(Exception):
"""The mint could not be reached (network transport failure).
@@ -116,339 +149,6 @@ class TokenConsumedError(Exception):
"""
class MintRateLimitedError(httpx.HTTPStatusError):
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
class Wallet(_CashuWallet):
"""Cashu wallet adapter that preserves rate-limit status information.
Cashu's default response adapter converts JSON error bodies into plain
``Exception`` instances before calling ``raise_for_status``. Intercept 429
here so Routstr's fallback and cooldown policy can use the real status
without unreliable message matching.
"""
@staticmethod
def raise_on_error_request(resp: httpx.Response) -> None:
if resp.status_code == 429:
raise MintRateLimitedError(
"Cashu mint rate limited",
request=resp.request,
response=resp,
)
_CashuWallet.raise_on_error_request(resp)
# httpx base classes cover their subclasses. HTTPStatusError is excluded on
# purpose — that means the mint answered, just with an error status.
_MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
_TRANSPORT_EXC_TYPES: tuple[type[BaseException], ...] = (
httpx.NetworkError,
httpx.TimeoutException,
ConnectionError, # refused/reset/aborted
socket.gaierror, # DNS failure
asyncio.TimeoutError,
)
class _MintRateGuard:
"""Limit concurrency and remember per-mint rate-limit cooldowns."""
_guards: dict[str, "_MintRateGuard"] = {}
@classmethod
def get(cls, mint_url: str) -> "_MintRateGuard":
concurrency = settings.mint_max_concurrency
guard = cls._guards.get(mint_url)
if guard is None or guard._max_concurrency != concurrency:
guard = cls(mint_url, concurrency)
cls._guards[mint_url] = guard
return guard
def __init__(self, mint_url: str, max_concurrency: int):
self._mint_url = mint_url
self._max_concurrency = max_concurrency
self._semaphore = (
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
)
self._cooldown_until = 0.0
self._cooldown_reason: str | None = None
self._consecutive_rate_limits = 0
self._needs_probe = False
self._probe_lock = asyncio.Lock()
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
deadline = time.monotonic() + max(0.0, delay)
if deadline >= self._cooldown_until:
self._cooldown_until = deadline
if reason is not None:
self._cooldown_reason = reason
elif self._cooldown_reason is None and reason is not None:
self._cooldown_reason = reason
self._needs_probe = True
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
remaining = self.cooldown_remaining()
if remaining > 0 and self._cooldown_reason == "rate_limited":
minimum = min(
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
)
if minimum > remaining:
self.apply_cooldown(minimum, reason="rate_limited")
return minimum
return remaining
self._consecutive_rate_limits += 1
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
self.apply_cooldown(delay, reason="rate_limited")
return delay
def cooldown_remaining(self) -> float:
return max(0.0, self._cooldown_until - time.monotonic())
def cooldown_reason(self) -> str | None:
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
async def _wait_for_cooldown(self) -> None:
while True:
deadline = self._cooldown_until
wait = max(0.0, deadline - time.monotonic())
if wait <= 0:
return
logger.debug(
"Mint rate guard: cooling down",
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
)
await asyncio.sleep(wait)
if self._cooldown_until <= deadline:
return
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
await self._wait_for_cooldown()
logger.warning(
"Mint cooldown ended; sending one probe request",
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
)
try:
result = await factory()
except Exception as error:
# Keep queued callers behind the probe. On a rate-limit,
# re-apply the *same* cooldown the caller already set rather
# than calling apply_rate_limit_cooldown() — the probe is a
# recovery check, not a new request that should escalate the
# exponential backoff counter.
if _is_mint_rate_limited(error):
retry_after = None
if isinstance(error, httpx.HTTPStatusError):
retry_after = _parse_retry_after(error.response.headers)
delay = max(
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS,
retry_after or 0.0,
)
self.apply_cooldown(delay, reason="rate_limited")
else:
self.apply_cooldown(1.0)
logger.warning(
"Mint cooldown probe failed",
extra={
"event": "mint_cooldown_probe_failed",
"mint_url": self._mint_url,
"error": str(error),
"error_type": type(error).__name__,
"cooldown_seconds": round(self.cooldown_remaining(), 2),
"consecutive_rate_limits": self._consecutive_rate_limits,
},
)
raise
self._needs_probe = False
self._cooldown_until = 0.0
self._cooldown_reason = None
self._consecutive_rate_limits = 0
logger.warning(
"Mint cooldown probe succeeded; restoring normal concurrency",
extra={
"event": "mint_cooldown_probe_succeeded",
"mint_url": self._mint_url,
},
)
return result
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
while True:
if self._needs_probe or self.cooldown_remaining() > 0:
async with self._probe_lock:
if self.cooldown_remaining() > 0:
self._needs_probe = True
if self._needs_probe:
return await self._run_probe(factory)
continue
if self._semaphore is None:
return await factory()
async with self._semaphore:
if self._needs_probe:
continue
return await factory()
def _mint_cooldown_remaining(mint_url: str) -> float:
return _MintRateGuard.get(mint_url).cooldown_remaining()
def _mint_cooldown_reason(mint_url: str) -> str | None:
return _MintRateGuard.get(mint_url).cooldown_reason()
def _is_mint_rate_limited(error: BaseException) -> bool:
"""True if the mint returned an HTTP 429 (Too Many Requests).
Only matches ``httpx.HTTPStatusError`` with status code 429 — never
classifies based on the exception's message text. Substring matching
on ``"rate limit"`` / ``"too many requests"`` was removed because it
catches unrelated errors (e.g. a 503 whose body happens to mention
"database rate exceeded"), which triggers unnecessary exponential
backoff and can block state recovery indefinitely.
"""
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, httpx.HTTPStatusError):
if current.response.status_code == 429:
return True
current = current.__cause__ or current.__context__
return False
async def _mint_operation(
factory: Callable[[], Awaitable[Any]],
*,
op_name: str = "mint_operation",
mint_url: str = "",
retry_timeouts: bool = True,
retry_on_rate_limit: bool = True,
) -> Any:
"""Run a mint operation with bounded concurrency and adaptive cooldown.
The timeout applies to each network attempt. Queueing, cooldown, and retry
backoff are deliberately outside it so the shipped 60-second 429 cooldown
is not cancelled by the 30-second operation timeout. ``factory`` must
return a fresh coroutine for every retry.
When ``retry_on_rate_limit`` is False a 429 is not retried in-place — the
cooldown is still applied to the per-mint guard (so subsequent operations on
that mint wait), but the exception is re-raised so the caller (typically
``_request_mint_with_fallback``) can immediately try a different mint.
"""
guard = _MintRateGuard.get(mint_url) if mint_url else None
timeout = settings.mint_operation_timeout_seconds
max_attempts = settings.mint_retry_max_attempts + 1
async def timed_factory() -> Any:
if timeout > 0:
return await asyncio.wait_for(factory(), timeout=timeout)
return await factory()
async def invoke() -> Any:
if guard is not None:
return await guard.run(timed_factory)
return await timed_factory()
async def run_with_retries() -> Any:
for attempt in range(max_attempts):
try:
return await invoke()
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
if retry_timeouts and attempt < max_attempts - 1:
backoff = (2**attempt) + (time.monotonic() % 1.0)
logger.warning(
"Mint operation timed out, retrying",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"backoff_seconds": round(backoff, 2),
},
)
await asyncio.sleep(backoff)
continue
raise httpx.TimeoutException(
f"{op_name} timed out (attempts: {attempt + 1})"
) from exc
except Exception as exc:
if not _is_mint_rate_limited(exc):
raise
# Apply cooldown to the guard regardless — even when we're
# about to re-raise for fallback, the guard must remember that
# this mint is rate-limited for future operations.
backoff = (2**attempt) + (time.monotonic() % 1.0)
if isinstance(exc, httpx.HTTPStatusError):
retry_after = _parse_retry_after(exc.response.headers)
if retry_after is not None:
backoff = max(retry_after, backoff)
cooldown = backoff
if guard is not None:
cooldown = guard.apply_rate_limit_cooldown(backoff)
# When the caller has a fallback strategy (trusted-mint
# list), re-raise immediately so the caller can try the next
# mint instead of waiting through this mint's cooldown.
if not retry_on_rate_limit:
logger.warning(
"Mint rate-limited, skipping retries for fallback",
extra={
"op_name": op_name,
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
raise
if attempt >= max_attempts - 1:
raise
logger.warning(
"Mint rate-limited, applying cooldown",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
if guard is None:
await asyncio.sleep(cooldown)
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
return await run_with_retries()
def _parse_retry_after(headers: Any) -> float | None:
"""Parse a Retry-After header (delta-seconds form) into seconds."""
raw = headers.get("retry-after") or headers.get("Retry-After")
if raw is None:
return None
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return None
def is_source_mint_connection_error(error: BaseException) -> bool:
seen: set[int] = set()
current: BaseException | None = error
@@ -475,7 +175,7 @@ def is_mint_connection_error(error: BaseException) -> bool:
return False
if isinstance(current, MintConnectionError):
return True
if isinstance(current, _TRANSPORT_EXC_TYPES):
if isinstance(current, MINT_TRANSPORT_EXCEPTIONS):
return True
current = current.__cause__ or current.__context__
return False
@@ -519,7 +219,7 @@ def classify_redemption_error(
"The mint that issued this Cashu token is unreachable; the token cannot be redeemed at another mint",
"cashu_source_mint_unreachable",
)
if _is_mint_rate_limited(error):
if is_mint_rate_limited(error):
return (
"mint_rate_limited",
503,
@@ -605,14 +305,14 @@ async def _redeem_same_mint(
drifts insolvent.
"""
try:
await _mint_operation(
await run_mint_operation(
lambda: wallet.load_mint(keyset_id=token_obj.keysets[0]),
op_name="redeem_load_mint",
mint_url=token_obj.mint,
)
wallet.verify_proofs_dleq(token_obj.proofs)
input_fees = wallet.get_fees_for_proofs(token_obj.proofs)
await _mint_operation(
await run_mint_operation(
lambda: wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True),
op_name="redeem_split",
mint_url=token_obj.mint,
@@ -655,7 +355,7 @@ async def recieve_token(
destinations = list(
dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
)
logger.warning(
logger.info(
"Cashu cross-mint swap required",
extra={
"event": "cashu_swap_started",
@@ -667,7 +367,7 @@ async def recieve_token(
)
return await swap_to_trusted_mint(token_obj, wallet)
logger.warning(
logger.info(
"Trying same-mint Cashu redemption",
extra={
"event": "cashu_same_mint_redemption",
@@ -769,12 +469,12 @@ async def find_trusted_mint_with_funds(
balances: dict[str, int] = {}
for mint_url in candidates:
if _mint_cooldown_remaining(mint_url) > 0:
if mint_cooldown_remaining(mint_url) > 0:
continue
try:
wallet = await get_wallet(mint_url, unit, retry_on_rate_limit=False)
except Exception as error:
if is_mint_connection_error(error) or _is_mint_rate_limited(error):
if is_mint_connection_error(error) or is_mint_rate_limited(error):
balances[mint_url] = 0
continue
raise
@@ -819,6 +519,17 @@ def _net_minted_amount(amount_msat: int, token_unit: str, fees: int) -> int:
return int(remaining_msat)
def _melt_definitively_failed(error: Exception) -> bool:
"""Return whether the mint authoritatively rejected the Lightning payment.
Cashu releases the reserved proofs for these responses, so the token remains
reusable. Transport failures and unknown errors are deliberately excluded:
after dispatch their payment outcome may still be pending or paid.
"""
message = str(error).strip()
return message.lower() == "could not pay invoice." or "(Code: 20004)" in message
def _melt_insufficient_shortfall(error: Exception) -> int | None:
"""
Classify a melt failure: return the observed shortfall (in the token unit)
@@ -871,7 +582,7 @@ async def _request_mint_with_fallback(
f"Token value is too small after fee deduction or unit conversion."
)
candidates = list(dict.fromkeys([settings.primary_mint, *settings.cashu_mints]))
logger.warning(
logger.info(
"Trying trusted destination mints",
extra={
"event": "cashu_destination_candidates",
@@ -883,7 +594,7 @@ async def _request_mint_with_fallback(
)
tried: list[str] = []
for candidate_index, mint_url in enumerate(candidates, start=1):
cooldown = _mint_cooldown_remaining(mint_url)
cooldown = mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.warning(
@@ -898,7 +609,7 @@ async def _request_mint_with_fallback(
},
)
continue
logger.warning(
logger.info(
"Trying destination mint",
extra={
"event": "cashu_destination_attempt",
@@ -917,13 +628,13 @@ async def _request_mint_with_fallback(
settings.primary_mint_unit,
retry_on_rate_limit=False,
)
quote = await _mint_operation(
quote = await run_mint_operation(
lambda: wallet.request_mint(amount),
op_name=op_name,
mint_url=mint_url,
retry_on_rate_limit=False,
)
logger.warning(
logger.info(
"Destination mint selected",
extra={
"event": "cashu_destination_selected",
@@ -937,12 +648,12 @@ async def _request_mint_with_fallback(
except Exception as error:
tried.append(f"{mint_url}: {type(error).__name__}")
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
rate_limited = is_mint_rate_limited(error)
if not connection_failure and not rate_limited:
raise
if connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
_MINT_TRANSPORT_COOLDOWN_SECONDS, reason="unreachable"
MintRateGuard.get(mint_url).apply_cooldown(
MINT_TRANSPORT_COOLDOWN_SECONDS, reason="unreachable"
)
logger.warning(
"Destination mint failed",
@@ -993,7 +704,7 @@ async def _calculate_swap_amount(
if token_mint_url == settings.primary_mint:
logger.info(
"swap_to_primary_mint: skipping fee estimation (same mint)",
"swap_to_trusted_mint: skipping fee estimation (same mint)",
extra={"minted_amount": receive_amount},
)
return int(receive_amount)
@@ -1005,7 +716,7 @@ async def _calculate_swap_amount(
# logs. Guard early with full diagnostic context instead.
if receive_amount <= 0:
logger.error(
"swap_to_primary_mint: receive_amount is zero or negative, cannot estimate fees",
"swap_to_trusted_mint: receive_amount is zero or negative, cannot estimate fees",
extra={
"amount_msat": amount_msat,
"token_unit": token_unit,
@@ -1022,7 +733,7 @@ async def _calculate_swap_amount(
)
logger.info(
"swap_to_primary_mint: estimating fees",
"swap_to_trusted_mint: estimating fees",
extra={
"dummy_amount": receive_amount,
"unit": settings.primary_mint_unit,
@@ -1040,7 +751,7 @@ async def _calculate_swap_amount(
primary_wallet=primary_wallet,
)
stage = "source_fee_quote"
dummy_melt_quote = await _mint_operation(
dummy_melt_quote = await run_mint_operation(
lambda: token_wallet.melt_quote(dummy_mint_quote.request),
op_name="swap_fee_est_melt_quote",
mint_url=token_mint_url,
@@ -1055,7 +766,7 @@ async def _calculate_swap_amount(
raise ValueError(f"Fees ({total_fees} {token_unit}) exceed token amount")
logger.info(
"swap_to_primary_mint: fee estimation result",
"swap_to_trusted_mint: fee estimation result",
extra={
"token_amount_sat": _msats_to_sats(amount_msat),
"estimated_fee": total_fees,
@@ -1105,10 +816,62 @@ async def _calculate_swap_amount(
raise ValueError(f"Failed to estimate fees: {e}") from e
async def _reconcile_ambiguous_melt(
wallet: Wallet, quote_id: str, proofs: list[Proof]
) -> bool:
"""Confirm a dispatched melt is paid or conservatively mark it ambiguous.
A PAID quote is authoritative and does not require a proof-state lookup.
Every other immediate snapshot remains unsafe to retry: an in-flight
Lightning payment can still move UNPAID/UNSPENT to PENDING or PAID after the
cancelled HTTP request returns.
"""
try:
quote = await run_mint_operation(
lambda: wallet.get_melt_quote(quote_id),
op_name="reconcile_swap_melt_quote",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except Exception as error:
raise TokenConsumedError(
"Source melt outcome is unknown; reconciliation required"
) from error
if quote is not None and quote.state == MeltQuoteState.paid:
return True
try:
proof_response = await run_mint_operation(
lambda: wallet.check_proof_state(proofs),
op_name="reconcile_swap_proofs",
mint_url=str(wallet.url),
retry_timeouts=False,
)
proof_states = [state.state.value for state in proof_response.states]
except Exception:
proof_states = []
quote_state = getattr(getattr(quote, "state", None), "value", "unknown")
raise TokenConsumedError(
"Source melt outcome is ambiguous; reconciliation required "
f"(quote_state={quote_state}, proof_states={proof_states})"
)
async def _confirm_melt_paid(
wallet: Wallet, quote_id: str, proofs: list[Proof], response: object
) -> bool:
"""Accept a melt response only when PAID is explicit or reconciled."""
if getattr(response, "state", None) == MeltQuoteState.paid:
return True
return await _reconcile_ambiguous_melt(wallet, quote_id, proofs)
async def swap_to_trusted_mint(
token_obj: Token, token_wallet: Wallet
) -> tuple[int, str, str]:
logger.warning(
logger.info(
"Starting Cashu cross-mint swap",
extra={
"event": "cashu_swap_started",
@@ -1135,7 +898,7 @@ async def swap_to_trusted_mint(
# NUT-02 input fee still applies; _redeem_same_mint accounts for it.
if token_obj.mint == settings.primary_mint:
logger.info(
"swap_to_primary_mint: token already on primary mint, skipping swap",
"swap_to_trusted_mint: token already on primary mint, skipping swap",
extra={
"mint": token_obj.mint,
"amount": token_amount,
@@ -1166,7 +929,7 @@ async def swap_to_trusted_mint(
attempt += 1
if minted_amount <= 0:
logger.error(
"swap_to_primary_mint: minted_amount is zero or negative before requesting quote",
"swap_to_trusted_mint: minted_amount is zero or negative before requesting quote",
extra={
"minted_amount": minted_amount,
"attempt": attempt,
@@ -1188,7 +951,7 @@ async def swap_to_trusted_mint(
primary_wallet=primary_wallet,
)
logger.info(
"swap_to_primary_mint: mint quote received",
"swap_to_trusted_mint: mint quote received",
extra={
"mint_quote_id": mint_quote.quote,
"attempt": attempt,
@@ -1196,7 +959,7 @@ async def swap_to_trusted_mint(
},
)
logger.warning(
logger.info(
"Requesting melt quote from source mint",
extra={
"event": "cashu_source_melt_quote_attempt",
@@ -1206,7 +969,7 @@ async def swap_to_trusted_mint(
},
)
try:
melt_quote = await _mint_operation(
melt_quote = await run_mint_operation(
lambda: token_wallet.melt_quote(mint_quote.request),
op_name="swap_melt_quote",
mint_url=token_obj.mint,
@@ -1232,7 +995,7 @@ async def swap_to_trusted_mint(
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
logger.info(
"swap_to_primary_mint: melt quote received",
"swap_to_trusted_mint: melt quote received",
extra={
"melt_quote_id": melt_quote.quote,
"melt_amount": melt_quote.amount,
@@ -1252,7 +1015,7 @@ async def swap_to_trusted_mint(
)
if attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
logger.warning(
"swap_to_primary_mint: insufficient token amount for melt fees",
"swap_to_trusted_mint: insufficient token amount for melt fees",
extra={
"token_amount": token_amount,
"melt_amount": melt_quote.amount,
@@ -1269,7 +1032,7 @@ async def swap_to_trusted_mint(
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve} + input_fees: {input_fees})"
)
logger.warning(
"swap_to_primary_mint: melt quote exceeds token amount, retrying",
"swap_to_trusted_mint: melt quote exceeds token amount, retrying",
extra={
"total_needed": total_needed,
"token_amount": token_amount,
@@ -1281,7 +1044,7 @@ async def swap_to_trusted_mint(
continue
try:
_ = await _mint_operation(
melt_response = await run_mint_operation(
lambda: token_wallet.melt(
proofs=token_obj.proofs,
invoice=mint_quote.request,
@@ -1292,36 +1055,45 @@ async def swap_to_trusted_mint(
mint_url=token_obj.mint,
retry_timeouts=False,
)
await _confirm_melt_paid(
token_wallet, melt_quote.quote, token_obj.proofs, melt_response
)
except Exception as e:
# A down mint won't fix itself by retrying with a smaller amount.
if is_mint_connection_error(e):
logger.error(
"Source mint became unreachable during melt",
extra={
"event": "cashu_source_mint_unreachable",
"stage": "source_melt",
"error": str(e),
"error_type": type(e).__name__,
"source_mint": token_obj.mint,
"destination_mint": dest_mint_url,
"attempt": attempt,
},
)
raise SourceMintConnectionError(
"Issuing Cashu mint is unreachable"
) from e
shortfall = _melt_insufficient_shortfall(e)
recomputed = 0
if shortfall is not None:
observed_extra_fee += shortfall
recomputed = _net_minted_amount(
amount_msat,
token_obj.unit,
melt_quote.fee_reserve + input_fees + observed_extra_fee,
)
if shortfall is None or attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
if shortfall is None:
if isinstance(e, TokenConsumedError):
raise
if _melt_definitively_failed(e):
raise ValueError(
f"Failed to melt token from foreign mint {token_obj.mint}: {e}"
) from e
if is_mint_connection_error(e):
await _reconcile_ambiguous_melt(
token_wallet, melt_quote.quote, token_obj.proofs
)
logger.info(
"Source melt reconciled as paid; minting on destination",
extra={
"event": "cashu_source_melt_reconciled_paid",
"source_mint": token_obj.mint,
"destination_mint": dest_mint_url,
"melt_quote_id": melt_quote.quote,
},
)
break
raise TokenConsumedError(
"Source melt failed after dispatch; outcome requires reconciliation"
) from e
observed_extra_fee += shortfall
recomputed = _net_minted_amount(
amount_msat,
token_obj.unit,
melt_quote.fee_reserve + input_fees + observed_extra_fee,
)
if attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
logger.error(
"swap_to_primary_mint: melt failed",
"swap_to_trusted_mint: melt failed",
extra={
"error": str(e),
"error_type": type(e).__name__,
@@ -1336,7 +1108,7 @@ async def swap_to_trusted_mint(
f"Failed to melt token from foreign mint {token_obj.mint}: {e}"
) from e
logger.warning(
"swap_to_primary_mint: mint demanded more than quoted at melt, retrying",
"swap_to_trusted_mint: mint demanded more than quoted at melt, retrying",
extra={
"shortfall": shortfall,
"retry_minted_amount": recomputed,
@@ -1348,7 +1120,7 @@ async def swap_to_trusted_mint(
break
logger.warning(
logger.info(
"Source melt succeeded; minting on destination",
extra={
"event": "cashu_destination_mint_attempt",
@@ -1361,9 +1133,9 @@ async def swap_to_trusted_mint(
await dest_wallet.load_proofs(reload=True)
pre_mint_balance = dest_wallet.available_balance.amount
try:
_ = await _mint_operation(
_ = await run_mint_operation(
lambda: dest_wallet.mint(minted_amount, quote_id=mint_quote.quote),
op_name="swap_mint_on_primary",
op_name="swap_mint_on_destination",
mint_url=dest_mint_url,
retry_timeouts=False,
)
@@ -1373,7 +1145,7 @@ async def swap_to_trusted_mint(
# bump_secret_derivation ran locally. Recover orphaned proofs and
# advance the counter so the next request derives fresh secrets.
logger.warning(
"swap_to_primary_mint: outputs already signed — recovering orphaned proofs",
"swap_to_trusted_mint: outputs already signed — recovering orphaned proofs",
extra={
"mint_quote_id": mint_quote.quote,
"minted_amount": minted_amount,
@@ -1388,7 +1160,7 @@ async def swap_to_trusted_mint(
post_recovery_balance = dest_wallet.available_balance.amount
balance_gained = post_recovery_balance - pre_mint_balance
logger.info(
"swap_to_primary_mint: recovery scan completed",
"swap_to_trusted_mint: recovery scan completed",
extra={
"pre_mint_balance": pre_mint_balance,
"post_recovery_balance": post_recovery_balance,
@@ -1411,7 +1183,7 @@ async def swap_to_trusted_mint(
raise
except Exception as recovery_err:
logger.error(
"swap_to_primary_mint: recovery failed",
"swap_to_trusted_mint: recovery failed",
extra={"error": str(recovery_err)},
)
raise TokenConsumedError(
@@ -1419,7 +1191,7 @@ async def swap_to_trusted_mint(
) from e
else:
logger.error(
"swap_to_primary_mint: mint on primary failed after successful melt",
"swap_to_trusted_mint: mint on primary failed after successful melt",
extra={
"error": str(e),
"error_type": type(e).__name__,
@@ -1432,7 +1204,7 @@ async def swap_to_trusted_mint(
"Mint on primary failed after successful melt"
) from e
logger.warning(
logger.info(
"Cashu cross-mint swap completed",
extra={
"event": "cashu_swap_completed",
@@ -1595,13 +1367,13 @@ async def get_wallet(
now = time.monotonic()
last = _wallet_last_load.get(id)
if last is None or now - last >= _WALLOAD_RELOAD_MIN_INTERVAL_SECONDS:
await _mint_operation(
await run_mint_operation(
lambda: _wallets[id].load_mint(),
op_name="load_mint",
mint_url=mint_url,
retry_on_rate_limit=retry_on_rate_limit,
)
await _mint_operation(
await run_mint_operation(
lambda: _wallets[id].load_proofs(reload=True),
op_name="load_proofs",
mint_url=mint_url,
@@ -1640,7 +1412,7 @@ async def slow_filter_spend_proofs(
batch_size = 1000
for i in range(0, len(proofs), batch_size):
pb = proofs[i : i + batch_size]
proof_states = await _mint_operation(
proof_states = await run_mint_operation(
lambda: wallet.check_proof_state(pb),
op_name="check_proof_state",
mint_url=str(wallet.url),
@@ -1652,12 +1424,7 @@ async def slow_filter_spend_proofs(
else:
_spent_proofs.append(proof)
if _spent_proofs:
await _mint_operation(
lambda: wallet.set_reserved_for_send(_spent_proofs, reserved=True),
op_name="set_reserved_spent_proofs",
mint_url=str(wallet.url),
retry_timeouts=False,
)
await wallet.set_reserved_for_send(_spent_proofs, reserved=True)
return _proofs
@@ -1686,7 +1453,7 @@ async def _get_supported_mint_units(mint_url: str) -> list[str]:
return cached[1]
wallet = await get_wallet(mint_url, settings.primary_mint_unit, load=False)
keysets = await _mint_operation(
keysets = await run_mint_operation(
lambda: wallet._get_keysets(),
op_name="get_mint_keysets",
mint_url=mint_url,
@@ -1750,7 +1517,7 @@ async def fetch_all_balances(
mint_units[mint_url] = await _get_supported_mint_units(mint_url)
except Exception as error:
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
rate_limited = is_mint_rate_limited(error)
error_code = (
"rate_limited"
if rate_limited
@@ -1759,12 +1526,12 @@ async def fetch_all_balances(
else "mint_error"
)
if connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
MintRateGuard.get(mint_url).apply_cooldown(
_BALANCE_FETCH_RETRY_SECONDS, reason="unreachable"
)
retry_delay = max(
_BALANCE_FETCH_RETRY_SECONDS,
_mint_cooldown_remaining(mint_url),
mint_cooldown_remaining(mint_url),
)
discovery_errors.append(
_balance_error(
@@ -1819,9 +1586,9 @@ async def fetch_all_balances(
retry_after_seconds=failure[0] - now,
)
cooldown = _mint_cooldown_remaining(mint_url)
cooldown = mint_cooldown_remaining(mint_url)
if cooldown > 0:
error_code = _mint_cooldown_reason(mint_url) or "cooldown"
error_code = mint_cooldown_reason(mint_url) or "cooldown"
error = {
"rate_limited": "Mint is rate limited",
"unreachable": "Mint is unreachable",
@@ -1846,7 +1613,7 @@ async def fetch_all_balances(
proofs = await slow_filter_spend_proofs(proofs, wallet)
except Exception as error:
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
rate_limited = is_mint_rate_limited(error)
error_code = (
"rate_limited"
if rate_limited
@@ -1855,16 +1622,16 @@ async def fetch_all_balances(
else "mint_error"
)
if rate_limited:
_MintRateGuard.get(mint_url).apply_rate_limit_cooldown(
MintRateGuard.get(mint_url).apply_rate_limit_cooldown(
_BALANCE_FETCH_RETRY_SECONDS
)
elif connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
MintRateGuard.get(mint_url).apply_cooldown(
_BALANCE_FETCH_RETRY_SECONDS, reason=error_code
)
retry_delay = max(
_BALANCE_FETCH_RETRY_SECONDS,
_mint_cooldown_remaining(mint_url),
mint_cooldown_remaining(mint_url),
)
_balance_fetch_failures[key] = (
time.monotonic() + retry_delay,
+18 -13
View File
@@ -208,25 +208,30 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
@pytest.mark.asyncio
async def test_five_percent_mint_fallback_headroom_is_admitted_and_reserved(
async def test_full_model_maximum_is_required_and_reserved(
integration_session: AsyncSession,
) -> None:
from routstr.auth import pay_for_request, validate_bearer_key
from routstr.payment.helpers import apply_mint_fee_allowance
key = _key(balance=95_000)
integration_session.add(key)
short_key = _key(balance=95_000)
exact_key = _key(balance=100_000)
integration_session.add(short_key)
integration_session.add(exact_key)
await integration_session.commit()
admission_cost = apply_mint_fee_allowance(100_000)
validated = await validate_bearer_key(
f"sk-{key.hashed_key}", integration_session, min_cost=admission_cost
)
await pay_for_request(validated, admission_cost, integration_session)
with pytest.raises(HTTPException) as insufficient:
await validate_bearer_key(
f"sk-{short_key.hashed_key}", integration_session, min_cost=100_000
)
assert insufficient.value.status_code == 402
await integration_session.refresh(key)
assert admission_cost == 95_000
assert key.reserved_balance == 95_000
validated = await validate_bearer_key(
f"sk-{exact_key.hashed_key}", integration_session, min_cost=100_000
)
await pay_for_request(validated, 100_000, integration_session)
await integration_session.refresh(exact_key)
assert exact_key.reserved_balance == 100_000
# ---------------------------------------------------------------------------
@@ -288,7 +293,7 @@ async def test_http_402_response_shape_on_insufficient_balance(
error = body["detail"]["error"]
assert error["code"] == "insufficient_balance"
assert error["type"] == "insufficient_quota"
assert "591.744 sats (591744 msats) required" in error["message"]
assert "622.888 sats (622888 msats) required" in error["message"]
assert "20.32 sats (20320 msats) available" in error["message"]
# Balance must be completely untouched
+5 -2
View File
@@ -20,6 +20,7 @@ from collections.abc import Callable
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import MeltQuoteState
from httpx import AsyncClient, Response
from routstr.core.settings import settings
@@ -81,7 +82,9 @@ def _make_swap_mocks(
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
)
)
mock_token_wallet.melt = AsyncMock(return_value=Mock())
mock_token_wallet.melt = AsyncMock(
return_value=Mock(state=MeltQuoteState.paid)
)
return mock_token, mock_token_wallet, mock_primary_wallet
@@ -144,7 +147,7 @@ async def test_topup_retries_when_melt_demands_more_than_quoted(
"Mint Error: not enough inputs provided for melt. "
"Provided: 179, needed: 180 (Code: 11000)"
),
Mock(),
Mock(state=MeltQuoteState.paid),
]
response = await _post_topup(
+3 -3
View File
@@ -161,7 +161,7 @@ async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=10),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
first = await fetch_all_balances(units=["sat"])
@@ -180,7 +180,7 @@ async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=71),
patch("routstr.mint.time.monotonic", return_value=71),
patch("routstr.wallet.logger.warning"),
):
await fetch_all_balances(units=["sat"])
@@ -219,7 +219,7 @@ async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
patch.object(settings, "primary_mint", mint),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.time.monotonic", return_value=10),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
details, *_ = await fetch_all_balances(units=["sat", "msat"])
+2
View File
@@ -155,6 +155,7 @@ async def test_non_pending_invoice_is_not_minted() -> None:
get_wallet.assert_not_awaited()
session.commit.assert_awaited_once()
assert _invoice_settlement_locks == {}
@pytest.mark.asyncio
@@ -212,3 +213,4 @@ async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
assert invoice.status == "paid"
finalize.assert_awaited_once()
assert _invoice_settlement_locks == {}
+101 -43
View File
@@ -1,70 +1,127 @@
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
timeout as LNURLError instead of hanging.
"""
"""LNURL melt attempts must not misclassify ambiguous payment outcomes."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from cashu.core.base import MeltQuoteState
from routstr.payment import lnurl
from routstr.core.settings import settings
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
LNURL_DATA = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
def _wallet() -> tuple[MagicMock, list[MagicMock]]:
proofs = [MagicMock(amount=1000)]
wallet = MagicMock()
wallet = MagicMock(url="https://mint.test")
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
return wallet, proofs
def _lnurl_patches() -> tuple[Any, Any]:
return (
patch(
"routstr.payment.lnurl.get_lnurl_data",
AsyncMock(return_value=LNURL_DATA),
),
patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
),
)
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_timeout_keeps_unpaid_outcome_ambiguous() -> None:
wallet, proofs = _wallet()
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5) # far longer than the patched timeout
await asyncio.sleep(5)
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.unpaid)
)
data_patch, invoice_patch = _lnurl_patches()
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
pytest.raises(LNURLError, match="outcome is ambiguous"),
):
with pytest.raises(LNURLError, match="Melt timed out"):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
wallet.set_reserved_for_melt.assert_not_called()
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
"""A prompt melt still returns the net amount, unaffected by the guard."""
proofs = [MagicMock(amount=1000)]
async def test_raw_send_to_lnurl_timeout_reconciled_paid_is_success() -> None:
wallet, proofs = _wallet()
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
wallet.melt = AsyncMock(return_value=MagicMock())
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5)
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.paid)
)
data_patch, invoice_patch = _lnurl_patches()
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
assert paid > 0
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_pending_response_stays_ambiguous() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.pending))
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.pending)
)
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
pytest.raises(LNURLError, match="outcome is ambiguous"),
):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_on_explicit_paid_response() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.paid))
wallet.get_melt_quote = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
@@ -72,3 +129,4 @@ async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
assert paid > 0
wallet.melt.assert_awaited_once()
wallet.get_melt_quote.assert_not_awaited()
+91
View File
@@ -0,0 +1,91 @@
from unittest.mock import AsyncMock, Mock
import pytest
from cashu.core.base import MeltQuoteState, ProofSpentState
from routstr.wallet import (
TokenConsumedError,
_confirm_melt_paid,
_reconcile_ambiguous_melt,
)
@pytest.mark.asyncio
async def test_paid_quote_is_authoritative_when_proof_lookup_would_fail() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
check_proof_state=AsyncMock(side_effect=RuntimeError("proof API unavailable")),
)
assert await _reconcile_ambiguous_melt(wallet, "quote-1", [Mock()]) is True
wallet.check_proof_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_timeout_snapshot_unpaid_unspent_remains_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.unpaid)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.unspent)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _reconcile_ambiguous_melt(wallet, "quote-2", [Mock()])
@pytest.mark.asyncio
async def test_successful_pending_melt_response_requires_reconciliation() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.pending)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.pending)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _confirm_melt_paid(
wallet,
"quote-pending",
[Mock()],
Mock(state=MeltQuoteState.pending),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("quote_state", "proof_state"),
[
(MeltQuoteState.pending, ProofSpentState.pending),
(MeltQuoteState.unpaid, ProofSpentState.spent),
(MeltQuoteState.unpaid, ProofSpentState.pending),
],
)
async def test_ambiguous_or_consumed_melt_is_never_reported_unspent(
quote_state: MeltQuoteState, proof_state: ProofSpentState
) -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=quote_state)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=proof_state)])
),
)
with pytest.raises(TokenConsumedError, match="reconciliation required"):
await _reconcile_ambiguous_melt(wallet, "quote-3", [Mock()])
@pytest.mark.asyncio
async def test_failed_melt_reconciliation_is_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(side_effect=RuntimeError("mint unavailable")),
check_proof_state=AsyncMock(),
)
with pytest.raises(TokenConsumedError, match="outcome is unknown"):
await _reconcile_ambiguous_melt(wallet, "quote-4", [Mock()])
+65
View File
@@ -0,0 +1,65 @@
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import Unit
from routstr.mint import (
MintCooldownError,
MintRateGuard,
MintRateLimitedError,
fail_fast_mint_operations,
)
from routstr.wallet import Wallet
@pytest.mark.asyncio
async def test_cooldown_fails_fast_while_wallet_mutation_scope_is_held() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(3600, reason="rate_limited")
operation = AsyncMock(return_value="should not run")
with (
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
pytest.raises(MintCooldownError) as caught,
):
async with fail_fast_mint_operations():
await guard.run(operation)
assert caught.value.retry_after_seconds > 0
operation.assert_not_awaited()
sleep.assert_not_awaited()
@pytest.mark.asyncio
async def test_cashu_429_dispatches_through_wallet_override() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
429,
request=request,
json={"detail": "too many requests", "code": 42900},
)
wallet = object.__new__(Wallet)
wallet.url = "http://mint:3338"
wallet.db = Mock()
wallet.keysets = {"loaded": Mock()}
wallet.mint_info = Mock()
wallet.mint_info.requires_blind_auth_path.return_value = False
wallet.mint_info.requires_clear_auth_path.return_value = False
wallet.auth_db = None
wallet.auth_keyset_id = None
real_client = httpx.AsyncClient
def client_factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
return real_client(
transport=httpx.MockTransport(handler),
base_url=str(kwargs["base_url"]),
)
with (
patch("cashu.wallet.v1_api.httpx.AsyncClient", side_effect=client_factory),
pytest.raises(MintRateLimitedError),
):
await wallet.mint_quote(1, Unit.sat)
+1 -15
View File
@@ -7,21 +7,7 @@ os.environ["UPSTREAM_BASE_URL"] = "http://test"
os.environ["UPSTREAM_API_KEY"] = "test"
from routstr.core.settings import settings # noqa: E402
from routstr.payment.helpers import ( # noqa: E402
apply_mint_fee_allowance,
get_max_cost_for_model,
)
def test_mint_fee_allowance_reserves_five_percent_fallback_headroom() -> None:
# Interim policy: Routstr may pay hidden cross-mint Lightning fees when a
# trusted-mint fallback is required.
assert apply_mint_fee_allowance(124_886) == 118_642
def test_mint_fee_allowance_never_drops_below_minimum() -> None:
with patch.object(settings, "min_request_msat", 100):
assert apply_mint_fee_allowance(50) == 100
from routstr.payment.helpers import get_max_cost_for_model # noqa: E402
async def test_get_max_cost_for_model_known() -> None:
+1 -1
View File
@@ -422,4 +422,4 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
with pytest.raises(asyncio.CancelledError):
await proxy_module.proxy(request, "v1/chat/completions", session=session)
revert_mock.assert_awaited_once_with(key, session, 950, reservation_snapshot)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
+1 -1
View File
@@ -406,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
assert RAW_ORG_ID not in serialized
assert "org-[REDACTED]" in serialized
# Single upstream failed -> reservation reverted exactly once (no double-charge).
revert_mock.assert_awaited_once_with(key, session, 950, reservation)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation)
+47 -54
View File
@@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import MeltQuoteState
from routstr.core.db import ApiKey
from routstr.wallet import (
@@ -639,7 +640,9 @@ def _make_swap_mocks(
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
)
)
mock_token_wallet.melt = AsyncMock(return_value=Mock())
mock_token_wallet.melt = AsyncMock(
return_value=Mock(state=MeltQuoteState.paid)
)
return mock_token, mock_token_wallet, mock_primary_wallet
@@ -770,7 +773,7 @@ async def test_swap_retries_when_melt_demands_more_than_quoted() -> None:
"Mint Error: not enough inputs provided for melt. "
"Provided: 179, needed: 180 (Code: 11000)"
),
Mock(),
Mock(state=MeltQuoteState.paid),
]
from routstr.core.settings import settings
@@ -801,7 +804,7 @@ async def test_swap_retries_on_cdk_unbalanced_error() -> None:
)
mock_token_wallet.melt.side_effect = [
Exception("Mint Error: Transaction unbalanced: 179, 178, 2 (Code: 11005)"),
Mock(),
Mock(state=MeltQuoteState.paid),
]
from routstr.core.settings import settings
@@ -1522,22 +1525,32 @@ async def test_swap_fee_estimation_transport_error_raises_mint_connection_error(
@pytest.mark.asyncio
async def test_swap_melt_transport_error_raises_mint_connection_error() -> None:
"""A transport failure during melt is surfaced as MintConnectionError and
is NOT retried — the mint is down, not demanding higher fees."""
async def test_swap_melt_transport_error_is_never_reported_reusable() -> None:
"""A timed-out melt remains ambiguous even when an immediate snapshot says
UNPAID/UNSPENT, so callers must not receive the original token for retry."""
from routstr.wallet import swap_to_primary_mint
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
1000, fee_reserves=[10, 10]
)
mock_token_wallet.melt = AsyncMock(side_effect=httpx.ConnectTimeout("timed out"))
from cashu.core.base import MeltQuoteState, ProofSpentState
mock_token_wallet.get_melt_quote = AsyncMock(
return_value=Mock(state=MeltQuoteState.unpaid)
)
mock_token_wallet.check_proof_state = AsyncMock(
return_value=Mock(
states=[Mock(state=ProofSpentState.unspent) for _ in mock_token.proofs]
)
)
from routstr.core.settings import settings
with patch.object(settings, "primary_mint", "http://primary:3338"):
with patch.object(settings, "primary_mint_unit", "sat"):
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
with pytest.raises(MintConnectionError):
with pytest.raises(TokenConsumedError, match="ambiguous"):
await swap_to_primary_mint(mock_token, mock_token_wallet)
assert mock_token_wallet.melt.call_count == 1
@@ -1595,8 +1608,8 @@ async def test_mint_rate_guard_waits_for_adaptive_cooldown() -> None:
guard._cooldown_until = 15.0
operation = AsyncMock(return_value="ok")
with patch("routstr.wallet.time.monotonic", return_value=10.0):
with patch("routstr.wallet.asyncio.sleep", AsyncMock()) as sleep:
with patch("routstr.mint.time.monotonic", return_value=10.0):
with patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep:
assert await guard.run(operation) == "ok"
sleep.assert_awaited_once_with(5.0)
@@ -1611,7 +1624,7 @@ async def test_mint_rate_guard_exponentially_backs_off_repeated_429s() -> None:
expected_delays = [60, 120, 240, 480, 960, 1920, 3840, 7680, 15360, 25200]
now = 0.0
with patch("routstr.wallet.time.monotonic") as monotonic:
with patch("routstr.mint.time.monotonic") as monotonic:
for index, expected in enumerate(expected_delays, start=1):
monotonic.return_value = now
assert guard.apply_rate_limit_cooldown(60) == expected
@@ -1682,8 +1695,8 @@ async def test_mint_rate_guard_keeps_cooldown_when_concurrency_is_unlimited() ->
operation = AsyncMock(return_value="ok")
with (
patch.object(settings, "mint_max_concurrency", 0),
patch("routstr.wallet.time.monotonic", return_value=0),
patch("routstr.wallet.asyncio.sleep", AsyncMock()) as sleep,
patch("routstr.mint.time.monotonic", return_value=0),
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
):
guard = _MintRateGuard.get("http://mint:3338")
guard.apply_cooldown(5)
@@ -1715,8 +1728,8 @@ async def test_mint_operation_honors_retry_after_as_minimum() -> None:
with patch.object(settings, "mint_retry_max_attempts", 1):
with patch.object(settings, "mint_operation_timeout_seconds", 0):
with patch.object(settings, "mint_max_concurrency", 1):
with patch("routstr.wallet.time.monotonic", return_value=0.1):
with patch("routstr.wallet.asyncio.sleep", sleep):
with patch("routstr.mint.time.monotonic", return_value=0.1):
with patch("routstr.mint.asyncio.sleep", sleep):
result = await _mint_operation(
factory, mint_url="http://mint:3338"
)
@@ -1734,7 +1747,7 @@ async def test_mint_operation_timeout_excludes_adaptive_cooldown() -> None:
with (
patch.object(settings, "mint_max_concurrency", 1),
patch.object(settings, "mint_operation_timeout_seconds", 0.01),
patch("routstr.wallet.asyncio.sleep", AsyncMock()) as sleep,
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
):
guard = _MintRateGuard.get("http://mint:3338")
guard.apply_cooldown(60)
@@ -1763,7 +1776,7 @@ async def test_default_timeout_allows_retry_after_rate_limit_cooldown() -> None:
patch.object(settings, "mint_retry_max_attempts", 3),
patch.object(settings, "mint_operation_timeout_seconds", 30),
patch.object(settings, "mint_max_concurrency", 1),
patch("routstr.wallet.asyncio.sleep", AsyncMock()),
patch("routstr.mint.asyncio.sleep", AsyncMock()),
):
assert await _mint_operation(operation, mint_url="http://mint:3338") == "ok"
@@ -1780,7 +1793,7 @@ async def test_mint_operation_retries_httpx_timeout_only_when_safe() -> None:
with patch.object(settings, "mint_retry_max_attempts", 2):
with patch.object(settings, "mint_operation_timeout_seconds", 0):
with patch("routstr.wallet.asyncio.sleep", AsyncMock()):
with patch("routstr.mint.asyncio.sleep", AsyncMock()):
assert await _mint_operation(retrying) == "ok"
with pytest.raises(httpx.TimeoutException):
await _mint_operation(non_retrying, retry_timeouts=False)
@@ -1802,7 +1815,7 @@ async def test_get_wallet_initializes_and_loads_once_concurrently() -> None:
) as create:
# A fresh wallet must load even when the host has been up for less than
# the reload interval.
with patch("routstr.wallet.time.monotonic", return_value=10.0):
with patch("routstr.mint.time.monotonic", return_value=10.0):
first, second = await asyncio.gather(
get_wallet("http://mint:3338"), get_wallet("http://mint:3338")
)
@@ -1833,7 +1846,7 @@ async def test_get_wallet_can_surface_429_without_retrying() -> None:
patch("routstr.wallet.Wallet.with_db", AsyncMock(return_value=wallet)),
patch.object(settings, "mint_retry_max_attempts", 3),
patch.object(settings, "mint_operation_timeout_seconds", 0),
patch("routstr.wallet.asyncio.sleep", AsyncMock()) as sleep,
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
):
with pytest.raises(httpx.HTTPStatusError):
await get_wallet("http://mint:3338", retry_on_rate_limit=False)
@@ -1965,7 +1978,7 @@ async def test_swap_falls_back_when_primary_wallet_cannot_load() -> None:
melt_quote=AsyncMock(
return_value=Mock(quote="melt_q", amount=990, fee_reserve=10)
),
melt=AsyncMock(return_value=Mock()),
melt=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
)
mint_quote = Mock(quote="mint_q_secondary", request="lnbc1secondary")
@@ -1994,6 +2007,7 @@ async def test_swap_falls_back_when_primary_wallet_cannot_load() -> None:
patch("asyncio.sleep", AsyncMock()),
patch("routstr.wallet.get_wallet", side_effect=mock_get),
patch("routstr.wallet.logger.warning") as warning,
patch("routstr.wallet.logger.info") as info,
):
amount, unit, mint_url = await swap_to_primary_mint(token, source_wallet)
@@ -2003,7 +2017,7 @@ async def test_swap_falls_back_when_primary_wallet_cannot_load() -> None:
assert any(call.args[0] == secondary for call in mock_get.await_args_list)
events = {
call.kwargs["extra"]["event"]
for call in warning.call_args_list
for call in [*warning.call_args_list, *info.call_args_list]
if "extra" in call.kwargs and "event" in call.kwargs["extra"]
}
assert "cashu_destination_failed" in events
@@ -2181,8 +2195,8 @@ async def test_wallet_fallback_skips_mint_during_cooldown() -> None:
patch.object(settings, "cashu_mints", [primary, secondary]),
patch.object(settings, "mint_max_concurrency", 0),
patch.object(settings, "mint_operation_timeout_seconds", 0),
patch("routstr.wallet.time.monotonic", return_value=10),
patch("routstr.wallet.asyncio.sleep", AsyncMock()) as sleep,
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
patch(
"routstr.wallet.get_wallet",
AsyncMock(side_effect=lambda mint, *args, **kwargs: wallets[mint]),
@@ -2378,49 +2392,28 @@ def test_classify_500_with_rate_limit_text_is_not_mint_rate_limited() -> None:
@pytest.mark.asyncio
async def test_probe_does_not_escalate_consecutive_rate_limits() -> None:
"""When a probe fails with a rate limit, _consecutive_rate_limits should
NOT increment — the probe is a recovery check, not a new request."""
from routstr.wallet import _MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, _MintRateGuard
from routstr.mint import MintRateGuard
guard = _MintRateGuard("http://mint", max_concurrency=0)
# Simulate initial rate limit: apply_rate_limit_cooldown increments counter
guard = MintRateGuard("http://mint", max_concurrency=0)
guard.apply_rate_limit_cooldown()
assert guard._consecutive_rate_limits == 1
cooldown_before = guard._cooldown_until
assert cooldown_before > 0
guard._cooldown_until = 0.0
# Simulate probe failure: _run_probe uses apply_cooldown, NOT
# apply_rate_limit_cooldown, so the counter stays at 1.
guard.apply_cooldown(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, reason="rate_limited")
assert guard._consecutive_rate_limits == 1 # unchanged!
with pytest.raises(httpx.HTTPStatusError):
await guard.run(AsyncMock(side_effect=_http_429_error()))
assert guard._consecutive_rate_limits == 1
assert guard._needs_probe is True
@pytest.mark.asyncio
async def test_probe_recovery_resets_consecutive_rate_limits() -> None:
"""A successful probe resets _consecutive_rate_limits to 0."""
from routstr.wallet import _MintRateGuard
from routstr.mint import MintRateGuard
guard = _MintRateGuard("http://mint", max_concurrency=0)
# First rate limit: increments to 1, sets 60s cooldown.
guard = MintRateGuard("http://mint", max_concurrency=0)
guard.apply_rate_limit_cooldown()
assert guard._consecutive_rate_limits == 1
# Manually expire the cooldown so the next call creates a fresh one.
guard._cooldown_until = 0.0
guard._cooldown_reason = None
# Second rate limit (after cooldown expired): increments to 2.
guard.apply_rate_limit_cooldown()
assert guard._consecutive_rate_limits == 2
# Simulate a successful probe by resetting (as _run_probe does)
guard._needs_probe = False
guard._cooldown_until = 0.0
guard._cooldown_reason = None
guard._consecutive_rate_limits = 0
assert await guard.run(AsyncMock(return_value="ok")) == "ok"
assert guard._consecutive_rate_limits == 0
assert guard._needs_probe is False