Files
routstr-core/routstr/wallet.py
T

2313 lines
88 KiB
Python

import asyncio
import fcntl
import os
import re
import socket
import time
import typing
from contextlib import asynccontextmanager
from contextvars import ContextVar
from pathlib import Path
from typing import Any, AsyncGenerator, Awaitable, Callable, TypedDict
import httpx
from cashu.core.base import MintQuote, Proof, Token
from cashu.core.mint_info import MintInfo as _CashuMintInfo
from cashu.wallet.helpers import deserialize_token_from_string
from cashu.wallet.wallet import Wallet as _CashuWallet
from pydantic_core import PydanticUndefined
from sqlmodel import col, select, update
from .core import db, get_logger
from .core.db import store_cashu_transaction_with_retry as store_cashu_transaction
from .core.settings import settings
from .payment.lnurl import raw_send_to_lnurl
# cashu still declares Optional[X] without explicit defaults on MintInfo.
# Under pydantic v2 those are required, but real mints omit many of them.
# Default Optional fields to None at import time so balance fetches don't 422.
for _name, _field in _CashuMintInfo.model_fields.items():
_annot = _field.annotation
_is_optional = typing.get_origin(_annot) is typing.Union and type(
None
) in typing.get_args(_annot)
if _is_optional and _field.default is PydanticUndefined:
_field.default = None
_CashuMintInfo.model_rebuild(force=True)
logger = get_logger(__name__)
# Preserve the real scheduler yield even when payout tests patch asyncio.sleep.
_scheduler_sleep = asyncio.sleep
_WALLET_OPERATION_LOCK = Path(".wallet") / ".routstr-operation.lock"
_wallet_operation_depth: ContextVar[int] = ContextVar(
"wallet_operation_depth", default=0
)
@asynccontextmanager
async def wallet_operation_guard() -> AsyncGenerator[None, None]:
"""Serialize proof mutation and owner payout across local worker processes."""
depth = _wallet_operation_depth.get()
if depth:
token = _wallet_operation_depth.set(depth + 1)
try:
yield
finally:
_wallet_operation_depth.reset(token)
return
_WALLET_OPERATION_LOCK.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(_WALLET_OPERATION_LOCK, os.O_CREAT | os.O_RDWR, 0o600)
acquired = False
depth_token = None
try:
while not acquired:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
acquired = True
except BlockingIOError:
await _scheduler_sleep(0.05)
depth_token = _wallet_operation_depth.set(1)
yield
finally:
if depth_token is not None:
_wallet_operation_depth.reset(depth_token)
if acquired:
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
def _sats_to_msats(amount: int) -> int:
return amount * 1000
def _msats_to_sats(amount: int) -> int:
return amount // 1000
def _mints_to_inspect() -> list[str]:
"""Return configured mints plus the primary mint, without duplicates."""
mint_urls = list(settings.cashu_mints)
if settings.primary_mint and settings.primary_mint not in mint_urls:
mint_urls.append(settings.primary_mint)
return mint_urls
class MintConnectionError(Exception):
"""The mint could not be reached (network transport failure).
Maps to a 503, not a 4xx: the token is fine, the mint is just unavailable.
"""
class SourceMintConnectionError(MintConnectionError):
"""The mint that issued the incoming proofs cannot be reached."""
class TokenConsumedError(Exception):
"""A failure that happened AFTER the token's proofs were spent (melt
succeeded, or redemption already returned) — e.g. minting on the primary
mint or the DB credit then failed.
Non-retryable: the same token will not work again. Seals the cause chain so
a transport error underneath is never re-surfaced as a retryable
mint_unreachable.
"""
class MintRateLimitedError(httpx.HTTPStatusError):
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
class Wallet(_CashuWallet):
"""Cashu wallet adapter that preserves rate-limit status information.
Cashu's default response adapter converts JSON error bodies into plain
``Exception`` instances before calling ``raise_for_status``. Intercept 429
here so Routstr's fallback and cooldown policy can use the real status
without unreliable message matching.
"""
@staticmethod
def raise_on_error_request(resp: httpx.Response) -> None:
if resp.status_code == 429:
raise MintRateLimitedError(
"Cashu mint rate limited",
request=resp.request,
response=resp,
)
_CashuWallet.raise_on_error_request(resp)
# httpx base classes cover their subclasses. HTTPStatusError is excluded on
# purpose — that means the mint answered, just with an error status.
_MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
_TRANSPORT_EXC_TYPES: tuple[type[BaseException], ...] = (
httpx.NetworkError,
httpx.TimeoutException,
ConnectionError, # refused/reset/aborted
socket.gaierror, # DNS failure
asyncio.TimeoutError,
)
class _MintRateGuard:
"""Limit concurrency and remember per-mint rate-limit cooldowns."""
_guards: dict[str, "_MintRateGuard"] = {}
@classmethod
def get(cls, mint_url: str) -> "_MintRateGuard":
concurrency = settings.mint_max_concurrency
guard = cls._guards.get(mint_url)
if guard is None or guard._max_concurrency != concurrency:
guard = cls(mint_url, concurrency)
cls._guards[mint_url] = guard
return guard
def __init__(self, mint_url: str, max_concurrency: int):
self._mint_url = mint_url
self._max_concurrency = max_concurrency
self._semaphore = (
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
)
self._cooldown_until = 0.0
self._cooldown_reason: str | None = None
self._consecutive_rate_limits = 0
self._needs_probe = False
self._probe_lock = asyncio.Lock()
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
deadline = time.monotonic() + max(0.0, delay)
if deadline >= self._cooldown_until:
self._cooldown_until = deadline
if reason is not None:
self._cooldown_reason = reason
elif self._cooldown_reason is None and reason is not None:
self._cooldown_reason = reason
self._needs_probe = True
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
remaining = self.cooldown_remaining()
if remaining > 0 and self._cooldown_reason == "rate_limited":
minimum = min(
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
)
if minimum > remaining:
self.apply_cooldown(minimum, reason="rate_limited")
return minimum
return remaining
self._consecutive_rate_limits += 1
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
self.apply_cooldown(delay, reason="rate_limited")
return delay
def cooldown_remaining(self) -> float:
return max(0.0, self._cooldown_until - time.monotonic())
def cooldown_reason(self) -> str | None:
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
async def _wait_for_cooldown(self) -> None:
while True:
deadline = self._cooldown_until
wait = max(0.0, deadline - time.monotonic())
if wait <= 0:
return
logger.debug(
"Mint rate guard: cooling down",
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
)
await asyncio.sleep(wait)
if self._cooldown_until <= deadline:
return
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
await self._wait_for_cooldown()
logger.warning(
"Mint cooldown ended; sending one probe request",
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
)
try:
result = await factory()
except Exception as error:
# Keep queued callers behind the probe. On a rate-limit,
# re-apply the *same* cooldown the caller already set rather
# than calling apply_rate_limit_cooldown() — the probe is a
# recovery check, not a new request that should escalate the
# exponential backoff counter.
if _is_mint_rate_limited(error):
retry_after = None
if isinstance(error, httpx.HTTPStatusError):
retry_after = _parse_retry_after(error.response.headers)
delay = max(
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS,
retry_after or 0.0,
)
self.apply_cooldown(delay, reason="rate_limited")
else:
self.apply_cooldown(1.0)
logger.warning(
"Mint cooldown probe failed",
extra={
"event": "mint_cooldown_probe_failed",
"mint_url": self._mint_url,
"error": str(error),
"error_type": type(error).__name__,
"cooldown_seconds": round(self.cooldown_remaining(), 2),
"consecutive_rate_limits": self._consecutive_rate_limits,
},
)
raise
self._needs_probe = False
self._cooldown_until = 0.0
self._cooldown_reason = None
self._consecutive_rate_limits = 0
logger.warning(
"Mint cooldown probe succeeded; restoring normal concurrency",
extra={
"event": "mint_cooldown_probe_succeeded",
"mint_url": self._mint_url,
},
)
return result
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
while True:
if self._needs_probe or self.cooldown_remaining() > 0:
async with self._probe_lock:
if self.cooldown_remaining() > 0:
self._needs_probe = True
if self._needs_probe:
return await self._run_probe(factory)
continue
if self._semaphore is None:
return await factory()
async with self._semaphore:
if self._needs_probe:
continue
return await factory()
def _mint_cooldown_remaining(mint_url: str) -> float:
return _MintRateGuard.get(mint_url).cooldown_remaining()
def _mint_cooldown_reason(mint_url: str) -> str | None:
return _MintRateGuard.get(mint_url).cooldown_reason()
def _is_mint_rate_limited(error: BaseException) -> bool:
"""True if the mint returned an HTTP 429 (Too Many Requests).
Only matches ``httpx.HTTPStatusError`` with status code 429 — never
classifies based on the exception's message text. Substring matching
on ``"rate limit"`` / ``"too many requests"`` was removed because it
catches unrelated errors (e.g. a 503 whose body happens to mention
"database rate exceeded"), which triggers unnecessary exponential
backoff and can block state recovery indefinitely.
"""
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, httpx.HTTPStatusError):
if current.response.status_code == 429:
return True
current = current.__cause__ or current.__context__
return False
async def _mint_operation(
factory: Callable[[], Awaitable[Any]],
*,
op_name: str = "mint_operation",
mint_url: str = "",
retry_timeouts: bool = True,
retry_on_rate_limit: bool = True,
) -> Any:
"""Run a mint operation with bounded concurrency and adaptive cooldown.
The timeout applies to each network attempt. Queueing, cooldown, and retry
backoff are deliberately outside it so the shipped 60-second 429 cooldown
is not cancelled by the 30-second operation timeout. ``factory`` must
return a fresh coroutine for every retry.
When ``retry_on_rate_limit`` is False a 429 is not retried in-place — the
cooldown is still applied to the per-mint guard (so subsequent operations on
that mint wait), but the exception is re-raised so the caller (typically
``_request_mint_with_fallback``) can immediately try a different mint.
"""
guard = _MintRateGuard.get(mint_url) if mint_url else None
timeout = settings.mint_operation_timeout_seconds
max_attempts = settings.mint_retry_max_attempts + 1
async def timed_factory() -> Any:
if timeout > 0:
return await asyncio.wait_for(factory(), timeout=timeout)
return await factory()
async def invoke() -> Any:
if guard is not None:
return await guard.run(timed_factory)
return await timed_factory()
async def run_with_retries() -> Any:
for attempt in range(max_attempts):
try:
return await invoke()
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
if retry_timeouts and attempt < max_attempts - 1:
backoff = (2**attempt) + (time.monotonic() % 1.0)
logger.warning(
"Mint operation timed out, retrying",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"backoff_seconds": round(backoff, 2),
},
)
await asyncio.sleep(backoff)
continue
raise httpx.TimeoutException(
f"{op_name} timed out (attempts: {attempt + 1})"
) from exc
except Exception as exc:
if not _is_mint_rate_limited(exc):
raise
# Apply cooldown to the guard regardless — even when we're
# about to re-raise for fallback, the guard must remember that
# this mint is rate-limited for future operations.
backoff = (2**attempt) + (time.monotonic() % 1.0)
if isinstance(exc, httpx.HTTPStatusError):
retry_after = _parse_retry_after(exc.response.headers)
if retry_after is not None:
backoff = max(retry_after, backoff)
cooldown = backoff
if guard is not None:
cooldown = guard.apply_rate_limit_cooldown(backoff)
# When the caller has a fallback strategy (trusted-mint
# list), re-raise immediately so the caller can try the next
# mint instead of waiting through this mint's cooldown.
if not retry_on_rate_limit:
logger.warning(
"Mint rate-limited, skipping retries for fallback",
extra={
"op_name": op_name,
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
raise
if attempt >= max_attempts - 1:
raise
logger.warning(
"Mint rate-limited, applying cooldown",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
if guard is None:
await asyncio.sleep(cooldown)
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
return await run_with_retries()
def _parse_retry_after(headers: Any) -> float | None:
"""Parse a Retry-After header (delta-seconds form) into seconds."""
raw = headers.get("retry-after") or headers.get("Retry-After")
if raw is None:
return None
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return None
def is_source_mint_connection_error(error: BaseException) -> bool:
seen: set[int] = set()
current: BaseException | None = error
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, SourceMintConnectionError):
return True
current = current.__cause__ or current.__context__
return False
def is_mint_connection_error(error: BaseException) -> bool:
"""True if ``error`` (or anything in its cause/context chain) is a mint
transport failure. Walks the chain because some sites re-raise transport
errors wrapped in ValueError/MintConnectionError; matches on TYPE, not text.
"""
seen: set[int] = set()
current: BaseException | None = error
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, TokenConsumedError):
# Sealed: the token was already spent, so whatever transport error
# sits underneath must not make this look retryable.
return False
if isinstance(current, MintConnectionError):
return True
if isinstance(current, _TRANSPORT_EXC_TYPES):
return True
current = current.__cause__ or current.__context__
return False
# Redemption ``code`` values whose token is spent/consumed/unusable — the
# X-Cashu path must NOT echo the original token for these (echoing invites a
# retry with a token that can never succeed again).
SPENT_TOKEN_CODES: frozenset[str] = frozenset(
{
"cashu_token_already_spent",
"cashu_token_consumed",
"cashu_token_zero_value",
"internal_error",
}
)
def classify_redemption_error(
error: Exception,
) -> tuple[str, int, str, str] | None:
"""Map a token-redemption failure to ``(type, status, message, code)``.
Single source of truth for every endpoint that redeems a token (bearer,
X-Cashu, top-up) so the same failure yields the same taxonomy everywhere.
``type`` and ``code`` are stable client contract; ``message`` is sanitized
(raw error text stays in logs). Returns None for an unclassified internal
fault — the caller emits a generic 500.
"""
if isinstance(error, TokenConsumedError):
return (
"token_consumed",
500,
"Token was redeemed but could not be credited; do not retry",
"cashu_token_consumed",
)
if is_source_mint_connection_error(error):
return (
"mint_unreachable",
503,
"The mint that issued this Cashu token is unreachable; the token cannot be redeemed at another mint",
"cashu_source_mint_unreachable",
)
if _is_mint_rate_limited(error):
return (
"mint_rate_limited",
503,
"Cashu mint rate-limited; retry after cooldown",
"cashu_mint_rate_limited",
)
if is_mint_connection_error(error):
return (
"mint_unreachable",
503,
"Cashu mint is unreachable",
"cashu_mint_unreachable",
)
lowered = str(error).lower()
if "already spent" in lowered:
return (
"token_already_spent",
400,
"Cashu token already spent",
"cashu_token_already_spent",
)
if (
"insufficient" in lowered
or "melt fee" in lowered
or "exceed token amount" in lowered
or "estimate fees" in lowered
):
return (
"mint_error",
422,
"Token value is too small to cover swap fees",
"cashu_token_swap_fees_exceed_amount",
)
if "failed to melt" in lowered:
return (
"mint_error",
422,
"Failed to swap token from foreign mint",
"cashu_foreign_mint_swap_failed",
)
if ("invalid" in lowered or "decode" in lowered) and "token" in lowered:
# Anchored to "token" so internal faults whose text merely contains
# "invalid"/"decode" fall through to the 500 branch, not a token error.
return (
"invalid_token",
400,
"Invalid Cashu token",
"invalid_cashu_token",
)
if "must be positive" in lowered or "yielded no value" in lowered:
# Redeemed to <= 0 (empty/dust token, or value fully consumed by fees).
# Consumed, so non-retryable, but its own code — not the generic bucket.
return (
"cashu_error",
400,
"Failed to redeem Cashu token: token yielded no value",
"cashu_token_zero_value",
)
if isinstance(error, ValueError):
return (
"cashu_error",
400,
"Failed to redeem Cashu token",
"cashu_token_redemption_failed",
)
return None
async def get_balance(unit: str) -> int:
wallet = await get_wallet(settings.primary_mint, unit)
return wallet.available_balance.amount
async def _redeem_same_mint(
wallet: Wallet, token_obj: Token
) -> tuple[int, str, str]: # amount, unit, mint_url
"""Redeem proofs at their own issuing mint (no cross-mint swap).
split() re-mints the incoming proofs into fresh ones we own so the sender
can't double-spend them. With include_fees=True the mint deducts its NUT-02
per-proof input fee, so we end up holding only `amount - input_fees`. Credit
that, not the face value, or routstr over-credits the user and its wallet
drifts insolvent.
"""
try:
await _mint_operation(
lambda: wallet.load_mint(keyset_id=token_obj.keysets[0]),
op_name="redeem_load_mint",
mint_url=token_obj.mint,
)
wallet.verify_proofs_dleq(token_obj.proofs)
input_fees = wallet.get_fees_for_proofs(token_obj.proofs)
await _mint_operation(
lambda: wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True),
op_name="redeem_split",
mint_url=token_obj.mint,
retry_timeouts=False,
)
except Exception as error:
if is_mint_connection_error(error):
logger.warning(
"Same-mint redemption failed; client must use a different token",
extra={
"event": "cashu_same_mint_redemption_failed",
"source_mint": token_obj.mint,
"source_unit": token_obj.unit,
"source_amount": token_obj.amount,
"cross_mint_fallback_attempted": False,
"action": "retry_with_token_from_another_mint",
"error": str(error),
"error_type": type(error).__name__,
},
)
raise SourceMintConnectionError(
"Issuing Cashu mint is unreachable"
) from error
raise
return int(token_obj.amount) - input_fees, token_obj.unit, token_obj.mint
async def recieve_token(
token: str,
) -> tuple[int, str, str]: # amount, unit, mint_url
token_obj = deserialize_token_from_string(token)
if len(token_obj.keysets) > 1:
raise ValueError("Multiple keysets per token currently not supported")
wallet = await get_wallet(token_obj.mint, token_obj.unit, load=False)
wallet.keyset_id = token_obj.keysets[0]
if token_obj.mint not in settings.cashu_mints:
destinations = list(
dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
)
logger.warning(
"Cashu cross-mint swap required",
extra={
"event": "cashu_swap_started",
"source_mint": token_obj.mint,
"source_unit": token_obj.unit,
"source_amount": token_obj.amount,
"destination_candidates": destinations,
},
)
return await swap_to_trusted_mint(token_obj, wallet)
logger.warning(
"Trying same-mint Cashu redemption",
extra={
"event": "cashu_same_mint_redemption",
"source_mint": token_obj.mint,
"source_unit": token_obj.unit,
"source_amount": token_obj.amount,
"cross_mint_fallback_on_connection_failure": False,
},
)
return await _redeem_same_mint(wallet, token_obj)
async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int, str]:
"""Create a token from the preferred mint or another funded trusted mint."""
effective_mint_url = await find_trusted_mint_with_funds(amount, unit, mint_url)
wallet = await get_wallet(effective_mint_url, unit)
proofs = get_proofs_per_mint_and_unit(
wallet, effective_mint_url, unit, not_reserved=True
)
proofs_for_mint = sum(proof.amount for proof in proofs)
all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved)
all_mint_urls = list({k.mint_url for k in wallet.keysets.values()})
proof_summary = {
f"{k.mint_url}/{k.unit.name}": sum(
p.amount for p in wallet.proofs if p.id == k.id
)
for k in wallet.keysets.values()
}
# Show ALL proofs in DB by keyset_id, regardless of whether the loaded wallet
# knows about that keyset. This reveals proofs orphaned under stale keysets.
raw_proofs_by_keyset: dict[str, int] = {}
for p in wallet.proofs:
raw_proofs_by_keyset[p.id] = raw_proofs_by_keyset.get(p.id, 0) + p.amount
logger.info(
f"send: proof inventory | mint={effective_mint_url} unit={unit} amount={amount} "
f"primary_mint={settings.primary_mint} liquid_proofs_for_mint={proofs_for_mint} "
f"reserved_proofs_for_mint={reserved_for_mint} "
f"all_mints={all_mint_urls} by_keyset={proof_summary} "
f"raw_proofs_by_keyset_id={raw_proofs_by_keyset} "
f"total_wallet_proofs={sum(p.amount for p in wallet.proofs)}"
)
# Reserve proofs only after serialization succeeds — if serialize_proofs or
# swap_to_send fails mid-way, proofs stay unreserved so dashboard balance
# doesn't go negative.
send_proofs, _ = await wallet.select_to_send(
proofs, amount, set_reserved=False, include_fees=False
)
try:
token = await wallet.serialize_proofs(
send_proofs, include_dleq=False, legacy=False, memo=None
)
except Exception:
await wallet.set_reserved_for_send(send_proofs, reserved=False)
raise
await wallet.set_reserved_for_send(send_proofs, reserved=True)
return amount, token
async def send_token(amount: int, unit: str, mint_url: str | None = None) -> str:
_, token = await send(amount, unit, mint_url)
return token
async def release_token_reservation(token: str) -> None:
"""Release a token that was created locally but never handed off."""
token_obj = deserialize_token_from_string(token)
wallet = await get_wallet(token_obj.mint, token_obj.unit, load=False)
await wallet.set_reserved_for_send(token_obj.proofs, reserved=False)
secrets = {proof.secret for proof in token_obj.proofs}
for proof in token_obj.proofs:
proof.reserved = False
for proof in wallet.proofs:
if proof.secret in secrets:
proof.reserved = False
def token_mint_url(token: str, fallback: str | None = None) -> str:
try:
return str(deserialize_token_from_string(token).mint)
except Exception:
if fallback is None:
raise
return fallback
async def find_trusted_mint_with_funds(
amount: int, unit: str, preferred_mint: str | None = None
) -> str:
"""Choose a trusted mint that can cover a refund without waiting on cooldown."""
trusted = list(dict.fromkeys([settings.primary_mint, *settings.cashu_mints]))
candidates: list[str] = []
if preferred_mint in trusted:
candidates.append(preferred_mint)
candidates.extend(mint for mint in trusted if mint not in candidates)
balances: dict[str, int] = {}
for mint_url in candidates:
if _mint_cooldown_remaining(mint_url) > 0:
continue
try:
wallet = await get_wallet(mint_url, unit, retry_on_rate_limit=False)
except Exception as error:
if is_mint_connection_error(error) or _is_mint_rate_limited(error):
balances[mint_url] = 0
continue
raise
proofs = get_proofs_per_mint_and_unit(wallet, mint_url, unit, not_reserved=True)
balances[mint_url] = sum(proof.amount for proof in proofs)
if balances[mint_url] >= amount:
return mint_url
raise ValueError(
f"No trusted mint has {amount} {unit} available; balances={balances}"
)
# A foreign mint's fee_reserve is a non-binding estimate (NUT-05): the mint may
# demand more when re-quoting or at melt execution. Instead of padding the
# estimate with a safety buffer (which strands the margin at the foreign mint
# on every swap), the swap retries with the amount recomputed from the fees the
# mint actually demands, up to this many attempts.
_MAX_SWAP_ATTEMPTS = 3
_MINT_ERROR_CODE_RE = re.compile(r"\(Code: (\d+)\)")
_MELT_SHORTFALL_RE = re.compile(r"Provided: (\d+), needed: (\d+)")
# Insufficient-melt-inputs failures differ across mint implementations. 11005 is
# the registered "Transaction is not balanced" code (cdk), specific enough to
# trust on the code alone. 11000 is nutshell's generic, unregistered
# TransactionError covering many unrelated failures, so it only counts as a fee
# shortfall alongside the "not enough inputs" detail text. With no code suffix at
# all, that same text is the only signal.
def _net_minted_amount(amount_msat: int, token_unit: str, fees: int) -> int:
"""
Convert the token value minus fees (given in the token unit) into an
amount in the primary mint's unit.
"""
fee_msat = _sats_to_msats(fees) if token_unit == "sat" else fees
remaining_msat = amount_msat - fee_msat
if settings.primary_mint_unit == "sat":
return _msats_to_sats(remaining_msat)
return int(remaining_msat)
def _melt_insufficient_shortfall(error: Exception) -> int | None:
"""
Classify a melt failure: return the observed shortfall (in the token unit)
when the mint rejected the inputs as insufficient, or None when the failure
is unrelated to fees and must not be retried (e.g. a Lightning payment
failure, where a smaller invoice would not help).
Cashu errors carry no structured amounts (NUT-00 defines only detail/code,
flattened to "Mint Error: <detail> (Code: <code>)" by cashu-py), so the
classification uses the code and the shortfall must be inferred: the
"Provided: X, needed: Y" amounts are nutshell-specific free text and only
refine the shortfall when present; otherwise shrink one unit at a time.
"""
message = str(error)
code_match = _MINT_ERROR_CODE_RE.search(message)
code = code_match.group(1) if code_match is not None else None
has_shortfall_text = "not enough inputs" in message.lower()
match code:
case "11005": # registered TransactionUnbalanced: trust the code
pass
case "11000" if has_shortfall_text: # generic nutshell error: needs the text
pass
case None if has_shortfall_text: # no code suffix: text is the only signal
pass
case _: # other codes, a bare 11000, or no signal: must not retry
return None
amounts = _MELT_SHORTFALL_RE.search(message)
if amounts is not None:
provided, needed = int(amounts.group(1)), int(amounts.group(2))
if needed > provided:
return needed - provided
return 1
async def _request_mint_with_fallback(
amount: int, *, op_name: str, primary_wallet: Wallet | None = None
) -> tuple[Wallet, str, MintQuote]:
"""Try request_mint on the primary mint, fall back to other trusted mints
on transport or rate-limit failure. Returns the wallet, mint_url, and quote.
Guards against amount <= 0: the cashu library's PostMintQuoteRequest
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
cryptic validation error deep in the stack. Fail fast with context.
"""
if amount <= 0:
raise ValueError(
f"_request_mint_with_fallback({op_name}): amount must be > 0, got {amount}. "
f"Token value is too small after fee deduction or unit conversion."
)
candidates = list(dict.fromkeys([settings.primary_mint, *settings.cashu_mints]))
logger.warning(
"Trying trusted destination mints",
extra={
"event": "cashu_destination_candidates",
"op_name": op_name,
"amount": amount,
"unit": settings.primary_mint_unit,
"candidates": candidates,
},
)
tried: list[str] = []
for candidate_index, mint_url in enumerate(candidates, start=1):
cooldown = _mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.warning(
"Skipping unavailable destination mint",
extra={
"event": "cashu_destination_skipped",
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"op_name": op_name,
"candidate_index": candidate_index,
"candidate_count": len(candidates),
},
)
continue
logger.warning(
"Trying destination mint",
extra={
"event": "cashu_destination_attempt",
"mint_url": mint_url,
"op_name": op_name,
"candidate_index": candidate_index,
"candidate_count": len(candidates),
},
)
try:
if mint_url == settings.primary_mint and primary_wallet is not None:
wallet = primary_wallet
else:
wallet = await get_wallet(
mint_url,
settings.primary_mint_unit,
retry_on_rate_limit=False,
)
quote = await _mint_operation(
lambda: wallet.request_mint(amount),
op_name=op_name,
mint_url=mint_url,
retry_on_rate_limit=False,
)
logger.warning(
"Destination mint selected",
extra={
"event": "cashu_destination_selected",
"mint_url": mint_url,
"op_name": op_name,
"candidate_index": candidate_index,
"fallback_used": candidate_index > 1,
},
)
return wallet, mint_url, quote
except Exception as error:
tried.append(f"{mint_url}: {type(error).__name__}")
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
if not connection_failure and not rate_limited:
raise
if connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
_MINT_TRANSPORT_COOLDOWN_SECONDS, reason="unreachable"
)
logger.warning(
"Destination mint failed",
extra={
"event": "cashu_destination_failed",
"failed_mint": mint_url,
"error": str(error),
"error_type": type(error).__name__,
"connection_failure": connection_failure,
"rate_limited": rate_limited,
"tried": tried,
"op_name": op_name,
"candidate_index": candidate_index,
"candidate_count": len(candidates),
},
)
continue
logger.error(
"All trusted destination mints failed",
extra={
"event": "cashu_destination_exhausted",
"op_name": op_name,
"amount": amount,
"unit": settings.primary_mint_unit,
"candidates": candidates,
"tried": tried,
},
)
raise MintConnectionError(f"All mints failed for {op_name}: {tried}")
async def _calculate_swap_amount(
amount_msat: int,
token_unit: str,
token_mint_url: str,
token_wallet: Wallet,
primary_wallet: Wallet | None,
proofs: list,
) -> int:
"""
Calculate the amount to mint on the primary mint after accounting for
melt fees and NUT-02 input fees on the foreign mint.
"""
if settings.primary_mint_unit == "sat":
receive_amount = _msats_to_sats(amount_msat)
else:
receive_amount = amount_msat
if token_mint_url == settings.primary_mint:
logger.info(
"swap_to_primary_mint: skipping fee estimation (same mint)",
extra={"minted_amount": receive_amount},
)
return int(receive_amount)
# The cashu library's PostMintQuoteRequest enforces amount > 0 (Pydantic
# Field(gt=0)). When the token's face value in the primary mint's unit
# truncates to 0 (e.g. < 1000 msat with a "sat" primary unit), calling
# request_mint(0) raises a validation error that is cryptic in production
# logs. Guard early with full diagnostic context instead.
if receive_amount <= 0:
logger.error(
"swap_to_primary_mint: receive_amount is zero or negative, cannot estimate fees",
extra={
"amount_msat": amount_msat,
"token_unit": token_unit,
"token_mint_url": token_mint_url,
"primary_mint": settings.primary_mint,
"primary_mint_unit": settings.primary_mint_unit,
"receive_amount": receive_amount,
},
)
raise ValueError(
f"Token amount ({amount_msat} msat, unit={token_unit}) is too small to "
f"swap to primary mint ({settings.primary_mint}, unit={settings.primary_mint_unit}): "
f"receive_amount={receive_amount}. Minimum 1 {settings.primary_mint_unit} required."
)
logger.info(
"swap_to_primary_mint: estimating fees",
extra={
"dummy_amount": receive_amount,
"unit": settings.primary_mint_unit,
"token_mint_url": token_mint_url,
"primary_mint": settings.primary_mint,
"amount_msat": amount_msat,
},
)
stage = "destination_fee_quote"
try:
_, _, dummy_mint_quote = await _request_mint_with_fallback(
receive_amount,
op_name="swap_fee_est_mint_quote",
primary_wallet=primary_wallet,
)
stage = "source_fee_quote"
dummy_melt_quote = await _mint_operation(
lambda: token_wallet.melt_quote(dummy_mint_quote.request),
op_name="swap_fee_est_melt_quote",
mint_url=token_mint_url,
)
fee_reserve = dummy_melt_quote.fee_reserve
input_fees = token_wallet.get_fees_for_proofs(proofs)
total_fees = fee_reserve + input_fees
minted_amount = _net_minted_amount(amount_msat, token_unit, total_fees)
if minted_amount <= 0:
raise ValueError(f"Fees ({total_fees} {token_unit}) exceed token amount")
logger.info(
"swap_to_primary_mint: fee estimation result",
extra={
"token_amount_sat": _msats_to_sats(amount_msat),
"estimated_fee": total_fees,
"estimated_fee_unit": token_unit,
"input_fees": input_fees,
"minted_amount": minted_amount,
"minted_unit": settings.primary_mint_unit,
"fee_reserve": fee_reserve,
"token_mint_url": token_mint_url,
"primary_mint": settings.primary_mint,
},
)
return minted_amount
except Exception as e:
logger.error(
"Cashu swap fee estimation failed",
extra={
"event": "cashu_swap_fee_estimation_failed",
"stage": stage,
"error": str(e),
"error_type": type(e).__name__,
"amount_msat": amount_msat,
"token_unit": token_unit,
"token_mint_url": token_mint_url,
"primary_mint": settings.primary_mint,
"primary_mint_unit": settings.primary_mint_unit,
"receive_amount": receive_amount,
},
)
if is_mint_connection_error(e):
if stage == "source_fee_quote":
logger.error(
"Source mint is unreachable; destination fallback cannot spend its proofs",
extra={
"event": "cashu_source_mint_unreachable",
"source_mint": token_mint_url,
"stage": stage,
"fallback_possible": False,
"reason": "cashu_proofs_are_bound_to_the_issuing_mint",
},
)
raise SourceMintConnectionError(
"Issuing Cashu mint is unreachable"
) from e
raise MintConnectionError("Cashu mint is unreachable") from e
raise ValueError(f"Failed to estimate fees: {e}") from e
async def swap_to_trusted_mint(
token_obj: Token, token_wallet: Wallet
) -> tuple[int, str, str]:
logger.warning(
"Starting Cashu cross-mint swap",
extra={
"event": "cashu_swap_started",
"source_mint": token_obj.mint,
"token_amount": token_obj.amount,
"unit": token_obj.unit,
"primary_mint": settings.primary_mint,
},
)
# Ensure amount is an integer
if not isinstance(token_obj.amount, int):
token_amount = int(token_obj.amount)
else:
token_amount = token_obj.amount
if token_obj.unit == "sat":
amount_msat = _sats_to_msats(token_amount)
elif token_obj.unit == "msat":
amount_msat = token_amount
else:
raise ValueError("Invalid unit")
# If the token is already from the primary mint, we don't need a cross-mint
# swap — redeem it same-mint. There's no melt/Lightning fee, but the mint's
# NUT-02 input fee still applies; _redeem_same_mint accounts for it.
if token_obj.mint == settings.primary_mint:
logger.info(
"swap_to_primary_mint: token already on primary mint, skipping swap",
extra={
"mint": token_obj.mint,
"amount": token_amount,
"unit": token_obj.unit,
},
)
return await _redeem_same_mint(token_wallet, token_obj)
primary_wallet: Wallet | None = None
minted_amount = await _calculate_swap_amount(
amount_msat,
token_obj.unit,
token_obj.mint,
token_wallet,
primary_wallet,
token_obj.proofs,
)
# The estimate above is non-binding: the mint may demand a higher fee on the
# real quote or reject the melt outright. Retry the quote/melt cycle with the
# amount recomputed from the fees the mint actually demands.
observed_extra_fee = 0
attempt = 0
dest_wallet = primary_wallet
dest_mint_url = settings.primary_mint
while True:
attempt += 1
if minted_amount <= 0:
logger.error(
"swap_to_primary_mint: minted_amount is zero or negative before requesting quote",
extra={
"minted_amount": minted_amount,
"attempt": attempt,
"foreign_mint": token_obj.mint,
"token_amount": token_amount,
"token_unit": token_obj.unit,
"amount_msat": amount_msat,
"observed_extra_fee": observed_extra_fee,
"primary_mint": settings.primary_mint,
},
)
raise ValueError(
f"Cannot swap token ({token_amount} {token_obj.unit}) from {token_obj.mint}: "
f"minted_amount={minted_amount} after fee deduction (attempt {attempt})"
)
dest_wallet, dest_mint_url, mint_quote = await _request_mint_with_fallback(
minted_amount,
op_name="swap_request_mint",
primary_wallet=primary_wallet,
)
logger.info(
"swap_to_primary_mint: mint quote received",
extra={
"mint_quote_id": mint_quote.quote,
"attempt": attempt,
"dest_mint": dest_mint_url,
},
)
logger.warning(
"Requesting melt quote from source mint",
extra={
"event": "cashu_source_melt_quote_attempt",
"source_mint": token_obj.mint,
"destination_mint": dest_mint_url,
"attempt": attempt,
},
)
try:
melt_quote = await _mint_operation(
lambda: token_wallet.melt_quote(mint_quote.request),
op_name="swap_melt_quote",
mint_url=token_obj.mint,
)
except Exception as error:
if is_mint_connection_error(error):
logger.error(
"Source mint is unreachable; destination fallback cannot spend its proofs",
extra={
"event": "cashu_source_mint_unreachable",
"source_mint": token_obj.mint,
"destination_mint": dest_mint_url,
"stage": "source_melt_quote",
"error": str(error),
"error_type": type(error).__name__,
"attempt": attempt,
},
)
raise SourceMintConnectionError(
"Issuing Cashu mint is unreachable"
) from error
raise
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
logger.info(
"swap_to_primary_mint: melt quote received",
extra={
"melt_quote_id": melt_quote.quote,
"melt_amount": melt_quote.amount,
"melt_fee_reserve": melt_quote.fee_reserve,
"input_fees": input_fees,
"total_needed": total_needed,
"token_amount": token_amount,
"attempt": attempt,
},
)
if total_needed > token_amount:
recomputed = _net_minted_amount(
amount_msat,
token_obj.unit,
melt_quote.fee_reserve + input_fees + observed_extra_fee,
)
if attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
logger.warning(
"swap_to_primary_mint: insufficient token amount for melt fees",
extra={
"token_amount": token_amount,
"melt_amount": melt_quote.amount,
"melt_fee_reserve": melt_quote.fee_reserve,
"input_fees": input_fees,
"total_needed": total_needed,
"shortfall": total_needed - token_amount,
"attempts": attempt,
},
)
raise ValueError(
f"Token amount ({token_amount} {token_obj.unit}) is insufficient to cover "
f"melt fees. Needed: {total_needed} {token_obj.unit} "
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve} + input_fees: {input_fees})"
)
logger.warning(
"swap_to_primary_mint: melt quote exceeds token amount, retrying",
extra={
"total_needed": total_needed,
"token_amount": token_amount,
"retry_minted_amount": recomputed,
"attempt": attempt,
},
)
minted_amount = recomputed
continue
try:
_ = await _mint_operation(
lambda: token_wallet.melt(
proofs=token_obj.proofs,
invoice=mint_quote.request,
fee_reserve_sat=melt_quote.fee_reserve,
quote_id=melt_quote.quote,
),
op_name="swap_melt",
mint_url=token_obj.mint,
retry_timeouts=False,
)
except Exception as e:
# A down mint won't fix itself by retrying with a smaller amount.
if is_mint_connection_error(e):
logger.error(
"Source mint became unreachable during melt",
extra={
"event": "cashu_source_mint_unreachable",
"stage": "source_melt",
"error": str(e),
"error_type": type(e).__name__,
"source_mint": token_obj.mint,
"destination_mint": dest_mint_url,
"attempt": attempt,
},
)
raise SourceMintConnectionError(
"Issuing Cashu mint is unreachable"
) from e
shortfall = _melt_insufficient_shortfall(e)
recomputed = 0
if shortfall is not None:
observed_extra_fee += shortfall
recomputed = _net_minted_amount(
amount_msat,
token_obj.unit,
melt_quote.fee_reserve + input_fees + observed_extra_fee,
)
if shortfall is None or attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
logger.error(
"swap_to_primary_mint: melt failed",
extra={
"error": str(e),
"error_type": type(e).__name__,
"foreign_mint": token_obj.mint,
"token_amount": token_amount,
"melt_quote_id": melt_quote.quote,
"total_needed": total_needed,
"attempts": attempt,
},
)
raise ValueError(
f"Failed to melt token from foreign mint {token_obj.mint}: {e}"
) from e
logger.warning(
"swap_to_primary_mint: mint demanded more than quoted at melt, retrying",
extra={
"shortfall": shortfall,
"retry_minted_amount": recomputed,
"attempt": attempt,
},
)
minted_amount = recomputed
continue
break
logger.warning(
"Source melt succeeded; minting on destination",
extra={
"event": "cashu_destination_mint_attempt",
"minted_amount": minted_amount,
"mint_quote_id": mint_quote.quote,
"dest_mint": dest_mint_url,
},
)
await dest_wallet.load_proofs(reload=True)
pre_mint_balance = dest_wallet.available_balance.amount
try:
_ = await _mint_operation(
lambda: dest_wallet.mint(minted_amount, quote_id=mint_quote.quote),
op_name="swap_mint_on_primary",
mint_url=dest_mint_url,
retry_timeouts=False,
)
except Exception as e:
if "11003" in str(e) or "outputs already signed" in str(e).lower():
# Previous mint call signed outputs at the mint but failed before
# bump_secret_derivation ran locally. Recover orphaned proofs and
# advance the counter so the next request derives fresh secrets.
logger.warning(
"swap_to_primary_mint: outputs already signed — recovering orphaned proofs",
extra={
"mint_quote_id": mint_quote.quote,
"minted_amount": minted_amount,
},
)
try:
for keyset_id in dest_wallet.keysets:
await dest_wallet.restore_tokens_for_keyset(
keyset_id, to=1, batch=25
)
await dest_wallet.load_proofs(reload=True)
post_recovery_balance = dest_wallet.available_balance.amount
balance_gained = post_recovery_balance - pre_mint_balance
logger.info(
"swap_to_primary_mint: recovery scan completed",
extra={
"pre_mint_balance": pre_mint_balance,
"post_recovery_balance": post_recovery_balance,
"balance_gained": balance_gained,
"expected": minted_amount,
},
)
if balance_gained < minted_amount:
# Recovery scan ran but did NOT restore the orphaned proofs
# (mint reports them as spent — they're stuck). Refuse to
# credit the API key balance for proofs we don't actually hold.
raise TokenConsumedError(
f"Swap recovery failed: mint signed outputs but proofs are "
f"unrecoverable (mint reports them spent). "
f"Expected {minted_amount}, recovered {balance_gained}. "
f"Local wallet DB ('.wallet/') state is corrupted — "
f"the counter for keyset is stuck at a bad index range."
)
except TokenConsumedError:
raise
except Exception as recovery_err:
logger.error(
"swap_to_primary_mint: recovery failed",
extra={"error": str(recovery_err)},
)
raise TokenConsumedError(
f"Mint on primary failed and recovery unsuccessful: {e}"
) from e
else:
logger.error(
"swap_to_primary_mint: mint on primary failed after successful melt",
extra={
"error": str(e),
"error_type": type(e).__name__,
"minted_amount": minted_amount,
"mint_quote_id": mint_quote.quote,
},
)
# Foreign proofs already melted (spent) — non-retryable.
raise TokenConsumedError(
"Mint on primary failed after successful melt"
) from e
logger.warning(
"Cashu cross-mint swap completed",
extra={
"event": "cashu_swap_completed",
"source_mint": token_obj.mint,
"dest_mint": dest_mint_url,
"original_amount": token_amount,
"minted_amount": minted_amount,
"unit": settings.primary_mint_unit,
},
)
return int(minted_amount), settings.primary_mint_unit, dest_mint_url
async def swap_to_primary_mint(
token_obj: Token, token_wallet: Wallet
) -> tuple[int, str, str]:
"""Backward-compatible alias for callers using the old function name."""
return await swap_to_trusted_mint(token_obj, token_wallet)
async def credit_balance(
cashu_token: str, key: db.ApiKey, session: db.AsyncSession
) -> int:
async with wallet_operation_guard():
return await _credit_balance_locked(cashu_token, key, session)
async def _credit_balance_locked(
cashu_token: str, key: db.ApiKey, session: db.AsyncSession
) -> int:
logger.info(
"Starting Cashu balance credit",
extra={
"event": "cashu_credit_started",
"key_hash": key.hashed_key[:8],
},
)
try:
amount, unit, mint_url = await recieve_token(cashu_token)
original_amount = amount
original_unit = unit
logger.info(
"credit_balance: Token redeemed successfully",
extra={"amount": amount, "unit": unit, "mint_url": mint_url},
)
if unit == "sat":
amount = _sats_to_msats(amount)
logger.info(
"credit_balance: Converted to msat", extra={"amount_msat": amount}
)
# Guard against zero/negative redemptions (empty or dust tokens, or
# swap-to-primary-mint amounts that net to <= 0 after fees). Raising here
# — before the UPDATE/commit below — leaves any freshly-created, still
# uncommitted ApiKey row to be rolled back when the request session
# closes, instead of persisting an orphan key with balance 0.
if amount <= 0:
logger.error(
"credit_balance: Redeemed amount is zero or negative; refusing to credit",
extra={"amount": amount, "unit": unit, "mint_url": mint_url},
)
raise ValueError(
f"Redeemed token amount must be positive, got {amount} msats"
)
logger.info(
"credit_balance: Updating balance",
extra={"old_balance": key.balance, "credit_amount": amount},
)
# The token is already redeemed (spent) here, so any crediting failure
# is post-redemption and non-retryable — surface it as TokenConsumedError
# (a key that vanished mid-flight, or an unexpected DB fault), never a
# retryable/token-error taxonomy.
try:
# Atomic UPDATE to prevent race conditions during concurrent topups.
stmt = (
update(db.ApiKey)
.where(col(db.ApiKey.hashed_key) == key.hashed_key)
.values(balance=(db.ApiKey.balance) + amount)
)
result = await session.exec(stmt) # type: ignore[call-overload]
# If pruning removed this key after redemption, do not commit a no-op
# balance update and pretend the top-up succeeded.
if (getattr(result, "rowcount", 0) or 0) == 0:
raise TokenConsumedError(
"Token redeemed but the API key disappeared before the "
"credit could be recorded"
)
await session.commit()
await session.refresh(key)
# refresh() starts a read transaction; release it before the
# transaction-history write opens its own session below.
await session.commit()
except TokenConsumedError:
raise
except Exception as db_error:
raise TokenConsumedError(
"Token redeemed but crediting the balance failed"
) from db_error
logger.info(
"credit_balance: Balance updated successfully",
extra={"new_balance": key.balance},
)
try:
await store_cashu_transaction(
token=cashu_token,
amount=original_amount,
unit=original_unit,
mint_url=mint_url,
typ="in",
source="apikey",
api_key_hashed_key=key.hashed_key,
)
except Exception:
pass
else:
logger.debug(
"Cashu token successfully redeemed and stored",
extra={"amount": amount, "unit": unit, "mint_url": mint_url},
)
return amount
except Exception as e:
logger.error(
"credit_balance: Error during token redemption",
extra={"error": str(e), "error_type": type(e).__name__},
)
raise
_wallets: dict[str, Wallet] = {}
_wallet_last_load: dict[str, float] = {}
_wallet_load_locks: dict[str, asyncio.Lock] = {}
# Minimum seconds between full mint info + proof reloads for the same
# wallet. Prevents redundant mint API calls when get_wallet(load=True)
# is called rapidly by multiple background tasks (balance fetch, payout,
# auto-topup all hitting get_wallet within the same cycle).
_WALLOAD_RELOAD_MIN_INTERVAL_SECONDS = 30
async def get_wallet(
mint_url: str,
unit: str = "sat",
load: bool = True,
retry_on_rate_limit: bool = True,
) -> Wallet:
global _wallets, _wallet_last_load, _wallet_load_locks
id = f"{mint_url}_{unit}"
lock = _wallet_load_locks.setdefault(id, asyncio.Lock())
async with lock:
if id not in _wallets:
_wallets[id] = await Wallet.with_db(mint_url, db=".wallet", unit=unit)
if load:
now = time.monotonic()
last = _wallet_last_load.get(id)
if last is None or now - last >= _WALLOAD_RELOAD_MIN_INTERVAL_SECONDS:
await _mint_operation(
lambda: _wallets[id].load_mint(),
op_name="load_mint",
mint_url=mint_url,
retry_on_rate_limit=retry_on_rate_limit,
)
await _mint_operation(
lambda: _wallets[id].load_proofs(reload=True),
op_name="load_proofs",
mint_url=mint_url,
retry_on_rate_limit=retry_on_rate_limit,
)
_wallet_last_load[id] = time.monotonic()
return _wallets[id]
def get_proofs_per_mint_and_unit(
wallet: Wallet, mint_url: str, unit: str, not_reserved: bool = False
) -> list[Proof]:
valid_keyset_ids = [
k.id
for k in wallet.keysets.values()
if k.mint_url == mint_url and k.unit.name == unit
]
proofs = [p for p in wallet.proofs if p.id in valid_keyset_ids]
if not_reserved:
proofs = [p for p in proofs if not p.reserved]
return proofs
async def slow_filter_spend_proofs(
proofs: list[Proof],
wallet: Wallet,
*,
retry_on_rate_limit: bool = True,
) -> list[Proof]:
if not proofs:
return []
_proofs = []
_spent_proofs = []
# Keep proof-state checks in large batches. Mint quotas count HTTP requests,
# so smaller batches make balance reads slower and more likely to hit 429s.
batch_size = 1000
for i in range(0, len(proofs), batch_size):
pb = proofs[i : i + batch_size]
proof_states = await _mint_operation(
lambda: wallet.check_proof_state(pb),
op_name="check_proof_state",
mint_url=str(wallet.url),
retry_on_rate_limit=retry_on_rate_limit,
)
for proof, state in zip(pb, proof_states.states):
if str(state.state) != "spent":
_proofs.append(proof)
else:
_spent_proofs.append(proof)
if _spent_proofs:
await _mint_operation(
lambda: wallet.set_reserved_for_send(_spent_proofs, reserved=True),
op_name="set_reserved_spent_proofs",
mint_url=str(wallet.url),
retry_timeouts=False,
)
return _proofs
class BalanceDetail(TypedDict, total=False):
mint_url: str
unit: str
wallet_balance: int
user_balance: int
owner_balance: int
error: str
error_code: str
retry_after_seconds: float
_BALANCE_FETCH_RETRY_SECONDS = 60.0
_MINT_UNITS_CACHE_SECONDS = 300.0
_balance_fetch_failures: dict[tuple[str, str], tuple[float, str, str]] = {}
_balance_fetch_locks: dict[str, asyncio.Lock] = {}
_mint_supported_units: dict[str, tuple[float, list[str]]] = {}
async def _get_supported_mint_units(mint_url: str) -> list[str]:
now = time.monotonic()
cached = _mint_supported_units.get(mint_url)
if cached is not None and now < cached[0]:
return cached[1]
wallet = await get_wallet(mint_url, settings.primary_mint_unit, load=False)
keysets = await _mint_operation(
lambda: wallet._get_keysets(),
op_name="get_mint_keysets",
mint_url=mint_url,
retry_on_rate_limit=False,
)
units: list[str] = []
for keyset in keysets:
if not keyset.active or keyset.unit is None:
continue
unit = keyset.unit if isinstance(keyset.unit, str) else keyset.unit.name
if unit and unit not in units:
units.append(unit)
if not units:
units = [settings.primary_mint_unit]
elif settings.primary_mint_unit in units:
units.remove(settings.primary_mint_unit)
units.insert(0, settings.primary_mint_unit)
_mint_supported_units[mint_url] = (
time.monotonic() + _MINT_UNITS_CACHE_SECONDS,
units,
)
return units
def _balance_error(
mint_url: str,
unit: str,
error: str,
*,
error_code: str,
retry_after_seconds: float | None = None,
) -> BalanceDetail:
detail: BalanceDetail = {
"mint_url": mint_url,
"unit": unit,
"wallet_balance": 0,
"user_balance": 0,
"owner_balance": 0,
"error": error,
"error_code": error_code,
}
if retry_after_seconds is not None:
detail["retry_after_seconds"] = round(max(0.0, retry_after_seconds), 2)
return detail
async def fetch_all_balances(
units: list[str] | None = None,
) -> tuple[list[BalanceDetail], int, int, int]:
"""Fetch balances for all trusted mints without holding DB connections during I/O."""
mint_urls = _mints_to_inspect()
mint_units: dict[str, list[str]] = {}
discovery_errors: list[BalanceDetail] = []
if units is not None:
mint_units = {mint_url: units for mint_url in mint_urls}
else:
for mint_url in mint_urls:
try:
mint_units[mint_url] = await _get_supported_mint_units(mint_url)
except Exception as error:
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
error_code = (
"rate_limited"
if rate_limited
else "unreachable"
if connection_failure
else "mint_error"
)
if connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
_BALANCE_FETCH_RETRY_SECONDS, reason="unreachable"
)
retry_delay = max(
_BALANCE_FETCH_RETRY_SECONDS,
_mint_cooldown_remaining(mint_url),
)
discovery_errors.append(
_balance_error(
mint_url,
settings.primary_mint_unit,
str(error),
error_code=error_code,
retry_after_seconds=retry_delay,
)
)
mint_units[mint_url] = []
if not connection_failure and not rate_limited:
logger.warning(
"Unable to discover mint units",
extra={
"mint_url": mint_url,
"error": str(error),
"error_type": type(error).__name__,
},
)
# Read all liabilities in one short-lived transaction, then release the
# connection before starting concurrent mint network requests.
user_balances: dict[tuple[str, str], int] = {}
liabilities_error: str | None = None
query_units = list(
dict.fromkeys(unit for mint_url in mint_urls for unit in mint_units[mint_url])
)
try:
async with db.create_session() as session:
user_balances = await db.balances_by_mint_and_unit(
session, mint_urls, query_units
)
except Exception as error:
logger.error("Error reading user balances", extra={"error": str(error)})
liabilities_error = str(error)
mint_check_limit = asyncio.Semaphore(settings.mint_operation_concurrency)
async def fetch_balance(mint_url: str, unit: str) -> BalanceDetail:
key = (mint_url, unit)
lock = _balance_fetch_locks.setdefault(mint_url, asyncio.Lock())
async with lock:
now = time.monotonic()
failure = _balance_fetch_failures.get(key)
if failure is not None and now < failure[0]:
return _balance_error(
mint_url,
unit,
failure[1],
error_code=failure[2],
retry_after_seconds=failure[0] - now,
)
cooldown = _mint_cooldown_remaining(mint_url)
if cooldown > 0:
error_code = _mint_cooldown_reason(mint_url) or "cooldown"
error = {
"rate_limited": "Mint is rate limited",
"unreachable": "Mint is unreachable",
}.get(error_code, "Mint cooldown is active")
_balance_fetch_failures[key] = (now + cooldown, error, error_code)
return _balance_error(
mint_url,
unit,
error,
error_code=error_code,
retry_after_seconds=cooldown,
)
try:
async with mint_check_limit:
wallet = await get_wallet(
mint_url, unit, retry_on_rate_limit=False
)
proofs = get_proofs_per_mint_and_unit(
wallet, mint_url, unit, not_reserved=True
)
proofs = await slow_filter_spend_proofs(proofs, wallet)
except Exception as error:
connection_failure = is_mint_connection_error(error)
rate_limited = _is_mint_rate_limited(error)
error_code = (
"rate_limited"
if rate_limited
else "unreachable"
if connection_failure
else "mint_error"
)
if rate_limited:
_MintRateGuard.get(mint_url).apply_rate_limit_cooldown(
_BALANCE_FETCH_RETRY_SECONDS
)
elif connection_failure:
_MintRateGuard.get(mint_url).apply_cooldown(
_BALANCE_FETCH_RETRY_SECONDS, reason=error_code
)
retry_delay = max(
_BALANCE_FETCH_RETRY_SECONDS,
_mint_cooldown_remaining(mint_url),
)
_balance_fetch_failures[key] = (
time.monotonic() + retry_delay,
str(error),
error_code,
)
logger.warning(
"Unable to refresh mint balance",
extra={
"mint_url": mint_url,
"unit": unit,
"error": str(error),
"connection_failure": connection_failure,
"rate_limited": rate_limited,
"mint_cooldown_applied": connection_failure or rate_limited,
"retry_seconds": round(retry_delay, 2),
},
)
return _balance_error(
mint_url,
unit,
str(error),
error_code=error_code,
retry_after_seconds=retry_delay,
)
_balance_fetch_failures.pop(key, None)
user_balance = user_balances.get((mint_url, unit), 0)
if unit == "sat":
user_balance = _msats_to_sats(user_balance)
proofs_balance = sum(proof.amount for proof in proofs)
return {
"mint_url": mint_url,
"unit": unit,
"wallet_balance": proofs_balance,
"user_balance": user_balance,
"owner_balance": proofs_balance - user_balance,
}
tasks = [
fetch_balance(mint_url, unit)
for mint_url in mint_urls
for unit in mint_units[mint_url]
]
balance_details = discovery_errors + list(await asyncio.gather(*tasks))
total_wallet_balance_sats = 0
total_user_balance_sats = 0
for detail in balance_details:
if detail.get("error"):
continue
unit = detail["unit"]
total_wallet_balance_sats += (
detail["wallet_balance"]
if unit == "sat"
else _msats_to_sats(detail["wallet_balance"])
)
if liabilities_error is None:
total_user_balance_sats += (
detail["user_balance"]
if unit == "sat"
else _msats_to_sats(detail["user_balance"])
)
if liabilities_error is None:
owner_balance = total_wallet_balance_sats - total_user_balance_sats
else:
owner_balance = 0
for detail in balance_details:
detail["user_balance"] = 0
detail["owner_balance"] = 0
detail.setdefault("error", liabilities_error)
return (
balance_details,
total_wallet_balance_sats,
total_user_balance_sats,
owner_balance,
)
async def _payout_mint_and_unit(mint_url: str, unit: str) -> None:
"""Send only conservatively proven owner funds for one wallet."""
try:
wallet = await get_wallet(mint_url, unit)
proofs = get_proofs_per_mint_and_unit(
wallet, mint_url, unit, not_reserved=True
)
proofs = await slow_filter_spend_proofs(proofs, wallet)
await asyncio.sleep(5)
except Exception as e:
logger.error(
f"Error sending payout: {type(e).__name__}",
extra={"error": str(e), "mint_url": mint_url, "unit": unit},
)
return
# Fetch liability after the proofs snapshot and settle delay while the
# wallet operation guard excludes concurrent proof mutation and crediting.
try:
async with db.create_session() as session:
# ApiKey stores a refund preference, not funding provenance. Until
# liabilities have a durable per-credit ledger, subtract the total
# liability from every wallet rather than risk calling customer
# funds owner profit on the wrong mint.
user_balance = await db.total_user_liability(session)
except Exception as e:
logger.error(
f"Error in periodic payout cycle: {type(e).__name__}",
extra={"error": str(e), "mint_url": mint_url, "unit": unit},
)
return
try:
if unit == "sat":
user_balance = _msats_to_sats(user_balance)
proofs_balance = sum(proof.amount for proof in proofs)
available_balance = proofs_balance - user_balance
min_amount = (
settings.min_payout_sat
if unit == "sat"
else _sats_to_msats(settings.min_payout_sat)
)
if available_balance > min_amount:
amount_received = await raw_send_to_lnurl(
wallet,
proofs,
settings.receive_ln_address,
unit,
amount=available_balance,
)
logger.info(
"Payout sent successfully",
extra={
"mint_url": mint_url,
"unit": unit,
"balance": available_balance,
"amount_received": amount_received,
},
)
except Exception as e:
logger.error(
f"Error sending payout: {type(e).__name__}",
extra={"error": str(e), "mint_url": mint_url, "unit": unit},
)
async def periodic_payout() -> None:
while True:
await asyncio.sleep(settings.payout_interval_seconds)
try:
if not settings.receive_ln_address:
continue
for mint_url in _mints_to_inspect():
for unit in ["sat", "msat"]:
# Proof mutation, liability observation, and sending are one
# cross-process critical section. Credits take the same lock.
async with wallet_operation_guard():
await _payout_mint_and_unit(mint_url, unit)
except Exception as e:
logger.error(
f"Error in periodic payout cycle: {type(e).__name__}",
extra={"error": str(e)},
)
async def _set_refund_sweep_state(
refund_id: str,
*,
predicates: tuple[typing.Any, ...] = (),
**values: object,
) -> int:
async with db.create_session() as session:
result = await session.exec( # type: ignore[call-overload]
update(db.CashuTransaction)
.where(col(db.CashuTransaction.id) == refund_id, *predicates)
.values(**values)
)
await session.commit()
return int(result.rowcount or 0)
async def _refund_sweep_once(cutoff: int) -> None:
claim_cutoff = int(time.time()) - settings.refund_sweep_claim_timeout_seconds
claim_available = col(db.CashuTransaction.sweep_started_at).is_(None) | (
col(db.CashuTransaction.sweep_started_at) < claim_cutoff
)
async with db.create_session() as session:
stmt = select(db.CashuTransaction).where(
db.CashuTransaction.type == "out",
db.CashuTransaction.collected == False, # noqa: E712
db.CashuTransaction.swept == False, # noqa: E712
db.CashuTransaction.created_at < cutoff,
claim_available,
)
results = await session.exec(stmt)
refunds = results.all()
for refund in refunds:
reclaimed_stale_claim = refund.sweep_started_at is not None
claim_started_at = int(time.time())
claimed = await _set_refund_sweep_state(
refund.id,
predicates=(
col(db.CashuTransaction.swept) == False, # noqa: E712
col(db.CashuTransaction.collected) == False, # noqa: E712
claim_available,
),
sweep_started_at=claim_started_at,
)
if claimed != 1:
continue
claim_owned = col(db.CashuTransaction.sweep_started_at) == claim_started_at
redeemed = False
try:
await recieve_token(refund.token)
redeemed = True
finalized = await _set_refund_sweep_state(
refund.id,
predicates=(claim_owned,),
swept=True,
sweep_started_at=None,
)
if finalized == 1:
logger.info(
"Swept uncollected refund",
extra={
"id": refund.id,
"amount": refund.amount,
"unit": refund.unit,
},
)
else:
logger.critical(
"Refund token swept after claim ownership changed; manual reconciliation required",
extra={"id": refund.id},
)
except BaseException as e:
if redeemed or isinstance(e, TokenConsumedError):
# The token was spent, or the redemption outcome is known to be
# post-spend. Retain the claim so a stale retry classifies
# "already spent" as swept, never as a client collection.
logger.critical(
"Refund token spent but sweep checkpoint was not completed; manual reconciliation required",
extra={"id": refund.id},
exc_info=isinstance(e, Exception),
)
if not isinstance(e, Exception):
raise
continue
error_msg = str(e).lower()
if isinstance(e, Exception) and "already spent" in error_msg:
if reclaimed_stale_claim:
# A prior worker may have redeemed the token and crashed
# before finalizing. Treat the ambiguous stale claim as a
# completed sweep rather than misreporting client collection.
updated = await _set_refund_sweep_state(
refund.id,
predicates=(claim_owned,),
swept=True,
sweep_started_at=None,
)
else:
updated = await _set_refund_sweep_state(
refund.id,
predicates=(claim_owned,),
collected=True,
swept=False,
sweep_started_at=None,
)
if updated == 1:
logger.info(
"Refund token was already spent",
extra={
"id": refund.id,
"reclaimed_stale_claim": reclaimed_stale_claim,
},
)
else:
logger.warning(
"Refund claim ownership changed before spent-token checkpoint",
extra={"id": refund.id},
)
else:
# Once redemption starts, an exception cannot prove the token
# was not spent (for example, a melt may land before the
# response is lost). Retain the claim so a stale retry treats
# an "already spent" result as a completed sweep.
logger.critical(
"Refund token redemption outcome is unknown; retaining sweep claim for reconciliation",
extra={"id": refund.id, "error": str(e)},
exc_info=isinstance(e, Exception),
)
if not isinstance(e, Exception):
raise
async def refund_sweep_once() -> None:
"""Sweep eligible uncollected refund tokens once."""
cutoff = int(time.time()) - settings.refund_sweep_ttl_seconds
await _refund_sweep_once(cutoff)
async def periodic_refund_sweep() -> None:
while True:
await asyncio.sleep(60 * 60) # every hour
try:
await refund_sweep_once()
except Exception as e:
logger.error(
"Error in periodic refund sweep",
extra={"error": str(e), "error_type": type(e).__name__},
)
async def periodic_routstr_fee_payout() -> None:
from .auth import (
ROUTSTR_FEE_DEFAULT_PAYOUT,
ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS,
ROUTSTR_LN_ADDRESS,
)
if not ROUTSTR_LN_ADDRESS:
logger.info("ROUTSTR_LN_ADDRESS not set, skipping fee payout")
return
while True:
await asyncio.sleep(ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS)
try:
async with db.create_session() as session:
fee = await db.get_routstr_fee(session)
if fee.payout_in_progress_msats:
logger.critical(
"Routstr fee payout requires manual reconciliation",
extra={
"payout_in_progress_msats": fee.payout_in_progress_msats,
"payout_started_at": fee.payout_started_at,
},
)
continue
accumulated_sats = _msats_to_sats(fee.accumulated_msats)
if accumulated_sats < ROUTSTR_FEE_DEFAULT_PAYOUT:
continue
paid_msats = _sats_to_msats(accumulated_sats)
# Wallet/proof preparation cannot send funds, so do it before the
# durable checkpoint. A preparation failure must not strand an
# in-progress payout that requires manual reconciliation.
wallet = await get_wallet(settings.primary_mint, "sat")
proofs = get_proofs_per_mint_and_unit(
wallet, settings.primary_mint, "sat", not_reserved=True
)
async with db.create_session() as session:
payout_checkpointed = await db.reset_routstr_fee(session, paid_msats)
if not payout_checkpointed:
logger.warning("Routstr fee payout was already claimed")
continue
try:
amount_received = await raw_send_to_lnurl(
wallet,
proofs,
ROUTSTR_LN_ADDRESS,
"sat",
amount=accumulated_sats,
)
except BaseException as e:
logger.critical(
"Routstr fee payout outcome is unknown; manual reconciliation required",
extra={"payout_in_progress_msats": paid_msats},
exc_info=isinstance(e, Exception),
)
if not isinstance(e, Exception):
raise
continue
try:
async with db.create_session() as session:
payout_completed = await db.complete_routstr_fee_payout(
session, paid_msats
)
except BaseException as e:
logger.critical(
"Routstr fee payout sent but checkpoint was not completed",
extra={"payout_in_progress_msats": paid_msats},
exc_info=isinstance(e, Exception),
)
if not isinstance(e, Exception):
raise
continue
if not payout_completed:
logger.critical(
"Routstr fee payout sent but checkpoint was not completed",
extra={"payout_in_progress_msats": paid_msats},
)
continue
logger.info(
"Routstr fee payout sent",
extra={
"accumulated_sats": accumulated_sats,
"amount_received": amount_received,
},
)
except Exception as e:
logger.error(
f"Error in Routstr fee payout: {type(e).__name__}",
extra={"error": str(e)},
)
async def send_to_lnurl(amount: int, unit: str, mint: str, address: str) -> int:
mint = await find_trusted_mint_with_funds(amount, unit, mint)
wallet = await get_wallet(mint, unit)
available = get_proofs_per_mint_and_unit(wallet, mint, unit, not_reserved=True)
proofs, _ = await wallet.select_to_send(available, amount, set_reserved=True)
return await raw_send_to_lnurl(wallet, proofs, address, unit)
# class Payment:
# """
# Stores all cashu payment related data
# """
# def __init__(self, token: str) -> None:
# self.initial_token = token
# amount, unit, mint_url = self.parse_token(token)
# self.amount = amount
# self.unit = unit
# self.mint_url = mint_url
# self.claimed_proofs = redeem_to_proofs(token)
# def parse_token(self, token: str) -> tuple[int, CurrencyUnit, str]:
# raise NotImplementedError
# def refund_full(self) -> None:
# raise NotImplementedError
# def refund_partial(self, amount: int) -> None:
# raise NotImplementedError