diff --git a/migrations/versions/add_mint_url_to_lightning_invoices.py b/migrations/versions/add_mint_url_to_lightning_invoices.py new file mode 100644 index 00000000..d3eb71b9 --- /dev/null +++ b/migrations/versions/add_mint_url_to_lightning_invoices.py @@ -0,0 +1,21 @@ +"""add mint_url to lightning_invoices + +Revision ID: add_mint_url_li +Revises: c6d7e8f9a0b1 +Create Date: 2026-07-10 02:00:00.000000 +""" +import sqlalchemy as sa +from alembic import op + +revision = "add_mint_url_li" +down_revision = "c6d7e8f9a0b1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("lightning_invoices", "mint_url") diff --git a/routstr/core/db.py b/routstr/core/db.py index 586f467e..6cd5b593 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -225,6 +225,9 @@ class LightningInvoice(SQLModel, table=True): # type: ignore default=None, description="Associated API key hash for topup operations" ) purpose: str = Field(description="create or topup") + mint_url: str | None = Field( + default=None, description="Mint URL where the quote was created (fallback tracking)" + ) created_at: int = Field( default_factory=lambda: int(time.time()), description="Unix timestamp" ) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 3a144e10..88797ff2 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -49,6 +49,23 @@ class Settings(BaseSettings): payout_interval_seconds: int = Field( default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS" ) + # Timeout (seconds) for individual mint API operations (melt, mint, swap, + # checkstate). When a mint is slow or rate-limiting, operations are + # cancelled after this delay instead of hanging indefinitely. + mint_operation_timeout_seconds: int = Field( + default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS" + ) + # Maximum mint API requests per minute, per mint URL. Nutshell mints + # (e.g. Minibits) enforce 20/min/IP on transaction endpoints (mint, melt, + # swap, quotes) and 60/min/IP globally. 20 stays under the transaction + # bucket since most calls here are transaction ops. 0 = unlimited. + mint_max_requests_per_minute: int = Field( + default=20, ge=0, env="MINT_MAX_REQUESTS_PER_MINUTE" + ) + # Max retries when a mint returns 429 or times out (exponential backoff). + mint_retry_max_attempts: int = Field( + default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS" + ) # Pricing # Default behavior: derive pricing from MODELS diff --git a/routstr/lightning.py b/routstr/lightning.py index b0bbc63b..24c21dde 100644 --- a/routstr/lightning.py +++ b/routstr/lightning.py @@ -11,7 +11,13 @@ from sqlmodel.ext.asyncio.session import AsyncSession from .core.db import ApiKey, LightningInvoice, create_session, get_session from .core.logging import get_logger from .core.settings import settings -from .wallet import get_wallet +from .wallet import ( + MintConnectionError, + _is_mint_rate_limited, + _mint_operation, + get_wallet, + is_mint_connection_error, +) logger = get_logger(__name__) @@ -64,12 +70,44 @@ class InvoiceRecoverRequest(BaseModel): bolt11: str = Field(description="BOLT11 invoice string") +async def _request_mint_with_fallback( + amount_sats: int, +) -> tuple[str, str, str]: + """Primary first, fall back to other trusted mints on rate-limit/transport failure.""" + tried: list[str] = [] + candidates = [settings.primary_mint] + [ + m for m in settings.cashu_mints if m != settings.primary_mint + ] + for mint_url in candidates: + try: + wallet = await get_wallet(mint_url, "sat") + quote = await _mint_operation( + lambda: wallet.request_mint(amount_sats), + op_name="request_mint_invoice", + mint_url=mint_url, + ) + return quote.request, quote.quote, mint_url + except Exception as e: + tried.append(f"{mint_url}: {type(e).__name__}") + if not is_mint_connection_error(e) and not _is_mint_rate_limited(e): + raise + logger.warning( + "request_mint failed, trying fallback mint", + extra={ + "failed_mint": mint_url, + "error": str(e), + "tried": tried, + }, + ) + continue + raise MintConnectionError(f"All mints failed for request_mint: {tried}") + + async def generate_lightning_invoice( amount_sats: int, description: str -) -> tuple[str, str]: - wallet = await get_wallet(settings.primary_mint, "sat") - quote = await wallet.request_mint(amount_sats) - return quote.request, quote.quote +) -> tuple[str, str, str]: + bolt11, payment_hash, mint_url = await _request_mint_with_fallback(amount_sats) + return bolt11, payment_hash, mint_url def generate_invoice_id() -> str: @@ -99,7 +137,7 @@ async def create_invoice( try: description = f"Routstr {request.purpose} {request.amount_sats} sats" - bolt11, payment_hash = await generate_lightning_invoice( + bolt11, payment_hash, mint_url = await generate_lightning_invoice( request.amount_sats, description ) @@ -115,6 +153,7 @@ async def create_invoice( status="pending", api_key_hash=api_key_token[3:] if api_key_token else None, purpose=request.purpose, + mint_url=mint_url, balance_limit=request.balance_limit, balance_limit_reset=request.balance_limit_reset, validity_date=request.validity_date, @@ -223,9 +262,14 @@ async def check_invoice_payment( invoice: LightningInvoice, session: AsyncSession ) -> None: try: - wallet = await get_wallet(settings.primary_mint, "sat") + mint_url = invoice.mint_url or settings.primary_mint + wallet = await get_wallet(mint_url, "sat") - mint_status = await wallet.get_mint_quote(invoice.payment_hash) + mint_status = await _mint_operation( + lambda: wallet.get_mint_quote(invoice.payment_hash), + op_name="get_mint_quote", + mint_url=mint_url, + ) if mint_status.paid: invoice.status = "paid" @@ -258,8 +302,13 @@ async def check_invoice_payment( async def create_api_key_from_invoice( invoice: LightningInvoice, session: AsyncSession ) -> ApiKey: - wallet = await get_wallet(settings.primary_mint, "sat") - await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash) + mint_url = invoice.mint_url or settings.primary_mint + wallet = await get_wallet(mint_url, "sat") + await _mint_operation( + lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash), + op_name="invoice_mint_create", + mint_url=mint_url, + ) dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}" hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest() @@ -268,7 +317,7 @@ async def create_api_key_from_invoice( hashed_key=hashed_key, balance=invoice.amount_sats * 1000, # Convert to msats refund_currency="sat", - refund_mint_url=settings.primary_mint, + refund_mint_url=mint_url, balance_limit=invoice.balance_limit, balance_limit_reset=invoice.balance_limit_reset, validity_date=invoice.validity_date, @@ -283,8 +332,13 @@ async def create_api_key_from_invoice( async def topup_api_key_from_invoice( invoice: LightningInvoice, session: AsyncSession ) -> None: - wallet = await get_wallet(settings.primary_mint, "sat") - await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash) + mint_url = invoice.mint_url or settings.primary_mint + wallet = await get_wallet(mint_url, "sat") + await _mint_operation( + lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash), + op_name="invoice_mint_topup", + mint_url=mint_url, + ) if not invoice.api_key_hash: raise ValueError("No API key associated with topup invoice") @@ -297,7 +351,9 @@ async def topup_api_key_from_invoice( await session.flush() -INVOICE_WATCH_INTERVAL_SECONDS = 5 +# Nutshell mints throttle Lightning backend lookups to once per 10s per +# quote, so polling faster just burns the global request budget for nothing. +INVOICE_WATCH_INTERVAL_SECONDS = 10 INVOICE_WATCH_BATCH_LIMIT = 100 diff --git a/routstr/payment/lnurl.py b/routstr/payment/lnurl.py index 26cf580d..3e67412d 100644 --- a/routstr/payment/lnurl.py +++ b/routstr/payment/lnurl.py @@ -1,11 +1,15 @@ from __future__ import annotations +import asyncio import math from typing import TypedDict import httpx from cashu.wallet.wallet import Proof, Wallet +from ..core.settings import settings +from ..wallet import _mint_operation + try: from bech32 import bech32_decode, convertbits # type: ignore except ModuleNotFoundError: # pragma: no cover – allow runtime miss @@ -215,15 +219,23 @@ async def raw_send_to_lnurl( lnurl_data["callback_url"], final_amount ) - melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice) + melt_quote_resp = await _mint_operation( + lambda: wallet.melt_quote(invoice=bolt11_invoice), + op_name="lnurl_melt_quote", + mint_url=str(wallet.url), + ) if amount: proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True) - _ = await wallet.melt( - proofs=proofs, - invoice=bolt11_invoice, - fee_reserve_sat=melt_quote_resp.fee_reserve, - quote_id=melt_quote_resp.quote, + _ = await _mint_operation( + lambda: wallet.melt( + proofs=proofs, + invoice=bolt11_invoice, + fee_reserve_sat=melt_quote_resp.fee_reserve, + quote_id=melt_quote_resp.quote, + ), + op_name="lnurl_melt", + mint_url=str(wallet.url), ) return final_amount diff --git a/routstr/wallet.py b/routstr/wallet.py index ccf7877f..6de0a6c9 100644 --- a/routstr/wallet.py +++ b/routstr/wallet.py @@ -16,7 +16,6 @@ from sqlmodel import col, select, update from .core import db, get_logger from .core.db import store_cashu_transaction from .core.settings import settings -from .payment.lnurl import raw_send_to_lnurl # cashu still declares Optional[X] without explicit defaults on MintInfo. # Under pydantic v2 those are required, but real mints omit many of them. @@ -62,6 +61,163 @@ _TRANSPORT_EXC_TYPES: tuple[type[BaseException], ...] = ( ) +class _MintRateLimiter: + """Per-mint token-bucket rate limiter. + + Enforces a maximum number of mint API requests per minute per mint URL. + When the bucket is empty, callers block until a token is available. + This prevents the node runner from being rate-limited or blocked by + mints that enforce request quotas. + """ + + _limiters: dict[str, "_MintRateLimiter"] = {} + + @classmethod + def get(cls, mint_url: str) -> "_MintRateLimiter | None": + rpm = settings.mint_max_requests_per_minute + if rpm <= 0: + return None + if mint_url not in cls._limiters: + cls._limiters[mint_url] = cls(mint_url, rpm) + return cls._limiters[mint_url] + + def __init__(self, mint_url: str, max_per_minute: int): + self._mint_url = mint_url + self._max = max_per_minute + # Refill rate: tokens per second + self._refill_rate = max_per_minute / 60.0 + self._tokens: float = float(max_per_minute) + self._last_refill = time.monotonic() + self._lock = asyncio.Lock() + + async def acquire(self) -> None: + async with self._lock: + now = time.monotonic() + elapsed = now - self._last_refill + self._tokens = min(self._max, self._tokens + elapsed * self._refill_rate) + self._last_refill = now + if self._tokens < 1: + wait = (1 - self._tokens) / self._refill_rate + logger.debug( + "Mint rate limiter: throttling", + extra={ + "mint_url": self._mint_url, + "wait_seconds": round(wait, 2), + "tokens_available": round(self._tokens, 2), + }, + ) + await asyncio.sleep(wait) + self._tokens = 0 + else: + self._tokens -= 1 + + +def _is_mint_rate_limited(error: BaseException) -> bool: + """True if the mint returned a 429 or rate-limit indication.""" + current: BaseException | None = error + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + if isinstance(current, httpx.HTTPStatusError): + if current.response.status_code == 429: + return True + lowered = str(current).lower() + if "rate limit" in lowered or "too many requests" in lowered: + return True + current = current.__cause__ or current.__context__ + return False + + +async def _mint_operation( + factory, *, op_name: str = "mint_operation", mint_url: str = "" +): + """Wrap a mint API callable with rate limiting, timeout, and retry. + + ``factory`` must be a zero-arg callable that returns a fresh coroutine + each call — a pre-created coroutine can only be awaited once, so on retry + the original would be dead. + """ + limiter = _MintRateLimiter.get(mint_url) if mint_url else None + timeout = settings.mint_operation_timeout_seconds + max_attempts = settings.mint_retry_max_attempts + 1 + + last_exc: Exception | None = None + for attempt in range(max_attempts): + if limiter is not None: + await limiter.acquire() + + try: + if timeout > 0: + return await asyncio.wait_for(factory(), timeout=timeout) + return await factory() + except asyncio.TimeoutError as exc: + last_exc = exc + if attempt < max_attempts - 1: + backoff = (2 ** attempt) + (time.monotonic() % 1.0) + logger.warning( + "Mint operation timed out, retrying", + extra={ + "op_name": op_name, + "mint_url": mint_url, + "attempt": attempt + 1, + "backoff_seconds": round(backoff, 2), + }, + ) + await asyncio.sleep(backoff) + continue + raise httpx.TimeoutException( + f"{op_name} timed out after {timeout}s (retried {attempt + 1}x)" + ) from exc + except httpx.HTTPStatusError as exc: + if _is_mint_rate_limited(exc) and attempt < max_attempts - 1: + backoff = (2 ** attempt) + (time.monotonic() % 1.0) + retry_after = _parse_retry_after(exc.response.headers) + if retry_after is not None: + backoff = min(retry_after, backoff * 2) + logger.warning( + "Mint returned 429, backing off", + extra={ + "op_name": op_name, + "mint_url": mint_url, + "attempt": attempt + 1, + "backoff_seconds": round(backoff, 2), + }, + ) + await asyncio.sleep(backoff) + continue + raise + except Exception as exc: + if _is_mint_rate_limited(exc) and attempt < max_attempts - 1: + backoff = (2 ** attempt) + (time.monotonic() % 1.0) + logger.warning( + "Mint rate-limited, backing off", + extra={ + "op_name": op_name, + "mint_url": mint_url, + "attempt": attempt + 1, + "backoff_seconds": round(backoff, 2), + }, + ) + await asyncio.sleep(backoff) + continue + raise + + if last_exc: + raise last_exc + raise RuntimeError(f"{op_name}: exhausted retries unexpectedly") + + +def _parse_retry_after(headers) -> float | None: + """Parse a Retry-After header (delta-seconds form) into seconds.""" + raw = headers.get("retry-after") or headers.get("Retry-After") + if raw is None: + return None + try: + return float(str(raw).strip()) + except (TypeError, ValueError): + return None + + def is_mint_connection_error(error: BaseException) -> bool: """True if ``error`` (or anything in its cause/context chain) is a mint transport failure. Walks the chain because some sites re-raise transport @@ -192,10 +348,18 @@ async def _redeem_same_mint( that, not the face value, or routstr over-credits the user and its wallet drifts insolvent. """ - await wallet.load_mint(keyset_id=token_obj.keysets[0]) + await _mint_operation( + lambda: wallet.load_mint(keyset_id=token_obj.keysets[0]), + op_name="redeem_load_mint", + mint_url=token_obj.mint, + ) wallet.verify_proofs_dleq(token_obj.proofs) input_fees = wallet.get_fees_for_proofs(token_obj.proofs) - await wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True) + await _mint_operation( + lambda: wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True), + op_name="redeem_split", + mint_url=token_obj.mint, + ) return int(token_obj.amount) - input_fees, token_obj.unit, token_obj.mint @@ -341,6 +505,44 @@ def _melt_insufficient_shortfall(error: Exception) -> int | None: return 1 +async def _request_mint_with_fallback( + amount: int, *, op_name: str, primary_wallet: Wallet | None = None +) -> tuple[Wallet, str, object]: + """Try request_mint on the primary mint, fall back to other trusted mints + on transport or rate-limit failure. Returns the wallet, mint_url, and quote.""" + candidates = [settings.primary_mint] + [ + m for m in settings.cashu_mints if m != settings.primary_mint + ] + tried: list[str] = [] + for mint_url in candidates: + try: + if mint_url == settings.primary_mint and primary_wallet is not None: + wallet = primary_wallet + else: + wallet = await get_wallet(mint_url, settings.primary_mint_unit) + quote = await _mint_operation( + lambda: wallet.request_mint(amount), + op_name=op_name, + mint_url=mint_url, + ) + return wallet, mint_url, quote + except Exception as e: + tried.append(f"{mint_url}: {type(e).__name__}") + if not is_mint_connection_error(e) and not _is_mint_rate_limited(e): + raise + logger.warning( + "request_mint failed, trying fallback mint", + extra={ + "failed_mint": mint_url, + "error": str(e), + "tried": tried, + "op_name": op_name, + }, + ) + continue + raise MintConnectionError(f"All mints failed for {op_name}: {tried}") + + async def _calculate_swap_amount( amount_msat: int, token_unit: str, @@ -374,8 +576,16 @@ async def _calculate_swap_amount( ) try: - dummy_mint_quote = await primary_wallet.request_mint(receive_amount) - dummy_melt_quote = await token_wallet.melt_quote(dummy_mint_quote.request) + _, _, dummy_mint_quote = await _request_mint_with_fallback( + receive_amount, + op_name="swap_fee_est_mint_quote", + primary_wallet=primary_wallet, + ) + dummy_melt_quote = await _mint_operation( + lambda: token_wallet.melt_quote(dummy_mint_quote.request), + op_name="swap_fee_est_melt_quote", + mint_url=token_mint_url, + ) fee_reserve = dummy_melt_quote.fee_reserve input_fees = token_wallet.get_fees_for_proofs(proofs) @@ -462,15 +672,23 @@ async def swap_to_primary_mint( # amount recomputed from the fees the mint actually demands. observed_extra_fee = 0 attempt = 0 + dest_wallet = primary_wallet + dest_mint_url = settings.primary_mint while True: attempt += 1 - mint_quote = await primary_wallet.request_mint(minted_amount) + dest_wallet, dest_mint_url, mint_quote = await _request_mint_with_fallback( + minted_amount, op_name="swap_request_mint", primary_wallet=primary_wallet + ) logger.info( "swap_to_primary_mint: mint quote received", - extra={"mint_quote_id": mint_quote.quote, "attempt": attempt}, + extra={"mint_quote_id": mint_quote.quote, "attempt": attempt, "dest_mint": dest_mint_url}, ) - melt_quote = await token_wallet.melt_quote(mint_quote.request) + melt_quote = await _mint_operation( + lambda: token_wallet.melt_quote(mint_quote.request), + op_name="swap_melt_quote", + mint_url=token_obj.mint, + ) input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs) total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees logger.info( @@ -523,11 +741,15 @@ async def swap_to_primary_mint( continue try: - _ = await token_wallet.melt( - proofs=token_obj.proofs, - invoice=mint_quote.request, - fee_reserve_sat=melt_quote.fee_reserve, - quote_id=melt_quote.quote, + _ = await _mint_operation( + lambda: token_wallet.melt( + proofs=token_obj.proofs, + invoice=mint_quote.request, + fee_reserve_sat=melt_quote.fee_reserve, + quote_id=melt_quote.quote, + ), + op_name="swap_melt", + mint_url=token_obj.mint, ) except Exception as e: # A down mint won't fix itself by retrying with a smaller amount. @@ -576,14 +798,18 @@ async def swap_to_primary_mint( break logger.info( - "swap_to_primary_mint: melt succeeded, minting on primary", - extra={"minted_amount": minted_amount, "mint_quote_id": mint_quote.quote}, + "swap_to_primary_mint: melt succeeded, minting on destination", + extra={"minted_amount": minted_amount, "mint_quote_id": mint_quote.quote, "dest_mint": dest_mint_url}, ) - await primary_wallet.load_proofs(reload=True) - pre_mint_balance = primary_wallet.available_balance.amount + await dest_wallet.load_proofs(reload=True) + pre_mint_balance = dest_wallet.available_balance.amount try: - _ = await primary_wallet.mint(minted_amount, quote_id=mint_quote.quote) + _ = await _mint_operation( + lambda: dest_wallet.mint(minted_amount, quote_id=mint_quote.quote), + op_name="swap_mint_on_primary", + mint_url=dest_mint_url, + ) except Exception as e: if "11003" in str(e) or "outputs already signed" in str(e).lower(): # Previous mint call signed outputs at the mint but failed before @@ -594,10 +820,10 @@ async def swap_to_primary_mint( extra={"mint_quote_id": mint_quote.quote, "minted_amount": minted_amount}, ) try: - for keyset_id in primary_wallet.keysets: - await primary_wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25) - await primary_wallet.load_proofs(reload=True) - post_recovery_balance = primary_wallet.available_balance.amount + for keyset_id in dest_wallet.keysets: + await dest_wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25) + await dest_wallet.load_proofs(reload=True) + post_recovery_balance = dest_wallet.available_balance.amount balance_gained = post_recovery_balance - pre_mint_balance logger.info( "swap_to_primary_mint: recovery scan completed", @@ -648,14 +874,14 @@ async def swap_to_primary_mint( "swap_to_primary_mint: completed successfully", extra={ "foreign_mint": token_obj.mint, - "primary_mint": settings.primary_mint, + "dest_mint": dest_mint_url, "original_amount": token_amount, "minted_amount": minted_amount, "unit": settings.primary_mint_unit, }, ) - return int(minted_amount), settings.primary_mint_unit, settings.primary_mint + return int(minted_amount), settings.primary_mint_unit, dest_mint_url async def credit_balance( @@ -760,17 +986,35 @@ async def credit_balance( _wallets: dict[str, Wallet] = {} +_wallet_last_load: dict[str, float] = {} +# Minimum seconds between full mint info + proof reloads for the same +# wallet. Prevents redundant mint API calls when get_wallet(load=True) +# is called rapidly by multiple background tasks (balance fetch, payout, +# auto-topup all hitting get_wallet within the same cycle). +_WALLOAD_RELOAD_MIN_INTERVAL_SECONDS = 30 async def get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Wallet: - global _wallets + global _wallets, _wallet_last_load id = f"{mint_url}_{unit}" if id not in _wallets: _wallets[id] = await Wallet.with_db(mint_url, db=".wallet", unit=unit) if load: - await _wallets[id].load_mint() - await _wallets[id].load_proofs(reload=True) + now = time.monotonic() + last = _wallet_last_load.get(id, 0) + if now - last >= _WALLOAD_RELOAD_MIN_INTERVAL_SECONDS: + await _mint_operation( + lambda: _wallets[id].load_mint(), + op_name="load_mint", + mint_url=mint_url, + ) + await _mint_operation( + lambda: _wallets[id].load_proofs(reload=True), + op_name="load_proofs", + mint_url=mint_url, + ) + _wallet_last_load[id] = now return _wallets[id] @@ -788,20 +1032,35 @@ def get_proofs_per_mint_and_unit( return proofs -async def slow_filter_spend_proofs(proofs: list[Proof], wallet: Wallet) -> list[Proof]: +async def slow_filter_spend_proofs( + proofs: list[Proof], wallet: Wallet +) -> list[Proof]: if not proofs: return [] _proofs = [] _spent_proofs = [] - for i in range(0, len(proofs), 1000): - pb = proofs[i : i + 1000] - proof_states = await wallet.check_proof_state(pb) + # Smaller batch size to reduce per-request load on the mint. + # 1000 proofs per batch was too aggressive and triggered rate limits + # on mints with strict request quotas. + batch_size = 100 + for i in range(0, len(proofs), batch_size): + pb = proofs[i : i + batch_size] + proof_states = await _mint_operation( + lambda: wallet.check_proof_state(pb), + op_name="check_proof_state", + mint_url=str(wallet.url), + ) for proof, state in zip(pb, proof_states.states): if str(state.state) != "spent": _proofs.append(proof) else: _spent_proofs.append(proof) - await wallet.set_reserved_for_send(_spent_proofs, reserved=True) + if _spent_proofs: + await _mint_operation( + lambda: wallet.set_reserved_for_send(_spent_proofs, reserved=True), + op_name="set_reserved_spent_proofs", + mint_url=str(wallet.url), + ) return _proofs @@ -923,6 +1182,8 @@ async def periodic_payout() -> None: if not settings.receive_ln_address: continue try: + from .payment.lnurl import raw_send_to_lnurl + async with db.create_session() as session: for mint_url in settings.cashu_mints: for unit in ["sat", "msat"]: @@ -1037,6 +1298,8 @@ async def periodic_routstr_fee_payout() -> None: while True: await asyncio.sleep(ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS) try: + from .payment.lnurl import raw_send_to_lnurl + async with db.create_session() as session: fee = await db.get_routstr_fee(session) accumulated_sats = fee.accumulated_msats // 1000 @@ -1065,6 +1328,8 @@ async def periodic_routstr_fee_payout() -> None: async def send_to_lnurl(amount: int, unit: str, mint: str, address: str) -> int: + from .payment.lnurl import raw_send_to_lnurl + wallet = await get_wallet(mint, unit) proofs = wallet._get_proofs_per_keyset(wallet.proofs)[wallet.keyset_id] proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True) diff --git a/tests/integration/test_lightning_invoice_rip08.py b/tests/integration/test_lightning_invoice_rip08.py index 29301a42..35f1f1e7 100644 --- a/tests/integration/test_lightning_invoice_rip08.py +++ b/tests/integration/test_lightning_invoice_rip08.py @@ -26,11 +26,12 @@ async def patch_invoice_generation() -> Any: """Stub out `generate_lightning_invoice` so no mint round-trip is needed.""" counter = {"n": 0} - async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]: + async def fake_generate(amount_sats: int, description: str) -> tuple[str, str, str]: counter["n"] += 1 return ( f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}", f"payment_hash_{counter['n']}", + "http://localhost:3338", ) with patch( diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 3bb36a28..cc35e63a 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -1321,3 +1321,244 @@ async def test_swap_melt_transport_error_raises_mint_connection_error() -> None: await swap_to_primary_mint(mock_token, mock_token_wallet) assert mock_token_wallet.melt.call_count == 1 + + +# --------------------------------------------------------------------------- +# _mint_operation factory + retry +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_mint_operation_factory_retry_succeeds() -> None: + """_mint_operation accepts a zero-arg factory, not a dead coroutine. + A factory that raises twice then succeeds must be retried and return.""" + from routstr.core.settings import settings + from routstr.wallet import _mint_operation + + calls = 0 + + async def factory(): + nonlocal calls + calls += 1 + if calls < 3: + raise TimeoutError("timeout") + return "ok" + + with patch.object(settings, "mint_retry_max_attempts", 3): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch("asyncio.sleep", AsyncMock()): + result = await _mint_operation( + factory, op_name="test_retry", mint_url="http://mint:3338" + ) + + assert calls == 3 + assert result == "ok" + + +@pytest.mark.asyncio +async def test_mint_operation_factory_retry_exhausted() -> None: + """When the factory always times out, _mint_operation raises + httpx.TimeoutException after mint_retry_max_attempts + 1 attempts.""" + from routstr.core.settings import settings + from routstr.wallet import _mint_operation + + calls = 0 + + async def factory(): + nonlocal calls + calls += 1 + raise TimeoutError("always timeout") + + with patch.object(settings, "mint_retry_max_attempts", 2): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch("asyncio.sleep", AsyncMock()): + with pytest.raises(httpx.TimeoutException): + await _mint_operation( + factory, op_name="test_exhaust", mint_url="http://mint:3338" + ) + + assert calls == 3 # max_attempts(2) + 1 + + +# --------------------------------------------------------------------------- +# Trusted-mint fallback +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lightning_mint_fallback_for_topups() -> None: + """When the primary mint is unreachable, _request_mint_with_fallback + falls back to a secondary trusted mint.""" + from routstr.core.settings import settings + from routstr.lightning import _request_mint_with_fallback + + primary = "http://primary:3338" + secondary = "http://secondary:3338" + + mock_primary_wallet = Mock() + mock_primary_wallet.request_mint = AsyncMock( + side_effect=httpx.ConnectError("primary down") + ) + + mock_quote = Mock() + mock_quote.request = "lnbc1secondary" + mock_quote.quote = "quote_secondary" + mock_secondary_wallet = Mock() + mock_secondary_wallet.request_mint = AsyncMock(return_value=mock_quote) + + wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet} + mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m]) + + with patch.object(settings, "primary_mint", primary): + with patch.object(settings, "cashu_mints", [primary, secondary]): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch("routstr.lightning.get_wallet", side_effect=mock_get): + bolt11, quote_id, mint_url = await _request_mint_with_fallback( + 1000 + ) + + assert mint_url == secondary + assert bolt11 == "lnbc1secondary" + assert quote_id == "quote_secondary" + mock_primary_wallet.request_mint.assert_called_once() + mock_secondary_wallet.request_mint.assert_called_once() + + +@pytest.mark.asyncio +async def test_swap_falls_back_to_secondary_mint() -> None: + """When the primary mint is unreachable, swap_to_primary_mint falls back + to a secondary trusted mint as the swap destination.""" + from routstr.core.settings import settings + from routstr.wallet import _wallet_last_load, _wallets, swap_to_primary_mint + + _wallets.clear() + _wallet_last_load.clear() + + primary = "http://primary:3338" + secondary = "http://secondary:3338" + foreign = "http://foreign:3338" + + mock_token = Mock() + mock_token.mint = foreign + mock_token.unit = "sat" + mock_token.amount = 1000 + mock_token.keysets = ["keyset1"] + mock_token.proofs = [Mock(amount=1000)] + + mock_token_wallet = Mock() + mock_token_wallet.load_mint = AsyncMock() + mock_token_wallet.load_proofs = AsyncMock() + mock_token_wallet.get_fees_for_proofs = Mock(return_value=0) + mock_token_wallet.melt_quote = AsyncMock( + return_value=Mock(quote="melt_q", amount=990, fee_reserve=10) + ) + mock_token_wallet.melt = AsyncMock(return_value=Mock()) + + mock_primary_wallet = Mock() + mock_primary_wallet.request_mint = AsyncMock( + side_effect=httpx.ConnectError("primary down") + ) + + mint_quote = Mock(quote="mint_q_secondary", request="lnbc1secondary") + mock_secondary_wallet = Mock() + mock_secondary_wallet.load_mint = AsyncMock() + mock_secondary_wallet.load_proofs = AsyncMock() + mock_secondary_wallet.available_balance = Mock(amount=0) + mock_secondary_wallet.keysets = ["ks_secondary"] + mock_secondary_wallet.restore_tokens_for_keyset = AsyncMock() + mock_secondary_wallet.request_mint = AsyncMock(return_value=mint_quote) + mock_secondary_wallet.mint = AsyncMock(return_value=Mock()) + + wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet} + mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m]) + + with patch.object(settings, "primary_mint", primary): + with patch.object(settings, "primary_mint_unit", "sat"): + with patch.object(settings, "cashu_mints", [primary, secondary]): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch("asyncio.sleep", AsyncMock()): + with patch( + "routstr.wallet.get_wallet", side_effect=mock_get + ): + amount, unit, mint_url = ( + await swap_to_primary_mint( + mock_token, mock_token_wallet + ) + ) + + assert mint_url == secondary + assert amount == 990 # 1000 - 10 fee_reserve + assert unit == "sat" + mock_secondary_wallet.mint.assert_called_once() + mock_primary_wallet.mint.assert_not_called() + + +@pytest.mark.asyncio +async def test_lightning_mint_fallback_on_429() -> None: + """A 429 from the primary mint should trigger fallback to a secondary, + not just transport errors.""" + from routstr.core.settings import settings + from routstr.lightning import _request_mint_with_fallback + + primary = "http://primary:3338" + secondary = "http://secondary:3338" + + mock_resp = Mock(status_code=429, headers={}) + mock_resp.raise_for_status = Mock(side_effect=httpx.HTTPStatusError( + "rate limited", request=Mock(), response=mock_resp + )) + mock_primary_wallet = Mock() + mock_primary_wallet.request_mint = AsyncMock( + side_effect=httpx.HTTPStatusError("rate limited", request=Mock(), response=mock_resp) + ) + + mock_quote = Mock(request="lnbc1secondary", quote="quote_secondary") + mock_secondary_wallet = Mock() + mock_secondary_wallet.request_mint = AsyncMock(return_value=mock_quote) + + wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet} + mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m]) + + with patch.object(settings, "primary_mint", primary): + with patch.object(settings, "cashu_mints", [primary, secondary]): + with patch.object(settings, "mint_retry_max_attempts", 0): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch("routstr.lightning.get_wallet", side_effect=mock_get): + bolt11, quote_id, mint_url = await _request_mint_with_fallback(1000) + + assert mint_url == secondary + mock_secondary_wallet.request_mint.assert_called_once() + + +@pytest.mark.asyncio +async def test_lightning_mint_fallback_all_fail() -> None: + """When every trusted mint fails, _request_mint_with_fallback raises + MintConnectionError instead of trying indefinitely.""" + from routstr.core.settings import settings + from routstr.lightning import _request_mint_with_fallback + from routstr.wallet import MintConnectionError + + primary = "http://primary:3338" + secondary = "http://secondary:3338" + + mock_primary_wallet = Mock() + mock_primary_wallet.request_mint = AsyncMock(side_effect=httpx.ConnectError("down")) + mock_secondary_wallet = Mock() + mock_secondary_wallet.request_mint = AsyncMock(side_effect=httpx.ConnectError("down")) + + wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet} + mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m]) + + with patch.object(settings, "primary_mint", primary): + with patch.object(settings, "cashu_mints", [primary, secondary]): + with patch.object(settings, "mint_retry_max_attempts", 0): + with patch.object(settings, "mint_max_requests_per_minute", 0): + with patch.object(settings, "mint_operation_timeout_seconds", 0): + with patch("routstr.lightning.get_wallet", side_effect=mock_get): + with pytest.raises(MintConnectionError): + await _request_mint_with_fallback(1000)