mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-08-09 02:54:37 +00:00
fix: per-mint rate limiting, trusted-mint fallback, and retry factory fix
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
"""add mint_url to lightning_invoices
|
||||
|
||||
Revision ID: add_mint_url_li
|
||||
Revises: c6d7e8f9a0b1
|
||||
Create Date: 2026-07-10 02:00:00.000000
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "add_mint_url_li"
|
||||
down_revision = "c6d7e8f9a0b1"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("lightning_invoices", "mint_url")
|
||||
@@ -225,6 +225,9 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
default=None, description="Associated API key hash for topup operations"
|
||||
)
|
||||
purpose: str = Field(description="create or topup")
|
||||
mint_url: str | None = Field(
|
||||
default=None, description="Mint URL where the quote was created (fallback tracking)"
|
||||
)
|
||||
created_at: int = Field(
|
||||
default_factory=lambda: int(time.time()), description="Unix timestamp"
|
||||
)
|
||||
|
||||
@@ -49,6 +49,23 @@ class Settings(BaseSettings):
|
||||
payout_interval_seconds: int = Field(
|
||||
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
|
||||
)
|
||||
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
|
||||
# checkstate). When a mint is slow or rate-limiting, operations are
|
||||
# cancelled after this delay instead of hanging indefinitely.
|
||||
mint_operation_timeout_seconds: int = Field(
|
||||
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
|
||||
)
|
||||
# Maximum mint API requests per minute, per mint URL. Nutshell mints
|
||||
# (e.g. Minibits) enforce 20/min/IP on transaction endpoints (mint, melt,
|
||||
# swap, quotes) and 60/min/IP globally. 20 stays under the transaction
|
||||
# bucket since most calls here are transaction ops. 0 = unlimited.
|
||||
mint_max_requests_per_minute: int = Field(
|
||||
default=20, ge=0, env="MINT_MAX_REQUESTS_PER_MINUTE"
|
||||
)
|
||||
# Max retries when a mint returns 429 or times out (exponential backoff).
|
||||
mint_retry_max_attempts: int = Field(
|
||||
default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS"
|
||||
)
|
||||
|
||||
# Pricing
|
||||
# Default behavior: derive pricing from MODELS
|
||||
|
||||
+70
-14
@@ -11,7 +11,13 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from .core.db import ApiKey, LightningInvoice, create_session, get_session
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .wallet import get_wallet
|
||||
from .wallet import (
|
||||
MintConnectionError,
|
||||
_is_mint_rate_limited,
|
||||
_mint_operation,
|
||||
get_wallet,
|
||||
is_mint_connection_error,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -64,12 +70,44 @@ class InvoiceRecoverRequest(BaseModel):
|
||||
bolt11: str = Field(description="BOLT11 invoice string")
|
||||
|
||||
|
||||
async def _request_mint_with_fallback(
|
||||
amount_sats: int,
|
||||
) -> tuple[str, str, str]:
|
||||
"""Primary first, fall back to other trusted mints on rate-limit/transport failure."""
|
||||
tried: list[str] = []
|
||||
candidates = [settings.primary_mint] + [
|
||||
m for m in settings.cashu_mints if m != settings.primary_mint
|
||||
]
|
||||
for mint_url in candidates:
|
||||
try:
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
quote = await _mint_operation(
|
||||
lambda: wallet.request_mint(amount_sats),
|
||||
op_name="request_mint_invoice",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
return quote.request, quote.quote, mint_url
|
||||
except Exception as e:
|
||||
tried.append(f"{mint_url}: {type(e).__name__}")
|
||||
if not is_mint_connection_error(e) and not _is_mint_rate_limited(e):
|
||||
raise
|
||||
logger.warning(
|
||||
"request_mint failed, trying fallback mint",
|
||||
extra={
|
||||
"failed_mint": mint_url,
|
||||
"error": str(e),
|
||||
"tried": tried,
|
||||
},
|
||||
)
|
||||
continue
|
||||
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
|
||||
|
||||
|
||||
async def generate_lightning_invoice(
|
||||
amount_sats: int, description: str
|
||||
) -> tuple[str, str]:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
quote = await wallet.request_mint(amount_sats)
|
||||
return quote.request, quote.quote
|
||||
) -> tuple[str, str, str]:
|
||||
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(amount_sats)
|
||||
return bolt11, payment_hash, mint_url
|
||||
|
||||
|
||||
def generate_invoice_id() -> str:
|
||||
@@ -99,7 +137,7 @@ async def create_invoice(
|
||||
|
||||
try:
|
||||
description = f"Routstr {request.purpose} {request.amount_sats} sats"
|
||||
bolt11, payment_hash = await generate_lightning_invoice(
|
||||
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
|
||||
request.amount_sats, description
|
||||
)
|
||||
|
||||
@@ -115,6 +153,7 @@ async def create_invoice(
|
||||
status="pending",
|
||||
api_key_hash=api_key_token[3:] if api_key_token else None,
|
||||
purpose=request.purpose,
|
||||
mint_url=mint_url,
|
||||
balance_limit=request.balance_limit,
|
||||
balance_limit_reset=request.balance_limit_reset,
|
||||
validity_date=request.validity_date,
|
||||
@@ -223,9 +262,14 @@ async def check_invoice_payment(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
try:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
|
||||
mint_status = await wallet.get_mint_quote(invoice.payment_hash)
|
||||
mint_status = await _mint_operation(
|
||||
lambda: wallet.get_mint_quote(invoice.payment_hash),
|
||||
op_name="get_mint_quote",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
|
||||
if mint_status.paid:
|
||||
invoice.status = "paid"
|
||||
@@ -258,8 +302,13 @@ async def check_invoice_payment(
|
||||
async def create_api_key_from_invoice(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> ApiKey:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
await _mint_operation(
|
||||
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
|
||||
op_name="invoice_mint_create",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
@@ -268,7 +317,7 @@ async def create_api_key_from_invoice(
|
||||
hashed_key=hashed_key,
|
||||
balance=invoice.amount_sats * 1000, # Convert to msats
|
||||
refund_currency="sat",
|
||||
refund_mint_url=settings.primary_mint,
|
||||
refund_mint_url=mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
@@ -283,8 +332,13 @@ async def create_api_key_from_invoice(
|
||||
async def topup_api_key_from_invoice(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
await _mint_operation(
|
||||
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
|
||||
op_name="invoice_mint_topup",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
|
||||
if not invoice.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
@@ -297,7 +351,9 @@ async def topup_api_key_from_invoice(
|
||||
await session.flush()
|
||||
|
||||
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 5
|
||||
# Nutshell mints throttle Lightning backend lookups to once per 10s per
|
||||
# quote, so polling faster just burns the global request budget for nothing.
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 10
|
||||
INVOICE_WATCH_BATCH_LIMIT = 100
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from typing import TypedDict
|
||||
|
||||
import httpx
|
||||
from cashu.wallet.wallet import Proof, Wallet
|
||||
|
||||
from ..core.settings import settings
|
||||
from ..wallet import _mint_operation
|
||||
|
||||
try:
|
||||
from bech32 import bech32_decode, convertbits # type: ignore
|
||||
except ModuleNotFoundError: # pragma: no cover – allow runtime miss
|
||||
@@ -215,15 +219,23 @@ async def raw_send_to_lnurl(
|
||||
lnurl_data["callback_url"], final_amount
|
||||
)
|
||||
|
||||
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
|
||||
melt_quote_resp = await _mint_operation(
|
||||
lambda: wallet.melt_quote(invoice=bolt11_invoice),
|
||||
op_name="lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
|
||||
if amount:
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
_ = await wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
_ = await _mint_operation(
|
||||
lambda: wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
),
|
||||
op_name="lnurl_melt",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
return final_amount
|
||||
|
||||
+297
-32
@@ -16,7 +16,6 @@ from sqlmodel import col, select, update
|
||||
from .core import db, get_logger
|
||||
from .core.db import store_cashu_transaction
|
||||
from .core.settings import settings
|
||||
from .payment.lnurl import raw_send_to_lnurl
|
||||
|
||||
# cashu still declares Optional[X] without explicit defaults on MintInfo.
|
||||
# Under pydantic v2 those are required, but real mints omit many of them.
|
||||
@@ -62,6 +61,163 @@ _TRANSPORT_EXC_TYPES: tuple[type[BaseException], ...] = (
|
||||
)
|
||||
|
||||
|
||||
class _MintRateLimiter:
|
||||
"""Per-mint token-bucket rate limiter.
|
||||
|
||||
Enforces a maximum number of mint API requests per minute per mint URL.
|
||||
When the bucket is empty, callers block until a token is available.
|
||||
This prevents the node runner from being rate-limited or blocked by
|
||||
mints that enforce request quotas.
|
||||
"""
|
||||
|
||||
_limiters: dict[str, "_MintRateLimiter"] = {}
|
||||
|
||||
@classmethod
|
||||
def get(cls, mint_url: str) -> "_MintRateLimiter | None":
|
||||
rpm = settings.mint_max_requests_per_minute
|
||||
if rpm <= 0:
|
||||
return None
|
||||
if mint_url not in cls._limiters:
|
||||
cls._limiters[mint_url] = cls(mint_url, rpm)
|
||||
return cls._limiters[mint_url]
|
||||
|
||||
def __init__(self, mint_url: str, max_per_minute: int):
|
||||
self._mint_url = mint_url
|
||||
self._max = max_per_minute
|
||||
# Refill rate: tokens per second
|
||||
self._refill_rate = max_per_minute / 60.0
|
||||
self._tokens: float = float(max_per_minute)
|
||||
self._last_refill = time.monotonic()
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self) -> None:
|
||||
async with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_refill
|
||||
self._tokens = min(self._max, self._tokens + elapsed * self._refill_rate)
|
||||
self._last_refill = now
|
||||
if self._tokens < 1:
|
||||
wait = (1 - self._tokens) / self._refill_rate
|
||||
logger.debug(
|
||||
"Mint rate limiter: throttling",
|
||||
extra={
|
||||
"mint_url": self._mint_url,
|
||||
"wait_seconds": round(wait, 2),
|
||||
"tokens_available": round(self._tokens, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
self._tokens = 0
|
||||
else:
|
||||
self._tokens -= 1
|
||||
|
||||
|
||||
def _is_mint_rate_limited(error: BaseException) -> bool:
|
||||
"""True if the mint returned a 429 or rate-limit indication."""
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
if isinstance(current, httpx.HTTPStatusError):
|
||||
if current.response.status_code == 429:
|
||||
return True
|
||||
lowered = str(current).lower()
|
||||
if "rate limit" in lowered or "too many requests" in lowered:
|
||||
return True
|
||||
current = current.__cause__ or current.__context__
|
||||
return False
|
||||
|
||||
|
||||
async def _mint_operation(
|
||||
factory, *, op_name: str = "mint_operation", mint_url: str = ""
|
||||
):
|
||||
"""Wrap a mint API callable with rate limiting, timeout, and retry.
|
||||
|
||||
``factory`` must be a zero-arg callable that returns a fresh coroutine
|
||||
each call — a pre-created coroutine can only be awaited once, so on retry
|
||||
the original would be dead.
|
||||
"""
|
||||
limiter = _MintRateLimiter.get(mint_url) if mint_url else None
|
||||
timeout = settings.mint_operation_timeout_seconds
|
||||
max_attempts = settings.mint_retry_max_attempts + 1
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_attempts):
|
||||
if limiter is not None:
|
||||
await limiter.acquire()
|
||||
|
||||
try:
|
||||
if timeout > 0:
|
||||
return await asyncio.wait_for(factory(), timeout=timeout)
|
||||
return await factory()
|
||||
except asyncio.TimeoutError as exc:
|
||||
last_exc = exc
|
||||
if attempt < max_attempts - 1:
|
||||
backoff = (2 ** attempt) + (time.monotonic() % 1.0)
|
||||
logger.warning(
|
||||
"Mint operation timed out, retrying",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"backoff_seconds": round(backoff, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
raise httpx.TimeoutException(
|
||||
f"{op_name} timed out after {timeout}s (retried {attempt + 1}x)"
|
||||
) from exc
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if _is_mint_rate_limited(exc) and attempt < max_attempts - 1:
|
||||
backoff = (2 ** attempt) + (time.monotonic() % 1.0)
|
||||
retry_after = _parse_retry_after(exc.response.headers)
|
||||
if retry_after is not None:
|
||||
backoff = min(retry_after, backoff * 2)
|
||||
logger.warning(
|
||||
"Mint returned 429, backing off",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"backoff_seconds": round(backoff, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
raise
|
||||
except Exception as exc:
|
||||
if _is_mint_rate_limited(exc) and attempt < max_attempts - 1:
|
||||
backoff = (2 ** attempt) + (time.monotonic() % 1.0)
|
||||
logger.warning(
|
||||
"Mint rate-limited, backing off",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"backoff_seconds": round(backoff, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
raise
|
||||
|
||||
if last_exc:
|
||||
raise last_exc
|
||||
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
|
||||
|
||||
|
||||
def _parse_retry_after(headers) -> float | None:
|
||||
"""Parse a Retry-After header (delta-seconds form) into seconds."""
|
||||
raw = headers.get("retry-after") or headers.get("Retry-After")
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def is_mint_connection_error(error: BaseException) -> bool:
|
||||
"""True if ``error`` (or anything in its cause/context chain) is a mint
|
||||
transport failure. Walks the chain because some sites re-raise transport
|
||||
@@ -192,10 +348,18 @@ async def _redeem_same_mint(
|
||||
that, not the face value, or routstr over-credits the user and its wallet
|
||||
drifts insolvent.
|
||||
"""
|
||||
await wallet.load_mint(keyset_id=token_obj.keysets[0])
|
||||
await _mint_operation(
|
||||
lambda: wallet.load_mint(keyset_id=token_obj.keysets[0]),
|
||||
op_name="redeem_load_mint",
|
||||
mint_url=token_obj.mint,
|
||||
)
|
||||
wallet.verify_proofs_dleq(token_obj.proofs)
|
||||
input_fees = wallet.get_fees_for_proofs(token_obj.proofs)
|
||||
await wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True)
|
||||
await _mint_operation(
|
||||
lambda: wallet.split(proofs=token_obj.proofs, amount=0, include_fees=True),
|
||||
op_name="redeem_split",
|
||||
mint_url=token_obj.mint,
|
||||
)
|
||||
return int(token_obj.amount) - input_fees, token_obj.unit, token_obj.mint
|
||||
|
||||
|
||||
@@ -341,6 +505,44 @@ def _melt_insufficient_shortfall(error: Exception) -> int | None:
|
||||
return 1
|
||||
|
||||
|
||||
async def _request_mint_with_fallback(
|
||||
amount: int, *, op_name: str, primary_wallet: Wallet | None = None
|
||||
) -> tuple[Wallet, str, object]:
|
||||
"""Try request_mint on the primary mint, fall back to other trusted mints
|
||||
on transport or rate-limit failure. Returns the wallet, mint_url, and quote."""
|
||||
candidates = [settings.primary_mint] + [
|
||||
m for m in settings.cashu_mints if m != settings.primary_mint
|
||||
]
|
||||
tried: list[str] = []
|
||||
for mint_url in candidates:
|
||||
try:
|
||||
if mint_url == settings.primary_mint and primary_wallet is not None:
|
||||
wallet = primary_wallet
|
||||
else:
|
||||
wallet = await get_wallet(mint_url, settings.primary_mint_unit)
|
||||
quote = await _mint_operation(
|
||||
lambda: wallet.request_mint(amount),
|
||||
op_name=op_name,
|
||||
mint_url=mint_url,
|
||||
)
|
||||
return wallet, mint_url, quote
|
||||
except Exception as e:
|
||||
tried.append(f"{mint_url}: {type(e).__name__}")
|
||||
if not is_mint_connection_error(e) and not _is_mint_rate_limited(e):
|
||||
raise
|
||||
logger.warning(
|
||||
"request_mint failed, trying fallback mint",
|
||||
extra={
|
||||
"failed_mint": mint_url,
|
||||
"error": str(e),
|
||||
"tried": tried,
|
||||
"op_name": op_name,
|
||||
},
|
||||
)
|
||||
continue
|
||||
raise MintConnectionError(f"All mints failed for {op_name}: {tried}")
|
||||
|
||||
|
||||
async def _calculate_swap_amount(
|
||||
amount_msat: int,
|
||||
token_unit: str,
|
||||
@@ -374,8 +576,16 @@ async def _calculate_swap_amount(
|
||||
)
|
||||
|
||||
try:
|
||||
dummy_mint_quote = await primary_wallet.request_mint(receive_amount)
|
||||
dummy_melt_quote = await token_wallet.melt_quote(dummy_mint_quote.request)
|
||||
_, _, dummy_mint_quote = await _request_mint_with_fallback(
|
||||
receive_amount,
|
||||
op_name="swap_fee_est_mint_quote",
|
||||
primary_wallet=primary_wallet,
|
||||
)
|
||||
dummy_melt_quote = await _mint_operation(
|
||||
lambda: token_wallet.melt_quote(dummy_mint_quote.request),
|
||||
op_name="swap_fee_est_melt_quote",
|
||||
mint_url=token_mint_url,
|
||||
)
|
||||
|
||||
fee_reserve = dummy_melt_quote.fee_reserve
|
||||
input_fees = token_wallet.get_fees_for_proofs(proofs)
|
||||
@@ -462,15 +672,23 @@ async def swap_to_primary_mint(
|
||||
# amount recomputed from the fees the mint actually demands.
|
||||
observed_extra_fee = 0
|
||||
attempt = 0
|
||||
dest_wallet = primary_wallet
|
||||
dest_mint_url = settings.primary_mint
|
||||
while True:
|
||||
attempt += 1
|
||||
mint_quote = await primary_wallet.request_mint(minted_amount)
|
||||
dest_wallet, dest_mint_url, mint_quote = await _request_mint_with_fallback(
|
||||
minted_amount, op_name="swap_request_mint", primary_wallet=primary_wallet
|
||||
)
|
||||
logger.info(
|
||||
"swap_to_primary_mint: mint quote received",
|
||||
extra={"mint_quote_id": mint_quote.quote, "attempt": attempt},
|
||||
extra={"mint_quote_id": mint_quote.quote, "attempt": attempt, "dest_mint": dest_mint_url},
|
||||
)
|
||||
|
||||
melt_quote = await token_wallet.melt_quote(mint_quote.request)
|
||||
melt_quote = await _mint_operation(
|
||||
lambda: token_wallet.melt_quote(mint_quote.request),
|
||||
op_name="swap_melt_quote",
|
||||
mint_url=token_obj.mint,
|
||||
)
|
||||
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
|
||||
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
|
||||
logger.info(
|
||||
@@ -523,11 +741,15 @@ async def swap_to_primary_mint(
|
||||
continue
|
||||
|
||||
try:
|
||||
_ = await token_wallet.melt(
|
||||
proofs=token_obj.proofs,
|
||||
invoice=mint_quote.request,
|
||||
fee_reserve_sat=melt_quote.fee_reserve,
|
||||
quote_id=melt_quote.quote,
|
||||
_ = await _mint_operation(
|
||||
lambda: token_wallet.melt(
|
||||
proofs=token_obj.proofs,
|
||||
invoice=mint_quote.request,
|
||||
fee_reserve_sat=melt_quote.fee_reserve,
|
||||
quote_id=melt_quote.quote,
|
||||
),
|
||||
op_name="swap_melt",
|
||||
mint_url=token_obj.mint,
|
||||
)
|
||||
except Exception as e:
|
||||
# A down mint won't fix itself by retrying with a smaller amount.
|
||||
@@ -576,14 +798,18 @@ async def swap_to_primary_mint(
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"swap_to_primary_mint: melt succeeded, minting on primary",
|
||||
extra={"minted_amount": minted_amount, "mint_quote_id": mint_quote.quote},
|
||||
"swap_to_primary_mint: melt succeeded, minting on destination",
|
||||
extra={"minted_amount": minted_amount, "mint_quote_id": mint_quote.quote, "dest_mint": dest_mint_url},
|
||||
)
|
||||
|
||||
await primary_wallet.load_proofs(reload=True)
|
||||
pre_mint_balance = primary_wallet.available_balance.amount
|
||||
await dest_wallet.load_proofs(reload=True)
|
||||
pre_mint_balance = dest_wallet.available_balance.amount
|
||||
try:
|
||||
_ = await primary_wallet.mint(minted_amount, quote_id=mint_quote.quote)
|
||||
_ = await _mint_operation(
|
||||
lambda: dest_wallet.mint(minted_amount, quote_id=mint_quote.quote),
|
||||
op_name="swap_mint_on_primary",
|
||||
mint_url=dest_mint_url,
|
||||
)
|
||||
except Exception as e:
|
||||
if "11003" in str(e) or "outputs already signed" in str(e).lower():
|
||||
# Previous mint call signed outputs at the mint but failed before
|
||||
@@ -594,10 +820,10 @@ async def swap_to_primary_mint(
|
||||
extra={"mint_quote_id": mint_quote.quote, "minted_amount": minted_amount},
|
||||
)
|
||||
try:
|
||||
for keyset_id in primary_wallet.keysets:
|
||||
await primary_wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await primary_wallet.load_proofs(reload=True)
|
||||
post_recovery_balance = primary_wallet.available_balance.amount
|
||||
for keyset_id in dest_wallet.keysets:
|
||||
await dest_wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await dest_wallet.load_proofs(reload=True)
|
||||
post_recovery_balance = dest_wallet.available_balance.amount
|
||||
balance_gained = post_recovery_balance - pre_mint_balance
|
||||
logger.info(
|
||||
"swap_to_primary_mint: recovery scan completed",
|
||||
@@ -648,14 +874,14 @@ async def swap_to_primary_mint(
|
||||
"swap_to_primary_mint: completed successfully",
|
||||
extra={
|
||||
"foreign_mint": token_obj.mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"dest_mint": dest_mint_url,
|
||||
"original_amount": token_amount,
|
||||
"minted_amount": minted_amount,
|
||||
"unit": settings.primary_mint_unit,
|
||||
},
|
||||
)
|
||||
|
||||
return int(minted_amount), settings.primary_mint_unit, settings.primary_mint
|
||||
return int(minted_amount), settings.primary_mint_unit, dest_mint_url
|
||||
|
||||
|
||||
async def credit_balance(
|
||||
@@ -760,17 +986,35 @@ async def credit_balance(
|
||||
|
||||
|
||||
_wallets: dict[str, Wallet] = {}
|
||||
_wallet_last_load: dict[str, float] = {}
|
||||
# Minimum seconds between full mint info + proof reloads for the same
|
||||
# wallet. Prevents redundant mint API calls when get_wallet(load=True)
|
||||
# is called rapidly by multiple background tasks (balance fetch, payout,
|
||||
# auto-topup all hitting get_wallet within the same cycle).
|
||||
_WALLOAD_RELOAD_MIN_INTERVAL_SECONDS = 30
|
||||
|
||||
|
||||
async def get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Wallet:
|
||||
global _wallets
|
||||
global _wallets, _wallet_last_load
|
||||
id = f"{mint_url}_{unit}"
|
||||
if id not in _wallets:
|
||||
_wallets[id] = await Wallet.with_db(mint_url, db=".wallet", unit=unit)
|
||||
|
||||
if load:
|
||||
await _wallets[id].load_mint()
|
||||
await _wallets[id].load_proofs(reload=True)
|
||||
now = time.monotonic()
|
||||
last = _wallet_last_load.get(id, 0)
|
||||
if now - last >= _WALLOAD_RELOAD_MIN_INTERVAL_SECONDS:
|
||||
await _mint_operation(
|
||||
lambda: _wallets[id].load_mint(),
|
||||
op_name="load_mint",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
await _mint_operation(
|
||||
lambda: _wallets[id].load_proofs(reload=True),
|
||||
op_name="load_proofs",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
_wallet_last_load[id] = now
|
||||
return _wallets[id]
|
||||
|
||||
|
||||
@@ -788,20 +1032,35 @@ def get_proofs_per_mint_and_unit(
|
||||
return proofs
|
||||
|
||||
|
||||
async def slow_filter_spend_proofs(proofs: list[Proof], wallet: Wallet) -> list[Proof]:
|
||||
async def slow_filter_spend_proofs(
|
||||
proofs: list[Proof], wallet: Wallet
|
||||
) -> list[Proof]:
|
||||
if not proofs:
|
||||
return []
|
||||
_proofs = []
|
||||
_spent_proofs = []
|
||||
for i in range(0, len(proofs), 1000):
|
||||
pb = proofs[i : i + 1000]
|
||||
proof_states = await wallet.check_proof_state(pb)
|
||||
# Smaller batch size to reduce per-request load on the mint.
|
||||
# 1000 proofs per batch was too aggressive and triggered rate limits
|
||||
# on mints with strict request quotas.
|
||||
batch_size = 100
|
||||
for i in range(0, len(proofs), batch_size):
|
||||
pb = proofs[i : i + batch_size]
|
||||
proof_states = await _mint_operation(
|
||||
lambda: wallet.check_proof_state(pb),
|
||||
op_name="check_proof_state",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
for proof, state in zip(pb, proof_states.states):
|
||||
if str(state.state) != "spent":
|
||||
_proofs.append(proof)
|
||||
else:
|
||||
_spent_proofs.append(proof)
|
||||
await wallet.set_reserved_for_send(_spent_proofs, reserved=True)
|
||||
if _spent_proofs:
|
||||
await _mint_operation(
|
||||
lambda: wallet.set_reserved_for_send(_spent_proofs, reserved=True),
|
||||
op_name="set_reserved_spent_proofs",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
return _proofs
|
||||
|
||||
|
||||
@@ -923,6 +1182,8 @@ async def periodic_payout() -> None:
|
||||
if not settings.receive_ln_address:
|
||||
continue
|
||||
try:
|
||||
from .payment.lnurl import raw_send_to_lnurl
|
||||
|
||||
async with db.create_session() as session:
|
||||
for mint_url in settings.cashu_mints:
|
||||
for unit in ["sat", "msat"]:
|
||||
@@ -1037,6 +1298,8 @@ async def periodic_routstr_fee_payout() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS)
|
||||
try:
|
||||
from .payment.lnurl import raw_send_to_lnurl
|
||||
|
||||
async with db.create_session() as session:
|
||||
fee = await db.get_routstr_fee(session)
|
||||
accumulated_sats = fee.accumulated_msats // 1000
|
||||
@@ -1065,6 +1328,8 @@ async def periodic_routstr_fee_payout() -> None:
|
||||
|
||||
|
||||
async def send_to_lnurl(amount: int, unit: str, mint: str, address: str) -> int:
|
||||
from .payment.lnurl import raw_send_to_lnurl
|
||||
|
||||
wallet = await get_wallet(mint, unit)
|
||||
proofs = wallet._get_proofs_per_keyset(wallet.proofs)[wallet.keyset_id]
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
@@ -26,11 +26,12 @@ async def patch_invoice_generation() -> Any:
|
||||
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
|
||||
counter = {"n": 0}
|
||||
|
||||
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
|
||||
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str, str]:
|
||||
counter["n"] += 1
|
||||
return (
|
||||
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
|
||||
f"payment_hash_{counter['n']}",
|
||||
"http://localhost:3338",
|
||||
)
|
||||
|
||||
with patch(
|
||||
|
||||
@@ -1321,3 +1321,244 @@ async def test_swap_melt_transport_error_raises_mint_connection_error() -> None:
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
assert mock_token_wallet.melt.call_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _mint_operation factory + retry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mint_operation_factory_retry_succeeds() -> None:
|
||||
"""_mint_operation accepts a zero-arg factory, not a dead coroutine.
|
||||
A factory that raises twice then succeeds must be retried and return."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _mint_operation
|
||||
|
||||
calls = 0
|
||||
|
||||
async def factory():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls < 3:
|
||||
raise TimeoutError("timeout")
|
||||
return "ok"
|
||||
|
||||
with patch.object(settings, "mint_retry_max_attempts", 3):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
result = await _mint_operation(
|
||||
factory, op_name="test_retry", mint_url="http://mint:3338"
|
||||
)
|
||||
|
||||
assert calls == 3
|
||||
assert result == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mint_operation_factory_retry_exhausted() -> None:
|
||||
"""When the factory always times out, _mint_operation raises
|
||||
httpx.TimeoutException after mint_retry_max_attempts + 1 attempts."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _mint_operation
|
||||
|
||||
calls = 0
|
||||
|
||||
async def factory():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise TimeoutError("always timeout")
|
||||
|
||||
with patch.object(settings, "mint_retry_max_attempts", 2):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
await _mint_operation(
|
||||
factory, op_name="test_exhaust", mint_url="http://mint:3338"
|
||||
)
|
||||
|
||||
assert calls == 3 # max_attempts(2) + 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trusted-mint fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lightning_mint_fallback_for_topups() -> None:
|
||||
"""When the primary mint is unreachable, _request_mint_with_fallback
|
||||
falls back to a secondary trusted mint."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.lightning import _request_mint_with_fallback
|
||||
|
||||
primary = "http://primary:3338"
|
||||
secondary = "http://secondary:3338"
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=httpx.ConnectError("primary down")
|
||||
)
|
||||
|
||||
mock_quote = Mock()
|
||||
mock_quote.request = "lnbc1secondary"
|
||||
mock_quote.quote = "quote_secondary"
|
||||
mock_secondary_wallet = Mock()
|
||||
mock_secondary_wallet.request_mint = AsyncMock(return_value=mock_quote)
|
||||
|
||||
wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet}
|
||||
mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m])
|
||||
|
||||
with patch.object(settings, "primary_mint", primary):
|
||||
with patch.object(settings, "cashu_mints", [primary, secondary]):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch("routstr.lightning.get_wallet", side_effect=mock_get):
|
||||
bolt11, quote_id, mint_url = await _request_mint_with_fallback(
|
||||
1000
|
||||
)
|
||||
|
||||
assert mint_url == secondary
|
||||
assert bolt11 == "lnbc1secondary"
|
||||
assert quote_id == "quote_secondary"
|
||||
mock_primary_wallet.request_mint.assert_called_once()
|
||||
mock_secondary_wallet.request_mint.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_falls_back_to_secondary_mint() -> None:
|
||||
"""When the primary mint is unreachable, swap_to_primary_mint falls back
|
||||
to a secondary trusted mint as the swap destination."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _wallet_last_load, _wallets, swap_to_primary_mint
|
||||
|
||||
_wallets.clear()
|
||||
_wallet_last_load.clear()
|
||||
|
||||
primary = "http://primary:3338"
|
||||
secondary = "http://secondary:3338"
|
||||
foreign = "http://foreign:3338"
|
||||
|
||||
mock_token = Mock()
|
||||
mock_token.mint = foreign
|
||||
mock_token.unit = "sat"
|
||||
mock_token.amount = 1000
|
||||
mock_token.keysets = ["keyset1"]
|
||||
mock_token.proofs = [Mock(amount=1000)]
|
||||
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
mock_token_wallet.melt_quote = AsyncMock(
|
||||
return_value=Mock(quote="melt_q", amount=990, fee_reserve=10)
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(return_value=Mock())
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=httpx.ConnectError("primary down")
|
||||
)
|
||||
|
||||
mint_quote = Mock(quote="mint_q_secondary", request="lnbc1secondary")
|
||||
mock_secondary_wallet = Mock()
|
||||
mock_secondary_wallet.load_mint = AsyncMock()
|
||||
mock_secondary_wallet.load_proofs = AsyncMock()
|
||||
mock_secondary_wallet.available_balance = Mock(amount=0)
|
||||
mock_secondary_wallet.keysets = ["ks_secondary"]
|
||||
mock_secondary_wallet.restore_tokens_for_keyset = AsyncMock()
|
||||
mock_secondary_wallet.request_mint = AsyncMock(return_value=mint_quote)
|
||||
mock_secondary_wallet.mint = AsyncMock(return_value=Mock())
|
||||
|
||||
wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet}
|
||||
mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m])
|
||||
|
||||
with patch.object(settings, "primary_mint", primary):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch.object(settings, "cashu_mints", [primary, secondary]):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
with patch(
|
||||
"routstr.wallet.get_wallet", side_effect=mock_get
|
||||
):
|
||||
amount, unit, mint_url = (
|
||||
await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
)
|
||||
|
||||
assert mint_url == secondary
|
||||
assert amount == 990 # 1000 - 10 fee_reserve
|
||||
assert unit == "sat"
|
||||
mock_secondary_wallet.mint.assert_called_once()
|
||||
mock_primary_wallet.mint.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lightning_mint_fallback_on_429() -> None:
|
||||
"""A 429 from the primary mint should trigger fallback to a secondary,
|
||||
not just transport errors."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.lightning import _request_mint_with_fallback
|
||||
|
||||
primary = "http://primary:3338"
|
||||
secondary = "http://secondary:3338"
|
||||
|
||||
mock_resp = Mock(status_code=429, headers={})
|
||||
mock_resp.raise_for_status = Mock(side_effect=httpx.HTTPStatusError(
|
||||
"rate limited", request=Mock(), response=mock_resp
|
||||
))
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError("rate limited", request=Mock(), response=mock_resp)
|
||||
)
|
||||
|
||||
mock_quote = Mock(request="lnbc1secondary", quote="quote_secondary")
|
||||
mock_secondary_wallet = Mock()
|
||||
mock_secondary_wallet.request_mint = AsyncMock(return_value=mock_quote)
|
||||
|
||||
wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet}
|
||||
mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m])
|
||||
|
||||
with patch.object(settings, "primary_mint", primary):
|
||||
with patch.object(settings, "cashu_mints", [primary, secondary]):
|
||||
with patch.object(settings, "mint_retry_max_attempts", 0):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch("routstr.lightning.get_wallet", side_effect=mock_get):
|
||||
bolt11, quote_id, mint_url = await _request_mint_with_fallback(1000)
|
||||
|
||||
assert mint_url == secondary
|
||||
mock_secondary_wallet.request_mint.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lightning_mint_fallback_all_fail() -> None:
|
||||
"""When every trusted mint fails, _request_mint_with_fallback raises
|
||||
MintConnectionError instead of trying indefinitely."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.lightning import _request_mint_with_fallback
|
||||
from routstr.wallet import MintConnectionError
|
||||
|
||||
primary = "http://primary:3338"
|
||||
secondary = "http://secondary:3338"
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.request_mint = AsyncMock(side_effect=httpx.ConnectError("down"))
|
||||
mock_secondary_wallet = Mock()
|
||||
mock_secondary_wallet.request_mint = AsyncMock(side_effect=httpx.ConnectError("down"))
|
||||
|
||||
wallets_map = {primary: mock_primary_wallet, secondary: mock_secondary_wallet}
|
||||
mock_get = AsyncMock(side_effect=lambda m, *a, **kw: wallets_map[m])
|
||||
|
||||
with patch.object(settings, "primary_mint", primary):
|
||||
with patch.object(settings, "cashu_mints", [primary, secondary]):
|
||||
with patch.object(settings, "mint_retry_max_attempts", 0):
|
||||
with patch.object(settings, "mint_max_requests_per_minute", 0):
|
||||
with patch.object(settings, "mint_operation_timeout_seconds", 0):
|
||||
with patch("routstr.lightning.get_wallet", side_effect=mock_get):
|
||||
with pytest.raises(MintConnectionError):
|
||||
await _request_mint_with_fallback(1000)
|
||||
|
||||
Reference in New Issue
Block a user