Compare commits

...
Author SHA1 Message Date
9qeklajc d97076ae13 fix: reconcile uncertain payout timeouts 2026-07-12 14:59:19 +02:00
4 changed files with 237 additions and 43 deletions
+61 -5
View File
@@ -5,12 +5,14 @@ import math
from typing import TypedDict
import httpx
from cashu.core.base import MeltQuoteState
from cashu.wallet.wallet import Proof, Wallet
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
# very slow mint can block a melt (and any caller, e.g. the payout loop)
# indefinitely. Bound it here so callers fail instead of hanging forever.
# indefinitely. Bound both the melt and the follow-up quote reconciliation.
MELT_TIMEOUT_SECONDS = 60
MELT_RECONCILE_TIMEOUT_SECONDS = 10
try:
from bech32 import bech32_decode, convertbits # type: ignore
@@ -31,6 +33,60 @@ class LNURLError(Exception):
"""LNURL related errors."""
class MeltOutcomeUncertainError(LNURLError):
"""A timed-out melt whose final state could not be established safely."""
def __init__(self, quote_id: str, reason: str) -> None:
self.quote_id = quote_id
super().__init__(f"Melt outcome uncertain for quote {quote_id}: {reason}")
async def _reconcile_timed_out_melt(
wallet: Wallet, proofs: list[Proof], quote_id: str
) -> None:
"""Resolve a timed-out melt without making reserved proofs spendable early."""
try:
quote = await asyncio.wait_for(
wallet.get_melt_quote(quote_id),
timeout=MELT_RECONCILE_TIMEOUT_SECONDS,
)
except Exception as error:
raise MeltOutcomeUncertainError(
quote_id, f"quote reconciliation failed: {type(error).__name__}: {error}"
) from error
if quote is None:
raise MeltOutcomeUncertainError(quote_id, "mint returned no quote")
if quote.state == MeltQuoteState.paid:
try:
# get_melt_quote updates the wallet database; explicitly invalidate
# the known-spent inputs so in-memory state cannot expose them.
await wallet.invalidate(proofs)
await wallet.load_proofs(reload=True)
except Exception as error:
raise MeltOutcomeUncertainError(
quote_id,
f"local reconciliation failed: {type(error).__name__}: {error}",
) from error
return
if quote.state == MeltQuoteState.unpaid:
try:
await wallet.set_reserved_for_melt(
proofs, reserved=False, quote_id=None
)
await wallet.load_proofs(reload=True)
except Exception as error:
raise MeltOutcomeUncertainError(
quote_id,
f"local reconciliation failed: {type(error).__name__}: {error}",
) from error
raise LNURLError(f"Melt timed out and quote {quote_id} is unpaid")
raise MeltOutcomeUncertainError(quote_id, f"mint reports {quote.state.value}")
async def decode_lnurl(lnurl: str) -> str:
"""Decode LNURL to get the actual URL.
@@ -236,8 +292,8 @@ async def raw_send_to_lnurl(
),
timeout=MELT_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError as e:
raise LNURLError(
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
) from e
except asyncio.TimeoutError:
await _reconcile_timed_out_melt(
wallet, proofs, melt_quote_resp.quote
)
return final_amount
+12 -1
View File
@@ -16,7 +16,7 @@ from sqlmodel import col, select, update
from .core import db, get_logger
from .core.db import store_cashu_transaction
from .core.settings import settings
from .payment.lnurl import raw_send_to_lnurl
from .payment.lnurl import MeltOutcomeUncertainError, raw_send_to_lnurl
# cashu still declares Optional[X] without explicit defaults on MintInfo.
# Under pydantic v2 those are required, but real mints omit many of them.
@@ -972,6 +972,17 @@ async def periodic_payout() -> None:
"amount_received": amount_received,
},
)
except MeltOutcomeUncertainError as e:
logger.error(
"Payout melt outcome uncertain",
extra={
"error": str(e),
"quote_id": e.quote_id,
"mint_url": mint_url,
"unit": unit,
"address": settings.receive_ln_address,
},
)
except Exception as e:
logger.error(
f"Error sending payout: {type(e).__name__}",
+118 -37
View File
@@ -1,74 +1,155 @@
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
timeout as LNURLError instead of hanging.
"""
"""Timeout reconciliation for LNURL melts."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from cashu.core.base import MeltQuoteState
from routstr.payment import lnurl
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
from routstr.payment.lnurl import (
LNURLError,
MeltOutcomeUncertainError,
raw_send_to_lnurl,
)
PROOFS = [MagicMock(amount=1000)]
LNURL_DATA = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
proofs = [MagicMock(amount=1000)]
def _wallet(state: MeltQuoteState | None) -> MagicMock:
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
wallet.select_to_send = AsyncMock(return_value=(PROOFS, None))
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5) # far longer than the patched timeout
await asyncio.sleep(5)
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=None if state is None else MagicMock(state=state)
)
wallet.invalidate = AsyncMock()
wallet.set_reserved_for_melt = AsyncMock()
wallet.load_proofs = AsyncMock()
return wallet
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
async def _send(wallet: MagicMock) -> int:
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.01), patch.object(
lnurl, "MELT_RECONCILE_TIMEOUT_SECONDS", 0.01
), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=LNURL_DATA)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
):
with pytest.raises(LNURLError, match="Melt timed out"):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
return await raw_send_to_lnurl(
wallet, PROOFS, "owner@ln.tld", "sat", amount=1000
)
@pytest.mark.asyncio
async def test_timeout_paid_quote_invalidates_proofs_and_reloads() -> None:
wallet = _wallet(MeltQuoteState.paid)
paid = await _send(wallet)
assert paid > 0
wallet.get_melt_quote.assert_awaited_once_with("q")
wallet.invalidate.assert_awaited_once_with(PROOFS)
wallet.load_proofs.assert_awaited_once_with(reload=True)
wallet.set_reserved_for_melt.assert_not_awaited()
@pytest.mark.asyncio
async def test_timeout_unpaid_quote_releases_proofs_and_reloads() -> None:
wallet = _wallet(MeltQuoteState.unpaid)
with pytest.raises(LNURLError, match="quote q is unpaid"):
await _send(wallet)
wallet.set_reserved_for_melt.assert_awaited_once_with(
PROOFS, reserved=False, quote_id=None
)
wallet.load_proofs.assert_awaited_once_with(reload=True)
wallet.invalidate.assert_not_awaited()
@pytest.mark.asyncio
async def test_timeout_pending_quote_preserves_reservation() -> None:
wallet = _wallet(MeltQuoteState.pending)
with pytest.raises(MeltOutcomeUncertainError, match="mint reports PENDING"):
await _send(wallet)
wallet.invalidate.assert_not_awaited()
wallet.set_reserved_for_melt.assert_not_awaited()
wallet.load_proofs.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("quote_result", "match"),
[
(None, "mint returned no quote"),
(RuntimeError("mint unavailable"), "quote reconciliation failed"),
],
)
async def test_reconciliation_failure_preserves_reservation(
quote_result: Any, match: str
) -> None:
wallet = _wallet(None)
if isinstance(quote_result, Exception):
wallet.get_melt_quote.side_effect = quote_result
else:
wallet.get_melt_quote.return_value = quote_result
with pytest.raises(MeltOutcomeUncertainError, match=match):
await _send(wallet)
wallet.invalidate.assert_not_awaited()
wallet.set_reserved_for_melt.assert_not_awaited()
wallet.load_proofs.assert_not_awaited()
@pytest.mark.asyncio
async def test_reconciliation_query_has_its_own_timeout() -> None:
wallet = _wallet(None)
async def _hang_quote(quote_id: str) -> None:
await asyncio.sleep(5)
wallet.get_melt_quote.side_effect = _hang_quote
with pytest.raises(MeltOutcomeUncertainError, match="quote reconciliation failed"):
await _send(wallet)
wallet.invalidate.assert_not_awaited()
wallet.set_reserved_for_melt.assert_not_awaited()
wallet.load_proofs.assert_not_awaited()
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
"""A prompt melt still returns the net amount, unaffected by the guard."""
proofs = [MagicMock(amount=1000)]
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
wallet = _wallet(None)
wallet.melt = AsyncMock(return_value=MagicMock())
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=LNURL_DATA)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
wallet, PROOFS, "owner@ln.tld", "sat", amount=1000
)
assert paid > 0
wallet.melt.assert_awaited_once()
wallet.get_melt_quote.assert_not_awaited()
+46
View File
@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from routstr.payment.lnurl import MeltOutcomeUncertainError
from routstr.wallet import periodic_payout
# Sentinel interval used to break the otherwise-infinite payout loop after
@@ -152,3 +153,48 @@ async def test_periodic_payout_handles_session_creation_failure() -> None:
extra = logger.error.call_args.kwargs["extra"]
assert message == "Error in periodic payout cycle: RuntimeError"
assert extra == {"error": "db unavailable"}
@pytest.mark.asyncio
async def test_periodic_payout_logs_uncertain_melt_with_full_context() -> None:
"""Operators can identify and reconcile a payout with an unknown outcome."""
from routstr.core.settings import settings
logger = MagicMock()
uncertain = MeltOutcomeUncertainError("quote-123", "mint reports PENDING")
with patch.object(settings, "cashu_mints", ["http://mint:3338"]), patch.object(
settings, "primary_mint", "http://mint:3338"
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
settings, "payout_interval_seconds", _INTERVAL
), patch.object(settings, "min_payout_sat", 10), patch(
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
), patch("routstr.wallet.db.create_session", _fake_session), patch(
"routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())
), patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
), patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
), patch(
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
), patch(
"routstr.wallet.raw_send_to_lnurl", AsyncMock(side_effect=uncertain)
), patch("routstr.wallet.logger", logger):
with pytest.raises(_LoopBreak):
await periodic_payout()
uncertain_logs = [
call
for call in logger.error.call_args_list
if call.args[0] == "Payout melt outcome uncertain"
]
assert uncertain_logs
assert uncertain_logs[0].kwargs["extra"] == {
"error": str(uncertain),
"quote_id": "quote-123",
"mint_url": "http://mint:3338",
"unit": "sat",
"address": "owner@ln.tld",
}