From d3426a68c008b0527e76f2b008c22283e45162d2 Mon Sep 17 00:00:00 2001 From: Paperclip Deployment Engineer Date: Wed, 15 Jul 2026 15:56:33 +0000 Subject: [PATCH] better fallback handling --- src/daemon/wallet/index.ts | 140 +++++++++++++++++++------ src/daemon/wallet/sdk-mint-fallback.ts | 115 ++++++++++++++++++++ tests/mint-fallback.test.ts | 86 +++++++++++++++ tests/sdk-mint-fallback.test.ts | 77 +++++++++++++- tests/wallet-send-fallback.test.ts | 51 +++++++++ 5 files changed, 436 insertions(+), 33 deletions(-) create mode 100644 tests/wallet-send-fallback.test.ts diff --git a/src/daemon/wallet/index.ts b/src/daemon/wallet/index.ts index 10c5bdc..caac7c1 100644 --- a/src/daemon/wallet/index.ts +++ b/src/daemon/wallet/index.ts @@ -1,5 +1,4 @@ import { getDecodedToken, Amount } from "@cashu/cashu-ts"; -import { InsufficientBalanceError } from "@routstr/sdk"; import { WalletConnect } from "applesauce-wallet-connect"; import { RelayPool } from "applesauce-relay"; import { logger } from "../../utils/logger"; @@ -18,6 +17,100 @@ export function decodeCashuTokenAmount(token: string): { return { amount, unit }; } +type SendCashuOptions = { + maxRetries?: number; + retryDelayMs?: number; + minimumAmountSats?: number; + fallbackAmounts?: number[]; +}; + +const MIN_PROVIDER_TOKEN_AMOUNT_SATS = 2; + +export function getSameMintSendAmounts( + amount: number, + availableBalance: number, +): number[] { + const requestedAmount = Math.max( + Math.ceil(amount), + MIN_PROVIDER_TOKEN_AMOUNT_SATS, + ); + const available = Math.floor(availableBalance); + const candidates = [requestedAmount]; + + if (available <= requestedAmount) return candidates; + + let denomination = 2 ** Math.ceil(Math.log2(requestedAmount)); + while (denomination <= available) { + if (!candidates.includes(denomination)) candidates.push(denomination); + denomination *= 2; + } + + // The entire ready balance is always worth trying last: even when no single + // power-of-two proof exists, all ready proofs together may be selectable. + if (!candidates.includes(available)) candidates.push(available); + return candidates; +} + +/** + * Create a token from exactly the requested mint. + * + * The SDK associates the returned token with `mintUrl`. Falling back to another + * mint here makes that association false and can send providers a token from a + * mint they cannot reach. Cross-mint fallback belongs above this adapter, where + * the actual mint URL remains part of the request state. + */ +export async function sendCashuFromMint( + client: Pick, + mintUrl: string, + amount: number, + options: SendCashuOptions = {}, +): Promise { + const maxRetries = options.maxRetries ?? 3; + const retryDelayMs = options.retryDelayMs ?? 5000; + const minimumAmountSats = options.minimumAmountSats ?? MIN_PROVIDER_TOKEN_AMOUNT_SATS; + const sendAmounts = [ + Math.max(amount, minimumAmountSats), + ...(options.fallbackAmounts ?? []), + ].filter((candidate, index, all) => candidate > 0 && all.indexOf(candidate) === index); + const retryErrorPattern = "Proof already reserved by operation"; + let lastInsufficientProofsError: unknown; + + for (const [amountIndex, sendAmount] of sendAmounts.entries()) { + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await client.sendCashu(sendAmount, mintUrl); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const shouldRetry = + attempt < maxRetries && errorMessage.includes(retryErrorPattern); + + if (shouldRetry) { + logger.log( + `sendToken attempt ${attempt + 1} failed with reserved proof error for ${mintUrl}, retrying in ${retryDelayMs / 1000}s...`, + ); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + continue; + } + + if ( + errorMessage.includes("Not enough proofs") && + amountIndex + 1 < sendAmounts.length + ) { + lastInsufficientProofsError = error; + logger.warn( + `sendToken: ${mintUrl} cannot compose ${sendAmount} sats; trying ${sendAmounts[amountIndex + 1]} sats from the same mint`, + ); + break; + } + + throw error; + } + } + } + + throw lastInsufficientProofsError ?? new Error("sendToken failed after max retries"); +} + export interface WalletAdapterOptions { cocodPath?: string | null; walletClient?: CocodClient; @@ -268,38 +361,21 @@ export async function createWalletAdapter( return options.getAutoRefillConfig?.() ?? options.autoRefill; }, async sendToken(mintUrl: string, amount: number): Promise { - const maxRetries = 3; - const retryDelayMs = 5000; - const retryErrorPattern = "Proof already reserved by operation"; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return await client.sendCashu(amount, mintUrl); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - - const shouldRetry = - attempt < maxRetries && errorMessage.includes(retryErrorPattern); - - if (shouldRetry) { - logger.log( - `sendToken attempt ${attempt + 1} failed with reserved proof error, retrying in ${retryDelayMs / 1000}s...`, - ); - await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); - continue; - } - - if (errorMessage.includes("Not enough proofs")) { - throw new InsufficientBalanceError(amount, 0); - } - - logger.error("Error in walletAdapter sendToken:", error); - throw error; - } + try { + const balances: Record = await syncMintState().catch( + () => ({}), + ); + const sendAmounts = getSameMintSendAmounts( + amount, + balances[mintUrl] ?? amount, + ); + return await sendCashuFromMint(client, mintUrl, sendAmounts[0]!, { + fallbackAmounts: sendAmounts.slice(1), + }); + } catch (error) { + logger.error("Error in walletAdapter sendToken:", error); + throw error; } - - throw new Error("sendToken failed after max retries"); }, async receiveToken(token: string): Promise<{ success: boolean; diff --git a/src/daemon/wallet/sdk-mint-fallback.ts b/src/daemon/wallet/sdk-mint-fallback.ts index eef336c..387c690 100644 --- a/src/daemon/wallet/sdk-mint-fallback.ts +++ b/src/daemon/wallet/sdk-mint-fallback.ts @@ -33,6 +33,7 @@ type WalletAdapterLike = { }; const PATCH_MARKER = Symbol.for("routstrd.mintFallbackTopUpPatched"); +const ERROR_RETRY_PATCH_MARKER = Symbol.for("routstrd.mintFallbackErrorRetryPatched"); function uniqueMintUrls(mints: Array): string[] { const seen = new Set(); @@ -51,6 +52,10 @@ function isMintUnreachableTopUpResult(result: TopUpResult): boolean { (isMintUnreachableError(result.message) || isMintUnreachableError(result.error)); } +function isMintUnreachableResponse(status: number, responseBody: unknown): boolean { + return status >= 500 && isMintUnreachableError(responseBody); +} + async function getTopUpMintCandidates( initialMintUrl: string, walletClient: CocodClient, @@ -74,6 +79,8 @@ export function installMintFallbackTopUp( walletAdapter: WalletAdapterLike, logger: LoggerLike, ): void { + installMintUnreachableErrorRetry(client, walletClient, walletAdapter, logger); + const balanceManager = client.getBalanceManager() as BalanceManagerLike & { [PATCH_MARKER]?: boolean; }; @@ -112,3 +119,111 @@ export function installMintFallbackTopUp( balanceManager[PATCH_MARKER] = true; } + +export function installMintUnreachableErrorRetry( + client: RoutstrClientLike, + walletClient: CocodClient, + walletAdapter: WalletAdapterLike, + logger: LoggerLike, +): void { + const patchedClient = client as RoutstrClientLike & Record & { [ERROR_RETRY_PATCH_MARKER]?: boolean }; + if (patchedClient[ERROR_RETRY_PATCH_MARKER]) return; + + const originalHandleErrorResponse = patchedClient._handleErrorResponse; + if (typeof originalHandleErrorResponse !== "function") return; + + patchedClient._handleErrorResponse = async function patchedHandleErrorResponse( + this: Record, + params: Record, + token: string, + status: number, + requestId: string | undefined, + xCashuRefundToken: string | undefined, + responseBody: unknown, + retryCount = 0, + ): Promise { + const mode = this.mode; + const baseUrl = params?.baseUrl; + const initialMintUrl = params?.mintUrl; + + if ( + mode === "apikeys" && + baseUrl && + !params?.mintFallbackAttempted && + isMintUnreachableResponse(status, responseBody) + ) { + const candidates = (await getTopUpMintCandidates( + initialMintUrl, + walletClient, + walletAdapter, + )).filter((mintUrl) => mintUrl !== initialMintUrl); + + if (candidates.length > 0) { + logger.warn( + `[wallet] Provider ${baseUrl} rejected the stored API key because its source mint is unreachable. ` + + `Trying ${candidates.length} fallback mint(s) before provider failover.`, + ); + } + + for (const [index, mintUrl] of candidates.entries()) { + try { + this.storageAdapter?.removeApiKey?.(baseUrl); + logger.log( + `[wallet] Retrying ${baseUrl} with a fresh API key from fallback mint ${mintUrl} ` + + `(${index + 1}/${candidates.length})...`, + ); + + const spendResult = await this._spendToken({ + mintUrl, + amount: params.requiredSats, + baseUrl, + }); + + const retryToken = spendResult.token; + if (!retryToken) { + throw new Error("Fresh API key creation returned no token"); + } + + const retryResponse = await this._makeRequest({ + ...params, + mintUrl, + token: retryToken, + requiredSats: params.requiredSats, + headers: this._withAuthAndTinfoilHeaders( + params.baseHeaders, + retryToken, + params.tinfoilEnabled, + params.selectedModel?.id, + ), + retryCount: retryCount + 1, + mintFallbackAttempted: true, + }); + + retryResponse.initialTokenBalanceInSats = spendResult.tokenBalanceUnit === "msat" + ? spendResult.tokenBalance / 1_000 + : spendResult.tokenBalance; + retryResponse.initialTokenBalanceUnknown = spendResult.tokenBalanceUnknown; + return retryResponse; + } catch (error) { + logger.warn( + `[wallet] Fallback mint ${mintUrl} could not create a replacement API key for ${baseUrl}: ` + + (error instanceof Error ? error.message : String(error)), + ); + } + } + } + + return originalHandleErrorResponse.call( + this, + params, + token, + status, + requestId, + xCashuRefundToken, + responseBody, + retryCount, + ); + }; + + patchedClient[ERROR_RETRY_PATCH_MARKER] = true; +} diff --git a/tests/mint-fallback.test.ts b/tests/mint-fallback.test.ts index cd64ce2..a3e2e3a 100644 --- a/tests/mint-fallback.test.ts +++ b/tests/mint-fallback.test.ts @@ -4,6 +4,10 @@ import { isMintUnreachableError, receiveBolt11WithMintFallback, } from "../src/daemon/wallet/mint-fallback"; +import { + getSameMintSendAmounts, + sendCashuFromMint, +} from "../src/daemon/wallet"; function createClient(receiveBolt11: CocodClient["receiveBolt11"]): CocodClient { return { @@ -21,6 +25,88 @@ function createClient(receiveBolt11: CocodClient["receiveBolt11"]): CocodClient }; } +describe("wallet token mint identity", () => { + test("builds bounded same-mint denomination fallbacks", () => { + expect(getSameMintSendAmounts(17, 4103)).toEqual([ + 17, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 4103, + ]); + expect(getSameMintSendAmounts(1, 1)).toEqual([2]); + }); + + test("creates at least two sats so proof fees cannot consume the whole token", async () => { + const amounts: number[] = []; + const client = { + sendCashu: async (amount: number) => { + amounts.push(amount); + return "two-sat-token"; + }, + }; + + await expect( + sendCashuFromMint(client, "https://mint-a.example", 1, { + maxRetries: 0, + retryDelayMs: 0, + }), + ).resolves.toBe("two-sat-token"); + + expect(amounts).toEqual([2]); + }); + + test("never silently creates a token from a different mint", async () => { + const attempts: string[] = []; + const client = { + sendCashu: async (_amount: number, mintUrl?: string) => { + attempts.push(mintUrl || ""); + throw new Error("Not enough proofs to send"); + }, + }; + + await expect( + sendCashuFromMint(client, "https://mint-a.example", 21, { + maxRetries: 0, + retryDelayMs: 0, + }), + ).rejects.toThrow("Not enough proofs"); + + expect(attempts).toEqual(["https://mint-a.example"]); + }); + + test("retries only the same mint for temporarily reserved proofs", async () => { + const attempts: string[] = []; + const client = { + sendCashu: async (_amount: number, mintUrl?: string) => { + attempts.push(mintUrl || ""); + if (attempts.length === 1) { + throw new Error("Proof already reserved by operation"); + } + return "token-from-mint-a"; + }, + }; + + const token = await sendCashuFromMint( + client, + "https://mint-a.example", + 21, + { maxRetries: 1, retryDelayMs: 0 }, + ); + + expect(token).toBe("token-from-mint-a"); + expect(attempts).toEqual([ + "https://mint-a.example", + "https://mint-a.example", + ]); + }); +}); + describe("mint fallback", () => { test("detects mint_unreachable in nested error payloads", () => { expect(isMintUnreachableError(new Error("mint_unreachable"))).toBe(true); diff --git a/tests/sdk-mint-fallback.test.ts b/tests/sdk-mint-fallback.test.ts index 840e3c3..16cfc48 100644 --- a/tests/sdk-mint-fallback.test.ts +++ b/tests/sdk-mint-fallback.test.ts @@ -1,6 +1,9 @@ import { describe, expect, test } from "bun:test"; import type { CocodClient } from "../src/daemon/wallet/cocod-client"; -import { installMintFallbackTopUp } from "../src/daemon/wallet/sdk-mint-fallback"; +import { + installMintFallbackTopUp, + installMintUnreachableErrorRetry, +} from "../src/daemon/wallet/sdk-mint-fallback"; function createCocodClient(mints: string[]): CocodClient { return { @@ -56,6 +59,78 @@ describe("SDK top-up mint fallback", () => { expect(attempts).toEqual(["https://mint-a.example", "https://mint-b.example"]); }); + test("replaces an API key from a fallback mint before provider failover on mint_unreachable", async () => { + const removedApiKeys: string[] = []; + const spendAttempts: string[] = []; + const retryRequests: Array<{ baseUrl: string; mintUrl: string; token: string; mintFallbackAttempted?: boolean }> = []; + const client = { + mode: "apikeys", + getBalanceManager: () => ({ topUp: async () => ({ success: true }) }), + storageAdapter: { + removeApiKey: (baseUrl: string) => removedApiKeys.push(baseUrl), + }, + _spendToken: async (options: { mintUrl: string; baseUrl: string }) => { + spendAttempts.push(options.mintUrl); + return { + token: `token-from-${options.mintUrl}`, + tokenBalance: 42, + tokenBalanceUnit: "sat", + tokenBalanceUnknown: false, + }; + }, + _withAuthAndTinfoilHeaders: (_headers: unknown, token: string) => ({ authorization: token }), + _makeRequest: async (options: { baseUrl: string; mintUrl: string; token: string }) => { + retryRequests.push(options); + return { ok: true }; + }, + _handleErrorResponse: async () => { + throw new Error("provider failover should not run"); + }, + }; + const walletAdapter = { + getBalances: async () => ({ "https://mint-b.example": 100 }), + }; + + installMintUnreachableErrorRetry( + client, + createCocodClient(["https://mint-a.example", "https://mint-b.example"]), + walletAdapter, + { log: () => undefined, warn: () => undefined }, + ); + + const result = await client._handleErrorResponse( + { + baseUrl: "https://provider.example", + mintUrl: "https://mint-a.example", + requiredSats: 21, + baseHeaders: {}, + tinfoilEnabled: false, + selectedModel: { id: "glm-5.2" }, + }, + "old-token", + 503, + "request-id", + undefined, + JSON.stringify({ detail: { error: { type: "mint_unreachable" } } }), + ); + + expect(result).toMatchObject({ + ok: true, + initialTokenBalanceInSats: 42, + initialTokenBalanceUnknown: false, + }); + expect(removedApiKeys).toEqual(["https://provider.example"]); + expect(spendAttempts).toEqual(["https://mint-b.example"]); + expect(retryRequests).toMatchObject([ + { + baseUrl: "https://provider.example", + mintUrl: "https://mint-b.example", + token: "token-from-https://mint-b.example", + mintFallbackAttempted: true, + }, + ]); + }); + test("does not retry provider top-up on unrelated failure", async () => { const attempts: string[] = []; const balanceManager = { diff --git a/tests/wallet-send-fallback.test.ts b/tests/wallet-send-fallback.test.ts new file mode 100644 index 0000000..fe56e26 --- /dev/null +++ b/tests/wallet-send-fallback.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { createWalletAdapter } from "../src/daemon/wallet"; +import type { CocodClient } from "../src/daemon/wallet/cocod-client"; + +function createClient(overrides: Partial): CocodClient { + return { + ping: async () => true, + getStatus: async () => "UNLOCKED", + unlock: async () => "ok", + getBalances: async () => ({}), + receiveCashu: async () => "ok", + receiveBolt11: async () => "invoice", + sendCashu: async () => "token", + sendBolt11: async () => "ok", + listMints: async () => [], + addMint: async () => "ok", + getMintInfo: async () => ({}), + ...overrides, + }; +} + +describe("wallet sendToken mint identity", () => { + test("uses a larger denomination from the same mint when the exact amount cannot be composed", async () => { + const attempts: Array<{ amount: number; mintUrl: string }> = []; + const client = createClient({ + getBalances: async () => ({ + "https://mint.cubabitcoin.org": 4104, + "https://mint.minibits.cash/Bitcoin": 61409, + }), + listMints: async () => [ + "https://mint.minibits.cash/Bitcoin", + "https://mint.cubabitcoin.org", + ], + sendCashu: async (amount, mintUrl) => { + attempts.push({ amount, mintUrl: mintUrl || "" }); + if (amount === 1024) return "cashu-larger-same-mint-token"; + throw new Error("Not enough proofs"); + }, + }); + + const walletAdapter = await createWalletAdapter({ walletClient: client }); + + await expect( + walletAdapter.sendToken("https://mint.cubabitcoin.org", 612), + ).resolves.toBe("cashu-larger-same-mint-token"); + expect(attempts).toEqual([ + { amount: 612, mintUrl: "https://mint.cubabitcoin.org" }, + { amount: 1024, mintUrl: "https://mint.cubabitcoin.org" }, + ]); + }); +});