better fallback handling

src/daemon/wallet/index.ts

@@ -1,5 +1,4 @@
 import { getDecodedToken, Amount } from "@cashu/cashu-ts";
-import { InsufficientBalanceError } from "@routstr/sdk";
 import { WalletConnect } from "applesauce-wallet-connect";
 import { RelayPool } from "applesauce-relay";
 import { logger } from "../../utils/logger";
@@ -18,6 +17,100 @@ export function decodeCashuTokenAmount(token: string): {
   return { amount, unit };
 }
 
+type SendCashuOptions = {
+  maxRetries?: number;
+  retryDelayMs?: number;
+  minimumAmountSats?: number;
+  fallbackAmounts?: number[];
+};
+
+const MIN_PROVIDER_TOKEN_AMOUNT_SATS = 2;
+
+export function getSameMintSendAmounts(
+  amount: number,
+  availableBalance: number,
+): number[] {
+  const requestedAmount = Math.max(
+    Math.ceil(amount),
+    MIN_PROVIDER_TOKEN_AMOUNT_SATS,
+  );
+  const available = Math.floor(availableBalance);
+  const candidates = [requestedAmount];
+
+  if (available <= requestedAmount) return candidates;
+
+  let denomination = 2 ** Math.ceil(Math.log2(requestedAmount));
+  while (denomination <= available) {
+    if (!candidates.includes(denomination)) candidates.push(denomination);
+    denomination *= 2;
+  }
+
+  // The entire ready balance is always worth trying last: even when no single
+  // power-of-two proof exists, all ready proofs together may be selectable.
+  if (!candidates.includes(available)) candidates.push(available);
+  return candidates;
+}
+
+/**
+ * Create a token from exactly the requested mint.
+ *
+ * The SDK associates the returned token with `mintUrl`. Falling back to another
+ * mint here makes that association false and can send providers a token from a
+ * mint they cannot reach. Cross-mint fallback belongs above this adapter, where
+ * the actual mint URL remains part of the request state.
+ */
+export async function sendCashuFromMint(
+  client: Pick<CocodClient, "sendCashu">,
+  mintUrl: string,
+  amount: number,
+  options: SendCashuOptions = {},
+): Promise<string> {
+  const maxRetries = options.maxRetries ?? 3;
+  const retryDelayMs = options.retryDelayMs ?? 5000;
+  const minimumAmountSats = options.minimumAmountSats ?? MIN_PROVIDER_TOKEN_AMOUNT_SATS;
+  const sendAmounts = [
+    Math.max(amount, minimumAmountSats),
+    ...(options.fallbackAmounts ?? []),
+  ].filter((candidate, index, all) => candidate > 0 && all.indexOf(candidate) === index);
+  const retryErrorPattern = "Proof already reserved by operation";
+  let lastInsufficientProofsError: unknown;
+
+  for (const [amountIndex, sendAmount] of sendAmounts.entries()) {
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        return await client.sendCashu(sendAmount, mintUrl);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        const shouldRetry =
+          attempt < maxRetries && errorMessage.includes(retryErrorPattern);
+
+        if (shouldRetry) {
+          logger.log(
+            `sendToken attempt ${attempt + 1} failed with reserved proof error for ${mintUrl}, retrying in ${retryDelayMs / 1000}s...`,
+          );
+          await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+          continue;
+        }
+
+        if (
+          errorMessage.includes("Not enough proofs") &&
+          amountIndex + 1 < sendAmounts.length
+        ) {
+          lastInsufficientProofsError = error;
+          logger.warn(
+            `sendToken: ${mintUrl} cannot compose ${sendAmount} sats; trying ${sendAmounts[amountIndex + 1]} sats from the same mint`,
+          );
+          break;
+        }
+
+        throw error;
+      }
+    }
+  }
+
+  throw lastInsufficientProofsError ?? new Error("sendToken failed after max retries");
+}
+
 export interface WalletAdapterOptions {
   cocodPath?: string | null;
   walletClient?: CocodClient;
@@ -268,38 +361,21 @@ export async function createWalletAdapter(
       return options.getAutoRefillConfig?.() ?? options.autoRefill;
     },
     async sendToken(mintUrl: string, amount: number): Promise<string> {
-      const maxRetries = 3;
-      const retryDelayMs = 5000;
-      const retryErrorPattern = "Proof already reserved by operation";
-
-      for (let attempt = 0; attempt <= maxRetries; attempt++) {
-        try {
-          return await client.sendCashu(amount, mintUrl);
-        } catch (error) {
-          const errorMessage =
-            error instanceof Error ? error.message : String(error);
-
-          const shouldRetry =
-            attempt < maxRetries && errorMessage.includes(retryErrorPattern);
-
-          if (shouldRetry) {
-            logger.log(
-              `sendToken attempt ${attempt + 1} failed with reserved proof error, retrying in ${retryDelayMs / 1000}s...`,
-            );
-            await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
-            continue;
-          }
-
-          if (errorMessage.includes("Not enough proofs")) {
-            throw new InsufficientBalanceError(amount, 0);
-          }
-
-          logger.error("Error in walletAdapter sendToken:", error);
-          throw error;
-        }
+      try {
+        const balances: Record<string, number> = await syncMintState().catch(
+          () => ({}),
+        );
+        const sendAmounts = getSameMintSendAmounts(
+          amount,
+          balances[mintUrl] ?? amount,
+        );
+        return await sendCashuFromMint(client, mintUrl, sendAmounts[0]!, {
+          fallbackAmounts: sendAmounts.slice(1),
+        });
+      } catch (error) {
+        logger.error("Error in walletAdapter sendToken:", error);
+        throw error;
       }
-
-      throw new Error("sendToken failed after max retries");
     },
     async receiveToken(token: string): Promise<{
       success: boolean;

src/daemon/wallet/sdk-mint-fallback.ts

@@ -33,6 +33,7 @@ type WalletAdapterLike = {
 };
 
 const PATCH_MARKER = Symbol.for("routstrd.mintFallbackTopUpPatched");
+const ERROR_RETRY_PATCH_MARKER = Symbol.for("routstrd.mintFallbackErrorRetryPatched");
 
 function uniqueMintUrls(mints: Array<string | undefined | null>): string[] {
   const seen = new Set<string>();
@@ -51,6 +52,10 @@ function isMintUnreachableTopUpResult(result: TopUpResult): boolean {
     (isMintUnreachableError(result.message) || isMintUnreachableError(result.error));
 }
 
+function isMintUnreachableResponse(status: number, responseBody: unknown): boolean {
+  return status >= 500 && isMintUnreachableError(responseBody);
+}
+
 async function getTopUpMintCandidates(
   initialMintUrl: string,
   walletClient: CocodClient,
@@ -74,6 +79,8 @@ export function installMintFallbackTopUp(
   walletAdapter: WalletAdapterLike,
   logger: LoggerLike,
 ): void {
+  installMintUnreachableErrorRetry(client, walletClient, walletAdapter, logger);
+
   const balanceManager = client.getBalanceManager() as BalanceManagerLike & {
     [PATCH_MARKER]?: boolean;
   };
@@ -112,3 +119,111 @@ export function installMintFallbackTopUp(
 
   balanceManager[PATCH_MARKER] = true;
 }
+
+export function installMintUnreachableErrorRetry(
+  client: RoutstrClientLike,
+  walletClient: CocodClient,
+  walletAdapter: WalletAdapterLike,
+  logger: LoggerLike,
+): void {
+  const patchedClient = client as RoutstrClientLike & Record<string, any> & { [ERROR_RETRY_PATCH_MARKER]?: boolean };
+  if (patchedClient[ERROR_RETRY_PATCH_MARKER]) return;
+
+  const originalHandleErrorResponse = patchedClient._handleErrorResponse;
+  if (typeof originalHandleErrorResponse !== "function") return;
+
+  patchedClient._handleErrorResponse = async function patchedHandleErrorResponse(
+    this: Record<string, any>,
+    params: Record<string, any>,
+    token: string,
+    status: number,
+    requestId: string | undefined,
+    xCashuRefundToken: string | undefined,
+    responseBody: unknown,
+    retryCount = 0,
+  ): Promise<unknown> {
+    const mode = this.mode;
+    const baseUrl = params?.baseUrl;
+    const initialMintUrl = params?.mintUrl;
+
+    if (
+      mode === "apikeys" &&
+      baseUrl &&
+      !params?.mintFallbackAttempted &&
+      isMintUnreachableResponse(status, responseBody)
+    ) {
+      const candidates = (await getTopUpMintCandidates(
+        initialMintUrl,
+        walletClient,
+        walletAdapter,
+      )).filter((mintUrl) => mintUrl !== initialMintUrl);
+
+      if (candidates.length > 0) {
+        logger.warn(
+          `[wallet] Provider ${baseUrl} rejected the stored API key because its source mint is unreachable. ` +
+            `Trying ${candidates.length} fallback mint(s) before provider failover.`,
+        );
+      }
+
+      for (const [index, mintUrl] of candidates.entries()) {
+        try {
+          this.storageAdapter?.removeApiKey?.(baseUrl);
+          logger.log(
+            `[wallet] Retrying ${baseUrl} with a fresh API key from fallback mint ${mintUrl} ` +
+              `(${index + 1}/${candidates.length})...`,
+          );
+
+          const spendResult = await this._spendToken({
+            mintUrl,
+            amount: params.requiredSats,
+            baseUrl,
+          });
+
+          const retryToken = spendResult.token;
+          if (!retryToken) {
+            throw new Error("Fresh API key creation returned no token");
+          }
+
+          const retryResponse = await this._makeRequest({
+            ...params,
+            mintUrl,
+            token: retryToken,
+            requiredSats: params.requiredSats,
+            headers: this._withAuthAndTinfoilHeaders(
+              params.baseHeaders,
+              retryToken,
+              params.tinfoilEnabled,
+              params.selectedModel?.id,
+            ),
+            retryCount: retryCount + 1,
+            mintFallbackAttempted: true,
+          });
+
+          retryResponse.initialTokenBalanceInSats = spendResult.tokenBalanceUnit === "msat"
+            ? spendResult.tokenBalance / 1_000
+            : spendResult.tokenBalance;
+          retryResponse.initialTokenBalanceUnknown = spendResult.tokenBalanceUnknown;
+          return retryResponse;
+        } catch (error) {
+          logger.warn(
+            `[wallet] Fallback mint ${mintUrl} could not create a replacement API key for ${baseUrl}: ` +
+              (error instanceof Error ? error.message : String(error)),
+          );
+        }
+      }
+    }
+
+    return originalHandleErrorResponse.call(
+      this,
+      params,
+      token,
+      status,
+      requestId,
+      xCashuRefundToken,
+      responseBody,
+      retryCount,
+    );
+  };
+
+  patchedClient[ERROR_RETRY_PATCH_MARKER] = true;
+}

tests/mint-fallback.test.ts

@@ -4,6 +4,10 @@ import {
   isMintUnreachableError,
   receiveBolt11WithMintFallback,
 } from "../src/daemon/wallet/mint-fallback";
+import {
+  getSameMintSendAmounts,
+  sendCashuFromMint,
+} from "../src/daemon/wallet";
 
 function createClient(receiveBolt11: CocodClient["receiveBolt11"]): CocodClient {
   return {
@@ -21,6 +25,88 @@ function createClient(receiveBolt11: CocodClient["receiveBolt11"]): CocodClient
   };
 }
 
+describe("wallet token mint identity", () => {
+  test("builds bounded same-mint denomination fallbacks", () => {
+    expect(getSameMintSendAmounts(17, 4103)).toEqual([
+      17,
+      32,
+      64,
+      128,
+      256,
+      512,
+      1024,
+      2048,
+      4096,
+      4103,
+    ]);
+    expect(getSameMintSendAmounts(1, 1)).toEqual([2]);
+  });
+
+  test("creates at least two sats so proof fees cannot consume the whole token", async () => {
+    const amounts: number[] = [];
+    const client = {
+      sendCashu: async (amount: number) => {
+        amounts.push(amount);
+        return "two-sat-token";
+      },
+    };
+
+    await expect(
+      sendCashuFromMint(client, "https://mint-a.example", 1, {
+        maxRetries: 0,
+        retryDelayMs: 0,
+      }),
+    ).resolves.toBe("two-sat-token");
+
+    expect(amounts).toEqual([2]);
+  });
+
+  test("never silently creates a token from a different mint", async () => {
+    const attempts: string[] = [];
+    const client = {
+      sendCashu: async (_amount: number, mintUrl?: string) => {
+        attempts.push(mintUrl || "");
+        throw new Error("Not enough proofs to send");
+      },
+    };
+
+    await expect(
+      sendCashuFromMint(client, "https://mint-a.example", 21, {
+        maxRetries: 0,
+        retryDelayMs: 0,
+      }),
+    ).rejects.toThrow("Not enough proofs");
+
+    expect(attempts).toEqual(["https://mint-a.example"]);
+  });
+
+  test("retries only the same mint for temporarily reserved proofs", async () => {
+    const attempts: string[] = [];
+    const client = {
+      sendCashu: async (_amount: number, mintUrl?: string) => {
+        attempts.push(mintUrl || "");
+        if (attempts.length === 1) {
+          throw new Error("Proof already reserved by operation");
+        }
+        return "token-from-mint-a";
+      },
+    };
+
+    const token = await sendCashuFromMint(
+      client,
+      "https://mint-a.example",
+      21,
+      { maxRetries: 1, retryDelayMs: 0 },
+    );
+
+    expect(token).toBe("token-from-mint-a");
+    expect(attempts).toEqual([
+      "https://mint-a.example",
+      "https://mint-a.example",
+    ]);
+  });
+});
+
 describe("mint fallback", () => {
   test("detects mint_unreachable in nested error payloads", () => {
     expect(isMintUnreachableError(new Error("mint_unreachable"))).toBe(true);

tests/sdk-mint-fallback.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, test } from "bun:test";
 import type { CocodClient } from "../src/daemon/wallet/cocod-client";
-import { installMintFallbackTopUp } from "../src/daemon/wallet/sdk-mint-fallback";
+import {
+  installMintFallbackTopUp,
+  installMintUnreachableErrorRetry,
+} from "../src/daemon/wallet/sdk-mint-fallback";
 
 function createCocodClient(mints: string[]): CocodClient {
   return {
@@ -56,6 +59,78 @@ describe("SDK top-up mint fallback", () => {
     expect(attempts).toEqual(["https://mint-a.example", "https://mint-b.example"]);
   });
 
+  test("replaces an API key from a fallback mint before provider failover on mint_unreachable", async () => {
+    const removedApiKeys: string[] = [];
+    const spendAttempts: string[] = [];
+    const retryRequests: Array<{ baseUrl: string; mintUrl: string; token: string; mintFallbackAttempted?: boolean }> = [];
+    const client = {
+      mode: "apikeys",
+      getBalanceManager: () => ({ topUp: async () => ({ success: true }) }),
+      storageAdapter: {
+        removeApiKey: (baseUrl: string) => removedApiKeys.push(baseUrl),
+      },
+      _spendToken: async (options: { mintUrl: string; baseUrl: string }) => {
+        spendAttempts.push(options.mintUrl);
+        return {
+          token: `token-from-${options.mintUrl}`,
+          tokenBalance: 42,
+          tokenBalanceUnit: "sat",
+          tokenBalanceUnknown: false,
+        };
+      },
+      _withAuthAndTinfoilHeaders: (_headers: unknown, token: string) => ({ authorization: token }),
+      _makeRequest: async (options: { baseUrl: string; mintUrl: string; token: string }) => {
+        retryRequests.push(options);
+        return { ok: true };
+      },
+      _handleErrorResponse: async () => {
+        throw new Error("provider failover should not run");
+      },
+    };
+    const walletAdapter = {
+      getBalances: async () => ({ "https://mint-b.example": 100 }),
+    };
+
+    installMintUnreachableErrorRetry(
+      client,
+      createCocodClient(["https://mint-a.example", "https://mint-b.example"]),
+      walletAdapter,
+      { log: () => undefined, warn: () => undefined },
+    );
+
+    const result = await client._handleErrorResponse(
+      {
+        baseUrl: "https://provider.example",
+        mintUrl: "https://mint-a.example",
+        requiredSats: 21,
+        baseHeaders: {},
+        tinfoilEnabled: false,
+        selectedModel: { id: "glm-5.2" },
+      },
+      "old-token",
+      503,
+      "request-id",
+      undefined,
+      JSON.stringify({ detail: { error: { type: "mint_unreachable" } } }),
+    );
+
+    expect(result).toMatchObject({
+      ok: true,
+      initialTokenBalanceInSats: 42,
+      initialTokenBalanceUnknown: false,
+    });
+    expect(removedApiKeys).toEqual(["https://provider.example"]);
+    expect(spendAttempts).toEqual(["https://mint-b.example"]);
+    expect(retryRequests).toMatchObject([
+      {
+        baseUrl: "https://provider.example",
+        mintUrl: "https://mint-b.example",
+        token: "token-from-https://mint-b.example",
+        mintFallbackAttempted: true,
+      },
+    ]);
+  });
+
   test("does not retry provider top-up on unrelated failure", async () => {
     const attempts: string[] = [];
     const balanceManager = {

tests/wallet-send-fallback.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, test } from "bun:test";
+import { createWalletAdapter } from "../src/daemon/wallet";
+import type { CocodClient } from "../src/daemon/wallet/cocod-client";
+
+function createClient(overrides: Partial<CocodClient>): CocodClient {
+  return {
+    ping: async () => true,
+    getStatus: async () => "UNLOCKED",
+    unlock: async () => "ok",
+    getBalances: async () => ({}),
+    receiveCashu: async () => "ok",
+    receiveBolt11: async () => "invoice",
+    sendCashu: async () => "token",
+    sendBolt11: async () => "ok",
+    listMints: async () => [],
+    addMint: async () => "ok",
+    getMintInfo: async () => ({}),
+    ...overrides,
+  };
+}
+
+describe("wallet sendToken mint identity", () => {
+  test("uses a larger denomination from the same mint when the exact amount cannot be composed", async () => {
+    const attempts: Array<{ amount: number; mintUrl: string }> = [];
+    const client = createClient({
+      getBalances: async () => ({
+        "https://mint.cubabitcoin.org": 4104,
+        "https://mint.minibits.cash/Bitcoin": 61409,
+      }),
+      listMints: async () => [
+        "https://mint.minibits.cash/Bitcoin",
+        "https://mint.cubabitcoin.org",
+      ],
+      sendCashu: async (amount, mintUrl) => {
+        attempts.push({ amount, mintUrl: mintUrl || "" });
+        if (amount === 1024) return "cashu-larger-same-mint-token";
+        throw new Error("Not enough proofs");
+      },
+    });
+
+    const walletAdapter = await createWalletAdapter({ walletClient: client });
+
+    await expect(
+      walletAdapter.sendToken("https://mint.cubabitcoin.org", 612),
+    ).resolves.toBe("cashu-larger-same-mint-token");
+    expect(attempts).toEqual([
+      { amount: 612, mintUrl: "https://mint.cubabitcoin.org" },
+      { amount: 1024, mintUrl: "https://mint.cubabitcoin.org" },
+    ]);
+  });
+});