From f5bf6d9f1cd4c67c80013c75d0fd19687384fc67 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:30:34 +0800 Subject: [PATCH 1/7] feat(wallet): recover PAID mint quotes that were paid but never issued A mint quote can be PAID at the mint while its local coco operation is still pending (the Lightning payment landed before expiry while the daemon was down) or even terminally failed. routstrd could strand those sats in two ways: - `routstrd wallet cleanup` failed expired pending quotes locally without asking the mint, so a quote paid before its invoice expired was marked failed and then skipped by every recovery sweep. - nothing re-issued a paid-but-unissued quote: coco's refresh turns the mint's expired-quote error into a terminal failure, and NUT-09 restore returns nothing when amount_issued is still 0. Changes: - add `failExpiredMintQuoteIfUnpaid`, the single mint-confirmed decision now used by both startup recovery and `wallet cleanup`. Cleanup gains `--force` for the old local-fail behaviour and reports `leftForRecovery`. - add `runMintQuoteRecovery` and pure selection/classification helpers in `mint-quote-recovery.ts`, exposed as `routstrd wallet recover` and `POST /wallet/recover`. It asks the mint per quote: PAID is minted from the operation's stored outputs, ISSUED restores proofs, UNPAID is left pending, and quotes the mint can no longer issue are reported instead of dropped. Failed operations are re-opened only by explicit operation id (`--include-failed`), since coco's pending listing never returns them, and a named failed operation is judged by the mint rather than a stale local observation. - add `createRunQueue` so explicit recovery requests are serialized, and a shared `outstanding` map so a quote check or finalize that outlives its timeout keeps blocking a retry until it settles. coco details the implementation is careful about: - `ops.mint.finalize` does not throw when the mint refuses (expired quote) or when an already-issued quote's proofs cannot be restored: it returns a terminal operation. Recovery inspects the returned state and error, counts only finalized-without-error as recovered, reserves `terminal` for failed/finalized-with-error, and leaves anything still pending to a later run. - `transitionToPending` spreads whatever it is handed and the sqlite repository rewrites every column, so `reopenFailedMintOperation` reloads and passes the full persisted row, and holds coco's per-operation lock across the read-check-write. That lock is fail-fast (it throws `OperationInProgressError` rather than waiting), so a re-open either holds the lock - blocking coco's own execute/finalize/recover paths, which share it - or writes nothing. Scope is narrow: `recordPendingObservation` and `failPendingOperation` write without that lock, so the claim is only that a re-open cannot clobber a concurrent executing/recovery pass. This shims private coco internals and is written against @cashu/coco-core 1.0.1, so it fails closed on a missing method and the integration tests must be re-run on any coco bump; changed behaviour under an unchanged name is not detectable by the guard alone. Dependency reproducibility and an upstream public locked reopen API remain release considerations. Known interop gap (not a supported path): NUT-09 allows `null` entries in the positional `signatures` array for unsigned outputs, but cashu-ts 3.7.1, which coco depends on, dereferences every entry while normalising amounts and so throws. A null-containing restore response therefore credits nothing and the operation is retried later. The fake mint keeps a switch for that shape and an explicit regression pins the current behaviour; this should be tracked upstream. Filtering the fixture to signed outputs for the success-path scenarios does not claim null responses are handled. Tests: - `mint-quote-recovery.manager.test.ts`: the production helper against a real Manager + sqlite service - full-row preservation, the fail-fast lock (including refusing to write while a processor holds it), and the `mint-op:pending` event. - `mint-quote-recovery.fake-mint.test.ts` + `testing/fake-mint.ts`: a real Manager over HTTP against an in-process mint signing with genuine secp256k1 blind signatures. Covers expired-but-PAID issuance with the operation's own blinded outputs (once), ISSUED restore without double credit, a mint refusal (20007) as terminal without credit, an issued-but-unrestorable quote as terminal without credit, the null-signature interop gap, a locked operation counted as busy, in-flight issuance not minted twice, a hung check tracked until it drains, coexistence with coco's own operation watcher/processor, and the composed failed -> re-open -> PAID issue -> spendable path with a stale UNPAID observation recovered by explicit id. - `createRunQueue` unit tests, adapter-backed helper tests, fail-closed behaviour, and the classify/select unit tests. --- src/cli.ts | 133 +++- src/daemon/http/index.ts | 36 + src/daemon/wallet/coco-client.test.ts | 665 ++++++++++++++++++ src/daemon/wallet/coco-client.ts | 638 +++++++++++++++-- src/daemon/wallet/cocod-client.ts | 45 ++ .../wallet/mint-operation-reopen.test.ts | 163 +++++ .../mint-quote-recovery.fake-mint.test.ts | 356 ++++++++++ .../mint-quote-recovery.manager.test.ts | 159 +++++ src/daemon/wallet/mint-quote-recovery.test.ts | 97 +++ src/daemon/wallet/mint-quote-recovery.ts | 116 +++ src/daemon/wallet/testing/fake-mint.ts | 347 +++++++++ 11 files changed, 2705 insertions(+), 50 deletions(-) create mode 100644 src/daemon/wallet/mint-operation-reopen.test.ts create mode 100644 src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts create mode 100644 src/daemon/wallet/mint-quote-recovery.manager.test.ts create mode 100644 src/daemon/wallet/mint-quote-recovery.test.ts create mode 100644 src/daemon/wallet/mint-quote-recovery.ts create mode 100644 src/daemon/wallet/testing/fake-mint.ts diff --git a/src/cli.ts b/src/cli.ts index c25f35e..21c6d54 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2014,16 +2014,22 @@ walletCmd .option("--mint-url ", "Only clean up operations for this mint URL") .option( "--min-age ", - "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)", + "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are checked with their mint)", "168", ) .option("--dry-run", "Report what would be cleaned without applying changes", false) + .option( + "--force", + "Fail expired mint quotes without confirming UNPAID with the mint (may strand paid quotes)", + false, + ) .option("-y, --yes", "Skip confirmation prompt", false) .action( async (options: { mintUrl?: string; minAge: string; dryRun: boolean; + force: boolean; yes: boolean; }) => { const minAgeHours = Number.parseFloat(options.minAge); @@ -2039,7 +2045,7 @@ walletCmd }); const answer = await new Promise((resolve) => { rl.question( - "This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", + "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", (value: string) => { rl.close(); resolve(value.trim().toLowerCase()); @@ -2061,6 +2067,7 @@ walletCmd mintUrl: options.mintUrl, minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000), dryRun: options.dryRun === true, + force: options.force === true, }, }); @@ -2073,6 +2080,7 @@ walletCmd | { dryRun?: boolean; failedMintQuotes?: number; + leftForRecovery?: number; reclaimedSends?: number; cancelledMelts?: number; skipped?: number; @@ -2086,6 +2094,9 @@ walletCmd console.log( ` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`, ); + console.log( + ` Expired quotes kept for recovery (paid/issued/unverified): ${output.leftForRecovery ?? 0}`, + ); console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`); console.log( ` Prepared melts cancelled: ${output.cancelledMelts ?? 0}`, @@ -2115,6 +2126,124 @@ walletCmd }, ); +walletCmd + .command("recover") + .description( + "Re-issue PAID mint quotes whose sats were never claimed by checking each quote with its mint", + ) + .option( + "--op ", + "Recover only this operation id (repeatable; required to target failed operations)", + (value: string, previous: string[]) => [...previous, value], + [] as string[], + ) + .option( + "--include-failed", + "Also re-open operations coco already gave up on (requires --op)", + false, + ) + .option("-y, --yes", "Skip confirmation prompt", false) + .action( + async (options: { + op: string[]; + includeFailed: boolean; + yes: boolean; + }) => { + const operationIds = options.op ?? []; + if (options.includeFailed && operationIds.length === 0) { + console.error( + "--include-failed can only target operations named with --op", + ); + process.exit(1); + } + + if (!options.yes) { + const rl = require("readline").createInterface({ + input: process.stdin, + output: process.stdout, + }); + const prompt = + operationIds.length > 0 + ? `Recover ${operationIds.length} mint quote operation(s)? [y/N] ` + : "Check every pending mint quote with its mint and claim any paid sats? [y/N] "; + const answer = await new Promise((resolve) => { + rl.question(prompt, (value: string) => { + rl.close(); + resolve(value.trim().toLowerCase()); + }); + }); + if (answer !== "y" && answer !== "yes") { + console.log("Aborted."); + return; + } + } + + try { + await ensureDaemonRunning(); + + const result = await callDaemon("/wallet/recover", { + method: "POST", + body: { + operationIds: operationIds.length > 0 ? operationIds : undefined, + includeFailed: options.includeFailed === true, + }, + }); + + if (result.error) { + console.log(result.error); + process.exit(1); + } + + const output = result.output as + | { + checked?: number; + recovered?: number; + waiting?: number; + terminal?: number; + reopened?: number; + retryable?: number; + busy?: number; + errors?: Array<{ operationId: string; error: string }>; + } + | undefined; + + if (output) { + console.log("Mint quote recovery:"); + console.log(` Checked with mint: ${output.checked ?? 0}`); + console.log( + ` Recovered (paid sats claimed): ${output.recovered ?? 0}`, + ); + console.log(` Still unpaid: ${output.waiting ?? 0}`); + console.log(` No longer issuable: ${output.terminal ?? 0}`); + console.log( + ` Re-opened failed operations: ${output.reopened ?? 0}`, + ); + console.log(` Left for a later run: ${output.retryable ?? 0}`); + console.log( + ` Skipped (recovery still running): ${output.busy ?? 0}`, + ); + if (output.errors && output.errors.length > 0) { + console.log("\nErrors:"); + for (const e of output.errors) { + console.log(` - ${e.operationId}: ${e.error}`); + } + } + } + } catch (error) { + const message = (error as Error).message; + if ( + message?.includes("fetch failed") || + message?.includes("Connection refused") + ) { + console.error("Daemon is not running"); + process.exit(1); + } + console.error(message); + process.exit(1); + } + }, + ); + const walletReceiveCmd = walletCmd .command("receive") .description("Wallet receive operations"); diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 05349bb..7cee3d5 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -283,6 +283,18 @@ function optionalStringField( return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function optionalStringArrayField( + body: Record, + field: string, +): string[] | undefined { + const value = body[field]; + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { + throw new CocodHttpError(400, `'${field}' must be an array of strings.`); + } + return value as string[]; +} + function getCurrentMode(deps: DaemonDeps): ClientMode { const stateMode = deps.store.getState()?.mode; return stateMode || deps.mode || "apikeys"; @@ -473,6 +485,30 @@ export function createDaemonRequestHandler(deps: { ? body.minAgeMs : undefined, dryRun: body.dryRun === true, + force: body.force === true, + }); + return { output: result }; + }); + return; + } + + if (req.method === "POST" && url.pathname === "/wallet/recover") { + await respond(res, async () => { + if (!deps.walletClient.recoverMintQuotes) { + throw new CocodHttpError( + 501, + "Mint quote recovery is not supported by this wallet client.", + ); + } + + const body = await readJsonBody(req); + const result = await deps.walletClient.recoverMintQuotes({ + operationIds: optionalStringArrayField(body, "operationIds"), + includeFailed: body.includeFailed === true, + timeoutMs: + typeof body.timeoutMs === "number" && Number.isFinite(body.timeoutMs) + ? body.timeoutMs + : undefined, }); return { output: result }; }); diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index 959e798..fc03b5e 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -14,12 +14,17 @@ import { assertLegacyCocodNotRunning, claimLegacyCocodPidFile, createCocoClient, + createRunQueue, DEFAULT_TRUSTED_MINT_URLS, + failExpiredMintQuoteIfUnpaid, isZombieProcess, + reopenFailedMintOperation, + runMintQuoteRecovery, settleExpiredMintQuotes, settlePendingMintQuotes, stopLegacyCocod, type ExpiredMintQuoteSource, + type MintQuoteRecoverySource, type PendingMintQuoteSource, type PendingMintSweepState, } from "./coco-client"; @@ -901,3 +906,663 @@ describe("settlePendingMintQuotes", () => { expect(logged.mock.calls[0]?.[0]).toContain("21 sat minted"); }); }); + +describe("createRunQueue", () => { + it("runs tasks strictly one after another", async () => { + const enqueue = createRunQueue(); + const order: string[] = []; + let active = 0; + let maxActive = 0; + const task = (name: string, delay: number) => async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`${name}:start`); + await new Promise((resolve) => setTimeout(resolve, delay)); + order.push(`${name}:end`); + active--; + return name; + }; + + const results = await Promise.all([ + enqueue(task("a", 20)), + enqueue(task("b", 1)), + enqueue(task("c", 1)), + ]); + + expect(results).toEqual(["a", "b", "c"]); + expect(maxActive).toBe(1); + expect(order).toEqual([ + "a:start", + "a:end", + "b:start", + "b:end", + "c:start", + "c:end", + ]); + }); + + it("keeps the chain alive after a rejected task", async () => { + const enqueue = createRunQueue(); + + const failed = enqueue(async () => { + throw new Error("boom"); + }); + const next = enqueue(async () => "ok"); + + await expect(failed).rejects.toThrow("boom"); + expect(await next).toBe("ok"); + }); +}); + +describe("failExpiredMintQuoteIfUnpaid", () => { + function fakeMintService(observe: (id: string) => Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>) { + const failPendingOperation = mock( + async ( + _op: { id: string }, + _failure: { reason: string; retryable?: boolean; observedAt: number }, + ) => ({}), + ); + return { + mintService: { + observePendingOperation: mock(observe), + failPendingOperation, + }, + failPendingOperation, + }; + } + + it("fails a quote its mint confirms unpaid", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => ({ + category: "waiting", + })); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("failed"); + expect(failPendingOperation).toHaveBeenCalledTimes(1); + expect(failPendingOperation.mock.calls[0]?.[1]?.reason).toContain( + "confirmed unpaid by mint", + ); + }); + + it.each(["ready", "completed", "terminal"] as const)( + "leaves a quote observed as %s for recovery", + async (category) => { + const { mintService, failPendingOperation } = fakeMintService( + async () => ({ category }), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result).toEqual({ outcome: "leftForRecovery", category }); + expect(failPendingOperation).not.toHaveBeenCalled(); + }, + ); + + it("leaves a quote pending when the mint cannot be reached", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => { + throw new Error("Network request failed"); + }); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); + + it("gives up waiting on a hung mint without failing the quote", async () => { + const { mintService, failPendingOperation } = fakeMintService( + () => new Promise(() => {}), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 20); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); +}); + + +describe("reopenFailedMintOperation", () => { + /** + * Mirrors coco's OperationIdLock, which is fail-fast: acquiring an id that is + * already locked throws OperationInProgressError instead of waiting. + */ + function makeLock() { + let held = false; + return { + get held() { + return held; + }, + async acquire() { + if (held) { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + } + held = true; + return () => { + held = false; + }; + }, + }; + } + + function fakeService( + current: Record | null, + hooks: { + lock?: ReturnType; + onWrite?: (lock: ReturnType) => void; + } = {}, + ) { + const lock = hooks.lock ?? makeLock(); + const transitionToPending = mock( + async (_op: Record, _error?: string) => { + hooks.onWrite?.(lock); + return {}; + }, + ); + return { + service: { + acquireOperationLock: mock(async (_id: string) => lock.acquire()), + getOperation: mock(async (_id: string) => current), + transitionToPending, + }, + transitionToPending, + lock, + }; + } + + it("re-opens a failed operation with the full persisted row", async () => { + // coco spreads whatever it is handed and the sqlite repository rewrites + // every column, so a partial object would erase the stored outputs. + const row = { + id: "op-1", + state: "failed", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + method: "bolt11", + amount: 210_000, + unit: "sat", + request: "lnbc...", + expiry: 1_800_000_000, + outputDataJson: "[{\"secret\":\"abc\"}]", + terminalFailure: { reason: "expired" }, + }; + const { service, transitionToPending } = fakeService(row); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(transitionToPending).toHaveBeenCalledTimes(1); + const passed = transitionToPending.mock.calls[0]?.[0]; + expect(passed).toMatchObject({ + id: "op-1", + quoteId: "quote-1", + amount: 210_000, + unit: "sat", + outputDataJson: "[{\"secret\":\"abc\"}]", + }); + // The stale terminal marker must not survive the re-open. + expect(passed?.terminalFailure).toBeUndefined(); + }); + + it("does nothing when the operation is no longer failed", async () => { + const { service, transitionToPending, lock } = fakeService({ + id: "op-1", + state: "finalized", + }); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + expect(transitionToPending).not.toHaveBeenCalled(); + // The lock must be released even on the no-op path. + expect(lock.held).toBe(false); + }); + + it("throws when the operation is missing", async () => { + const { service, lock } = fakeService(null); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + "not found", + ); + expect(lock.held).toBe(false); + }); + + it("fails closed when coco no longer exposes the operation lock", async () => { + const transitionToPending = mock( + async (_op: Record, _error?: string) => ({}), + ); + const service = { + getOperation: mock(async () => ({ id: "op-1", state: "failed" })), + transitionToPending, + } as unknown as Parameters[0]; + + await expect( + reopenFailedMintOperation(service, "op-1"), + ).rejects.toThrow("acquireOperationLock"); + expect(transitionToPending).not.toHaveBeenCalled(); + }); + + it("holds the operation lock across read-check-write", async () => { + const lock = makeLock(); + const order: string[] = []; + const service = { + acquireOperationLock: mock(async (_id: string) => { + order.push("lock"); + const release = await lock.acquire(); + return () => { + order.push("unlock"); + release(); + }; + }), + getOperation: mock(async (_id: string) => { + expect(lock.held).toBe(true); + order.push("read"); + return { id: "op-1", state: "failed", quoteId: "quote-1" }; + }), + transitionToPending: mock(async () => { + expect(lock.held).toBe(true); + order.push("write"); + return {}; + }), + }; + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(order).toEqual(["lock", "read", "write", "unlock"]); + expect(lock.held).toBe(false); + }); + + it("refuses to re-open while the operation lock is held elsewhere", async () => { + const lock = makeLock(); + const release = await lock.acquire(); + const { service, transitionToPending } = fakeService( + { id: "op-1", state: "failed" }, + { lock }, + ); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /in progress/, + ); + expect(transitionToPending).not.toHaveBeenCalled(); + release(); + }); +}); + +describe("runMintQuoteRecovery", () => { + function mintOp(overrides: Record = {}) { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state: "pending", + amount: 210_000, + expiry: 0, + ...overrides, + }; + } + + function fakeSource( + ops: Array>, + behavior: { + observe?: (id: string) => Promise<{ + category: "waiting" | "ready" | "completed" | "terminal"; + }>; + finalize?: (id: string) => Promise; + reopen?: (id: string) => Promise; + } = {}, + ) { + const finalize = mock( + behavior.finalize ?? + (async (_id: string) => ({ state: "finalized" })), + ); + const observePendingOperation = mock( + behavior.observe ?? (async () => ({ category: "waiting" as const })), + ); + const reopenFailedOperation = mock( + behavior.reopen ?? (async (_id: string) => true), + ); + const byId = new Map(ops.map((op) => [op.id as string, op])); + const source = { + ops: { + mint: { + listPending: async () => + ops.filter( + (op) => op.state === "pending" || op.state === "executing", + ), + get: async (id: string) => byId.get(id) ?? null, + finalize, + }, + }, + mintOperationService: { observePendingOperation }, + reopenFailedOperation, + } as unknown as MintQuoteRecoverySource; + return { source, finalize, observePendingOperation, reopenFailedOperation }; + } + + it("mints the stored outputs for a quote the mint reports PAID", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 1, waiting: 0 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(finalize.mock.calls[0]?.[0]).toBe("op-1"); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("leaves an unpaid quote pending", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "waiting" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 0, waiting: 1 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports a quote the mint can no longer issue", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "terminal" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ terminal: 1, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("retries later when the mint is unreachable", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => { + throw new Error("fetch failed"); + }, + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toHaveLength(1); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("does not count a failed finalize as a recovery", async () => { + // coco returns a terminal operation instead of throwing when the mint + // refuses, so a fulfilled finalize is not evidence that sats were claimed. + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ + state: "failed", + error: "Recovered: quote quote-1 expired while executing mint", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors[0]?.error).toContain("expired"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("does not count a finalized-with-error operation as a recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + finalize: async () => ({ + state: "finalized", + error: "Recovered issued quote quote-1 but no proofs could be restored", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors).toHaveLength(1); + }); + + it("bounds finalize so one hung mint cannot block recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const started = Date.now(); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + + expect(Date.now() - started).toBeLessThan(5_000); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + }); + + it("recovers an interrupted mint without re-checking the quote", async () => { + const { source, finalize, observePendingOperation } = fakeSource([ + mintOp({ state: "executing" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it("skips failed operations unless the caller opts in", async () => { + const { source, reopenFailedOperation } = fakeSource([ + mintOp({ state: "failed", lastObservedRemoteState: "PAID" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 0, recovered: 0, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("re-opens a named failed operation, then mints it", async () => { + const { source, reopenFailedOperation, finalize } = fakeSource( + [mintOp({ state: "failed", lastObservedRemoteState: "PAID" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledWith("op-1"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("re-opens a named failed operation even without a PAID observation", async () => { + // The old local-fail bug left quotes with a stale or missing observation, + // which is exactly when an operator needs to retry them. + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledTimes(1); + }); + + it("skips an operation that is no longer failed when re-opened", async () => { + const { source, finalize } = fakeSource( + [mintOp({ state: "failed" })], + { reopen: async () => false }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 0, checked: 0, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports an unknown operation id instead of throwing", async () => { + const { source } = fakeSource([]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["missing"], + }); + + expect(result.checked).toBe(0); + expect(result.errors).toEqual([ + { operationId: "missing", error: "operation not found" }, + ]); + }); + + it("deduplicates repeated operation ids", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1", "op-1"], + }); + + expect(result.checked).toBe(1); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("ignores finalized operations even when targeted", async () => { + const { source, finalize } = fakeSource([mintOp({ state: "finalized" })]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + }); + + expect(result.checked).toBe(0); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("leaves a non-terminal finalize result for a later run, not terminal", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ state: "pending" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(result.errors[0]?.error).toContain("will retry"); + }); + + it("skips operations whose earlier recovery is still in flight", async () => { + const outstanding = new Map>([ + ["op-1", new Promise(() => {})], + ]); + const { source, finalize, observePendingOperation } = fakeSource( + [mintOp()], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { outstanding }); + + expect(result).toMatchObject({ busy: 1, checked: 0, recovered: 0 }); + expect(observePendingOperation).not.toHaveBeenCalled(); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("keeps a timed-out finalize registered so a retry waits", async () => { + const outstanding = new Map>(); + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has("op-1")).toBe(true); + // The abandoned mint request must not be retried underneath. + expect(second).toMatchObject({ busy: 1, checked: 0 }); + }); + + it("does not re-open a failed operation whose recovery is in flight", async () => { + const outstanding = new Map>([ + ["op-1", new Promise(() => {})], + ]); + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + {}, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + outstanding, + }); + + expect(result).toMatchObject({ busy: 1, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("counts an in-progress operation as busy rather than retryable", async () => { + const { source } = fakeSource( + [mintOp({ state: "failed" })], + { + reopen: async () => { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + }, + }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ busy: 1, retryable: 0, reopened: 0 }); + }); + + it("keeps a timed-out quote check registered so a retry waits", async () => { + // observePendingOperation is not read-only, so a hung check must not be + // retried underneath: it could persist a stale observation later. + const outstanding = new Map>(); + const { source, finalize } = fakeSource([mintOp()], { + observe: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, checked: 1 }); + expect(outstanding.has("op-1")).toBe(true); + expect(second).toMatchObject({ busy: 1, checked: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); +}); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index c8077c8..ebfec2d 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -37,6 +37,11 @@ import type { WalletRecoveryProgress, } from "./cocod-client"; import { selectCleanupOperations } from "./cleanup"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; import { clearInterruptedReceiveReservations, deleteReceiveTokenReservation, @@ -579,6 +584,96 @@ interface MintOperationServiceCleanup { observePendingOperation( operationId: string, ): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>; + /** + * Acquire coco's per-operation lock for `operationId` and return its release + * function. coco's execute/finalize/recover paths take the same lock, so + * holding it across a read-check-write makes the transition atomic with + * respect to them. + */ + acquireOperationLock(operationId: string): Promise<() => void>; + /** Reload a mint operation row, or null when it no longer exists. */ + getOperation(operationId: string): Promise | null>; + /** + * Put a terminally failed operation back into `pending`. + * + * coco keeps this private, and it spreads whatever it is handed into the row + * it writes. The sqlite repository rewrites every column, so callers MUST + * pass a freshly reloaded full row: a partial object such as `{ id }` would + * erase `outputDataJson` and make the paid sats unrecoverable. + */ + transitionToPending( + op: Record, + error?: string, + ): Promise; +} + +/** + * Re-open a terminally failed mint operation so recovery can retry it. + * + * Two details make this safe: + * + * - The persisted row is reloaded and handed to coco in full. coco spreads + * whatever it is given and the sqlite repository rewrites every column, so a + * partial object would be rejected by the NOT NULL schema or, on a more + * permissive adapter, erase the stored outputs. + * - The read-check-write runs under coco's per-operation lock, the same lock + * coco's execute/finalize/recover paths take. Reloading alone only narrows + * the race: without the lock two concurrent recoveries could both see + * `failed` and the slower one would clobber a newer state. + * + * The lock is fail-fast rather than wait-based: coco's `OperationIdLock.acquire` + * throws `OperationInProgressError` when the id is already locked. So either + * this helper holds the lock - and coco's own execute/finalize/recover paths + * cannot interleave, because acquiring would throw for them too - or it throws + * and writes nothing. It never waits, and never writes without the lock, which + * is why a stale `failed` snapshot cannot clobber a newer state. + * + * Scope of that lock, in this coco version: `recordPendingObservation` and + * `failPendingOperation` write without taking it. The justified claim is + * therefore narrow - a re-open cannot clobber a concurrent executing/recovery + * pass - not a general guarantee against every watcher write. + * + * This is a compatibility shim over private coco internals, so it fails closed: + * if any of the expected methods are missing it throws before writing. That + * check only catches removals, not changed behaviour under the same name: it + * was written against @cashu/coco-core 1.0.1, so any coco bump must re-run the + * real-Manager and fake-mint integration tests. The long-term fix is an + * upstream public `reopenFailedOperation(id)` that takes the same lock, reloads + * the full row, preserves the outputs and emits the usual events. + */ +export async function reopenFailedMintOperation( + service: Pick< + MintOperationServiceCleanup, + "acquireOperationLock" | "getOperation" | "transitionToPending" + >, + operationId: string, +): Promise { + for (const method of [ + "acquireOperationLock", + "getOperation", + "transitionToPending", + ] as const) { + if (typeof service[method] !== "function") { + throw new Error( + `coco mintOperationService.${method} is unavailable; refusing to re-open a failed mint operation`, + ); + } + } + const release = await service.acquireOperationLock(operationId); + try { + const current = await service.getOperation(operationId); + if (!current) throw new Error(`Operation ${operationId} not found`); + if (current.state !== "failed") return false; + // Clearing the terminal-failure marker keeps the re-opened row from + // looking terminally failed to readers that inspect it alongside `state`. + await service.transitionToPending( + { ...current, terminalFailure: undefined }, + undefined, + ); + return true; + } finally { + release(); + } } export interface CreateCocoClientOptions { @@ -671,6 +766,57 @@ export interface ExpiredMintSettlement { unobserved: number; } +/** Outcome of asking a mint about one expired pending quote. */ +export type ExpiredMintQuoteOutcome = + | "failed" + | "leftForRecovery" + | "unobserved"; + +/** + * Decide one expired pending quote's fate by asking the mint. + * + * Expiry alone does not prove the quote was never paid: the Lightning payment + * can land just before expiry while the daemon is down, leaving no local + * observation. A quote the mint still reports UNPAID can never be issued and + * is safe to fail locally; anything else (PAID/ISSUED, or a mint that cannot + * answer) stays pending so recovery can still claim the sats. + */ +export async function failExpiredMintQuoteIfUnpaid( + mintService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" | "failPendingOperation" + >, + operationId: string, + timeoutMs: number, +): Promise<{ + outcome: ExpiredMintQuoteOutcome; + category?: "waiting" | "ready" | "completed" | "terminal"; + error?: unknown; +}> { + try { + const observation = await withTimeout( + mintService.observePendingOperation(operationId), + timeoutMs, + ); + if (observation.category !== "waiting") { + return { outcome: "leftForRecovery", category: observation.category }; + } + // The mint confirms the expired quote is still unpaid: it can never be + // issued now, so failing it locally cannot strand funds. + await mintService.failPendingOperation( + { id: operationId }, + { + reason: "Expired mint quote confirmed unpaid by mint", + retryable: false, + observedAt: Date.now(), + }, + ); + return { outcome: "failed", category: observation.category }; + } catch (error) { + return { outcome: "unobserved", error }; + } +} + /** * Settle expired pending mint quotes before the mint recovery sweep runs. * @@ -726,45 +872,37 @@ export async function settleExpiredMintQuotes( break; } - try { - const result = await withTimeout( - source.mintOperationService.observePendingOperation(op.id), - remainingMs, + const check = await failExpiredMintQuoteIfUnpaid( + source.mintOperationService, + op.id, + remainingMs, + ); + if (check.outcome === "failed") { + settlement.failed++; + } else if (check.outcome === "leftForRecovery") { + // PAID/ISSUED (or terminally failed) at the mint: normal recovery + // must see this quote so paid proofs get claimed. + settlement.leftForRecovery++; + const observed = + check.category === "ready" + ? "was paid at the mint" + : check.category === "completed" + ? "was already issued at the mint" + : "failed terminally at the mint"; + startupProgress( + `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, ); - if (result.category === "waiting") { - // The mint confirms the expired quote is still unpaid: it can never - // be issued now, so failing it locally cannot strand funds. - await source.mintOperationService.failPendingOperation( - { id: op.id }, - { - reason: "Expired mint quote confirmed unpaid by mint", - retryable: false, - observedAt: Date.now(), - }, - ); - settlement.failed++; - } else { - // PAID/ISSUED (or terminally failed) at the mint: normal recovery - // must see this quote so paid proofs get claimed. - settlement.leftForRecovery++; - const observed = - result.category === "ready" - ? "was paid at the mint" - : result.category === "completed" - ? "was already issued at the mint" - : "failed terminally at the mint"; - startupProgress( - `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, - ); - } - } catch (error) { + } else { // Mint unreachable, too slow, or the quote unknown to it: leave the // operation pending so a later startup can still recover it. settlement.unobserved++; logger.warn("Could not check expired mint quote; leaving it pending", { operationId: op.id, mintUrl: op.mintUrl, - error: error instanceof Error ? error.message : String(error), + error: + check.error instanceof Error + ? check.error.message + : String(check.error), }); } } @@ -772,6 +910,345 @@ export async function settleExpiredMintQuotes( return settlement; } +/** + * Source for explicit PAID mint-quote recovery. + * + * Unlike the startup sweeps this also accepts caller-supplied operation ids so + * an operator can target a quote coco already gave up on (state `failed`). + */ +export interface MintQuoteRecoverySource { + ops: { + mint: { + listPending(): Promise; + get(operationId: string): Promise; + finalize(operationId: string): Promise; + }; + }; + mintOperationService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" + >; + /** + * Re-open a failed operation so it can be recovered; false when it is no + * longer failed. Implementations must reload the full row (see + * `reopenFailedMintOperation`). + */ + reopenFailedOperation(operationId: string): Promise; +} + +export interface MintQuoteRecoveryOptions { + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Per-quote budget for observing the mint and finalizing the operation. */ + timeoutMs?: number; + /** + * Re-open failed operations instead of skipping them. Only applies to + * operations named by `operationIds`: coco's pending listing never returns + * failed operations, so they can only be recovered by explicit id. + */ + includeFailed?: boolean; + /** + * In-flight recovery work keyed by operation id, shared across runs. + * withTimeout does not cancel the underlying request, so a timed-out quote + * check or finalize must keep blocking a retry until it actually settles. + */ + outstanding?: Map>; +} + +export interface MintQuoteRecoveryResult { + /** Operations recovery acted on. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** + * Quotes that ended terminally: the mint can no longer issue them, or coco + * finalised them without recovering any proofs. + */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** + * Operations left to a later run: the mint was unreachable, the per-quote + * budget ran out, or the operation ended in a non-terminal state. + */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + +/** + * Run async tasks strictly one after another. + * + * Used to serialize explicit wallet recovery: two concurrent requests must not + * both snapshot the same failed operation, and a retry must not start + * underneath work that outlived its timeout. A rejected task never breaks the + * chain for the next one. + */ +export function createRunQueue(): (run: () => Promise) => Promise { + let tail: Promise = Promise.resolve(); + return (run: () => Promise): Promise => { + const result = tail.then(run, run); + tail = result.then( + () => undefined, + () => undefined, + ); + return result; + }; +} + +/** Per-quote budget for the mint round-trip during explicit recovery. */ +const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000; + +/** + * Recover mint quotes whose sats are PAID at the mint but were never claimed. + * + * For every target the mint is asked for the current quote state, and only it + * decides the outcome: PAID quotes have their stored outputs submitted, ISSUED + * quotes have their signatures restored (NUT-09), UNPAID quotes are left + * pending, and quotes the mint can no longer issue are reported rather than + * silently dropped. Anything the mint cannot answer is retried later. + * + * `finalize()` does not throw when the mint refuses to issue or when an + * already-issued quote's proofs cannot be restored: it returns a terminal + * operation instead. Recovery therefore inspects the returned operation's + * state and error and only counts a genuine finalized-without-error as + * recovered. + * + * Failed operations are skipped unless `includeFailed` is set, and they can + * only be targeted by explicit id because coco's pending listing never returns + * them. Re-opening an operation is a mutation, so it happens only here, never + * during startup recovery. + * + * Callers should serialize their own invocations and pass a shared + * `outstanding` map: `timeoutMs` bounds the wait but does not cancel the + * request behind it, so both a timed-out quote check and a timed-out finalize + * keep blocking a retry until they actually settle. + */ +export async function runMintQuoteRecovery( + source: MintQuoteRecoverySource, + options: MintQuoteRecoveryOptions = {}, + onProgress?: (message: string) => void, +): Promise { + const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS; + const outstanding = + options.outstanding ?? new Map>(); + const result: MintQuoteRecoveryResult = { + checked: 0, + recovered: 0, + waiting: 0, + terminal: 0, + reopened: 0, + retryable: 0, + busy: 0, + errors: [], + }; + const messageOf = (error: unknown) => + error instanceof Error ? error.message : String(error); + /** coco's fail-fast operation lock rejected the call: another holder exists. */ + const isInProgress = (error: unknown) => + error instanceof Error && error.name === "OperationInProgressError"; + /** + * Register in-flight work for an operation. Entries are cleared only once the + * work actually settles (withTimeout does not cancel the request behind it), + * so a timed-out call keeps blocking a retry. The identity check stops a late + * settlement from clearing a newer entry for the same operation. + */ + const track = (operationId: string, work: Promise) => { + outstanding.set(operationId, work); + const clear = () => { + if (outstanding.get(operationId) === work) { + outstanding.delete(operationId); + } + }; + void work.then(clear, clear); + }; + + let targets: MintQuoteRecoveryCandidate[]; + if (options.operationIds && options.operationIds.length > 0) { + targets = []; + const seen = new Set(); + for (const operationId of options.operationIds) { + if (seen.has(operationId)) continue; + seen.add(operationId); + try { + const op = await source.ops.mint.get(operationId); + if (!op) { + result.errors.push({ operationId, error: "operation not found" }); + continue; + } + targets.push(op); + } catch (error) { + result.errors.push({ operationId, error: messageOf(error) }); + } + } + } else { + targets = await source.ops.mint.listPending(); + } + + const { pending, failed } = selectMintQuotesForRecovery({ + mints: targets, + includeFailed: options.includeFailed === true, + }); + + for (const op of failed) { + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(op.id)) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + continue; + } + try { + if (!(await source.reopenFailedOperation(op.id))) { + onProgress?.(`${label}: no longer failed; skipped`); + continue; + } + result.reopened++; + onProgress?.(`${label}: re-opened failed operation for recovery`); + } catch (error) { + // coco's operation lock is fail-fast, so an in-progress error means a + // processor or another recovery holds the operation right now. + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery holds it; skipped`); + } else { + result.retryable++; + onProgress?.(`${label}: could not re-open: ${messageOf(error)}`); + } + result.errors.push({ operationId: op.id, error: messageOf(error) }); + continue; + } + await recoverOne(op); + } + + for (const op of pending) await recoverOne(op); + + return result; + + async function recoverOne(op: MintQuoteRecoveryCandidate): Promise { + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(op.id)) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + return; + } + result.checked++; + // One budget per operation, shared by the mint check and the finalize, so + // a slow mint cannot silently double the documented per-quote wait. + const deadlineAt = Date.now() + timeoutMs; + const remaining = () => Math.max(1, deadlineAt - Date.now()); + + if (op.state === "executing") { + // A crash mid-mint can leave outputs already signed at the mint; + // finalize recovers them instead of minting a second time. + await finalizeAndClassify( + op.id, + label, + "recovered interrupted mint", + remaining, + ); + return; + } + + let observation: { + category: "waiting" | "ready" | "completed" | "terminal"; + }; + // observePendingOperation is not read-only: it emits quote-state-changed, + // persists the observation and can fail a terminal operation. Track it too, + // so a timed-out check cannot be retried and then persist a stale read. + const check = source.mintOperationService.observePendingOperation(op.id); + track(op.id, check); + try { + observation = await withTimeout(check, remaining()); + } catch (error) { + result.retryable++; + result.errors.push({ operationId: op.id, error: messageOf(error) }); + onProgress?.(`${label}: could not check with mint: ${messageOf(error)}`); + return; + } + + const decision = classifyMintQuoteObservation(observation.category); + if (decision.action === "finalize") { + await finalizeAndClassify( + op.id, + label, + decision.observedRemoteState === "PAID" + ? `paid, minting proofs (${op.amount} sat)` + : `already issued, restoring proofs (${op.amount} sat)`, + remaining, + ); + } else if (decision.action === "waiting") { + result.waiting++; + onProgress?.(`${label}: mint reports UNPAID; left pending`); + } else { + // coco records the mint's terminal verdict by failing the operation. + result.terminal++; + onProgress?.(`${label}: mint can no longer issue this quote`); + } + } + + /** + * Run finalize and classify its result. coco returns a terminal operation + * rather than throwing when the mint refuses (for example an expired quote) + * or when an already-issued quote's proofs could not be restored, so a + * fulfilled promise is not by itself evidence that sats were recovered. + */ + async function finalizeAndClassify( + operationId: string, + label: string, + successMessage: string, + remaining: () => number, + ): Promise { + const work = source.ops.mint.finalize(operationId); + track(operationId, work); + let terminal: { state?: string; error?: string } | null | undefined; + try { + terminal = (await withTimeout(work, remaining())) as + | { state?: string; error?: string } + | null + | undefined; + } catch (error) { + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery is working on it; skipped`); + } else { + result.retryable++; + onProgress?.(`${label}: could not finish recovery: ${messageOf(error)}`); + } + result.errors.push({ operationId, error: messageOf(error) }); + return; + } + if (terminal?.state === "finalized" && !terminal.error) { + result.recovered++; + onProgress?.(`${label}: ${successMessage}`); + return; + } + if ( + terminal?.state === "failed" || + (terminal?.state === "finalized" && terminal.error) + ) { + result.terminal++; + const detail = + terminal.error ?? `left in state ${terminal.state ?? "unknown"}`; + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: not recovered: ${detail}`); + return; + } + // Pending/executing/unknown: coco may still be working on the operation, + // so leave it to a later run rather than calling it terminal. + result.retryable++; + result.errors.push({ + operationId, + error: `left in state ${terminal?.state ?? "unknown"}; will retry`, + }); + onProgress?.( + `${label}: still ${terminal?.state ?? "unknown"}; left for a later run`, + ); + } +} + const PENDING_MINT_SWEEP_INTERVAL_MS = 15_000; /** Per-quote wait inside a sweep, so one stalled mint cannot starve the rest. */ const PENDING_MINT_CHECK_TIMEOUT_MS = 10_000; @@ -1397,6 +1874,10 @@ export async function createCocoClient( }; let disposed = false; + // Explicit recovery runs are serialized, and finalize work that outlives its + // timeout stays in the map so a retry waits for it. + const enqueueRecovery = createRunQueue(); + const recoveryOutstanding = new Map>(); /** * Block a value-moving operation until background recovery has settled. @@ -1748,6 +2229,7 @@ export async function createCocoClient( await waitForRecovery(); const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000; const dryRun = options.dryRun === true; + const force = options.force === true; const nowMs = Date.now(); const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([ @@ -1775,6 +2257,8 @@ export async function createCocoClient( }); const errors: WalletCleanupResult["errors"] = []; + let failedMintQuotes = 0; + let leftForRecovery = 0; if (!dryRun) { const mintService = ( @@ -1784,20 +2268,49 @@ export async function createCocoClient( ).mintOperationService; for (const op of selection.mintsToFail) { - try { - await mintService.failPendingOperation( - { id: op.id }, - { - reason: "Expired unpaid mint quote cleaned up by routstrd", - retryable: false, - observedAt: nowMs, - }, - ); - } catch (error) { - errors.push({ - operationId: op.id, - error: error instanceof Error ? error.message : String(error), - }); + if (force) { + // Legacy behaviour: fail the quote locally without asking the mint. + try { + await mintService.failPendingOperation( + { id: op.id }, + { + reason: "Expired mint quote cleaned up by routstrd (forced)", + retryable: false, + observedAt: nowMs, + }, + ); + failedMintQuotes++; + } catch (error) { + errors.push({ + operationId: op.id, + error: error instanceof Error ? error.message : String(error), + }); + } + continue; + } + // Expiry alone does not prove the quote was never paid: the + // Lightning payment can land before expiry while the daemon is down. + // Confirm UNPAID with the mint before failing, exactly as startup + // recovery does; paid quotes are left for recovery to finalize. + const check = await failExpiredMintQuoteIfUnpaid( + mintService, + op.id, + EXPIRED_MINT_OBSERVATION_DEADLINE_MS, + ); + if (check.outcome === "failed") { + failedMintQuotes++; + } else { + leftForRecovery++; + if (check.outcome === "unobserved") { + errors.push({ + operationId: op.id, + error: `could not confirm quote state with mint: ${ + check.error instanceof Error + ? check.error.message + : String(check.error) + }`, + }); + } } } @@ -1824,8 +2337,12 @@ export async function createCocoClient( } } + const failedMintQuoteCount = dryRun + ? selection.mintsToFail.length + : failedMintQuotes; const actedOn = - selection.mintsToFail.length + + failedMintQuoteCount + + leftForRecovery + selection.sendsToReclaim.length + selection.meltsToCancel.length; const skipped = @@ -1834,12 +2351,37 @@ export async function createCocoClient( return { dryRun, - failedMintQuotes: selection.mintsToFail.length, + failedMintQuotes: failedMintQuoteCount, + leftForRecovery, reclaimedSends: selection.sendsToReclaim.length, cancelledMelts: selection.meltsToCancel.length, skipped, errors, }; }, + + async recoverMintQuotes(options, onProgress) { + await waitForRecovery(); + const service = ( + coco as unknown as { + mintOperationService: MintOperationServiceCleanup; + } + ).mintOperationService; + // Serialize explicit recovery: two concurrent requests must not both + // snapshot the same failed operation, and a retry must not start + // underneath a finalize that outlived its timeout. + return enqueueRecovery(() => + runMintQuoteRecovery( + { + ops: coco.ops as unknown as MintQuoteRecoverySource["ops"], + mintOperationService: service, + reopenFailedOperation: (operationId) => + reopenFailedMintOperation(service, operationId), + }, + { ...options, outstanding: recoveryOutstanding }, + onProgress, + ), + ); + }, }; } diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index d338506..b53bf88 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -83,6 +83,12 @@ export interface WalletCleanupOptions { minAgeMs?: number; /** Report what would be cleaned without applying changes. */ dryRun?: boolean; + /** + * Fail expired mint quotes without confirming UNPAID with the mint. Only for + * operators who accept the risk of stranding a quote that was paid before + * its invoice expired; recovery is the safe default. + */ + force?: boolean; } /** Summary of a wallet cleanup run. */ @@ -90,6 +96,8 @@ export interface WalletCleanupResult { dryRun: boolean; /** Number of expired pending mint quotes marked as failed. */ failedMintQuotes: number; + /** Expired quotes whose mint reported PAID/ISSUED, left for recovery. */ + leftForRecovery: number; /** Number of stale pending send operations reclaimed. */ reclaimedSends: number; /** Number of stale prepared melt operations cancelled. */ @@ -120,6 +128,35 @@ export interface MintQuoteStatus { error?: string; } +/** Options for explicit PAID mint-quote recovery. */ +export interface WalletMintQuoteRecoveryOptions { + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Re-open failed operations instead of skipping them. */ + includeFailed?: boolean; + /** Per-quote mint timeout in milliseconds. */ + timeoutMs?: number; +} + +/** Summary of a PAID mint-quote recovery run. */ +export interface WalletMintQuoteRecoveryResult { + /** Operations whose quote state was checked with the mint. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** Quotes the mint can no longer issue. */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** Operations left to a later run (mint unreachable, budget spent, non-terminal). */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + export interface CocodClient { ping(): Promise; getStatus(): Promise; @@ -152,6 +189,14 @@ export interface CocodClient { cleanupStuckOperations?( options?: WalletCleanupOptions, ): Promise; + /** + * Re-issue PAID mint quotes whose sats were never claimed, optionally + * targeting specific operations (including ones coco already failed). + */ + recoverMintQuotes?( + options?: WalletMintQuoteRecoveryOptions, + onProgress?: (message: string) => void, + ): Promise; /** Report background wallet recovery progress, when the wallet supports it. */ getRecoveryProgress?(): Promise; } diff --git a/src/daemon/wallet/mint-operation-reopen.test.ts b/src/daemon/wallet/mint-operation-reopen.test.ts new file mode 100644 index 0000000..9197fec --- /dev/null +++ b/src/daemon/wallet/mint-operation-reopen.test.ts @@ -0,0 +1,163 @@ +/** + * Re-opening a failed mint operation is the one genuinely destructive step of + * PAID-quote recovery, because coco's private `transitionToPending` spreads + * whatever it is handed and `SqliteMintOperationRepository.update` rewrites + * every column. A partial object such as `{ id }` is therefore rejected by the + * NOT NULL schema, and on a more permissive adapter would overwrite `quoteId`, + * `amount`, `request`, `lastObservedRemoteState` and `outputDataJson` with NULL, + * destroying the material needed to claim the paid sats. + * + * These tests drive the production `reopenFailedMintOperation` helper against a + * real coco sqlite repository through adapter-backed getOperation and + * transitionToPending implementations, so a regression at the helper/service + * boundary is caught rather than a mock standing in for it. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow(state = "failed") { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state, + createdAt: 1_000, + updatedAt: 2_000, + error: state === "failed" ? "expired" : undefined, + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: + state === "failed" ? { reason: "expired", observedAt: 3_000 } : undefined, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation against real coco sqlite", () => { + let dir: string | undefined; + let database: Database | undefined; + + afterEach(() => { + database?.close(); + database = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function repos() { + dir = mkdtempSync(join(tmpdir(), "routstrd-reopen-")); + database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + return repositories; + } + + function readRow(repositories: SqliteRepositories, id: string) { + return repositories.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + /** + * Adapter-backed stand-in for the private coco service methods the helper + * uses: getOperation reads and transitionToPending mirrors coco's + * spread-and-update implementation. + */ + function serviceOver(repositories: SqliteRepositories) { + return { + acquireOperationLock: async (_id: string) => () => {}, + getOperation: (id: string) => readRow(repositories, id), + transitionToPending: async ( + op: Record, + error?: string, + ) => { + await repositories.mintOperationRepository.update({ + ...op, + state: "pending", + error, + } as never); + }, + }; + } + + it("re-opens a failed row without losing quote metadata or stored outputs", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(true); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.unit).toBe("sat"); + expect(row?.request).toBe("lnbc1example"); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + }); + + it("is a no-op when the operation is no longer failed", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow("finalized") as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(false); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("finalized"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + }); + + it("rejects a partial row, which is why the helper reloads in full", async () => { + // Locks in the reason for reloading. If a future coco version accepts + // partial updates this fails, and the helper can be simplified rather than + // silently losing paid sats. + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + await expect( + repositories.mintOperationRepository.update({ + id: "op-1", + state: "pending", + updatedAt: Date.now(), + } as never), + ).rejects.toThrow(); + + const unchanged = await readRow(repositories, "op-1"); + expect(unchanged?.state).toBe("failed"); + expect(unchanged?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts new file mode 100644 index 0000000..20715b5 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -0,0 +1,356 @@ +/** + * End-to-end PAID mint-quote recovery against a real coco Manager, real sqlite + * and a real in-process mint that produces genuine blind signatures. + * + * Nothing here mocks the wallet: a quote is created through coco, the mint is + * told what to report, and the production `runMintQuoteRecovery` drives the + * outcome. These are the release-gating scenarios for the feature, and they are + * the only tests that exercise issuance and NUT-09 restore over HTTP. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint"; +import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client"; + +type AnyRecord = Record; + +interface Booted { + manager: Manager; + repositories: SqliteRepositories; + mint: FakeMint; + /** Build the recovery source the production function expects. */ + source: () => AnyRecord; + spendable: () => Promise; + close: () => Promise; +} + +async function boot(options: { quoteExpiry?: number | null } = {}) { + const mint = new FakeMint(); + mint.quoteExpiry = options.quoteExpiry ?? null; + mint.start(); + const dir = mkdtempSync(join(tmpdir(), "routstrd-fakemint-")); + const database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + const manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await manager.mint.addMint(mint.url, { trusted: true }); + + const service = (manager as unknown as { mintOperationService: AnyRecord }) + .mintOperationService; + + const booted: Booted = { + manager, + repositories, + mint, + spendable: async () => { + const balances = (await manager.wallet.balances.byMint()) as Record< + string, + { spendable: number } + >; + return balances[mint.url]?.spendable ?? 0; + }, + source: () => ({ + ops: { + mint: { + listPending: () => manager.ops.mint.listPending(), + get: (id: string) => manager.ops.mint.get(id), + finalize: (id: string) => manager.ops.mint.finalize(id), + }, + }, + mintOperationService: service, + reopenFailedOperation: (id: string) => + reopenFailedMintOperation(service as never, id), + }), + close: async () => { + await manager.dispose().catch(() => undefined); + database.close(); + mint.stop(); + rmSync(dir, { recursive: true, force: true }); + }, + }; + + return booted; +} + +async function prepareQuote(booted: Booted, amount: number) { + const op = (await booted.manager.ops.mint.prepare({ + mintUrl: booted.mint.url, + amount, + method: "bolt11", + } as never)) as unknown as AnyRecord; + return op; +} + +function outputsOf(op: AnyRecord) { + // coco stores mint outputs as { keep, send }, like the on-disk output JSON. + const outputData = (op.outputData as AnyRecord).keep as Array<{ + blindedMessage: { amount: unknown; id: string; B_: string }; + }>; + return outputData.map((output) => ({ + amount: Number(String(output.blindedMessage.amount)), + id: output.blindedMessage.id, + B_: output.blindedMessage.B_, + })); +} + +let booted: Booted | undefined; +afterEach(async () => { + await booted?.close(); + booted = undefined; +}); + +describe("PAID mint quote recovery with a real Manager and mint", () => { + it("issues an expired-but-PAID quote with the operation's own outputs, once", async () => { + // The motivating case: the invoice expired, but the mint says PAID and has + // issued nothing. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 210_000); + const expectedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ checked: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(210_000); + // Issuance used exactly the blinded outputs stored on the operation. + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...expectedOutputs].sort(), + ); + + // A second run must not mint again or double-credit. + const again = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(again).toMatchObject({ checked: 0, recovered: 0 }); + expect(booted.mint.requests).toHaveLength(1); + expect(await booted.spendable()).toBe(210_000); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 210_000); + // Another wallet issued it: the signatures exist at the mint for the very + // outputs this operation stored. + booted.mint.signFor(op.quoteId as string, outputsOf(op)); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 1, terminal: 0, retryable: 0 }); + expect(await booted.spendable()).toBe(210_000); + }); + + it("reports terminal without credit when the mint refuses issuance", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + booted.mint.mintError = { code: QUOTE_EXPIRED, detail: "quote expired" }; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect((result.errors as unknown[]).length).toBeGreaterThan(0); + expect(await booted.spendable()).toBe(0); + }); + + it("reports terminal without credit when an issued quote cannot be restored", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint, but the signatures for our outputs are gone. + booted.mint.markIssued(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("does not credit a quote when the mint returns null NUT-09 signatures", async () => { + // KNOWN INTEROP GAP, not a supported path. NUT-09 permits `null` in the + // positional `signatures` array for outputs the mint never signed, but + // cashu-ts 3.7.1 - which coco depends on - dereferences every entry while + // normalising amounts, so the wallet throws instead of skipping the null. + // Recovery therefore surfaces an error and credits nothing, leaving the + // operation for a later run. + // + // This test pins the current behaviour so the gap cannot quietly disappear. + // Revisit (and change this expectation) once coco's cashu-ts parses + // positional nulls, and file/track it upstream in the meantime. + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint with nothing signed for this operation's outputs. + booted.mint.markIssued(op.quoteId as string); + booted.mint.restoreIncludesNulls = true; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("recovers a failed operation whose stale history says UNPAID", async () => { + // The composed path the feature exists for: a real prepared operation, + // failed locally, with a stale UNPAID observation from the old local-fail + // behaviour, recovered by explicit id and issued with its own outputs. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + const storedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const row = (await booted.repositories.mintOperationRepository.getById( + op.id as string, + )) as unknown as AnyRecord; + expect(row.outputData).toBeDefined(); + await booted.repositories.mintOperationRepository.update({ + ...row, + state: "failed", + lastObservedRemoteState: "UNPAID", + error: "Expired unpaid mint quote cleaned up by routstrd", + terminalFailure: { reason: "expired", observedAt: Date.now() }, + } as never); + + // A purely local decision would skip this row; the mint has the last word. + const result = (await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + includeFailed: true, + })) as unknown as Record; + + expect(result).toMatchObject({ reopened: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...storedOutputs].sort(), + ); + }); + + it("counts a locked operation as busy instead of minting underneath it", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + const service = ( + booted.manager as unknown as { + mintOperationService: { acquireOperationLock(id: string): Promise<() => void> }; + } + ).mintOperationService; + + // Hold the operation, as a processor or another recovery would. + const release = await service.acquireOperationLock(op.id as string); + const blocked = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(blocked).toMatchObject({ recovered: 0 }); + expect((blocked.busy ?? 0) + (blocked.retryable ?? 0)).toBeGreaterThan(0); + expect(booted.mint.requests).toHaveLength(0); + expect(await booted.spendable()).toBe(0); + + release(); + const after = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(after).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("does not mint a second time while issuance is in flight", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Hold the mint's response so the first recovery is visibly in flight. + let releaseGate!: () => void; + booted.mint.gate = new Promise((resolve) => { + releaseGate = resolve; + }); + const first = runMintQuoteRecovery( + booted.source() as never, + ) as unknown as Promise>; + + for (let i = 0; i < 400 && booted.mint.requests.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(booted.mint.requests).toHaveLength(1); + + // A concurrent run must not issue again while that request is outstanding. + await runMintQuoteRecovery(booted.source() as never); + expect(booted.mint.requests).toHaveLength(1); + + releaseGate(); + expect(await first).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("coexists with coco's own mint operation watcher and processor", async () => { + booted = await boot({ quoteExpiry: -60 }); + // The real background machinery coco uses to settle pending mint quotes. + await booted.manager.enableMintOperationWatcher(); + await booted.manager.enableMintOperationProcessor(); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Either our recovery or coco's processor may win; both are safe. + await runMintQuoteRecovery(booted.source() as never); + + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + + // Give the processor time to act and confirm nothing is credited twice. + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + }); + + it("tracks a hung quote check so a later run waits instead of re-reading", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + let releaseObserve!: () => void; + booted.mint.observeGate = new Promise((resolve) => { + releaseObserve = resolve; + }); + const outstanding = new Map>(); + + const first = (await runMintQuoteRecovery(booted.source() as never, { + timeoutMs: 30, + outstanding, + })) as unknown as Record; + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has(op.id as string)).toBe(true); + + const second = (await runMintQuoteRecovery(booted.source() as never, { + outstanding, + })) as unknown as Record; + expect(second).toMatchObject({ checked: 0, busy: 1 }); + + // Once the held check settles the tracking drains, and recovery proceeds. + releaseObserve(); + for (let i = 0; i < 400 && outstanding.size > 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(outstanding.size).toBe(0); + + const third = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(third).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.manager.test.ts b/src/daemon/wallet/mint-quote-recovery.manager.test.ts new file mode 100644 index 0000000..5c779bf --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.manager.test.ts @@ -0,0 +1,159 @@ +/** + * Real-Manager integration for re-opening a failed mint operation. + * + * The unit tests exercise `reopenFailedMintOperation` against a hand-written + * service double, which cannot catch a change in coco's own + * `MintOperationService.transitionToPending` semantics or in its per-operation + * lock. This drives the production helper against an actual coco `Manager` + * backed by sqlite, so the private-service boundary that the helper depends on + * is exercised for real: full-row preservation, the shared operation lock, and + * the `mint-op:pending` event. + * + * No mint or network access is involved; nothing here enables the mint watcher + * or processor, which the fake-mint integration covers. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow() { + return { + id: "op-1", + mintUrl: "https://mint.invalid", + quoteId: "quote-1", + state: "failed", + createdAt: 1_000, + updatedAt: 2_000, + error: "expired", + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: { reason: "expired", observedAt: 3_000 }, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation with a real coco Manager", () => { + let dir: string | undefined; + let database: Database | undefined; + let manager: Manager | undefined; + let repositories: SqliteRepositories | undefined; + + afterEach(async () => { + await manager?.dispose().catch(() => undefined); + manager = undefined; + database?.close(); + database = undefined; + repositories = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function boot() { + dir = mkdtempSync(join(tmpdir(), "routstrd-manager-")); + database = new Database(join(dir, "coco.db")); + repositories = new SqliteRepositories({ database }); + await repositories.init(); + manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await repositories.mintOperationRepository.create(failedRow() as never); + const service = ( + manager as unknown as { + mintOperationService: { + acquireOperationLock(id: string): Promise<() => void>; + getOperation(id: string): Promise | null>; + transitionToPending( + op: Record, + error?: string, + ): Promise; + }; + } + ).mintOperationService; + const eventBus = ( + manager as unknown as { + eventBus: { on(event: string, handler: () => void): () => void }; + } + ).eventBus; + return { service, eventBus }; + } + + function readRow(id: string) { + return repositories!.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + it("re-opens through the real service and emits mint-op:pending", async () => { + const { service, eventBus } = await boot(); + const events: string[] = []; + const off = eventBus.on("mint-op:pending", () => events.push("pending")); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + off(); + + expect(reopened).toBe(true); + const row = await readRow("op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + expect(events).toEqual(["pending"]); + }); + + it("refuses to re-open while coco's operation lock is held", async () => { + const { service } = await boot(); + // coco's OperationIdLock is fail-fast: a holder blocks the re-open by + // making it throw, and nothing is written. + const release = await service.acquireOperationLock("op-1"); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /already in progress/, + ); + const blocked = await readRow("op-1"); + expect(blocked?.state).toBe("failed"); + expect(blocked?.outputData).toEqual(OUTPUT_DATA); + + release(); + expect(await reopenFailedMintOperation(service, "op-1")).toBe(true); + const reopened = await readRow("op-1"); + expect(reopened?.state).toBe("pending"); + expect(reopened?.outputData).toEqual(OUTPUT_DATA); + }); + + it("leaves an operation a processor finalized alone", async () => { + const { service } = await boot(); + // Emulate a processor winning the race while holding the same lock. + const release = await service.acquireOperationLock("op-1"); + const row = await readRow("op-1"); + await repositories!.mintOperationRepository.update({ + ...row, + state: "finalized", + } as never); + release(); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + const after = await readRow("op-1"); + expect(after?.state).toBe("finalized"); + expect(after?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.test.ts b/src/daemon/wallet/mint-quote-recovery.test.ts new file mode 100644 index 0000000..eac5de9 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "bun:test"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; + +function mint( + overrides: Partial = {}, +): MintQuoteRecoveryCandidate { + return { + id: "mint-1", + mintUrl: "https://mint.example", + quoteId: "quote-1", + state: "pending", + amount: 1000, + expiry: 0, + ...overrides, + }; +} + +describe("classifyMintQuoteObservation", () => { + it("finalizes a paid-but-unissued quote by minting its stored outputs", () => { + expect(classifyMintQuoteObservation("ready")).toEqual({ + action: "finalize", + observedRemoteState: "PAID", + }); + }); + + it("finalizes an already-issued quote by restoring its proofs", () => { + expect(classifyMintQuoteObservation("completed")).toEqual({ + action: "finalize", + observedRemoteState: "ISSUED", + }); + }); + + it("leaves an unpaid quote alone", () => { + expect(classifyMintQuoteObservation("waiting")).toEqual({ + action: "waiting", + }); + }); + + it("reports a quote the mint can no longer issue", () => { + expect(classifyMintQuoteObservation("terminal")).toEqual({ + action: "terminal", + }); + }); +}); + +describe("selectMintQuotesForRecovery", () => { + it("selects every pending quote because only the mint knows if it was paid", () => { + const result = selectMintQuotesForRecovery({ + mints: [ + mint({ id: "expired", expiry: 1 }), + mint({ id: "fresh" }), + mint({ id: "observed-unpaid", lastObservedRemoteState: "UNPAID" }), + mint({ id: "observed-paid", lastObservedRemoteState: "PAID" }), + ], + }); + expect(result.pending.map((op) => op.id)).toEqual([ + "expired", + "fresh", + "observed-unpaid", + "observed-paid", + ]); + }); + + it("recovers executing operations left behind by a crash mid-mint", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "executing", state: "executing" })], + }); + expect(result.pending.map((op) => op.id)).toEqual(["executing"]); + }); + + it("ignores finalized operations", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "done", state: "finalized" })], + }); + expect(result.pending).toEqual([]); + expect(result.failed).toEqual([]); + }); + + it("does not re-open failed operations by default", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + }); + expect(result.failed).toEqual([]); + }); + + it("returns failed operations when the caller opts in", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + includeFailed: true, + }); + expect(result.failed.map((op) => op.id)).toEqual(["given-up"]); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.ts b/src/daemon/wallet/mint-quote-recovery.ts new file mode 100644 index 0000000..be53b95 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.ts @@ -0,0 +1,116 @@ +/** + * Pure helpers for PAID mint-quote recovery. + * + * A mint quote can be PAID at the mint while its local operation is still + * `pending` (the Lightning payment landed before expiry while the daemon was + * down, so no local observation was ever recorded) or even terminally + * `failed` (coco gives up when the mint refuses to sign, for example after the + * invoice expiry). The paid sats are claimable either way: NUT-04 lets the + * holder submit outputs for any quote id while `amount_issued < amount_paid`. + * + * Recovery therefore has to ask the mint what it thinks, then re-issue the + * quote. These helpers decide *what* to do from a remote observation; the + * actual state transitions are applied by the in-process coco wallet client + * so coco-core's operation services emit their normal events and release + * proof reservations. Keeping the decisions here makes them unit testable + * without a wallet database or network access. + */ + +/** Subset of coco's mint operation rows that recovery needs. */ +export interface MintQuoteRecoveryCandidate { + id: string; + mintUrl: string; + quoteId?: string; + state: string; + /** Quote amount in sats. */ + amount: number; + /** Quote expiry in epoch seconds. `0` means unknown/not applicable. */ + expiry: number; + /** Last quote state observed from the mint (UNPAID, PAID, ISSUED). */ + lastObservedRemoteState?: string; + error?: string; +} + +/** Coco's classification of a fresh remote quote check. */ +export type PendingMintCheckCategory = + | "waiting" + | "ready" + | "completed" + | "terminal"; + +/** What recovery should do with a quote after checking it with the mint. */ +export type MintQuoteRecoveryDecision = + | { action: "finalize"; observedRemoteState: "PAID" | "ISSUED" } + | { action: "waiting" } + | { action: "terminal" }; + +/** + * Map a remote quote check onto a recovery action. + * + * - `ready` means the mint reports the quote PAID but never issued: submit the + * operation's stored outputs to claim the sats. + * - `completed` means the mint already issued it: recover the signatures + * (NUT-09) instead of minting again. + * - `waiting` means the mint still reports the quote UNPAID: nothing is + * claimable, so leave the operation alone. + * - `terminal` means the quote can no longer be issued (for example the mint + * refused an expired quote). coco persists that verdict as a failed + * operation, so recovery must report it rather than treat it as progress. + */ +export function classifyMintQuoteObservation( + category: PendingMintCheckCategory, +): MintQuoteRecoveryDecision { + switch (category) { + case "ready": + return { action: "finalize", observedRemoteState: "PAID" }; + case "completed": + return { action: "finalize", observedRemoteState: "ISSUED" }; + case "waiting": + return { action: "waiting" }; + case "terminal": + return { action: "terminal" }; + } +} + +export interface MintQuoteRecoverySelectionOptions< + T extends MintQuoteRecoveryCandidate, +> { + mints: T[]; + /** + * Also consider terminally failed operations. Off by default: re-opening a + * failed operation is a mutation, so only an explicit user-invoked recovery + * may do it. Startup recovery must never resurrect quotes on its own. + */ + includeFailed?: boolean; +} + +export interface MintQuoteRecoverySelection< + T extends MintQuoteRecoveryCandidate, +> { + /** Pending (or executing) operations that need a fresh mint observation. */ + pending: T[]; + /** Failed operations the caller may re-open and retry. */ + failed: T[]; +} + +/** + * Split operations into those recovery should check and those that were + * already given up on. + * + * Every `pending` operation is selected: only the mint knows whether an + * expired quote was paid before the local invoice ran out. `executing` + * operations are recovered too, since a crash mid-mint leaves outputs that + * may already be signed. + */ +export function selectMintQuotesForRecovery< + T extends MintQuoteRecoveryCandidate, +>(options: MintQuoteRecoverySelectionOptions): MintQuoteRecoverySelection { + const { mints, includeFailed = false } = options; + const pending = mints.filter( + (op) => op.state === "pending" || op.state === "executing", + ); + const failed = includeFailed + ? mints.filter((op) => op.state === "failed") + : []; + return { pending, failed }; +} diff --git a/src/daemon/wallet/testing/fake-mint.ts b/src/daemon/wallet/testing/fake-mint.ts new file mode 100644 index 0000000..3960ee9 --- /dev/null +++ b/src/daemon/wallet/testing/fake-mint.ts @@ -0,0 +1,347 @@ +/** + * In-process Cashu mint for integration tests. + * + * Implements just enough of NUT-01/02/04/06/07/09 to drive a real coco + * `Manager` against real HTTP: keyset publication, bolt11 mint quotes, minting + * blinded outputs with a real secp256k1 blind signature, NUT-09 restore and + * NUT-07 proof states. No Lightning, no network, no NPC. + * + * The signing keys and signatures are genuine (`@cashu/cashu-ts` mint-side + * helpers), so a wallet that receives these signatures can unblind and verify + * them exactly as with a production mint. + */ +import { + createBlindSignature, + createNewMintKeys, + pointFromHex, +} from "@cashu/cashu-ts"; + +/** NUT error codes used by the scenarios. */ +export const QUOTE_EXPIRED = 20007; +export const ALREADY_ISSUED = 20002; + +export type FakeQuoteState = "UNPAID" | "PAID" | "ISSUED"; + +export interface FakeMintQuote { + quote: string; + request: string; + amount: number; + unit: string; + state: FakeQuoteState; + /** Epoch seconds, or null for a quote that never expires. */ + expiry: number | null; + amountPaid: number; + amountIssued: number; + pubkey: null; +} + +export interface FakeMintRequest { + quote: string; + outputs: Array<{ amount: number; id: string; B_: string }>; +} + +interface StoredSignature { + amount: number; + id: string; + C_: string; +} + +const toHex = (bytes: Uint8Array) => Buffer.from(bytes).toString("hex"); + +export class FakeMint { + readonly keysetId: string; + readonly keysByAmount: Record; + readonly requests: FakeMintRequest[] = []; + /** Every output the mint has ever signed, keyed by B_. */ + readonly signed = new Map(); + + /** When set, POST /v1/mint/bolt11 fails with this NUT error. */ + mintError: { code: number; detail: string } | null = null; + /** When set, quote creation returns this expiry (epoch seconds). */ + quoteExpiry: number | null = 3_600; + /** + * Awaited before responding to a mint request, so a test can hold minting + * open and interleave another recovery attempt. + */ + gate: Promise | null = null; + /** Awaited before answering a quote-state check, to hold observe open. */ + observeGate: Promise | null = null; + /** + * When true, POST /v1/restore answers with NUT-09's spec-legal positional + * arrays, including `null` for outputs the mint never signed. + * + * This is a known interop gap, not a supported path: cashu-ts 3.7.1 (which + * coco depends on) dereferences every entry of `signatures` while normalising + * amounts, so a `null` makes the wallet throw instead of skipping it. The + * switch exists so tests keep that behaviour visible; see + * mint-quote-recovery.fake-mint.test.ts. + */ + restoreIncludesNulls = false; + + private readonly quotes = new Map(); + private counter = 0; + private server?: ReturnType; + + constructor() { + const pair = createNewMintKeys(20, new Uint8Array(32).fill(9)); + this.keysetId = pair.keysetId; + this.keysByAmount = Object.fromEntries( + Object.entries(pair.pubKeys).map(([amount, key]) => [ + amount, + typeof key === "string" ? key : toHex(key), + ]), + ); + this.privKeys = Object.fromEntries( + Object.entries(pair.privKeys) as Array<[string, Uint8Array]>, + ); + } + + private readonly privKeys: Record; + + get url(): string { + if (!this.server) throw new Error("fake mint not started"); + return `http://127.0.0.1:${this.server.port}`; + } + + start(): void { + this.server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: (request) => this.handle(request), + }); + } + + stop(): void { + this.server?.stop(true); + this.server = undefined; + } + + /** Test control: the mint sees the invoice as paid but has issued nothing. */ + markPaid(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "PAID"; + quote.amountPaid = quote.amount; + } + + /** + * Test control: mark the quote issued without going through the mint + * endpoint, simulating a wallet that lost the signatures. + */ + markIssued(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "ISSUED"; + quote.amountPaid = quote.amount; + quote.amountIssued = quote.amount; + } + + /** Test control: sign outputs directly, as if another wallet had issued them. */ + signFor(quoteId: string, outputs: FakeMintRequest["outputs"]): void { + for (const output of outputs) { + this.signOutput(output); + } + this.markIssued(quoteId); + } + + getQuote(quoteId: string): FakeMintQuote | undefined { + return this.quotes.get(quoteId); + } + + private quoteBody(quote: FakeMintQuote) { + return { + quote: quote.quote, + request: quote.request, + amount: quote.amount, + unit: quote.unit, + state: quote.state, + expiry: quote.expiry, + amount_paid: quote.amountPaid, + amount_issued: quote.amountIssued, + pubkey: quote.pubkey, + }; + } + + private signOutput(output: { + amount: number; + id: string; + B_: string; + }): StoredSignature { + const privKey = this.privKeys[String(output.amount)]; + if (!privKey) { + throw new Error(`fake mint has no key for amount ${output.amount}`); + } + const signature = createBlindSignature( + pointFromHex(output.B_), + privKey, + this.keysetId, + ); + const stored: StoredSignature = { + amount: output.amount, + id: this.keysetId, + C_: toHex(signature.C_.toBytes(false)), + }; + this.signed.set(output.B_, stored); + return stored; + } + + private json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + } + + private error(code: number, detail: string, status = 400): Response { + return this.json({ code, detail }, status); + } + + private async handle(request: Request): Promise { + const { pathname } = new URL(request.url); + const body = async () => { + try { + return (await request.json()) as Record; + } catch { + return {}; + } + }; + + if (pathname === "/v1/info") { + return this.json({ + name: "fake-mint", + version: "0.0.1", + nuts: { + 4: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 5: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 7: { supported: true }, + 9: { supported: true }, + }, + }); + } + + if (pathname === "/v1/keys" || pathname.startsWith("/v1/keys/")) { + return this.json({ + keysets: [ + { id: this.keysetId, unit: "sat", keys: this.keysByAmount }, + ], + }); + } + + if (pathname === "/v1/keysets") { + return this.json({ + keysets: [{ id: this.keysetId, unit: "sat", active: true }], + }); + } + + if (pathname === "/v1/mint/quote/bolt11" && request.method === "POST") { + const input = await body(); + const amount = Number(input.amount); + const unit = typeof input.unit === "string" ? input.unit : "sat"; + const quote: FakeMintQuote = { + quote: `fake-quote-${++this.counter}`, + request: `lnbcfake${this.counter}`, + amount, + unit, + state: "UNPAID", + expiry: + this.quoteExpiry === null + ? null + : Math.floor(Date.now() / 1000) + this.quoteExpiry, + amountPaid: 0, + amountIssued: 0, + pubkey: null, + }; + this.quotes.set(quote.quote, quote); + return this.json(this.quoteBody(quote)); + } + + const quoteMatch = pathname.match(/^\/v1\/mint\/quote\/bolt11\/(.+)$/); + if (quoteMatch?.[1] && request.method === "GET") { + if (this.observeGate) await this.observeGate; + const quote = this.quotes.get(decodeURIComponent(quoteMatch[1])); + if (!quote) return this.error(50000, "Unknown quote"); + return this.json(this.quoteBody(quote)); + } + + if (pathname === "/v1/mint/bolt11" && request.method === "POST") { + const input = await body(); + const quoteId = String(input.quote); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + this.requests.push({ quote: quoteId, outputs }); + if (this.gate) await this.gate; + if (this.mintError) { + return this.error(this.mintError.code, this.mintError.detail); + } + const quote = this.quotes.get(quoteId); + if (!quote) return this.error(50000, "Unknown quote"); + if (quote.state === "ISSUED" || quote.amountIssued > 0) { + return this.error(ALREADY_ISSUED, "Quote already issued"); + } + if (quote.state !== "PAID") { + return this.error(20001, "Quote is not paid"); + } + const total = outputs.reduce((sum, o) => sum + Number(o.amount), 0); + if (total > quote.amountPaid - quote.amountIssued) { + return this.error(10002, "Outputs exceed the paid amount"); + } + const signatures = outputs.map((output) => ({ + amount: output.amount, + id: this.keysetId, + C_: this.signOutput(output).C_, + })); + quote.state = "ISSUED"; + quote.amountIssued += total; + return this.json({ signatures }); + } + + if (pathname === "/v1/restore" && request.method === "POST") { + const input = await body(); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + if (this.restoreIncludesNulls) { + // Spec-legal NUT-09 shape, including nulls. Kept behind a switch + // because it is currently unusable with coco's cashu-ts version. + return this.json({ + outputs, + signatures: outputs.map( + (output) => this.signed.get(output.B_) ?? null, + ), + }); + } + // Return only the signed outputs; coco matches them by B_ and treats the + // rest as "nothing to restore". + const signed = outputs.filter((output) => this.signed.has(output.B_)); + return this.json({ + outputs: signed, + signatures: signed.map((output) => this.signed.get(output.B_)), + }); + } + + if (pathname === "/v1/checkstate" && request.method === "POST") { + const input = await body(); + const ys = (input.Ys ?? []) as string[]; + return this.json({ + states: ys.map((Y) => ({ Y, state: "UNSPENT", witness: null })), + }); + } + + return this.error(404, `fake mint has no route for ${pathname}`, 404); + } +} From 6d19c0b2da4ed1115a09cac50b6de8eb62144b4b Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:58:09 +0800 Subject: [PATCH 2/7] feat(history): filter by transaction type and look up entries by id Add -t/--type (repeatable or comma-separated: send, receive, mint, melt) and -i/--id to `routstrd history`. Single-id lookups print one summary line, or full details with --verbose. The /wallet/history endpoint now accepts `type` and `id` query params; type filtering scans pages and applies offset/limit after filtering so pagination stays correct. Adds getHistoryEntryById to the wallet client interface and a shared list of valid history types. --- SKILL.md | 2 + src/cli.ts | 66 ++++++++++-- src/daemon/http/history.test.ts | 165 ++++++++++++++++++++++++++++++ src/daemon/http/index.ts | 58 ++++++++++- src/daemon/wallet/coco-client.ts | 4 + src/daemon/wallet/cocod-client.ts | 5 + src/utils/history.test.ts | 20 ++++ src/utils/history.ts | 9 ++ 8 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 src/daemon/http/history.test.ts create mode 100644 src/utils/history.test.ts create mode 100644 src/utils/history.ts diff --git a/SKILL.md b/SKILL.md index 7c47eb5..0302042 100644 --- a/SKILL.md +++ b/SKILL.md @@ -121,6 +121,8 @@ Show wallet transaction history. |--------|---------|-------------| | `-n, --limit ` | 50 | Number of entries to show | | `--offset ` | 0 | Number of entries to skip | +| `-t, --type ` | all | Filter by transaction type (`send`, `receive`, `mint`, `melt`). Repeatable or comma-separated | +| `-i, --id ` | | Show a single transaction by its ID (prints one summary line; add `--verbose` for full details) | | `-v, --verbose` | false | Show full details including encoded Cashu tokens | | `--json` | false | Output raw JSON with token objects (no encoding) | diff --git a/src/cli.ts b/src/cli.ts index c25f35e..a2be293 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,6 +58,10 @@ import { } from "./daemon/wallet/paths"; import * as QRCode from "qrcode"; import { normalizeNostrPubkey, npubFromPubkey, npubFromSecretKey } from "./utils/nip98"; +import { + HISTORY_ENTRY_TYPES, + isHistoryEntryType, +} from "./utils/history"; import { generateSecretKey, nip19 } from "nostr-tools"; import { generateMnemonic } from "@scure/bip39"; import { wordlist } from "@scure/bip39/wordlists/english.js"; @@ -1789,17 +1793,50 @@ program .description("Show wallet transaction history") .option("-n, --limit ", "Number of entries to show", "50") .option("--offset ", "Number of entries to skip", "0") - .option("-v, --verbose", "Show full details including encoded Cashu tokens") + .option( + "-t, --type ", + "Filter by transaction type (send, receive, mint, melt). Repeatable or comma-separated", + ) + .option("-i, --id ", "Show a single transaction by its ID") + .option("-v, --verbose", "Show full details instead of the summary line") .option("--json", "Output raw JSON with token objects (no encoding)") - .action(async (options: { limit: string; offset: string; verbose: boolean; json: boolean }) => { + .action(async (options: { + limit: string; + offset: string; + type?: string[]; + id?: string; + verbose: boolean; + json: boolean; + }) => { await ensureDaemonRunning(); const limit = Math.min(parseInt(options.limit, 10) || 50, 1000); const offset = parseInt(options.offset, 10) || 0; - const result = await callDaemon( - `/wallet/history?offset=${offset}&limit=${limit}`, + // Support both repeated flags (-t send -t mint) and comma-separated + // values (-t send,mint). + const requestedTypes = (options.type ?? []) + .flatMap((value) => value.split(",")) + .map((value) => value.trim().toLowerCase()) + .filter((value) => value.length > 0); + const unknownTypes = requestedTypes.filter( + (value) => !isHistoryEntryType(value), ); + if (unknownTypes.length > 0) { + console.error( + `Unknown transaction type: ${unknownTypes.join(", ")}. Valid types: ${HISTORY_ENTRY_TYPES.join(", ")}.`, + ); + process.exit(1); + } + + const query = new URLSearchParams({ + offset: String(offset), + limit: String(limit), + }); + if (requestedTypes.length > 0) query.set("type", requestedTypes.join(",")); + if (options.id) query.set("id", options.id); + + const result = await callDaemon(`/wallet/history?${query.toString()}`); if (result.error) { console.log(result.error); @@ -1812,7 +1849,15 @@ program const entries = data?.entries || []; if (entries.length === 0) { - console.log("No transaction history yet."); + if (options.id) { + console.log(`No transaction found with ID ${options.id}.`); + } else if (requestedTypes.length > 0) { + console.log( + `No ${requestedTypes.join(", ")} transactions found.`, + ); + } else { + console.log("No transaction history yet."); + } return; } @@ -1869,10 +1914,13 @@ program const pad = (s: string, w: number) => s.padEnd(w); const sep = Object.values(widths).map((w) => "-".repeat(w)).join(" | "); - console.log( - `${pad(idCol, widths.id)} | ${pad(timeCol, widths.time)} | ${pad(typeCol, widths.type)} | ${pad(mintCol, widths.mint)} | ${pad(amtCol, widths.amount)}`, - ); - console.log(sep); + // A single transaction lookup prints just its summary line. + if (!options.id) { + console.log( + `${pad(idCol, widths.id)} | ${pad(timeCol, widths.time)} | ${pad(typeCol, widths.type)} | ${pad(mintCol, widths.mint)} | ${pad(amtCol, widths.amount)}`, + ); + console.log(sep); + } for (const row of rows) { console.log( diff --git a/src/daemon/http/history.test.ts b/src/daemon/http/history.test.ts new file mode 100644 index 0000000..43d755c --- /dev/null +++ b/src/daemon/http/history.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "bun:test"; +import { EventEmitter } from "events"; +import type { HistoryEntry } from "@cashu/coco-core"; +import { + createDaemonRequestHandler, + getHistoryByTypes, + parseHistoryTypes, +} from "./index"; + +const MINT_URL = "https://mint.example/"; + +function makeEntry( + id: string, + type: HistoryEntry["type"], + createdAt: number, +): HistoryEntry { + return { + id, + type, + createdAt, + mintUrl: MINT_URL, + unit: "sat", + amount: 21, + } as HistoryEntry; +} + +const ENTRIES: HistoryEntry[] = [ + makeEntry("send-1", "send", 4), + makeEntry("receive-1", "receive", 3), + makeEntry("mint-1", "mint", 2), + makeEntry("melt-1", "melt", 1), + makeEntry("send-2", "send", 0), +]; + +function makeWalletClient(entries: HistoryEntry[] = ENTRIES) { + return { + getHistory: async (offset = 0, limit = 50) => + entries.slice(offset, offset + limit), + getHistoryEntryById: async (id: string) => + entries.find((entry) => entry.id === id) ?? null, + }; +} + +function makeReq(method: string, path: string) { + const req = new EventEmitter() as any; + req.method = method; + req.url = path; + req.headers = { host: "localhost" }; + return req; +} + +function makeRes() { + const res: any = { + status: 0, + body: "", + writeHead(status: number) { + res.status = status; + return res; + }, + end(chunk?: string) { + if (chunk) res.body += chunk; + return res; + }, + json() { + return JSON.parse(res.body); + }, + }; + return res; +} + +async function callHistory(path: string, entries?: HistoryEntry[]) { + const handler = createDaemonRequestHandler({ + walletClient: makeWalletClient(entries), + } as any); + const res = makeRes(); + await handler(makeReq("GET", path), res); + return res; +} + +describe("parseHistoryTypes", () => { + it("splits, trims, and lowercases comma-separated values", () => { + expect(parseHistoryTypes(" Send , MINT ")).toEqual(["send", "mint"]); + }); + + it("returns an empty list for missing or blank input", () => { + expect(parseHistoryTypes(null)).toEqual([]); + expect(parseHistoryTypes("")).toEqual([]); + expect(parseHistoryTypes(" , ")).toEqual([]); + }); +}); + +describe("getHistoryByTypes", () => { + it("filters by type and applies offset/limit after filtering", async () => { + const client = makeWalletClient(); + expect(await getHistoryByTypes(client, ["send"], 0, 10)).toEqual([ + ENTRIES[0]!, + ENTRIES[4]!, + ]); + expect(await getHistoryByTypes(client, ["send"], 1, 1)).toEqual([ + ENTRIES[4]!, + ]); + }); + + it("scans past the page size boundary to find matches", async () => { + const filler = Array.from({ length: 250 }, (_, index) => + makeEntry(`receive-${index}`, "receive", index), + ); + const target = makeEntry("mint-late", "mint", -1); + const client = makeWalletClient([...filler, target]); + + expect(await getHistoryByTypes(client, ["mint"], 0, 10)).toEqual([target]); + }); + + it("returns no entries when nothing matches", async () => { + const client = makeWalletClient([makeEntry("mint-1", "mint", 1)]); + expect(await getHistoryByTypes(client, ["send"], 0, 10)).toEqual([]); + }); +}); + +describe("GET /wallet/history", () => { + it("returns all entries without a filter", async () => { + const res = await callHistory("/wallet/history"); + expect(res.status).toBe(200); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-1", + "receive-1", + "mint-1", + "melt-1", + "send-2", + ]); + }); + + it("filters entries by a comma-separated type list", async () => { + const res = await callHistory("/wallet/history?type=send,melt"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-1", + "melt-1", + "send-2", + ]); + }); + + it("honors offset/limit for filtered results", async () => { + const res = await callHistory("/wallet/history?type=send&offset=1&limit=1"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-2", + ]); + }); + + it("looks up a single entry by id", async () => { + const res = await callHistory("/wallet/history?id=mint-1"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "mint-1", + ]); + }); + + it("returns an empty list when the id is unknown", async () => { + const res = await callHistory("/wallet/history?id=does-not-exist"); + const body = res.json(); + expect(body.output.entries).toEqual([]); + }); +}); diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 05349bb..8979aa7 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -375,6 +375,46 @@ function makeSdkLogger(...parts: string[]): SdkLogger { }; } +/** + * Parse a `type` query parameter into a list of normalized transaction types. + * Accepts comma-separated values and is case-insensitive. + */ +export function parseHistoryTypes(raw: string | null | undefined): string[] { + if (!raw) return []; + return raw + .split(",") + .map((value) => value.trim().toLowerCase()) + .filter((value) => value.length > 0); +} + +/** Page size used when scanning history to apply a type filter. */ +const HISTORY_TYPE_SCAN_PAGE_SIZE = 200; + +/** + * Return history entries matching `types`, with offset/limit applied after + * filtering. The coco history repository only paginates by raw position, so we + * scan pages until enough matches accumulate (or history is exhausted). + */ +export async function getHistoryByTypes( + client: Pick, + types: string[], + offset: number, + limit: number, +): Promise { + const wanted = new Set(types); + const matched: HistoryEntry[] = []; + let scanOffset = 0; + while (matched.length < offset + limit) { + const page = await client.getHistory(scanOffset, HISTORY_TYPE_SCAN_PAGE_SIZE); + for (const entry of page) { + if (wanted.has(entry.type)) matched.push(entry); + } + if (page.length < HISTORY_TYPE_SCAN_PAGE_SIZE) break; + scanOffset += HISTORY_TYPE_SCAN_PAGE_SIZE; + } + return matched.slice(offset, offset + limit); +} + export function createDaemonRequestHandler(deps: { provider: string | null; server: { close(cb?: () => void): void }; @@ -596,9 +636,25 @@ export function createDaemonRequestHandler(deps: { await respond(res, async () => { const offsetParam = url.searchParams.get("offset"); const limitParam = url.searchParams.get("limit"); + const idParam = url.searchParams.get("id")?.trim() || ""; + const types = parseHistoryTypes(url.searchParams.get("type")); const offset = offsetParam ? parseInt(offsetParam, 10) || 0 : 0; const limit = limitParam ? parseInt(limitParam, 10) || 50 : 50; - const entries = await deps.walletClient.getHistory(offset, limit); + + let entries: HistoryEntry[]; + if (idParam) { + const entry = await deps.walletClient.getHistoryEntryById(idParam); + entries = entry ? [entry] : []; + } else if (types.length > 0) { + entries = await getHistoryByTypes( + deps.walletClient, + types, + offset, + limit, + ); + } else { + entries = await deps.walletClient.getHistory(offset, limit); + } const encoded = entries.map((entry: HistoryEntry) => { const base = { ...entry } as Record; diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index c8077c8..6fce529 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -1707,6 +1707,10 @@ export async function createCocoClient( return coco.history.getPaginatedHistory(offset, limit); }, + async getHistoryEntryById(id: string): Promise { + return coco.history.getHistoryEntryById(id); + }, + async getNpcAddress(): Promise { const info = await npcApi().getInfo(); const name = diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index d338506..14feedf 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -142,6 +142,8 @@ export interface CocodClient { /** Release resources held by in-process wallet implementations. */ dispose?(): Promise; getHistory(offset?: number, limit?: number): Promise; + /** Look up a single transaction by its history entry ID. */ + getHistoryEntryById(id: string): Promise; /** NPC (npubx.cash) Lightning address for this wallet. */ getNpcAddress(): Promise; /** Claim an NPC username; pass confirm=true to pay the claim fee from the wallet. */ @@ -465,6 +467,9 @@ export function createCocodClient( async getHistory(_offset?: number, _limit?: number): Promise { return []; }, + async getHistoryEntryById(_id: string): Promise { + return null; + }, async getNpcAddress(): Promise { const address = await callDaemon("/npc/address"); if (typeof address !== "string" || !address.trim()) { diff --git a/src/utils/history.test.ts b/src/utils/history.test.ts new file mode 100644 index 0000000..581ce28 --- /dev/null +++ b/src/utils/history.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "bun:test"; +import { HISTORY_ENTRY_TYPES, isHistoryEntryType } from "./history"; + +describe("history entry types", () => { + it("lists the four supported transaction types", () => { + expect([...HISTORY_ENTRY_TYPES]).toEqual([ + "mint", + "melt", + "send", + "receive", + ]); + }); + + it("recognizes known types and rejects unknown ones", () => { + expect(isHistoryEntryType("send")).toBe(true); + expect(isHistoryEntryType("melt")).toBe(true); + expect(isHistoryEntryType("SEND")).toBe(false); + expect(isHistoryEntryType("refund")).toBe(false); + }); +}); diff --git a/src/utils/history.ts b/src/utils/history.ts new file mode 100644 index 0000000..3640655 --- /dev/null +++ b/src/utils/history.ts @@ -0,0 +1,9 @@ +/** Transaction types understood by the wallet history type filter. */ +export const HISTORY_ENTRY_TYPES = ["mint", "melt", "send", "receive"] as const; + +export type HistoryEntryType = (typeof HISTORY_ENTRY_TYPES)[number]; + +/** True when `value` is a recognized history transaction type. */ +export function isHistoryEntryType(value: string): value is HistoryEntryType { + return (HISTORY_ENTRY_TYPES as readonly string[]).includes(value); +} From 425dc2b8485809dc6f95649b8e2f0a141c5e8154 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:11:29 +0800 Subject: [PATCH 3/7] fix(wallet): address mint recovery review and clarify scope --- docs/wallet-mint-recovery.md | 71 +++++++++++++++++++ src/cli.ts | 19 +++-- src/daemon/http/index.ts | 33 ++++++--- src/daemon/http/wallet-recovery.test.ts | 49 +++++++++++++ src/daemon/wallet/cleanup.test.ts | 13 +++- src/daemon/wallet/cleanup.ts | 18 ++++- src/daemon/wallet/coco-client.test.ts | 42 +++++++++++ src/daemon/wallet/coco-client.ts | 35 ++++++--- src/daemon/wallet/cocod-client.ts | 6 +- .../mint-quote-recovery.fake-mint.test.ts | 30 ++++++++ src/daemon/wallet/mint-quote-recovery.ts | 11 +-- 11 files changed, 296 insertions(+), 31 deletions(-) create mode 100644 docs/wallet-mint-recovery.md create mode 100644 src/daemon/http/wallet-recovery.test.ts diff --git a/docs/wallet-mint-recovery.md b/docs/wallet-mint-recovery.md new file mode 100644 index 0000000..cb344ef --- /dev/null +++ b/docs/wallet-mint-recovery.md @@ -0,0 +1,71 @@ +# Mint quote recovery: scope and troubleshooting + +`routstrd wallet recover` explicitly retries mint operations through coco using +**their existing stored outputs**. It can restore signatures when a quote is +already issued, and reopen failed operations when explicitly requested: + +```sh +routstrd history --json +routstrd wallet recover --op --include-failed +``` + +Failed operations require explicit IDs over both HTTP and the CLI. A successful +re-run on an already finalized operation is a no-op. Requests that exceed their +wait budget are not cancelled; explicit retries skip the operation while the +underlying work is outstanding. + +## What this fixes—and what it does not + +Coco already checks pending quotes on startup and the daemon periodically +refreshes them. A quote paid while the daemon was offline does not, by itself, +require a new issuance implementation. + +This change makes normal cleanup confirm UNPAID with the mint before failing an +expired quote. PAID, ISSUED and unverified quotes remain pending. It also gives +operators a recovery path for operations previously marked failed. + +It does **not** replace rejected outputs with fresh outputs on an active keyset. +An inactive-keyset rejection can therefore remain retryable with zero recovery. +Recovery reports coco's persisted mint error when available, rather than only a +generic “remains pending” error. + +Do not infer that the production incidents were caused by keyset retirement. +Before claiming those incidents are fixed, collect: + +- The affected operation IDs, quote IDs, state and persisted `error`. +- A fresh remote quote state and, where provided, paid/issued amounts. +- The keyset IDs in the stored outputs and the mint's current keyset metadata. +- A reproduction showing existing recovery fails and the proposed fix succeeds. + +Inspect persisted operation data through a read-only database copy; do not edit +rows or run recovery scripts concurrently with a daemon against the same wallet. +Never share the mnemonic, output secrets, or full wallet database in a PR. + +A future fresh-output path must preserve original outputs for uncertain issuance +and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate +with coco's watcher/processor. It needs its own integration tests before handling +real funds. + +## Cleanup preview and force + +`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not +confirmed failures. `failedMintQuotes` and `leftForRecovery` are zero because no +mint check or cleanup transition was performed. Send/melt counts remain planned +cleanup counts in dry-run mode. + +`--force` deliberately bypasses mint confirmation and can strand paid sats in a +failed operation. Prefer normal cleanup. Forced operations can be retried with +`--op --include-failed`, but recovery still depends on the mint +accepting their stored outputs or restoring their signatures. + +## Integration and release notes + +The reopen helper uses private coco-core 1.0.1 methods. Retain real-Manager and +HTTP fake-mint coverage, use frozen dependency installs, and re-run integration +tests on coco upgrades. A controlled low-value live-mint smoke test remains +recommended before release. + +PR #118 removes `cocod-client.ts`. When integrating that change, move recovery +and cleanup contracts into its replacement `wallet-client.ts`, rename HTTP error +references accordingly, and make recovery mandatory for the in-process client. +This follow-up does not pull in #118's unrelated removal. diff --git a/src/cli.ts b/src/cli.ts index 21c6d54..b81efd6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2045,7 +2045,9 @@ walletCmd }); const answer = await new Promise((resolve) => { rl.question( - "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", + options.force + ? "WARNING: --force fails expired mint quotes WITHOUT checking the mint and may strand paid sats. It also reclaims old pending sends and cancels prepared melts. Continue? [y/N] " + : "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", (value: string) => { rl.close(); resolve(value.trim().toLowerCase()); @@ -2080,6 +2082,7 @@ walletCmd | { dryRun?: boolean; failedMintQuotes?: number; + mintQuoteCandidates?: number; leftForRecovery?: number; reclaimedSends?: number; cancelledMelts?: number; @@ -2091,9 +2094,13 @@ walletCmd if (output) { const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:"; console.log(prefix); - console.log( - ` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`, - ); + if (output.dryRun) { + console.log( + ` Expired mint quote candidates (not checked with mint): ${output.mintQuoteCandidates ?? 0}`, + ); + } else { + console.log(` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`); + } console.log( ` Expired quotes kept for recovery (paid/issued/unverified): ${output.leftForRecovery ?? 0}`, ); @@ -2129,11 +2136,11 @@ walletCmd walletCmd .command("recover") .description( - "Re-issue PAID mint quotes whose sats were never claimed by checking each quote with its mint", + "Retry mint quotes using their stored outputs (does not replace rejected outputs)", ) .option( "--op ", - "Recover only this operation id (repeatable; required to target failed operations)", + "Recover this operation id (repeatable; find IDs with routstrd history --json)", (value: string, previous: string[]) => [...previous, value], [] as string[], ) diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 7cee3d5..adb8ea6 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -289,10 +289,13 @@ function optionalStringArrayField( ): string[] | undefined { const value = body[field]; if (value === undefined) return undefined; - if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { - throw new CocodHttpError(400, `'${field}' must be an array of strings.`); + if ( + !Array.isArray(value) || + value.some((item) => typeof item !== "string" || !item.trim()) + ) { + throw new CocodHttpError(400, `'${field}' must be an array of non-empty strings.`); } - return value as string[]; + return value.map((item: string) => item.trim()); } function getCurrentMode(deps: DaemonDeps): ClientMode { @@ -502,13 +505,27 @@ export function createDaemonRequestHandler(deps: { } const body = await readJsonBody(req); + const operationIds = optionalStringArrayField(body, "operationIds"); + if (body.includeFailed === true && !operationIds?.length) { + throw new CocodHttpError( + 400, + "'includeFailed' requires non-empty 'operationIds'.", + ); + } + if ( + body.timeoutMs !== undefined && + (typeof body.timeoutMs !== "number" || + !Number.isFinite(body.timeoutMs) || body.timeoutMs <= 0) + ) { + throw new CocodHttpError( + 400, + "'timeoutMs' must be a positive finite number.", + ); + } const result = await deps.walletClient.recoverMintQuotes({ - operationIds: optionalStringArrayField(body, "operationIds"), + operationIds, includeFailed: body.includeFailed === true, - timeoutMs: - typeof body.timeoutMs === "number" && Number.isFinite(body.timeoutMs) - ? body.timeoutMs - : undefined, + timeoutMs: body.timeoutMs as number | undefined, }); return { output: result }; }); diff --git a/src/daemon/http/wallet-recovery.test.ts b/src/daemon/http/wallet-recovery.test.ts new file mode 100644 index 0000000..b49a984 --- /dev/null +++ b/src/daemon/http/wallet-recovery.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, mock } from "bun:test"; +import { EventEmitter } from "node:events"; +import { createDaemonRequestHandler } from "./index"; + +async function recover(body: unknown) { + const recoverMintQuotes = mock(async (_options: unknown) => ({ recovered: 0 })); + const handler = createDaemonRequestHandler({ walletClient: { recoverMintQuotes } } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from(JSON.stringify(body))); + req.emit("end"); + }); + await handler(req, res as never); + return { res, recoverMintQuotes }; +} + +describe("POST /wallet/recover validation", () => { + it.each([{}, { operationIds: [] }])("rejects includeFailed without explicit IDs: %j", async (body) => { + const { res, recoverMintQuotes } = await recover({ ...body, includeFailed: true }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([[""], [" "], [42]])("rejects invalid operation IDs: %j", async (operationIds) => { + const { res, recoverMintQuotes } = await recover({ operationIds }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([0, -1, "1000"])("rejects invalid timeout %j", async (timeoutMs) => { + const { res, recoverMintQuotes } = await recover({ timeoutMs }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it("passes normalized explicit IDs and a positive timeout", async () => { + const { res, recoverMintQuotes } = await recover({ operationIds: [" op-1 "], includeFailed: true, timeoutMs: 1000 }); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledWith({ operationIds: ["op-1"], includeFailed: true, timeoutMs: 1000 }); + }); + it("still permits checking pending quotes without IDs", async () => { + const { res, recoverMintQuotes } = await recover({}); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/daemon/wallet/cleanup.test.ts b/src/daemon/wallet/cleanup.test.ts index 381ab4e..b7a8568 100644 --- a/src/daemon/wallet/cleanup.test.ts +++ b/src/daemon/wallet/cleanup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; const NOW_MS = 1_800_000_000_000; const DAY_MS = 24 * 60 * 60 * 1000; @@ -166,3 +166,14 @@ describe("selectCleanupOperations", () => { expect(result.meltsToCancel).toEqual([]); }); }); + +describe("mint cleanup reporting", () => { + it("reports dry-run candidates, not confirmed failures", () => { + expect(summarizeMintCleanup({ dryRun: true, candidates: 3, failed: 0, leftForRecovery: 0 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 0, leftForRecovery: 0 }); + }); + it("reports only actual failures in a real run", () => { + expect(summarizeMintCleanup({ dryRun: false, candidates: 3, failed: 1, leftForRecovery: 2 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 1, leftForRecovery: 2 }); + }); +}); diff --git a/src/daemon/wallet/cleanup.ts b/src/daemon/wallet/cleanup.ts index bbf24c2..1ff48ea 100644 --- a/src/daemon/wallet/cleanup.ts +++ b/src/daemon/wallet/cleanup.ts @@ -64,8 +64,8 @@ export interface CleanupSelection< * can have happened before expiry while the daemon was down, leaving no * local observation. Callers that fail quotes automatically at startup must * therefore confirm UNPAID with the mint first (see - * settleExpiredMintQuotes in coco-client.ts); only the explicit, - * user-invoked cleanup command may fail candidates purely locally. + * failExpiredMintQuoteIfUnpaid in coco-client.ts). Explicit cleanup follows + * the same rule unless the operator opts into unsafe `--force` behaviour. * - Pending sends are reclaimed (rolled back) only when they are older than * `minAgeMs`, so we never roll back a token that a receiver might still * legitimately claim. @@ -102,3 +102,17 @@ export function selectCleanupOperations< return { mintsToFail, sendsToReclaim, meltsToCancel }; } + +/** Keep a local-only dry-run preview distinct from mint-confirmed outcomes. */ +export function summarizeMintCleanup(input: { + dryRun: boolean; + candidates: number; + failed: number; + leftForRecovery: number; +}) { + return { + mintQuoteCandidates: input.candidates, + failedMintQuotes: input.dryRun ? 0 : input.failed, + leftForRecovery: input.dryRun ? 0 : input.leftForRecovery, + }; +} diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index fc03b5e..3711300 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -1333,6 +1333,28 @@ describe("runMintQuoteRecovery", () => { expect(result.errors).toHaveLength(1); }); + it("falls back to the original failure when diagnostic lookup fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = async () => { throw new Error("lookup failed"); }; + const result = await runMintQuoteRecovery(source); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + + it("bounds a hung diagnostic lookup after finalize fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = () => new Promise(() => {}); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + it("bounds finalize so one hung mint cannot block recovery", async () => { const { source } = fakeSource([mintOp()], { observe: async () => ({ category: "ready" }), @@ -1369,6 +1391,26 @@ describe("runMintQuoteRecovery", () => { expect(reopenFailedOperation).not.toHaveBeenCalled(); }); + it("requires explicit IDs when including failed operations", async () => { + const { source, reopenFailedOperation, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { includeFailed: true })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + await expect(runMintQuoteRecovery(source, { includeFailed: true, operationIds: [] })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it.each([0, -1, NaN, Infinity])("rejects invalid recovery timeout %s", async (timeoutMs) => { + const { source, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { timeoutMs })).rejects.toThrow( + "timeoutMs must be a positive finite number", + ); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + it("re-opens a named failed operation, then mints it", async () => { const { source, reopenFailedOperation, finalize } = fakeSource( [mintOp({ state: "failed", lastObservedRemoteState: "PAID" })], diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index ebfec2d..900088d 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -36,7 +36,7 @@ import type { WalletCleanupResult, WalletRecoveryProgress, } from "./cocod-client"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; import { classifyMintQuoteObservation, selectMintQuotesForRecovery, @@ -1032,7 +1032,13 @@ export async function runMintQuoteRecovery( options: MintQuoteRecoveryOptions = {}, onProgress?: (message: string) => void, ): Promise { + if (options.includeFailed && !options.operationIds?.length) { + throw new Error("includeFailed requires explicit operationIds"); + } const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error("timeoutMs must be a positive finite number"); + } const outstanding = options.outstanding ?? new Map>(); const result: MintQuoteRecoveryResult = { @@ -1215,7 +1221,18 @@ export async function runMintQuoteRecovery( onProgress?.(`${label}: another recovery is working on it; skipped`); } else { result.retryable++; - onProgress?.(`${label}: could not finish recovery: ${messageOf(error)}`); + // finalize can throw a generic "remains pending" error after coco has + // persisted the actionable mint rejection (for example inactive keyset). + const current = await withTimeout( + source.ops.mint.get(operationId), + remaining(), + ).catch(() => null); + const detail = current?.state === "pending" && current.error + ? current.error + : messageOf(error); + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: could not finish recovery: ${detail}`); + return; } result.errors.push({ operationId, error: messageOf(error) }); return; @@ -2337,11 +2354,14 @@ export async function createCocoClient( } } - const failedMintQuoteCount = dryRun - ? selection.mintsToFail.length - : failedMintQuotes; + const mintSummary = summarizeMintCleanup({ + dryRun, + candidates: selection.mintsToFail.length, + failed: failedMintQuotes, + leftForRecovery, + }); const actedOn = - failedMintQuoteCount + + (dryRun ? mintSummary.mintQuoteCandidates : mintSummary.failedMintQuotes) + leftForRecovery + selection.sendsToReclaim.length + selection.meltsToCancel.length; @@ -2351,8 +2371,7 @@ export async function createCocoClient( return { dryRun, - failedMintQuotes: failedMintQuoteCount, - leftForRecovery, + ...mintSummary, reclaimedSends: selection.sendsToReclaim.length, cancelledMelts: selection.meltsToCancel.length, skipped, diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index b53bf88..1caee85 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -94,9 +94,11 @@ export interface WalletCleanupOptions { /** Summary of a wallet cleanup run. */ export interface WalletCleanupResult { dryRun: boolean; - /** Number of expired pending mint quotes marked as failed. */ + /** Expired quotes selected for checking; dry runs do not contact the mint. */ + mintQuoteCandidates: number; + /** Number actually marked failed (always zero in a dry run). */ failedMintQuotes: number; - /** Expired quotes whose mint reported PAID/ISSUED, left for recovery. */ + /** Expired quotes kept pending because they are paid/issued or unverified. */ leftForRecovery: number; /** Number of stale pending send operations reclaimed. */ reclaimedSends: number; diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts index 20715b5..2554665 100644 --- a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -134,6 +134,36 @@ describe("PAID mint quote recovery with a real Manager and mint", () => { expect(await booted.spendable()).toBe(210_000); }); + it("existing coco recovery already issues expired paid pending quotes", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + booted.mint.markPaid(op.quoteId as string); + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(100); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("ISSUED"); + }); + + it("keeps rejected stored outputs and reports the actionable mint error", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + const outputs = outputsOf(op); + booted.mint.markPaid(op.quoteId as string); + // Model the mint refusing the stored outputs, not invoice expiry. This is + // not evidence that the production quotes used an inactive keyset. + booted.mint.mintError = { code: 12001, detail: "keyset id inactive." }; + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(0); + const result = await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + }); + expect(result).toMatchObject({ recovered: 0, retryable: 1 }); + expect(result.errors.some((entry) => entry.error.includes("keyset id inactive"))).toBe(true); + expect(await booted.spendable()).toBe(0); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("PAID"); + expect(outputsOf(await booted.manager.ops.mint.get(op.id as string) as unknown as AnyRecord)).toEqual(outputs); + for (const request of booted.mint.requests) expect(request.outputs).toEqual(outputs); + }); + it("restores proofs for a quote already issued at the mint", async () => { booted = await boot({ quoteExpiry: null }); const op = await prepareQuote(booted, 210_000); diff --git a/src/daemon/wallet/mint-quote-recovery.ts b/src/daemon/wallet/mint-quote-recovery.ts index be53b95..eb620b2 100644 --- a/src/daemon/wallet/mint-quote-recovery.ts +++ b/src/daemon/wallet/mint-quote-recovery.ts @@ -5,11 +5,14 @@ * `pending` (the Lightning payment landed before expiry while the daemon was * down, so no local observation was ever recorded) or even terminally * `failed` (coco gives up when the mint refuses to sign, for example after the - * invoice expiry). The paid sats are claimable either way: NUT-04 lets the - * holder submit outputs for any quote id while `amount_issued < amount_paid`. + * invoice expiry). Claimability still depends on the mint accepting issuance. + * This feature retries the stored outputs or restores their signatures; it + * does not regenerate outputs rejected by the mint (for example an inactive + * keyset). coco already reconciles pending paid quotes at startup and in the + * periodic sweep. The new capability is operator-targeted recovery, including + * explicitly reopening failed operations, alongside safer cleanup. * - * Recovery therefore has to ask the mint what it thinks, then re-issue the - * quote. These helpers decide *what* to do from a remote observation; the + * Recovery asks the mint what it thinks, then retries issuance or restore. These helpers decide *what* to do from a remote observation; the * actual state transitions are applied by the in-process coco wallet client * so coco-core's operation services emit their normal events and release * proof reservations. Keeping the decisions here makes them unit testable From e848f2d521e1c5c3c14e070af136849a63447361 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:54:21 +0800 Subject: [PATCH 4/7] fix(wallet): give post-finalize diagnostic read its own budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diagnostic ops.mint.get that fetches coco's persisted mint error was bound to remaining() — often ~1ms after a slow mint consumed the per-op budget, so the actionable error silently fell back to the generic message. It is a local DB read, not a mint round-trip, so it now gets its own 250ms bound (still hang-safe). Adds a regression test for the nearly-spent-budget case. --- src/daemon/wallet/coco-client.test.ts | 26 ++++++++++++++++++++++++++ src/daemon/wallet/coco-client.ts | 13 +++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index 3711300..e18e7c2 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -1355,6 +1355,32 @@ describe("runMintQuoteRecovery", () => { expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); }); + it("surfaces the persisted mint error even when the mint budget is nearly spent", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { + // Consume almost the whole per-op mint budget before throwing: exactly + // the slow-mint case where a diagnostic bound to remaining() would + // starve and silently fall back to the generic message. + await new Promise((resolve) => setTimeout(resolve, 25)); + throw new Error("remains pending"); + }, + }); + source.ops.mint.get = async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { + ...mintOp(), + state: "pending", + error: "keyset id inactive.", + }; + }; + const result = await runMintQuoteRecovery(source, { timeoutMs: 30 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([ + { operationId: "op-1", error: "keyset id inactive." }, + ]); + }); + it("bounds finalize so one hung mint cannot block recovery", async () => { const { source } = fakeSource([mintOp()], { observe: async () => ({ category: "ready" }), diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 900088d..970516e 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -1002,6 +1002,14 @@ export function createRunQueue(): (run: () => Promise) => Promise { /** Per-quote budget for the mint round-trip during explicit recovery. */ const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000; +/** + * Bound for the local post-finalize diagnostic read. This is a database + * lookup, not a mint round-trip, so it gets its own small budget: the per-op + * mint budget is often already spent when finalize throws, and a starved + * diagnostic would silently fall back to the generic error message. + */ +const DIAGNOSTIC_LOOKUP_TIMEOUT_MS = 250; + /** * Recover mint quotes whose sats are PAID at the mint but were never claimed. * @@ -1142,7 +1150,8 @@ export async function runMintQuoteRecovery( } result.checked++; // One budget per operation, shared by the mint check and the finalize, so - // a slow mint cannot silently double the documented per-quote wait. + // a slow mint cannot silently double the documented per-quote wait. The + // local post-finalize diagnostic read is exempt (DIAGNOSTIC_LOOKUP_TIMEOUT_MS). const deadlineAt = Date.now() + timeoutMs; const remaining = () => Math.max(1, deadlineAt - Date.now()); @@ -1225,7 +1234,7 @@ export async function runMintQuoteRecovery( // persisted the actionable mint rejection (for example inactive keyset). const current = await withTimeout( source.ops.mint.get(operationId), - remaining(), + Math.max(remaining(), DIAGNOSTIC_LOOKUP_TIMEOUT_MS), ).catch(() => null); const detail = current?.state === "pending" && current.error ? current.error From b97a4e37988af1e3705380960da6ef1f7ce9012e Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:38:55 +0530 Subject: [PATCH 5/7] wallet: probe mint reachability and gate recovery per mint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Startup recovery swept every non-terminal operation through coco's global recovery.run(), so each op at an unreachable mint cost a full network timeout (one 'Send recovery: mint unreachable' line per op per startup), and waitForRecovery gated ALL value-moving operations on the global sweep — a dead mint blocked sends from healthy mints too. - recovery-probe.ts: enumerate stuck ops across send/melt/receive/mint, probe each affected mint once in parallel (GET /v1/info, 2s), and drive recovery per op for reachable mints only. Dead-mint ops stay parked, exactly as coco's own 'will retry later' path leaves them. - runWalletRecovery keeps coco's global sweeps when every mint answers (they also clean up init ops and orphaned reservations); the targeted driver only runs when a mint is dead. Executing sends use coco's private tryRecover* via cast, matching the mintOperationService precedent. - createRecoveryGate: per-mint waitForRecovery. Mints with no stuck ops are never gated; sendCashu/sendBolt11/receiveBolt11 resolve their target mint before gating. Global recovery errors still poison all callers. - startRecoveryRecheck: re-run targeted recovery every 5 minutes so a mint that comes back is recovered without a restart and mid-session stuck ops get reconciled. Transition-based logging only (down once, back once); idle ticks cost one DB query and no network. Receive ops stay startup-only because recovering competing receives needs the startup dedup classification. Known limitation: in degraded mode, init-op and orphaned-reservation housekeeping is deferred to a startup where all stuck mints answer. --- src/daemon/wallet/coco-client.test.ts | 71 ++++++ src/daemon/wallet/coco-client.ts | 187 ++++++++++++-- src/daemon/wallet/recovery-probe.test.ts | 279 ++++++++++++++++++++ src/daemon/wallet/recovery-probe.ts | 312 +++++++++++++++++++++++ 4 files changed, 831 insertions(+), 18 deletions(-) create mode 100644 src/daemon/wallet/recovery-probe.test.ts create mode 100644 src/daemon/wallet/recovery-probe.ts diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index e18e7c2..5da56a9 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -14,6 +14,7 @@ import { assertLegacyCocodNotRunning, claimLegacyCocodPidFile, createCocoClient, + createRecoveryGate, createRunQueue, DEFAULT_TRUSTED_MINT_URLS, failExpiredMintQuoteIfUnpaid, @@ -1634,3 +1635,73 @@ describe("runMintQuoteRecovery", () => { expect(finalize).not.toHaveBeenCalled(); }); }); + +describe("createRecoveryGate", () => { + const HEALTHY = "https://healthy.example.com"; + const STUCK = "https://stuck.example.com"; + + function settled(promise: Promise): Promise { + return Promise.race([ + promise.then(() => true, () => true), + new Promise((resolve) => setTimeout(() => resolve(false), 25)), + ]); + } + + it("lets a mint without stuck operations proceed while another mint recovers", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + // Recovery is still running (complete() never called), yet the healthy + // mint must not be blocked by the stuck one. + await gate.waitForRecovery(HEALTHY); + }); + + it("holds a mint with stuck operations until recovery completes", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery(STUCK); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); + + it("waits for the stuck-mint enumeration before deciding", async () => { + const gate = createRecoveryGate(); + const waiting = gate.waitForRecovery(HEALTHY); + expect(await settled(waiting)).toBe(false); + gate.publishStuckMints(new Set([STUCK])); + await waiting; + }); + + it("holds callers without a target mint until recovery completes", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery(); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); + + it("poisons every caller after a recovery failure", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + gate.fail("disk exploded"); + await expect(gate.waitForRecovery(HEALTHY)).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + await expect(gate.waitForRecovery(STUCK)).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + await expect(gate.waitForRecovery()).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + }); + + it("falls back to the global gate for unparseable mint URLs", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery("not a url"); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); +}); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index e0bc59e..60c0f2e 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -42,6 +42,14 @@ import { selectMintQuotesForRecovery, type MintQuoteRecoveryCandidate, } from "./mint-quote-recovery"; +import { + collectStuckOperations, + probeMintReachability, + runTargetedRecovery, + startRecoveryRecheck, + type SendRecoveryService, + type StuckOperation, +} from "./recovery-probe"; import { clearInterruptedReceiveReservations, deleteReceiveTokenReservation, @@ -1534,6 +1542,85 @@ interface RecoveryPhaseProgress { failedMintQuotes: number; } +/** + * Coco keeps per-operation recovery private on its services; routstrd already + * reaches into the Manager the same way for `mintOperationService`. Send is + * the only family whose public `refresh()` cannot recover executing ops. + */ +function sendRecoveryServiceOf(coco: Manager): SendRecoveryService { + return (coco as unknown as { sendOperationService: SendRecoveryService }) + .sendOperationService; +} + +/** + * Gate for value-moving wallet operations while startup recovery runs. + * + * The gate is per mint: once recovery has enumerated stuck operations + * (publishStuckMints), callers whose target mint has none proceed immediately + * — a dead or slow mint must not stall spends from a healthy one. Callers + * without a target mint, or whose mint has stuck operations, wait for the + * full sweep. fail() poisons every caller; reads are never gated. + */ +export interface RecoveryGate { + waitForRecovery(mintUrl?: string): Promise; + publishStuckMints(mints: Set): void; + complete(): void; + fail(error: string): void; +} + +export function createRecoveryGate(): RecoveryGate { + let stuckMints: Set | undefined; + let done = false; + let error: string | undefined; + let mintsResolve: (() => void) | undefined; + const mintsPromise = new Promise((resolve) => { + mintsResolve = resolve; + }); + let doneResolve: (() => void) | undefined; + const donePromise = new Promise((resolve) => { + doneResolve = resolve; + }); + + return { + async waitForRecovery(mintUrl?: string): Promise { + if (mintUrl) { + let normalized: string | undefined; + try { + normalized = normalizeMintUrl(mintUrl); + } catch { + // Unparseable URL falls back to the global gate. + normalized = undefined; + } + if (normalized) { + await mintsPromise; + if (!stuckMints?.has(normalized)) { + if (error) throw new Error(`Wallet is not ready: ${error}`); + return; + } + } + } + if (!done) await donePromise; + if (error) throw new Error(`Wallet is not ready: ${error}`); + }, + publishStuckMints(mints: Set): void { + if (stuckMints) return; + stuckMints = mints; + mintsResolve?.(); + }, + complete(): void { + done = true; + mintsResolve?.(); + doneResolve?.(); + }, + fail(message: string): void { + done = true; + error = message; + mintsResolve?.(); + doneResolve?.(); + }, + }; +} + /** * Run the wallet recovery sweeps in order, reporting phase changes. * @@ -1545,6 +1632,7 @@ async function runWalletRecovery( coco: Manager, onProgress: (progress: RecoveryPhaseProgress) => void, receiveOperationIds?: string[], + onStuckMintsKnown?: (mints: Set) => void, ): Promise { surfacingRecoveryProgress = true; let failedMintQuotes = 0; @@ -1571,19 +1659,60 @@ async function runWalletRecovery( } onProgress({ phase: "Settled expired mint quotes", failedMintQuotes }); + // Probe every mint that has stuck operations once, up front, so a dead + // mint costs a single short probe instead of a network timeout per + // operation per sweep. Healthy-mint operations are recovered per op; + // dead-mint operations stay parked exactly as coco's own "will retry + // later" path would leave them. + onProgress({ phase: "Probing mints", failedMintQuotes }); + const stuckOperations = await collectStuckOperations(coco.ops); + // Value-moving operations gate per mint on this set (see waitForRecovery); + // publish it as soon as it is known so healthy mints unblock immediately. + onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl))); + const unreachableMints = await probeMintReachability( + [...new Set(stuckOperations.map((op) => op.mintUrl))], + ); + for (const mintUrl of unreachableMints) { + const count = stuckOperations.filter((op) => op.mintUrl === mintUrl).length; + startupProgress( + `Skipping recovery for unreachable mint: ${mintUrl} (${count} op${count === 1 ? "" : "s"})`, + ); + } + const degraded = unreachableMints.size > 0; + const targeted = (kinds: Array) => + runTargetedRecovery(coco.ops, sendRecoveryServiceOf(coco), { + kinds, + stuckOperations, + unreachableMints, + }); + + // Happy path (every mint reachable) keeps coco's global sweeps: they also + // clean up init operations and orphaned proof reservations, which the + // per-op driver cannot enumerate. The targeted driver only runs when a + // dead mint would otherwise tax every stuck op with a network timeout. onProgress({ phase: "Send recovery", failedMintQuotes }); - await coco.ops.send.recovery.run(); + if (!degraded) await coco.ops.send.recovery.run(); + else await targeted(["send"]); onProgress({ phase: "Melt recovery", failedMintQuotes }); - await coco.ops.melt.recovery.run(); + if (!degraded) await coco.ops.melt.recovery.run(); + else await targeted(["melt"]); onProgress({ phase: "Receive recovery", failedMintQuotes }); if (receiveOperationIds) { // The pre-check already classified every executing receive by unique // input set. Recover only the conclusive retained operations; unresolved // groups stay untouched instead of falling back to Coco 1's expensive - // per-row sweep on this startup. + // per-row sweep on this startup. Operations at mints the probe found + // unreachable are skipped rather than costing their 15s timeout each. + const mintByOperation = new Map( + stuckOperations + .filter((op) => op.kind === "receive") + .map((op) => [op.id, op.mintUrl]), + ); for (const operationId of receiveOperationIds) { + const mintUrl = mintByOperation.get(operationId); + if (mintUrl && unreachableMints.has(mintUrl)) continue; try { await withTimeout(coco.ops.receive.refresh(operationId), 15_000); } catch (error) { @@ -1593,12 +1722,15 @@ async function runWalletRecovery( }); } } - } else { + } else if (!degraded) { await coco.ops.receive.recovery.run(); + } else { + await targeted(["receive"]); } onProgress({ phase: "Mint recovery", failedMintQuotes }); - await coco.recoverPendingMintOperations(); + if (!degraded) await coco.recoverPendingMintOperations(); + else await targeted(["mint"]); onProgress({ phase: "done", failedMintQuotes }); } finally { @@ -1658,9 +1790,11 @@ export async function createCocoClient( }; let recoveryResolve: (() => void) | undefined; let stopPendingMintSweep: (() => Promise) | undefined; + let stopRecoveryRecheck: (() => Promise) | undefined; const recoveryPromise = new Promise((resolve) => { recoveryResolve = resolve; }); + const recoveryGate = createRecoveryGate(); try { startupProgress("Opening Cashu wallet database..."); @@ -1858,13 +1992,33 @@ export async function createCocoClient( } }, receiveRecoveryOperationIds, + (mints) => recoveryGate.publishStuckMints(mints), ) .then(async () => { await syncReceiveReservations(); recoveryDone = true; recoveryPhase = "done"; + recoveryGate.complete(); recoveryResolve?.(); startupProgress("Wallet recovery complete."); + // Re-check stuck operations periodically: a mint that comes back has + // its parked operations recovered without a daemon restart, and + // operations stuck mid-session are reconciled too. Receive stays + // startup-only because recovering competing receives safely requires + // the startup dedup classification (see receive-dedup.ts). + stopRecoveryRecheck = startRecoveryRecheck( + coco!.ops, + sendRecoveryServiceOf(coco!), + { + kinds: ["send", "melt", "mint"], + onMintDown: (mintUrl, opCount) => + logger.warn( + `Mint ${mintUrl} is unreachable; ${opCount} stuck operation(s) will keep retrying`, + ), + onMintBack: (mintUrl) => + logger.log(`Mint ${mintUrl} is reachable again; resumed recovering its operations`), + }, + ); stopPendingMintSweep = startPendingMintSweep({ ops: coco!.ops, wallet: coco!.wallet, @@ -1877,6 +2031,7 @@ export async function createCocoClient( recoveryDone = true; recoveryPhase = "error"; recoveryError = error instanceof Error ? error.message : String(error); + recoveryGate.fail(recoveryError); recoveryResolve?.(); startupProgress(`Wallet recovery failed: ${recoveryError}`); }); @@ -1905,16 +2060,11 @@ export async function createCocoClient( const enqueueRecovery = createRunQueue(); const recoveryOutstanding = new Map>(); - /** - * Block a value-moving operation until background recovery has settled. - * Reads stay ungated so the daemon can report balances/status immediately. - */ - const waitForRecovery = async (): Promise => { - if (!recoveryDone) await recoveryPromise; - if (recoveryError) { - throw new Error(`Wallet is not ready: ${recoveryError}`); - } - }; + // Block a value-moving operation until background recovery has settled for + // its target mint (see createRecoveryGate). Reads stay ungated so the + // daemon can report balances/status immediately. + const waitForRecovery = (mintUrl?: string): Promise => + recoveryGate.waitForRecovery(mintUrl); return { async ping(): Promise { @@ -2094,13 +2244,13 @@ export async function createCocoClient( }, async receiveBolt11(amount: number, mintUrl?: string) { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for Lightning invoice"); } + await waitForRecovery(targetMint); const op = await coco.ops.mint.prepare({ mintUrl: targetMint, amount, @@ -2126,13 +2276,13 @@ export async function createCocoClient( }, async sendCashu(amount: number, mintUrl?: string): Promise { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for sending"); } + await waitForRecovery(targetMint); const prepared = await coco.ops.send.prepare({ mintUrl: targetMint, amount, @@ -2142,13 +2292,13 @@ export async function createCocoClient( }, async sendBolt11(invoice: string, mintUrl?: string): Promise { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for Lightning payment"); } + await waitForRecovery(targetMint); const prepared = await coco.ops.melt.prepare({ mintUrl: targetMint, method: "bolt11", @@ -2194,6 +2344,7 @@ export async function createCocoClient( async dispose(): Promise { if (disposed) return; disposed = true; + await stopRecoveryRecheck?.(); try { // Let any in-flight recovery settle before closing the database from // underneath it. The recovery promise resolves on success or failure. diff --git a/src/daemon/wallet/recovery-probe.test.ts b/src/daemon/wallet/recovery-probe.test.ts new file mode 100644 index 0000000..2468cb9 --- /dev/null +++ b/src/daemon/wallet/recovery-probe.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it, mock } from "bun:test"; +import { + collectStuckOperations, + probeMintReachability, + runTargetedRecovery, + startRecoveryRecheck, + type StuckOperationSource, + type SendRecoveryService, +} from "./recovery-probe"; + +interface FakeOp { + id: string; + mintUrl: string; + state: string; +} + +function op(id: string, mintUrl: string, state = "pending"): FakeOp { + return { id, mintUrl, state }; +} + +interface FakeSource extends StuckOperationSource { + stuck: Record<"send" | "melt" | "receive" | "mint", FakeOp[]>; + refreshed: Record<"send" | "melt" | "receive" | "mint", string[]>; + failOnRefresh?: Set; +} + +function makeSource( + stuck: Partial>, +): FakeSource { + const refreshed: FakeSource["refreshed"] = { + send: [], + melt: [], + receive: [], + mint: [], + }; + const full = { + send: stuck.send ?? [], + melt: stuck.melt ?? [], + receive: stuck.receive ?? [], + mint: stuck.mint ?? [], + }; + const family = (kind: keyof typeof full) => ({ + listInFlight: async () => full[kind] as never, + refresh: async (id: string) => { + refreshed[kind].push(id); + if (full[kind].some((o) => o.id === id && o.id.startsWith("boom"))) { + throw new Error("mint rejected the operation"); + } + return full[kind].find((o) => o.id === id) as never; + }, + }); + return { + stuck: full, + refreshed, + send: family("send") as FakeSource["send"], + melt: family("melt") as FakeSource["melt"], + receive: family("receive") as FakeSource["receive"], + mint: family("mint") as FakeSource["mint"], + }; +} + +function makeSendService(): SendRecoveryService & { + initRecovered: string[]; + executingRecovered: string[]; +} { + const initRecovered: string[] = []; + const executingRecovered: string[] = []; + return { + initRecovered, + executingRecovered, + tryRecoverInitOperation: async (raw) => { + initRecovered.push((raw as FakeOp).id); + }, + tryRecoverExecutingOperation: async (raw) => { + executingRecovered.push((raw as FakeOp).id); + }, + }; +} + +/** fetch stub: mints whose URL contains "dead" hang/fail; others answer. */ +function makeFetch(deadPredicate: (url: string) => boolean) { + const calls: string[] = []; + const fetchImpl = (async (url: string | URL | Request) => { + const href = String(url); + calls.push(href); + if (deadPredicate(href)) throw new Error("connect ECONNREFUSED"); + return new Response("{}", { status: 200 }); + }) as unknown as typeof fetch; + return { calls, fetchImpl }; +} + +const isDeadUrl = (url: string) => url.includes("dead"); + +const liveFetch = () => makeFetch(() => false); + +describe("collectStuckOperations", () => { + it("aggregates all four operation families with normalized mint URLs", async () => { + const source = makeSource({ + send: [op("s1", "https://mint.example.com/")], + melt: [op("m1", "https://mint.example.com")], + receive: [op("r1", "https://other.example.com", "executing")], + mint: [op("q1", "https://mint.example.com")], + }); + const stuck = await collectStuckOperations(source); + expect(stuck).toHaveLength(4); + expect(stuck.map((s) => s.kind).sort()).toEqual([ + "melt", + "mint", + "receive", + "send", + ]); + // Trailing slash normalized so the same mint dedupes to one probe target. + const mintUrls = new Set(stuck.map((s) => s.mintUrl)); + expect(mintUrls.size).toBe(2); + }); + + it("returns empty when nothing is stuck", async () => { + const stuck = await collectStuckOperations(makeSource({})); + expect(stuck).toEqual([]); + }); +}); + +describe("probeMintReachability", () => { + it("marks only mints whose fetch fails as unreachable", async () => { + const deadFetch = makeFetch(isDeadUrl); + const unreachable = await probeMintReachability( + ["https://live.example.com", "https://dead.example.com"], + { fetchImpl: deadFetch.fetchImpl }, + ); + expect([...unreachable]).toEqual(["https://dead.example.com"]); + expect(deadFetch.calls).toHaveLength(2); + expect(deadFetch.calls.every((c) => c.endsWith("/v1/info"))).toBe(true); + }); + + it("treats HTTP error responses as reachable", async () => { + const fetchImpl = (async () => + new Response("oops", { status: 500 })) as unknown as typeof fetch; + const unreachable = await probeMintReachability(["https://live.example.com"], { + fetchImpl, + }); + expect(unreachable.size).toBe(0); + }); +}); + +describe("runTargetedRecovery", () => { + it("does not probe when nothing is stuck", async () => { + const deadFetch = makeFetch(isDeadUrl); + const result = await runTargetedRecovery(makeSource({}), makeSendService(), { + fetchImpl: deadFetch.fetchImpl, + }); + expect(result).toMatchObject({ recovered: 0, skipped: 0, failed: 0 }); + expect(deadFetch.calls).toHaveLength(0); + }); + + it("skips operations at unreachable mints and recovers the rest", async () => { + const source = makeSource({ + send: [op("s1", "https://dead.example.com"), op("s2", "https://live.example.com")], + melt: [op("m1", "https://dead.example.com"), op("m2", "https://live.example.com")], + }); + const deadFetch = makeFetch(isDeadUrl); + const skippedMints: Array<[string, number]> = []; + const result = await runTargetedRecovery(source, makeSendService(), { + fetchImpl: deadFetch.fetchImpl, + onSkippedMint: (mintUrl, count) => skippedMints.push([mintUrl, count]), + }); + expect(result.recovered).toBe(2); + expect(result.skipped).toBe(2); + expect(result.failed).toBe(0); + expect(skippedMints).toEqual([["https://dead.example.com", 2]]); + expect(result.skippedMints.get("https://dead.example.com")).toBe(2); + // Only live-mint operations were driven. + expect(source.refreshed.send).toEqual(["s2"]); + expect(source.refreshed.melt).toEqual(["m2"]); + }); + + it("recovers pending sends via refresh and executing sends via the service", async () => { + const source = makeSource({ + send: [ + op("pending-1", "https://live.example.com", "pending"), + op("exec-1", "https://live.example.com", "executing"), + op("init-1", "https://live.example.com", "init"), + ], + }); + const sendService = makeSendService(); + const result = await runTargetedRecovery(source, sendService, { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.recovered).toBe(3); + expect(source.refreshed.send).toEqual(["pending-1"]); + expect(sendService.executingRecovered).toEqual(["exec-1"]); + expect(sendService.initRecovered).toEqual(["init-1"]); + }); + + it("counts per-operation failures at reachable mints and continues", async () => { + const source = makeSource({ + melt: [op("boom-1", "https://live.example.com"), op("m2", "https://live.example.com")], + }); + const result = await runTargetedRecovery(source, makeSendService(), { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.recovered).toBe(1); + expect(result.failed).toBe(1); + expect(source.refreshed.melt).toEqual(["boom-1", "m2"]); + }); + + it("respects the kinds filter and a pre-computed unreachable set", async () => { + const deadFetch = makeFetch(isDeadUrl); + const source = makeSource({ + send: [op("s1", "https://dead.example.com")], + mint: [op("q1", "https://dead.example.com")], + }); + const result = await runTargetedRecovery(source, makeSendService(), { + kinds: ["mint"], + unreachableMints: new Set(["https://dead.example.com"]), + fetchImpl: deadFetch.fetchImpl, + }); + // No probe ran (pre-computed set) and the send op was not even counted. + expect(deadFetch.calls).toHaveLength(0); + expect(result.skipped).toBe(1); + expect(source.refreshed.send).toEqual([]); + }); +}); + +describe("startRecoveryRecheck", () => { + it("recovers stuck operations on the interval and stops cleanly", async () => { + const source = makeSource({ + send: [op("s1", "https://live.example.com")], + }); + const stop = startRecoveryRecheck(source, makeSendService(), { + intervalMs: 20, + fetchImpl: liveFetch().fetchImpl, + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + await stop(); + expect(source.refreshed.send.length).toBeGreaterThanOrEqual(1); + const after = source.refreshed.send.length; + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(source.refreshed.send.length).toBe(after); + }); + + it("is idle without network traffic when nothing is stuck", async () => { + const { calls, fetchImpl } = makeFetch(() => false); + const stop = startRecoveryRecheck(makeSource({}), makeSendService(), { + intervalMs: 20, + fetchImpl, + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + await stop(); + expect(calls).toHaveLength(0); + }); + + it("reports mint down/up transitions once instead of repeating", async () => { + let dead = true; + const { fetchImpl } = makeFetch(() => dead); + const source = makeSource({ + melt: [op("m1", "https://flaky.example.com")], + }); + const down: string[] = []; + const back: string[] = []; + const stop = startRecoveryRecheck(source, makeSendService(), { + intervalMs: 20, + fetchImpl, + onMintDown: (mintUrl) => down.push(mintUrl), + onMintBack: (mintUrl) => back.push(mintUrl), + }); + await new Promise((resolve) => setTimeout(resolve, 90)); + // Dead across several ticks: reported once, never re-probed per op. + expect(down).toEqual(["https://flaky.example.com"]); + expect(back).toEqual([]); + expect(source.refreshed.melt).toEqual([]); + + dead = false; + await new Promise((resolve) => setTimeout(resolve, 90)); + await stop(); + expect(down).toEqual(["https://flaky.example.com"]); + expect(back).toEqual(["https://flaky.example.com"]); + expect(source.refreshed.melt.length).toBeGreaterThanOrEqual(1); + }); +}); diff --git a/src/daemon/wallet/recovery-probe.ts b/src/daemon/wallet/recovery-probe.ts new file mode 100644 index 0000000..952b3ab --- /dev/null +++ b/src/daemon/wallet/recovery-probe.ts @@ -0,0 +1,312 @@ +/** + * Mint reachability probing and targeted (per-operation) wallet recovery. + * + * Coco's global recovery sweeps walk every non-terminal operation one at a + * time; each operation at an unreachable mint costs a full network timeout. + * This module probes every mint that has stuck operations once, in parallel, + * and drives recovery per operation only for mints that answer — so one dead + * mint costs a single short probe instead of N sequential timeouts, and its + * operations stay parked (exactly as coco's "will retry later" path leaves + * them) until the mint comes back. + */ +import { normalizeMintUrl, type Manager } from "@cashu/coco-core"; +import { logger } from "../../utils/logger"; + +/** Short probe: a mint that cannot answer /v1/info in 2s slows every op. */ +export const MINT_PROBE_TIMEOUT_MS = 2_000; +/** How often stuck operations are re-checked (and dead mints re-probed). */ +export const RECOVERY_RECHECK_INTERVAL_MS = 300_000; + +type OpsApi = Manager["ops"]; + +/** Structural subset of the ops APIs used to enumerate and recover operations. */ +export interface StuckOperationSource { + send: Pick; + melt: Pick; + receive: Pick; + mint: Pick; +} + +export type StuckOperationKind = "send" | "melt" | "receive" | "mint"; + +export interface StuckOperation { + kind: StuckOperationKind; + id: string; + /** Normalized mint URL. */ + mintUrl: string; + state: string; + /** The operation object as returned by the API (needed by service-level recovery). */ + raw: unknown; +} + +/** + * The per-operation send recovery entry points coco keeps private. Send is the + * only operation family whose public `refresh()` does not cover `executing` + * operations; routstrd already reaches into coco internals the same way for + * `mintOperationService` (see coco-client.ts). + */ +export interface SendRecoveryService { + tryRecoverInitOperation(op: unknown): Promise; + tryRecoverExecutingOperation(op: unknown): Promise; +} + +export interface RecoveryRunResult { + /** Operations at reachable mints whose recovery completed. */ + recovered: number; + /** Operations skipped because their mint did not answer the probe. */ + skipped: number; + /** Operations at reachable mints whose recovery still failed. */ + failed: number; + /** Unreachable mint URL -> number of operations skipped there. */ + skippedMints: Map; +} + +export interface TargetedRecoveryOptions { + probeTimeoutMs?: number; + fetchImpl?: typeof fetch; + /** Operation families to recover. Defaults to all four. */ + kinds?: StuckOperationKind[]; + /** Pre-collected operations (e.g. from startup gating); default: enumerate now. */ + stuckOperations?: StuckOperation[]; + /** Pre-probed unreachable mints; default: probe now. */ + unreachableMints?: Set; + /** Called once per unreachable mint with the number of skipped operations. */ + onSkippedMint?: (mintUrl: string, opCount: number) => void; +} + +function asStuckOperations( + kind: StuckOperationKind, + ops: Array<{ id: string; mintUrl: string; state: string }>, +): StuckOperation[] { + const stuck: StuckOperation[] = []; + for (const op of ops) { + try { + stuck.push({ + kind, + id: op.id, + mintUrl: normalizeMintUrl(op.mintUrl), + state: op.state, + raw: op, + }); + } catch { + // Unparseable mint URL: keep the operation recoverable by treating it as + // reachable (probe only covers successfully normalized URLs). + stuck.push({ kind, id: op.id, mintUrl: op.mintUrl, state: op.state, raw: op }); + } + } + return stuck; +} + +/** + * Enumerate every non-terminal operation across all four operation families. + * Returns [] quickly when nothing is stuck, which is the common case. + */ +export async function collectStuckOperations( + source: StuckOperationSource, +): Promise { + const [sends, melts, receives, mints] = await Promise.all([ + source.send.listInFlight(), + source.melt.listInFlight(), + source.receive.listInFlight(), + source.mint.listInFlight(), + ]); + return [ + ...asStuckOperations("send", sends), + ...asStuckOperations("melt", melts), + ...asStuckOperations("receive", receives), + ...asStuckOperations("mint", mints), + ]; +} + +/** + * Probe each mint once, in parallel, and return the set of mint URLs that did + * not answer `GET /v1/info` in time. A mint that answers with an HTTP error is + * still "reachable" — its operations will fail with a real mint error instead + * of a network timeout, which is the information recovery needs. + */ +export async function probeMintReachability( + mintUrls: string[], + options: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}, +): Promise> { + const fetcher = options.fetchImpl ?? fetch; + const timeoutMs = options.timeoutMs ?? MINT_PROBE_TIMEOUT_MS; + const unreachable = new Set(); + + await Promise.all( + mintUrls.map(async (mintUrl) => { + try { + await fetcher(new URL("/v1/info", mintUrl).toString(), { + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (error) { + logger.debug("Mint did not answer recovery probe", { + mintUrl, + error: error instanceof Error ? error.message : String(error), + }); + unreachable.add(mintUrl); + } + }), + ); + + return unreachable; +} + +async function recoverStuckOperation( + source: StuckOperationSource, + sendService: SendRecoveryService, + op: StuckOperation, +): Promise { + switch (op.kind) { + case "send": + if (op.state === "pending") { + // Public API: actively re-checks the proofs with the mint. + await source.send.refresh(op.id); + } else if (op.state === "executing") { + // No public per-op path exists for executing sends (coco keeps it + // private); tryRecover* swallows per-op errors and leaves the + // operation for the next pass, matching the global sweep's behavior. + await sendService.tryRecoverExecutingOperation(op.raw); + } else if (op.state === "init") { + await sendService.tryRecoverInitOperation(op.raw); + } + // prepared / rolling_back: the global sweep only warns; nothing to do. + return; + case "melt": + // refresh() covers both pending and executing melt operations. + await source.melt.refresh(op.id); + return; + case "receive": + // refresh() actively recovers executing receive operations. + await source.receive.refresh(op.id); + return; + case "mint": + // refresh() covers both pending and executing mint operations. + await source.mint.refresh(op.id); + return; + } +} + +/** + * Recover every stuck operation whose mint answers a reachability probe, + * skipping operations at unreachable mints. Operations are recovered + * sequentially per mint (matching the global sweep's ordering guarantees); + * skipped operations are left untouched for a later pass, which is exactly + * what coco's own "Could not reach mint for recovery, will retry later" path + * does with them. + */ +export async function runTargetedRecovery( + source: StuckOperationSource, + sendService: SendRecoveryService, + options: TargetedRecoveryOptions = {}, +): Promise { + const result: RecoveryRunResult = { + recovered: 0, + skipped: 0, + failed: 0, + skippedMints: new Map(), + }; + + const kinds = options.kinds ?? ["send", "melt", "receive", "mint"]; + const stuck = (options.stuckOperations ?? (await collectStuckOperations(source))).filter( + (op) => kinds.includes(op.kind), + ); + if (stuck.length === 0) return result; + + const unreachable = + options.unreachableMints ?? + (await probeMintReachability([...new Set(stuck.map((op) => op.mintUrl))], { + timeoutMs: options.probeTimeoutMs, + fetchImpl: options.fetchImpl, + })); + + for (const mintUrl of unreachable) { + const count = stuck.filter((op) => op.mintUrl === mintUrl).length; + result.skippedMints.set(mintUrl, count); + result.skipped += count; + options.onSkippedMint?.(mintUrl, count); + } + + for (const op of stuck) { + if (unreachable.has(op.mintUrl)) continue; + try { + await recoverStuckOperation(source, sendService, op); + result.recovered++; + } catch (error) { + // Same semantics as coco's tryRecover*: leave the operation for the + // next pass. A reachable mint can still reject a specific operation. + result.failed++; + logger.warn("Targeted operation recovery did not complete", { + kind: op.kind, + operationId: op.id, + mintUrl: op.mintUrl, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return result; +} + +export interface RecoveryRecheckOptions extends TargetedRecoveryOptions { + intervalMs?: number; + /** Called when a mint transitions unreachable -> reachable with recovered op count. */ + onMintBack?: (mintUrl: string) => void; + /** Called when a mint transitions reachable -> unreachable. */ + onMintDown?: (mintUrl: string, opCount: number) => void; +} + +/** + * Periodically re-run targeted recovery so a mint that comes back online has + * its parked operations recovered without a daemon restart, and operations + * that get stuck mid-session are reconciled too. Idle ticks (no stuck + * operations) cost one DB query and no network traffic. Logging is + * transition-based: a mint that stays dead produces no repeated output. + * + * Returns a stop function that waits for any in-flight tick. + */ +export function startRecoveryRecheck( + source: StuckOperationSource, + sendService: SendRecoveryService, + options: RecoveryRecheckOptions = {}, +): () => Promise { + const intervalMs = options.intervalMs ?? RECOVERY_RECHECK_INTERVAL_MS; + let stopped = false; + let timer: ReturnType | undefined; + let inFlight: Promise = Promise.resolve(); + const knownDead = new Set(); + + const tick = async () => { + if (stopped) return; + inFlight = (async () => { + const result = await runTargetedRecovery(source, sendService, { + ...options, + onSkippedMint: (mintUrl, opCount) => { + if (!knownDead.has(mintUrl)) { + knownDead.add(mintUrl); + options.onMintDown?.(mintUrl, opCount); + } + options.onSkippedMint?.(mintUrl, opCount); + }, + }); + for (const mintUrl of [...knownDead]) { + if (!result.skippedMints.has(mintUrl)) { + knownDead.delete(mintUrl); + options.onMintBack?.(mintUrl); + } + } + })().catch((error: unknown) => { + logger.warn( + `Stuck-operation recheck failed: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + await inFlight; + if (!stopped) timer = setTimeout(tick, intervalMs); + }; + timer = setTimeout(tick, intervalMs); + + return async () => { + stopped = true; + if (timer) clearTimeout(timer); + await inFlight; + }; +} From 6ff7fe2de37fb02ccfb39b00cd8676caf93fe6e1 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:45:21 +0800 Subject: [PATCH 6/7] wallet: gate recovery per mint only in degraded mode, guard live ops Review fixes for the mint reachability probe (#116): - Open the per-mint recovery gate only on the degraded path, after the probe and local housekeeping finish. On the happy path every value-moving caller waits for the global sweeps as before, so coco's send/melt recovery never runs beside a live execute. - Skip operations whose per-operation lock is held (diagnostics.isLocked) in the targeted driver, and drive sends only in pending/executing states, so recovery can never race a swap that execute() still holds. Drive sends through recoverExecutingOperation instead of the error-swallowing tryRecover* wrappers. - Run coco's local-only crash cleanup (init operations, orphaned proof reservations) on the degraded path before the gate opens, so a mint that never returns cannot leave that housekeeping undone forever. - Probe with a path-preserving /v1/info join so subpath mints (e.g. https://host/Bitcoin) are checked at their real endpoint, and classify malformed persisted URLs as unreachable. - Drop the 5-minute stuck-operation recheck: it introduced the live-send race, an unbounded shutdown wait, and duplicate polling next to the pending-mint sweep. Parked operations are recovered on the next startup instead. - Count recovery attempts rather than completions, since the driver cannot observe tryRecover*-style swallowed errors. Adds recovery-integration.test.ts with real coco Manager + SQLite regression tests: live execute vs targeted recovery, gate ordering on both paths, and local housekeeping without network. --- src/daemon/wallet/coco-client.ts | 75 ++++---- .../wallet/recovery-integration.test.ts | 174 ++++++++++++++++++ src/daemon/wallet/recovery-probe.test.ts | 110 +++++------ src/daemon/wallet/recovery-probe.ts | 103 ++--------- 4 files changed, 276 insertions(+), 186 deletions(-) create mode 100644 src/daemon/wallet/recovery-integration.test.ts diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 60c0f2e..8fae8c9 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -46,7 +46,6 @@ import { collectStuckOperations, probeMintReachability, runTargetedRecovery, - startRecoveryRecheck, type SendRecoveryService, type StuckOperation, } from "./recovery-probe"; @@ -1552,12 +1551,38 @@ function sendRecoveryServiceOf(coco: Manager): SendRecoveryService { .sendOperationService; } +/** Local-only crash cleanup, run before the degraded gate opens. */ +export async function cleanupLocalRecoveryState( + coco: Manager, + repo: SqliteRepositories, +): Promise { + // Coco 1.0.1 implements these as local repository/proof operations only. + // Keep this version-sensitive bridge together with the send recovery bridge. + const services = coco as unknown as Record; + cleanupOrphanedReservations(): Promise; + }>; + const families = [ + ["send", repo.sendOperationRepository], + ["melt", repo.meltOperationRepository], + ["receive", repo.receiveOperationRepository], + ["mint", repo.mintOperationRepository], + ] as const; + for (const [kind, repository] of families) { + for (const op of await repository.getByState("init")) { + await services[`${kind}OperationService`]!.recoverInitOperation(op); + } + } + await services.sendOperationService!.cleanupOrphanedReservations(); +} + /** * Gate for value-moving wallet operations while startup recovery runs. * - * The gate is per mint: once recovery has enumerated stuck operations - * (publishStuckMints), callers whose target mint has none proceed immediately - * — a dead or slow mint must not stall spends from a healthy one. Callers + * On degraded startup only, publishStuckMints opens the gate for callers + * whose target mint has no stuck operations after probing and local cleanup. + * A dead mint must not stall spends from a healthy one. On the happy path + * all callers wait until the global sweeps finish. Callers * without a target mint, or whose mint has stuck operations, wait for the * full sweep. fail() poisons every caller; reads are never gated. */ @@ -1628,11 +1653,12 @@ export function createRecoveryGate(): RecoveryGate { * are failed locally so `recoverPendingMintOperations()` skips them, while * paid/issued and unreachable-mint quotes stay pending for the sweep. */ -async function runWalletRecovery( +export async function runWalletRecovery( coco: Manager, onProgress: (progress: RecoveryPhaseProgress) => void, receiveOperationIds?: string[], onStuckMintsKnown?: (mints: Set) => void, + options: { cleanupLocalState?: () => Promise; fetchImpl?: typeof fetch } = {}, ): Promise { surfacingRecoveryProgress = true; let failedMintQuotes = 0; @@ -1666,11 +1692,9 @@ async function runWalletRecovery( // later" path would leave them. onProgress({ phase: "Probing mints", failedMintQuotes }); const stuckOperations = await collectStuckOperations(coco.ops); - // Value-moving operations gate per mint on this set (see waitForRecovery); - // publish it as soon as it is known so healthy mints unblock immediately. - onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl))); const unreachableMints = await probeMintReachability( [...new Set(stuckOperations.map((op) => op.mintUrl))], + { fetchImpl: options.fetchImpl }, ); for (const mintUrl of unreachableMints) { const count = stuckOperations.filter((op) => op.mintUrl === mintUrl).length; @@ -1679,6 +1703,13 @@ async function runWalletRecovery( ); } const degraded = unreachableMints.size > 0; + if (degraded) { + // Local-only housekeeping must finish before any new operation is allowed. + await options.cleanupLocalState?.(); + // Global sweeps enumerate fresh state and are unsafe beside live sends. + // Only the snapshot-based degraded path may open the per-mint gate. + onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl))); + } const targeted = (kinds: Array) => runTargetedRecovery(coco.ops, sendRecoveryServiceOf(coco), { kinds, @@ -1687,9 +1718,10 @@ async function runWalletRecovery( }); // Happy path (every mint reachable) keeps coco's global sweeps: they also - // clean up init operations and orphaned proof reservations, which the - // per-op driver cannot enumerate. The targeted driver only runs when a - // dead mint would otherwise tax every stuck op with a network timeout. + // clean up init operations and orphaned proof reservations. Degraded + // startup runs that local housekeeping before opening its gate, and only + // drives the previously collected snapshot when a dead mint would + // otherwise tax every stuck op with a network timeout. onProgress({ phase: "Send recovery", failedMintQuotes }); if (!degraded) await coco.ops.send.recovery.run(); else await targeted(["send"]); @@ -1790,7 +1822,6 @@ export async function createCocoClient( }; let recoveryResolve: (() => void) | undefined; let stopPendingMintSweep: (() => Promise) | undefined; - let stopRecoveryRecheck: (() => Promise) | undefined; const recoveryPromise = new Promise((resolve) => { recoveryResolve = resolve; }); @@ -1993,6 +2024,7 @@ export async function createCocoClient( }, receiveRecoveryOperationIds, (mints) => recoveryGate.publishStuckMints(mints), + { cleanupLocalState: () => cleanupLocalRecoveryState(coco!, repo) }, ) .then(async () => { await syncReceiveReservations(); @@ -2001,24 +2033,6 @@ export async function createCocoClient( recoveryGate.complete(); recoveryResolve?.(); startupProgress("Wallet recovery complete."); - // Re-check stuck operations periodically: a mint that comes back has - // its parked operations recovered without a daemon restart, and - // operations stuck mid-session are reconciled too. Receive stays - // startup-only because recovering competing receives safely requires - // the startup dedup classification (see receive-dedup.ts). - stopRecoveryRecheck = startRecoveryRecheck( - coco!.ops, - sendRecoveryServiceOf(coco!), - { - kinds: ["send", "melt", "mint"], - onMintDown: (mintUrl, opCount) => - logger.warn( - `Mint ${mintUrl} is unreachable; ${opCount} stuck operation(s) will keep retrying`, - ), - onMintBack: (mintUrl) => - logger.log(`Mint ${mintUrl} is reachable again; resumed recovering its operations`), - }, - ); stopPendingMintSweep = startPendingMintSweep({ ops: coco!.ops, wallet: coco!.wallet, @@ -2344,7 +2358,6 @@ export async function createCocoClient( async dispose(): Promise { if (disposed) return; disposed = true; - await stopRecoveryRecheck?.(); try { // Let any in-flight recovery settle before closing the database from // underneath it. The recovery promise resolves on success or failure. diff --git a/src/daemon/wallet/recovery-integration.test.ts b/src/daemon/wallet/recovery-integration.test.ts new file mode 100644 index 0000000..59aa30e --- /dev/null +++ b/src/daemon/wallet/recovery-integration.test.ts @@ -0,0 +1,174 @@ +import { expect, it, spyOn } from "bun:test"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { Database } from "bun:sqlite"; +import { runTargetedRecovery, type SendRecoveryService } from "./recovery-probe"; + +import { + cleanupLocalRecoveryState, + createRecoveryGate, + runWalletRecovery, +} from "./coco-client"; + +const MINT = "https://mint.example.com"; +const OP = "op-live"; + +it("targeted recovery leaves a send that execute() holds alone", async () => { + const repo = new SqliteRepositories({ database: new Database(":memory:") }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const internals = coco as unknown as { + sendOperationService: SendRecoveryService; + walletService: { getWalletWithActiveKeysetId: (m: string) => Promise }; + }; + + let swapStarted!: () => void; + const started = new Promise((r) => (swapStarted = r)); + let finishSwap!: (v: { send: unknown[]; keep: unknown[] }) => void; + const swap = new Promise<{ send: unknown[]; keep: unknown[] }>((r) => (finishSwap = r)); + internals.walletService.getWalletWithActiveKeysetId = async () => ({ + wallet: { + unit: "sat", + send: async () => (swapStarted(), swap), + checkProofsStates: async () => [{ state: "UNSPENT" }], + getFeesForProofs: () => 0, + }, + }); + + await repo.proofRepository.saveProofs(MINT, [ + { id: "00aa", amount: 8, secret: "in-1", C: "02aa", mintUrl: MINT, state: "ready" } as never, + ]); + await repo.proofRepository.reserveProofs(MINT, ["in-1"], OP); + await repo.sendOperationRepository.create({ + id: OP, mintUrl: MINT, amount: 8, state: "prepared", method: "default", methodData: {}, + createdAt: Date.now(), updatedAt: Date.now(), needsSwap: true, fee: 0, inputAmount: 8, + inputProofSecrets: ["in-1"], + outputData: { + keep: [], + send: [{ + blindedMessage: { amount: 8, id: "00aa", B_: "02" + "11".repeat(32) }, + blindingFactor: "01", + secret: Buffer.from("out-1").toString("hex"), + }], + }, + } as never); + + const live = coco.ops.send.execute(OP); + await started; // swap is at the mint + await runTargetedRecovery(coco.ops, internals.sendOperationService, { + kinds: ["send"], + fetchImpl: (async () => new Response("{}")) as unknown as unknown as typeof fetch, + }); + // On 8005aeb this is "rolled_back" and "in-1" is no longer reserved. + expect((await coco.ops.send.get(OP))?.state).toBe("executing"); + + finishSwap({ send: [{ id: "00aa", amount: 8, secret: "out-1", C: "02bb" }], keep: [] }); + await live; + expect((await coco.ops.send.get(OP))?.state).toBe("pending"); +}); + + +it("keeps healthy-mint callers gated while happy-path global recovery runs", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const gate = createRecoveryGate(); + let enter!: () => void; + const entered = new Promise((resolve) => { enter = resolve; }); + let release!: () => void; + const barrier = new Promise((resolve) => { release = resolve; }); + const spies = [ + spyOn(coco.ops.send.recovery, "run").mockImplementation(async () => { enter(); await barrier; }), + spyOn(coco.ops.melt.recovery, "run").mockResolvedValue(undefined), + spyOn(coco.ops.receive.recovery, "run").mockResolvedValue(undefined), + spyOn(coco, "recoverPendingMintOperations").mockResolvedValue(undefined), + ]; + try { + const recovery = runWalletRecovery(coco, () => {}, undefined, + (mints) => gate.publishStuckMints(mints), + { fetchImpl: (async () => new Response("{}")) as unknown as typeof fetch }, + ).then(() => gate.complete()); + await entered; + let released = false; + const waiting = gate.waitForRecovery(MINT).then(() => { released = true; }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(released).toBe(false); + release(); + await recovery; + await waiting; + expect(released).toBe(true); + } finally { + release(); + for (const spy of spies) spy.mockRestore(); + db.close(); + } +}); + +it("degraded startup finishes local housekeeping before opening the per-mint gate", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + await repo.sendOperationRepository.create({ + id: "stuck", mintUrl: MINT, amount: 8, state: "rolling_back", + method: "default", methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + const gate = createRecoveryGate(); + const events: string[] = []; + const sweep = spyOn(coco.ops.send.recovery, "run"); + try { + const recovery = runWalletRecovery(coco, () => {}, [], (mints) => { + events.push("gate"); + gate.publishStuckMints(mints); + }, { + fetchImpl: (async () => { throw new Error("offline"); }) as unknown as unknown as typeof fetch, + cleanupLocalState: async () => { events.push("cleanup"); }, + }).then(() => gate.complete()); + await gate.waitForRecovery("https://healthy.example.com"); + expect(events).toEqual(["cleanup", "gate"]); + await recovery; + expect(sweep).not.toHaveBeenCalled(); + expect((await coco.ops.send.get("stuck"))?.state).toBe("rolling_back"); + } finally { + sweep.mockRestore(); + db.close(); + } +}); + +it("local housekeeping cleans init sends and orphaned reservations without network", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + try { + await repo.sendOperationRepository.create({ + id: "init-send", mintUrl: MINT, amount: 8, state: "init", method: "default", + methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + for (const [id, repository] of [ + ["init-melt", repo.meltOperationRepository], + ["init-receive", repo.receiveOperationRepository], + ["init-mint", repo.mintOperationRepository], + ] as const) { + await repository.create({ + id, mintUrl: MINT, amount: 8, state: "init", method: "bolt11", + methodData: {}, inputProofs: [], createdAt: Date.now(), updatedAt: Date.now(), + } as never); + } + await repo.proofRepository.saveProofs(MINT, [ + { id: "00aa", amount: 8, secret: "init-input", C: "02aa", mintUrl: MINT, state: "ready" }, + { id: "00aa", amount: 8, secret: "orphan-input", C: "02aa", mintUrl: MINT, state: "ready" }, + ] as never); + await repo.proofRepository.reserveProofs(MINT, ["init-input"], "init-send"); + await repo.proofRepository.reserveProofs(MINT, ["orphan-input"], "missing-send"); + await cleanupLocalRecoveryState(coco, repo); + expect(await coco.ops.send.get("init-send")).toBeNull(); + expect(await repo.proofRepository.getReservedProofs()).toEqual([]); + expect(await repo.meltOperationRepository.getById("init-melt")).toBeNull(); + expect(await repo.receiveOperationRepository.getById("init-receive")).toBeNull(); + expect(await repo.mintOperationRepository.getById("init-mint")).toBeNull(); + } finally { + db.close(); + } +}); diff --git a/src/daemon/wallet/recovery-probe.test.ts b/src/daemon/wallet/recovery-probe.test.ts index 2468cb9..0be1a06 100644 --- a/src/daemon/wallet/recovery-probe.test.ts +++ b/src/daemon/wallet/recovery-probe.test.ts @@ -1,9 +1,8 @@ -import { describe, expect, it, mock } from "bun:test"; +import { describe, expect, it } from "bun:test"; import { collectStuckOperations, probeMintReachability, runTargetedRecovery, - startRecoveryRecheck, type StuckOperationSource, type SendRecoveryService, } from "./recovery-probe"; @@ -40,6 +39,7 @@ function makeSource( mint: stuck.mint ?? [], }; const family = (kind: keyof typeof full) => ({ + diagnostics: { isLocked: () => false }, listInFlight: async () => full[kind] as never, refresh: async (id: string) => { refreshed[kind].push(id); @@ -60,18 +60,12 @@ function makeSource( } function makeSendService(): SendRecoveryService & { - initRecovered: string[]; executingRecovered: string[]; } { - const initRecovered: string[] = []; const executingRecovered: string[] = []; return { - initRecovered, executingRecovered, - tryRecoverInitOperation: async (raw) => { - initRecovered.push((raw as FakeOp).id); - }, - tryRecoverExecutingOperation: async (raw) => { + recoverExecutingOperation: async (raw) => { executingRecovered.push((raw as FakeOp).id); }, }; @@ -148,7 +142,7 @@ describe("runTargetedRecovery", () => { const result = await runTargetedRecovery(makeSource({}), makeSendService(), { fetchImpl: deadFetch.fetchImpl, }); - expect(result).toMatchObject({ recovered: 0, skipped: 0, failed: 0 }); + expect(result).toMatchObject({ attempted: 0, skipped: 0, failed: 0 }); expect(deadFetch.calls).toHaveLength(0); }); @@ -163,7 +157,7 @@ describe("runTargetedRecovery", () => { fetchImpl: deadFetch.fetchImpl, onSkippedMint: (mintUrl, count) => skippedMints.push([mintUrl, count]), }); - expect(result.recovered).toBe(2); + expect(result.attempted).toBe(2); expect(result.skipped).toBe(2); expect(result.failed).toBe(0); expect(skippedMints).toEqual([["https://dead.example.com", 2]]); @@ -178,17 +172,15 @@ describe("runTargetedRecovery", () => { send: [ op("pending-1", "https://live.example.com", "pending"), op("exec-1", "https://live.example.com", "executing"), - op("init-1", "https://live.example.com", "init"), ], }); const sendService = makeSendService(); const result = await runTargetedRecovery(source, sendService, { fetchImpl: liveFetch().fetchImpl, }); - expect(result.recovered).toBe(3); + expect(result.attempted).toBe(2); expect(source.refreshed.send).toEqual(["pending-1"]); expect(sendService.executingRecovered).toEqual(["exec-1"]); - expect(sendService.initRecovered).toEqual(["init-1"]); }); it("counts per-operation failures at reachable mints and continues", async () => { @@ -198,7 +190,7 @@ describe("runTargetedRecovery", () => { const result = await runTargetedRecovery(source, makeSendService(), { fetchImpl: liveFetch().fetchImpl, }); - expect(result.recovered).toBe(1); + expect(result.attempted).toBe(2); expect(result.failed).toBe(1); expect(source.refreshed.melt).toEqual(["boom-1", "m2"]); }); @@ -221,59 +213,39 @@ describe("runTargetedRecovery", () => { }); }); -describe("startRecoveryRecheck", () => { - it("recovers stuck operations on the interval and stops cleanly", async () => { - const source = makeSource({ - send: [op("s1", "https://live.example.com")], - }); - const stop = startRecoveryRecheck(source, makeSendService(), { - intervalMs: 20, - fetchImpl: liveFetch().fetchImpl, - }); - await new Promise((resolve) => setTimeout(resolve, 60)); - await stop(); - expect(source.refreshed.send.length).toBeGreaterThanOrEqual(1); - const after = source.refreshed.send.length; - await new Promise((resolve) => setTimeout(resolve, 60)); - expect(source.refreshed.send.length).toBe(after); - }); - it("is idle without network traffic when nothing is stuck", async () => { - const { calls, fetchImpl } = makeFetch(() => false); - const stop = startRecoveryRecheck(makeSource({}), makeSendService(), { - intervalMs: 20, - fetchImpl, - }); - await new Promise((resolve) => setTimeout(resolve, 60)); - await stop(); - expect(calls).toHaveLength(0); - }); - - it("reports mint down/up transitions once instead of repeating", async () => { - let dead = true; - const { fetchImpl } = makeFetch(() => dead); - const source = makeSource({ - melt: [op("m1", "https://flaky.example.com")], - }); - const down: string[] = []; - const back: string[] = []; - const stop = startRecoveryRecheck(source, makeSendService(), { - intervalMs: 20, - fetchImpl, - onMintDown: (mintUrl) => down.push(mintUrl), - onMintBack: (mintUrl) => back.push(mintUrl), - }); - await new Promise((resolve) => setTimeout(resolve, 90)); - // Dead across several ticks: reported once, never re-probed per op. - expect(down).toEqual(["https://flaky.example.com"]); - expect(back).toEqual([]); - expect(source.refreshed.melt).toEqual([]); - - dead = false; - await new Promise((resolve) => setTimeout(resolve, 90)); - await stop(); - expect(down).toEqual(["https://flaky.example.com"]); - expect(back).toEqual(["https://flaky.example.com"]); - expect(source.refreshed.melt.length).toBeGreaterThanOrEqual(1); - }); +it("preserves subpath mint URLs when probing", async () => { + const { fetchImpl, calls } = liveFetch(); + await probeMintReachability(["https://mint.example.com/Bitcoin/"], { fetchImpl }); + expect(calls).toEqual(["https://mint.example.com/Bitcoin/v1/info"]); +}); + +it("does not drive locked operations or count rolling-back sends as attempts", async () => { + const source = makeSource({ send: [ + op("live", "https://mint.example.com", "executing"), + op("rollback", "https://mint.example.com", "rolling_back"), + ] }); + source.send.diagnostics.isLocked = (id) => id === "live"; + const service = makeSendService(); + const result = await runTargetedRecovery(source, service, { fetchImpl: liveFetch().fetchImpl }); + expect(result.attempted).toBe(0); + expect(service.executingRecovered).toEqual([]); +}); + +it("classifies malformed persisted URLs as unreachable without fetching", async () => { + const { fetchImpl, calls } = liveFetch(); + const unreachable = await probeMintReachability(["not-a-url"], { fetchImpl }); + expect([...unreachable]).toEqual(["not-a-url"]); + expect(calls).toEqual([]); +}); + +it("bounds a hanging probe with an abort signal", async () => { + const fetchImpl = (async (_url: unknown, options: RequestInit) => + new Promise((_resolve, reject) => { + options.signal!.addEventListener("abort", () => reject(options.signal!.reason), { once: true }); + })) as unknown as typeof fetch; + const unreachable = await probeMintReachability(["https://slow.example.com"], { + fetchImpl, timeoutMs: 10, + }); + expect([...unreachable]).toEqual(["https://slow.example.com"]); }); diff --git a/src/daemon/wallet/recovery-probe.ts b/src/daemon/wallet/recovery-probe.ts index 952b3ab..9dce35f 100644 --- a/src/daemon/wallet/recovery-probe.ts +++ b/src/daemon/wallet/recovery-probe.ts @@ -7,24 +7,22 @@ * and drives recovery per operation only for mints that answer — so one dead * mint costs a single short probe instead of N sequential timeouts, and its * operations stay parked (exactly as coco's "will retry later" path leaves - * them) until the mint comes back. + * them) until a later startup finds the mint reachable. */ import { normalizeMintUrl, type Manager } from "@cashu/coco-core"; import { logger } from "../../utils/logger"; /** Short probe: a mint that cannot answer /v1/info in 2s slows every op. */ export const MINT_PROBE_TIMEOUT_MS = 2_000; -/** How often stuck operations are re-checked (and dead mints re-probed). */ -export const RECOVERY_RECHECK_INTERVAL_MS = 300_000; type OpsApi = Manager["ops"]; /** Structural subset of the ops APIs used to enumerate and recover operations. */ export interface StuckOperationSource { - send: Pick; - melt: Pick; - receive: Pick; - mint: Pick; + send: Pick; + melt: Pick; + receive: Pick; + mint: Pick; } export type StuckOperationKind = "send" | "melt" | "receive" | "mint"; @@ -46,13 +44,12 @@ export interface StuckOperation { * `mintOperationService` (see coco-client.ts). */ export interface SendRecoveryService { - tryRecoverInitOperation(op: unknown): Promise; - tryRecoverExecutingOperation(op: unknown): Promise; + recoverExecutingOperation(op: unknown): Promise; } export interface RecoveryRunResult { - /** Operations at reachable mints whose recovery completed. */ - recovered: number; + /** Operations for which recovery was attempted (not necessarily completed). */ + attempted: number; /** Operations skipped because their mint did not answer the probe. */ skipped: number; /** Operations at reachable mints whose recovery still failed. */ @@ -89,8 +86,8 @@ function asStuckOperations( raw: op, }); } catch { - // Unparseable mint URL: keep the operation recoverable by treating it as - // reachable (probe only covers successfully normalized URLs). + // Preserve malformed persisted URLs; the probe will classify them as + // unreachable rather than attempting recovery against an invalid URL. stuck.push({ kind, id: op.id, mintUrl: op.mintUrl, state: op.state, raw: op }); } } @@ -135,7 +132,7 @@ export async function probeMintReachability( await Promise.all( mintUrls.map(async (mintUrl) => { try { - await fetcher(new URL("/v1/info", mintUrl).toString(), { + await fetcher(`${normalizeMintUrl(mintUrl)}/v1/info`, { signal: AbortSignal.timeout(timeoutMs), }); } catch (error) { @@ -162,12 +159,8 @@ async function recoverStuckOperation( // Public API: actively re-checks the proofs with the mint. await source.send.refresh(op.id); } else if (op.state === "executing") { - // No public per-op path exists for executing sends (coco keeps it - // private); tryRecover* swallows per-op errors and leaves the - // operation for the next pass, matching the global sweep's behavior. - await sendService.tryRecoverExecutingOperation(op.raw); - } else if (op.state === "init") { - await sendService.tryRecoverInitOperation(op.raw); + // Startup snapshot only; skip live operations in the driver below. + await sendService.recoverExecutingOperation(op.raw); } // prepared / rolling_back: the global sweep only warns; nothing to do. return; @@ -200,7 +193,7 @@ export async function runTargetedRecovery( options: TargetedRecoveryOptions = {}, ): Promise { const result: RecoveryRunResult = { - recovered: 0, + attempted: 0, skipped: 0, failed: 0, skippedMints: new Map(), @@ -228,9 +221,11 @@ export async function runTargetedRecovery( for (const op of stuck) { if (unreachable.has(op.mintUrl)) continue; + if (source[op.kind].diagnostics.isLocked(op.id)) continue; + if (op.kind === "send" && !["pending", "executing"].includes(op.state)) continue; + result.attempted++; try { await recoverStuckOperation(source, sendService, op); - result.recovered++; } catch (error) { // Same semantics as coco's tryRecover*: leave the operation for the // next pass. A reachable mint can still reject a specific operation. @@ -246,67 +241,3 @@ export async function runTargetedRecovery( return result; } - -export interface RecoveryRecheckOptions extends TargetedRecoveryOptions { - intervalMs?: number; - /** Called when a mint transitions unreachable -> reachable with recovered op count. */ - onMintBack?: (mintUrl: string) => void; - /** Called when a mint transitions reachable -> unreachable. */ - onMintDown?: (mintUrl: string, opCount: number) => void; -} - -/** - * Periodically re-run targeted recovery so a mint that comes back online has - * its parked operations recovered without a daemon restart, and operations - * that get stuck mid-session are reconciled too. Idle ticks (no stuck - * operations) cost one DB query and no network traffic. Logging is - * transition-based: a mint that stays dead produces no repeated output. - * - * Returns a stop function that waits for any in-flight tick. - */ -export function startRecoveryRecheck( - source: StuckOperationSource, - sendService: SendRecoveryService, - options: RecoveryRecheckOptions = {}, -): () => Promise { - const intervalMs = options.intervalMs ?? RECOVERY_RECHECK_INTERVAL_MS; - let stopped = false; - let timer: ReturnType | undefined; - let inFlight: Promise = Promise.resolve(); - const knownDead = new Set(); - - const tick = async () => { - if (stopped) return; - inFlight = (async () => { - const result = await runTargetedRecovery(source, sendService, { - ...options, - onSkippedMint: (mintUrl, opCount) => { - if (!knownDead.has(mintUrl)) { - knownDead.add(mintUrl); - options.onMintDown?.(mintUrl, opCount); - } - options.onSkippedMint?.(mintUrl, opCount); - }, - }); - for (const mintUrl of [...knownDead]) { - if (!result.skippedMints.has(mintUrl)) { - knownDead.delete(mintUrl); - options.onMintBack?.(mintUrl); - } - } - })().catch((error: unknown) => { - logger.warn( - `Stuck-operation recheck failed: ${error instanceof Error ? error.message : String(error)}`, - ); - }); - await inFlight; - if (!stopped) timer = setTimeout(tick, intervalMs); - }; - timer = setTimeout(tick, intervalMs); - - return async () => { - stopped = true; - if (timer) clearTimeout(timer); - await inFlight; - }; -} From 00c823519d3f3d66b49de952e5373fd1462361f6 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:36:30 +0800 Subject: [PATCH 7/7] wallet: shared recovery tracking, bounded waits, safe shutdown - Share one wallet-lifetime tracker (family:id keys) across startup settlement, targeted recovery, explicit mint-quote recovery, and the pending-mint sweep, so a timed-out observation is never retried underneath. - Bound targeted recovery per operation (15s) and per pass (60s); a timeout reports timedOut and keeps the underlying work tracked. - Executing-send recovery holds coco's per-operation lock until the underlying drive actually settles, not until the caller times out. - Dispose now rejects new/queued recovery, drains queued and in-flight work, and only then closes coco/database/ownership; a bounded caller wait reports incomplete shutdown instead of closing early. - Fail-closed checks for coco private recovery internals before any local cleanup writes. - Remove the periodic recheck; recovery stays startup- or operator-triggered. --- src/daemon/http/index.ts | 13 + src/daemon/http/wallet-recovery.test.ts | 50 +++ src/daemon/wallet/coco-client.test.ts | 31 +- src/daemon/wallet/coco-client.ts | 285 +++++++++++------- src/daemon/wallet/cocod-client.ts | 22 ++ .../mint-quote-recovery.fake-mint.test.ts | 2 +- .../wallet/recovery-integration.test.ts | 85 +++++- src/daemon/wallet/recovery-probe.test.ts | 201 +++++++++++- src/daemon/wallet/recovery-probe.ts | 85 +++++- src/daemon/wallet/recovery-work.test.ts | 41 +++ src/daemon/wallet/recovery-work.ts | 45 +++ 11 files changed, 742 insertions(+), 118 deletions(-) create mode 100644 src/daemon/wallet/recovery-work.test.ts create mode 100644 src/daemon/wallet/recovery-work.ts diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 04d15e9..31cae7c 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -572,6 +572,19 @@ export function createDaemonRequestHandler(deps: { return; } + if (req.method === "POST" && url.pathname === "/wallet/recover/operations") { + await respond(res, async () => { + if (!deps.walletClient.recoverStuckOperations) { + throw new CocodHttpError( + 501, + "Stuck operation recovery is not supported by this wallet client.", + ); + } + return { output: await deps.walletClient.recoverStuckOperations() }; + }); + return; + } + if (req.method === "POST" && url.pathname === "/wallet/receive/cashu") { await respond(res, async () => { const body = await readJsonBody(req); diff --git a/src/daemon/http/wallet-recovery.test.ts b/src/daemon/http/wallet-recovery.test.ts index b49a984..4c61737 100644 --- a/src/daemon/http/wallet-recovery.test.ts +++ b/src/daemon/http/wallet-recovery.test.ts @@ -47,3 +47,53 @@ describe("POST /wallet/recover validation", () => { expect(recoverMintQuotes).toHaveBeenCalledTimes(1); }); }); + +async function recoverOperations() { + const recoverStuckOperations = mock(async () => ({ + attempted: 1, + busy: 0, + skipped: 0, + failed: 0, + skippedMints: {}, + })); + const handler = createDaemonRequestHandler({ walletClient: { recoverStuckOperations } } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover/operations", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from("{}")); + req.emit("end"); + }); + await handler(req, res as never); + return { res, recoverStuckOperations }; +} + +describe("POST /wallet/recover/operations", () => { + it("drives stuck-operation recovery and returns the summary", async () => { + const { res, recoverStuckOperations } = await recoverOperations(); + expect(res.status).toBe(200); + expect(recoverStuckOperations).toHaveBeenCalledTimes(1); + expect(JSON.parse(res.body).output).toMatchObject({ attempted: 1, busy: 0 }); + }); + + it("returns 501 when the wallet client does not support it", async () => { + const handler = createDaemonRequestHandler({ walletClient: {} } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover/operations", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from("{}")); + req.emit("end"); + }); + await handler(req, res as never); + expect(res.status).toBe(501); + }); +}); diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index 5da56a9..b020237 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -681,6 +681,29 @@ describe("settleExpiredMintQuotes", () => { expect(observePendingOperation).not.toHaveBeenCalled(); expect(failPendingOperation).not.toHaveBeenCalled(); }); + + it("skips quotes at probe-unreachable mints without spending the budget", async () => { + const ops = [ + pendingMintOp({ id: "dead-1", mintUrl: "https://dead.example.com" }), + pendingMintOp({ id: "dead-2", mintUrl: "https://dead.example.com/" }), + pendingMintOp({ id: "live-1", mintUrl: "https://live.example.com" }), + ]; + const { source, observePendingOperation, failPendingOperation } = + fakeSource(ops); + + const result = await settleExpiredMintQuotes( + source, + NOW_MS, + undefined, + { unreachableMints: new Set(["https://dead.example.com"]) }, + ); + + expect(result).toEqual({ failed: 1, leftForRecovery: 0, unobserved: 2 }); + // Only the reachable mint was asked anything. + expect(observePendingOperation).toHaveBeenCalledTimes(1); + expect(observePendingOperation.mock.calls[0]?.[0]).toBe("live-1"); + expect(failPendingOperation).toHaveBeenCalledTimes(1); + }); }); describe("settlePendingMintQuotes", () => { @@ -1537,7 +1560,7 @@ describe("runMintQuoteRecovery", () => { it("skips operations whose earlier recovery is still in flight", async () => { const outstanding = new Map>([ - ["op-1", new Promise(() => {})], + ["mint:op-1", new Promise(() => {})], ]); const { source, finalize, observePendingOperation } = fakeSource( [mintOp()], @@ -1568,14 +1591,14 @@ describe("runMintQuoteRecovery", () => { }); expect(first).toMatchObject({ retryable: 1, recovered: 0 }); - expect(outstanding.has("op-1")).toBe(true); + expect(outstanding.has("mint:op-1")).toBe(true); // The abandoned mint request must not be retried underneath. expect(second).toMatchObject({ busy: 1, checked: 0 }); }); it("does not re-open a failed operation whose recovery is in flight", async () => { const outstanding = new Map>([ - ["op-1", new Promise(() => {})], + ["mint:op-1", new Promise(() => {})], ]); const { source, reopenFailedOperation } = fakeSource( [mintOp({ state: "failed" })], @@ -1630,7 +1653,7 @@ describe("runMintQuoteRecovery", () => { }); expect(first).toMatchObject({ retryable: 1, checked: 1 }); - expect(outstanding.has("op-1")).toBe(true); + expect(outstanding.has("mint:op-1")).toBe(true); expect(second).toMatchObject({ busy: 1, checked: 0 }); expect(finalize).not.toHaveBeenCalled(); }); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 8fae8c9..bbd1f84 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -1,3 +1,4 @@ +import { recoveryKey, trackRecovery, drainRecoveryWork, waitForRecoveryWork, createRecoveryDisposer, type RecoveryWork } from "./recovery-work"; import { Manager, OperationInProgressError, @@ -731,6 +732,7 @@ const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000; /** Rejects when `timeoutMs` elapses before `promise` settles. */ function withTimeout(promise: Promise, timeoutMs: number): Promise { + if (timeoutMs === Infinity) return promise; let timer: ReturnType | undefined; const timeout = new Promise((_resolve, reject) => { timer = setTimeout( @@ -845,6 +847,7 @@ export async function settleExpiredMintQuotes( source: ExpiredMintQuoteSource, nowMs: number, deadlineMs: number = EXPIRED_MINT_OBSERVATION_DEADLINE_MS, + options: { unreachableMints?: Set; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {}, ): Promise { const pendingMints = await source.ops.mint.listPending(); const selection = selectCleanupOperations({ @@ -865,6 +868,25 @@ export async function settleExpiredMintQuotes( const startedAt = Date.now(); for (const op of candidates) { + if (options.shouldStop?.() || options.outstanding?.has(recoveryKey("mint", op.id))) { + settlement.unobserved++; continue; + } + // A mint the startup probe already found unreachable cannot answer an + // observation either; skip it without spending the shared wall-clock + // budget, leaving the quote pending for a later startup. + if (options.unreachableMints) { + let mintUrl = op.mintUrl; + try { + mintUrl = normalizeMintUrl(op.mintUrl); + } catch { + // Malformed persisted URL: probe keys are raw for those, and the + // observation below would fail anyway, landing in `unobserved`. + } + if (options.unreachableMints.has(mintUrl)) { + settlement.unobserved++; + continue; + } + } const remainingMs = deadlineMs - (Date.now() - startedAt); if (remainingMs <= 0) { const skipped = @@ -879,11 +901,12 @@ export async function settleExpiredMintQuotes( break; } - const check = await failExpiredMintQuoteIfUnpaid( - source.mintOperationService, - op.id, - remainingMs, - ); + // Track the complete observation/failure chain, not just its bounded wait. + const work = failExpiredMintQuoteIfUnpaid(source.mintOperationService, op.id, Infinity); + if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("mint", op.id), work); + const check = await waitForRecoveryWork(work, remainingMs).catch(error => ({ + outcome: "unobserved" as const, error, category: undefined, + })); if (check.outcome === "failed") { settlement.failed++; } else if (check.outcome === "leftForRecovery") { @@ -944,6 +967,7 @@ export interface MintQuoteRecoverySource { } export interface MintQuoteRecoveryOptions { + shouldStop?: () => boolean; /** Target only these operation ids (may include failed operations). */ operationIds?: string[]; /** Per-quote budget for observing the mint and finalizing the operation. */ @@ -955,7 +979,7 @@ export interface MintQuoteRecoveryOptions { */ includeFailed?: boolean; /** - * In-flight recovery work keyed by operation id, shared across runs. + * In-flight recovery work keyed by family:id (mint:), shared across runs. * withTimeout does not cancel the underlying request, so a timed-out quote * check or finalize must keep blocking a retry until it actually settles. */ @@ -994,9 +1018,9 @@ export interface MintQuoteRecoveryResult { * underneath work that outlived its timeout. A rejected task never breaks the * chain for the next one. */ -export function createRunQueue(): (run: () => Promise) => Promise { +export function createRunQueue(): ((run: () => Promise) => Promise) & { drain(): Promise } { let tail: Promise = Promise.resolve(); - return (run: () => Promise): Promise => { + const enqueue = (run: () => Promise): Promise => { const result = tail.then(run, run); tail = result.then( () => undefined, @@ -1004,6 +1028,7 @@ export function createRunQueue(): (run: () => Promise) => Promise { ); return result; }; + return Object.assign(enqueue, { drain: async () => { await tail; } }); } /** Per-quote budget for the mint round-trip during explicit recovery. */ @@ -1071,21 +1096,8 @@ export async function runMintQuoteRecovery( /** coco's fail-fast operation lock rejected the call: another holder exists. */ const isInProgress = (error: unknown) => error instanceof Error && error.name === "OperationInProgressError"; - /** - * Register in-flight work for an operation. Entries are cleared only once the - * work actually settles (withTimeout does not cancel the request behind it), - * so a timed-out call keeps blocking a retry. The identity check stops a late - * settlement from clearing a newer entry for the same operation. - */ - const track = (operationId: string, work: Promise) => { - outstanding.set(operationId, work); - const clear = () => { - if (outstanding.get(operationId) === work) { - outstanding.delete(operationId); - } - }; - void work.then(clear, clear); - }; + const track = (operationId: string, work: Promise) => + trackRecovery(outstanding, recoveryKey("mint", operationId), work); let targets: MintQuoteRecoveryCandidate[]; if (options.operationIds && options.operationIds.length > 0) { @@ -1115,8 +1127,9 @@ export async function runMintQuoteRecovery( }); for (const op of failed) { + if (options.shouldStop?.()) break; const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; - if (outstanding.has(op.id)) { + if (outstanding.has(recoveryKey("mint", op.id))) { result.busy++; onProgress?.(`${label}: an earlier recovery is still running; skipped`); continue; @@ -1144,13 +1157,16 @@ export async function runMintQuoteRecovery( await recoverOne(op); } - for (const op of pending) await recoverOne(op); + for (const op of pending) { + if (options.shouldStop?.()) break; + await recoverOne(op); + } return result; async function recoverOne(op: MintQuoteRecoveryCandidate): Promise { const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; - if (outstanding.has(op.id)) { + if (outstanding.has(recoveryKey("mint", op.id))) { result.busy++; onProgress?.(`${label}: an earlier recovery is still running; skipped`); return; @@ -1305,6 +1321,7 @@ export interface PendingMintSweepOptions { deadlineMs?: number; checkTimeoutMs?: number; state?: PendingMintSweepState; + shouldStop?: () => boolean; } type PendingMintOutcome = "unreachable" | "other"; @@ -1326,21 +1343,21 @@ export async function settlePendingMintQuotes( const ordered = [...pending.slice(resumeAt), ...pending.slice(0, resumeAt)]; const startedAt = Date.now(); for (const op of ordered) { + if (options.shouldStop?.()) break; const remainingMs = deadlineMs - (Date.now() - startedAt); - if (state.outstanding.has(op.id) || remainingMs <= 0) { + if (state.outstanding.has(recoveryKey("mint", op.id)) || remainingMs <= 0) { unreachable++; continue; } state.after = op.id; - // Report on the refresh itself so a late result is still logged. + // Track refresh AND its late-result reporting mutations as one lifetime. const settled = source.ops.mint .refresh(op.id) .then( (result) => reportPendingMintRefresh(source, op, result, nowMs), (error) => reportPendingMintRefreshError(source, op, error), - ) - .finally(() => state.outstanding.delete(op.id)); - state.outstanding.set(op.id, settled); + ); + trackRecovery(state.outstanding, recoveryKey("mint", op.id), settled); try { const outcome = await withTimeout(settled, Math.min(checkTimeoutMs, remainingMs)); if (outcome === "unreachable") unreachable++; @@ -1416,16 +1433,16 @@ async function reportPendingMintRefreshError( * still running after that fails against the closed database and is picked * up by startup recovery. */ -function startPendingMintSweep(source: PendingMintQuoteSource): () => Promise { +function startPendingMintSweep(source: PendingMintQuoteSource, outstanding: RecoveryWork): () => Promise { let stopped = false; let timer: ReturnType | undefined; let inFlight: Promise = Promise.resolve(); let unreachableBefore = 0; - const state: PendingMintSweepState = { outstanding: new Map() }; + const state: PendingMintSweepState = { outstanding }; const tick = async () => { if (stopped) return; - inFlight = settlePendingMintQuotes(source, Date.now(), { state }).then( + inFlight = settlePendingMintQuotes(source, Date.now(), { state, shouldStop: () => stopped }).then( ({ unreachable }) => { // Report a mint becoming unreachable, or reachable again, once. if (unreachable > 0 && unreachableBefore === 0) { @@ -1559,21 +1576,37 @@ export async function cleanupLocalRecoveryState( // Coco 1.0.1 implements these as local repository/proof operations only. // Keep this version-sensitive bridge together with the send recovery bridge. const services = coco as unknown as Record; - cleanupOrphanedReservations(): Promise; - }>; + recoverInitOperation?(op: unknown): Promise; + cleanupOrphanedReservations?(): Promise; + } | undefined>; const families = [ ["send", repo.sendOperationRepository], ["melt", repo.meltOperationRepository], ["receive", repo.receiveOperationRepository], ["mint", repo.mintOperationRepository], ] as const; - for (const [kind, repository] of families) { - for (const op of await repository.getByState("init")) { - await services[`${kind}OperationService`]!.recoverInitOperation(op); + // Fail closed the way reopenFailedMintOperation does: a coco bump that + // removes or renames these privates must stop recovery with a clear error + // before anything is written, not crash halfway through the loop with the + // cleanup half-applied. + for (const [kind] of families) { + if (typeof services[`${kind}OperationService`]?.recoverInitOperation !== "function") { + throw new Error( + `coco ${kind}OperationService.recoverInitOperation is unavailable; refusing local recovery cleanup`, + ); } } - await services.sendOperationService!.cleanupOrphanedReservations(); + if (typeof services.sendOperationService?.cleanupOrphanedReservations !== "function") { + throw new Error( + "coco sendOperationService.cleanupOrphanedReservations is unavailable; refusing local recovery cleanup", + ); + } + for (const [kind, repository] of families) { + for (const op of await repository.getByState("init")) { + await services[`${kind}OperationService`]!.recoverInitOperation!(op); + } + } + await services.sendOperationService!.cleanupOrphanedReservations!(); } /** @@ -1649,7 +1682,7 @@ export function createRecoveryGate(): RecoveryGate { /** * Run the wallet recovery sweeps in order, reporting phase changes. * - * Expired mint quotes are settled first: quotes their mint confirms as unpaid + * Probe first, then settle expired mint quotes: quotes confirmed as unpaid * are failed locally so `recoverPendingMintOperations()` skips them, while * paid/issued and unreachable-mint quotes stay pending for the sweep. */ @@ -1658,38 +1691,16 @@ export async function runWalletRecovery( onProgress: (progress: RecoveryPhaseProgress) => void, receiveOperationIds?: string[], onStuckMintsKnown?: (mints: Set) => void, - options: { cleanupLocalState?: () => Promise; fetchImpl?: typeof fetch } = {}, + options: { cleanupLocalState?: () => Promise; fetchImpl?: typeof fetch; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {}, ): Promise { surfacingRecoveryProgress = true; let failedMintQuotes = 0; try { - onProgress({ phase: "Settling expired mint quotes", failedMintQuotes }); - const settlement = await settleExpiredMintQuotes( - { - ops: coco.ops, - mintOperationService: ( - coco as unknown as { - mintOperationService: MintOperationServiceCleanup; - } - ).mintOperationService, - }, - Date.now(), - ); - failedMintQuotes = settlement.failed; - if (settlement.leftForRecovery > 0 || settlement.unobserved > 0) { - startupProgress( - `Expired mint quotes: ${settlement.failed} failed locally, ` + - `${settlement.leftForRecovery} paid/issued (kept for recovery), ` + - `${settlement.unobserved} unverifiable (kept pending).`, - ); - } - onProgress({ phase: "Settled expired mint quotes", failedMintQuotes }); - - // Probe every mint that has stuck operations once, up front, so a dead - // mint costs a single short probe instead of a network timeout per - // operation per sweep. Healthy-mint operations are recovered per op; - // dead-mint operations stay parked exactly as coco's own "will retry - // later" path would leave them. + // Probe every mint that has stuck operations once, FIRST, so a dead mint + // costs a single short probe instead of taxing settlement's observation + // budget plus a network timeout per operation per sweep. Healthy-mint + // operations are recovered per op; dead-mint operations stay parked + // exactly as coco's own "will retry later" path would leave them. onProgress({ phase: "Probing mints", failedMintQuotes }); const stuckOperations = await collectStuckOperations(coco.ops); const unreachableMints = await probeMintReachability( @@ -1710,9 +1721,39 @@ export async function runWalletRecovery( // Only the snapshot-based degraded path may open the per-mint gate. onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl))); } + + // Settlement runs after the gate opens and only spends its observation + // budget on mints the probe found reachable; dead-mint quotes stay + // pending untouched. It only reads and locally fails long-expired quotes, + // so it cannot conflict with live operations the gate just admitted. + onProgress({ phase: "Settling expired mint quotes", failedMintQuotes }); + const settlement = await settleExpiredMintQuotes( + { + ops: coco.ops, + mintOperationService: ( + coco as unknown as { + mintOperationService: MintOperationServiceCleanup; + } + ).mintOperationService, + }, + Date.now(), + undefined, + { unreachableMints, outstanding: options.outstanding, shouldStop: options.shouldStop }, + ); + failedMintQuotes = settlement.failed; + if (settlement.leftForRecovery > 0 || settlement.unobserved > 0) { + startupProgress( + `Expired mint quotes: ${settlement.failed} failed locally, ` + + `${settlement.leftForRecovery} paid/issued (kept for recovery), ` + + `${settlement.unobserved} unverifiable (kept pending).`, + ); + } + onProgress({ phase: "Settled expired mint quotes", failedMintQuotes }); const targeted = (kinds: Array) => runTargetedRecovery(coco.ops, sendRecoveryServiceOf(coco), { kinds, + outstanding: options.outstanding, + shouldStop: options.shouldStop, stuckOperations, unreachableMints, }); @@ -1743,10 +1784,13 @@ export async function runWalletRecovery( .map((op) => [op.id, op.mintUrl]), ); for (const operationId of receiveOperationIds) { + if (options.shouldStop?.()) break; const mintUrl = mintByOperation.get(operationId); if (mintUrl && unreachableMints.has(mintUrl)) continue; try { - await withTimeout(coco.ops.receive.refresh(operationId), 15_000); + const work = coco.ops.receive.refresh(operationId); + if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("receive", operationId), work); + await withTimeout(work, 15_000); } catch (error) { logger.warn("Targeted receive recovery did not complete", { operationId, @@ -1761,7 +1805,11 @@ export async function runWalletRecovery( } onProgress({ phase: "Mint recovery", failedMintQuotes }); - if (!degraded) await coco.recoverPendingMintOperations(); + // A settlement wait may have timed out while an unlocked observation + // still runs. Never let a fresh global mint sweep observe it again. + if (!degraded && ![...(options.outstanding?.keys() ?? [])].some(key => key.startsWith("mint:"))) { + await coco.recoverPendingMintOperations(); + } else await targeted(["mint"]); onProgress({ phase: "done", failedMintQuotes }); @@ -1826,6 +1874,9 @@ export async function createCocoClient( recoveryResolve = resolve; }); const recoveryGate = createRecoveryGate(); + let disposed = false; + const enqueueRecovery = createRunQueue(); + const recoveryOutstanding: RecoveryWork = new Map(); try { startupProgress("Opening Cashu wallet database..."); @@ -2024,7 +2075,7 @@ export async function createCocoClient( }, receiveRecoveryOperationIds, (mints) => recoveryGate.publishStuckMints(mints), - { cleanupLocalState: () => cleanupLocalRecoveryState(coco!, repo) }, + { cleanupLocalState: () => cleanupLocalRecoveryState(coco!, repo), outstanding: recoveryOutstanding, shouldStop: () => disposed }, ) .then(async () => { await syncReceiveReservations(); @@ -2039,7 +2090,7 @@ export async function createCocoClient( mintOperationService: ( coco as unknown as { mintOperationService: MintOperationServiceCleanup } ).mintOperationService, - }); + }, recoveryOutstanding); }) .catch((error) => { recoveryDone = true; @@ -2068,17 +2119,32 @@ export async function createCocoClient( return api; }; - let disposed = false; - // Explicit recovery runs are serialized, and finalize work that outlives its - // timeout stays in the map so a retry waits for it. - const enqueueRecovery = createRunQueue(); - const recoveryOutstanding = new Map>(); + const assertOpen = () => { if (disposed) throw new Error("Wallet is shutting down"); }; // Block a value-moving operation until background recovery has settled for // its target mint (see createRecoveryGate). Reads stay ungated so the // daemon can report balances/status immediately. - const waitForRecovery = (mintUrl?: string): Promise => - recoveryGate.waitForRecovery(mintUrl); + const waitForRecovery = async (mintUrl?: string): Promise => { + assertOpen(); + await recoveryGate.waitForRecovery(mintUrl); + assertOpen(); + }; + + const disposeRecovery = createRecoveryDisposer( + () => { disposed = true; }, + async () => { + await recoveryPromise; + await stopPendingMintSweep?.(); + await enqueueRecovery.drain(); + await drainRecoveryWork(recoveryOutstanding); + }, + async () => { + await coco.dispose(); + database.close(); + releaseLegacyPidClaim(); + releaseWalletPidClaim(); + }, + ); return { async ping(): Promise { @@ -2356,22 +2422,10 @@ export async function createCocoClient( }, async dispose(): Promise { - if (disposed) return; - disposed = true; - try { - // Let any in-flight recovery settle before closing the database from - // underneath it. The recovery promise resolves on success or failure. - await recoveryPromise; - await stopPendingMintSweep?.(); - await coco.dispose(); - } finally { - try { - database.close(); - } finally { - releaseLegacyPidClaim(); - releaseWalletPidClaim(); - } - } + await disposeRecovery().catch(error => { + logger.warn("Wallet shutdown incomplete; database and ownership retained until recovery settles"); + throw error; + }); }, async getHistory(offset?: number, limit?: number): Promise { @@ -2566,18 +2620,45 @@ export async function createCocoClient( // Serialize explicit recovery: two concurrent requests must not both // snapshot the same failed operation, and a retry must not start // underneath a finalize that outlived its timeout. - return enqueueRecovery(() => - runMintQuoteRecovery( + return enqueueRecovery(() => { + assertOpen(); + return runMintQuoteRecovery( { ops: coco.ops as unknown as MintQuoteRecoverySource["ops"], mintOperationService: service, reopenFailedOperation: (operationId) => reopenFailedMintOperation(service, operationId), }, - { ...options, outstanding: recoveryOutstanding }, + { ...options, outstanding: recoveryOutstanding, shouldStop: () => disposed }, onProgress, - ), - ); + ); + }); + }, + + async recoverStuckOperations() { + await waitForRecovery(); + // Serialized against explicit mint-quote recovery (and itself) through + // the same queue and lifetime tracker, so timed-out passes cannot retry the same + // operation. Receive stays startup-only: recovering competing receives + // safely requires the startup dedup classification (receive-dedup.ts). + // Operations a live execute holds come back as busy via coco's + // fail-fast operation lock, never driven underneath it. + const result = await enqueueRecovery(() => { + assertOpen(); + return runTargetedRecovery(coco!.ops, sendRecoveryServiceOf(coco!), { + kinds: ["send", "melt", "mint"], + outstanding: recoveryOutstanding, + shouldStop: () => disposed, + }); + }); + return { + timedOut: result.timedOut, + attempted: result.attempted, + busy: result.busy, + skipped: result.skipped, + failed: result.failed, + skippedMints: Object.fromEntries(result.skippedMints), + }; }, }; } diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index 2fab2c6..93d4d49 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -159,6 +159,22 @@ export interface WalletMintQuoteRecoveryResult { errors: Array<{ operationId: string; error: string }>; } +/** Summary of a stuck-operation (send/melt/mint) recovery run. */ +export interface WalletStuckOperationRecoveryResult { + /** Timed-out waits; the underlying operation remains tracked. */ + timedOut: number; + /** Operations for which recovery was attempted (not necessarily completed). */ + attempted: number; + /** Locked operations or unfinished work from another pass; retry later. */ + busy: number; + /** Operations skipped for unreachable mints, shutdown, or pass budget exhaustion. */ + skipped: number; + /** Operations at reachable mints whose recovery still failed. */ + failed: number; + /** Unreachable mint URL -> number of operations skipped there. */ + skippedMints: Record; +} + export interface CocodClient { ping(): Promise; getStatus(): Promise; @@ -201,6 +217,12 @@ export interface CocodClient { options?: WalletMintQuoteRecoveryOptions, onProgress?: (message: string) => void, ): Promise; + /** + * Recover stuck send/melt/mint operations whose mints answer a + * reachability probe. Operations a live execute holds are reported busy, + * never driven. Receive stays startup-only (receive dedup classification). + */ + recoverStuckOperations?(): Promise; /** Report background wallet recovery progress, when the wallet supports it. */ getRecoveryProgress?(): Promise; } diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts index 2554665..69b0c06 100644 --- a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -363,7 +363,7 @@ describe("PAID mint quote recovery with a real Manager and mint", () => { outstanding, })) as unknown as Record; expect(first).toMatchObject({ retryable: 1, recovered: 0 }); - expect(outstanding.has(op.id as string)).toBe(true); + expect(outstanding.has(`mint:${op.id}`)).toBe(true); const second = (await runMintQuoteRecovery(booted.source() as never, { outstanding, diff --git a/src/daemon/wallet/recovery-integration.test.ts b/src/daemon/wallet/recovery-integration.test.ts index 59aa30e..f279f93 100644 --- a/src/daemon/wallet/recovery-integration.test.ts +++ b/src/daemon/wallet/recovery-integration.test.ts @@ -55,12 +55,15 @@ it("targeted recovery leaves a send that execute() holds alone", async () => { const live = coco.ops.send.execute(OP); await started; // swap is at the mint - await runTargetedRecovery(coco.ops, internals.sendOperationService, { + const result = await runTargetedRecovery(coco.ops, internals.sendOperationService, { kinds: ["send"], fetchImpl: (async () => new Response("{}")) as unknown as unknown as typeof fetch, }); // On 8005aeb this is "rolled_back" and "in-1" is no longer reserved. expect((await coco.ops.send.get(OP))?.state).toBe("executing"); + // The live execute holds coco's per-operation lock: busy, never driven. + expect(result.attempted).toBe(0); + expect(result.busy).toBe(1); finishSwap({ send: [{ id: "00aa", amount: 8, secret: "out-1", C: "02bb" }], keep: [] }); await live; @@ -136,6 +139,65 @@ it("degraded startup finishes local housekeeping before opening the per-mint gat } }); +it("degraded startup probes before settlement and never asks a dead mint", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const expiredRow = (id: string, mintUrl: string) => ({ + id, mintUrl, quoteId: `q-${id}`, state: "pending", + createdAt: 1_000, updatedAt: 2_000, method: "bolt11", + methodData: { method: "bolt11", data: {} }, amount: 100, unit: "sat", + request: "lnbc1example", expiry: 1_000_000, // epoch seconds, long past + }); + await repo.mintOperationRepository.create(expiredRow("dead-quote", "https://dead.example.com") as never); + await repo.mintOperationRepository.create(expiredRow("live-quote", "https://live.example.com") as never); + const internals = coco as unknown as { + mintOperationService: { + observePendingOperation(id: string): Promise<{ category: "waiting" }>; + failPendingOperation(op: unknown, failure: unknown): Promise; + }; + }; + const events: string[] = []; + const observe = spyOn(internals.mintOperationService, "observePendingOperation") + .mockImplementation(async (id: string) => { + events.push(`observe:${id}`); + return { category: "waiting" }; + }); + // Fail the row for real (as the production service would) so the targeted + // mint pass sees state "failed" and refresh() returns without re-observing. + const fail = spyOn(internals.mintOperationService, "failPendingOperation") + .mockImplementation(async (op: unknown) => { + const id = (op as { id: string }).id; + const row = await repo.mintOperationRepository.getById(id); + if (row) await repo.mintOperationRepository.update({ ...row, state: "failed" } as never); + return {} as never; + }); + const gate = createRecoveryGate(); + try { + const recovery = runWalletRecovery(coco, () => {}, [], (mints) => { + events.push("gate"); + gate.publishStuckMints(mints); + }, { + fetchImpl: (async (url: unknown) => { + if (String(url).includes("dead.example.com")) throw new Error("offline"); + return new Response("{}"); + }) as unknown as typeof fetch, + cleanupLocalState: async () => { events.push("cleanup"); }, + }).then(() => gate.complete()); + await recovery; + // The dead mint was probed, never asked to observe its quote; the gate + // opened before settlement spent anything on the live mint. + expect(events).toEqual(["cleanup", "gate", "observe:live-quote"]); + expect(observe).toHaveBeenCalledTimes(1); + expect(fail).toHaveBeenCalledTimes(1); + } finally { + observe.mockRestore(); + fail.mockRestore(); + db.close(); + } +}); + it("local housekeeping cleans init sends and orphaned reservations without network", async () => { const db = new Database(":memory:"); const repo = new SqliteRepositories({ database: db }); @@ -172,3 +234,24 @@ it("local housekeeping cleans init sends and orphaned reservations without netwo db.close(); } }); + +it("cleanup checks all private methods before making any local writes", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const internals = coco as unknown as { mintOperationService: { recoverInitOperation: unknown } }; + const original = internals.mintOperationService.recoverInitOperation; + try { + await repo.sendOperationRepository.create({ + id: "untouched-init", mintUrl: MINT, amount: 8, state: "init", method: "default", + methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + internals.mintOperationService.recoverInitOperation = undefined; + await expect(cleanupLocalRecoveryState(coco, repo)).rejects.toThrow("mintOperationService.recoverInitOperation is unavailable"); + expect((await repo.sendOperationRepository.getById("untouched-init"))?.state).toBe("init"); + } finally { + internals.mintOperationService.recoverInitOperation = original; + db.close(); + } +}); diff --git a/src/daemon/wallet/recovery-probe.test.ts b/src/daemon/wallet/recovery-probe.test.ts index 0be1a06..18b7ca8 100644 --- a/src/daemon/wallet/recovery-probe.test.ts +++ b/src/daemon/wallet/recovery-probe.test.ts @@ -7,6 +7,8 @@ import { type SendRecoveryService, } from "./recovery-probe"; +import { runMintQuoteRecovery, settlePendingMintQuotes, settleExpiredMintQuotes } from "./coco-client"; + interface FakeOp { id: string; mintUrl: string; @@ -52,7 +54,11 @@ function makeSource( return { stuck: full, refreshed, - send: family("send") as FakeSource["send"], + send: { + ...family("send"), + get: async (id: string) => + (full.send.find((o) => o.id === id) ?? null) as never, + } as FakeSource["send"], melt: family("melt") as FakeSource["melt"], receive: family("receive") as FakeSource["receive"], mint: family("mint") as FakeSource["mint"], @@ -61,12 +67,24 @@ function makeSource( function makeSendService(): SendRecoveryService & { executingRecovered: string[]; + /** Lock events in order, e.g. "acquire:op-1", "release:op-1". */ + lockLog: string[]; } { const executingRecovered: string[] = []; + const lockLog: string[] = []; return { executingRecovered, + lockLog, + acquireOperationLock: async (id) => { + lockLog.push(`acquire:${id}`); + return () => { + lockLog.push(`release:${id}`); + }; + }, recoverExecutingOperation: async (raw) => { - executingRecovered.push((raw as FakeOp).id); + const id = (raw as FakeOp).id; + executingRecovered.push(id); + lockLog.push(`recover:${id}`); }, }; } @@ -181,6 +199,54 @@ describe("runTargetedRecovery", () => { expect(result.attempted).toBe(2); expect(source.refreshed.send).toEqual(["pending-1"]); expect(sendService.executingRecovered).toEqual(["exec-1"]); + // The executing send is driven under coco's per-operation lock. + expect(sendService.lockLog).toEqual([ + "acquire:exec-1", + "recover:exec-1", + "release:exec-1", + ]); + }); + + it("re-reads state under the lock and skips a send that left executing", async () => { + const source = makeSource({ + send: [op("s1", "https://live.example.com", "pending")], + }); + const sendService = makeSendService(); + // Snapshot taken while the op was still executing; it has since settled. + const result = await runTargetedRecovery(source, sendService, { + stuckOperations: [ + { + kind: "send", + id: "s1", + mintUrl: "https://live.example.com", + state: "executing", + raw: op("s1", "https://live.example.com", "executing"), + }, + ], + unreachableMints: new Set(), + }); + expect(result.attempted).toBe(1); + expect(sendService.executingRecovered).toEqual([]); + // The lock is still acquired and released around the re-read. + expect(sendService.lockLog).toEqual(["acquire:s1", "release:s1"]); + }); + + it("counts a send as busy when a live execute wins the lock race", async () => { + const source = makeSource({ + send: [op("s1", "https://live.example.com", "executing")], + }); + const sendService = makeSendService(); + sendService.acquireOperationLock = async () => { + const error = new Error("operation in progress"); + error.name = "OperationInProgressError"; + throw error; + }; + const result = await runTargetedRecovery(source, sendService, { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.busy).toBe(1); + expect(result.failed).toBe(0); + expect(sendService.executingRecovered).toEqual([]); }); it("counts per-operation failures at reachable mints and continues", async () => { @@ -229,7 +295,9 @@ it("does not drive locked operations or count rolling-back sends as attempts", a const service = makeSendService(); const result = await runTargetedRecovery(source, service, { fetchImpl: liveFetch().fetchImpl }); expect(result.attempted).toBe(0); + expect(result.busy).toBe(1); expect(service.executingRecovered).toEqual([]); + expect(service.lockLog).toEqual([]); }); it("classifies malformed persisted URLs as unreachable without fetching", async () => { @@ -249,3 +317,132 @@ it("bounds a hanging probe with an abort signal", async () => { }); expect([...unreachable]).toEqual(["https://slow.example.com"]); }); + + +it("shares timed-out mint observations across quote recovery, targeted recovery and the sweep", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + let finish!: (value: { category: "waiting" }) => void; + const observation = new Promise<{ category: "waiting" }>(r => { finish = r; }); + const outstanding = new Map>(); + const quoteSource = { + ops: { mint: { + listPending: async () => [{ ...source.stuck.mint[0]!, method: "bolt11" }], + get: async () => null, + finalize: async () => ({ state: "finalized" }), + } }, + mintOperationService: { observePendingOperation: () => observation }, + reopenFailedOperation: async () => false, + }; + try { + await runMintQuoteRecovery(quoteSource as never, { outstanding, timeoutMs: 5 }); + expect(outstanding.has("mint:q1")).toBe(true); + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.busy).toBe(1); + await settlePendingMintQuotes({ + ops: { mint: { ...source.mint, listPending: async () => source.stuck.mint as never } }, + wallet: { balances: { byMint: async () => ({}) } }, + mintOperationService: { failPendingOperation: async () => ({}) }, + } as never, Date.now(), { state: { outstanding } }); + expect(source.refreshed.mint).toEqual([]); + } finally { finish({ category: "waiting" }); await observation; } +}); + +it("tracks a hung targeted mint so quote recovery skips it while unrelated operations proceed", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")], melt: [op("m1", "https://live.example.com")] }); + let finish!: (value: never) => void; + source.mint.refresh = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + try { + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, timeoutMs: 5, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(source.refreshed.melt).toEqual(["m1"]); + const quote = await runMintQuoteRecovery({ + ops: { mint: { listPending: async () => [{ ...source.stuck.mint[0]!, method: "bolt11" }] } }, + } as never, { outstanding }); + expect(quote.busy).toBe(1); + } finally { finish({ state: "pending" } as never); } +}); + +it("retains executing-send lock until a timed-out underlying drive actually finishes", async () => { + const source = makeSource({ send: [op("s1", "https://live.example.com", "executing")] }); + const service = makeSendService(); + let finish!: () => void; + service.recoverExecutingOperation = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + const result = await runTargetedRecovery(source, service, { + outstanding, timeoutMs: 5, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(service.lockLog).toEqual(["acquire:s1"]); + const retry = await runTargetedRecovery(source, service, { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(retry.busy).toBe(1); + const actual = outstanding.get("send:s1")!; + finish(); + await actual; + expect(service.lockLog).toEqual(["acquire:s1", "release:s1"]); + expect(outstanding.size).toBe(0); +}); + +it("refuses missing executing-send internals without driving the operation", async () => { + const source = makeSource({ send: [op("s1", "https://live.example.com", "executing")] }); + const result = await runTargetedRecovery(source, {} as SendRecoveryService, { fetchImpl: liveFetch().fetchImpl }); + expect(result.failed).toBe(1); +}); + + +it("startup settlement keeps its timed-out observation visible to manual recovery", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + let finish!: (value: { category: "ready" }) => void; + const observation = new Promise<{ category: "ready" }>(r => { finish = r; }); + const outstanding = new Map>(); + const settlement = await settleExpiredMintQuotes({ + ops: { mint: { listPending: async () => [{ ...source.stuck.mint[0]!, expiry: 1, updatedAt: 1 }] } }, + mintOperationService: { observePendingOperation: () => observation }, + } as never, Date.now(), 5, { outstanding }); + expect(settlement.unobserved).toBe(1); + const result = await runTargetedRecovery(source, makeSendService(), { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(result.busy).toBe(1); + expect(source.refreshed.mint).toEqual([]); + const actual = outstanding.get("mint:q1")!; + finish({ category: "ready" }); + await actual; + expect(outstanding.size).toBe(0); +}); + +it("deadline and shutdown leave remaining operations untouched", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + const result = await runTargetedRecovery(source, makeSendService(), { + shouldStop: () => true, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.skipped).toBe(1); + expect(result.attempted).toBe(0); + expect(source.refreshed.mint).toEqual([]); +}); + +it("a pass budget bounds total drive waits, not just each operation", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com"), op("q2", "https://live.example.com")] }); + let finish!: (value: never) => void; + source.mint.refresh = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, timeoutMs: 100, deadlineMs: 10, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(result.attempted).toBe(1); + expect(result.skipped).toBe(1); + const actual = outstanding.get("mint:q1")!; + finish({ state: "pending" } as never); + await actual; +}); + +it("unfinished work in another operation family does not block the same bare id", async () => { + const source = makeSource({ mint: [op("same-id", "https://live.example.com")] }); + const outstanding = new Map>([["send:same-id", new Promise(() => {})]]); + const result = await runTargetedRecovery(source, makeSendService(), { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(result.busy).toBe(0); + expect(source.refreshed.mint).toEqual(["same-id"]); +}); diff --git a/src/daemon/wallet/recovery-probe.ts b/src/daemon/wallet/recovery-probe.ts index 9dce35f..3dddfaa 100644 --- a/src/daemon/wallet/recovery-probe.ts +++ b/src/daemon/wallet/recovery-probe.ts @@ -7,9 +7,11 @@ * and drives recovery per operation only for mints that answer — so one dead * mint costs a single short probe instead of N sequential timeouts, and its * operations stay parked (exactly as coco's "will retry later" path leaves - * them) until a later startup finds the mint reachable. + * them) until an explicit mid-session recovery (see recoverStuckOperations + * in coco-client.ts) or a later startup finds the mint reachable. */ import { normalizeMintUrl, type Manager } from "@cashu/coco-core"; +import { recoveryKey, trackRecovery, waitForRecoveryWork, RecoveryWaitTimeout, type RecoveryWork } from "./recovery-work"; import { logger } from "../../utils/logger"; /** Short probe: a mint that cannot answer /v1/info in 2s slows every op. */ @@ -19,7 +21,7 @@ type OpsApi = Manager["ops"]; /** Structural subset of the ops APIs used to enumerate and recover operations. */ export interface StuckOperationSource { - send: Pick; + send: Pick; melt: Pick; receive: Pick; mint: Pick; @@ -45,20 +47,37 @@ export interface StuckOperation { */ export interface SendRecoveryService { recoverExecutingOperation(op: unknown): Promise; + /** + * coco's per-operation lock. It is fail-fast: acquiring an id a live + * execute/finalize/recover already holds throws OperationInProgressError + * instead of waiting. Holding it across the state re-read and the drive + * makes executing-send recovery atomic against a live execute — the same + * pattern reopenFailedMintOperation uses for mint operations + * (see coco-client.ts). + */ + acquireOperationLock(operationId: string): Promise<() => void>; } export interface RecoveryRunResult { /** Operations for which recovery was attempted (not necessarily completed). */ attempted: number; - /** Operations skipped because their mint did not answer the probe. */ + /** Timed-out waits, also included in attempted; underlying work remains tracked. */ + timedOut: number; + /** Locked operations or unfinished work from another pass; retry later. */ + busy: number; + /** Operations skipped for unreachable mints, shutdown, or pass budget exhaustion. */ skipped: number; - /** Operations at reachable mints whose recovery still failed. */ + /** Attempts that threw a non-busy, non-timeout error. */ failed: number; /** Unreachable mint URL -> number of operations skipped there. */ skippedMints: Map; } export interface TargetedRecoveryOptions { + outstanding?: RecoveryWork; + timeoutMs?: number; + deadlineMs?: number; + shouldStop?: () => boolean; probeTimeoutMs?: number; fetchImpl?: typeof fetch; /** Operation families to recover. Defaults to all four. */ @@ -159,8 +178,31 @@ async function recoverStuckOperation( // Public API: actively re-checks the proofs with the mint. await source.send.refresh(op.id); } else if (op.state === "executing") { - // Startup snapshot only; skip live operations in the driver below. - await sendService.recoverExecutingOperation(op.raw); + // Fail closed: the lock and the drive are private coco internals + // (written against coco-core 1.0.1), so a coco bump that renames them + // must fail this operation loudly instead of corrupting it. + if ( + typeof sendService.acquireOperationLock !== "function" || + typeof sendService.recoverExecutingOperation !== "function" + ) { + throw new Error( + "coco sendOperationService recovery internals are unavailable; refusing to recover an executing send", + ); + } + // recoverExecutingOperation takes no lock and does no state re-read, + // so take coco's fail-fast per-operation lock first (throws + // OperationInProgressError when a live execute holds the operation — + // the driver counts that as busy, not failed) and re-read the state + // under it: the snapshot op must still be executing before we drive. + const release = await sendService.acquireOperationLock(op.id); + try { + const latest = await source.send.get(op.id); + if (latest?.state === "executing") { + await sendService.recoverExecutingOperation(latest); + } + } finally { + release(); + } } // prepared / rolling_back: the global sweep only warns; nothing to do. return; @@ -194,11 +236,20 @@ export async function runTargetedRecovery( ): Promise { const result: RecoveryRunResult = { attempted: 0, + timedOut: 0, + busy: 0, skipped: 0, failed: 0, skippedMints: new Map(), }; + const timeoutMs = options.timeoutMs ?? 15_000; + const deadlineMs = options.deadlineMs ?? 60_000; + if (![timeoutMs, deadlineMs].every(n => Number.isFinite(n) && n > 0)) { + throw new Error("Recovery budgets must be positive finite numbers"); + } + const deadline = Date.now() + deadlineMs; + const outstanding = options.outstanding ?? new Map(); const kinds = options.kinds ?? ["send", "melt", "receive", "mint"]; const stuck = (options.stuckOperations ?? (await collectStuckOperations(source))).filter( (op) => kinds.includes(op.kind), @@ -221,12 +272,30 @@ export async function runTargetedRecovery( for (const op of stuck) { if (unreachable.has(op.mintUrl)) continue; - if (source[op.kind].diagnostics.isLocked(op.id)) continue; + if (options.shouldStop?.() || Date.now() >= deadline) { result.skipped++; continue; } + const key = recoveryKey(op.kind, op.id); + if (outstanding.has(key)) { result.busy++; continue; } + // Cheap pre-filter for live operations; the lock inside + // recoverStuckOperation is what actually makes the drive atomic. + if (source[op.kind].diagnostics.isLocked(op.id)) { + result.busy++; + continue; + } if (op.kind === "send" && !["pending", "executing"].includes(op.state)) continue; result.attempted++; try { - await recoverStuckOperation(source, sendService, op); + const work = trackRecovery(outstanding, key, recoverStuckOperation(source, sendService, op)); + await waitForRecoveryWork(work, Math.min(timeoutMs, Math.max(1, deadline - Date.now()))); } catch (error) { + if (error instanceof RecoveryWaitTimeout) { result.timedOut++; continue; } + // A live execute grabbed the operation between the isLocked pre-filter + // and the lock acquisition: busy, not failed — leave it for a later + // pass. Name-matched like runMintQuoteRecovery does, because the error + // crosses a package boundary. + if (error instanceof Error && error.name === "OperationInProgressError") { + result.busy++; + continue; + } // Same semantics as coco's tryRecover*: leave the operation for the // next pass. A reachable mint can still reject a specific operation. result.failed++; diff --git a/src/daemon/wallet/recovery-work.test.ts b/src/daemon/wallet/recovery-work.test.ts new file mode 100644 index 0000000..06bfb51 --- /dev/null +++ b/src/daemon/wallet/recovery-work.test.ts @@ -0,0 +1,41 @@ +import { expect, it } from "bun:test"; +import { createRunQueue } from "./coco-client"; +import { createRecoveryDisposer, drainRecoveryWork, trackRecovery } from "./recovery-work"; + +it("incomplete shutdown retains resources until late writes settle; retries join disposal", async () => { + const work = new Map>(); + let finish!: () => void; + const events: string[] = []; + trackRecovery(work, "mint:q", new Promise(r => { finish = r; }).then(() => { events.push("write"); })); + const dispose = createRecoveryDisposer(() => {}, () => drainRecoveryWork(work), async () => { events.push("close"); }, 5); + await expect(dispose()).rejects.toThrow("Timed out"); + expect(events).toEqual([]); + finish(); + await dispose(); + expect(events).toEqual(["write", "close"]); + await dispose(); + expect(events).toEqual(["write", "close"]); +}); + +it("shutdown drains active queue work and queued callbacks reject before touching DB", async () => { + const queue = createRunQueue(); + let disposed = false; + let finish!: () => void; + const events: string[] = []; + let started!: () => void; + const ready = new Promise(r => { started = r; }); + const active = queue(() => new Promise(r => { finish = r; started(); }).then(() => { events.push("write"); })); + await ready; + const queued = queue(async () => { + if (disposed) throw new Error("Wallet is shutting down"); + events.push("unexpected"); + }); + const rejected = queued.catch(error => error); + const dispose = createRecoveryDisposer(() => { disposed = true; }, () => queue.drain(), async () => { events.push("close"); }, 5); + await expect(dispose()).rejects.toThrow("Timed out"); + finish(); + await active; + expect((await rejected).message).toContain("shutting down"); + await dispose(); + expect(events).toEqual(["write", "close"]); +}); diff --git a/src/daemon/wallet/recovery-work.ts b/src/daemon/wallet/recovery-work.ts new file mode 100644 index 0000000..a776f74 --- /dev/null +++ b/src/daemon/wallet/recovery-work.ts @@ -0,0 +1,45 @@ +/** A timed-out wait is not cancellation: retain work until it actually settles. */ +export type RecoveryWork = Map>; +export const recoveryKey = (kind: string, id: string): string => `${kind}:${id}`; + +export function trackRecovery(work: RecoveryWork, key: string, promise: Promise): Promise { + work.set(key, promise); + const clear = () => { if (work.get(key) === promise) work.delete(key); }; + void promise.then(clear, clear); + return promise; +} + +export class RecoveryWaitTimeout extends Error { + constructor() { super("Timed out waiting for recovery; underlying work is still tracked"); } +} + +export async function waitForRecoveryWork(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new RecoveryWaitTimeout()), timeoutMs); + })]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** Drain actual work, including work registered while an earlier task settles. */ +export async function drainRecoveryWork(work: RecoveryWork): Promise { + while (work.size) await Promise.allSettled([...work.values()]); +} + +/** Timeout reports incomplete disposal; actual cleanup continues safely. */ +export function createRecoveryDisposer( + quiesce: () => void, + settle: () => Promise, + close: () => Promise, + timeoutMs = 30_000, +): () => Promise { + let disposal: Promise | undefined; + return async () => { + quiesce(); + disposal ??= (async () => { await settle(); await close(); })(); + await waitForRecoveryWork(disposal, timeoutMs); + }; +}