diff --git a/SKILL.md b/SKILL.md index 7c47eb5..0302042 100644 --- a/SKILL.md +++ b/SKILL.md @@ -121,6 +121,8 @@ Show wallet transaction history. |--------|---------|-------------| | `-n, --limit ` | 50 | Number of entries to show | | `--offset ` | 0 | Number of entries to skip | +| `-t, --type ` | all | Filter by transaction type (`send`, `receive`, `mint`, `melt`). Repeatable or comma-separated | +| `-i, --id ` | | Show a single transaction by its ID (prints one summary line; add `--verbose` for full details) | | `-v, --verbose` | false | Show full details including encoded Cashu tokens | | `--json` | false | Output raw JSON with token objects (no encoding) | diff --git a/docs/wallet-mint-recovery.md b/docs/wallet-mint-recovery.md new file mode 100644 index 0000000..cb344ef --- /dev/null +++ b/docs/wallet-mint-recovery.md @@ -0,0 +1,71 @@ +# Mint quote recovery: scope and troubleshooting + +`routstrd wallet recover` explicitly retries mint operations through coco using +**their existing stored outputs**. It can restore signatures when a quote is +already issued, and reopen failed operations when explicitly requested: + +```sh +routstrd history --json +routstrd wallet recover --op --include-failed +``` + +Failed operations require explicit IDs over both HTTP and the CLI. A successful +re-run on an already finalized operation is a no-op. Requests that exceed their +wait budget are not cancelled; explicit retries skip the operation while the +underlying work is outstanding. + +## What this fixes—and what it does not + +Coco already checks pending quotes on startup and the daemon periodically +refreshes them. A quote paid while the daemon was offline does not, by itself, +require a new issuance implementation. + +This change makes normal cleanup confirm UNPAID with the mint before failing an +expired quote. PAID, ISSUED and unverified quotes remain pending. It also gives +operators a recovery path for operations previously marked failed. + +It does **not** replace rejected outputs with fresh outputs on an active keyset. +An inactive-keyset rejection can therefore remain retryable with zero recovery. +Recovery reports coco's persisted mint error when available, rather than only a +generic “remains pending” error. + +Do not infer that the production incidents were caused by keyset retirement. +Before claiming those incidents are fixed, collect: + +- The affected operation IDs, quote IDs, state and persisted `error`. +- A fresh remote quote state and, where provided, paid/issued amounts. +- The keyset IDs in the stored outputs and the mint's current keyset metadata. +- A reproduction showing existing recovery fails and the proposed fix succeeds. + +Inspect persisted operation data through a read-only database copy; do not edit +rows or run recovery scripts concurrently with a daemon against the same wallet. +Never share the mnemonic, output secrets, or full wallet database in a PR. + +A future fresh-output path must preserve original outputs for uncertain issuance +and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate +with coco's watcher/processor. It needs its own integration tests before handling +real funds. + +## Cleanup preview and force + +`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not +confirmed failures. `failedMintQuotes` and `leftForRecovery` are zero because no +mint check or cleanup transition was performed. Send/melt counts remain planned +cleanup counts in dry-run mode. + +`--force` deliberately bypasses mint confirmation and can strand paid sats in a +failed operation. Prefer normal cleanup. Forced operations can be retried with +`--op --include-failed`, but recovery still depends on the mint +accepting their stored outputs or restoring their signatures. + +## Integration and release notes + +The reopen helper uses private coco-core 1.0.1 methods. Retain real-Manager and +HTTP fake-mint coverage, use frozen dependency installs, and re-run integration +tests on coco upgrades. A controlled low-value live-mint smoke test remains +recommended before release. + +PR #118 removes `cocod-client.ts`. When integrating that change, move recovery +and cleanup contracts into its replacement `wallet-client.ts`, rename HTTP error +references accordingly, and make recovery mandatory for the in-process client. +This follow-up does not pull in #118's unrelated removal. diff --git a/src/cli.ts b/src/cli.ts index c25f35e..badd231 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,6 +58,10 @@ import { } from "./daemon/wallet/paths"; import * as QRCode from "qrcode"; import { normalizeNostrPubkey, npubFromPubkey, npubFromSecretKey } from "./utils/nip98"; +import { + HISTORY_ENTRY_TYPES, + isHistoryEntryType, +} from "./utils/history"; import { generateSecretKey, nip19 } from "nostr-tools"; import { generateMnemonic } from "@scure/bip39"; import { wordlist } from "@scure/bip39/wordlists/english.js"; @@ -1789,17 +1793,50 @@ program .description("Show wallet transaction history") .option("-n, --limit ", "Number of entries to show", "50") .option("--offset ", "Number of entries to skip", "0") - .option("-v, --verbose", "Show full details including encoded Cashu tokens") + .option( + "-t, --type ", + "Filter by transaction type (send, receive, mint, melt). Repeatable or comma-separated", + ) + .option("-i, --id ", "Show a single transaction by its ID") + .option("-v, --verbose", "Show full details instead of the summary line") .option("--json", "Output raw JSON with token objects (no encoding)") - .action(async (options: { limit: string; offset: string; verbose: boolean; json: boolean }) => { + .action(async (options: { + limit: string; + offset: string; + type?: string[]; + id?: string; + verbose: boolean; + json: boolean; + }) => { await ensureDaemonRunning(); const limit = Math.min(parseInt(options.limit, 10) || 50, 1000); const offset = parseInt(options.offset, 10) || 0; - const result = await callDaemon( - `/wallet/history?offset=${offset}&limit=${limit}`, + // Support both repeated flags (-t send -t mint) and comma-separated + // values (-t send,mint). + const requestedTypes = (options.type ?? []) + .flatMap((value) => value.split(",")) + .map((value) => value.trim().toLowerCase()) + .filter((value) => value.length > 0); + const unknownTypes = requestedTypes.filter( + (value) => !isHistoryEntryType(value), ); + if (unknownTypes.length > 0) { + console.error( + `Unknown transaction type: ${unknownTypes.join(", ")}. Valid types: ${HISTORY_ENTRY_TYPES.join(", ")}.`, + ); + process.exit(1); + } + + const query = new URLSearchParams({ + offset: String(offset), + limit: String(limit), + }); + if (requestedTypes.length > 0) query.set("type", requestedTypes.join(",")); + if (options.id) query.set("id", options.id); + + const result = await callDaemon(`/wallet/history?${query.toString()}`); if (result.error) { console.log(result.error); @@ -1812,7 +1849,15 @@ program const entries = data?.entries || []; if (entries.length === 0) { - console.log("No transaction history yet."); + if (options.id) { + console.log(`No transaction found with ID ${options.id}.`); + } else if (requestedTypes.length > 0) { + console.log( + `No ${requestedTypes.join(", ")} transactions found.`, + ); + } else { + console.log("No transaction history yet."); + } return; } @@ -1869,10 +1914,13 @@ program const pad = (s: string, w: number) => s.padEnd(w); const sep = Object.values(widths).map((w) => "-".repeat(w)).join(" | "); - console.log( - `${pad(idCol, widths.id)} | ${pad(timeCol, widths.time)} | ${pad(typeCol, widths.type)} | ${pad(mintCol, widths.mint)} | ${pad(amtCol, widths.amount)}`, - ); - console.log(sep); + // A single transaction lookup prints just its summary line. + if (!options.id) { + console.log( + `${pad(idCol, widths.id)} | ${pad(timeCol, widths.time)} | ${pad(typeCol, widths.type)} | ${pad(mintCol, widths.mint)} | ${pad(amtCol, widths.amount)}`, + ); + console.log(sep); + } for (const row of rows) { console.log( @@ -2014,16 +2062,22 @@ walletCmd .option("--mint-url ", "Only clean up operations for this mint URL") .option( "--min-age ", - "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)", + "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are checked with their mint)", "168", ) .option("--dry-run", "Report what would be cleaned without applying changes", false) + .option( + "--force", + "Fail expired mint quotes without confirming UNPAID with the mint (may strand paid quotes)", + false, + ) .option("-y, --yes", "Skip confirmation prompt", false) .action( async (options: { mintUrl?: string; minAge: string; dryRun: boolean; + force: boolean; yes: boolean; }) => { const minAgeHours = Number.parseFloat(options.minAge); @@ -2039,7 +2093,9 @@ walletCmd }); const answer = await new Promise((resolve) => { rl.question( - "This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", + options.force + ? "WARNING: --force fails expired mint quotes WITHOUT checking the mint and may strand paid sats. It also reclaims old pending sends and cancels prepared melts. Continue? [y/N] " + : "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", (value: string) => { rl.close(); resolve(value.trim().toLowerCase()); @@ -2061,6 +2117,7 @@ walletCmd mintUrl: options.mintUrl, minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000), dryRun: options.dryRun === true, + force: options.force === true, }, }); @@ -2073,6 +2130,8 @@ walletCmd | { dryRun?: boolean; failedMintQuotes?: number; + mintQuoteCandidates?: number; + leftForRecovery?: number; reclaimedSends?: number; cancelledMelts?: number; skipped?: number; @@ -2083,8 +2142,15 @@ walletCmd if (output) { const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:"; console.log(prefix); + if (output.dryRun) { + console.log( + ` Expired mint quote candidates (not checked with mint): ${output.mintQuoteCandidates ?? 0}`, + ); + } else { + console.log(` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`); + } console.log( - ` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`, + ` Expired quotes kept for recovery (paid/issued/unverified): ${output.leftForRecovery ?? 0}`, ); console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`); console.log( @@ -2115,6 +2181,124 @@ walletCmd }, ); +walletCmd + .command("recover") + .description( + "Retry mint quotes using their stored outputs (does not replace rejected outputs)", + ) + .option( + "--op ", + "Recover this operation id (repeatable; find IDs with routstrd history --json)", + (value: string, previous: string[]) => [...previous, value], + [] as string[], + ) + .option( + "--include-failed", + "Also re-open operations coco already gave up on (requires --op)", + false, + ) + .option("-y, --yes", "Skip confirmation prompt", false) + .action( + async (options: { + op: string[]; + includeFailed: boolean; + yes: boolean; + }) => { + const operationIds = options.op ?? []; + if (options.includeFailed && operationIds.length === 0) { + console.error( + "--include-failed can only target operations named with --op", + ); + process.exit(1); + } + + if (!options.yes) { + const rl = require("readline").createInterface({ + input: process.stdin, + output: process.stdout, + }); + const prompt = + operationIds.length > 0 + ? `Recover ${operationIds.length} mint quote operation(s)? [y/N] ` + : "Check every pending mint quote with its mint and claim any paid sats? [y/N] "; + const answer = await new Promise((resolve) => { + rl.question(prompt, (value: string) => { + rl.close(); + resolve(value.trim().toLowerCase()); + }); + }); + if (answer !== "y" && answer !== "yes") { + console.log("Aborted."); + return; + } + } + + try { + await ensureDaemonRunning(); + + const result = await callDaemon("/wallet/recover", { + method: "POST", + body: { + operationIds: operationIds.length > 0 ? operationIds : undefined, + includeFailed: options.includeFailed === true, + }, + }); + + if (result.error) { + console.log(result.error); + process.exit(1); + } + + const output = result.output as + | { + checked?: number; + recovered?: number; + waiting?: number; + terminal?: number; + reopened?: number; + retryable?: number; + busy?: number; + errors?: Array<{ operationId: string; error: string }>; + } + | undefined; + + if (output) { + console.log("Mint quote recovery:"); + console.log(` Checked with mint: ${output.checked ?? 0}`); + console.log( + ` Recovered (paid sats claimed): ${output.recovered ?? 0}`, + ); + console.log(` Still unpaid: ${output.waiting ?? 0}`); + console.log(` No longer issuable: ${output.terminal ?? 0}`); + console.log( + ` Re-opened failed operations: ${output.reopened ?? 0}`, + ); + console.log(` Left for a later run: ${output.retryable ?? 0}`); + console.log( + ` Skipped (recovery still running): ${output.busy ?? 0}`, + ); + if (output.errors && output.errors.length > 0) { + console.log("\nErrors:"); + for (const e of output.errors) { + console.log(` - ${e.operationId}: ${e.error}`); + } + } + } + } catch (error) { + const message = (error as Error).message; + if ( + message?.includes("fetch failed") || + message?.includes("Connection refused") + ) { + console.error("Daemon is not running"); + process.exit(1); + } + console.error(message); + process.exit(1); + } + }, + ); + const walletReceiveCmd = walletCmd .command("receive") .description("Wallet receive operations"); diff --git a/src/daemon/http/history.test.ts b/src/daemon/http/history.test.ts new file mode 100644 index 0000000..43d755c --- /dev/null +++ b/src/daemon/http/history.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "bun:test"; +import { EventEmitter } from "events"; +import type { HistoryEntry } from "@cashu/coco-core"; +import { + createDaemonRequestHandler, + getHistoryByTypes, + parseHistoryTypes, +} from "./index"; + +const MINT_URL = "https://mint.example/"; + +function makeEntry( + id: string, + type: HistoryEntry["type"], + createdAt: number, +): HistoryEntry { + return { + id, + type, + createdAt, + mintUrl: MINT_URL, + unit: "sat", + amount: 21, + } as HistoryEntry; +} + +const ENTRIES: HistoryEntry[] = [ + makeEntry("send-1", "send", 4), + makeEntry("receive-1", "receive", 3), + makeEntry("mint-1", "mint", 2), + makeEntry("melt-1", "melt", 1), + makeEntry("send-2", "send", 0), +]; + +function makeWalletClient(entries: HistoryEntry[] = ENTRIES) { + return { + getHistory: async (offset = 0, limit = 50) => + entries.slice(offset, offset + limit), + getHistoryEntryById: async (id: string) => + entries.find((entry) => entry.id === id) ?? null, + }; +} + +function makeReq(method: string, path: string) { + const req = new EventEmitter() as any; + req.method = method; + req.url = path; + req.headers = { host: "localhost" }; + return req; +} + +function makeRes() { + const res: any = { + status: 0, + body: "", + writeHead(status: number) { + res.status = status; + return res; + }, + end(chunk?: string) { + if (chunk) res.body += chunk; + return res; + }, + json() { + return JSON.parse(res.body); + }, + }; + return res; +} + +async function callHistory(path: string, entries?: HistoryEntry[]) { + const handler = createDaemonRequestHandler({ + walletClient: makeWalletClient(entries), + } as any); + const res = makeRes(); + await handler(makeReq("GET", path), res); + return res; +} + +describe("parseHistoryTypes", () => { + it("splits, trims, and lowercases comma-separated values", () => { + expect(parseHistoryTypes(" Send , MINT ")).toEqual(["send", "mint"]); + }); + + it("returns an empty list for missing or blank input", () => { + expect(parseHistoryTypes(null)).toEqual([]); + expect(parseHistoryTypes("")).toEqual([]); + expect(parseHistoryTypes(" , ")).toEqual([]); + }); +}); + +describe("getHistoryByTypes", () => { + it("filters by type and applies offset/limit after filtering", async () => { + const client = makeWalletClient(); + expect(await getHistoryByTypes(client, ["send"], 0, 10)).toEqual([ + ENTRIES[0]!, + ENTRIES[4]!, + ]); + expect(await getHistoryByTypes(client, ["send"], 1, 1)).toEqual([ + ENTRIES[4]!, + ]); + }); + + it("scans past the page size boundary to find matches", async () => { + const filler = Array.from({ length: 250 }, (_, index) => + makeEntry(`receive-${index}`, "receive", index), + ); + const target = makeEntry("mint-late", "mint", -1); + const client = makeWalletClient([...filler, target]); + + expect(await getHistoryByTypes(client, ["mint"], 0, 10)).toEqual([target]); + }); + + it("returns no entries when nothing matches", async () => { + const client = makeWalletClient([makeEntry("mint-1", "mint", 1)]); + expect(await getHistoryByTypes(client, ["send"], 0, 10)).toEqual([]); + }); +}); + +describe("GET /wallet/history", () => { + it("returns all entries without a filter", async () => { + const res = await callHistory("/wallet/history"); + expect(res.status).toBe(200); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-1", + "receive-1", + "mint-1", + "melt-1", + "send-2", + ]); + }); + + it("filters entries by a comma-separated type list", async () => { + const res = await callHistory("/wallet/history?type=send,melt"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-1", + "melt-1", + "send-2", + ]); + }); + + it("honors offset/limit for filtered results", async () => { + const res = await callHistory("/wallet/history?type=send&offset=1&limit=1"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "send-2", + ]); + }); + + it("looks up a single entry by id", async () => { + const res = await callHistory("/wallet/history?id=mint-1"); + const body = res.json(); + expect(body.output.entries.map((e: HistoryEntry) => e.id)).toEqual([ + "mint-1", + ]); + }); + + it("returns an empty list when the id is unknown", async () => { + const res = await callHistory("/wallet/history?id=does-not-exist"); + const body = res.json(); + expect(body.output.entries).toEqual([]); + }); +}); diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 05349bb..31cae7c 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -283,6 +283,21 @@ function optionalStringField( return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function optionalStringArrayField( + body: Record, + field: string, +): string[] | undefined { + const value = body[field]; + if (value === undefined) return undefined; + if ( + !Array.isArray(value) || + value.some((item) => typeof item !== "string" || !item.trim()) + ) { + throw new CocodHttpError(400, `'${field}' must be an array of non-empty strings.`); + } + return value.map((item: string) => item.trim()); +} + function getCurrentMode(deps: DaemonDeps): ClientMode { const stateMode = deps.store.getState()?.mode; return stateMode || deps.mode || "apikeys"; @@ -375,6 +390,46 @@ function makeSdkLogger(...parts: string[]): SdkLogger { }; } +/** + * Parse a `type` query parameter into a list of normalized transaction types. + * Accepts comma-separated values and is case-insensitive. + */ +export function parseHistoryTypes(raw: string | null | undefined): string[] { + if (!raw) return []; + return raw + .split(",") + .map((value) => value.trim().toLowerCase()) + .filter((value) => value.length > 0); +} + +/** Page size used when scanning history to apply a type filter. */ +const HISTORY_TYPE_SCAN_PAGE_SIZE = 200; + +/** + * Return history entries matching `types`, with offset/limit applied after + * filtering. The coco history repository only paginates by raw position, so we + * scan pages until enough matches accumulate (or history is exhausted). + */ +export async function getHistoryByTypes( + client: Pick, + types: string[], + offset: number, + limit: number, +): Promise { + const wanted = new Set(types); + const matched: HistoryEntry[] = []; + let scanOffset = 0; + while (matched.length < offset + limit) { + const page = await client.getHistory(scanOffset, HISTORY_TYPE_SCAN_PAGE_SIZE); + for (const entry of page) { + if (wanted.has(entry.type)) matched.push(entry); + } + if (page.length < HISTORY_TYPE_SCAN_PAGE_SIZE) break; + scanOffset += HISTORY_TYPE_SCAN_PAGE_SIZE; + } + return matched.slice(offset, offset + limit); +} + export function createDaemonRequestHandler(deps: { provider: string | null; server: { close(cb?: () => void): void }; @@ -473,12 +528,63 @@ export function createDaemonRequestHandler(deps: { ? body.minAgeMs : undefined, dryRun: body.dryRun === true, + force: body.force === true, }); return { output: result }; }); return; } + if (req.method === "POST" && url.pathname === "/wallet/recover") { + await respond(res, async () => { + if (!deps.walletClient.recoverMintQuotes) { + throw new CocodHttpError( + 501, + "Mint quote recovery is not supported by this wallet client.", + ); + } + + const body = await readJsonBody(req); + const operationIds = optionalStringArrayField(body, "operationIds"); + if (body.includeFailed === true && !operationIds?.length) { + throw new CocodHttpError( + 400, + "'includeFailed' requires non-empty 'operationIds'.", + ); + } + if ( + body.timeoutMs !== undefined && + (typeof body.timeoutMs !== "number" || + !Number.isFinite(body.timeoutMs) || body.timeoutMs <= 0) + ) { + throw new CocodHttpError( + 400, + "'timeoutMs' must be a positive finite number.", + ); + } + const result = await deps.walletClient.recoverMintQuotes({ + operationIds, + includeFailed: body.includeFailed === true, + timeoutMs: body.timeoutMs as number | undefined, + }); + return { output: result }; + }); + return; + } + + if (req.method === "POST" && url.pathname === "/wallet/recover/operations") { + await respond(res, async () => { + if (!deps.walletClient.recoverStuckOperations) { + throw new CocodHttpError( + 501, + "Stuck operation recovery is not supported by this wallet client.", + ); + } + return { output: await deps.walletClient.recoverStuckOperations() }; + }); + return; + } + if (req.method === "POST" && url.pathname === "/wallet/receive/cashu") { await respond(res, async () => { const body = await readJsonBody(req); @@ -596,9 +702,25 @@ export function createDaemonRequestHandler(deps: { await respond(res, async () => { const offsetParam = url.searchParams.get("offset"); const limitParam = url.searchParams.get("limit"); + const idParam = url.searchParams.get("id")?.trim() || ""; + const types = parseHistoryTypes(url.searchParams.get("type")); const offset = offsetParam ? parseInt(offsetParam, 10) || 0 : 0; const limit = limitParam ? parseInt(limitParam, 10) || 50 : 50; - const entries = await deps.walletClient.getHistory(offset, limit); + + let entries: HistoryEntry[]; + if (idParam) { + const entry = await deps.walletClient.getHistoryEntryById(idParam); + entries = entry ? [entry] : []; + } else if (types.length > 0) { + entries = await getHistoryByTypes( + deps.walletClient, + types, + offset, + limit, + ); + } else { + entries = await deps.walletClient.getHistory(offset, limit); + } const encoded = entries.map((entry: HistoryEntry) => { const base = { ...entry } as Record; diff --git a/src/daemon/http/wallet-recovery.test.ts b/src/daemon/http/wallet-recovery.test.ts new file mode 100644 index 0000000..4c61737 --- /dev/null +++ b/src/daemon/http/wallet-recovery.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, mock } from "bun:test"; +import { EventEmitter } from "node:events"; +import { createDaemonRequestHandler } from "./index"; + +async function recover(body: unknown) { + const recoverMintQuotes = mock(async (_options: unknown) => ({ recovered: 0 })); + const handler = createDaemonRequestHandler({ walletClient: { recoverMintQuotes } } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from(JSON.stringify(body))); + req.emit("end"); + }); + await handler(req, res as never); + return { res, recoverMintQuotes }; +} + +describe("POST /wallet/recover validation", () => { + it.each([{}, { operationIds: [] }])("rejects includeFailed without explicit IDs: %j", async (body) => { + const { res, recoverMintQuotes } = await recover({ ...body, includeFailed: true }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([[""], [" "], [42]])("rejects invalid operation IDs: %j", async (operationIds) => { + const { res, recoverMintQuotes } = await recover({ operationIds }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([0, -1, "1000"])("rejects invalid timeout %j", async (timeoutMs) => { + const { res, recoverMintQuotes } = await recover({ timeoutMs }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it("passes normalized explicit IDs and a positive timeout", async () => { + const { res, recoverMintQuotes } = await recover({ operationIds: [" op-1 "], includeFailed: true, timeoutMs: 1000 }); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledWith({ operationIds: ["op-1"], includeFailed: true, timeoutMs: 1000 }); + }); + it("still permits checking pending quotes without IDs", async () => { + const { res, recoverMintQuotes } = await recover({}); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledTimes(1); + }); +}); + +async function recoverOperations() { + const recoverStuckOperations = mock(async () => ({ + attempted: 1, + busy: 0, + skipped: 0, + failed: 0, + skippedMints: {}, + })); + const handler = createDaemonRequestHandler({ walletClient: { recoverStuckOperations } } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover/operations", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from("{}")); + req.emit("end"); + }); + await handler(req, res as never); + return { res, recoverStuckOperations }; +} + +describe("POST /wallet/recover/operations", () => { + it("drives stuck-operation recovery and returns the summary", async () => { + const { res, recoverStuckOperations } = await recoverOperations(); + expect(res.status).toBe(200); + expect(recoverStuckOperations).toHaveBeenCalledTimes(1); + expect(JSON.parse(res.body).output).toMatchObject({ attempted: 1, busy: 0 }); + }); + + it("returns 501 when the wallet client does not support it", async () => { + const handler = createDaemonRequestHandler({ walletClient: {} } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover/operations", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from("{}")); + req.emit("end"); + }); + await handler(req, res as never); + expect(res.status).toBe(501); + }); +}); diff --git a/src/daemon/wallet/cleanup.test.ts b/src/daemon/wallet/cleanup.test.ts index 381ab4e..b7a8568 100644 --- a/src/daemon/wallet/cleanup.test.ts +++ b/src/daemon/wallet/cleanup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; const NOW_MS = 1_800_000_000_000; const DAY_MS = 24 * 60 * 60 * 1000; @@ -166,3 +166,14 @@ describe("selectCleanupOperations", () => { expect(result.meltsToCancel).toEqual([]); }); }); + +describe("mint cleanup reporting", () => { + it("reports dry-run candidates, not confirmed failures", () => { + expect(summarizeMintCleanup({ dryRun: true, candidates: 3, failed: 0, leftForRecovery: 0 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 0, leftForRecovery: 0 }); + }); + it("reports only actual failures in a real run", () => { + expect(summarizeMintCleanup({ dryRun: false, candidates: 3, failed: 1, leftForRecovery: 2 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 1, leftForRecovery: 2 }); + }); +}); diff --git a/src/daemon/wallet/cleanup.ts b/src/daemon/wallet/cleanup.ts index bbf24c2..1ff48ea 100644 --- a/src/daemon/wallet/cleanup.ts +++ b/src/daemon/wallet/cleanup.ts @@ -64,8 +64,8 @@ export interface CleanupSelection< * can have happened before expiry while the daemon was down, leaving no * local observation. Callers that fail quotes automatically at startup must * therefore confirm UNPAID with the mint first (see - * settleExpiredMintQuotes in coco-client.ts); only the explicit, - * user-invoked cleanup command may fail candidates purely locally. + * failExpiredMintQuoteIfUnpaid in coco-client.ts). Explicit cleanup follows + * the same rule unless the operator opts into unsafe `--force` behaviour. * - Pending sends are reclaimed (rolled back) only when they are older than * `minAgeMs`, so we never roll back a token that a receiver might still * legitimately claim. @@ -102,3 +102,17 @@ export function selectCleanupOperations< return { mintsToFail, sendsToReclaim, meltsToCancel }; } + +/** Keep a local-only dry-run preview distinct from mint-confirmed outcomes. */ +export function summarizeMintCleanup(input: { + dryRun: boolean; + candidates: number; + failed: number; + leftForRecovery: number; +}) { + return { + mintQuoteCandidates: input.candidates, + failedMintQuotes: input.dryRun ? 0 : input.failed, + leftForRecovery: input.dryRun ? 0 : input.leftForRecovery, + }; +} diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index 959e798..b020237 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -14,12 +14,18 @@ import { assertLegacyCocodNotRunning, claimLegacyCocodPidFile, createCocoClient, + createRecoveryGate, + createRunQueue, DEFAULT_TRUSTED_MINT_URLS, + failExpiredMintQuoteIfUnpaid, isZombieProcess, + reopenFailedMintOperation, + runMintQuoteRecovery, settleExpiredMintQuotes, settlePendingMintQuotes, stopLegacyCocod, type ExpiredMintQuoteSource, + type MintQuoteRecoverySource, type PendingMintQuoteSource, type PendingMintSweepState, } from "./coco-client"; @@ -675,6 +681,29 @@ describe("settleExpiredMintQuotes", () => { expect(observePendingOperation).not.toHaveBeenCalled(); expect(failPendingOperation).not.toHaveBeenCalled(); }); + + it("skips quotes at probe-unreachable mints without spending the budget", async () => { + const ops = [ + pendingMintOp({ id: "dead-1", mintUrl: "https://dead.example.com" }), + pendingMintOp({ id: "dead-2", mintUrl: "https://dead.example.com/" }), + pendingMintOp({ id: "live-1", mintUrl: "https://live.example.com" }), + ]; + const { source, observePendingOperation, failPendingOperation } = + fakeSource(ops); + + const result = await settleExpiredMintQuotes( + source, + NOW_MS, + undefined, + { unreachableMints: new Set(["https://dead.example.com"]) }, + ); + + expect(result).toEqual({ failed: 1, leftForRecovery: 0, unobserved: 2 }); + // Only the reachable mint was asked anything. + expect(observePendingOperation).toHaveBeenCalledTimes(1); + expect(observePendingOperation.mock.calls[0]?.[0]).toBe("live-1"); + expect(failPendingOperation).toHaveBeenCalledTimes(1); + }); }); describe("settlePendingMintQuotes", () => { @@ -901,3 +930,801 @@ describe("settlePendingMintQuotes", () => { expect(logged.mock.calls[0]?.[0]).toContain("21 sat minted"); }); }); + +describe("createRunQueue", () => { + it("runs tasks strictly one after another", async () => { + const enqueue = createRunQueue(); + const order: string[] = []; + let active = 0; + let maxActive = 0; + const task = (name: string, delay: number) => async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`${name}:start`); + await new Promise((resolve) => setTimeout(resolve, delay)); + order.push(`${name}:end`); + active--; + return name; + }; + + const results = await Promise.all([ + enqueue(task("a", 20)), + enqueue(task("b", 1)), + enqueue(task("c", 1)), + ]); + + expect(results).toEqual(["a", "b", "c"]); + expect(maxActive).toBe(1); + expect(order).toEqual([ + "a:start", + "a:end", + "b:start", + "b:end", + "c:start", + "c:end", + ]); + }); + + it("keeps the chain alive after a rejected task", async () => { + const enqueue = createRunQueue(); + + const failed = enqueue(async () => { + throw new Error("boom"); + }); + const next = enqueue(async () => "ok"); + + await expect(failed).rejects.toThrow("boom"); + expect(await next).toBe("ok"); + }); +}); + +describe("failExpiredMintQuoteIfUnpaid", () => { + function fakeMintService(observe: (id: string) => Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>) { + const failPendingOperation = mock( + async ( + _op: { id: string }, + _failure: { reason: string; retryable?: boolean; observedAt: number }, + ) => ({}), + ); + return { + mintService: { + observePendingOperation: mock(observe), + failPendingOperation, + }, + failPendingOperation, + }; + } + + it("fails a quote its mint confirms unpaid", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => ({ + category: "waiting", + })); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("failed"); + expect(failPendingOperation).toHaveBeenCalledTimes(1); + expect(failPendingOperation.mock.calls[0]?.[1]?.reason).toContain( + "confirmed unpaid by mint", + ); + }); + + it.each(["ready", "completed", "terminal"] as const)( + "leaves a quote observed as %s for recovery", + async (category) => { + const { mintService, failPendingOperation } = fakeMintService( + async () => ({ category }), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result).toEqual({ outcome: "leftForRecovery", category }); + expect(failPendingOperation).not.toHaveBeenCalled(); + }, + ); + + it("leaves a quote pending when the mint cannot be reached", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => { + throw new Error("Network request failed"); + }); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); + + it("gives up waiting on a hung mint without failing the quote", async () => { + const { mintService, failPendingOperation } = fakeMintService( + () => new Promise(() => {}), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 20); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); +}); + + +describe("reopenFailedMintOperation", () => { + /** + * Mirrors coco's OperationIdLock, which is fail-fast: acquiring an id that is + * already locked throws OperationInProgressError instead of waiting. + */ + function makeLock() { + let held = false; + return { + get held() { + return held; + }, + async acquire() { + if (held) { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + } + held = true; + return () => { + held = false; + }; + }, + }; + } + + function fakeService( + current: Record | null, + hooks: { + lock?: ReturnType; + onWrite?: (lock: ReturnType) => void; + } = {}, + ) { + const lock = hooks.lock ?? makeLock(); + const transitionToPending = mock( + async (_op: Record, _error?: string) => { + hooks.onWrite?.(lock); + return {}; + }, + ); + return { + service: { + acquireOperationLock: mock(async (_id: string) => lock.acquire()), + getOperation: mock(async (_id: string) => current), + transitionToPending, + }, + transitionToPending, + lock, + }; + } + + it("re-opens a failed operation with the full persisted row", async () => { + // coco spreads whatever it is handed and the sqlite repository rewrites + // every column, so a partial object would erase the stored outputs. + const row = { + id: "op-1", + state: "failed", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + method: "bolt11", + amount: 210_000, + unit: "sat", + request: "lnbc...", + expiry: 1_800_000_000, + outputDataJson: "[{\"secret\":\"abc\"}]", + terminalFailure: { reason: "expired" }, + }; + const { service, transitionToPending } = fakeService(row); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(transitionToPending).toHaveBeenCalledTimes(1); + const passed = transitionToPending.mock.calls[0]?.[0]; + expect(passed).toMatchObject({ + id: "op-1", + quoteId: "quote-1", + amount: 210_000, + unit: "sat", + outputDataJson: "[{\"secret\":\"abc\"}]", + }); + // The stale terminal marker must not survive the re-open. + expect(passed?.terminalFailure).toBeUndefined(); + }); + + it("does nothing when the operation is no longer failed", async () => { + const { service, transitionToPending, lock } = fakeService({ + id: "op-1", + state: "finalized", + }); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + expect(transitionToPending).not.toHaveBeenCalled(); + // The lock must be released even on the no-op path. + expect(lock.held).toBe(false); + }); + + it("throws when the operation is missing", async () => { + const { service, lock } = fakeService(null); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + "not found", + ); + expect(lock.held).toBe(false); + }); + + it("fails closed when coco no longer exposes the operation lock", async () => { + const transitionToPending = mock( + async (_op: Record, _error?: string) => ({}), + ); + const service = { + getOperation: mock(async () => ({ id: "op-1", state: "failed" })), + transitionToPending, + } as unknown as Parameters[0]; + + await expect( + reopenFailedMintOperation(service, "op-1"), + ).rejects.toThrow("acquireOperationLock"); + expect(transitionToPending).not.toHaveBeenCalled(); + }); + + it("holds the operation lock across read-check-write", async () => { + const lock = makeLock(); + const order: string[] = []; + const service = { + acquireOperationLock: mock(async (_id: string) => { + order.push("lock"); + const release = await lock.acquire(); + return () => { + order.push("unlock"); + release(); + }; + }), + getOperation: mock(async (_id: string) => { + expect(lock.held).toBe(true); + order.push("read"); + return { id: "op-1", state: "failed", quoteId: "quote-1" }; + }), + transitionToPending: mock(async () => { + expect(lock.held).toBe(true); + order.push("write"); + return {}; + }), + }; + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(order).toEqual(["lock", "read", "write", "unlock"]); + expect(lock.held).toBe(false); + }); + + it("refuses to re-open while the operation lock is held elsewhere", async () => { + const lock = makeLock(); + const release = await lock.acquire(); + const { service, transitionToPending } = fakeService( + { id: "op-1", state: "failed" }, + { lock }, + ); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /in progress/, + ); + expect(transitionToPending).not.toHaveBeenCalled(); + release(); + }); +}); + +describe("runMintQuoteRecovery", () => { + function mintOp(overrides: Record = {}) { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state: "pending", + amount: 210_000, + expiry: 0, + ...overrides, + }; + } + + function fakeSource( + ops: Array>, + behavior: { + observe?: (id: string) => Promise<{ + category: "waiting" | "ready" | "completed" | "terminal"; + }>; + finalize?: (id: string) => Promise; + reopen?: (id: string) => Promise; + } = {}, + ) { + const finalize = mock( + behavior.finalize ?? + (async (_id: string) => ({ state: "finalized" })), + ); + const observePendingOperation = mock( + behavior.observe ?? (async () => ({ category: "waiting" as const })), + ); + const reopenFailedOperation = mock( + behavior.reopen ?? (async (_id: string) => true), + ); + const byId = new Map(ops.map((op) => [op.id as string, op])); + const source = { + ops: { + mint: { + listPending: async () => + ops.filter( + (op) => op.state === "pending" || op.state === "executing", + ), + get: async (id: string) => byId.get(id) ?? null, + finalize, + }, + }, + mintOperationService: { observePendingOperation }, + reopenFailedOperation, + } as unknown as MintQuoteRecoverySource; + return { source, finalize, observePendingOperation, reopenFailedOperation }; + } + + it("mints the stored outputs for a quote the mint reports PAID", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 1, waiting: 0 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(finalize.mock.calls[0]?.[0]).toBe("op-1"); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("leaves an unpaid quote pending", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "waiting" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 0, waiting: 1 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports a quote the mint can no longer issue", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "terminal" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ terminal: 1, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("retries later when the mint is unreachable", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => { + throw new Error("fetch failed"); + }, + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toHaveLength(1); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("does not count a failed finalize as a recovery", async () => { + // coco returns a terminal operation instead of throwing when the mint + // refuses, so a fulfilled finalize is not evidence that sats were claimed. + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ + state: "failed", + error: "Recovered: quote quote-1 expired while executing mint", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors[0]?.error).toContain("expired"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("does not count a finalized-with-error operation as a recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + finalize: async () => ({ + state: "finalized", + error: "Recovered issued quote quote-1 but no proofs could be restored", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors).toHaveLength(1); + }); + + it("falls back to the original failure when diagnostic lookup fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = async () => { throw new Error("lookup failed"); }; + const result = await runMintQuoteRecovery(source); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + + it("bounds a hung diagnostic lookup after finalize fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = () => new Promise(() => {}); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + + it("surfaces the persisted mint error even when the mint budget is nearly spent", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { + // Consume almost the whole per-op mint budget before throwing: exactly + // the slow-mint case where a diagnostic bound to remaining() would + // starve and silently fall back to the generic message. + await new Promise((resolve) => setTimeout(resolve, 25)); + throw new Error("remains pending"); + }, + }); + source.ops.mint.get = async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { + ...mintOp(), + state: "pending", + error: "keyset id inactive.", + }; + }; + const result = await runMintQuoteRecovery(source, { timeoutMs: 30 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([ + { operationId: "op-1", error: "keyset id inactive." }, + ]); + }); + + it("bounds finalize so one hung mint cannot block recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const started = Date.now(); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + + expect(Date.now() - started).toBeLessThan(5_000); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + }); + + it("recovers an interrupted mint without re-checking the quote", async () => { + const { source, finalize, observePendingOperation } = fakeSource([ + mintOp({ state: "executing" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it("skips failed operations unless the caller opts in", async () => { + const { source, reopenFailedOperation } = fakeSource([ + mintOp({ state: "failed", lastObservedRemoteState: "PAID" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 0, recovered: 0, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("requires explicit IDs when including failed operations", async () => { + const { source, reopenFailedOperation, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { includeFailed: true })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + await expect(runMintQuoteRecovery(source, { includeFailed: true, operationIds: [] })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it.each([0, -1, NaN, Infinity])("rejects invalid recovery timeout %s", async (timeoutMs) => { + const { source, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { timeoutMs })).rejects.toThrow( + "timeoutMs must be a positive finite number", + ); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it("re-opens a named failed operation, then mints it", async () => { + const { source, reopenFailedOperation, finalize } = fakeSource( + [mintOp({ state: "failed", lastObservedRemoteState: "PAID" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledWith("op-1"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("re-opens a named failed operation even without a PAID observation", async () => { + // The old local-fail bug left quotes with a stale or missing observation, + // which is exactly when an operator needs to retry them. + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledTimes(1); + }); + + it("skips an operation that is no longer failed when re-opened", async () => { + const { source, finalize } = fakeSource( + [mintOp({ state: "failed" })], + { reopen: async () => false }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 0, checked: 0, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports an unknown operation id instead of throwing", async () => { + const { source } = fakeSource([]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["missing"], + }); + + expect(result.checked).toBe(0); + expect(result.errors).toEqual([ + { operationId: "missing", error: "operation not found" }, + ]); + }); + + it("deduplicates repeated operation ids", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1", "op-1"], + }); + + expect(result.checked).toBe(1); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("ignores finalized operations even when targeted", async () => { + const { source, finalize } = fakeSource([mintOp({ state: "finalized" })]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + }); + + expect(result.checked).toBe(0); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("leaves a non-terminal finalize result for a later run, not terminal", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ state: "pending" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(result.errors[0]?.error).toContain("will retry"); + }); + + it("skips operations whose earlier recovery is still in flight", async () => { + const outstanding = new Map>([ + ["mint:op-1", new Promise(() => {})], + ]); + const { source, finalize, observePendingOperation } = fakeSource( + [mintOp()], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { outstanding }); + + expect(result).toMatchObject({ busy: 1, checked: 0, recovered: 0 }); + expect(observePendingOperation).not.toHaveBeenCalled(); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("keeps a timed-out finalize registered so a retry waits", async () => { + const outstanding = new Map>(); + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has("mint:op-1")).toBe(true); + // The abandoned mint request must not be retried underneath. + expect(second).toMatchObject({ busy: 1, checked: 0 }); + }); + + it("does not re-open a failed operation whose recovery is in flight", async () => { + const outstanding = new Map>([ + ["mint:op-1", new Promise(() => {})], + ]); + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + {}, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + outstanding, + }); + + expect(result).toMatchObject({ busy: 1, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("counts an in-progress operation as busy rather than retryable", async () => { + const { source } = fakeSource( + [mintOp({ state: "failed" })], + { + reopen: async () => { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + }, + }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ busy: 1, retryable: 0, reopened: 0 }); + }); + + it("keeps a timed-out quote check registered so a retry waits", async () => { + // observePendingOperation is not read-only, so a hung check must not be + // retried underneath: it could persist a stale observation later. + const outstanding = new Map>(); + const { source, finalize } = fakeSource([mintOp()], { + observe: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, checked: 1 }); + expect(outstanding.has("mint:op-1")).toBe(true); + expect(second).toMatchObject({ busy: 1, checked: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); +}); + +describe("createRecoveryGate", () => { + const HEALTHY = "https://healthy.example.com"; + const STUCK = "https://stuck.example.com"; + + function settled(promise: Promise): Promise { + return Promise.race([ + promise.then(() => true, () => true), + new Promise((resolve) => setTimeout(() => resolve(false), 25)), + ]); + } + + it("lets a mint without stuck operations proceed while another mint recovers", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + // Recovery is still running (complete() never called), yet the healthy + // mint must not be blocked by the stuck one. + await gate.waitForRecovery(HEALTHY); + }); + + it("holds a mint with stuck operations until recovery completes", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery(STUCK); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); + + it("waits for the stuck-mint enumeration before deciding", async () => { + const gate = createRecoveryGate(); + const waiting = gate.waitForRecovery(HEALTHY); + expect(await settled(waiting)).toBe(false); + gate.publishStuckMints(new Set([STUCK])); + await waiting; + }); + + it("holds callers without a target mint until recovery completes", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery(); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); + + it("poisons every caller after a recovery failure", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + gate.fail("disk exploded"); + await expect(gate.waitForRecovery(HEALTHY)).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + await expect(gate.waitForRecovery(STUCK)).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + await expect(gate.waitForRecovery()).rejects.toThrow( + "Wallet is not ready: disk exploded", + ); + }); + + it("falls back to the global gate for unparseable mint URLs", async () => { + const gate = createRecoveryGate(); + gate.publishStuckMints(new Set([STUCK])); + const waiting = gate.waitForRecovery("not a url"); + expect(await settled(waiting)).toBe(false); + gate.complete(); + await waiting; + }); +}); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 30a7854..869bcb1 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -1,4 +1,5 @@ import { withTimeout as withRequestTimeout } from "../../utils/with-timeout"; +import { recoveryKey, trackRecovery, drainRecoveryWork, waitForRecoveryWork, createRecoveryDisposer, type RecoveryWork } from "./recovery-work"; import { Manager, OperationInProgressError, @@ -37,7 +38,19 @@ import type { WalletCleanupResult, WalletRecoveryProgress, } from "./cocod-client"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; +import { + collectStuckOperations, + probeMintReachability, + runTargetedRecovery, + type SendRecoveryService, + type StuckOperation, +} from "./recovery-probe"; import { clearInterruptedReceiveReservations, deleteReceiveTokenReservation, @@ -580,6 +593,96 @@ interface MintOperationServiceCleanup { observePendingOperation( operationId: string, ): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>; + /** + * Acquire coco's per-operation lock for `operationId` and return its release + * function. coco's execute/finalize/recover paths take the same lock, so + * holding it across a read-check-write makes the transition atomic with + * respect to them. + */ + acquireOperationLock(operationId: string): Promise<() => void>; + /** Reload a mint operation row, or null when it no longer exists. */ + getOperation(operationId: string): Promise | null>; + /** + * Put a terminally failed operation back into `pending`. + * + * coco keeps this private, and it spreads whatever it is handed into the row + * it writes. The sqlite repository rewrites every column, so callers MUST + * pass a freshly reloaded full row: a partial object such as `{ id }` would + * erase `outputDataJson` and make the paid sats unrecoverable. + */ + transitionToPending( + op: Record, + error?: string, + ): Promise; +} + +/** + * Re-open a terminally failed mint operation so recovery can retry it. + * + * Two details make this safe: + * + * - The persisted row is reloaded and handed to coco in full. coco spreads + * whatever it is given and the sqlite repository rewrites every column, so a + * partial object would be rejected by the NOT NULL schema or, on a more + * permissive adapter, erase the stored outputs. + * - The read-check-write runs under coco's per-operation lock, the same lock + * coco's execute/finalize/recover paths take. Reloading alone only narrows + * the race: without the lock two concurrent recoveries could both see + * `failed` and the slower one would clobber a newer state. + * + * The lock is fail-fast rather than wait-based: coco's `OperationIdLock.acquire` + * throws `OperationInProgressError` when the id is already locked. So either + * this helper holds the lock - and coco's own execute/finalize/recover paths + * cannot interleave, because acquiring would throw for them too - or it throws + * and writes nothing. It never waits, and never writes without the lock, which + * is why a stale `failed` snapshot cannot clobber a newer state. + * + * Scope of that lock, in this coco version: `recordPendingObservation` and + * `failPendingOperation` write without taking it. The justified claim is + * therefore narrow - a re-open cannot clobber a concurrent executing/recovery + * pass - not a general guarantee against every watcher write. + * + * This is a compatibility shim over private coco internals, so it fails closed: + * if any of the expected methods are missing it throws before writing. That + * check only catches removals, not changed behaviour under the same name: it + * was written against @cashu/coco-core 1.0.1, so any coco bump must re-run the + * real-Manager and fake-mint integration tests. The long-term fix is an + * upstream public `reopenFailedOperation(id)` that takes the same lock, reloads + * the full row, preserves the outputs and emits the usual events. + */ +export async function reopenFailedMintOperation( + service: Pick< + MintOperationServiceCleanup, + "acquireOperationLock" | "getOperation" | "transitionToPending" + >, + operationId: string, +): Promise { + for (const method of [ + "acquireOperationLock", + "getOperation", + "transitionToPending", + ] as const) { + if (typeof service[method] !== "function") { + throw new Error( + `coco mintOperationService.${method} is unavailable; refusing to re-open a failed mint operation`, + ); + } + } + const release = await service.acquireOperationLock(operationId); + try { + const current = await service.getOperation(operationId); + if (!current) throw new Error(`Operation ${operationId} not found`); + if (current.state !== "failed") return false; + // Clearing the terminal-failure marker keeps the re-opened row from + // looking terminally failed to readers that inspect it alongside `state`. + await service.transitionToPending( + { ...current, terminalFailure: undefined }, + undefined, + ); + return true; + } finally { + release(); + } } export interface CreateCocoClientOptions { @@ -630,6 +733,7 @@ const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000; /** Rejects when `timeoutMs` elapses before `promise` settles. */ function withTimeout(promise: Promise, timeoutMs: number): Promise { + if (timeoutMs === Infinity) return promise; return withRequestTimeout(promise, timeoutMs, "Timed out contacting mint"); } @@ -663,6 +767,57 @@ export interface ExpiredMintSettlement { unobserved: number; } +/** Outcome of asking a mint about one expired pending quote. */ +export type ExpiredMintQuoteOutcome = + | "failed" + | "leftForRecovery" + | "unobserved"; + +/** + * Decide one expired pending quote's fate by asking the mint. + * + * Expiry alone does not prove the quote was never paid: the Lightning payment + * can land just before expiry while the daemon is down, leaving no local + * observation. A quote the mint still reports UNPAID can never be issued and + * is safe to fail locally; anything else (PAID/ISSUED, or a mint that cannot + * answer) stays pending so recovery can still claim the sats. + */ +export async function failExpiredMintQuoteIfUnpaid( + mintService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" | "failPendingOperation" + >, + operationId: string, + timeoutMs: number, +): Promise<{ + outcome: ExpiredMintQuoteOutcome; + category?: "waiting" | "ready" | "completed" | "terminal"; + error?: unknown; +}> { + try { + const observation = await withTimeout( + mintService.observePendingOperation(operationId), + timeoutMs, + ); + if (observation.category !== "waiting") { + return { outcome: "leftForRecovery", category: observation.category }; + } + // The mint confirms the expired quote is still unpaid: it can never be + // issued now, so failing it locally cannot strand funds. + await mintService.failPendingOperation( + { id: operationId }, + { + reason: "Expired mint quote confirmed unpaid by mint", + retryable: false, + observedAt: Date.now(), + }, + ); + return { outcome: "failed", category: observation.category }; + } catch (error) { + return { outcome: "unobserved", error }; + } +} + /** * Settle expired pending mint quotes before the mint recovery sweep runs. * @@ -684,6 +839,7 @@ export async function settleExpiredMintQuotes( source: ExpiredMintQuoteSource, nowMs: number, deadlineMs: number = EXPIRED_MINT_OBSERVATION_DEADLINE_MS, + options: { unreachableMints?: Set; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {}, ): Promise { const pendingMints = await source.ops.mint.listPending(); const selection = selectCleanupOperations({ @@ -704,6 +860,25 @@ export async function settleExpiredMintQuotes( const startedAt = Date.now(); for (const op of candidates) { + if (options.shouldStop?.() || options.outstanding?.has(recoveryKey("mint", op.id))) { + settlement.unobserved++; continue; + } + // A mint the startup probe already found unreachable cannot answer an + // observation either; skip it without spending the shared wall-clock + // budget, leaving the quote pending for a later startup. + if (options.unreachableMints) { + let mintUrl = op.mintUrl; + try { + mintUrl = normalizeMintUrl(op.mintUrl); + } catch { + // Malformed persisted URL: probe keys are raw for those, and the + // observation below would fail anyway, landing in `unobserved`. + } + if (options.unreachableMints.has(mintUrl)) { + settlement.unobserved++; + continue; + } + } const remainingMs = deadlineMs - (Date.now() - startedAt); if (remainingMs <= 0) { const skipped = @@ -718,45 +893,38 @@ export async function settleExpiredMintQuotes( break; } - try { - const result = await withTimeout( - source.mintOperationService.observePendingOperation(op.id), - remainingMs, + // Track the complete observation/failure chain, not just its bounded wait. + const work = failExpiredMintQuoteIfUnpaid(source.mintOperationService, op.id, Infinity); + if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("mint", op.id), work); + const check = await waitForRecoveryWork(work, remainingMs).catch(error => ({ + outcome: "unobserved" as const, error, category: undefined, + })); + if (check.outcome === "failed") { + settlement.failed++; + } else if (check.outcome === "leftForRecovery") { + // PAID/ISSUED (or terminally failed) at the mint: normal recovery + // must see this quote so paid proofs get claimed. + settlement.leftForRecovery++; + const observed = + check.category === "ready" + ? "was paid at the mint" + : check.category === "completed" + ? "was already issued at the mint" + : "failed terminally at the mint"; + startupProgress( + `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, ); - if (result.category === "waiting") { - // The mint confirms the expired quote is still unpaid: it can never - // be issued now, so failing it locally cannot strand funds. - await source.mintOperationService.failPendingOperation( - { id: op.id }, - { - reason: "Expired mint quote confirmed unpaid by mint", - retryable: false, - observedAt: Date.now(), - }, - ); - settlement.failed++; - } else { - // PAID/ISSUED (or terminally failed) at the mint: normal recovery - // must see this quote so paid proofs get claimed. - settlement.leftForRecovery++; - const observed = - result.category === "ready" - ? "was paid at the mint" - : result.category === "completed" - ? "was already issued at the mint" - : "failed terminally at the mint"; - startupProgress( - `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, - ); - } - } catch (error) { + } else { // Mint unreachable, too slow, or the quote unknown to it: leave the // operation pending so a later startup can still recover it. settlement.unobserved++; logger.warn("Could not check expired mint quote; leaving it pending", { operationId: op.id, mintUrl: op.mintUrl, - error: error instanceof Error ? error.message : String(error), + error: + check.error instanceof Error + ? check.error.message + : String(check.error), }); } } @@ -764,6 +932,364 @@ export async function settleExpiredMintQuotes( return settlement; } +/** + * Source for explicit PAID mint-quote recovery. + * + * Unlike the startup sweeps this also accepts caller-supplied operation ids so + * an operator can target a quote coco already gave up on (state `failed`). + */ +export interface MintQuoteRecoverySource { + ops: { + mint: { + listPending(): Promise; + get(operationId: string): Promise; + finalize(operationId: string): Promise; + }; + }; + mintOperationService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" + >; + /** + * Re-open a failed operation so it can be recovered; false when it is no + * longer failed. Implementations must reload the full row (see + * `reopenFailedMintOperation`). + */ + reopenFailedOperation(operationId: string): Promise; +} + +export interface MintQuoteRecoveryOptions { + shouldStop?: () => boolean; + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Per-quote budget for observing the mint and finalizing the operation. */ + timeoutMs?: number; + /** + * Re-open failed operations instead of skipping them. Only applies to + * operations named by `operationIds`: coco's pending listing never returns + * failed operations, so they can only be recovered by explicit id. + */ + includeFailed?: boolean; + /** + * In-flight recovery work keyed by family:id (mint:), shared across runs. + * withTimeout does not cancel the underlying request, so a timed-out quote + * check or finalize must keep blocking a retry until it actually settles. + */ + outstanding?: Map>; +} + +export interface MintQuoteRecoveryResult { + /** Operations recovery acted on. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** + * Quotes that ended terminally: the mint can no longer issue them, or coco + * finalised them without recovering any proofs. + */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** + * Operations left to a later run: the mint was unreachable, the per-quote + * budget ran out, or the operation ended in a non-terminal state. + */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + +/** + * Run async tasks strictly one after another. + * + * Used to serialize explicit wallet recovery: two concurrent requests must not + * both snapshot the same failed operation, and a retry must not start + * underneath work that outlived its timeout. A rejected task never breaks the + * chain for the next one. + */ +export function createRunQueue(): ((run: () => Promise) => Promise) & { drain(): Promise } { + let tail: Promise = Promise.resolve(); + const enqueue = (run: () => Promise): Promise => { + const result = tail.then(run, run); + tail = result.then( + () => undefined, + () => undefined, + ); + return result; + }; + return Object.assign(enqueue, { drain: async () => { await tail; } }); +} + +/** Per-quote budget for the mint round-trip during explicit recovery. */ +const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000; + +/** + * Bound for the local post-finalize diagnostic read. This is a database + * lookup, not a mint round-trip, so it gets its own small budget: the per-op + * mint budget is often already spent when finalize throws, and a starved + * diagnostic would silently fall back to the generic error message. + */ +const DIAGNOSTIC_LOOKUP_TIMEOUT_MS = 250; + +/** + * Recover mint quotes whose sats are PAID at the mint but were never claimed. + * + * For every target the mint is asked for the current quote state, and only it + * decides the outcome: PAID quotes have their stored outputs submitted, ISSUED + * quotes have their signatures restored (NUT-09), UNPAID quotes are left + * pending, and quotes the mint can no longer issue are reported rather than + * silently dropped. Anything the mint cannot answer is retried later. + * + * `finalize()` does not throw when the mint refuses to issue or when an + * already-issued quote's proofs cannot be restored: it returns a terminal + * operation instead. Recovery therefore inspects the returned operation's + * state and error and only counts a genuine finalized-without-error as + * recovered. + * + * Failed operations are skipped unless `includeFailed` is set, and they can + * only be targeted by explicit id because coco's pending listing never returns + * them. Re-opening an operation is a mutation, so it happens only here, never + * during startup recovery. + * + * Callers should serialize their own invocations and pass a shared + * `outstanding` map: `timeoutMs` bounds the wait but does not cancel the + * request behind it, so both a timed-out quote check and a timed-out finalize + * keep blocking a retry until they actually settle. + */ +export async function runMintQuoteRecovery( + source: MintQuoteRecoverySource, + options: MintQuoteRecoveryOptions = {}, + onProgress?: (message: string) => void, +): Promise { + if (options.includeFailed && !options.operationIds?.length) { + throw new Error("includeFailed requires explicit operationIds"); + } + const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error("timeoutMs must be a positive finite number"); + } + const outstanding = + options.outstanding ?? new Map>(); + const result: MintQuoteRecoveryResult = { + checked: 0, + recovered: 0, + waiting: 0, + terminal: 0, + reopened: 0, + retryable: 0, + busy: 0, + errors: [], + }; + const messageOf = (error: unknown) => + error instanceof Error ? error.message : String(error); + /** coco's fail-fast operation lock rejected the call: another holder exists. */ + const isInProgress = (error: unknown) => + error instanceof Error && error.name === "OperationInProgressError"; + const track = (operationId: string, work: Promise) => + trackRecovery(outstanding, recoveryKey("mint", operationId), work); + + let targets: MintQuoteRecoveryCandidate[]; + if (options.operationIds && options.operationIds.length > 0) { + targets = []; + const seen = new Set(); + for (const operationId of options.operationIds) { + if (seen.has(operationId)) continue; + seen.add(operationId); + try { + const op = await source.ops.mint.get(operationId); + if (!op) { + result.errors.push({ operationId, error: "operation not found" }); + continue; + } + targets.push(op); + } catch (error) { + result.errors.push({ operationId, error: messageOf(error) }); + } + } + } else { + targets = await source.ops.mint.listPending(); + } + + const { pending, failed } = selectMintQuotesForRecovery({ + mints: targets, + includeFailed: options.includeFailed === true, + }); + + for (const op of failed) { + if (options.shouldStop?.()) break; + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(recoveryKey("mint", op.id))) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + continue; + } + try { + if (!(await source.reopenFailedOperation(op.id))) { + onProgress?.(`${label}: no longer failed; skipped`); + continue; + } + result.reopened++; + onProgress?.(`${label}: re-opened failed operation for recovery`); + } catch (error) { + // coco's operation lock is fail-fast, so an in-progress error means a + // processor or another recovery holds the operation right now. + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery holds it; skipped`); + } else { + result.retryable++; + onProgress?.(`${label}: could not re-open: ${messageOf(error)}`); + } + result.errors.push({ operationId: op.id, error: messageOf(error) }); + continue; + } + await recoverOne(op); + } + + for (const op of pending) { + if (options.shouldStop?.()) break; + await recoverOne(op); + } + + return result; + + async function recoverOne(op: MintQuoteRecoveryCandidate): Promise { + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(recoveryKey("mint", op.id))) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + return; + } + result.checked++; + // One budget per operation, shared by the mint check and the finalize, so + // a slow mint cannot silently double the documented per-quote wait. The + // local post-finalize diagnostic read is exempt (DIAGNOSTIC_LOOKUP_TIMEOUT_MS). + const deadlineAt = Date.now() + timeoutMs; + const remaining = () => Math.max(1, deadlineAt - Date.now()); + + if (op.state === "executing") { + // A crash mid-mint can leave outputs already signed at the mint; + // finalize recovers them instead of minting a second time. + await finalizeAndClassify( + op.id, + label, + "recovered interrupted mint", + remaining, + ); + return; + } + + let observation: { + category: "waiting" | "ready" | "completed" | "terminal"; + }; + // observePendingOperation is not read-only: it emits quote-state-changed, + // persists the observation and can fail a terminal operation. Track it too, + // so a timed-out check cannot be retried and then persist a stale read. + const check = source.mintOperationService.observePendingOperation(op.id); + track(op.id, check); + try { + observation = await withTimeout(check, remaining()); + } catch (error) { + result.retryable++; + result.errors.push({ operationId: op.id, error: messageOf(error) }); + onProgress?.(`${label}: could not check with mint: ${messageOf(error)}`); + return; + } + + const decision = classifyMintQuoteObservation(observation.category); + if (decision.action === "finalize") { + await finalizeAndClassify( + op.id, + label, + decision.observedRemoteState === "PAID" + ? `paid, minting proofs (${op.amount} sat)` + : `already issued, restoring proofs (${op.amount} sat)`, + remaining, + ); + } else if (decision.action === "waiting") { + result.waiting++; + onProgress?.(`${label}: mint reports UNPAID; left pending`); + } else { + // coco records the mint's terminal verdict by failing the operation. + result.terminal++; + onProgress?.(`${label}: mint can no longer issue this quote`); + } + } + + /** + * Run finalize and classify its result. coco returns a terminal operation + * rather than throwing when the mint refuses (for example an expired quote) + * or when an already-issued quote's proofs could not be restored, so a + * fulfilled promise is not by itself evidence that sats were recovered. + */ + async function finalizeAndClassify( + operationId: string, + label: string, + successMessage: string, + remaining: () => number, + ): Promise { + const work = source.ops.mint.finalize(operationId); + track(operationId, work); + let terminal: { state?: string; error?: string } | null | undefined; + try { + terminal = (await withTimeout(work, remaining())) as + | { state?: string; error?: string } + | null + | undefined; + } catch (error) { + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery is working on it; skipped`); + } else { + result.retryable++; + // finalize can throw a generic "remains pending" error after coco has + // persisted the actionable mint rejection (for example inactive keyset). + const current = await withTimeout( + source.ops.mint.get(operationId), + Math.max(remaining(), DIAGNOSTIC_LOOKUP_TIMEOUT_MS), + ).catch(() => null); + const detail = current?.state === "pending" && current.error + ? current.error + : messageOf(error); + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: could not finish recovery: ${detail}`); + return; + } + result.errors.push({ operationId, error: messageOf(error) }); + return; + } + if (terminal?.state === "finalized" && !terminal.error) { + result.recovered++; + onProgress?.(`${label}: ${successMessage}`); + return; + } + if ( + terminal?.state === "failed" || + (terminal?.state === "finalized" && terminal.error) + ) { + result.terminal++; + const detail = + terminal.error ?? `left in state ${terminal.state ?? "unknown"}`; + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: not recovered: ${detail}`); + return; + } + // Pending/executing/unknown: coco may still be working on the operation, + // so leave it to a later run rather than calling it terminal. + result.retryable++; + result.errors.push({ + operationId, + error: `left in state ${terminal?.state ?? "unknown"}; will retry`, + }); + onProgress?.( + `${label}: still ${terminal?.state ?? "unknown"}; left for a later run`, + ); + } +} + const PENDING_MINT_SWEEP_INTERVAL_MS = 15_000; /** Per-quote wait inside a sweep, so one stalled mint cannot starve the rest. */ const PENDING_MINT_CHECK_TIMEOUT_MS = 10_000; @@ -787,6 +1313,7 @@ export interface PendingMintSweepOptions { deadlineMs?: number; checkTimeoutMs?: number; state?: PendingMintSweepState; + shouldStop?: () => boolean; } type PendingMintOutcome = "unreachable" | "other"; @@ -808,21 +1335,21 @@ export async function settlePendingMintQuotes( const ordered = [...pending.slice(resumeAt), ...pending.slice(0, resumeAt)]; const startedAt = Date.now(); for (const op of ordered) { + if (options.shouldStop?.()) break; const remainingMs = deadlineMs - (Date.now() - startedAt); - if (state.outstanding.has(op.id) || remainingMs <= 0) { + if (state.outstanding.has(recoveryKey("mint", op.id)) || remainingMs <= 0) { unreachable++; continue; } state.after = op.id; - // Report on the refresh itself so a late result is still logged. + // Track refresh AND its late-result reporting mutations as one lifetime. const settled = source.ops.mint .refresh(op.id) .then( (result) => reportPendingMintRefresh(source, op, result, nowMs), (error) => reportPendingMintRefreshError(source, op, error), - ) - .finally(() => state.outstanding.delete(op.id)); - state.outstanding.set(op.id, settled); + ); + trackRecovery(state.outstanding, recoveryKey("mint", op.id), settled); try { const outcome = await withTimeout(settled, Math.min(checkTimeoutMs, remainingMs)); if (outcome === "unreachable") unreachable++; @@ -898,16 +1425,16 @@ async function reportPendingMintRefreshError( * still running after that fails against the closed database and is picked * up by startup recovery. */ -function startPendingMintSweep(source: PendingMintQuoteSource): () => Promise { +function startPendingMintSweep(source: PendingMintQuoteSource, outstanding: RecoveryWork): () => Promise { let stopped = false; let timer: ReturnType | undefined; let inFlight: Promise = Promise.resolve(); let unreachableBefore = 0; - const state: PendingMintSweepState = { outstanding: new Map() }; + const state: PendingMintSweepState = { outstanding }; const tick = async () => { if (stopped) return; - inFlight = settlePendingMintQuotes(source, Date.now(), { state }).then( + inFlight = settlePendingMintQuotes(source, Date.now(), { state, shouldStop: () => stopped }).then( ({ unreachable }) => { // Report a mint becoming unreachable, or reachable again, once. if (unreachable > 0 && unreachableBefore === 0) { @@ -1023,21 +1550,174 @@ interface RecoveryPhaseProgress { failedMintQuotes: number; } +/** + * Coco keeps per-operation recovery private on its services; routstrd already + * reaches into the Manager the same way for `mintOperationService`. Send is + * the only family whose public `refresh()` cannot recover executing ops. + */ +function sendRecoveryServiceOf(coco: Manager): SendRecoveryService { + return (coco as unknown as { sendOperationService: SendRecoveryService }) + .sendOperationService; +} + +/** Local-only crash cleanup, run before the degraded gate opens. */ +export async function cleanupLocalRecoveryState( + coco: Manager, + repo: SqliteRepositories, +): Promise { + // Coco 1.0.1 implements these as local repository/proof operations only. + // Keep this version-sensitive bridge together with the send recovery bridge. + const services = coco as unknown as Record; + cleanupOrphanedReservations?(): Promise; + } | undefined>; + const families = [ + ["send", repo.sendOperationRepository], + ["melt", repo.meltOperationRepository], + ["receive", repo.receiveOperationRepository], + ["mint", repo.mintOperationRepository], + ] as const; + // Fail closed the way reopenFailedMintOperation does: a coco bump that + // removes or renames these privates must stop recovery with a clear error + // before anything is written, not crash halfway through the loop with the + // cleanup half-applied. + for (const [kind] of families) { + if (typeof services[`${kind}OperationService`]?.recoverInitOperation !== "function") { + throw new Error( + `coco ${kind}OperationService.recoverInitOperation is unavailable; refusing local recovery cleanup`, + ); + } + } + if (typeof services.sendOperationService?.cleanupOrphanedReservations !== "function") { + throw new Error( + "coco sendOperationService.cleanupOrphanedReservations is unavailable; refusing local recovery cleanup", + ); + } + for (const [kind, repository] of families) { + for (const op of await repository.getByState("init")) { + await services[`${kind}OperationService`]!.recoverInitOperation!(op); + } + } + await services.sendOperationService!.cleanupOrphanedReservations!(); +} + +/** + * Gate for value-moving wallet operations while startup recovery runs. + * + * On degraded startup only, publishStuckMints opens the gate for callers + * whose target mint has no stuck operations after probing and local cleanup. + * A dead mint must not stall spends from a healthy one. On the happy path + * all callers wait until the global sweeps finish. Callers + * without a target mint, or whose mint has stuck operations, wait for the + * full sweep. fail() poisons every caller; reads are never gated. + */ +export interface RecoveryGate { + waitForRecovery(mintUrl?: string): Promise; + publishStuckMints(mints: Set): void; + complete(): void; + fail(error: string): void; +} + +export function createRecoveryGate(): RecoveryGate { + let stuckMints: Set | undefined; + let done = false; + let error: string | undefined; + let mintsResolve: (() => void) | undefined; + const mintsPromise = new Promise((resolve) => { + mintsResolve = resolve; + }); + let doneResolve: (() => void) | undefined; + const donePromise = new Promise((resolve) => { + doneResolve = resolve; + }); + + return { + async waitForRecovery(mintUrl?: string): Promise { + if (mintUrl) { + let normalized: string | undefined; + try { + normalized = normalizeMintUrl(mintUrl); + } catch { + // Unparseable URL falls back to the global gate. + normalized = undefined; + } + if (normalized) { + await mintsPromise; + if (!stuckMints?.has(normalized)) { + if (error) throw new Error(`Wallet is not ready: ${error}`); + return; + } + } + } + if (!done) await donePromise; + if (error) throw new Error(`Wallet is not ready: ${error}`); + }, + publishStuckMints(mints: Set): void { + if (stuckMints) return; + stuckMints = mints; + mintsResolve?.(); + }, + complete(): void { + done = true; + mintsResolve?.(); + doneResolve?.(); + }, + fail(message: string): void { + done = true; + error = message; + mintsResolve?.(); + doneResolve?.(); + }, + }; +} + /** * Run the wallet recovery sweeps in order, reporting phase changes. * - * Expired mint quotes are settled first: quotes their mint confirms as unpaid + * Probe first, then settle expired mint quotes: quotes confirmed as unpaid * are failed locally so `recoverPendingMintOperations()` skips them, while * paid/issued and unreachable-mint quotes stay pending for the sweep. */ -async function runWalletRecovery( +export async function runWalletRecovery( coco: Manager, onProgress: (progress: RecoveryPhaseProgress) => void, receiveOperationIds?: string[], + onStuckMintsKnown?: (mints: Set) => void, + options: { cleanupLocalState?: () => Promise; fetchImpl?: typeof fetch; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {}, ): Promise { surfacingRecoveryProgress = true; let failedMintQuotes = 0; try { + // Probe every mint that has stuck operations once, FIRST, so a dead mint + // costs a single short probe instead of taxing settlement's observation + // budget plus a network timeout per operation per sweep. Healthy-mint + // operations are recovered per op; dead-mint operations stay parked + // exactly as coco's own "will retry later" path would leave them. + onProgress({ phase: "Probing mints", failedMintQuotes }); + const stuckOperations = await collectStuckOperations(coco.ops); + const unreachableMints = await probeMintReachability( + [...new Set(stuckOperations.map((op) => op.mintUrl))], + { fetchImpl: options.fetchImpl }, + ); + for (const mintUrl of unreachableMints) { + const count = stuckOperations.filter((op) => op.mintUrl === mintUrl).length; + startupProgress( + `Skipping recovery for unreachable mint: ${mintUrl} (${count} op${count === 1 ? "" : "s"})`, + ); + } + const degraded = unreachableMints.size > 0; + if (degraded) { + // Local-only housekeeping must finish before any new operation is allowed. + await options.cleanupLocalState?.(); + // Global sweeps enumerate fresh state and are unsafe beside live sends. + // Only the snapshot-based degraded path may open the per-mint gate. + onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl))); + } + + // Settlement runs after the gate opens and only spends its observation + // budget on mints the probe found reachable; dead-mint quotes stay + // pending untouched. It only reads and locally fails long-expired quotes, + // so it cannot conflict with live operations the gate just admitted. onProgress({ phase: "Settling expired mint quotes", failedMintQuotes }); const settlement = await settleExpiredMintQuotes( { @@ -1049,6 +1729,8 @@ async function runWalletRecovery( ).mintOperationService, }, Date.now(), + undefined, + { unreachableMints, outstanding: options.outstanding, shouldStop: options.shouldStop }, ); failedMintQuotes = settlement.failed; if (settlement.leftForRecovery > 0 || settlement.unobserved > 0) { @@ -1059,22 +1741,48 @@ async function runWalletRecovery( ); } onProgress({ phase: "Settled expired mint quotes", failedMintQuotes }); + const targeted = (kinds: Array) => + runTargetedRecovery(coco.ops, sendRecoveryServiceOf(coco), { + kinds, + outstanding: options.outstanding, + shouldStop: options.shouldStop, + stuckOperations, + unreachableMints, + }); + // Happy path (every mint reachable) keeps coco's global sweeps: they also + // clean up init operations and orphaned proof reservations. Degraded + // startup runs that local housekeeping before opening its gate, and only + // drives the previously collected snapshot when a dead mint would + // otherwise tax every stuck op with a network timeout. onProgress({ phase: "Send recovery", failedMintQuotes }); - await coco.ops.send.recovery.run(); + if (!degraded) await coco.ops.send.recovery.run(); + else await targeted(["send"]); onProgress({ phase: "Melt recovery", failedMintQuotes }); - await coco.ops.melt.recovery.run(); + if (!degraded) await coco.ops.melt.recovery.run(); + else await targeted(["melt"]); onProgress({ phase: "Receive recovery", failedMintQuotes }); if (receiveOperationIds) { // The pre-check already classified every executing receive by unique // input set. Recover only the conclusive retained operations; unresolved // groups stay untouched instead of falling back to Coco 1's expensive - // per-row sweep on this startup. + // per-row sweep on this startup. Operations at mints the probe found + // unreachable are skipped rather than costing their 15s timeout each. + const mintByOperation = new Map( + stuckOperations + .filter((op) => op.kind === "receive") + .map((op) => [op.id, op.mintUrl]), + ); for (const operationId of receiveOperationIds) { + if (options.shouldStop?.()) break; + const mintUrl = mintByOperation.get(operationId); + if (mintUrl && unreachableMints.has(mintUrl)) continue; try { - await withTimeout(coco.ops.receive.refresh(operationId), 15_000); + const work = coco.ops.receive.refresh(operationId); + if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("receive", operationId), work); + await withTimeout(work, 15_000); } catch (error) { logger.warn("Targeted receive recovery did not complete", { operationId, @@ -1082,12 +1790,19 @@ async function runWalletRecovery( }); } } - } else { + } else if (!degraded) { await coco.ops.receive.recovery.run(); + } else { + await targeted(["receive"]); } onProgress({ phase: "Mint recovery", failedMintQuotes }); - await coco.recoverPendingMintOperations(); + // A settlement wait may have timed out while an unlocked observation + // still runs. Never let a fresh global mint sweep observe it again. + if (!degraded && ![...(options.outstanding?.keys() ?? [])].some(key => key.startsWith("mint:"))) { + await coco.recoverPendingMintOperations(); + } + else await targeted(["mint"]); onProgress({ phase: "done", failedMintQuotes }); } finally { @@ -1150,6 +1865,10 @@ export async function createCocoClient( const recoveryPromise = new Promise((resolve) => { recoveryResolve = resolve; }); + const recoveryGate = createRecoveryGate(); + let disposed = false; + const enqueueRecovery = createRunQueue(); + const recoveryOutstanding: RecoveryWork = new Map(); try { startupProgress("Opening Cashu wallet database..."); @@ -1347,11 +2066,14 @@ export async function createCocoClient( } }, receiveRecoveryOperationIds, + (mints) => recoveryGate.publishStuckMints(mints), + { cleanupLocalState: () => cleanupLocalRecoveryState(coco!, repo), outstanding: recoveryOutstanding, shouldStop: () => disposed }, ) .then(async () => { await syncReceiveReservations(); recoveryDone = true; recoveryPhase = "done"; + recoveryGate.complete(); recoveryResolve?.(); startupProgress("Wallet recovery complete."); stopPendingMintSweep = startPendingMintSweep({ @@ -1360,12 +2082,13 @@ export async function createCocoClient( mintOperationService: ( coco as unknown as { mintOperationService: MintOperationServiceCleanup } ).mintOperationService, - }); + }, recoveryOutstanding); }) .catch((error) => { recoveryDone = true; recoveryPhase = "error"; recoveryError = error instanceof Error ? error.message : String(error); + recoveryGate.fail(recoveryError); recoveryResolve?.(); startupProgress(`Wallet recovery failed: ${recoveryError}`); }); @@ -1388,19 +2111,33 @@ export async function createCocoClient( return api; }; - let disposed = false; + const assertOpen = () => { if (disposed) throw new Error("Wallet is shutting down"); }; - /** - * Block a value-moving operation until background recovery has settled. - * Reads stay ungated so the daemon can report balances/status immediately. - */ - const waitForRecovery = async (): Promise => { - if (!recoveryDone) await recoveryPromise; - if (recoveryError) { - throw new Error(`Wallet is not ready: ${recoveryError}`); - } + // Block a value-moving operation until background recovery has settled for + // its target mint (see createRecoveryGate). Reads stay ungated so the + // daemon can report balances/status immediately. + const waitForRecovery = async (mintUrl?: string): Promise => { + assertOpen(); + await recoveryGate.waitForRecovery(mintUrl); + assertOpen(); }; + const disposeRecovery = createRecoveryDisposer( + () => { disposed = true; }, + async () => { + await recoveryPromise; + await stopPendingMintSweep?.(); + await enqueueRecovery.drain(); + await drainRecoveryWork(recoveryOutstanding); + }, + async () => { + await coco.dispose(); + database.close(); + releaseLegacyPidClaim(); + releaseWalletPidClaim(); + }, + ); + return { async ping(): Promise { try { @@ -1579,13 +2316,13 @@ export async function createCocoClient( }, async receiveBolt11(amount: number, mintUrl?: string) { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for Lightning invoice"); } + await waitForRecovery(targetMint); const op = await coco.ops.mint.prepare({ mintUrl: targetMint, amount, @@ -1611,13 +2348,13 @@ export async function createCocoClient( }, async sendCashu(amount: number, mintUrl?: string): Promise { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for sending"); } + await waitForRecovery(targetMint); const prepared = await coco.ops.send.prepare({ mintUrl: targetMint, amount, @@ -1627,13 +2364,13 @@ export async function createCocoClient( }, async sendBolt11(invoice: string, mintUrl?: string): Promise { - await waitForRecovery(); const targetMint = mintUrl ? normalizeMintUrl(mintUrl) : walletConfig.defaultMintUrl; if (!targetMint) { throw new Error("No trusted mint available for Lightning payment"); } + await waitForRecovery(targetMint); const prepared = await coco.ops.melt.prepare({ mintUrl: targetMint, method: "bolt11", @@ -1677,28 +2414,20 @@ export async function createCocoClient( }, async dispose(): Promise { - if (disposed) return; - disposed = true; - try { - // Let any in-flight recovery settle before closing the database from - // underneath it. The recovery promise resolves on success or failure. - await recoveryPromise; - await stopPendingMintSweep?.(); - await coco.dispose(); - } finally { - try { - database.close(); - } finally { - releaseLegacyPidClaim(); - releaseWalletPidClaim(); - } - } + await disposeRecovery().catch(error => { + logger.warn("Wallet shutdown incomplete; database and ownership retained until recovery settles"); + throw error; + }); }, async getHistory(offset?: number, limit?: number): Promise { return coco.history.getPaginatedHistory(offset, limit); }, + async getHistoryEntryById(id: string): Promise { + return coco.history.getHistoryEntryById(id); + }, + async getNpcAddress(): Promise { const info = await npcApi().getInfo(); const name = @@ -1740,6 +2469,7 @@ export async function createCocoClient( await waitForRecovery(); const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000; const dryRun = options.dryRun === true; + const force = options.force === true; const nowMs = Date.now(); const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([ @@ -1767,6 +2497,8 @@ export async function createCocoClient( }); const errors: WalletCleanupResult["errors"] = []; + let failedMintQuotes = 0; + let leftForRecovery = 0; if (!dryRun) { const mintService = ( @@ -1776,20 +2508,49 @@ export async function createCocoClient( ).mintOperationService; for (const op of selection.mintsToFail) { - try { - await mintService.failPendingOperation( - { id: op.id }, - { - reason: "Expired unpaid mint quote cleaned up by routstrd", - retryable: false, - observedAt: nowMs, - }, - ); - } catch (error) { - errors.push({ - operationId: op.id, - error: error instanceof Error ? error.message : String(error), - }); + if (force) { + // Legacy behaviour: fail the quote locally without asking the mint. + try { + await mintService.failPendingOperation( + { id: op.id }, + { + reason: "Expired mint quote cleaned up by routstrd (forced)", + retryable: false, + observedAt: nowMs, + }, + ); + failedMintQuotes++; + } catch (error) { + errors.push({ + operationId: op.id, + error: error instanceof Error ? error.message : String(error), + }); + } + continue; + } + // Expiry alone does not prove the quote was never paid: the + // Lightning payment can land before expiry while the daemon is down. + // Confirm UNPAID with the mint before failing, exactly as startup + // recovery does; paid quotes are left for recovery to finalize. + const check = await failExpiredMintQuoteIfUnpaid( + mintService, + op.id, + EXPIRED_MINT_OBSERVATION_DEADLINE_MS, + ); + if (check.outcome === "failed") { + failedMintQuotes++; + } else { + leftForRecovery++; + if (check.outcome === "unobserved") { + errors.push({ + operationId: op.id, + error: `could not confirm quote state with mint: ${ + check.error instanceof Error + ? check.error.message + : String(check.error) + }`, + }); + } } } @@ -1816,8 +2577,15 @@ export async function createCocoClient( } } + const mintSummary = summarizeMintCleanup({ + dryRun, + candidates: selection.mintsToFail.length, + failed: failedMintQuotes, + leftForRecovery, + }); const actedOn = - selection.mintsToFail.length + + (dryRun ? mintSummary.mintQuoteCandidates : mintSummary.failedMintQuotes) + + leftForRecovery + selection.sendsToReclaim.length + selection.meltsToCancel.length; const skipped = @@ -1826,12 +2594,63 @@ export async function createCocoClient( return { dryRun, - failedMintQuotes: selection.mintsToFail.length, + ...mintSummary, reclaimedSends: selection.sendsToReclaim.length, cancelledMelts: selection.meltsToCancel.length, skipped, errors, }; }, + + async recoverMintQuotes(options, onProgress) { + await waitForRecovery(); + const service = ( + coco as unknown as { + mintOperationService: MintOperationServiceCleanup; + } + ).mintOperationService; + // Serialize explicit recovery: two concurrent requests must not both + // snapshot the same failed operation, and a retry must not start + // underneath a finalize that outlived its timeout. + return enqueueRecovery(() => { + assertOpen(); + return runMintQuoteRecovery( + { + ops: coco.ops as unknown as MintQuoteRecoverySource["ops"], + mintOperationService: service, + reopenFailedOperation: (operationId) => + reopenFailedMintOperation(service, operationId), + }, + { ...options, outstanding: recoveryOutstanding, shouldStop: () => disposed }, + onProgress, + ); + }); + }, + + async recoverStuckOperations() { + await waitForRecovery(); + // Serialized against explicit mint-quote recovery (and itself) through + // the same queue and lifetime tracker, so timed-out passes cannot retry the same + // operation. Receive stays startup-only: recovering competing receives + // safely requires the startup dedup classification (receive-dedup.ts). + // Operations a live execute holds come back as busy via coco's + // fail-fast operation lock, never driven underneath it. + const result = await enqueueRecovery(() => { + assertOpen(); + return runTargetedRecovery(coco!.ops, sendRecoveryServiceOf(coco!), { + kinds: ["send", "melt", "mint"], + outstanding: recoveryOutstanding, + shouldStop: () => disposed, + }); + }); + return { + timedOut: result.timedOut, + attempted: result.attempted, + busy: result.busy, + skipped: result.skipped, + failed: result.failed, + skippedMints: Object.fromEntries(result.skippedMints), + }; + }, }; } diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index d338506..93d4d49 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -83,13 +83,23 @@ export interface WalletCleanupOptions { minAgeMs?: number; /** Report what would be cleaned without applying changes. */ dryRun?: boolean; + /** + * Fail expired mint quotes without confirming UNPAID with the mint. Only for + * operators who accept the risk of stranding a quote that was paid before + * its invoice expired; recovery is the safe default. + */ + force?: boolean; } /** Summary of a wallet cleanup run. */ export interface WalletCleanupResult { dryRun: boolean; - /** Number of expired pending mint quotes marked as failed. */ + /** Expired quotes selected for checking; dry runs do not contact the mint. */ + mintQuoteCandidates: number; + /** Number actually marked failed (always zero in a dry run). */ failedMintQuotes: number; + /** Expired quotes kept pending because they are paid/issued or unverified. */ + leftForRecovery: number; /** Number of stale pending send operations reclaimed. */ reclaimedSends: number; /** Number of stale prepared melt operations cancelled. */ @@ -120,6 +130,51 @@ export interface MintQuoteStatus { error?: string; } +/** Options for explicit PAID mint-quote recovery. */ +export interface WalletMintQuoteRecoveryOptions { + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Re-open failed operations instead of skipping them. */ + includeFailed?: boolean; + /** Per-quote mint timeout in milliseconds. */ + timeoutMs?: number; +} + +/** Summary of a PAID mint-quote recovery run. */ +export interface WalletMintQuoteRecoveryResult { + /** Operations whose quote state was checked with the mint. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** Quotes the mint can no longer issue. */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** Operations left to a later run (mint unreachable, budget spent, non-terminal). */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + +/** Summary of a stuck-operation (send/melt/mint) recovery run. */ +export interface WalletStuckOperationRecoveryResult { + /** Timed-out waits; the underlying operation remains tracked. */ + timedOut: number; + /** Operations for which recovery was attempted (not necessarily completed). */ + attempted: number; + /** Locked operations or unfinished work from another pass; retry later. */ + busy: number; + /** Operations skipped for unreachable mints, shutdown, or pass budget exhaustion. */ + skipped: number; + /** Operations at reachable mints whose recovery still failed. */ + failed: number; + /** Unreachable mint URL -> number of operations skipped there. */ + skippedMints: Record; +} + export interface CocodClient { ping(): Promise; getStatus(): Promise; @@ -142,6 +197,8 @@ export interface CocodClient { /** Release resources held by in-process wallet implementations. */ dispose?(): Promise; getHistory(offset?: number, limit?: number): Promise; + /** Look up a single transaction by its history entry ID. */ + getHistoryEntryById(id: string): Promise; /** NPC (npubx.cash) Lightning address for this wallet. */ getNpcAddress(): Promise; /** Claim an NPC username; pass confirm=true to pay the claim fee from the wallet. */ @@ -152,6 +209,20 @@ export interface CocodClient { cleanupStuckOperations?( options?: WalletCleanupOptions, ): Promise; + /** + * Re-issue PAID mint quotes whose sats were never claimed, optionally + * targeting specific operations (including ones coco already failed). + */ + recoverMintQuotes?( + options?: WalletMintQuoteRecoveryOptions, + onProgress?: (message: string) => void, + ): Promise; + /** + * Recover stuck send/melt/mint operations whose mints answer a + * reachability probe. Operations a live execute holds are reported busy, + * never driven. Receive stays startup-only (receive dedup classification). + */ + recoverStuckOperations?(): Promise; /** Report background wallet recovery progress, when the wallet supports it. */ getRecoveryProgress?(): Promise; } @@ -465,6 +536,9 @@ export function createCocodClient( async getHistory(_offset?: number, _limit?: number): Promise { return []; }, + async getHistoryEntryById(_id: string): Promise { + return null; + }, async getNpcAddress(): Promise { const address = await callDaemon("/npc/address"); if (typeof address !== "string" || !address.trim()) { diff --git a/src/daemon/wallet/mint-operation-reopen.test.ts b/src/daemon/wallet/mint-operation-reopen.test.ts new file mode 100644 index 0000000..9197fec --- /dev/null +++ b/src/daemon/wallet/mint-operation-reopen.test.ts @@ -0,0 +1,163 @@ +/** + * Re-opening a failed mint operation is the one genuinely destructive step of + * PAID-quote recovery, because coco's private `transitionToPending` spreads + * whatever it is handed and `SqliteMintOperationRepository.update` rewrites + * every column. A partial object such as `{ id }` is therefore rejected by the + * NOT NULL schema, and on a more permissive adapter would overwrite `quoteId`, + * `amount`, `request`, `lastObservedRemoteState` and `outputDataJson` with NULL, + * destroying the material needed to claim the paid sats. + * + * These tests drive the production `reopenFailedMintOperation` helper against a + * real coco sqlite repository through adapter-backed getOperation and + * transitionToPending implementations, so a regression at the helper/service + * boundary is caught rather than a mock standing in for it. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow(state = "failed") { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state, + createdAt: 1_000, + updatedAt: 2_000, + error: state === "failed" ? "expired" : undefined, + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: + state === "failed" ? { reason: "expired", observedAt: 3_000 } : undefined, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation against real coco sqlite", () => { + let dir: string | undefined; + let database: Database | undefined; + + afterEach(() => { + database?.close(); + database = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function repos() { + dir = mkdtempSync(join(tmpdir(), "routstrd-reopen-")); + database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + return repositories; + } + + function readRow(repositories: SqliteRepositories, id: string) { + return repositories.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + /** + * Adapter-backed stand-in for the private coco service methods the helper + * uses: getOperation reads and transitionToPending mirrors coco's + * spread-and-update implementation. + */ + function serviceOver(repositories: SqliteRepositories) { + return { + acquireOperationLock: async (_id: string) => () => {}, + getOperation: (id: string) => readRow(repositories, id), + transitionToPending: async ( + op: Record, + error?: string, + ) => { + await repositories.mintOperationRepository.update({ + ...op, + state: "pending", + error, + } as never); + }, + }; + } + + it("re-opens a failed row without losing quote metadata or stored outputs", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(true); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.unit).toBe("sat"); + expect(row?.request).toBe("lnbc1example"); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + }); + + it("is a no-op when the operation is no longer failed", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow("finalized") as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(false); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("finalized"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + }); + + it("rejects a partial row, which is why the helper reloads in full", async () => { + // Locks in the reason for reloading. If a future coco version accepts + // partial updates this fails, and the helper can be simplified rather than + // silently losing paid sats. + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + await expect( + repositories.mintOperationRepository.update({ + id: "op-1", + state: "pending", + updatedAt: Date.now(), + } as never), + ).rejects.toThrow(); + + const unchanged = await readRow(repositories, "op-1"); + expect(unchanged?.state).toBe("failed"); + expect(unchanged?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts new file mode 100644 index 0000000..69b0c06 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -0,0 +1,386 @@ +/** + * End-to-end PAID mint-quote recovery against a real coco Manager, real sqlite + * and a real in-process mint that produces genuine blind signatures. + * + * Nothing here mocks the wallet: a quote is created through coco, the mint is + * told what to report, and the production `runMintQuoteRecovery` drives the + * outcome. These are the release-gating scenarios for the feature, and they are + * the only tests that exercise issuance and NUT-09 restore over HTTP. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint"; +import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client"; + +type AnyRecord = Record; + +interface Booted { + manager: Manager; + repositories: SqliteRepositories; + mint: FakeMint; + /** Build the recovery source the production function expects. */ + source: () => AnyRecord; + spendable: () => Promise; + close: () => Promise; +} + +async function boot(options: { quoteExpiry?: number | null } = {}) { + const mint = new FakeMint(); + mint.quoteExpiry = options.quoteExpiry ?? null; + mint.start(); + const dir = mkdtempSync(join(tmpdir(), "routstrd-fakemint-")); + const database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + const manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await manager.mint.addMint(mint.url, { trusted: true }); + + const service = (manager as unknown as { mintOperationService: AnyRecord }) + .mintOperationService; + + const booted: Booted = { + manager, + repositories, + mint, + spendable: async () => { + const balances = (await manager.wallet.balances.byMint()) as Record< + string, + { spendable: number } + >; + return balances[mint.url]?.spendable ?? 0; + }, + source: () => ({ + ops: { + mint: { + listPending: () => manager.ops.mint.listPending(), + get: (id: string) => manager.ops.mint.get(id), + finalize: (id: string) => manager.ops.mint.finalize(id), + }, + }, + mintOperationService: service, + reopenFailedOperation: (id: string) => + reopenFailedMintOperation(service as never, id), + }), + close: async () => { + await manager.dispose().catch(() => undefined); + database.close(); + mint.stop(); + rmSync(dir, { recursive: true, force: true }); + }, + }; + + return booted; +} + +async function prepareQuote(booted: Booted, amount: number) { + const op = (await booted.manager.ops.mint.prepare({ + mintUrl: booted.mint.url, + amount, + method: "bolt11", + } as never)) as unknown as AnyRecord; + return op; +} + +function outputsOf(op: AnyRecord) { + // coco stores mint outputs as { keep, send }, like the on-disk output JSON. + const outputData = (op.outputData as AnyRecord).keep as Array<{ + blindedMessage: { amount: unknown; id: string; B_: string }; + }>; + return outputData.map((output) => ({ + amount: Number(String(output.blindedMessage.amount)), + id: output.blindedMessage.id, + B_: output.blindedMessage.B_, + })); +} + +let booted: Booted | undefined; +afterEach(async () => { + await booted?.close(); + booted = undefined; +}); + +describe("PAID mint quote recovery with a real Manager and mint", () => { + it("issues an expired-but-PAID quote with the operation's own outputs, once", async () => { + // The motivating case: the invoice expired, but the mint says PAID and has + // issued nothing. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 210_000); + const expectedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ checked: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(210_000); + // Issuance used exactly the blinded outputs stored on the operation. + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...expectedOutputs].sort(), + ); + + // A second run must not mint again or double-credit. + const again = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(again).toMatchObject({ checked: 0, recovered: 0 }); + expect(booted.mint.requests).toHaveLength(1); + expect(await booted.spendable()).toBe(210_000); + }); + + it("existing coco recovery already issues expired paid pending quotes", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + booted.mint.markPaid(op.quoteId as string); + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(100); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("ISSUED"); + }); + + it("keeps rejected stored outputs and reports the actionable mint error", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + const outputs = outputsOf(op); + booted.mint.markPaid(op.quoteId as string); + // Model the mint refusing the stored outputs, not invoice expiry. This is + // not evidence that the production quotes used an inactive keyset. + booted.mint.mintError = { code: 12001, detail: "keyset id inactive." }; + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(0); + const result = await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + }); + expect(result).toMatchObject({ recovered: 0, retryable: 1 }); + expect(result.errors.some((entry) => entry.error.includes("keyset id inactive"))).toBe(true); + expect(await booted.spendable()).toBe(0); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("PAID"); + expect(outputsOf(await booted.manager.ops.mint.get(op.id as string) as unknown as AnyRecord)).toEqual(outputs); + for (const request of booted.mint.requests) expect(request.outputs).toEqual(outputs); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 210_000); + // Another wallet issued it: the signatures exist at the mint for the very + // outputs this operation stored. + booted.mint.signFor(op.quoteId as string, outputsOf(op)); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 1, terminal: 0, retryable: 0 }); + expect(await booted.spendable()).toBe(210_000); + }); + + it("reports terminal without credit when the mint refuses issuance", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + booted.mint.mintError = { code: QUOTE_EXPIRED, detail: "quote expired" }; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect((result.errors as unknown[]).length).toBeGreaterThan(0); + expect(await booted.spendable()).toBe(0); + }); + + it("reports terminal without credit when an issued quote cannot be restored", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint, but the signatures for our outputs are gone. + booted.mint.markIssued(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("does not credit a quote when the mint returns null NUT-09 signatures", async () => { + // KNOWN INTEROP GAP, not a supported path. NUT-09 permits `null` in the + // positional `signatures` array for outputs the mint never signed, but + // cashu-ts 3.7.1 - which coco depends on - dereferences every entry while + // normalising amounts, so the wallet throws instead of skipping the null. + // Recovery therefore surfaces an error and credits nothing, leaving the + // operation for a later run. + // + // This test pins the current behaviour so the gap cannot quietly disappear. + // Revisit (and change this expectation) once coco's cashu-ts parses + // positional nulls, and file/track it upstream in the meantime. + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint with nothing signed for this operation's outputs. + booted.mint.markIssued(op.quoteId as string); + booted.mint.restoreIncludesNulls = true; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("recovers a failed operation whose stale history says UNPAID", async () => { + // The composed path the feature exists for: a real prepared operation, + // failed locally, with a stale UNPAID observation from the old local-fail + // behaviour, recovered by explicit id and issued with its own outputs. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + const storedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const row = (await booted.repositories.mintOperationRepository.getById( + op.id as string, + )) as unknown as AnyRecord; + expect(row.outputData).toBeDefined(); + await booted.repositories.mintOperationRepository.update({ + ...row, + state: "failed", + lastObservedRemoteState: "UNPAID", + error: "Expired unpaid mint quote cleaned up by routstrd", + terminalFailure: { reason: "expired", observedAt: Date.now() }, + } as never); + + // A purely local decision would skip this row; the mint has the last word. + const result = (await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + includeFailed: true, + })) as unknown as Record; + + expect(result).toMatchObject({ reopened: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...storedOutputs].sort(), + ); + }); + + it("counts a locked operation as busy instead of minting underneath it", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + const service = ( + booted.manager as unknown as { + mintOperationService: { acquireOperationLock(id: string): Promise<() => void> }; + } + ).mintOperationService; + + // Hold the operation, as a processor or another recovery would. + const release = await service.acquireOperationLock(op.id as string); + const blocked = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(blocked).toMatchObject({ recovered: 0 }); + expect((blocked.busy ?? 0) + (blocked.retryable ?? 0)).toBeGreaterThan(0); + expect(booted.mint.requests).toHaveLength(0); + expect(await booted.spendable()).toBe(0); + + release(); + const after = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(after).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("does not mint a second time while issuance is in flight", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Hold the mint's response so the first recovery is visibly in flight. + let releaseGate!: () => void; + booted.mint.gate = new Promise((resolve) => { + releaseGate = resolve; + }); + const first = runMintQuoteRecovery( + booted.source() as never, + ) as unknown as Promise>; + + for (let i = 0; i < 400 && booted.mint.requests.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(booted.mint.requests).toHaveLength(1); + + // A concurrent run must not issue again while that request is outstanding. + await runMintQuoteRecovery(booted.source() as never); + expect(booted.mint.requests).toHaveLength(1); + + releaseGate(); + expect(await first).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("coexists with coco's own mint operation watcher and processor", async () => { + booted = await boot({ quoteExpiry: -60 }); + // The real background machinery coco uses to settle pending mint quotes. + await booted.manager.enableMintOperationWatcher(); + await booted.manager.enableMintOperationProcessor(); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Either our recovery or coco's processor may win; both are safe. + await runMintQuoteRecovery(booted.source() as never); + + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + + // Give the processor time to act and confirm nothing is credited twice. + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + }); + + it("tracks a hung quote check so a later run waits instead of re-reading", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + let releaseObserve!: () => void; + booted.mint.observeGate = new Promise((resolve) => { + releaseObserve = resolve; + }); + const outstanding = new Map>(); + + const first = (await runMintQuoteRecovery(booted.source() as never, { + timeoutMs: 30, + outstanding, + })) as unknown as Record; + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has(`mint:${op.id}`)).toBe(true); + + const second = (await runMintQuoteRecovery(booted.source() as never, { + outstanding, + })) as unknown as Record; + expect(second).toMatchObject({ checked: 0, busy: 1 }); + + // Once the held check settles the tracking drains, and recovery proceeds. + releaseObserve(); + for (let i = 0; i < 400 && outstanding.size > 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(outstanding.size).toBe(0); + + const third = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(third).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.manager.test.ts b/src/daemon/wallet/mint-quote-recovery.manager.test.ts new file mode 100644 index 0000000..5c779bf --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.manager.test.ts @@ -0,0 +1,159 @@ +/** + * Real-Manager integration for re-opening a failed mint operation. + * + * The unit tests exercise `reopenFailedMintOperation` against a hand-written + * service double, which cannot catch a change in coco's own + * `MintOperationService.transitionToPending` semantics or in its per-operation + * lock. This drives the production helper against an actual coco `Manager` + * backed by sqlite, so the private-service boundary that the helper depends on + * is exercised for real: full-row preservation, the shared operation lock, and + * the `mint-op:pending` event. + * + * No mint or network access is involved; nothing here enables the mint watcher + * or processor, which the fake-mint integration covers. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow() { + return { + id: "op-1", + mintUrl: "https://mint.invalid", + quoteId: "quote-1", + state: "failed", + createdAt: 1_000, + updatedAt: 2_000, + error: "expired", + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: { reason: "expired", observedAt: 3_000 }, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation with a real coco Manager", () => { + let dir: string | undefined; + let database: Database | undefined; + let manager: Manager | undefined; + let repositories: SqliteRepositories | undefined; + + afterEach(async () => { + await manager?.dispose().catch(() => undefined); + manager = undefined; + database?.close(); + database = undefined; + repositories = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function boot() { + dir = mkdtempSync(join(tmpdir(), "routstrd-manager-")); + database = new Database(join(dir, "coco.db")); + repositories = new SqliteRepositories({ database }); + await repositories.init(); + manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await repositories.mintOperationRepository.create(failedRow() as never); + const service = ( + manager as unknown as { + mintOperationService: { + acquireOperationLock(id: string): Promise<() => void>; + getOperation(id: string): Promise | null>; + transitionToPending( + op: Record, + error?: string, + ): Promise; + }; + } + ).mintOperationService; + const eventBus = ( + manager as unknown as { + eventBus: { on(event: string, handler: () => void): () => void }; + } + ).eventBus; + return { service, eventBus }; + } + + function readRow(id: string) { + return repositories!.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + it("re-opens through the real service and emits mint-op:pending", async () => { + const { service, eventBus } = await boot(); + const events: string[] = []; + const off = eventBus.on("mint-op:pending", () => events.push("pending")); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + off(); + + expect(reopened).toBe(true); + const row = await readRow("op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + expect(events).toEqual(["pending"]); + }); + + it("refuses to re-open while coco's operation lock is held", async () => { + const { service } = await boot(); + // coco's OperationIdLock is fail-fast: a holder blocks the re-open by + // making it throw, and nothing is written. + const release = await service.acquireOperationLock("op-1"); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /already in progress/, + ); + const blocked = await readRow("op-1"); + expect(blocked?.state).toBe("failed"); + expect(blocked?.outputData).toEqual(OUTPUT_DATA); + + release(); + expect(await reopenFailedMintOperation(service, "op-1")).toBe(true); + const reopened = await readRow("op-1"); + expect(reopened?.state).toBe("pending"); + expect(reopened?.outputData).toEqual(OUTPUT_DATA); + }); + + it("leaves an operation a processor finalized alone", async () => { + const { service } = await boot(); + // Emulate a processor winning the race while holding the same lock. + const release = await service.acquireOperationLock("op-1"); + const row = await readRow("op-1"); + await repositories!.mintOperationRepository.update({ + ...row, + state: "finalized", + } as never); + release(); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + const after = await readRow("op-1"); + expect(after?.state).toBe("finalized"); + expect(after?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.test.ts b/src/daemon/wallet/mint-quote-recovery.test.ts new file mode 100644 index 0000000..eac5de9 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "bun:test"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; + +function mint( + overrides: Partial = {}, +): MintQuoteRecoveryCandidate { + return { + id: "mint-1", + mintUrl: "https://mint.example", + quoteId: "quote-1", + state: "pending", + amount: 1000, + expiry: 0, + ...overrides, + }; +} + +describe("classifyMintQuoteObservation", () => { + it("finalizes a paid-but-unissued quote by minting its stored outputs", () => { + expect(classifyMintQuoteObservation("ready")).toEqual({ + action: "finalize", + observedRemoteState: "PAID", + }); + }); + + it("finalizes an already-issued quote by restoring its proofs", () => { + expect(classifyMintQuoteObservation("completed")).toEqual({ + action: "finalize", + observedRemoteState: "ISSUED", + }); + }); + + it("leaves an unpaid quote alone", () => { + expect(classifyMintQuoteObservation("waiting")).toEqual({ + action: "waiting", + }); + }); + + it("reports a quote the mint can no longer issue", () => { + expect(classifyMintQuoteObservation("terminal")).toEqual({ + action: "terminal", + }); + }); +}); + +describe("selectMintQuotesForRecovery", () => { + it("selects every pending quote because only the mint knows if it was paid", () => { + const result = selectMintQuotesForRecovery({ + mints: [ + mint({ id: "expired", expiry: 1 }), + mint({ id: "fresh" }), + mint({ id: "observed-unpaid", lastObservedRemoteState: "UNPAID" }), + mint({ id: "observed-paid", lastObservedRemoteState: "PAID" }), + ], + }); + expect(result.pending.map((op) => op.id)).toEqual([ + "expired", + "fresh", + "observed-unpaid", + "observed-paid", + ]); + }); + + it("recovers executing operations left behind by a crash mid-mint", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "executing", state: "executing" })], + }); + expect(result.pending.map((op) => op.id)).toEqual(["executing"]); + }); + + it("ignores finalized operations", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "done", state: "finalized" })], + }); + expect(result.pending).toEqual([]); + expect(result.failed).toEqual([]); + }); + + it("does not re-open failed operations by default", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + }); + expect(result.failed).toEqual([]); + }); + + it("returns failed operations when the caller opts in", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + includeFailed: true, + }); + expect(result.failed.map((op) => op.id)).toEqual(["given-up"]); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.ts b/src/daemon/wallet/mint-quote-recovery.ts new file mode 100644 index 0000000..eb620b2 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.ts @@ -0,0 +1,119 @@ +/** + * Pure helpers for PAID mint-quote recovery. + * + * A mint quote can be PAID at the mint while its local operation is still + * `pending` (the Lightning payment landed before expiry while the daemon was + * down, so no local observation was ever recorded) or even terminally + * `failed` (coco gives up when the mint refuses to sign, for example after the + * invoice expiry). Claimability still depends on the mint accepting issuance. + * This feature retries the stored outputs or restores their signatures; it + * does not regenerate outputs rejected by the mint (for example an inactive + * keyset). coco already reconciles pending paid quotes at startup and in the + * periodic sweep. The new capability is operator-targeted recovery, including + * explicitly reopening failed operations, alongside safer cleanup. + * + * Recovery asks the mint what it thinks, then retries issuance or restore. These helpers decide *what* to do from a remote observation; the + * actual state transitions are applied by the in-process coco wallet client + * so coco-core's operation services emit their normal events and release + * proof reservations. Keeping the decisions here makes them unit testable + * without a wallet database or network access. + */ + +/** Subset of coco's mint operation rows that recovery needs. */ +export interface MintQuoteRecoveryCandidate { + id: string; + mintUrl: string; + quoteId?: string; + state: string; + /** Quote amount in sats. */ + amount: number; + /** Quote expiry in epoch seconds. `0` means unknown/not applicable. */ + expiry: number; + /** Last quote state observed from the mint (UNPAID, PAID, ISSUED). */ + lastObservedRemoteState?: string; + error?: string; +} + +/** Coco's classification of a fresh remote quote check. */ +export type PendingMintCheckCategory = + | "waiting" + | "ready" + | "completed" + | "terminal"; + +/** What recovery should do with a quote after checking it with the mint. */ +export type MintQuoteRecoveryDecision = + | { action: "finalize"; observedRemoteState: "PAID" | "ISSUED" } + | { action: "waiting" } + | { action: "terminal" }; + +/** + * Map a remote quote check onto a recovery action. + * + * - `ready` means the mint reports the quote PAID but never issued: submit the + * operation's stored outputs to claim the sats. + * - `completed` means the mint already issued it: recover the signatures + * (NUT-09) instead of minting again. + * - `waiting` means the mint still reports the quote UNPAID: nothing is + * claimable, so leave the operation alone. + * - `terminal` means the quote can no longer be issued (for example the mint + * refused an expired quote). coco persists that verdict as a failed + * operation, so recovery must report it rather than treat it as progress. + */ +export function classifyMintQuoteObservation( + category: PendingMintCheckCategory, +): MintQuoteRecoveryDecision { + switch (category) { + case "ready": + return { action: "finalize", observedRemoteState: "PAID" }; + case "completed": + return { action: "finalize", observedRemoteState: "ISSUED" }; + case "waiting": + return { action: "waiting" }; + case "terminal": + return { action: "terminal" }; + } +} + +export interface MintQuoteRecoverySelectionOptions< + T extends MintQuoteRecoveryCandidate, +> { + mints: T[]; + /** + * Also consider terminally failed operations. Off by default: re-opening a + * failed operation is a mutation, so only an explicit user-invoked recovery + * may do it. Startup recovery must never resurrect quotes on its own. + */ + includeFailed?: boolean; +} + +export interface MintQuoteRecoverySelection< + T extends MintQuoteRecoveryCandidate, +> { + /** Pending (or executing) operations that need a fresh mint observation. */ + pending: T[]; + /** Failed operations the caller may re-open and retry. */ + failed: T[]; +} + +/** + * Split operations into those recovery should check and those that were + * already given up on. + * + * Every `pending` operation is selected: only the mint knows whether an + * expired quote was paid before the local invoice ran out. `executing` + * operations are recovered too, since a crash mid-mint leaves outputs that + * may already be signed. + */ +export function selectMintQuotesForRecovery< + T extends MintQuoteRecoveryCandidate, +>(options: MintQuoteRecoverySelectionOptions): MintQuoteRecoverySelection { + const { mints, includeFailed = false } = options; + const pending = mints.filter( + (op) => op.state === "pending" || op.state === "executing", + ); + const failed = includeFailed + ? mints.filter((op) => op.state === "failed") + : []; + return { pending, failed }; +} diff --git a/src/daemon/wallet/recovery-integration.test.ts b/src/daemon/wallet/recovery-integration.test.ts new file mode 100644 index 0000000..f279f93 --- /dev/null +++ b/src/daemon/wallet/recovery-integration.test.ts @@ -0,0 +1,257 @@ +import { expect, it, spyOn } from "bun:test"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { Database } from "bun:sqlite"; +import { runTargetedRecovery, type SendRecoveryService } from "./recovery-probe"; + +import { + cleanupLocalRecoveryState, + createRecoveryGate, + runWalletRecovery, +} from "./coco-client"; + +const MINT = "https://mint.example.com"; +const OP = "op-live"; + +it("targeted recovery leaves a send that execute() holds alone", async () => { + const repo = new SqliteRepositories({ database: new Database(":memory:") }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const internals = coco as unknown as { + sendOperationService: SendRecoveryService; + walletService: { getWalletWithActiveKeysetId: (m: string) => Promise }; + }; + + let swapStarted!: () => void; + const started = new Promise((r) => (swapStarted = r)); + let finishSwap!: (v: { send: unknown[]; keep: unknown[] }) => void; + const swap = new Promise<{ send: unknown[]; keep: unknown[] }>((r) => (finishSwap = r)); + internals.walletService.getWalletWithActiveKeysetId = async () => ({ + wallet: { + unit: "sat", + send: async () => (swapStarted(), swap), + checkProofsStates: async () => [{ state: "UNSPENT" }], + getFeesForProofs: () => 0, + }, + }); + + await repo.proofRepository.saveProofs(MINT, [ + { id: "00aa", amount: 8, secret: "in-1", C: "02aa", mintUrl: MINT, state: "ready" } as never, + ]); + await repo.proofRepository.reserveProofs(MINT, ["in-1"], OP); + await repo.sendOperationRepository.create({ + id: OP, mintUrl: MINT, amount: 8, state: "prepared", method: "default", methodData: {}, + createdAt: Date.now(), updatedAt: Date.now(), needsSwap: true, fee: 0, inputAmount: 8, + inputProofSecrets: ["in-1"], + outputData: { + keep: [], + send: [{ + blindedMessage: { amount: 8, id: "00aa", B_: "02" + "11".repeat(32) }, + blindingFactor: "01", + secret: Buffer.from("out-1").toString("hex"), + }], + }, + } as never); + + const live = coco.ops.send.execute(OP); + await started; // swap is at the mint + const result = await runTargetedRecovery(coco.ops, internals.sendOperationService, { + kinds: ["send"], + fetchImpl: (async () => new Response("{}")) as unknown as unknown as typeof fetch, + }); + // On 8005aeb this is "rolled_back" and "in-1" is no longer reserved. + expect((await coco.ops.send.get(OP))?.state).toBe("executing"); + // The live execute holds coco's per-operation lock: busy, never driven. + expect(result.attempted).toBe(0); + expect(result.busy).toBe(1); + + finishSwap({ send: [{ id: "00aa", amount: 8, secret: "out-1", C: "02bb" }], keep: [] }); + await live; + expect((await coco.ops.send.get(OP))?.state).toBe("pending"); +}); + + +it("keeps healthy-mint callers gated while happy-path global recovery runs", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const gate = createRecoveryGate(); + let enter!: () => void; + const entered = new Promise((resolve) => { enter = resolve; }); + let release!: () => void; + const barrier = new Promise((resolve) => { release = resolve; }); + const spies = [ + spyOn(coco.ops.send.recovery, "run").mockImplementation(async () => { enter(); await barrier; }), + spyOn(coco.ops.melt.recovery, "run").mockResolvedValue(undefined), + spyOn(coco.ops.receive.recovery, "run").mockResolvedValue(undefined), + spyOn(coco, "recoverPendingMintOperations").mockResolvedValue(undefined), + ]; + try { + const recovery = runWalletRecovery(coco, () => {}, undefined, + (mints) => gate.publishStuckMints(mints), + { fetchImpl: (async () => new Response("{}")) as unknown as typeof fetch }, + ).then(() => gate.complete()); + await entered; + let released = false; + const waiting = gate.waitForRecovery(MINT).then(() => { released = true; }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(released).toBe(false); + release(); + await recovery; + await waiting; + expect(released).toBe(true); + } finally { + release(); + for (const spy of spies) spy.mockRestore(); + db.close(); + } +}); + +it("degraded startup finishes local housekeeping before opening the per-mint gate", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + await repo.sendOperationRepository.create({ + id: "stuck", mintUrl: MINT, amount: 8, state: "rolling_back", + method: "default", methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + const gate = createRecoveryGate(); + const events: string[] = []; + const sweep = spyOn(coco.ops.send.recovery, "run"); + try { + const recovery = runWalletRecovery(coco, () => {}, [], (mints) => { + events.push("gate"); + gate.publishStuckMints(mints); + }, { + fetchImpl: (async () => { throw new Error("offline"); }) as unknown as unknown as typeof fetch, + cleanupLocalState: async () => { events.push("cleanup"); }, + }).then(() => gate.complete()); + await gate.waitForRecovery("https://healthy.example.com"); + expect(events).toEqual(["cleanup", "gate"]); + await recovery; + expect(sweep).not.toHaveBeenCalled(); + expect((await coco.ops.send.get("stuck"))?.state).toBe("rolling_back"); + } finally { + sweep.mockRestore(); + db.close(); + } +}); + +it("degraded startup probes before settlement and never asks a dead mint", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const expiredRow = (id: string, mintUrl: string) => ({ + id, mintUrl, quoteId: `q-${id}`, state: "pending", + createdAt: 1_000, updatedAt: 2_000, method: "bolt11", + methodData: { method: "bolt11", data: {} }, amount: 100, unit: "sat", + request: "lnbc1example", expiry: 1_000_000, // epoch seconds, long past + }); + await repo.mintOperationRepository.create(expiredRow("dead-quote", "https://dead.example.com") as never); + await repo.mintOperationRepository.create(expiredRow("live-quote", "https://live.example.com") as never); + const internals = coco as unknown as { + mintOperationService: { + observePendingOperation(id: string): Promise<{ category: "waiting" }>; + failPendingOperation(op: unknown, failure: unknown): Promise; + }; + }; + const events: string[] = []; + const observe = spyOn(internals.mintOperationService, "observePendingOperation") + .mockImplementation(async (id: string) => { + events.push(`observe:${id}`); + return { category: "waiting" }; + }); + // Fail the row for real (as the production service would) so the targeted + // mint pass sees state "failed" and refresh() returns without re-observing. + const fail = spyOn(internals.mintOperationService, "failPendingOperation") + .mockImplementation(async (op: unknown) => { + const id = (op as { id: string }).id; + const row = await repo.mintOperationRepository.getById(id); + if (row) await repo.mintOperationRepository.update({ ...row, state: "failed" } as never); + return {} as never; + }); + const gate = createRecoveryGate(); + try { + const recovery = runWalletRecovery(coco, () => {}, [], (mints) => { + events.push("gate"); + gate.publishStuckMints(mints); + }, { + fetchImpl: (async (url: unknown) => { + if (String(url).includes("dead.example.com")) throw new Error("offline"); + return new Response("{}"); + }) as unknown as typeof fetch, + cleanupLocalState: async () => { events.push("cleanup"); }, + }).then(() => gate.complete()); + await recovery; + // The dead mint was probed, never asked to observe its quote; the gate + // opened before settlement spent anything on the live mint. + expect(events).toEqual(["cleanup", "gate", "observe:live-quote"]); + expect(observe).toHaveBeenCalledTimes(1); + expect(fail).toHaveBeenCalledTimes(1); + } finally { + observe.mockRestore(); + fail.mockRestore(); + db.close(); + } +}); + +it("local housekeeping cleans init sends and orphaned reservations without network", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + try { + await repo.sendOperationRepository.create({ + id: "init-send", mintUrl: MINT, amount: 8, state: "init", method: "default", + methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + for (const [id, repository] of [ + ["init-melt", repo.meltOperationRepository], + ["init-receive", repo.receiveOperationRepository], + ["init-mint", repo.mintOperationRepository], + ] as const) { + await repository.create({ + id, mintUrl: MINT, amount: 8, state: "init", method: "bolt11", + methodData: {}, inputProofs: [], createdAt: Date.now(), updatedAt: Date.now(), + } as never); + } + await repo.proofRepository.saveProofs(MINT, [ + { id: "00aa", amount: 8, secret: "init-input", C: "02aa", mintUrl: MINT, state: "ready" }, + { id: "00aa", amount: 8, secret: "orphan-input", C: "02aa", mintUrl: MINT, state: "ready" }, + ] as never); + await repo.proofRepository.reserveProofs(MINT, ["init-input"], "init-send"); + await repo.proofRepository.reserveProofs(MINT, ["orphan-input"], "missing-send"); + await cleanupLocalRecoveryState(coco, repo); + expect(await coco.ops.send.get("init-send")).toBeNull(); + expect(await repo.proofRepository.getReservedProofs()).toEqual([]); + expect(await repo.meltOperationRepository.getById("init-melt")).toBeNull(); + expect(await repo.receiveOperationRepository.getById("init-receive")).toBeNull(); + expect(await repo.mintOperationRepository.getById("init-mint")).toBeNull(); + } finally { + db.close(); + } +}); + +it("cleanup checks all private methods before making any local writes", async () => { + const db = new Database(":memory:"); + const repo = new SqliteRepositories({ database: db }); + await repo.init(); + const coco = new Manager(repo, async () => new Uint8Array(64)); + const internals = coco as unknown as { mintOperationService: { recoverInitOperation: unknown } }; + const original = internals.mintOperationService.recoverInitOperation; + try { + await repo.sendOperationRepository.create({ + id: "untouched-init", mintUrl: MINT, amount: 8, state: "init", method: "default", + methodData: {}, createdAt: Date.now(), updatedAt: Date.now(), + } as never); + internals.mintOperationService.recoverInitOperation = undefined; + await expect(cleanupLocalRecoveryState(coco, repo)).rejects.toThrow("mintOperationService.recoverInitOperation is unavailable"); + expect((await repo.sendOperationRepository.getById("untouched-init"))?.state).toBe("init"); + } finally { + internals.mintOperationService.recoverInitOperation = original; + db.close(); + } +}); diff --git a/src/daemon/wallet/recovery-probe.test.ts b/src/daemon/wallet/recovery-probe.test.ts new file mode 100644 index 0000000..18b7ca8 --- /dev/null +++ b/src/daemon/wallet/recovery-probe.test.ts @@ -0,0 +1,448 @@ +import { describe, expect, it } from "bun:test"; +import { + collectStuckOperations, + probeMintReachability, + runTargetedRecovery, + type StuckOperationSource, + type SendRecoveryService, +} from "./recovery-probe"; + +import { runMintQuoteRecovery, settlePendingMintQuotes, settleExpiredMintQuotes } from "./coco-client"; + +interface FakeOp { + id: string; + mintUrl: string; + state: string; +} + +function op(id: string, mintUrl: string, state = "pending"): FakeOp { + return { id, mintUrl, state }; +} + +interface FakeSource extends StuckOperationSource { + stuck: Record<"send" | "melt" | "receive" | "mint", FakeOp[]>; + refreshed: Record<"send" | "melt" | "receive" | "mint", string[]>; + failOnRefresh?: Set; +} + +function makeSource( + stuck: Partial>, +): FakeSource { + const refreshed: FakeSource["refreshed"] = { + send: [], + melt: [], + receive: [], + mint: [], + }; + const full = { + send: stuck.send ?? [], + melt: stuck.melt ?? [], + receive: stuck.receive ?? [], + mint: stuck.mint ?? [], + }; + const family = (kind: keyof typeof full) => ({ + diagnostics: { isLocked: () => false }, + listInFlight: async () => full[kind] as never, + refresh: async (id: string) => { + refreshed[kind].push(id); + if (full[kind].some((o) => o.id === id && o.id.startsWith("boom"))) { + throw new Error("mint rejected the operation"); + } + return full[kind].find((o) => o.id === id) as never; + }, + }); + return { + stuck: full, + refreshed, + send: { + ...family("send"), + get: async (id: string) => + (full.send.find((o) => o.id === id) ?? null) as never, + } as FakeSource["send"], + melt: family("melt") as FakeSource["melt"], + receive: family("receive") as FakeSource["receive"], + mint: family("mint") as FakeSource["mint"], + }; +} + +function makeSendService(): SendRecoveryService & { + executingRecovered: string[]; + /** Lock events in order, e.g. "acquire:op-1", "release:op-1". */ + lockLog: string[]; +} { + const executingRecovered: string[] = []; + const lockLog: string[] = []; + return { + executingRecovered, + lockLog, + acquireOperationLock: async (id) => { + lockLog.push(`acquire:${id}`); + return () => { + lockLog.push(`release:${id}`); + }; + }, + recoverExecutingOperation: async (raw) => { + const id = (raw as FakeOp).id; + executingRecovered.push(id); + lockLog.push(`recover:${id}`); + }, + }; +} + +/** fetch stub: mints whose URL contains "dead" hang/fail; others answer. */ +function makeFetch(deadPredicate: (url: string) => boolean) { + const calls: string[] = []; + const fetchImpl = (async (url: string | URL | Request) => { + const href = String(url); + calls.push(href); + if (deadPredicate(href)) throw new Error("connect ECONNREFUSED"); + return new Response("{}", { status: 200 }); + }) as unknown as typeof fetch; + return { calls, fetchImpl }; +} + +const isDeadUrl = (url: string) => url.includes("dead"); + +const liveFetch = () => makeFetch(() => false); + +describe("collectStuckOperations", () => { + it("aggregates all four operation families with normalized mint URLs", async () => { + const source = makeSource({ + send: [op("s1", "https://mint.example.com/")], + melt: [op("m1", "https://mint.example.com")], + receive: [op("r1", "https://other.example.com", "executing")], + mint: [op("q1", "https://mint.example.com")], + }); + const stuck = await collectStuckOperations(source); + expect(stuck).toHaveLength(4); + expect(stuck.map((s) => s.kind).sort()).toEqual([ + "melt", + "mint", + "receive", + "send", + ]); + // Trailing slash normalized so the same mint dedupes to one probe target. + const mintUrls = new Set(stuck.map((s) => s.mintUrl)); + expect(mintUrls.size).toBe(2); + }); + + it("returns empty when nothing is stuck", async () => { + const stuck = await collectStuckOperations(makeSource({})); + expect(stuck).toEqual([]); + }); +}); + +describe("probeMintReachability", () => { + it("marks only mints whose fetch fails as unreachable", async () => { + const deadFetch = makeFetch(isDeadUrl); + const unreachable = await probeMintReachability( + ["https://live.example.com", "https://dead.example.com"], + { fetchImpl: deadFetch.fetchImpl }, + ); + expect([...unreachable]).toEqual(["https://dead.example.com"]); + expect(deadFetch.calls).toHaveLength(2); + expect(deadFetch.calls.every((c) => c.endsWith("/v1/info"))).toBe(true); + }); + + it("treats HTTP error responses as reachable", async () => { + const fetchImpl = (async () => + new Response("oops", { status: 500 })) as unknown as typeof fetch; + const unreachable = await probeMintReachability(["https://live.example.com"], { + fetchImpl, + }); + expect(unreachable.size).toBe(0); + }); +}); + +describe("runTargetedRecovery", () => { + it("does not probe when nothing is stuck", async () => { + const deadFetch = makeFetch(isDeadUrl); + const result = await runTargetedRecovery(makeSource({}), makeSendService(), { + fetchImpl: deadFetch.fetchImpl, + }); + expect(result).toMatchObject({ attempted: 0, skipped: 0, failed: 0 }); + expect(deadFetch.calls).toHaveLength(0); + }); + + it("skips operations at unreachable mints and recovers the rest", async () => { + const source = makeSource({ + send: [op("s1", "https://dead.example.com"), op("s2", "https://live.example.com")], + melt: [op("m1", "https://dead.example.com"), op("m2", "https://live.example.com")], + }); + const deadFetch = makeFetch(isDeadUrl); + const skippedMints: Array<[string, number]> = []; + const result = await runTargetedRecovery(source, makeSendService(), { + fetchImpl: deadFetch.fetchImpl, + onSkippedMint: (mintUrl, count) => skippedMints.push([mintUrl, count]), + }); + expect(result.attempted).toBe(2); + expect(result.skipped).toBe(2); + expect(result.failed).toBe(0); + expect(skippedMints).toEqual([["https://dead.example.com", 2]]); + expect(result.skippedMints.get("https://dead.example.com")).toBe(2); + // Only live-mint operations were driven. + expect(source.refreshed.send).toEqual(["s2"]); + expect(source.refreshed.melt).toEqual(["m2"]); + }); + + it("recovers pending sends via refresh and executing sends via the service", async () => { + const source = makeSource({ + send: [ + op("pending-1", "https://live.example.com", "pending"), + op("exec-1", "https://live.example.com", "executing"), + ], + }); + const sendService = makeSendService(); + const result = await runTargetedRecovery(source, sendService, { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.attempted).toBe(2); + expect(source.refreshed.send).toEqual(["pending-1"]); + expect(sendService.executingRecovered).toEqual(["exec-1"]); + // The executing send is driven under coco's per-operation lock. + expect(sendService.lockLog).toEqual([ + "acquire:exec-1", + "recover:exec-1", + "release:exec-1", + ]); + }); + + it("re-reads state under the lock and skips a send that left executing", async () => { + const source = makeSource({ + send: [op("s1", "https://live.example.com", "pending")], + }); + const sendService = makeSendService(); + // Snapshot taken while the op was still executing; it has since settled. + const result = await runTargetedRecovery(source, sendService, { + stuckOperations: [ + { + kind: "send", + id: "s1", + mintUrl: "https://live.example.com", + state: "executing", + raw: op("s1", "https://live.example.com", "executing"), + }, + ], + unreachableMints: new Set(), + }); + expect(result.attempted).toBe(1); + expect(sendService.executingRecovered).toEqual([]); + // The lock is still acquired and released around the re-read. + expect(sendService.lockLog).toEqual(["acquire:s1", "release:s1"]); + }); + + it("counts a send as busy when a live execute wins the lock race", async () => { + const source = makeSource({ + send: [op("s1", "https://live.example.com", "executing")], + }); + const sendService = makeSendService(); + sendService.acquireOperationLock = async () => { + const error = new Error("operation in progress"); + error.name = "OperationInProgressError"; + throw error; + }; + const result = await runTargetedRecovery(source, sendService, { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.busy).toBe(1); + expect(result.failed).toBe(0); + expect(sendService.executingRecovered).toEqual([]); + }); + + it("counts per-operation failures at reachable mints and continues", async () => { + const source = makeSource({ + melt: [op("boom-1", "https://live.example.com"), op("m2", "https://live.example.com")], + }); + const result = await runTargetedRecovery(source, makeSendService(), { + fetchImpl: liveFetch().fetchImpl, + }); + expect(result.attempted).toBe(2); + expect(result.failed).toBe(1); + expect(source.refreshed.melt).toEqual(["boom-1", "m2"]); + }); + + it("respects the kinds filter and a pre-computed unreachable set", async () => { + const deadFetch = makeFetch(isDeadUrl); + const source = makeSource({ + send: [op("s1", "https://dead.example.com")], + mint: [op("q1", "https://dead.example.com")], + }); + const result = await runTargetedRecovery(source, makeSendService(), { + kinds: ["mint"], + unreachableMints: new Set(["https://dead.example.com"]), + fetchImpl: deadFetch.fetchImpl, + }); + // No probe ran (pre-computed set) and the send op was not even counted. + expect(deadFetch.calls).toHaveLength(0); + expect(result.skipped).toBe(1); + expect(source.refreshed.send).toEqual([]); + }); +}); + + +it("preserves subpath mint URLs when probing", async () => { + const { fetchImpl, calls } = liveFetch(); + await probeMintReachability(["https://mint.example.com/Bitcoin/"], { fetchImpl }); + expect(calls).toEqual(["https://mint.example.com/Bitcoin/v1/info"]); +}); + +it("does not drive locked operations or count rolling-back sends as attempts", async () => { + const source = makeSource({ send: [ + op("live", "https://mint.example.com", "executing"), + op("rollback", "https://mint.example.com", "rolling_back"), + ] }); + source.send.diagnostics.isLocked = (id) => id === "live"; + const service = makeSendService(); + const result = await runTargetedRecovery(source, service, { fetchImpl: liveFetch().fetchImpl }); + expect(result.attempted).toBe(0); + expect(result.busy).toBe(1); + expect(service.executingRecovered).toEqual([]); + expect(service.lockLog).toEqual([]); +}); + +it("classifies malformed persisted URLs as unreachable without fetching", async () => { + const { fetchImpl, calls } = liveFetch(); + const unreachable = await probeMintReachability(["not-a-url"], { fetchImpl }); + expect([...unreachable]).toEqual(["not-a-url"]); + expect(calls).toEqual([]); +}); + +it("bounds a hanging probe with an abort signal", async () => { + const fetchImpl = (async (_url: unknown, options: RequestInit) => + new Promise((_resolve, reject) => { + options.signal!.addEventListener("abort", () => reject(options.signal!.reason), { once: true }); + })) as unknown as typeof fetch; + const unreachable = await probeMintReachability(["https://slow.example.com"], { + fetchImpl, timeoutMs: 10, + }); + expect([...unreachable]).toEqual(["https://slow.example.com"]); +}); + + +it("shares timed-out mint observations across quote recovery, targeted recovery and the sweep", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + let finish!: (value: { category: "waiting" }) => void; + const observation = new Promise<{ category: "waiting" }>(r => { finish = r; }); + const outstanding = new Map>(); + const quoteSource = { + ops: { mint: { + listPending: async () => [{ ...source.stuck.mint[0]!, method: "bolt11" }], + get: async () => null, + finalize: async () => ({ state: "finalized" }), + } }, + mintOperationService: { observePendingOperation: () => observation }, + reopenFailedOperation: async () => false, + }; + try { + await runMintQuoteRecovery(quoteSource as never, { outstanding, timeoutMs: 5 }); + expect(outstanding.has("mint:q1")).toBe(true); + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.busy).toBe(1); + await settlePendingMintQuotes({ + ops: { mint: { ...source.mint, listPending: async () => source.stuck.mint as never } }, + wallet: { balances: { byMint: async () => ({}) } }, + mintOperationService: { failPendingOperation: async () => ({}) }, + } as never, Date.now(), { state: { outstanding } }); + expect(source.refreshed.mint).toEqual([]); + } finally { finish({ category: "waiting" }); await observation; } +}); + +it("tracks a hung targeted mint so quote recovery skips it while unrelated operations proceed", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")], melt: [op("m1", "https://live.example.com")] }); + let finish!: (value: never) => void; + source.mint.refresh = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + try { + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, timeoutMs: 5, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(source.refreshed.melt).toEqual(["m1"]); + const quote = await runMintQuoteRecovery({ + ops: { mint: { listPending: async () => [{ ...source.stuck.mint[0]!, method: "bolt11" }] } }, + } as never, { outstanding }); + expect(quote.busy).toBe(1); + } finally { finish({ state: "pending" } as never); } +}); + +it("retains executing-send lock until a timed-out underlying drive actually finishes", async () => { + const source = makeSource({ send: [op("s1", "https://live.example.com", "executing")] }); + const service = makeSendService(); + let finish!: () => void; + service.recoverExecutingOperation = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + const result = await runTargetedRecovery(source, service, { + outstanding, timeoutMs: 5, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(service.lockLog).toEqual(["acquire:s1"]); + const retry = await runTargetedRecovery(source, service, { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(retry.busy).toBe(1); + const actual = outstanding.get("send:s1")!; + finish(); + await actual; + expect(service.lockLog).toEqual(["acquire:s1", "release:s1"]); + expect(outstanding.size).toBe(0); +}); + +it("refuses missing executing-send internals without driving the operation", async () => { + const source = makeSource({ send: [op("s1", "https://live.example.com", "executing")] }); + const result = await runTargetedRecovery(source, {} as SendRecoveryService, { fetchImpl: liveFetch().fetchImpl }); + expect(result.failed).toBe(1); +}); + + +it("startup settlement keeps its timed-out observation visible to manual recovery", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + let finish!: (value: { category: "ready" }) => void; + const observation = new Promise<{ category: "ready" }>(r => { finish = r; }); + const outstanding = new Map>(); + const settlement = await settleExpiredMintQuotes({ + ops: { mint: { listPending: async () => [{ ...source.stuck.mint[0]!, expiry: 1, updatedAt: 1 }] } }, + mintOperationService: { observePendingOperation: () => observation }, + } as never, Date.now(), 5, { outstanding }); + expect(settlement.unobserved).toBe(1); + const result = await runTargetedRecovery(source, makeSendService(), { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(result.busy).toBe(1); + expect(source.refreshed.mint).toEqual([]); + const actual = outstanding.get("mint:q1")!; + finish({ category: "ready" }); + await actual; + expect(outstanding.size).toBe(0); +}); + +it("deadline and shutdown leave remaining operations untouched", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com")] }); + const result = await runTargetedRecovery(source, makeSendService(), { + shouldStop: () => true, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.skipped).toBe(1); + expect(result.attempted).toBe(0); + expect(source.refreshed.mint).toEqual([]); +}); + +it("a pass budget bounds total drive waits, not just each operation", async () => { + const source = makeSource({ mint: [op("q1", "https://live.example.com"), op("q2", "https://live.example.com")] }); + let finish!: (value: never) => void; + source.mint.refresh = () => new Promise(r => { finish = r; }); + const outstanding = new Map>(); + const result = await runTargetedRecovery(source, makeSendService(), { + outstanding, timeoutMs: 100, deadlineMs: 10, fetchImpl: liveFetch().fetchImpl, + }); + expect(result.timedOut).toBe(1); + expect(result.attempted).toBe(1); + expect(result.skipped).toBe(1); + const actual = outstanding.get("mint:q1")!; + finish({ state: "pending" } as never); + await actual; +}); + +it("unfinished work in another operation family does not block the same bare id", async () => { + const source = makeSource({ mint: [op("same-id", "https://live.example.com")] }); + const outstanding = new Map>([["send:same-id", new Promise(() => {})]]); + const result = await runTargetedRecovery(source, makeSendService(), { outstanding, fetchImpl: liveFetch().fetchImpl }); + expect(result.busy).toBe(0); + expect(source.refreshed.mint).toEqual(["same-id"]); +}); diff --git a/src/daemon/wallet/recovery-probe.ts b/src/daemon/wallet/recovery-probe.ts new file mode 100644 index 0000000..3dddfaa --- /dev/null +++ b/src/daemon/wallet/recovery-probe.ts @@ -0,0 +1,312 @@ +/** + * Mint reachability probing and targeted (per-operation) wallet recovery. + * + * Coco's global recovery sweeps walk every non-terminal operation one at a + * time; each operation at an unreachable mint costs a full network timeout. + * This module probes every mint that has stuck operations once, in parallel, + * and drives recovery per operation only for mints that answer — so one dead + * mint costs a single short probe instead of N sequential timeouts, and its + * operations stay parked (exactly as coco's "will retry later" path leaves + * them) until an explicit mid-session recovery (see recoverStuckOperations + * in coco-client.ts) or a later startup finds the mint reachable. + */ +import { normalizeMintUrl, type Manager } from "@cashu/coco-core"; +import { recoveryKey, trackRecovery, waitForRecoveryWork, RecoveryWaitTimeout, type RecoveryWork } from "./recovery-work"; +import { logger } from "../../utils/logger"; + +/** Short probe: a mint that cannot answer /v1/info in 2s slows every op. */ +export const MINT_PROBE_TIMEOUT_MS = 2_000; + +type OpsApi = Manager["ops"]; + +/** Structural subset of the ops APIs used to enumerate and recover operations. */ +export interface StuckOperationSource { + send: Pick; + melt: Pick; + receive: Pick; + mint: Pick; +} + +export type StuckOperationKind = "send" | "melt" | "receive" | "mint"; + +export interface StuckOperation { + kind: StuckOperationKind; + id: string; + /** Normalized mint URL. */ + mintUrl: string; + state: string; + /** The operation object as returned by the API (needed by service-level recovery). */ + raw: unknown; +} + +/** + * The per-operation send recovery entry points coco keeps private. Send is the + * only operation family whose public `refresh()` does not cover `executing` + * operations; routstrd already reaches into coco internals the same way for + * `mintOperationService` (see coco-client.ts). + */ +export interface SendRecoveryService { + recoverExecutingOperation(op: unknown): Promise; + /** + * coco's per-operation lock. It is fail-fast: acquiring an id a live + * execute/finalize/recover already holds throws OperationInProgressError + * instead of waiting. Holding it across the state re-read and the drive + * makes executing-send recovery atomic against a live execute — the same + * pattern reopenFailedMintOperation uses for mint operations + * (see coco-client.ts). + */ + acquireOperationLock(operationId: string): Promise<() => void>; +} + +export interface RecoveryRunResult { + /** Operations for which recovery was attempted (not necessarily completed). */ + attempted: number; + /** Timed-out waits, also included in attempted; underlying work remains tracked. */ + timedOut: number; + /** Locked operations or unfinished work from another pass; retry later. */ + busy: number; + /** Operations skipped for unreachable mints, shutdown, or pass budget exhaustion. */ + skipped: number; + /** Attempts that threw a non-busy, non-timeout error. */ + failed: number; + /** Unreachable mint URL -> number of operations skipped there. */ + skippedMints: Map; +} + +export interface TargetedRecoveryOptions { + outstanding?: RecoveryWork; + timeoutMs?: number; + deadlineMs?: number; + shouldStop?: () => boolean; + probeTimeoutMs?: number; + fetchImpl?: typeof fetch; + /** Operation families to recover. Defaults to all four. */ + kinds?: StuckOperationKind[]; + /** Pre-collected operations (e.g. from startup gating); default: enumerate now. */ + stuckOperations?: StuckOperation[]; + /** Pre-probed unreachable mints; default: probe now. */ + unreachableMints?: Set; + /** Called once per unreachable mint with the number of skipped operations. */ + onSkippedMint?: (mintUrl: string, opCount: number) => void; +} + +function asStuckOperations( + kind: StuckOperationKind, + ops: Array<{ id: string; mintUrl: string; state: string }>, +): StuckOperation[] { + const stuck: StuckOperation[] = []; + for (const op of ops) { + try { + stuck.push({ + kind, + id: op.id, + mintUrl: normalizeMintUrl(op.mintUrl), + state: op.state, + raw: op, + }); + } catch { + // Preserve malformed persisted URLs; the probe will classify them as + // unreachable rather than attempting recovery against an invalid URL. + stuck.push({ kind, id: op.id, mintUrl: op.mintUrl, state: op.state, raw: op }); + } + } + return stuck; +} + +/** + * Enumerate every non-terminal operation across all four operation families. + * Returns [] quickly when nothing is stuck, which is the common case. + */ +export async function collectStuckOperations( + source: StuckOperationSource, +): Promise { + const [sends, melts, receives, mints] = await Promise.all([ + source.send.listInFlight(), + source.melt.listInFlight(), + source.receive.listInFlight(), + source.mint.listInFlight(), + ]); + return [ + ...asStuckOperations("send", sends), + ...asStuckOperations("melt", melts), + ...asStuckOperations("receive", receives), + ...asStuckOperations("mint", mints), + ]; +} + +/** + * Probe each mint once, in parallel, and return the set of mint URLs that did + * not answer `GET /v1/info` in time. A mint that answers with an HTTP error is + * still "reachable" — its operations will fail with a real mint error instead + * of a network timeout, which is the information recovery needs. + */ +export async function probeMintReachability( + mintUrls: string[], + options: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}, +): Promise> { + const fetcher = options.fetchImpl ?? fetch; + const timeoutMs = options.timeoutMs ?? MINT_PROBE_TIMEOUT_MS; + const unreachable = new Set(); + + await Promise.all( + mintUrls.map(async (mintUrl) => { + try { + await fetcher(`${normalizeMintUrl(mintUrl)}/v1/info`, { + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (error) { + logger.debug("Mint did not answer recovery probe", { + mintUrl, + error: error instanceof Error ? error.message : String(error), + }); + unreachable.add(mintUrl); + } + }), + ); + + return unreachable; +} + +async function recoverStuckOperation( + source: StuckOperationSource, + sendService: SendRecoveryService, + op: StuckOperation, +): Promise { + switch (op.kind) { + case "send": + if (op.state === "pending") { + // Public API: actively re-checks the proofs with the mint. + await source.send.refresh(op.id); + } else if (op.state === "executing") { + // Fail closed: the lock and the drive are private coco internals + // (written against coco-core 1.0.1), so a coco bump that renames them + // must fail this operation loudly instead of corrupting it. + if ( + typeof sendService.acquireOperationLock !== "function" || + typeof sendService.recoverExecutingOperation !== "function" + ) { + throw new Error( + "coco sendOperationService recovery internals are unavailable; refusing to recover an executing send", + ); + } + // recoverExecutingOperation takes no lock and does no state re-read, + // so take coco's fail-fast per-operation lock first (throws + // OperationInProgressError when a live execute holds the operation — + // the driver counts that as busy, not failed) and re-read the state + // under it: the snapshot op must still be executing before we drive. + const release = await sendService.acquireOperationLock(op.id); + try { + const latest = await source.send.get(op.id); + if (latest?.state === "executing") { + await sendService.recoverExecutingOperation(latest); + } + } finally { + release(); + } + } + // prepared / rolling_back: the global sweep only warns; nothing to do. + return; + case "melt": + // refresh() covers both pending and executing melt operations. + await source.melt.refresh(op.id); + return; + case "receive": + // refresh() actively recovers executing receive operations. + await source.receive.refresh(op.id); + return; + case "mint": + // refresh() covers both pending and executing mint operations. + await source.mint.refresh(op.id); + return; + } +} + +/** + * Recover every stuck operation whose mint answers a reachability probe, + * skipping operations at unreachable mints. Operations are recovered + * sequentially per mint (matching the global sweep's ordering guarantees); + * skipped operations are left untouched for a later pass, which is exactly + * what coco's own "Could not reach mint for recovery, will retry later" path + * does with them. + */ +export async function runTargetedRecovery( + source: StuckOperationSource, + sendService: SendRecoveryService, + options: TargetedRecoveryOptions = {}, +): Promise { + const result: RecoveryRunResult = { + attempted: 0, + timedOut: 0, + busy: 0, + skipped: 0, + failed: 0, + skippedMints: new Map(), + }; + + const timeoutMs = options.timeoutMs ?? 15_000; + const deadlineMs = options.deadlineMs ?? 60_000; + if (![timeoutMs, deadlineMs].every(n => Number.isFinite(n) && n > 0)) { + throw new Error("Recovery budgets must be positive finite numbers"); + } + const deadline = Date.now() + deadlineMs; + const outstanding = options.outstanding ?? new Map(); + const kinds = options.kinds ?? ["send", "melt", "receive", "mint"]; + const stuck = (options.stuckOperations ?? (await collectStuckOperations(source))).filter( + (op) => kinds.includes(op.kind), + ); + if (stuck.length === 0) return result; + + const unreachable = + options.unreachableMints ?? + (await probeMintReachability([...new Set(stuck.map((op) => op.mintUrl))], { + timeoutMs: options.probeTimeoutMs, + fetchImpl: options.fetchImpl, + })); + + for (const mintUrl of unreachable) { + const count = stuck.filter((op) => op.mintUrl === mintUrl).length; + result.skippedMints.set(mintUrl, count); + result.skipped += count; + options.onSkippedMint?.(mintUrl, count); + } + + for (const op of stuck) { + if (unreachable.has(op.mintUrl)) continue; + if (options.shouldStop?.() || Date.now() >= deadline) { result.skipped++; continue; } + const key = recoveryKey(op.kind, op.id); + if (outstanding.has(key)) { result.busy++; continue; } + // Cheap pre-filter for live operations; the lock inside + // recoverStuckOperation is what actually makes the drive atomic. + if (source[op.kind].diagnostics.isLocked(op.id)) { + result.busy++; + continue; + } + if (op.kind === "send" && !["pending", "executing"].includes(op.state)) continue; + result.attempted++; + try { + const work = trackRecovery(outstanding, key, recoverStuckOperation(source, sendService, op)); + await waitForRecoveryWork(work, Math.min(timeoutMs, Math.max(1, deadline - Date.now()))); + } catch (error) { + if (error instanceof RecoveryWaitTimeout) { result.timedOut++; continue; } + // A live execute grabbed the operation between the isLocked pre-filter + // and the lock acquisition: busy, not failed — leave it for a later + // pass. Name-matched like runMintQuoteRecovery does, because the error + // crosses a package boundary. + if (error instanceof Error && error.name === "OperationInProgressError") { + result.busy++; + continue; + } + // Same semantics as coco's tryRecover*: leave the operation for the + // next pass. A reachable mint can still reject a specific operation. + result.failed++; + logger.warn("Targeted operation recovery did not complete", { + kind: op.kind, + operationId: op.id, + mintUrl: op.mintUrl, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return result; +} diff --git a/src/daemon/wallet/recovery-work.test.ts b/src/daemon/wallet/recovery-work.test.ts new file mode 100644 index 0000000..06bfb51 --- /dev/null +++ b/src/daemon/wallet/recovery-work.test.ts @@ -0,0 +1,41 @@ +import { expect, it } from "bun:test"; +import { createRunQueue } from "./coco-client"; +import { createRecoveryDisposer, drainRecoveryWork, trackRecovery } from "./recovery-work"; + +it("incomplete shutdown retains resources until late writes settle; retries join disposal", async () => { + const work = new Map>(); + let finish!: () => void; + const events: string[] = []; + trackRecovery(work, "mint:q", new Promise(r => { finish = r; }).then(() => { events.push("write"); })); + const dispose = createRecoveryDisposer(() => {}, () => drainRecoveryWork(work), async () => { events.push("close"); }, 5); + await expect(dispose()).rejects.toThrow("Timed out"); + expect(events).toEqual([]); + finish(); + await dispose(); + expect(events).toEqual(["write", "close"]); + await dispose(); + expect(events).toEqual(["write", "close"]); +}); + +it("shutdown drains active queue work and queued callbacks reject before touching DB", async () => { + const queue = createRunQueue(); + let disposed = false; + let finish!: () => void; + const events: string[] = []; + let started!: () => void; + const ready = new Promise(r => { started = r; }); + const active = queue(() => new Promise(r => { finish = r; started(); }).then(() => { events.push("write"); })); + await ready; + const queued = queue(async () => { + if (disposed) throw new Error("Wallet is shutting down"); + events.push("unexpected"); + }); + const rejected = queued.catch(error => error); + const dispose = createRecoveryDisposer(() => { disposed = true; }, () => queue.drain(), async () => { events.push("close"); }, 5); + await expect(dispose()).rejects.toThrow("Timed out"); + finish(); + await active; + expect((await rejected).message).toContain("shutting down"); + await dispose(); + expect(events).toEqual(["write", "close"]); +}); diff --git a/src/daemon/wallet/recovery-work.ts b/src/daemon/wallet/recovery-work.ts new file mode 100644 index 0000000..a776f74 --- /dev/null +++ b/src/daemon/wallet/recovery-work.ts @@ -0,0 +1,45 @@ +/** A timed-out wait is not cancellation: retain work until it actually settles. */ +export type RecoveryWork = Map>; +export const recoveryKey = (kind: string, id: string): string => `${kind}:${id}`; + +export function trackRecovery(work: RecoveryWork, key: string, promise: Promise): Promise { + work.set(key, promise); + const clear = () => { if (work.get(key) === promise) work.delete(key); }; + void promise.then(clear, clear); + return promise; +} + +export class RecoveryWaitTimeout extends Error { + constructor() { super("Timed out waiting for recovery; underlying work is still tracked"); } +} + +export async function waitForRecoveryWork(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new RecoveryWaitTimeout()), timeoutMs); + })]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** Drain actual work, including work registered while an earlier task settles. */ +export async function drainRecoveryWork(work: RecoveryWork): Promise { + while (work.size) await Promise.allSettled([...work.values()]); +} + +/** Timeout reports incomplete disposal; actual cleanup continues safely. */ +export function createRecoveryDisposer( + quiesce: () => void, + settle: () => Promise, + close: () => Promise, + timeoutMs = 30_000, +): () => Promise { + let disposal: Promise | undefined; + return async () => { + quiesce(); + disposal ??= (async () => { await settle(); await close(); })(); + await waitForRecoveryWork(disposal, timeoutMs); + }; +} diff --git a/src/daemon/wallet/testing/fake-mint.ts b/src/daemon/wallet/testing/fake-mint.ts new file mode 100644 index 0000000..3960ee9 --- /dev/null +++ b/src/daemon/wallet/testing/fake-mint.ts @@ -0,0 +1,347 @@ +/** + * In-process Cashu mint for integration tests. + * + * Implements just enough of NUT-01/02/04/06/07/09 to drive a real coco + * `Manager` against real HTTP: keyset publication, bolt11 mint quotes, minting + * blinded outputs with a real secp256k1 blind signature, NUT-09 restore and + * NUT-07 proof states. No Lightning, no network, no NPC. + * + * The signing keys and signatures are genuine (`@cashu/cashu-ts` mint-side + * helpers), so a wallet that receives these signatures can unblind and verify + * them exactly as with a production mint. + */ +import { + createBlindSignature, + createNewMintKeys, + pointFromHex, +} from "@cashu/cashu-ts"; + +/** NUT error codes used by the scenarios. */ +export const QUOTE_EXPIRED = 20007; +export const ALREADY_ISSUED = 20002; + +export type FakeQuoteState = "UNPAID" | "PAID" | "ISSUED"; + +export interface FakeMintQuote { + quote: string; + request: string; + amount: number; + unit: string; + state: FakeQuoteState; + /** Epoch seconds, or null for a quote that never expires. */ + expiry: number | null; + amountPaid: number; + amountIssued: number; + pubkey: null; +} + +export interface FakeMintRequest { + quote: string; + outputs: Array<{ amount: number; id: string; B_: string }>; +} + +interface StoredSignature { + amount: number; + id: string; + C_: string; +} + +const toHex = (bytes: Uint8Array) => Buffer.from(bytes).toString("hex"); + +export class FakeMint { + readonly keysetId: string; + readonly keysByAmount: Record; + readonly requests: FakeMintRequest[] = []; + /** Every output the mint has ever signed, keyed by B_. */ + readonly signed = new Map(); + + /** When set, POST /v1/mint/bolt11 fails with this NUT error. */ + mintError: { code: number; detail: string } | null = null; + /** When set, quote creation returns this expiry (epoch seconds). */ + quoteExpiry: number | null = 3_600; + /** + * Awaited before responding to a mint request, so a test can hold minting + * open and interleave another recovery attempt. + */ + gate: Promise | null = null; + /** Awaited before answering a quote-state check, to hold observe open. */ + observeGate: Promise | null = null; + /** + * When true, POST /v1/restore answers with NUT-09's spec-legal positional + * arrays, including `null` for outputs the mint never signed. + * + * This is a known interop gap, not a supported path: cashu-ts 3.7.1 (which + * coco depends on) dereferences every entry of `signatures` while normalising + * amounts, so a `null` makes the wallet throw instead of skipping it. The + * switch exists so tests keep that behaviour visible; see + * mint-quote-recovery.fake-mint.test.ts. + */ + restoreIncludesNulls = false; + + private readonly quotes = new Map(); + private counter = 0; + private server?: ReturnType; + + constructor() { + const pair = createNewMintKeys(20, new Uint8Array(32).fill(9)); + this.keysetId = pair.keysetId; + this.keysByAmount = Object.fromEntries( + Object.entries(pair.pubKeys).map(([amount, key]) => [ + amount, + typeof key === "string" ? key : toHex(key), + ]), + ); + this.privKeys = Object.fromEntries( + Object.entries(pair.privKeys) as Array<[string, Uint8Array]>, + ); + } + + private readonly privKeys: Record; + + get url(): string { + if (!this.server) throw new Error("fake mint not started"); + return `http://127.0.0.1:${this.server.port}`; + } + + start(): void { + this.server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: (request) => this.handle(request), + }); + } + + stop(): void { + this.server?.stop(true); + this.server = undefined; + } + + /** Test control: the mint sees the invoice as paid but has issued nothing. */ + markPaid(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "PAID"; + quote.amountPaid = quote.amount; + } + + /** + * Test control: mark the quote issued without going through the mint + * endpoint, simulating a wallet that lost the signatures. + */ + markIssued(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "ISSUED"; + quote.amountPaid = quote.amount; + quote.amountIssued = quote.amount; + } + + /** Test control: sign outputs directly, as if another wallet had issued them. */ + signFor(quoteId: string, outputs: FakeMintRequest["outputs"]): void { + for (const output of outputs) { + this.signOutput(output); + } + this.markIssued(quoteId); + } + + getQuote(quoteId: string): FakeMintQuote | undefined { + return this.quotes.get(quoteId); + } + + private quoteBody(quote: FakeMintQuote) { + return { + quote: quote.quote, + request: quote.request, + amount: quote.amount, + unit: quote.unit, + state: quote.state, + expiry: quote.expiry, + amount_paid: quote.amountPaid, + amount_issued: quote.amountIssued, + pubkey: quote.pubkey, + }; + } + + private signOutput(output: { + amount: number; + id: string; + B_: string; + }): StoredSignature { + const privKey = this.privKeys[String(output.amount)]; + if (!privKey) { + throw new Error(`fake mint has no key for amount ${output.amount}`); + } + const signature = createBlindSignature( + pointFromHex(output.B_), + privKey, + this.keysetId, + ); + const stored: StoredSignature = { + amount: output.amount, + id: this.keysetId, + C_: toHex(signature.C_.toBytes(false)), + }; + this.signed.set(output.B_, stored); + return stored; + } + + private json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + } + + private error(code: number, detail: string, status = 400): Response { + return this.json({ code, detail }, status); + } + + private async handle(request: Request): Promise { + const { pathname } = new URL(request.url); + const body = async () => { + try { + return (await request.json()) as Record; + } catch { + return {}; + } + }; + + if (pathname === "/v1/info") { + return this.json({ + name: "fake-mint", + version: "0.0.1", + nuts: { + 4: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 5: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 7: { supported: true }, + 9: { supported: true }, + }, + }); + } + + if (pathname === "/v1/keys" || pathname.startsWith("/v1/keys/")) { + return this.json({ + keysets: [ + { id: this.keysetId, unit: "sat", keys: this.keysByAmount }, + ], + }); + } + + if (pathname === "/v1/keysets") { + return this.json({ + keysets: [{ id: this.keysetId, unit: "sat", active: true }], + }); + } + + if (pathname === "/v1/mint/quote/bolt11" && request.method === "POST") { + const input = await body(); + const amount = Number(input.amount); + const unit = typeof input.unit === "string" ? input.unit : "sat"; + const quote: FakeMintQuote = { + quote: `fake-quote-${++this.counter}`, + request: `lnbcfake${this.counter}`, + amount, + unit, + state: "UNPAID", + expiry: + this.quoteExpiry === null + ? null + : Math.floor(Date.now() / 1000) + this.quoteExpiry, + amountPaid: 0, + amountIssued: 0, + pubkey: null, + }; + this.quotes.set(quote.quote, quote); + return this.json(this.quoteBody(quote)); + } + + const quoteMatch = pathname.match(/^\/v1\/mint\/quote\/bolt11\/(.+)$/); + if (quoteMatch?.[1] && request.method === "GET") { + if (this.observeGate) await this.observeGate; + const quote = this.quotes.get(decodeURIComponent(quoteMatch[1])); + if (!quote) return this.error(50000, "Unknown quote"); + return this.json(this.quoteBody(quote)); + } + + if (pathname === "/v1/mint/bolt11" && request.method === "POST") { + const input = await body(); + const quoteId = String(input.quote); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + this.requests.push({ quote: quoteId, outputs }); + if (this.gate) await this.gate; + if (this.mintError) { + return this.error(this.mintError.code, this.mintError.detail); + } + const quote = this.quotes.get(quoteId); + if (!quote) return this.error(50000, "Unknown quote"); + if (quote.state === "ISSUED" || quote.amountIssued > 0) { + return this.error(ALREADY_ISSUED, "Quote already issued"); + } + if (quote.state !== "PAID") { + return this.error(20001, "Quote is not paid"); + } + const total = outputs.reduce((sum, o) => sum + Number(o.amount), 0); + if (total > quote.amountPaid - quote.amountIssued) { + return this.error(10002, "Outputs exceed the paid amount"); + } + const signatures = outputs.map((output) => ({ + amount: output.amount, + id: this.keysetId, + C_: this.signOutput(output).C_, + })); + quote.state = "ISSUED"; + quote.amountIssued += total; + return this.json({ signatures }); + } + + if (pathname === "/v1/restore" && request.method === "POST") { + const input = await body(); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + if (this.restoreIncludesNulls) { + // Spec-legal NUT-09 shape, including nulls. Kept behind a switch + // because it is currently unusable with coco's cashu-ts version. + return this.json({ + outputs, + signatures: outputs.map( + (output) => this.signed.get(output.B_) ?? null, + ), + }); + } + // Return only the signed outputs; coco matches them by B_ and treats the + // rest as "nothing to restore". + const signed = outputs.filter((output) => this.signed.has(output.B_)); + return this.json({ + outputs: signed, + signatures: signed.map((output) => this.signed.get(output.B_)), + }); + } + + if (pathname === "/v1/checkstate" && request.method === "POST") { + const input = await body(); + const ys = (input.Ys ?? []) as string[]; + return this.json({ + states: ys.map((Y) => ({ Y, state: "UNSPENT", witness: null })), + }); + } + + return this.error(404, `fake mint has no route for ${pathname}`, 404); + } +} diff --git a/src/utils/history.test.ts b/src/utils/history.test.ts new file mode 100644 index 0000000..581ce28 --- /dev/null +++ b/src/utils/history.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "bun:test"; +import { HISTORY_ENTRY_TYPES, isHistoryEntryType } from "./history"; + +describe("history entry types", () => { + it("lists the four supported transaction types", () => { + expect([...HISTORY_ENTRY_TYPES]).toEqual([ + "mint", + "melt", + "send", + "receive", + ]); + }); + + it("recognizes known types and rejects unknown ones", () => { + expect(isHistoryEntryType("send")).toBe(true); + expect(isHistoryEntryType("melt")).toBe(true); + expect(isHistoryEntryType("SEND")).toBe(false); + expect(isHistoryEntryType("refund")).toBe(false); + }); +}); diff --git a/src/utils/history.ts b/src/utils/history.ts new file mode 100644 index 0000000..3640655 --- /dev/null +++ b/src/utils/history.ts @@ -0,0 +1,9 @@ +/** Transaction types understood by the wallet history type filter. */ +export const HISTORY_ENTRY_TYPES = ["mint", "melt", "send", "receive"] as const; + +export type HistoryEntryType = (typeof HISTORY_ENTRY_TYPES)[number]; + +/** True when `value` is a recognized history transaction type. */ +export function isHistoryEntryType(value: string): value is HistoryEntryType { + return (HISTORY_ENTRY_TYPES as readonly string[]).includes(value); +}