diff --git a/docs/wallet-mint-recovery.md b/docs/wallet-mint-recovery.md new file mode 100644 index 0000000..cb344ef --- /dev/null +++ b/docs/wallet-mint-recovery.md @@ -0,0 +1,71 @@ +# Mint quote recovery: scope and troubleshooting + +`routstrd wallet recover` explicitly retries mint operations through coco using +**their existing stored outputs**. It can restore signatures when a quote is +already issued, and reopen failed operations when explicitly requested: + +```sh +routstrd history --json +routstrd wallet recover --op --include-failed +``` + +Failed operations require explicit IDs over both HTTP and the CLI. A successful +re-run on an already finalized operation is a no-op. Requests that exceed their +wait budget are not cancelled; explicit retries skip the operation while the +underlying work is outstanding. + +## What this fixes—and what it does not + +Coco already checks pending quotes on startup and the daemon periodically +refreshes them. A quote paid while the daemon was offline does not, by itself, +require a new issuance implementation. + +This change makes normal cleanup confirm UNPAID with the mint before failing an +expired quote. PAID, ISSUED and unverified quotes remain pending. It also gives +operators a recovery path for operations previously marked failed. + +It does **not** replace rejected outputs with fresh outputs on an active keyset. +An inactive-keyset rejection can therefore remain retryable with zero recovery. +Recovery reports coco's persisted mint error when available, rather than only a +generic “remains pending” error. + +Do not infer that the production incidents were caused by keyset retirement. +Before claiming those incidents are fixed, collect: + +- The affected operation IDs, quote IDs, state and persisted `error`. +- A fresh remote quote state and, where provided, paid/issued amounts. +- The keyset IDs in the stored outputs and the mint's current keyset metadata. +- A reproduction showing existing recovery fails and the proposed fix succeeds. + +Inspect persisted operation data through a read-only database copy; do not edit +rows or run recovery scripts concurrently with a daemon against the same wallet. +Never share the mnemonic, output secrets, or full wallet database in a PR. + +A future fresh-output path must preserve original outputs for uncertain issuance +and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate +with coco's watcher/processor. It needs its own integration tests before handling +real funds. + +## Cleanup preview and force + +`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not +confirmed failures. `failedMintQuotes` and `leftForRecovery` are zero because no +mint check or cleanup transition was performed. Send/melt counts remain planned +cleanup counts in dry-run mode. + +`--force` deliberately bypasses mint confirmation and can strand paid sats in a +failed operation. Prefer normal cleanup. Forced operations can be retried with +`--op --include-failed`, but recovery still depends on the mint +accepting their stored outputs or restoring their signatures. + +## Integration and release notes + +The reopen helper uses private coco-core 1.0.1 methods. Retain real-Manager and +HTTP fake-mint coverage, use frozen dependency installs, and re-run integration +tests on coco upgrades. A controlled low-value live-mint smoke test remains +recommended before release. + +PR #118 removes `cocod-client.ts`. When integrating that change, move recovery +and cleanup contracts into its replacement `wallet-client.ts`, rename HTTP error +references accordingly, and make recovery mandatory for the in-process client. +This follow-up does not pull in #118's unrelated removal. diff --git a/src/cli.ts b/src/cli.ts index a2be293..badd231 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2062,16 +2062,22 @@ walletCmd .option("--mint-url ", "Only clean up operations for this mint URL") .option( "--min-age ", - "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)", + "Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are checked with their mint)", "168", ) .option("--dry-run", "Report what would be cleaned without applying changes", false) + .option( + "--force", + "Fail expired mint quotes without confirming UNPAID with the mint (may strand paid quotes)", + false, + ) .option("-y, --yes", "Skip confirmation prompt", false) .action( async (options: { mintUrl?: string; minAge: string; dryRun: boolean; + force: boolean; yes: boolean; }) => { const minAgeHours = Number.parseFloat(options.minAge); @@ -2087,7 +2093,9 @@ walletCmd }); const answer = await new Promise((resolve) => { rl.question( - "This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", + options.force + ? "WARNING: --force fails expired mint quotes WITHOUT checking the mint and may strand paid sats. It also reclaims old pending sends and cancels prepared melts. Continue? [y/N] " + : "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ", (value: string) => { rl.close(); resolve(value.trim().toLowerCase()); @@ -2109,6 +2117,7 @@ walletCmd mintUrl: options.mintUrl, minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000), dryRun: options.dryRun === true, + force: options.force === true, }, }); @@ -2121,6 +2130,8 @@ walletCmd | { dryRun?: boolean; failedMintQuotes?: number; + mintQuoteCandidates?: number; + leftForRecovery?: number; reclaimedSends?: number; cancelledMelts?: number; skipped?: number; @@ -2131,8 +2142,15 @@ walletCmd if (output) { const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:"; console.log(prefix); + if (output.dryRun) { + console.log( + ` Expired mint quote candidates (not checked with mint): ${output.mintQuoteCandidates ?? 0}`, + ); + } else { + console.log(` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`); + } console.log( - ` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`, + ` Expired quotes kept for recovery (paid/issued/unverified): ${output.leftForRecovery ?? 0}`, ); console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`); console.log( @@ -2163,6 +2181,124 @@ walletCmd }, ); +walletCmd + .command("recover") + .description( + "Retry mint quotes using their stored outputs (does not replace rejected outputs)", + ) + .option( + "--op ", + "Recover this operation id (repeatable; find IDs with routstrd history --json)", + (value: string, previous: string[]) => [...previous, value], + [] as string[], + ) + .option( + "--include-failed", + "Also re-open operations coco already gave up on (requires --op)", + false, + ) + .option("-y, --yes", "Skip confirmation prompt", false) + .action( + async (options: { + op: string[]; + includeFailed: boolean; + yes: boolean; + }) => { + const operationIds = options.op ?? []; + if (options.includeFailed && operationIds.length === 0) { + console.error( + "--include-failed can only target operations named with --op", + ); + process.exit(1); + } + + if (!options.yes) { + const rl = require("readline").createInterface({ + input: process.stdin, + output: process.stdout, + }); + const prompt = + operationIds.length > 0 + ? `Recover ${operationIds.length} mint quote operation(s)? [y/N] ` + : "Check every pending mint quote with its mint and claim any paid sats? [y/N] "; + const answer = await new Promise((resolve) => { + rl.question(prompt, (value: string) => { + rl.close(); + resolve(value.trim().toLowerCase()); + }); + }); + if (answer !== "y" && answer !== "yes") { + console.log("Aborted."); + return; + } + } + + try { + await ensureDaemonRunning(); + + const result = await callDaemon("/wallet/recover", { + method: "POST", + body: { + operationIds: operationIds.length > 0 ? operationIds : undefined, + includeFailed: options.includeFailed === true, + }, + }); + + if (result.error) { + console.log(result.error); + process.exit(1); + } + + const output = result.output as + | { + checked?: number; + recovered?: number; + waiting?: number; + terminal?: number; + reopened?: number; + retryable?: number; + busy?: number; + errors?: Array<{ operationId: string; error: string }>; + } + | undefined; + + if (output) { + console.log("Mint quote recovery:"); + console.log(` Checked with mint: ${output.checked ?? 0}`); + console.log( + ` Recovered (paid sats claimed): ${output.recovered ?? 0}`, + ); + console.log(` Still unpaid: ${output.waiting ?? 0}`); + console.log(` No longer issuable: ${output.terminal ?? 0}`); + console.log( + ` Re-opened failed operations: ${output.reopened ?? 0}`, + ); + console.log(` Left for a later run: ${output.retryable ?? 0}`); + console.log( + ` Skipped (recovery still running): ${output.busy ?? 0}`, + ); + if (output.errors && output.errors.length > 0) { + console.log("\nErrors:"); + for (const e of output.errors) { + console.log(` - ${e.operationId}: ${e.error}`); + } + } + } + } catch (error) { + const message = (error as Error).message; + if ( + message?.includes("fetch failed") || + message?.includes("Connection refused") + ) { + console.error("Daemon is not running"); + process.exit(1); + } + console.error(message); + process.exit(1); + } + }, + ); + const walletReceiveCmd = walletCmd .command("receive") .description("Wallet receive operations"); diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 8979aa7..04d15e9 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -283,6 +283,21 @@ function optionalStringField( return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function optionalStringArrayField( + body: Record, + field: string, +): string[] | undefined { + const value = body[field]; + if (value === undefined) return undefined; + if ( + !Array.isArray(value) || + value.some((item) => typeof item !== "string" || !item.trim()) + ) { + throw new CocodHttpError(400, `'${field}' must be an array of non-empty strings.`); + } + return value.map((item: string) => item.trim()); +} + function getCurrentMode(deps: DaemonDeps): ClientMode { const stateMode = deps.store.getState()?.mode; return stateMode || deps.mode || "apikeys"; @@ -513,6 +528,44 @@ export function createDaemonRequestHandler(deps: { ? body.minAgeMs : undefined, dryRun: body.dryRun === true, + force: body.force === true, + }); + return { output: result }; + }); + return; + } + + if (req.method === "POST" && url.pathname === "/wallet/recover") { + await respond(res, async () => { + if (!deps.walletClient.recoverMintQuotes) { + throw new CocodHttpError( + 501, + "Mint quote recovery is not supported by this wallet client.", + ); + } + + const body = await readJsonBody(req); + const operationIds = optionalStringArrayField(body, "operationIds"); + if (body.includeFailed === true && !operationIds?.length) { + throw new CocodHttpError( + 400, + "'includeFailed' requires non-empty 'operationIds'.", + ); + } + if ( + body.timeoutMs !== undefined && + (typeof body.timeoutMs !== "number" || + !Number.isFinite(body.timeoutMs) || body.timeoutMs <= 0) + ) { + throw new CocodHttpError( + 400, + "'timeoutMs' must be a positive finite number.", + ); + } + const result = await deps.walletClient.recoverMintQuotes({ + operationIds, + includeFailed: body.includeFailed === true, + timeoutMs: body.timeoutMs as number | undefined, }); return { output: result }; }); diff --git a/src/daemon/http/wallet-recovery.test.ts b/src/daemon/http/wallet-recovery.test.ts new file mode 100644 index 0000000..b49a984 --- /dev/null +++ b/src/daemon/http/wallet-recovery.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, mock } from "bun:test"; +import { EventEmitter } from "node:events"; +import { createDaemonRequestHandler } from "./index"; + +async function recover(body: unknown) { + const recoverMintQuotes = mock(async (_options: unknown) => ({ recovered: 0 })); + const handler = createDaemonRequestHandler({ walletClient: { recoverMintQuotes } } as never); + const req = new EventEmitter() as any; + Object.assign(req, { method: "POST", url: "/wallet/recover", headers: { host: "localhost" } }); + const res = { + status: 0, body: "", + writeHead(status: number) { this.status = status; }, + end(chunk: string) { this.body = chunk; }, + }; + setImmediate(() => { + req.emit("data", Buffer.from(JSON.stringify(body))); + req.emit("end"); + }); + await handler(req, res as never); + return { res, recoverMintQuotes }; +} + +describe("POST /wallet/recover validation", () => { + it.each([{}, { operationIds: [] }])("rejects includeFailed without explicit IDs: %j", async (body) => { + const { res, recoverMintQuotes } = await recover({ ...body, includeFailed: true }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([[""], [" "], [42]])("rejects invalid operation IDs: %j", async (operationIds) => { + const { res, recoverMintQuotes } = await recover({ operationIds }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it.each([0, -1, "1000"])("rejects invalid timeout %j", async (timeoutMs) => { + const { res, recoverMintQuotes } = await recover({ timeoutMs }); + expect(res.status).toBe(400); + expect(recoverMintQuotes).not.toHaveBeenCalled(); + }); + it("passes normalized explicit IDs and a positive timeout", async () => { + const { res, recoverMintQuotes } = await recover({ operationIds: [" op-1 "], includeFailed: true, timeoutMs: 1000 }); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledWith({ operationIds: ["op-1"], includeFailed: true, timeoutMs: 1000 }); + }); + it("still permits checking pending quotes without IDs", async () => { + const { res, recoverMintQuotes } = await recover({}); + expect(res.status).toBe(200); + expect(recoverMintQuotes).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/daemon/wallet/cleanup.test.ts b/src/daemon/wallet/cleanup.test.ts index 381ab4e..b7a8568 100644 --- a/src/daemon/wallet/cleanup.test.ts +++ b/src/daemon/wallet/cleanup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; const NOW_MS = 1_800_000_000_000; const DAY_MS = 24 * 60 * 60 * 1000; @@ -166,3 +166,14 @@ describe("selectCleanupOperations", () => { expect(result.meltsToCancel).toEqual([]); }); }); + +describe("mint cleanup reporting", () => { + it("reports dry-run candidates, not confirmed failures", () => { + expect(summarizeMintCleanup({ dryRun: true, candidates: 3, failed: 0, leftForRecovery: 0 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 0, leftForRecovery: 0 }); + }); + it("reports only actual failures in a real run", () => { + expect(summarizeMintCleanup({ dryRun: false, candidates: 3, failed: 1, leftForRecovery: 2 })) + .toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 1, leftForRecovery: 2 }); + }); +}); diff --git a/src/daemon/wallet/cleanup.ts b/src/daemon/wallet/cleanup.ts index bbf24c2..1ff48ea 100644 --- a/src/daemon/wallet/cleanup.ts +++ b/src/daemon/wallet/cleanup.ts @@ -64,8 +64,8 @@ export interface CleanupSelection< * can have happened before expiry while the daemon was down, leaving no * local observation. Callers that fail quotes automatically at startup must * therefore confirm UNPAID with the mint first (see - * settleExpiredMintQuotes in coco-client.ts); only the explicit, - * user-invoked cleanup command may fail candidates purely locally. + * failExpiredMintQuoteIfUnpaid in coco-client.ts). Explicit cleanup follows + * the same rule unless the operator opts into unsafe `--force` behaviour. * - Pending sends are reclaimed (rolled back) only when they are older than * `minAgeMs`, so we never roll back a token that a receiver might still * legitimately claim. @@ -102,3 +102,17 @@ export function selectCleanupOperations< return { mintsToFail, sendsToReclaim, meltsToCancel }; } + +/** Keep a local-only dry-run preview distinct from mint-confirmed outcomes. */ +export function summarizeMintCleanup(input: { + dryRun: boolean; + candidates: number; + failed: number; + leftForRecovery: number; +}) { + return { + mintQuoteCandidates: input.candidates, + failedMintQuotes: input.dryRun ? 0 : input.failed, + leftForRecovery: input.dryRun ? 0 : input.leftForRecovery, + }; +} diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index 959e798..e18e7c2 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -14,12 +14,17 @@ import { assertLegacyCocodNotRunning, claimLegacyCocodPidFile, createCocoClient, + createRunQueue, DEFAULT_TRUSTED_MINT_URLS, + failExpiredMintQuoteIfUnpaid, isZombieProcess, + reopenFailedMintOperation, + runMintQuoteRecovery, settleExpiredMintQuotes, settlePendingMintQuotes, stopLegacyCocod, type ExpiredMintQuoteSource, + type MintQuoteRecoverySource, type PendingMintQuoteSource, type PendingMintSweepState, } from "./coco-client"; @@ -901,3 +906,731 @@ describe("settlePendingMintQuotes", () => { expect(logged.mock.calls[0]?.[0]).toContain("21 sat minted"); }); }); + +describe("createRunQueue", () => { + it("runs tasks strictly one after another", async () => { + const enqueue = createRunQueue(); + const order: string[] = []; + let active = 0; + let maxActive = 0; + const task = (name: string, delay: number) => async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`${name}:start`); + await new Promise((resolve) => setTimeout(resolve, delay)); + order.push(`${name}:end`); + active--; + return name; + }; + + const results = await Promise.all([ + enqueue(task("a", 20)), + enqueue(task("b", 1)), + enqueue(task("c", 1)), + ]); + + expect(results).toEqual(["a", "b", "c"]); + expect(maxActive).toBe(1); + expect(order).toEqual([ + "a:start", + "a:end", + "b:start", + "b:end", + "c:start", + "c:end", + ]); + }); + + it("keeps the chain alive after a rejected task", async () => { + const enqueue = createRunQueue(); + + const failed = enqueue(async () => { + throw new Error("boom"); + }); + const next = enqueue(async () => "ok"); + + await expect(failed).rejects.toThrow("boom"); + expect(await next).toBe("ok"); + }); +}); + +describe("failExpiredMintQuoteIfUnpaid", () => { + function fakeMintService(observe: (id: string) => Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>) { + const failPendingOperation = mock( + async ( + _op: { id: string }, + _failure: { reason: string; retryable?: boolean; observedAt: number }, + ) => ({}), + ); + return { + mintService: { + observePendingOperation: mock(observe), + failPendingOperation, + }, + failPendingOperation, + }; + } + + it("fails a quote its mint confirms unpaid", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => ({ + category: "waiting", + })); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("failed"); + expect(failPendingOperation).toHaveBeenCalledTimes(1); + expect(failPendingOperation.mock.calls[0]?.[1]?.reason).toContain( + "confirmed unpaid by mint", + ); + }); + + it.each(["ready", "completed", "terminal"] as const)( + "leaves a quote observed as %s for recovery", + async (category) => { + const { mintService, failPendingOperation } = fakeMintService( + async () => ({ category }), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result).toEqual({ outcome: "leftForRecovery", category }); + expect(failPendingOperation).not.toHaveBeenCalled(); + }, + ); + + it("leaves a quote pending when the mint cannot be reached", async () => { + const { mintService, failPendingOperation } = fakeMintService(async () => { + throw new Error("Network request failed"); + }); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); + + it("gives up waiting on a hung mint without failing the quote", async () => { + const { mintService, failPendingOperation } = fakeMintService( + () => new Promise(() => {}), + ); + + const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 20); + + expect(result.outcome).toBe("unobserved"); + expect(failPendingOperation).not.toHaveBeenCalled(); + }); +}); + + +describe("reopenFailedMintOperation", () => { + /** + * Mirrors coco's OperationIdLock, which is fail-fast: acquiring an id that is + * already locked throws OperationInProgressError instead of waiting. + */ + function makeLock() { + let held = false; + return { + get held() { + return held; + }, + async acquire() { + if (held) { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + } + held = true; + return () => { + held = false; + }; + }, + }; + } + + function fakeService( + current: Record | null, + hooks: { + lock?: ReturnType; + onWrite?: (lock: ReturnType) => void; + } = {}, + ) { + const lock = hooks.lock ?? makeLock(); + const transitionToPending = mock( + async (_op: Record, _error?: string) => { + hooks.onWrite?.(lock); + return {}; + }, + ); + return { + service: { + acquireOperationLock: mock(async (_id: string) => lock.acquire()), + getOperation: mock(async (_id: string) => current), + transitionToPending, + }, + transitionToPending, + lock, + }; + } + + it("re-opens a failed operation with the full persisted row", async () => { + // coco spreads whatever it is handed and the sqlite repository rewrites + // every column, so a partial object would erase the stored outputs. + const row = { + id: "op-1", + state: "failed", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + method: "bolt11", + amount: 210_000, + unit: "sat", + request: "lnbc...", + expiry: 1_800_000_000, + outputDataJson: "[{\"secret\":\"abc\"}]", + terminalFailure: { reason: "expired" }, + }; + const { service, transitionToPending } = fakeService(row); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(transitionToPending).toHaveBeenCalledTimes(1); + const passed = transitionToPending.mock.calls[0]?.[0]; + expect(passed).toMatchObject({ + id: "op-1", + quoteId: "quote-1", + amount: 210_000, + unit: "sat", + outputDataJson: "[{\"secret\":\"abc\"}]", + }); + // The stale terminal marker must not survive the re-open. + expect(passed?.terminalFailure).toBeUndefined(); + }); + + it("does nothing when the operation is no longer failed", async () => { + const { service, transitionToPending, lock } = fakeService({ + id: "op-1", + state: "finalized", + }); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + expect(transitionToPending).not.toHaveBeenCalled(); + // The lock must be released even on the no-op path. + expect(lock.held).toBe(false); + }); + + it("throws when the operation is missing", async () => { + const { service, lock } = fakeService(null); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + "not found", + ); + expect(lock.held).toBe(false); + }); + + it("fails closed when coco no longer exposes the operation lock", async () => { + const transitionToPending = mock( + async (_op: Record, _error?: string) => ({}), + ); + const service = { + getOperation: mock(async () => ({ id: "op-1", state: "failed" })), + transitionToPending, + } as unknown as Parameters[0]; + + await expect( + reopenFailedMintOperation(service, "op-1"), + ).rejects.toThrow("acquireOperationLock"); + expect(transitionToPending).not.toHaveBeenCalled(); + }); + + it("holds the operation lock across read-check-write", async () => { + const lock = makeLock(); + const order: string[] = []; + const service = { + acquireOperationLock: mock(async (_id: string) => { + order.push("lock"); + const release = await lock.acquire(); + return () => { + order.push("unlock"); + release(); + }; + }), + getOperation: mock(async (_id: string) => { + expect(lock.held).toBe(true); + order.push("read"); + return { id: "op-1", state: "failed", quoteId: "quote-1" }; + }), + transitionToPending: mock(async () => { + expect(lock.held).toBe(true); + order.push("write"); + return {}; + }), + }; + + const reopened = await reopenFailedMintOperation(service, "op-1"); + + expect(reopened).toBe(true); + expect(order).toEqual(["lock", "read", "write", "unlock"]); + expect(lock.held).toBe(false); + }); + + it("refuses to re-open while the operation lock is held elsewhere", async () => { + const lock = makeLock(); + const release = await lock.acquire(); + const { service, transitionToPending } = fakeService( + { id: "op-1", state: "failed" }, + { lock }, + ); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /in progress/, + ); + expect(transitionToPending).not.toHaveBeenCalled(); + release(); + }); +}); + +describe("runMintQuoteRecovery", () => { + function mintOp(overrides: Record = {}) { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state: "pending", + amount: 210_000, + expiry: 0, + ...overrides, + }; + } + + function fakeSource( + ops: Array>, + behavior: { + observe?: (id: string) => Promise<{ + category: "waiting" | "ready" | "completed" | "terminal"; + }>; + finalize?: (id: string) => Promise; + reopen?: (id: string) => Promise; + } = {}, + ) { + const finalize = mock( + behavior.finalize ?? + (async (_id: string) => ({ state: "finalized" })), + ); + const observePendingOperation = mock( + behavior.observe ?? (async () => ({ category: "waiting" as const })), + ); + const reopenFailedOperation = mock( + behavior.reopen ?? (async (_id: string) => true), + ); + const byId = new Map(ops.map((op) => [op.id as string, op])); + const source = { + ops: { + mint: { + listPending: async () => + ops.filter( + (op) => op.state === "pending" || op.state === "executing", + ), + get: async (id: string) => byId.get(id) ?? null, + finalize, + }, + }, + mintOperationService: { observePendingOperation }, + reopenFailedOperation, + } as unknown as MintQuoteRecoverySource; + return { source, finalize, observePendingOperation, reopenFailedOperation }; + } + + it("mints the stored outputs for a quote the mint reports PAID", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 1, waiting: 0 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(finalize.mock.calls[0]?.[0]).toBe("op-1"); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("leaves an unpaid quote pending", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "waiting" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 1, recovered: 0, waiting: 1 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports a quote the mint can no longer issue", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "terminal" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ terminal: 1, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("retries later when the mint is unreachable", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => { + throw new Error("fetch failed"); + }, + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toHaveLength(1); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("does not count a failed finalize as a recovery", async () => { + // coco returns a terminal operation instead of throwing when the mint + // refuses, so a fulfilled finalize is not evidence that sats were claimed. + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ + state: "failed", + error: "Recovered: quote quote-1 expired while executing mint", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors[0]?.error).toContain("expired"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("does not count a finalized-with-error operation as a recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "completed" }), + finalize: async () => ({ + state: "finalized", + error: "Recovered issued quote quote-1 but no proofs could be restored", + }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(result.errors).toHaveLength(1); + }); + + it("falls back to the original failure when diagnostic lookup fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = async () => { throw new Error("lookup failed"); }; + const result = await runMintQuoteRecovery(source); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + + it("bounds a hung diagnostic lookup after finalize fails", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { throw new Error("original failure"); }, + }); + source.ops.mint.get = () => new Promise(() => {}); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]); + }); + + it("surfaces the persisted mint error even when the mint budget is nearly spent", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => { + // Consume almost the whole per-op mint budget before throwing: exactly + // the slow-mint case where a diagnostic bound to remaining() would + // starve and silently fall back to the generic message. + await new Promise((resolve) => setTimeout(resolve, 25)); + throw new Error("remains pending"); + }, + }); + source.ops.mint.get = async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { + ...mintOp(), + state: "pending", + error: "keyset id inactive.", + }; + }; + const result = await runMintQuoteRecovery(source, { timeoutMs: 30 }); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + expect(result.errors).toEqual([ + { operationId: "op-1", error: "keyset id inactive." }, + ]); + }); + + it("bounds finalize so one hung mint cannot block recovery", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const started = Date.now(); + const result = await runMintQuoteRecovery(source, { timeoutMs: 20 }); + + expect(Date.now() - started).toBeLessThan(5_000); + expect(result).toMatchObject({ retryable: 1, recovered: 0 }); + }); + + it("recovers an interrupted mint without re-checking the quote", async () => { + const { source, finalize, observePendingOperation } = fakeSource([ + mintOp({ state: "executing" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 1 }); + expect(finalize).toHaveBeenCalledTimes(1); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it("skips failed operations unless the caller opts in", async () => { + const { source, reopenFailedOperation } = fakeSource([ + mintOp({ state: "failed", lastObservedRemoteState: "PAID" }), + ]); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ checked: 0, recovered: 0, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("requires explicit IDs when including failed operations", async () => { + const { source, reopenFailedOperation, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { includeFailed: true })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + await expect(runMintQuoteRecovery(source, { includeFailed: true, operationIds: [] })).rejects.toThrow( + "includeFailed requires explicit operationIds", + ); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it.each([0, -1, NaN, Infinity])("rejects invalid recovery timeout %s", async (timeoutMs) => { + const { source, observePendingOperation } = fakeSource([]); + await expect(runMintQuoteRecovery(source, { timeoutMs })).rejects.toThrow( + "timeoutMs must be a positive finite number", + ); + expect(observePendingOperation).not.toHaveBeenCalled(); + }); + + it("re-opens a named failed operation, then mints it", async () => { + const { source, reopenFailedOperation, finalize } = fakeSource( + [mintOp({ state: "failed", lastObservedRemoteState: "PAID" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledWith("op-1"); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("re-opens a named failed operation even without a PAID observation", async () => { + // The old local-fail bug left quotes with a stale or missing observation, + // which is exactly when an operator needs to retry them. + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 1, recovered: 1 }); + expect(reopenFailedOperation).toHaveBeenCalledTimes(1); + }); + + it("skips an operation that is no longer failed when re-opened", async () => { + const { source, finalize } = fakeSource( + [mintOp({ state: "failed" })], + { reopen: async () => false }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ reopened: 0, checked: 0, recovered: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("reports an unknown operation id instead of throwing", async () => { + const { source } = fakeSource([]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["missing"], + }); + + expect(result.checked).toBe(0); + expect(result.errors).toEqual([ + { operationId: "missing", error: "operation not found" }, + ]); + }); + + it("deduplicates repeated operation ids", async () => { + const { source, finalize } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + }); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1", "op-1"], + }); + + expect(result.checked).toBe(1); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("ignores finalized operations even when targeted", async () => { + const { source, finalize } = fakeSource([mintOp({ state: "finalized" })]); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + }); + + expect(result.checked).toBe(0); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("leaves a non-terminal finalize result for a later run, not terminal", async () => { + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: async () => ({ state: "pending" }), + }); + + const result = await runMintQuoteRecovery(source); + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(result.errors[0]?.error).toContain("will retry"); + }); + + it("skips operations whose earlier recovery is still in flight", async () => { + const outstanding = new Map>([ + ["op-1", new Promise(() => {})], + ]); + const { source, finalize, observePendingOperation } = fakeSource( + [mintOp()], + { observe: async () => ({ category: "ready" }) }, + ); + + const result = await runMintQuoteRecovery(source, { outstanding }); + + expect(result).toMatchObject({ busy: 1, checked: 0, recovered: 0 }); + expect(observePendingOperation).not.toHaveBeenCalled(); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("keeps a timed-out finalize registered so a retry waits", async () => { + const outstanding = new Map>(); + const { source } = fakeSource([mintOp()], { + observe: async () => ({ category: "ready" }), + finalize: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has("op-1")).toBe(true); + // The abandoned mint request must not be retried underneath. + expect(second).toMatchObject({ busy: 1, checked: 0 }); + }); + + it("does not re-open a failed operation whose recovery is in flight", async () => { + const outstanding = new Map>([ + ["op-1", new Promise(() => {})], + ]); + const { source, reopenFailedOperation } = fakeSource( + [mintOp({ state: "failed" })], + {}, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + outstanding, + }); + + expect(result).toMatchObject({ busy: 1, reopened: 0 }); + expect(reopenFailedOperation).not.toHaveBeenCalled(); + }); + + it("counts an in-progress operation as busy rather than retryable", async () => { + const { source } = fakeSource( + [mintOp({ state: "failed" })], + { + reopen: async () => { + const error = new Error("Operation op-1 is already in progress"); + error.name = "OperationInProgressError"; + throw error; + }, + }, + ); + + const result = await runMintQuoteRecovery(source, { + operationIds: ["op-1"], + includeFailed: true, + }); + + expect(result).toMatchObject({ busy: 1, retryable: 0, reopened: 0 }); + }); + + it("keeps a timed-out quote check registered so a retry waits", async () => { + // observePendingOperation is not read-only, so a hung check must not be + // retried underneath: it could persist a stale observation later. + const outstanding = new Map>(); + const { source, finalize } = fakeSource([mintOp()], { + observe: () => new Promise(() => {}), + }); + + const first = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + const second = await runMintQuoteRecovery(source, { + timeoutMs: 20, + outstanding, + }); + + expect(first).toMatchObject({ retryable: 1, checked: 1 }); + expect(outstanding.has("op-1")).toBe(true); + expect(second).toMatchObject({ busy: 1, checked: 0 }); + expect(finalize).not.toHaveBeenCalled(); + }); +}); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 6fce529..e0bc59e 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -36,7 +36,12 @@ import type { WalletCleanupResult, WalletRecoveryProgress, } from "./cocod-client"; -import { selectCleanupOperations } from "./cleanup"; +import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; import { clearInterruptedReceiveReservations, deleteReceiveTokenReservation, @@ -579,6 +584,96 @@ interface MintOperationServiceCleanup { observePendingOperation( operationId: string, ): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>; + /** + * Acquire coco's per-operation lock for `operationId` and return its release + * function. coco's execute/finalize/recover paths take the same lock, so + * holding it across a read-check-write makes the transition atomic with + * respect to them. + */ + acquireOperationLock(operationId: string): Promise<() => void>; + /** Reload a mint operation row, or null when it no longer exists. */ + getOperation(operationId: string): Promise | null>; + /** + * Put a terminally failed operation back into `pending`. + * + * coco keeps this private, and it spreads whatever it is handed into the row + * it writes. The sqlite repository rewrites every column, so callers MUST + * pass a freshly reloaded full row: a partial object such as `{ id }` would + * erase `outputDataJson` and make the paid sats unrecoverable. + */ + transitionToPending( + op: Record, + error?: string, + ): Promise; +} + +/** + * Re-open a terminally failed mint operation so recovery can retry it. + * + * Two details make this safe: + * + * - The persisted row is reloaded and handed to coco in full. coco spreads + * whatever it is given and the sqlite repository rewrites every column, so a + * partial object would be rejected by the NOT NULL schema or, on a more + * permissive adapter, erase the stored outputs. + * - The read-check-write runs under coco's per-operation lock, the same lock + * coco's execute/finalize/recover paths take. Reloading alone only narrows + * the race: without the lock two concurrent recoveries could both see + * `failed` and the slower one would clobber a newer state. + * + * The lock is fail-fast rather than wait-based: coco's `OperationIdLock.acquire` + * throws `OperationInProgressError` when the id is already locked. So either + * this helper holds the lock - and coco's own execute/finalize/recover paths + * cannot interleave, because acquiring would throw for them too - or it throws + * and writes nothing. It never waits, and never writes without the lock, which + * is why a stale `failed` snapshot cannot clobber a newer state. + * + * Scope of that lock, in this coco version: `recordPendingObservation` and + * `failPendingOperation` write without taking it. The justified claim is + * therefore narrow - a re-open cannot clobber a concurrent executing/recovery + * pass - not a general guarantee against every watcher write. + * + * This is a compatibility shim over private coco internals, so it fails closed: + * if any of the expected methods are missing it throws before writing. That + * check only catches removals, not changed behaviour under the same name: it + * was written against @cashu/coco-core 1.0.1, so any coco bump must re-run the + * real-Manager and fake-mint integration tests. The long-term fix is an + * upstream public `reopenFailedOperation(id)` that takes the same lock, reloads + * the full row, preserves the outputs and emits the usual events. + */ +export async function reopenFailedMintOperation( + service: Pick< + MintOperationServiceCleanup, + "acquireOperationLock" | "getOperation" | "transitionToPending" + >, + operationId: string, +): Promise { + for (const method of [ + "acquireOperationLock", + "getOperation", + "transitionToPending", + ] as const) { + if (typeof service[method] !== "function") { + throw new Error( + `coco mintOperationService.${method} is unavailable; refusing to re-open a failed mint operation`, + ); + } + } + const release = await service.acquireOperationLock(operationId); + try { + const current = await service.getOperation(operationId); + if (!current) throw new Error(`Operation ${operationId} not found`); + if (current.state !== "failed") return false; + // Clearing the terminal-failure marker keeps the re-opened row from + // looking terminally failed to readers that inspect it alongside `state`. + await service.transitionToPending( + { ...current, terminalFailure: undefined }, + undefined, + ); + return true; + } finally { + release(); + } } export interface CreateCocoClientOptions { @@ -671,6 +766,57 @@ export interface ExpiredMintSettlement { unobserved: number; } +/** Outcome of asking a mint about one expired pending quote. */ +export type ExpiredMintQuoteOutcome = + | "failed" + | "leftForRecovery" + | "unobserved"; + +/** + * Decide one expired pending quote's fate by asking the mint. + * + * Expiry alone does not prove the quote was never paid: the Lightning payment + * can land just before expiry while the daemon is down, leaving no local + * observation. A quote the mint still reports UNPAID can never be issued and + * is safe to fail locally; anything else (PAID/ISSUED, or a mint that cannot + * answer) stays pending so recovery can still claim the sats. + */ +export async function failExpiredMintQuoteIfUnpaid( + mintService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" | "failPendingOperation" + >, + operationId: string, + timeoutMs: number, +): Promise<{ + outcome: ExpiredMintQuoteOutcome; + category?: "waiting" | "ready" | "completed" | "terminal"; + error?: unknown; +}> { + try { + const observation = await withTimeout( + mintService.observePendingOperation(operationId), + timeoutMs, + ); + if (observation.category !== "waiting") { + return { outcome: "leftForRecovery", category: observation.category }; + } + // The mint confirms the expired quote is still unpaid: it can never be + // issued now, so failing it locally cannot strand funds. + await mintService.failPendingOperation( + { id: operationId }, + { + reason: "Expired mint quote confirmed unpaid by mint", + retryable: false, + observedAt: Date.now(), + }, + ); + return { outcome: "failed", category: observation.category }; + } catch (error) { + return { outcome: "unobserved", error }; + } +} + /** * Settle expired pending mint quotes before the mint recovery sweep runs. * @@ -726,45 +872,37 @@ export async function settleExpiredMintQuotes( break; } - try { - const result = await withTimeout( - source.mintOperationService.observePendingOperation(op.id), - remainingMs, + const check = await failExpiredMintQuoteIfUnpaid( + source.mintOperationService, + op.id, + remainingMs, + ); + if (check.outcome === "failed") { + settlement.failed++; + } else if (check.outcome === "leftForRecovery") { + // PAID/ISSUED (or terminally failed) at the mint: normal recovery + // must see this quote so paid proofs get claimed. + settlement.leftForRecovery++; + const observed = + check.category === "ready" + ? "was paid at the mint" + : check.category === "completed" + ? "was already issued at the mint" + : "failed terminally at the mint"; + startupProgress( + `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, ); - if (result.category === "waiting") { - // The mint confirms the expired quote is still unpaid: it can never - // be issued now, so failing it locally cannot strand funds. - await source.mintOperationService.failPendingOperation( - { id: op.id }, - { - reason: "Expired mint quote confirmed unpaid by mint", - retryable: false, - observedAt: Date.now(), - }, - ); - settlement.failed++; - } else { - // PAID/ISSUED (or terminally failed) at the mint: normal recovery - // must see this quote so paid proofs get claimed. - settlement.leftForRecovery++; - const observed = - result.category === "ready" - ? "was paid at the mint" - : result.category === "completed" - ? "was already issued at the mint" - : "failed terminally at the mint"; - startupProgress( - `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`, - ); - } - } catch (error) { + } else { // Mint unreachable, too slow, or the quote unknown to it: leave the // operation pending so a later startup can still recover it. settlement.unobserved++; logger.warn("Could not check expired mint quote; leaving it pending", { operationId: op.id, mintUrl: op.mintUrl, - error: error instanceof Error ? error.message : String(error), + error: + check.error instanceof Error + ? check.error.message + : String(check.error), }); } } @@ -772,6 +910,371 @@ export async function settleExpiredMintQuotes( return settlement; } +/** + * Source for explicit PAID mint-quote recovery. + * + * Unlike the startup sweeps this also accepts caller-supplied operation ids so + * an operator can target a quote coco already gave up on (state `failed`). + */ +export interface MintQuoteRecoverySource { + ops: { + mint: { + listPending(): Promise; + get(operationId: string): Promise; + finalize(operationId: string): Promise; + }; + }; + mintOperationService: Pick< + MintOperationServiceCleanup, + "observePendingOperation" + >; + /** + * Re-open a failed operation so it can be recovered; false when it is no + * longer failed. Implementations must reload the full row (see + * `reopenFailedMintOperation`). + */ + reopenFailedOperation(operationId: string): Promise; +} + +export interface MintQuoteRecoveryOptions { + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Per-quote budget for observing the mint and finalizing the operation. */ + timeoutMs?: number; + /** + * Re-open failed operations instead of skipping them. Only applies to + * operations named by `operationIds`: coco's pending listing never returns + * failed operations, so they can only be recovered by explicit id. + */ + includeFailed?: boolean; + /** + * In-flight recovery work keyed by operation id, shared across runs. + * withTimeout does not cancel the underlying request, so a timed-out quote + * check or finalize must keep blocking a retry until it actually settles. + */ + outstanding?: Map>; +} + +export interface MintQuoteRecoveryResult { + /** Operations recovery acted on. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** + * Quotes that ended terminally: the mint can no longer issue them, or coco + * finalised them without recovering any proofs. + */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** + * Operations left to a later run: the mint was unreachable, the per-quote + * budget ran out, or the operation ended in a non-terminal state. + */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + +/** + * Run async tasks strictly one after another. + * + * Used to serialize explicit wallet recovery: two concurrent requests must not + * both snapshot the same failed operation, and a retry must not start + * underneath work that outlived its timeout. A rejected task never breaks the + * chain for the next one. + */ +export function createRunQueue(): (run: () => Promise) => Promise { + let tail: Promise = Promise.resolve(); + return (run: () => Promise): Promise => { + const result = tail.then(run, run); + tail = result.then( + () => undefined, + () => undefined, + ); + return result; + }; +} + +/** Per-quote budget for the mint round-trip during explicit recovery. */ +const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000; + +/** + * Bound for the local post-finalize diagnostic read. This is a database + * lookup, not a mint round-trip, so it gets its own small budget: the per-op + * mint budget is often already spent when finalize throws, and a starved + * diagnostic would silently fall back to the generic error message. + */ +const DIAGNOSTIC_LOOKUP_TIMEOUT_MS = 250; + +/** + * Recover mint quotes whose sats are PAID at the mint but were never claimed. + * + * For every target the mint is asked for the current quote state, and only it + * decides the outcome: PAID quotes have their stored outputs submitted, ISSUED + * quotes have their signatures restored (NUT-09), UNPAID quotes are left + * pending, and quotes the mint can no longer issue are reported rather than + * silently dropped. Anything the mint cannot answer is retried later. + * + * `finalize()` does not throw when the mint refuses to issue or when an + * already-issued quote's proofs cannot be restored: it returns a terminal + * operation instead. Recovery therefore inspects the returned operation's + * state and error and only counts a genuine finalized-without-error as + * recovered. + * + * Failed operations are skipped unless `includeFailed` is set, and they can + * only be targeted by explicit id because coco's pending listing never returns + * them. Re-opening an operation is a mutation, so it happens only here, never + * during startup recovery. + * + * Callers should serialize their own invocations and pass a shared + * `outstanding` map: `timeoutMs` bounds the wait but does not cancel the + * request behind it, so both a timed-out quote check and a timed-out finalize + * keep blocking a retry until they actually settle. + */ +export async function runMintQuoteRecovery( + source: MintQuoteRecoverySource, + options: MintQuoteRecoveryOptions = {}, + onProgress?: (message: string) => void, +): Promise { + if (options.includeFailed && !options.operationIds?.length) { + throw new Error("includeFailed requires explicit operationIds"); + } + const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error("timeoutMs must be a positive finite number"); + } + const outstanding = + options.outstanding ?? new Map>(); + const result: MintQuoteRecoveryResult = { + checked: 0, + recovered: 0, + waiting: 0, + terminal: 0, + reopened: 0, + retryable: 0, + busy: 0, + errors: [], + }; + const messageOf = (error: unknown) => + error instanceof Error ? error.message : String(error); + /** coco's fail-fast operation lock rejected the call: another holder exists. */ + const isInProgress = (error: unknown) => + error instanceof Error && error.name === "OperationInProgressError"; + /** + * Register in-flight work for an operation. Entries are cleared only once the + * work actually settles (withTimeout does not cancel the request behind it), + * so a timed-out call keeps blocking a retry. The identity check stops a late + * settlement from clearing a newer entry for the same operation. + */ + const track = (operationId: string, work: Promise) => { + outstanding.set(operationId, work); + const clear = () => { + if (outstanding.get(operationId) === work) { + outstanding.delete(operationId); + } + }; + void work.then(clear, clear); + }; + + let targets: MintQuoteRecoveryCandidate[]; + if (options.operationIds && options.operationIds.length > 0) { + targets = []; + const seen = new Set(); + for (const operationId of options.operationIds) { + if (seen.has(operationId)) continue; + seen.add(operationId); + try { + const op = await source.ops.mint.get(operationId); + if (!op) { + result.errors.push({ operationId, error: "operation not found" }); + continue; + } + targets.push(op); + } catch (error) { + result.errors.push({ operationId, error: messageOf(error) }); + } + } + } else { + targets = await source.ops.mint.listPending(); + } + + const { pending, failed } = selectMintQuotesForRecovery({ + mints: targets, + includeFailed: options.includeFailed === true, + }); + + for (const op of failed) { + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(op.id)) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + continue; + } + try { + if (!(await source.reopenFailedOperation(op.id))) { + onProgress?.(`${label}: no longer failed; skipped`); + continue; + } + result.reopened++; + onProgress?.(`${label}: re-opened failed operation for recovery`); + } catch (error) { + // coco's operation lock is fail-fast, so an in-progress error means a + // processor or another recovery holds the operation right now. + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery holds it; skipped`); + } else { + result.retryable++; + onProgress?.(`${label}: could not re-open: ${messageOf(error)}`); + } + result.errors.push({ operationId: op.id, error: messageOf(error) }); + continue; + } + await recoverOne(op); + } + + for (const op of pending) await recoverOne(op); + + return result; + + async function recoverOne(op: MintQuoteRecoveryCandidate): Promise { + const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`; + if (outstanding.has(op.id)) { + result.busy++; + onProgress?.(`${label}: an earlier recovery is still running; skipped`); + return; + } + result.checked++; + // One budget per operation, shared by the mint check and the finalize, so + // a slow mint cannot silently double the documented per-quote wait. The + // local post-finalize diagnostic read is exempt (DIAGNOSTIC_LOOKUP_TIMEOUT_MS). + const deadlineAt = Date.now() + timeoutMs; + const remaining = () => Math.max(1, deadlineAt - Date.now()); + + if (op.state === "executing") { + // A crash mid-mint can leave outputs already signed at the mint; + // finalize recovers them instead of minting a second time. + await finalizeAndClassify( + op.id, + label, + "recovered interrupted mint", + remaining, + ); + return; + } + + let observation: { + category: "waiting" | "ready" | "completed" | "terminal"; + }; + // observePendingOperation is not read-only: it emits quote-state-changed, + // persists the observation and can fail a terminal operation. Track it too, + // so a timed-out check cannot be retried and then persist a stale read. + const check = source.mintOperationService.observePendingOperation(op.id); + track(op.id, check); + try { + observation = await withTimeout(check, remaining()); + } catch (error) { + result.retryable++; + result.errors.push({ operationId: op.id, error: messageOf(error) }); + onProgress?.(`${label}: could not check with mint: ${messageOf(error)}`); + return; + } + + const decision = classifyMintQuoteObservation(observation.category); + if (decision.action === "finalize") { + await finalizeAndClassify( + op.id, + label, + decision.observedRemoteState === "PAID" + ? `paid, minting proofs (${op.amount} sat)` + : `already issued, restoring proofs (${op.amount} sat)`, + remaining, + ); + } else if (decision.action === "waiting") { + result.waiting++; + onProgress?.(`${label}: mint reports UNPAID; left pending`); + } else { + // coco records the mint's terminal verdict by failing the operation. + result.terminal++; + onProgress?.(`${label}: mint can no longer issue this quote`); + } + } + + /** + * Run finalize and classify its result. coco returns a terminal operation + * rather than throwing when the mint refuses (for example an expired quote) + * or when an already-issued quote's proofs could not be restored, so a + * fulfilled promise is not by itself evidence that sats were recovered. + */ + async function finalizeAndClassify( + operationId: string, + label: string, + successMessage: string, + remaining: () => number, + ): Promise { + const work = source.ops.mint.finalize(operationId); + track(operationId, work); + let terminal: { state?: string; error?: string } | null | undefined; + try { + terminal = (await withTimeout(work, remaining())) as + | { state?: string; error?: string } + | null + | undefined; + } catch (error) { + if (isInProgress(error)) { + result.busy++; + onProgress?.(`${label}: another recovery is working on it; skipped`); + } else { + result.retryable++; + // finalize can throw a generic "remains pending" error after coco has + // persisted the actionable mint rejection (for example inactive keyset). + const current = await withTimeout( + source.ops.mint.get(operationId), + Math.max(remaining(), DIAGNOSTIC_LOOKUP_TIMEOUT_MS), + ).catch(() => null); + const detail = current?.state === "pending" && current.error + ? current.error + : messageOf(error); + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: could not finish recovery: ${detail}`); + return; + } + result.errors.push({ operationId, error: messageOf(error) }); + return; + } + if (terminal?.state === "finalized" && !terminal.error) { + result.recovered++; + onProgress?.(`${label}: ${successMessage}`); + return; + } + if ( + terminal?.state === "failed" || + (terminal?.state === "finalized" && terminal.error) + ) { + result.terminal++; + const detail = + terminal.error ?? `left in state ${terminal.state ?? "unknown"}`; + result.errors.push({ operationId, error: detail }); + onProgress?.(`${label}: not recovered: ${detail}`); + return; + } + // Pending/executing/unknown: coco may still be working on the operation, + // so leave it to a later run rather than calling it terminal. + result.retryable++; + result.errors.push({ + operationId, + error: `left in state ${terminal?.state ?? "unknown"}; will retry`, + }); + onProgress?.( + `${label}: still ${terminal?.state ?? "unknown"}; left for a later run`, + ); + } +} + const PENDING_MINT_SWEEP_INTERVAL_MS = 15_000; /** Per-quote wait inside a sweep, so one stalled mint cannot starve the rest. */ const PENDING_MINT_CHECK_TIMEOUT_MS = 10_000; @@ -1397,6 +1900,10 @@ export async function createCocoClient( }; let disposed = false; + // Explicit recovery runs are serialized, and finalize work that outlives its + // timeout stays in the map so a retry waits for it. + const enqueueRecovery = createRunQueue(); + const recoveryOutstanding = new Map>(); /** * Block a value-moving operation until background recovery has settled. @@ -1752,6 +2259,7 @@ export async function createCocoClient( await waitForRecovery(); const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000; const dryRun = options.dryRun === true; + const force = options.force === true; const nowMs = Date.now(); const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([ @@ -1779,6 +2287,8 @@ export async function createCocoClient( }); const errors: WalletCleanupResult["errors"] = []; + let failedMintQuotes = 0; + let leftForRecovery = 0; if (!dryRun) { const mintService = ( @@ -1788,20 +2298,49 @@ export async function createCocoClient( ).mintOperationService; for (const op of selection.mintsToFail) { - try { - await mintService.failPendingOperation( - { id: op.id }, - { - reason: "Expired unpaid mint quote cleaned up by routstrd", - retryable: false, - observedAt: nowMs, - }, - ); - } catch (error) { - errors.push({ - operationId: op.id, - error: error instanceof Error ? error.message : String(error), - }); + if (force) { + // Legacy behaviour: fail the quote locally without asking the mint. + try { + await mintService.failPendingOperation( + { id: op.id }, + { + reason: "Expired mint quote cleaned up by routstrd (forced)", + retryable: false, + observedAt: nowMs, + }, + ); + failedMintQuotes++; + } catch (error) { + errors.push({ + operationId: op.id, + error: error instanceof Error ? error.message : String(error), + }); + } + continue; + } + // Expiry alone does not prove the quote was never paid: the + // Lightning payment can land before expiry while the daemon is down. + // Confirm UNPAID with the mint before failing, exactly as startup + // recovery does; paid quotes are left for recovery to finalize. + const check = await failExpiredMintQuoteIfUnpaid( + mintService, + op.id, + EXPIRED_MINT_OBSERVATION_DEADLINE_MS, + ); + if (check.outcome === "failed") { + failedMintQuotes++; + } else { + leftForRecovery++; + if (check.outcome === "unobserved") { + errors.push({ + operationId: op.id, + error: `could not confirm quote state with mint: ${ + check.error instanceof Error + ? check.error.message + : String(check.error) + }`, + }); + } } } @@ -1828,8 +2367,15 @@ export async function createCocoClient( } } + const mintSummary = summarizeMintCleanup({ + dryRun, + candidates: selection.mintsToFail.length, + failed: failedMintQuotes, + leftForRecovery, + }); const actedOn = - selection.mintsToFail.length + + (dryRun ? mintSummary.mintQuoteCandidates : mintSummary.failedMintQuotes) + + leftForRecovery + selection.sendsToReclaim.length + selection.meltsToCancel.length; const skipped = @@ -1838,12 +2384,36 @@ export async function createCocoClient( return { dryRun, - failedMintQuotes: selection.mintsToFail.length, + ...mintSummary, reclaimedSends: selection.sendsToReclaim.length, cancelledMelts: selection.meltsToCancel.length, skipped, errors, }; }, + + async recoverMintQuotes(options, onProgress) { + await waitForRecovery(); + const service = ( + coco as unknown as { + mintOperationService: MintOperationServiceCleanup; + } + ).mintOperationService; + // Serialize explicit recovery: two concurrent requests must not both + // snapshot the same failed operation, and a retry must not start + // underneath a finalize that outlived its timeout. + return enqueueRecovery(() => + runMintQuoteRecovery( + { + ops: coco.ops as unknown as MintQuoteRecoverySource["ops"], + mintOperationService: service, + reopenFailedOperation: (operationId) => + reopenFailedMintOperation(service, operationId), + }, + { ...options, outstanding: recoveryOutstanding }, + onProgress, + ), + ); + }, }; } diff --git a/src/daemon/wallet/cocod-client.ts b/src/daemon/wallet/cocod-client.ts index 14feedf..2fab2c6 100644 --- a/src/daemon/wallet/cocod-client.ts +++ b/src/daemon/wallet/cocod-client.ts @@ -83,13 +83,23 @@ export interface WalletCleanupOptions { minAgeMs?: number; /** Report what would be cleaned without applying changes. */ dryRun?: boolean; + /** + * Fail expired mint quotes without confirming UNPAID with the mint. Only for + * operators who accept the risk of stranding a quote that was paid before + * its invoice expired; recovery is the safe default. + */ + force?: boolean; } /** Summary of a wallet cleanup run. */ export interface WalletCleanupResult { dryRun: boolean; - /** Number of expired pending mint quotes marked as failed. */ + /** Expired quotes selected for checking; dry runs do not contact the mint. */ + mintQuoteCandidates: number; + /** Number actually marked failed (always zero in a dry run). */ failedMintQuotes: number; + /** Expired quotes kept pending because they are paid/issued or unverified. */ + leftForRecovery: number; /** Number of stale pending send operations reclaimed. */ reclaimedSends: number; /** Number of stale prepared melt operations cancelled. */ @@ -120,6 +130,35 @@ export interface MintQuoteStatus { error?: string; } +/** Options for explicit PAID mint-quote recovery. */ +export interface WalletMintQuoteRecoveryOptions { + /** Target only these operation ids (may include failed operations). */ + operationIds?: string[]; + /** Re-open failed operations instead of skipping them. */ + includeFailed?: boolean; + /** Per-quote mint timeout in milliseconds. */ + timeoutMs?: number; +} + +/** Summary of a PAID mint-quote recovery run. */ +export interface WalletMintQuoteRecoveryResult { + /** Operations whose quote state was checked with the mint. */ + checked: number; + /** Operations whose paid sats were minted or restored. */ + recovered: number; + /** Quotes the mint still reports UNPAID; left pending. */ + waiting: number; + /** Quotes the mint can no longer issue. */ + terminal: number; + /** Failed operations moved back to pending before checking. */ + reopened: number; + /** Operations left to a later run (mint unreachable, budget spent, non-terminal). */ + retryable: number; + /** Operations skipped because an earlier recovery of them is still running. */ + busy: number; + errors: Array<{ operationId: string; error: string }>; +} + export interface CocodClient { ping(): Promise; getStatus(): Promise; @@ -154,6 +193,14 @@ export interface CocodClient { cleanupStuckOperations?( options?: WalletCleanupOptions, ): Promise; + /** + * Re-issue PAID mint quotes whose sats were never claimed, optionally + * targeting specific operations (including ones coco already failed). + */ + recoverMintQuotes?( + options?: WalletMintQuoteRecoveryOptions, + onProgress?: (message: string) => void, + ): Promise; /** Report background wallet recovery progress, when the wallet supports it. */ getRecoveryProgress?(): Promise; } diff --git a/src/daemon/wallet/mint-operation-reopen.test.ts b/src/daemon/wallet/mint-operation-reopen.test.ts new file mode 100644 index 0000000..9197fec --- /dev/null +++ b/src/daemon/wallet/mint-operation-reopen.test.ts @@ -0,0 +1,163 @@ +/** + * Re-opening a failed mint operation is the one genuinely destructive step of + * PAID-quote recovery, because coco's private `transitionToPending` spreads + * whatever it is handed and `SqliteMintOperationRepository.update` rewrites + * every column. A partial object such as `{ id }` is therefore rejected by the + * NOT NULL schema, and on a more permissive adapter would overwrite `quoteId`, + * `amount`, `request`, `lastObservedRemoteState` and `outputDataJson` with NULL, + * destroying the material needed to claim the paid sats. + * + * These tests drive the production `reopenFailedMintOperation` helper against a + * real coco sqlite repository through adapter-backed getOperation and + * transitionToPending implementations, so a regression at the helper/service + * boundary is caught rather than a mock standing in for it. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow(state = "failed") { + return { + id: "op-1", + mintUrl: "https://mint.example.com", + quoteId: "quote-1", + state, + createdAt: 1_000, + updatedAt: 2_000, + error: state === "failed" ? "expired" : undefined, + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: + state === "failed" ? { reason: "expired", observedAt: 3_000 } : undefined, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation against real coco sqlite", () => { + let dir: string | undefined; + let database: Database | undefined; + + afterEach(() => { + database?.close(); + database = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function repos() { + dir = mkdtempSync(join(tmpdir(), "routstrd-reopen-")); + database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + return repositories; + } + + function readRow(repositories: SqliteRepositories, id: string) { + return repositories.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + /** + * Adapter-backed stand-in for the private coco service methods the helper + * uses: getOperation reads and transitionToPending mirrors coco's + * spread-and-update implementation. + */ + function serviceOver(repositories: SqliteRepositories) { + return { + acquireOperationLock: async (_id: string) => () => {}, + getOperation: (id: string) => readRow(repositories, id), + transitionToPending: async ( + op: Record, + error?: string, + ) => { + await repositories.mintOperationRepository.update({ + ...op, + state: "pending", + error, + } as never); + }, + }; + } + + it("re-opens a failed row without losing quote metadata or stored outputs", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(true); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.unit).toBe("sat"); + expect(row?.request).toBe("lnbc1example"); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + }); + + it("is a no-op when the operation is no longer failed", async () => { + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow("finalized") as never, + ); + + const reopened = await reopenFailedMintOperation( + serviceOver(repositories), + "op-1", + ); + + expect(reopened).toBe(false); + const row = await readRow(repositories, "op-1"); + expect(row?.state).toBe("finalized"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + }); + + it("rejects a partial row, which is why the helper reloads in full", async () => { + // Locks in the reason for reloading. If a future coco version accepts + // partial updates this fails, and the helper can be simplified rather than + // silently losing paid sats. + const repositories = await repos(); + await repositories.mintOperationRepository.create( + failedRow() as never, + ); + + await expect( + repositories.mintOperationRepository.update({ + id: "op-1", + state: "pending", + updatedAt: Date.now(), + } as never), + ).rejects.toThrow(); + + const unchanged = await readRow(repositories, "op-1"); + expect(unchanged?.state).toBe("failed"); + expect(unchanged?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts new file mode 100644 index 0000000..2554665 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -0,0 +1,386 @@ +/** + * End-to-end PAID mint-quote recovery against a real coco Manager, real sqlite + * and a real in-process mint that produces genuine blind signatures. + * + * Nothing here mocks the wallet: a quote is created through coco, the mint is + * told what to report, and the production `runMintQuoteRecovery` drives the + * outcome. These are the release-gating scenarios for the feature, and they are + * the only tests that exercise issuance and NUT-09 restore over HTTP. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint"; +import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client"; + +type AnyRecord = Record; + +interface Booted { + manager: Manager; + repositories: SqliteRepositories; + mint: FakeMint; + /** Build the recovery source the production function expects. */ + source: () => AnyRecord; + spendable: () => Promise; + close: () => Promise; +} + +async function boot(options: { quoteExpiry?: number | null } = {}) { + const mint = new FakeMint(); + mint.quoteExpiry = options.quoteExpiry ?? null; + mint.start(); + const dir = mkdtempSync(join(tmpdir(), "routstrd-fakemint-")); + const database = new Database(join(dir, "coco.db")); + const repositories = new SqliteRepositories({ database }); + await repositories.init(); + const manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await manager.mint.addMint(mint.url, { trusted: true }); + + const service = (manager as unknown as { mintOperationService: AnyRecord }) + .mintOperationService; + + const booted: Booted = { + manager, + repositories, + mint, + spendable: async () => { + const balances = (await manager.wallet.balances.byMint()) as Record< + string, + { spendable: number } + >; + return balances[mint.url]?.spendable ?? 0; + }, + source: () => ({ + ops: { + mint: { + listPending: () => manager.ops.mint.listPending(), + get: (id: string) => manager.ops.mint.get(id), + finalize: (id: string) => manager.ops.mint.finalize(id), + }, + }, + mintOperationService: service, + reopenFailedOperation: (id: string) => + reopenFailedMintOperation(service as never, id), + }), + close: async () => { + await manager.dispose().catch(() => undefined); + database.close(); + mint.stop(); + rmSync(dir, { recursive: true, force: true }); + }, + }; + + return booted; +} + +async function prepareQuote(booted: Booted, amount: number) { + const op = (await booted.manager.ops.mint.prepare({ + mintUrl: booted.mint.url, + amount, + method: "bolt11", + } as never)) as unknown as AnyRecord; + return op; +} + +function outputsOf(op: AnyRecord) { + // coco stores mint outputs as { keep, send }, like the on-disk output JSON. + const outputData = (op.outputData as AnyRecord).keep as Array<{ + blindedMessage: { amount: unknown; id: string; B_: string }; + }>; + return outputData.map((output) => ({ + amount: Number(String(output.blindedMessage.amount)), + id: output.blindedMessage.id, + B_: output.blindedMessage.B_, + })); +} + +let booted: Booted | undefined; +afterEach(async () => { + await booted?.close(); + booted = undefined; +}); + +describe("PAID mint quote recovery with a real Manager and mint", () => { + it("issues an expired-but-PAID quote with the operation's own outputs, once", async () => { + // The motivating case: the invoice expired, but the mint says PAID and has + // issued nothing. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 210_000); + const expectedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ checked: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(210_000); + // Issuance used exactly the blinded outputs stored on the operation. + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...expectedOutputs].sort(), + ); + + // A second run must not mint again or double-credit. + const again = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(again).toMatchObject({ checked: 0, recovered: 0 }); + expect(booted.mint.requests).toHaveLength(1); + expect(await booted.spendable()).toBe(210_000); + }); + + it("existing coco recovery already issues expired paid pending quotes", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + booted.mint.markPaid(op.quoteId as string); + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(100); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("ISSUED"); + }); + + it("keeps rejected stored outputs and reports the actionable mint error", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 100); + const outputs = outputsOf(op); + booted.mint.markPaid(op.quoteId as string); + // Model the mint refusing the stored outputs, not invoice expiry. This is + // not evidence that the production quotes used an inactive keyset. + booted.mint.mintError = { code: 12001, detail: "keyset id inactive." }; + await booted.manager.recoverPendingMintOperations(); + expect(await booted.spendable()).toBe(0); + const result = await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + }); + expect(result).toMatchObject({ recovered: 0, retryable: 1 }); + expect(result.errors.some((entry) => entry.error.includes("keyset id inactive"))).toBe(true); + expect(await booted.spendable()).toBe(0); + expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("PAID"); + expect(outputsOf(await booted.manager.ops.mint.get(op.id as string) as unknown as AnyRecord)).toEqual(outputs); + for (const request of booted.mint.requests) expect(request.outputs).toEqual(outputs); + }); + + it("restores proofs for a quote already issued at the mint", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 210_000); + // Another wallet issued it: the signatures exist at the mint for the very + // outputs this operation stored. + booted.mint.signFor(op.quoteId as string, outputsOf(op)); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 1, terminal: 0, retryable: 0 }); + expect(await booted.spendable()).toBe(210_000); + }); + + it("reports terminal without credit when the mint refuses issuance", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + booted.mint.mintError = { code: QUOTE_EXPIRED, detail: "quote expired" }; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect((result.errors as unknown[]).length).toBeGreaterThan(0); + expect(await booted.spendable()).toBe(0); + }); + + it("reports terminal without credit when an issued quote cannot be restored", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint, but the signatures for our outputs are gone. + booted.mint.markIssued(op.quoteId as string); + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("does not credit a quote when the mint returns null NUT-09 signatures", async () => { + // KNOWN INTEROP GAP, not a supported path. NUT-09 permits `null` in the + // positional `signatures` array for outputs the mint never signed, but + // cashu-ts 3.7.1 - which coco depends on - dereferences every entry while + // normalising amounts, so the wallet throws instead of skipping the null. + // Recovery therefore surfaces an error and credits nothing, leaving the + // operation for a later run. + // + // This test pins the current behaviour so the gap cannot quietly disappear. + // Revisit (and change this expectation) once coco's cashu-ts parses + // positional nulls, and file/track it upstream in the meantime. + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 21_000); + // Issued at the mint with nothing signed for this operation's outputs. + booted.mint.markIssued(op.quoteId as string); + booted.mint.restoreIncludesNulls = true; + + const result = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 }); + expect(await booted.spendable()).toBe(0); + }); + + it("recovers a failed operation whose stale history says UNPAID", async () => { + // The composed path the feature exists for: a real prepared operation, + // failed locally, with a stale UNPAID observation from the old local-fail + // behaviour, recovered by explicit id and issued with its own outputs. + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + const storedOutputs = outputsOf(op).map((o) => o.B_); + booted.mint.markPaid(op.quoteId as string); + + const row = (await booted.repositories.mintOperationRepository.getById( + op.id as string, + )) as unknown as AnyRecord; + expect(row.outputData).toBeDefined(); + await booted.repositories.mintOperationRepository.update({ + ...row, + state: "failed", + lastObservedRemoteState: "UNPAID", + error: "Expired unpaid mint quote cleaned up by routstrd", + terminalFailure: { reason: "expired", observedAt: Date.now() }, + } as never); + + // A purely local decision would skip this row; the mint has the last word. + const result = (await runMintQuoteRecovery(booted.source() as never, { + operationIds: [op.id as string], + includeFailed: true, + })) as unknown as Record; + + expect(result).toMatchObject({ reopened: 1, recovered: 1, terminal: 0 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual( + [...storedOutputs].sort(), + ); + }); + + it("counts a locked operation as busy instead of minting underneath it", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + const service = ( + booted.manager as unknown as { + mintOperationService: { acquireOperationLock(id: string): Promise<() => void> }; + } + ).mintOperationService; + + // Hold the operation, as a processor or another recovery would. + const release = await service.acquireOperationLock(op.id as string); + const blocked = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + + expect(blocked).toMatchObject({ recovered: 0 }); + expect((blocked.busy ?? 0) + (blocked.retryable ?? 0)).toBeGreaterThan(0); + expect(booted.mint.requests).toHaveLength(0); + expect(await booted.spendable()).toBe(0); + + release(); + const after = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(after).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("does not mint a second time while issuance is in flight", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Hold the mint's response so the first recovery is visibly in flight. + let releaseGate!: () => void; + booted.mint.gate = new Promise((resolve) => { + releaseGate = resolve; + }); + const first = runMintQuoteRecovery( + booted.source() as never, + ) as unknown as Promise>; + + for (let i = 0; i < 400 && booted.mint.requests.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(booted.mint.requests).toHaveLength(1); + + // A concurrent run must not issue again while that request is outstanding. + await runMintQuoteRecovery(booted.source() as never); + expect(booted.mint.requests).toHaveLength(1); + + releaseGate(); + expect(await first).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests).toHaveLength(1); + }); + + it("coexists with coco's own mint operation watcher and processor", async () => { + booted = await boot({ quoteExpiry: -60 }); + // The real background machinery coco uses to settle pending mint quotes. + await booted.manager.enableMintOperationWatcher(); + await booted.manager.enableMintOperationProcessor(); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + // Either our recovery or coco's processor may win; both are safe. + await runMintQuoteRecovery(booted.source() as never); + + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + + // Give the processor time to act and confirm nothing is credited twice. + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(await booted.spendable()).toBe(21_000); + expect(booted.mint.requests.length).toBeLessThanOrEqual(1); + }); + + it("tracks a hung quote check so a later run waits instead of re-reading", async () => { + booted = await boot({ quoteExpiry: -60 }); + const op = await prepareQuote(booted, 21_000); + booted.mint.markPaid(op.quoteId as string); + + let releaseObserve!: () => void; + booted.mint.observeGate = new Promise((resolve) => { + releaseObserve = resolve; + }); + const outstanding = new Map>(); + + const first = (await runMintQuoteRecovery(booted.source() as never, { + timeoutMs: 30, + outstanding, + })) as unknown as Record; + expect(first).toMatchObject({ retryable: 1, recovered: 0 }); + expect(outstanding.has(op.id as string)).toBe(true); + + const second = (await runMintQuoteRecovery(booted.source() as never, { + outstanding, + })) as unknown as Record; + expect(second).toMatchObject({ checked: 0, busy: 1 }); + + // Once the held check settles the tracking drains, and recovery proceeds. + releaseObserve(); + for (let i = 0; i < 400 && outstanding.size > 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(outstanding.size).toBe(0); + + const third = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(third).toMatchObject({ recovered: 1 }); + expect(await booted.spendable()).toBe(21_000); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.manager.test.ts b/src/daemon/wallet/mint-quote-recovery.manager.test.ts new file mode 100644 index 0000000..5c779bf --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.manager.test.ts @@ -0,0 +1,159 @@ +/** + * Real-Manager integration for re-opening a failed mint operation. + * + * The unit tests exercise `reopenFailedMintOperation` against a hand-written + * service double, which cannot catch a change in coco's own + * `MintOperationService.transitionToPending` semantics or in its per-operation + * lock. This drives the production helper against an actual coco `Manager` + * backed by sqlite, so the private-service boundary that the helper depends on + * is exercised for real: full-row preservation, the shared operation lock, and + * the `mint-op:pending` event. + * + * No mint or network access is involved; nothing here enables the mint watcher + * or processor, which the fake-mint integration covers. + */ +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Manager } from "@cashu/coco-core"; +import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; +import { reopenFailedMintOperation } from "./coco-client"; + +const OUTPUT_DATA = [ + { + blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" }, + blindingFactor: "1234567890", + secret: "aabbccdd", + }, +]; + +function failedRow() { + return { + id: "op-1", + mintUrl: "https://mint.invalid", + quoteId: "quote-1", + state: "failed", + createdAt: 1_000, + updatedAt: 2_000, + error: "expired", + method: "bolt11", + methodData: { method: "bolt11", data: {} }, + amount: 210_000, + unit: "sat", + request: "lnbc1example", + expiry: 1_800_000_000, + pubkey: undefined, + lastObservedRemoteState: "PAID", + lastObservedRemoteStateAt: 3_000, + terminalFailure: { reason: "expired", observedAt: 3_000 }, + outputData: OUTPUT_DATA, + }; +} + +describe("reopenFailedMintOperation with a real coco Manager", () => { + let dir: string | undefined; + let database: Database | undefined; + let manager: Manager | undefined; + let repositories: SqliteRepositories | undefined; + + afterEach(async () => { + await manager?.dispose().catch(() => undefined); + manager = undefined; + database?.close(); + database = undefined; + repositories = undefined; + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + async function boot() { + dir = mkdtempSync(join(tmpdir(), "routstrd-manager-")); + database = new Database(join(dir, "coco.db")); + repositories = new SqliteRepositories({ database }); + await repositories.init(); + manager = new Manager(repositories, async () => new Uint8Array(64).fill(7)); + await repositories.mintOperationRepository.create(failedRow() as never); + const service = ( + manager as unknown as { + mintOperationService: { + acquireOperationLock(id: string): Promise<() => void>; + getOperation(id: string): Promise | null>; + transitionToPending( + op: Record, + error?: string, + ): Promise; + }; + } + ).mintOperationService; + const eventBus = ( + manager as unknown as { + eventBus: { on(event: string, handler: () => void): () => void }; + } + ).eventBus; + return { service, eventBus }; + } + + function readRow(id: string) { + return repositories!.mintOperationRepository.getById(id) as unknown as Promise< + Record | null + >; + } + + it("re-opens through the real service and emits mint-op:pending", async () => { + const { service, eventBus } = await boot(); + const events: string[] = []; + const off = eventBus.on("mint-op:pending", () => events.push("pending")); + + const reopened = await reopenFailedMintOperation(service, "op-1"); + off(); + + expect(reopened).toBe(true); + const row = await readRow("op-1"); + expect(row?.state).toBe("pending"); + expect(row?.quoteId).toBe("quote-1"); + expect(row?.amount).toBe(210_000); + expect(row?.lastObservedRemoteState).toBe("PAID"); + expect(row?.outputData).toEqual(OUTPUT_DATA); + expect(row?.terminalFailure ?? undefined).toBeUndefined(); + expect(events).toEqual(["pending"]); + }); + + it("refuses to re-open while coco's operation lock is held", async () => { + const { service } = await boot(); + // coco's OperationIdLock is fail-fast: a holder blocks the re-open by + // making it throw, and nothing is written. + const release = await service.acquireOperationLock("op-1"); + + await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow( + /already in progress/, + ); + const blocked = await readRow("op-1"); + expect(blocked?.state).toBe("failed"); + expect(blocked?.outputData).toEqual(OUTPUT_DATA); + + release(); + expect(await reopenFailedMintOperation(service, "op-1")).toBe(true); + const reopened = await readRow("op-1"); + expect(reopened?.state).toBe("pending"); + expect(reopened?.outputData).toEqual(OUTPUT_DATA); + }); + + it("leaves an operation a processor finalized alone", async () => { + const { service } = await boot(); + // Emulate a processor winning the race while holding the same lock. + const release = await service.acquireOperationLock("op-1"); + const row = await readRow("op-1"); + await repositories!.mintOperationRepository.update({ + ...row, + state: "finalized", + } as never); + release(); + + expect(await reopenFailedMintOperation(service, "op-1")).toBe(false); + const after = await readRow("op-1"); + expect(after?.state).toBe("finalized"); + expect(after?.outputData).toEqual(OUTPUT_DATA); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.test.ts b/src/daemon/wallet/mint-quote-recovery.test.ts new file mode 100644 index 0000000..eac5de9 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "bun:test"; +import { + classifyMintQuoteObservation, + selectMintQuotesForRecovery, + type MintQuoteRecoveryCandidate, +} from "./mint-quote-recovery"; + +function mint( + overrides: Partial = {}, +): MintQuoteRecoveryCandidate { + return { + id: "mint-1", + mintUrl: "https://mint.example", + quoteId: "quote-1", + state: "pending", + amount: 1000, + expiry: 0, + ...overrides, + }; +} + +describe("classifyMintQuoteObservation", () => { + it("finalizes a paid-but-unissued quote by minting its stored outputs", () => { + expect(classifyMintQuoteObservation("ready")).toEqual({ + action: "finalize", + observedRemoteState: "PAID", + }); + }); + + it("finalizes an already-issued quote by restoring its proofs", () => { + expect(classifyMintQuoteObservation("completed")).toEqual({ + action: "finalize", + observedRemoteState: "ISSUED", + }); + }); + + it("leaves an unpaid quote alone", () => { + expect(classifyMintQuoteObservation("waiting")).toEqual({ + action: "waiting", + }); + }); + + it("reports a quote the mint can no longer issue", () => { + expect(classifyMintQuoteObservation("terminal")).toEqual({ + action: "terminal", + }); + }); +}); + +describe("selectMintQuotesForRecovery", () => { + it("selects every pending quote because only the mint knows if it was paid", () => { + const result = selectMintQuotesForRecovery({ + mints: [ + mint({ id: "expired", expiry: 1 }), + mint({ id: "fresh" }), + mint({ id: "observed-unpaid", lastObservedRemoteState: "UNPAID" }), + mint({ id: "observed-paid", lastObservedRemoteState: "PAID" }), + ], + }); + expect(result.pending.map((op) => op.id)).toEqual([ + "expired", + "fresh", + "observed-unpaid", + "observed-paid", + ]); + }); + + it("recovers executing operations left behind by a crash mid-mint", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "executing", state: "executing" })], + }); + expect(result.pending.map((op) => op.id)).toEqual(["executing"]); + }); + + it("ignores finalized operations", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "done", state: "finalized" })], + }); + expect(result.pending).toEqual([]); + expect(result.failed).toEqual([]); + }); + + it("does not re-open failed operations by default", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + }); + expect(result.failed).toEqual([]); + }); + + it("returns failed operations when the caller opts in", () => { + const result = selectMintQuotesForRecovery({ + mints: [mint({ id: "given-up", state: "failed" })], + includeFailed: true, + }); + expect(result.failed.map((op) => op.id)).toEqual(["given-up"]); + }); +}); diff --git a/src/daemon/wallet/mint-quote-recovery.ts b/src/daemon/wallet/mint-quote-recovery.ts new file mode 100644 index 0000000..eb620b2 --- /dev/null +++ b/src/daemon/wallet/mint-quote-recovery.ts @@ -0,0 +1,119 @@ +/** + * Pure helpers for PAID mint-quote recovery. + * + * A mint quote can be PAID at the mint while its local operation is still + * `pending` (the Lightning payment landed before expiry while the daemon was + * down, so no local observation was ever recorded) or even terminally + * `failed` (coco gives up when the mint refuses to sign, for example after the + * invoice expiry). Claimability still depends on the mint accepting issuance. + * This feature retries the stored outputs or restores their signatures; it + * does not regenerate outputs rejected by the mint (for example an inactive + * keyset). coco already reconciles pending paid quotes at startup and in the + * periodic sweep. The new capability is operator-targeted recovery, including + * explicitly reopening failed operations, alongside safer cleanup. + * + * Recovery asks the mint what it thinks, then retries issuance or restore. These helpers decide *what* to do from a remote observation; the + * actual state transitions are applied by the in-process coco wallet client + * so coco-core's operation services emit their normal events and release + * proof reservations. Keeping the decisions here makes them unit testable + * without a wallet database or network access. + */ + +/** Subset of coco's mint operation rows that recovery needs. */ +export interface MintQuoteRecoveryCandidate { + id: string; + mintUrl: string; + quoteId?: string; + state: string; + /** Quote amount in sats. */ + amount: number; + /** Quote expiry in epoch seconds. `0` means unknown/not applicable. */ + expiry: number; + /** Last quote state observed from the mint (UNPAID, PAID, ISSUED). */ + lastObservedRemoteState?: string; + error?: string; +} + +/** Coco's classification of a fresh remote quote check. */ +export type PendingMintCheckCategory = + | "waiting" + | "ready" + | "completed" + | "terminal"; + +/** What recovery should do with a quote after checking it with the mint. */ +export type MintQuoteRecoveryDecision = + | { action: "finalize"; observedRemoteState: "PAID" | "ISSUED" } + | { action: "waiting" } + | { action: "terminal" }; + +/** + * Map a remote quote check onto a recovery action. + * + * - `ready` means the mint reports the quote PAID but never issued: submit the + * operation's stored outputs to claim the sats. + * - `completed` means the mint already issued it: recover the signatures + * (NUT-09) instead of minting again. + * - `waiting` means the mint still reports the quote UNPAID: nothing is + * claimable, so leave the operation alone. + * - `terminal` means the quote can no longer be issued (for example the mint + * refused an expired quote). coco persists that verdict as a failed + * operation, so recovery must report it rather than treat it as progress. + */ +export function classifyMintQuoteObservation( + category: PendingMintCheckCategory, +): MintQuoteRecoveryDecision { + switch (category) { + case "ready": + return { action: "finalize", observedRemoteState: "PAID" }; + case "completed": + return { action: "finalize", observedRemoteState: "ISSUED" }; + case "waiting": + return { action: "waiting" }; + case "terminal": + return { action: "terminal" }; + } +} + +export interface MintQuoteRecoverySelectionOptions< + T extends MintQuoteRecoveryCandidate, +> { + mints: T[]; + /** + * Also consider terminally failed operations. Off by default: re-opening a + * failed operation is a mutation, so only an explicit user-invoked recovery + * may do it. Startup recovery must never resurrect quotes on its own. + */ + includeFailed?: boolean; +} + +export interface MintQuoteRecoverySelection< + T extends MintQuoteRecoveryCandidate, +> { + /** Pending (or executing) operations that need a fresh mint observation. */ + pending: T[]; + /** Failed operations the caller may re-open and retry. */ + failed: T[]; +} + +/** + * Split operations into those recovery should check and those that were + * already given up on. + * + * Every `pending` operation is selected: only the mint knows whether an + * expired quote was paid before the local invoice ran out. `executing` + * operations are recovered too, since a crash mid-mint leaves outputs that + * may already be signed. + */ +export function selectMintQuotesForRecovery< + T extends MintQuoteRecoveryCandidate, +>(options: MintQuoteRecoverySelectionOptions): MintQuoteRecoverySelection { + const { mints, includeFailed = false } = options; + const pending = mints.filter( + (op) => op.state === "pending" || op.state === "executing", + ); + const failed = includeFailed + ? mints.filter((op) => op.state === "failed") + : []; + return { pending, failed }; +} diff --git a/src/daemon/wallet/testing/fake-mint.ts b/src/daemon/wallet/testing/fake-mint.ts new file mode 100644 index 0000000..3960ee9 --- /dev/null +++ b/src/daemon/wallet/testing/fake-mint.ts @@ -0,0 +1,347 @@ +/** + * In-process Cashu mint for integration tests. + * + * Implements just enough of NUT-01/02/04/06/07/09 to drive a real coco + * `Manager` against real HTTP: keyset publication, bolt11 mint quotes, minting + * blinded outputs with a real secp256k1 blind signature, NUT-09 restore and + * NUT-07 proof states. No Lightning, no network, no NPC. + * + * The signing keys and signatures are genuine (`@cashu/cashu-ts` mint-side + * helpers), so a wallet that receives these signatures can unblind and verify + * them exactly as with a production mint. + */ +import { + createBlindSignature, + createNewMintKeys, + pointFromHex, +} from "@cashu/cashu-ts"; + +/** NUT error codes used by the scenarios. */ +export const QUOTE_EXPIRED = 20007; +export const ALREADY_ISSUED = 20002; + +export type FakeQuoteState = "UNPAID" | "PAID" | "ISSUED"; + +export interface FakeMintQuote { + quote: string; + request: string; + amount: number; + unit: string; + state: FakeQuoteState; + /** Epoch seconds, or null for a quote that never expires. */ + expiry: number | null; + amountPaid: number; + amountIssued: number; + pubkey: null; +} + +export interface FakeMintRequest { + quote: string; + outputs: Array<{ amount: number; id: string; B_: string }>; +} + +interface StoredSignature { + amount: number; + id: string; + C_: string; +} + +const toHex = (bytes: Uint8Array) => Buffer.from(bytes).toString("hex"); + +export class FakeMint { + readonly keysetId: string; + readonly keysByAmount: Record; + readonly requests: FakeMintRequest[] = []; + /** Every output the mint has ever signed, keyed by B_. */ + readonly signed = new Map(); + + /** When set, POST /v1/mint/bolt11 fails with this NUT error. */ + mintError: { code: number; detail: string } | null = null; + /** When set, quote creation returns this expiry (epoch seconds). */ + quoteExpiry: number | null = 3_600; + /** + * Awaited before responding to a mint request, so a test can hold minting + * open and interleave another recovery attempt. + */ + gate: Promise | null = null; + /** Awaited before answering a quote-state check, to hold observe open. */ + observeGate: Promise | null = null; + /** + * When true, POST /v1/restore answers with NUT-09's spec-legal positional + * arrays, including `null` for outputs the mint never signed. + * + * This is a known interop gap, not a supported path: cashu-ts 3.7.1 (which + * coco depends on) dereferences every entry of `signatures` while normalising + * amounts, so a `null` makes the wallet throw instead of skipping it. The + * switch exists so tests keep that behaviour visible; see + * mint-quote-recovery.fake-mint.test.ts. + */ + restoreIncludesNulls = false; + + private readonly quotes = new Map(); + private counter = 0; + private server?: ReturnType; + + constructor() { + const pair = createNewMintKeys(20, new Uint8Array(32).fill(9)); + this.keysetId = pair.keysetId; + this.keysByAmount = Object.fromEntries( + Object.entries(pair.pubKeys).map(([amount, key]) => [ + amount, + typeof key === "string" ? key : toHex(key), + ]), + ); + this.privKeys = Object.fromEntries( + Object.entries(pair.privKeys) as Array<[string, Uint8Array]>, + ); + } + + private readonly privKeys: Record; + + get url(): string { + if (!this.server) throw new Error("fake mint not started"); + return `http://127.0.0.1:${this.server.port}`; + } + + start(): void { + this.server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: (request) => this.handle(request), + }); + } + + stop(): void { + this.server?.stop(true); + this.server = undefined; + } + + /** Test control: the mint sees the invoice as paid but has issued nothing. */ + markPaid(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "PAID"; + quote.amountPaid = quote.amount; + } + + /** + * Test control: mark the quote issued without going through the mint + * endpoint, simulating a wallet that lost the signatures. + */ + markIssued(quoteId: string): void { + const quote = this.quotes.get(quoteId); + if (!quote) throw new Error(`unknown fake quote ${quoteId}`); + quote.state = "ISSUED"; + quote.amountPaid = quote.amount; + quote.amountIssued = quote.amount; + } + + /** Test control: sign outputs directly, as if another wallet had issued them. */ + signFor(quoteId: string, outputs: FakeMintRequest["outputs"]): void { + for (const output of outputs) { + this.signOutput(output); + } + this.markIssued(quoteId); + } + + getQuote(quoteId: string): FakeMintQuote | undefined { + return this.quotes.get(quoteId); + } + + private quoteBody(quote: FakeMintQuote) { + return { + quote: quote.quote, + request: quote.request, + amount: quote.amount, + unit: quote.unit, + state: quote.state, + expiry: quote.expiry, + amount_paid: quote.amountPaid, + amount_issued: quote.amountIssued, + pubkey: quote.pubkey, + }; + } + + private signOutput(output: { + amount: number; + id: string; + B_: string; + }): StoredSignature { + const privKey = this.privKeys[String(output.amount)]; + if (!privKey) { + throw new Error(`fake mint has no key for amount ${output.amount}`); + } + const signature = createBlindSignature( + pointFromHex(output.B_), + privKey, + this.keysetId, + ); + const stored: StoredSignature = { + amount: output.amount, + id: this.keysetId, + C_: toHex(signature.C_.toBytes(false)), + }; + this.signed.set(output.B_, stored); + return stored; + } + + private json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + } + + private error(code: number, detail: string, status = 400): Response { + return this.json({ code, detail }, status); + } + + private async handle(request: Request): Promise { + const { pathname } = new URL(request.url); + const body = async () => { + try { + return (await request.json()) as Record; + } catch { + return {}; + } + }; + + if (pathname === "/v1/info") { + return this.json({ + name: "fake-mint", + version: "0.0.1", + nuts: { + 4: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 5: { + methods: [ + { + method: "bolt11", + unit: "sat", + min_amount: 1, + max_amount: 1_000_000, + }, + ], + }, + 7: { supported: true }, + 9: { supported: true }, + }, + }); + } + + if (pathname === "/v1/keys" || pathname.startsWith("/v1/keys/")) { + return this.json({ + keysets: [ + { id: this.keysetId, unit: "sat", keys: this.keysByAmount }, + ], + }); + } + + if (pathname === "/v1/keysets") { + return this.json({ + keysets: [{ id: this.keysetId, unit: "sat", active: true }], + }); + } + + if (pathname === "/v1/mint/quote/bolt11" && request.method === "POST") { + const input = await body(); + const amount = Number(input.amount); + const unit = typeof input.unit === "string" ? input.unit : "sat"; + const quote: FakeMintQuote = { + quote: `fake-quote-${++this.counter}`, + request: `lnbcfake${this.counter}`, + amount, + unit, + state: "UNPAID", + expiry: + this.quoteExpiry === null + ? null + : Math.floor(Date.now() / 1000) + this.quoteExpiry, + amountPaid: 0, + amountIssued: 0, + pubkey: null, + }; + this.quotes.set(quote.quote, quote); + return this.json(this.quoteBody(quote)); + } + + const quoteMatch = pathname.match(/^\/v1\/mint\/quote\/bolt11\/(.+)$/); + if (quoteMatch?.[1] && request.method === "GET") { + if (this.observeGate) await this.observeGate; + const quote = this.quotes.get(decodeURIComponent(quoteMatch[1])); + if (!quote) return this.error(50000, "Unknown quote"); + return this.json(this.quoteBody(quote)); + } + + if (pathname === "/v1/mint/bolt11" && request.method === "POST") { + const input = await body(); + const quoteId = String(input.quote); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + this.requests.push({ quote: quoteId, outputs }); + if (this.gate) await this.gate; + if (this.mintError) { + return this.error(this.mintError.code, this.mintError.detail); + } + const quote = this.quotes.get(quoteId); + if (!quote) return this.error(50000, "Unknown quote"); + if (quote.state === "ISSUED" || quote.amountIssued > 0) { + return this.error(ALREADY_ISSUED, "Quote already issued"); + } + if (quote.state !== "PAID") { + return this.error(20001, "Quote is not paid"); + } + const total = outputs.reduce((sum, o) => sum + Number(o.amount), 0); + if (total > quote.amountPaid - quote.amountIssued) { + return this.error(10002, "Outputs exceed the paid amount"); + } + const signatures = outputs.map((output) => ({ + amount: output.amount, + id: this.keysetId, + C_: this.signOutput(output).C_, + })); + quote.state = "ISSUED"; + quote.amountIssued += total; + return this.json({ signatures }); + } + + if (pathname === "/v1/restore" && request.method === "POST") { + const input = await body(); + const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"]; + if (this.restoreIncludesNulls) { + // Spec-legal NUT-09 shape, including nulls. Kept behind a switch + // because it is currently unusable with coco's cashu-ts version. + return this.json({ + outputs, + signatures: outputs.map( + (output) => this.signed.get(output.B_) ?? null, + ), + }); + } + // Return only the signed outputs; coco matches them by B_ and treats the + // rest as "nothing to restore". + const signed = outputs.filter((output) => this.signed.has(output.B_)); + return this.json({ + outputs: signed, + signatures: signed.map((output) => this.signed.get(output.B_)), + }); + } + + if (pathname === "/v1/checkstate" && request.method === "POST") { + const input = await body(); + const ys = (input.Ys ?? []) as string[]; + return this.json({ + states: ys.map((Y) => ({ Y, state: "UNSPENT", witness: null })), + }); + } + + return this.error(404, `fake mint has no route for ${pathname}`, 404); + } +}