diff --git a/docs/wallet-mint-recovery.md b/docs/wallet-mint-recovery.md index cb344ef..ca2eb05 100644 --- a/docs/wallet-mint-recovery.md +++ b/docs/wallet-mint-recovery.md @@ -46,6 +46,27 @@ and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate with coco's watcher/processor. It needs its own integration tests before handling real funds. +## Wallet doctor + +`routstrd wallet doctor` runs read-only health checks before the legacy +migration diagnosis: + +1. **Mint reachability** — a NUT-06 probe against every trusted mint. +2. **Recent unpaid quotes** — pending quotes from the last hour the mint + still reports UNPAID (an invoice awaiting payment; informational). +3. **Paid but not issued** — the stuck scenario this recovery feature + fixes. Each finding prints its `wallet recover --op ` remediation, + with `--include-failed` when the operation must be re-opened first. +4. **Stuck melts** — prepared melts holding reserved proofs, in-flight + melts, and failed melts whose input proofs were never released. + +The doctor never mutates wallet state: quote checks are plain NUT-04 reads, +not coco's observe-and-persist path, and remediation is always left to the +operator. It exits non-zero when money is provably at risk (unreachable +mint, paid-but-unissued quote, failed melt with locked proofs) or the +migration diagnosis finds a conflict, so it can gate scripts. When the +daemon is down only the offline migration section runs. + ## Cleanup preview and force `wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not diff --git a/src/cli-doctor.test.ts b/src/cli-doctor.test.ts new file mode 100644 index 0000000..c4cef98 --- /dev/null +++ b/src/cli-doctor.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "bun:test"; +import { renderWalletHealth } from "./cli"; +import type { WalletDoctorReport } from "./daemon/wallet/doctor"; + +const NOW = 1_800_000_000_000; + +function report(overrides: Partial = {}): WalletDoctorReport { + return { + generatedAt: NOW, + mints: [], + unpaidQuotes: [], + paidUnissued: [], + stuckMelts: [], + uncheckedQuotes: 0, + ...overrides, + }; +} + +describe("renderWalletHealth", () => { + it("renders an all-green report", () => { + const output = renderWalletHealth( + report({ + mints: [ + { mintUrl: "https://mint.example", reachable: true, latencyMs: 212 }, + ], + }), + ); + expect(output).toContain("Wallet health"); + expect(output).toContain("✓ https://mint.example — reachable (212 ms)"); + expect(output).toContain("none unpaid"); + expect(output).toContain("none stuck"); + }); + + it("renders unreachable mints with their error", () => { + const output = renderWalletHealth( + report({ + mints: [ + { mintUrl: "https://mint.down", reachable: false, error: "fetch failed" }, + ], + }), + ); + expect(output).toContain("✗ https://mint.down — unreachable: fetch failed"); + }); + + it("renders unpaid quotes with age and expiry", () => { + const output = renderWalletHealth( + report({ + unpaidQuotes: [ + { + operationId: "op-1234567890abcdef", + quoteId: "quote-abcdef1234567890", + mintUrl: "https://mint.example", + amount: 500, + ageMs: 5 * 60_000, + expiresInMs: 41 * 60_000, + }, + ], + }), + ); + expect(output).toContain("⚠ quote-ab… at https://mint.example — 500 sat, UNPAID"); + expect(output).toContain("created 5m ago, expires in 41m"); + }); + + it("renders paid-unissued findings with the recover remediation", () => { + const output = renderWalletHealth( + report({ + paidUnissued: [ + { + operationId: "op-1234567890abcdef", + quoteId: "quote-abcdef1234567890", + mintUrl: "https://mint.example", + amount: 1_000, + localState: "failed", + remoteState: "PAID", + error: "keyset id inactive.", + remediation: "routstrd wallet recover --op op-1234567890abcdef --include-failed", + }, + ], + }), + ); + expect(output).toContain("✗ op-12345… (quote quote-ab…)"); + expect(output).toContain("1,000 sat PAID at mint, local state failed"); + expect(output).toContain("mint error: keyset id inactive."); + expect(output).toContain( + "→ routstrd wallet recover --op op-1234567890abcdef --include-failed", + ); + }); + + it("renders stuck melts with locked totals and remediation", () => { + const output = renderWalletHealth( + report({ + stuckMelts: [ + { + operationId: "melt-1234567890", + mintUrl: "https://mint.example", + amount: 2_100, + feeReserve: 12, + ageMs: 3 * 86_400_000, + kind: "prepared", + lockedSecrets: 4, + remediation: "routstrd wallet cleanup", + }, + { + operationId: "melt-abcdef", + mintUrl: "https://mint.example", + amount: 50, + feeReserve: 2, + ageMs: 90_000, + kind: "failed-locked", + lockedSecrets: 1, + remediation: "restart the daemon", + }, + ], + }), + ); + expect(output).toContain( + "⚠ melt-123… at https://mint.example — 2,112 sat locked (4 proof(s)), prepared, payment never attempted, 3d old", + ); + expect(output).toContain("✗ melt-abcdef at"); + expect(output).toContain("failed but proofs still locked"); + }); + + it("notes quotes that could not be checked", () => { + const output = renderWalletHealth(report({ uncheckedQuotes: 3 })); + expect(output).toContain("3 quote(s) could not be checked with their mint"); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index c2da59e..f96a52a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -49,6 +49,11 @@ import { summarizeWalletDirectory, WalletMigrationConflictError, } from "./daemon/wallet/diagnostics"; +import { + doctorReportSeverity, + DOCTOR_RECENT_QUOTE_WINDOW_MS, + type WalletDoctorReport, +} from "./daemon/wallet/doctor"; import { legacyCocodDir, legacyCocodPidPath, @@ -2062,14 +2067,156 @@ walletCmd await handleDaemonCommand("/wallet/status"); }); +const doctorUseColor = !process.env.NO_COLOR && process.stdout.isTTY === true; +const doctorPaint = (code: string, text: string): string => + doctorUseColor ? `\x1b[${code}m${text}\x1b[0m` : text; +const doctorGreen = (text: string) => doctorPaint("32", text); +const doctorRed = (text: string) => doctorPaint("31", text); +const doctorYellow = (text: string) => doctorPaint("33", text); + +function shortenOperationId(id: string): string { + return id.length > 12 ? `${id.slice(0, 8)}…` : id; +} + +function formatSats(amount: number): string { + return amount.toLocaleString("en-US"); +} + +function formatDoctorDuration(ms: number): string { + const abs = Math.abs(ms); + if (abs < 60_000) return `${Math.max(1, Math.round(abs / 1000))}s`; + if (abs < 3_600_000) return `${Math.round(abs / 60_000)}m`; + if (abs < 86_400_000) return `${Math.round(abs / 3_600_000)}h`; + return `${Math.round(abs / 86_400_000)}d`; +} + +/** Render the daemon's live doctor report; colors are NO_COLOR/TTY aware. */ +export function renderWalletHealth(report: WalletDoctorReport): string { + const lines: string[] = ["Wallet health", "=============", ""]; + + lines.push("Mints"); + if (report.mints.length === 0) { + lines.push(" (no trusted mints)"); + } + for (const probe of report.mints) { + if (probe.reachable) { + const latency = + probe.latencyMs !== undefined ? ` (${probe.latencyMs} ms)` : ""; + lines.push(` ${doctorGreen("✓")} ${probe.mintUrl} — reachable${latency}`); + } else { + lines.push( + ` ${doctorRed("✗")} ${probe.mintUrl} — unreachable${probe.error ? `: ${probe.error}` : ""}`, + ); + } + } + + lines.push( + "", + `Mint quotes (last ${formatDoctorDuration(DOCTOR_RECENT_QUOTE_WINDOW_MS)})`, + ); + if (report.unpaidQuotes.length === 0) { + lines.push(" none unpaid"); + } + for (const quote of report.unpaidQuotes) { + const expiry = + quote.expiresInMs === undefined + ? "" + : quote.expiresInMs >= 0 + ? `, expires in ${formatDoctorDuration(quote.expiresInMs)}` + : `, expired ${formatDoctorDuration(quote.expiresInMs)} ago`; + lines.push( + ` ${doctorYellow("⚠")} ${shortenOperationId(quote.quoteId ?? quote.operationId)} at ${quote.mintUrl} — ${formatSats(quote.amount)} sat, UNPAID, created ${formatDoctorDuration(quote.ageMs)} ago${expiry}`, + ); + } + + lines.push("", "Paid but not issued"); + if (report.paidUnissued.length === 0) { + lines.push(" none"); + } + for (const finding of report.paidUnissued) { + lines.push( + ` ${doctorRed("✗")} ${shortenOperationId(finding.operationId)} (quote ${shortenOperationId(finding.quoteId ?? "?")}) at ${finding.mintUrl} — ${formatSats(finding.amount)} sat ${finding.remoteState} at mint, local state ${finding.localState}`, + ); + if (finding.error) lines.push(` mint error: ${finding.error}`); + lines.push(` → ${finding.remediation}`); + } + + lines.push("", "Melts"); + if (report.stuckMelts.length === 0) { + lines.push(" none stuck"); + } + for (const melt of report.stuckMelts) { + const icon = melt.kind === "failed-locked" ? doctorRed("✗") : doctorYellow("⚠"); + const what = + melt.kind === "prepared" + ? "prepared, payment never attempted" + : melt.kind === "in-flight" + ? "payment in flight" + : "failed but proofs still locked"; + lines.push( + ` ${icon} ${shortenOperationId(melt.operationId)} at ${melt.mintUrl} — ${formatSats(melt.amount + melt.feeReserve)} sat locked (${melt.lockedSecrets} proof(s)), ${what}, ${formatDoctorDuration(melt.ageMs)} old`, + ); + lines.push(` → ${melt.remediation}`); + } + + if (report.uncheckedQuotes > 0) { + lines.push( + "", + ` ${doctorYellow("⚠")} ${report.uncheckedQuotes} quote(s) could not be checked with their mint`, + ); + } + return lines.join("\n"); +} + walletCmd .command("doctor") - .description("Diagnose conflicting wallets (current routstrd wallet vs legacy cocod)") - .action(async () => { + .description( + "Wallet health checks (mint reachability, unpaid/stuck quotes, locked melts), then legacy wallet migration diagnosis", + ) + .option("--json", "Print the raw health report as JSON", false) + .action(async (options: { json: boolean }) => { const target = summarizeWalletDirectory(defaultWalletDir(), "canonical"); const source = summarizeWalletDirectory(legacyCocodDir(), "legacy"); - console.log(renderWalletDoctor(target, source)); - if (diagnoseWallets(target, source).conflict) process.exit(1); + const migrationConflict = diagnoseWallets(target, source).conflict; + + // Live health checks need the daemon; when it is down the offline + // migration diagnosis still runs, because a broken startup is exactly + // when the doctor gets invoked. + let report: WalletDoctorReport | undefined; + try { + const result = await callDaemon("/wallet/doctor"); + if (result.error) throw new Error(result.error); + report = result.output as WalletDoctorReport | undefined; + } catch { + report = undefined; + } + + if (options.json) { + console.log( + JSON.stringify( + { health: report ?? null, migrationConflict }, + null, + 2, + ), + ); + } else { + if (report) { + console.log(renderWalletHealth(report)); + } else { + console.log("Wallet health"); + console.log("============="); + console.log(""); + console.log( + " skipped: daemon is not running; live checks need it (routstrd start)", + ); + } + console.log(""); + console.log(renderWalletDoctor(target, source)); + } + + const healthCritical = + report !== undefined && doctorReportSeverity(report) === "critical"; + if (healthCritical || migrationConflict) process.exit(1); }); walletCmd diff --git a/src/daemon/http/index.ts b/src/daemon/http/index.ts index 503d754..1cef897 100644 --- a/src/daemon/http/index.ts +++ b/src/daemon/http/index.ts @@ -549,6 +549,19 @@ export function createDaemonRequestHandler(deps: { return; } + if (req.method === "GET" && url.pathname === "/wallet/doctor") { + await respond(res, async () => { + if (!deps.walletClient.diagnoseWallet) { + throw new WalletHttpError( + 501, + "Wallet doctor is not supported by this wallet client.", + ); + } + return { output: await deps.walletClient.diagnoseWallet() }; + }); + return; + } + if (req.method === "POST" && url.pathname === "/wallet/recover") { await respond(res, async () => { if (!deps.walletClient.recoverMintQuotes) { diff --git a/src/daemon/http/wallet-doctor.test.ts b/src/daemon/http/wallet-doctor.test.ts new file mode 100644 index 0000000..348c17e --- /dev/null +++ b/src/daemon/http/wallet-doctor.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it, mock } from "bun:test"; +import { EventEmitter } from "node:events"; +import { createDaemonRequestHandler } from "./index"; + +function get(path: string, walletClient: unknown) { + const handler = createDaemonRequestHandler({ walletClient } as never); + const req = new EventEmitter() as any; + Object.assign(req, { + method: "GET", + url: path, + headers: { host: "localhost" }, + }); + const res = { + status: 0, + body: "", + writeHead(status: number) { + this.status = status; + }, + end(chunk: string) { + this.body = chunk; + }, + }; + setImmediate(() => req.emit("end")); + return handler(req, res as never).then(() => res); +} + +describe("GET /wallet/doctor", () => { + it("returns the client's doctor report", async () => { + const report = { + generatedAt: 1_800_000_000_000, + mints: [{ mintUrl: "https://mint.example", reachable: true, latencyMs: 12 }], + unpaidQuotes: [], + paidUnissued: [], + stuckMelts: [], + uncheckedQuotes: 0, + }; + const diagnoseWallet = mock(async () => report); + const res = await get("/wallet/doctor", { diagnoseWallet }); + expect(res.status).toBe(200); + expect(JSON.parse(res.body)).toEqual({ output: report }); + expect(diagnoseWallet).toHaveBeenCalledTimes(1); + }); + + it("returns 501 when the wallet client has no doctor support", async () => { + const res = await get("/wallet/doctor", {}); + expect(res.status).toBe(501); + expect(JSON.parse(res.body).error).toContain("not supported"); + }); + + it("surfaces client failures as errors, not a fake-healthy report", async () => { + const diagnoseWallet = mock(async () => { + throw new Error("coco repositories unavailable"); + }); + const res = await get("/wallet/doctor", { diagnoseWallet }); + expect(res.status).toBe(500); + expect(JSON.parse(res.body).error).toContain( + "coco repositories unavailable", + ); + }); +}); diff --git a/src/daemon/wallet/coco-client.test.ts b/src/daemon/wallet/coco-client.test.ts index b020237..5548a89 100644 --- a/src/daemon/wallet/coco-client.test.ts +++ b/src/daemon/wallet/coco-client.test.ts @@ -21,6 +21,7 @@ import { isZombieProcess, reopenFailedMintOperation, runMintQuoteRecovery, + runWalletDoctor, settleExpiredMintQuotes, settlePendingMintQuotes, stopLegacyCocod, @@ -28,6 +29,7 @@ import { type MintQuoteRecoverySource, type PendingMintQuoteSource, type PendingMintSweepState, + type WalletDoctorSource, } from "./coco-client"; import { OperationInProgressError } from "@cashu/coco-core"; import { logger } from "../../utils/logger"; @@ -1728,3 +1730,215 @@ describe("createRecoveryGate", () => { await waiting; }); }); + +describe("runWalletDoctor", () => { + const NOW = Date.now(); + + function doctorMintOp(overrides: Record = {}) { + return { + id: "op-1", + mintUrl: "https://mint.example", + quoteId: "quote-1", + state: "pending", + amount: 500, + expiry: Math.floor(NOW / 1000) + 600, + createdAt: NOW - 5 * 60_000, + updatedAt: NOW - 60_000, + ...overrides, + }; + } + + function doctorMeltOp(overrides: Record = {}) { + return { + id: "melt-1", + mintUrl: "https://mint.example", + quoteId: "melt-quote-1", + state: "prepared", + amount: 2_100, + fee_reserve: 12, + inputProofSecrets: ["secret-a"], + createdAt: NOW - 3 * 24 * 3_600_000, + updatedAt: NOW - 3 * 24 * 3_600_000, + ...overrides, + }; + } + + function fakeDoctorSource(overrides: Partial = {}) { + const fetchQuoteState = mock(async (_mintUrl: string, _quoteId: string) => "UNPAID"); + const source: WalletDoctorSource = { + listTrustedMintUrls: async () => ["https://mint.example"], + listPendingMintOps: async () => [], + listFailedMintOps: async () => [], + listPreparedMelts: async () => [], + listInFlightMelts: async () => [], + listFailedMelts: async () => [], + getInflightProofSecrets: async () => [], + probeMint: async (mintUrl: string) => ({ + mintUrl, + reachable: true, + latencyMs: 5, + }), + fetchQuoteState, + ...overrides, + }; + return { source, fetchQuoteState }; + } + + it("reports a healthy wallet as all-green", async () => { + const { source } = fakeDoctorSource(); + const report = await runWalletDoctor(source); + expect(report.mints).toEqual([ + { mintUrl: "https://mint.example", reachable: true, latencyMs: 5 }, + ]); + expect(report.unpaidQuotes).toEqual([]); + expect(report.paidUnissued).toEqual([]); + expect(report.stuckMelts).toEqual([]); + expect(report.uncheckedQuotes).toBe(0); + }); + + it("lists a recent quote the mint still reports UNPAID", async () => { + const { source } = fakeDoctorSource({ + listPendingMintOps: async () => [doctorMintOp() as never], + }); + const report = await runWalletDoctor(source); + expect(report.unpaidQuotes).toHaveLength(1); + expect(report.unpaidQuotes[0]).toMatchObject({ + operationId: "op-1", + amount: 500, + }); + expect(report.paidUnissued).toEqual([]); + }); + + it("flags a pending quote the mint reports PAID with the recover command", async () => { + const { source } = fakeDoctorSource({ + listPendingMintOps: async () => [doctorMintOp() as never], + fetchQuoteState: async () => "PAID", + }); + const report = await runWalletDoctor(source); + expect(report.unpaidQuotes).toEqual([]); + expect(report.paidUnissued).toHaveLength(1); + expect(report.paidUnissued[0]).toMatchObject({ + operationId: "op-1", + remoteState: "PAID", + remediation: "routstrd wallet recover --op op-1", + }); + }); + + it("flags a failed quote last seen PAID with --include-failed", async () => { + const { source } = fakeDoctorSource({ + listFailedMintOps: async () => [ + doctorMintOp({ + id: "op-9", + state: "failed", + lastObservedRemoteState: "PAID", + }) as never, + ], + fetchQuoteState: async () => "PAID", + }); + const report = await runWalletDoctor(source); + expect(report.paidUnissued).toHaveLength(1); + expect(report.paidUnissued[0]?.remediation).toBe( + "routstrd wallet recover --op op-9 --include-failed", + ); + }); + + it("does not probe failed quotes last seen UNPAID", async () => { + const { source, fetchQuoteState } = fakeDoctorSource({ + listFailedMintOps: async () => [ + doctorMintOp({ + state: "failed", + lastObservedRemoteState: "UNPAID", + }) as never, + ], + }); + const report = await runWalletDoctor(source); + expect(fetchQuoteState).not.toHaveBeenCalled(); + expect(report.paidUnissued).toEqual([]); + expect(report.uncheckedQuotes).toBe(0); + }); + + it("probes each target quote only once", async () => { + const { source, fetchQuoteState } = fakeDoctorSource({ + listPendingMintOps: async () => [doctorMintOp() as never], + }); + fetchQuoteState.mockImplementation(async () => "PAID"); + await runWalletDoctor(source); + expect(fetchQuoteState).toHaveBeenCalledTimes(1); + expect(fetchQuoteState).toHaveBeenCalledWith( + "https://mint.example", + "quote-1", + ); + }); + + it("counts probe failures as unchecked instead of dropping them", async () => { + const { source } = fakeDoctorSource({ + listPendingMintOps: async () => [doctorMintOp() as never], + fetchQuoteState: async () => { + throw new Error("mint unreachable"); + }, + }); + const report = await runWalletDoctor(source); + expect(report.uncheckedQuotes).toBe(1); + expect(report.unpaidQuotes).toEqual([]); + expect(report.paidUnissued).toEqual([]); + }); + + it("counts quotes the probe budget never reached as unchecked", async () => { + const { source, fetchQuoteState } = fakeDoctorSource({ + listPendingMintOps: async () => [ + doctorMintOp() as never, + doctorMintOp({ id: "op-2", quoteId: "quote-2" }) as never, + ], + quoteProbeBudgetMs: 0, + }); + const report = await runWalletDoctor(source); + expect(fetchQuoteState).not.toHaveBeenCalled(); + expect(report.uncheckedQuotes).toBe(2); + }); + + it("passes unreachable mint probes through", async () => { + const { source } = fakeDoctorSource({ + probeMint: async (mintUrl: string) => ({ + mintUrl, + reachable: false, + error: "fetch failed", + }), + }); + const report = await runWalletDoctor(source); + expect(report.mints).toEqual([ + { mintUrl: "https://mint.example", reachable: false, error: "fetch failed" }, + ]); + }); + + it("flags old prepared melts and failed melts with locked proofs", async () => { + const { source } = fakeDoctorSource({ + listPreparedMelts: async () => [doctorMeltOp()], + listFailedMelts: async () => [ + doctorMeltOp({ id: "melt-2", state: "failed" }), + ], + getInflightProofSecrets: async () => ["secret-a"], + }); + const report = await runWalletDoctor(source); + expect(report.stuckMelts).toHaveLength(2); + const byKind = new Map(report.stuckMelts.map((melt) => [melt.kind, melt])); + expect(byKind.get("prepared")).toMatchObject({ + operationId: "melt-1", + feeReserve: 12, + }); + expect(byKind.get("failed-locked")).toMatchObject({ + operationId: "melt-2", + lockedSecrets: 1, + }); + }); + + it("ignores failed melts whose proofs were released", async () => { + const { source } = fakeDoctorSource({ + listFailedMelts: async () => [ + doctorMeltOp({ id: "melt-2", state: "failed" }), + ], + getInflightProofSecrets: async () => [], + }); + const report = await runWalletDoctor(source); + expect(report.stuckMelts).toEqual([]); + }); +}); diff --git a/src/daemon/wallet/coco-client.ts b/src/daemon/wallet/coco-client.ts index 48d366e..bac6697 100644 --- a/src/daemon/wallet/coco-client.ts +++ b/src/daemon/wallet/coco-client.ts @@ -39,6 +39,18 @@ import type { WalletRecoveryProgress, } from "./wallet-client"; import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup"; +import { + classifyPaidUnissued, + classifyStuckMelt, + mintQuoteStateToCategory, + selectPaidUnissuedCandidates, + selectRecentMintQuotes, + toUnpaidQuote, + type DoctorMeltOperation, + type DoctorMintOperation, + type DoctorMintProbe, + type WalletDoctorReport, +} from "./doctor"; import { classifyMintQuoteObservation, selectMintQuotesForRecovery, @@ -731,6 +743,31 @@ async function enableCocoManager(coco: Manager): Promise { */ const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000; +/** Per-request timeout for the doctor's NUT-06/NUT-04 mint probes. */ +const DOCTOR_MINT_PROBE_TIMEOUT_MS = 4_000; +/** Overall budget for the doctor's quote-state probe round. */ +const DOCTOR_QUOTE_PROBE_BUDGET_MS = 15_000; +/** How many doctor quote-state probes run at once. */ +const DOCTOR_QUOTE_PROBE_CONCURRENCY = 4; + +/** + * Private coco repositories the doctor reads failed operations and locked + * proofs through. Read-only access only; resolved fail-closed (see + * doctorRepositories) so a coco upgrade that renames them surfaces + * immediately instead of silently skipping health checks. + */ +interface MintOperationRepositoryDoctor { + getByState(state: string): Promise; +} + +interface MeltOperationRepositoryDoctor { + getByState(state: string): Promise>>; +} + +interface ProofRepositoryDoctor { + getInflightProofs(mintUrls?: string[]): Promise>; +} + /** Rejects when `timeoutMs` elapses before `promise` settles. */ function withTimeout(promise: Promise, timeoutMs: number): Promise { if (timeoutMs === Infinity) return promise; @@ -1810,6 +1847,240 @@ export async function runWalletRecovery( } } +/** + * Structural source for the wallet doctor, so the orchestration is testable + * without a wallet database or network access (mirrors + * MintQuoteRecoverySource). Every method is a READ: the doctor never + * observes-and-persists, finalizes, or fails operations. + */ +export interface WalletDoctorSource { + listTrustedMintUrls(): Promise; + /** Pending and executing mint operations. */ + listPendingMintOps(): Promise; + /** Terminally failed mint operations (via coco's private repository). */ + listFailedMintOps(): Promise; + listPreparedMelts(): Promise>>; + listInFlightMelts(): Promise>>; + listFailedMelts(): Promise>>; + /** Secrets of proofs currently locked (`inflight`). */ + getInflightProofSecrets(): Promise; + /** NUT-06 reachability probe for one mint. */ + probeMint(mintUrl: string): Promise; + /** Read-only NUT-04 quote state fetch (no persistence). */ + fetchQuoteState(mintUrl: string, quoteId: string): Promise; + /** Probe budget/concurrency overrides for tests. */ + quoteProbeBudgetMs?: number; + quoteProbeConcurrency?: number; +} + +/** + * Run the read-only wallet health checks behind `routstrd wallet doctor`. + * + * Four checks, all driven by the pure helpers in doctor.ts: + * + * 1. Mint reachability: every trusted mint gets a NUT-06 probe, in parallel. + * 2. Recent unpaid quotes: pending quotes created in the last hour whose + * mint still reports UNPAID - informational, an invoice awaiting payment. + * 3. Paid but not issued: the stuck scenario `wallet recover` remediates. + * Candidates share recovery's selection rules; the remote state is + * classified by the same helper recovery uses, so doctor and recovery + * never disagree about what is claimable. + * 4. Stuck melts: prepared melts holding reserved proofs, in-flight melts, + * and failed melts whose input proofs were never released. + * + * Quote-state probes are read-only NUT-04 fetches (never coco's + * observe-and-persist path), bounded by an overall budget and a concurrency + * limit so a slow mint cannot stall the whole report. Quotes that could not + * be checked are counted in `uncheckedQuotes` rather than silently dropped. + */ +export async function runWalletDoctor( + source: WalletDoctorSource, +): Promise { + const nowMs = Date.now(); + const report: WalletDoctorReport = { + generatedAt: nowMs, + mints: [], + unpaidQuotes: [], + paidUnissued: [], + stuckMelts: [], + uncheckedQuotes: 0, + }; + + const trustedMintUrls = await source.listTrustedMintUrls(); + report.mints = await Promise.all( + trustedMintUrls.map((mintUrl) => source.probeMint(mintUrl)), + ); + + const [pendingMintOps, failedMintOps] = await Promise.all([ + source.listPendingMintOps(), + source.listFailedMintOps(), + ]); + const recentQuotes = selectRecentMintQuotes(pendingMintOps, nowMs); + const recentIds = new Set(recentQuotes.map((op) => op.id)); + const candidates = selectPaidUnissuedCandidates([ + ...pendingMintOps, + ...failedMintOps, + ]); + + const targets = new Map(); + for (const op of [...recentQuotes, ...candidates]) { + if (!targets.has(op.id)) targets.set(op.id, op); + } + const queue = [...targets.values()]; + const budgetMs = source.quoteProbeBudgetMs ?? DOCTOR_QUOTE_PROBE_BUDGET_MS; + const concurrency = + source.quoteProbeConcurrency ?? DOCTOR_QUOTE_PROBE_CONCURRENCY; + const deadlineAt = nowMs + budgetMs; + let next = 0; + await Promise.all( + Array.from({ length: Math.min(concurrency, queue.length) }, async () => { + while (next < queue.length && Date.now() < deadlineAt) { + const op = queue[next++]; + if (!op) break; + try { + const remoteState = await source.fetchQuoteState( + op.mintUrl, + op.quoteId as string, + ); + if ( + recentIds.has(op.id) && + mintQuoteStateToCategory(remoteState) === "waiting" + ) { + report.unpaidQuotes.push(toUnpaidQuote(op, nowMs)); + } + const finding = classifyPaidUnissued(op, remoteState); + if (finding) report.paidUnissued.push(finding); + } catch { + report.uncheckedQuotes++; + } + } + }), + ); + report.uncheckedQuotes += queue.length - next; + // Concurrent workers push in completion order; sort for a stable report. + report.unpaidQuotes.sort((a, b) => a.operationId.localeCompare(b.operationId)); + report.paidUnissued.sort((a, b) => a.operationId.localeCompare(b.operationId)); + + const [preparedMelts, inFlightMelts, failedMelts, inflightSecretsList] = + await Promise.all([ + source.listPreparedMelts(), + source.listInFlightMelts(), + source.listFailedMelts(), + source.getInflightProofSecrets(), + ]); + const inflightSecrets = new Set(inflightSecretsList); + for (const raw of [...preparedMelts, ...inFlightMelts, ...failedMelts]) { + const finding = classifyStuckMelt(toDoctorMeltOperation(raw), { + nowMs, + inflightSecrets, + }); + if (finding) report.stuckMelts.push(finding); + } + report.stuckMelts.sort((a, b) => a.operationId.localeCompare(b.operationId)); + + return report; +} + +/** Map coco's melt operation rows onto the doctor's structural subset. */ +function toDoctorMeltOperation( + raw: Record, +): DoctorMeltOperation { + return { + id: String(raw.id), + mintUrl: String(raw.mintUrl), + quoteId: typeof raw.quoteId === "string" ? raw.quoteId : undefined, + state: String(raw.state), + amount: Number(raw.amount ?? 0), + feeReserve: Number(raw.fee_reserve ?? 0), + inputProofSecrets: Array.isArray(raw.inputProofSecrets) + ? raw.inputProofSecrets.map((secret) => String(secret)) + : [], + createdAt: Number(raw.createdAt ?? 0), + updatedAt: Number(raw.updatedAt ?? 0), + error: typeof raw.error === "string" ? raw.error : undefined, + }; +} + +/** + * Resolve coco's private repositories for the doctor. Fails closed - a coco + * upgrade that renames them must surface here and in the doctor tests, not + * silently skip the failed-operation and locked-proof checks. + */ +function doctorRepositories(coco: Manager): { + mintOps: MintOperationRepositoryDoctor; + meltOps: MeltOperationRepositoryDoctor; + proofs: ProofRepositoryDoctor; +} { + const inner = coco as unknown as { + mintOperationRepository?: MintOperationRepositoryDoctor; + meltOperationRepository?: MeltOperationRepositoryDoctor; + proofRepository?: ProofRepositoryDoctor; + }; + const checks: Array<[string, object | undefined, string]> = [ + ["mintOperationRepository", inner.mintOperationRepository, "getByState"], + ["meltOperationRepository", inner.meltOperationRepository, "getByState"], + ["proofRepository", inner.proofRepository, "getInflightProofs"], + ]; + for (const [name, repo, method] of checks) { + if ( + !repo || + typeof (repo as Record)[method] !== "function" + ) { + throw new Error( + `coco ${name}.${method} is unavailable; refusing to run wallet doctor with partial health checks`, + ); + } + } + return { + mintOps: inner.mintOperationRepository as MintOperationRepositoryDoctor, + meltOps: inner.meltOperationRepository as MeltOperationRepositoryDoctor, + proofs: inner.proofRepository as ProofRepositoryDoctor, + }; +} + +/** Read-only NUT-06 reachability probe for one mint. */ +export async function probeMintInfo(mintUrl: string): Promise { + const startedAt = Date.now(); + try { + const response = await fetch(`${mintUrl.replace(/\/+$/, "")}/v1/info`, { + signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return { + mintUrl, + reachable: true, + latencyMs: Date.now() - startedAt, + }; + } catch (error) { + return { + mintUrl, + reachable: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +/** + * Read-only NUT-04 quote state fetch. Unlike coco's + * observePendingOperation this persists nothing, which is what keeps the + * doctor free of side effects. + */ +export async function fetchMintQuoteState( + mintUrl: string, + quoteId: string, +): Promise { + const response = await fetch( + `${mintUrl.replace(/\/+$/, "")}/v1/mint/quote/bolt11/${encodeURIComponent(quoteId)}`, + { signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS) }, + ); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = (await response.json()) as { state?: unknown }; + if (typeof body.state !== "string") { + throw new Error("mint response missing quote state"); + } + return body.state; +} + export async function createCocoClient( options: CreateCocoClientOptions = {}, ): Promise { @@ -2602,6 +2873,39 @@ export async function createCocoClient( }; }, + async diagnoseWallet() { + const repos = doctorRepositories(coco); + return runWalletDoctor({ + listTrustedMintUrls: async () => + (await coco.mint.getAllTrustedMints()).map((mint) => mint.mintUrl), + listPendingMintOps: async () => { + // listInFlight can overlap with listPending; dedupe by id. + const byId = new Map(); + for (const op of [ + ...(await coco.ops.mint.listPending()), + ...(await coco.ops.mint.listInFlight()), + ] as unknown as DoctorMintOperation[]) { + byId.set(op.id, op); + } + return [...byId.values()]; + }, + listFailedMintOps: () => repos.mintOps.getByState("failed"), + listPreparedMelts: async () => + coco.ops.melt.listPrepared() as unknown as Array< + Record + >, + listInFlightMelts: async () => + coco.ops.melt.listInFlight() as unknown as Array< + Record + >, + listFailedMelts: () => repos.meltOps.getByState("failed"), + getInflightProofSecrets: async () => + (await repos.proofs.getInflightProofs()).map((proof) => proof.secret), + probeMint: probeMintInfo, + fetchQuoteState: fetchMintQuoteState, + }); + }, + async recoverMintQuotes(options, onProgress) { await waitForRecovery(); const service = ( diff --git a/src/daemon/wallet/doctor.ts b/src/daemon/wallet/doctor.ts index 5a04f76..386a14b 100644 --- a/src/daemon/wallet/doctor.ts +++ b/src/daemon/wallet/doctor.ts @@ -142,7 +142,7 @@ export interface WalletDoctorReport { paidUnissued: DoctorPaidUnissuedQuote[]; /** Melt operations holding locked proofs. */ stuckMelts: DoctorStuckMelt[]; - /** Quotes skipped because the probe budget ran out. */ + /** Quotes whose remote state could not be checked (probe failure or budget exhausted). */ uncheckedQuotes: number; } diff --git a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts index 69b0c06..593e795 100644 --- a/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts +++ b/src/daemon/wallet/mint-quote-recovery.fake-mint.test.ts @@ -15,7 +15,15 @@ import { join } from "node:path"; import { Manager } from "@cashu/coco-core"; import { SqliteRepositories } from "@cashu/coco-sqlite-bun"; import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint"; -import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client"; +import { + fetchMintQuoteState, + probeMintInfo, + reopenFailedMintOperation, + runMintQuoteRecovery, + runWalletDoctor, + type WalletDoctorSource, +} from "./coco-client"; +import type { DoctorMintOperation } from "./doctor"; type AnyRecord = Record; @@ -384,3 +392,101 @@ describe("PAID mint quote recovery with a real Manager and mint", () => { expect(await booted.spendable()).toBe(21_000); }); }); + +describe("wallet doctor with a real Manager and mint", () => { + function doctorSource(booted: Booted): WalletDoctorSource { + const manager = booted.manager; + const repos = manager as unknown as { + mintOperationRepository: { + getByState(state: string): Promise; + }; + meltOperationRepository: { + getByState(state: string): Promise; + }; + proofRepository: { + getInflightProofs(): Promise>; + }; + }; + return { + listTrustedMintUrls: async () => + (await manager.mint.getAllTrustedMints()).map((mint) => mint.mintUrl), + listPendingMintOps: async () => { + const byId = new Map(); + for (const op of [ + ...(await manager.ops.mint.listPending()), + ...(await manager.ops.mint.listInFlight()), + ] as unknown as DoctorMintOperation[]) { + byId.set(op.id, op); + } + return [...byId.values()]; + }, + listFailedMintOps: () => repos.mintOperationRepository.getByState("failed"), + listPreparedMelts: async () => + manager.ops.melt.listPrepared() as unknown as AnyRecord[], + listInFlightMelts: async () => + manager.ops.melt.listInFlight() as unknown as AnyRecord[], + listFailedMelts: () => repos.meltOperationRepository.getByState("failed"), + getInflightProofSecrets: async () => + (await repos.proofRepository.getInflightProofs()).map( + (proof) => proof.secret, + ), + probeMint: probeMintInfo, + fetchQuoteState: fetchMintQuoteState, + }; + } + + it("surfaces unpaid, then paid-unissued, then a clean bill after recovery", async () => { + booted = await boot({ quoteExpiry: null }); + const op = await prepareQuote(booted, 100); + + // Fresh quote, unpaid: the doctor reaches the mint and lists the quote. + const unpaid = await runWalletDoctor(doctorSource(booted)); + expect(unpaid.mints).toHaveLength(1); + expect(unpaid.mints[0]?.reachable).toBe(true); + expect(unpaid.paidUnissued).toEqual([]); + expect(unpaid.uncheckedQuotes).toBe(0); + expect(unpaid.unpaidQuotes).toHaveLength(1); + expect(unpaid.unpaidQuotes[0]).toMatchObject({ + operationId: op.id as string, + amount: 100, + }); + + // The quote gets paid while the daemon is "down": the doctor surfaces + // exactly the stuck scenario recovery exists for, with the recover hint. + booted.mint.markPaid(op.quoteId as string); + const stuck = await runWalletDoctor(doctorSource(booted)); + expect(stuck.unpaidQuotes).toEqual([]); + expect(stuck.paidUnissued).toHaveLength(1); + expect(stuck.paidUnissued[0]).toMatchObject({ + operationId: op.id as string, + remoteState: "PAID", + amount: 100, + remediation: `routstrd wallet recover --op ${op.id as string}`, + }); + + // Recovery claims the sats; the doctor goes back to all-green. + const recovered = (await runMintQuoteRecovery( + booted.source() as never, + )) as unknown as Record; + expect(recovered).toMatchObject({ recovered: 1 }); + const clean = await runWalletDoctor(doctorSource(booted)); + expect(clean.unpaidQuotes).toEqual([]); + expect(clean.paidUnissued).toEqual([]); + expect(clean.stuckMelts).toEqual([]); + expect(clean.uncheckedQuotes).toBe(0); + }); + + it("reports unreachable mints without failing the other checks", async () => { + booted = await boot({ quoteExpiry: null }); + await prepareQuote(booted, 100); + booted.mint.stop(); + + const report = await runWalletDoctor(doctorSource(booted)); + expect(report.mints[0]?.reachable).toBe(false); + // The quote probe fails too, and is counted rather than dropped. + expect(report.uncheckedQuotes).toBe(1); + expect(report.unpaidQuotes).toEqual([]); + expect(report.paidUnissued).toEqual([]); + booted.mint.start(); + }); +});