feat(wallet): add live health checks to wallet doctor

wire the doctor helpers from the previous commit into a read-only health
check surfaced by the daemon and the CLI:

- coco-client: runWalletDoctor orchestrates the four checks against an
  injected structural source (unit-testable like runMintQuoteRecovery);
  diagnoseWallet() wires it to coco - trusted mints, pending/in-flight
  mint ops, and failed ops + inflight proofs via coco's private
  repositories, resolved fail-closed so a coco upgrade that renames them
  surfaces immediately. Quote checks are plain NUT-04 reads (never coco's
  observe-and-persist path), bounded by an overall budget and a
  concurrency limit; uncheckable quotes are counted, not dropped.
- http: GET /wallet/doctor returns the report (501 when the wallet client
  has no doctor support).
- cli: wallet doctor now prints the health section first (mint
  reachability, recent unpaid quotes, paid-but-not-issued with recover
  remediation, stuck melts), then the existing offline migration
  diagnosis. NO_COLOR/TTY-aware colors, --json for scripting, exit 1 when
  money is provably at risk or the wallets conflict. When the daemon is
  down the live section is skipped rather than auto-starting the daemon.
- tests: HTTP route tests, runWalletDoctor unit tests with a fake source,
  and real-Manager + fake-mint integration tests covering the full
  unpaid -> paid-unissued -> recovered -> clean arc and unreachable-mint
  degradation.
- docs: wallet doctor section in wallet-mint-recovery.md.
This commit is contained in:
redshift
2026-10-04 20:52:16 +08:00
parent 50c69656bd
commit 9c75d2e202
9 changed files with 998 additions and 6 deletions
+21
View File
@@ -46,6 +46,27 @@ and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate
with coco's watcher/processor. It needs its own integration tests before handling
real funds.
## Wallet doctor
`routstrd wallet doctor` runs read-only health checks before the legacy
migration diagnosis:
1. **Mint reachability** — a NUT-06 probe against every trusted mint.
2. **Recent unpaid quotes** — pending quotes from the last hour the mint
still reports UNPAID (an invoice awaiting payment; informational).
3. **Paid but not issued** — the stuck scenario this recovery feature
fixes. Each finding prints its `wallet recover --op <id>` remediation,
with `--include-failed` when the operation must be re-opened first.
4. **Stuck melts** — prepared melts holding reserved proofs, in-flight
melts, and failed melts whose input proofs were never released.
The doctor never mutates wallet state: quote checks are plain NUT-04 reads,
not coco's observe-and-persist path, and remediation is always left to the
operator. It exits non-zero when money is provably at risk (unreachable
mint, paid-but-unissued quote, failed melt with locked proofs) or the
migration diagnosis finds a conflict, so it can gate scripts. When the
daemon is down only the offline migration section runs.
## Cleanup preview and force
`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not
+127
View File
@@ -0,0 +1,127 @@
import { describe, expect, it } from "bun:test";
import { renderWalletHealth } from "./cli";
import type { WalletDoctorReport } from "./daemon/wallet/doctor";
const NOW = 1_800_000_000_000;
function report(overrides: Partial<WalletDoctorReport> = {}): WalletDoctorReport {
return {
generatedAt: NOW,
mints: [],
unpaidQuotes: [],
paidUnissued: [],
stuckMelts: [],
uncheckedQuotes: 0,
...overrides,
};
}
describe("renderWalletHealth", () => {
it("renders an all-green report", () => {
const output = renderWalletHealth(
report({
mints: [
{ mintUrl: "https://mint.example", reachable: true, latencyMs: 212 },
],
}),
);
expect(output).toContain("Wallet health");
expect(output).toContain("✓ https://mint.example — reachable (212 ms)");
expect(output).toContain("none unpaid");
expect(output).toContain("none stuck");
});
it("renders unreachable mints with their error", () => {
const output = renderWalletHealth(
report({
mints: [
{ mintUrl: "https://mint.down", reachable: false, error: "fetch failed" },
],
}),
);
expect(output).toContain("✗ https://mint.down — unreachable: fetch failed");
});
it("renders unpaid quotes with age and expiry", () => {
const output = renderWalletHealth(
report({
unpaidQuotes: [
{
operationId: "op-1234567890abcdef",
quoteId: "quote-abcdef1234567890",
mintUrl: "https://mint.example",
amount: 500,
ageMs: 5 * 60_000,
expiresInMs: 41 * 60_000,
},
],
}),
);
expect(output).toContain("⚠ quote-ab… at https://mint.example — 500 sat, UNPAID");
expect(output).toContain("created 5m ago, expires in 41m");
});
it("renders paid-unissued findings with the recover remediation", () => {
const output = renderWalletHealth(
report({
paidUnissued: [
{
operationId: "op-1234567890abcdef",
quoteId: "quote-abcdef1234567890",
mintUrl: "https://mint.example",
amount: 1_000,
localState: "failed",
remoteState: "PAID",
error: "keyset id inactive.",
remediation: "routstrd wallet recover --op op-1234567890abcdef --include-failed",
},
],
}),
);
expect(output).toContain("✗ op-12345… (quote quote-ab…)");
expect(output).toContain("1,000 sat PAID at mint, local state failed");
expect(output).toContain("mint error: keyset id inactive.");
expect(output).toContain(
"→ routstrd wallet recover --op op-1234567890abcdef --include-failed",
);
});
it("renders stuck melts with locked totals and remediation", () => {
const output = renderWalletHealth(
report({
stuckMelts: [
{
operationId: "melt-1234567890",
mintUrl: "https://mint.example",
amount: 2_100,
feeReserve: 12,
ageMs: 3 * 86_400_000,
kind: "prepared",
lockedSecrets: 4,
remediation: "routstrd wallet cleanup",
},
{
operationId: "melt-abcdef",
mintUrl: "https://mint.example",
amount: 50,
feeReserve: 2,
ageMs: 90_000,
kind: "failed-locked",
lockedSecrets: 1,
remediation: "restart the daemon",
},
],
}),
);
expect(output).toContain(
"⚠ melt-123… at https://mint.example — 2,112 sat locked (4 proof(s)), prepared, payment never attempted, 3d old",
);
expect(output).toContain("✗ melt-abcdef at");
expect(output).toContain("failed but proofs still locked");
});
it("notes quotes that could not be checked", () => {
const output = renderWalletHealth(report({ uncheckedQuotes: 3 }));
expect(output).toContain("3 quote(s) could not be checked with their mint");
});
});
+151 -4
View File
@@ -49,6 +49,11 @@ import {
summarizeWalletDirectory,
WalletMigrationConflictError,
} from "./daemon/wallet/diagnostics";
import {
doctorReportSeverity,
DOCTOR_RECENT_QUOTE_WINDOW_MS,
type WalletDoctorReport,
} from "./daemon/wallet/doctor";
import {
legacyCocodDir,
legacyCocodPidPath,
@@ -2062,14 +2067,156 @@ walletCmd
await handleDaemonCommand("/wallet/status");
});
const doctorUseColor = !process.env.NO_COLOR && process.stdout.isTTY === true;
const doctorPaint = (code: string, text: string): string =>
doctorUseColor ? `\x1b[${code}m${text}\x1b[0m` : text;
const doctorGreen = (text: string) => doctorPaint("32", text);
const doctorRed = (text: string) => doctorPaint("31", text);
const doctorYellow = (text: string) => doctorPaint("33", text);
function shortenOperationId(id: string): string {
return id.length > 12 ? `${id.slice(0, 8)}…` : id;
}
function formatSats(amount: number): string {
return amount.toLocaleString("en-US");
}
function formatDoctorDuration(ms: number): string {
const abs = Math.abs(ms);
if (abs < 60_000) return `${Math.max(1, Math.round(abs / 1000))}s`;
if (abs < 3_600_000) return `${Math.round(abs / 60_000)}m`;
if (abs < 86_400_000) return `${Math.round(abs / 3_600_000)}h`;
return `${Math.round(abs / 86_400_000)}d`;
}
/** Render the daemon's live doctor report; colors are NO_COLOR/TTY aware. */
export function renderWalletHealth(report: WalletDoctorReport): string {
const lines: string[] = ["Wallet health", "=============", ""];
lines.push("Mints");
if (report.mints.length === 0) {
lines.push(" (no trusted mints)");
}
for (const probe of report.mints) {
if (probe.reachable) {
const latency =
probe.latencyMs !== undefined ? ` (${probe.latencyMs} ms)` : "";
lines.push(` ${doctorGreen("✓")} ${probe.mintUrl} — reachable${latency}`);
} else {
lines.push(
` ${doctorRed("✗")} ${probe.mintUrl} — unreachable${probe.error ? `: ${probe.error}` : ""}`,
);
}
}
lines.push(
"",
`Mint quotes (last ${formatDoctorDuration(DOCTOR_RECENT_QUOTE_WINDOW_MS)})`,
);
if (report.unpaidQuotes.length === 0) {
lines.push(" none unpaid");
}
for (const quote of report.unpaidQuotes) {
const expiry =
quote.expiresInMs === undefined
? ""
: quote.expiresInMs >= 0
? `, expires in ${formatDoctorDuration(quote.expiresInMs)}`
: `, expired ${formatDoctorDuration(quote.expiresInMs)} ago`;
lines.push(
` ${doctorYellow("⚠")} ${shortenOperationId(quote.quoteId ?? quote.operationId)} at ${quote.mintUrl} — ${formatSats(quote.amount)} sat, UNPAID, created ${formatDoctorDuration(quote.ageMs)} ago${expiry}`,
);
}
lines.push("", "Paid but not issued");
if (report.paidUnissued.length === 0) {
lines.push(" none");
}
for (const finding of report.paidUnissued) {
lines.push(
` ${doctorRed("✗")} ${shortenOperationId(finding.operationId)} (quote ${shortenOperationId(finding.quoteId ?? "?")}) at ${finding.mintUrl} — ${formatSats(finding.amount)} sat ${finding.remoteState} at mint, local state ${finding.localState}`,
);
if (finding.error) lines.push(` mint error: ${finding.error}`);
lines.push(` → ${finding.remediation}`);
}
lines.push("", "Melts");
if (report.stuckMelts.length === 0) {
lines.push(" none stuck");
}
for (const melt of report.stuckMelts) {
const icon = melt.kind === "failed-locked" ? doctorRed("✗") : doctorYellow("⚠");
const what =
melt.kind === "prepared"
? "prepared, payment never attempted"
: melt.kind === "in-flight"
? "payment in flight"
: "failed but proofs still locked";
lines.push(
` ${icon} ${shortenOperationId(melt.operationId)} at ${melt.mintUrl} — ${formatSats(melt.amount + melt.feeReserve)} sat locked (${melt.lockedSecrets} proof(s)), ${what}, ${formatDoctorDuration(melt.ageMs)} old`,
);
lines.push(` → ${melt.remediation}`);
}
if (report.uncheckedQuotes > 0) {
lines.push(
"",
` ${doctorYellow("⚠")} ${report.uncheckedQuotes} quote(s) could not be checked with their mint`,
);
}
return lines.join("\n");
}
walletCmd
.command("doctor")
.description("Diagnose conflicting wallets (current routstrd wallet vs legacy cocod)")
.action(async () => {
.description(
"Wallet health checks (mint reachability, unpaid/stuck quotes, locked melts), then legacy wallet migration diagnosis",
)
.option("--json", "Print the raw health report as JSON", false)
.action(async (options: { json: boolean }) => {
const target = summarizeWalletDirectory(defaultWalletDir(), "canonical");
const source = summarizeWalletDirectory(legacyCocodDir(), "legacy");
console.log(renderWalletDoctor(target, source));
if (diagnoseWallets(target, source).conflict) process.exit(1);
const migrationConflict = diagnoseWallets(target, source).conflict;
// Live health checks need the daemon; when it is down the offline
// migration diagnosis still runs, because a broken startup is exactly
// when the doctor gets invoked.
let report: WalletDoctorReport | undefined;
try {
const result = await callDaemon("/wallet/doctor");
if (result.error) throw new Error(result.error);
report = result.output as WalletDoctorReport | undefined;
} catch {
report = undefined;
}
if (options.json) {
console.log(
JSON.stringify(
{ health: report ?? null, migrationConflict },
null,
2,
),
);
} else {
if (report) {
console.log(renderWalletHealth(report));
} else {
console.log("Wallet health");
console.log("=============");
console.log("");
console.log(
" skipped: daemon is not running; live checks need it (routstrd start)",
);
}
console.log("");
console.log(renderWalletDoctor(target, source));
}
const healthCritical =
report !== undefined && doctorReportSeverity(report) === "critical";
if (healthCritical || migrationConflict) process.exit(1);
});
walletCmd
+13
View File
@@ -549,6 +549,19 @@ export function createDaemonRequestHandler(deps: {
return;
}
if (req.method === "GET" && url.pathname === "/wallet/doctor") {
await respond(res, async () => {
if (!deps.walletClient.diagnoseWallet) {
throw new WalletHttpError(
501,
"Wallet doctor is not supported by this wallet client.",
);
}
return { output: await deps.walletClient.diagnoseWallet() };
});
return;
}
if (req.method === "POST" && url.pathname === "/wallet/recover") {
await respond(res, async () => {
if (!deps.walletClient.recoverMintQuotes) {
+60
View File
@@ -0,0 +1,60 @@
import { describe, expect, it, mock } from "bun:test";
import { EventEmitter } from "node:events";
import { createDaemonRequestHandler } from "./index";
function get(path: string, walletClient: unknown) {
const handler = createDaemonRequestHandler({ walletClient } as never);
const req = new EventEmitter() as any;
Object.assign(req, {
method: "GET",
url: path,
headers: { host: "localhost" },
});
const res = {
status: 0,
body: "",
writeHead(status: number) {
this.status = status;
},
end(chunk: string) {
this.body = chunk;
},
};
setImmediate(() => req.emit("end"));
return handler(req, res as never).then(() => res);
}
describe("GET /wallet/doctor", () => {
it("returns the client's doctor report", async () => {
const report = {
generatedAt: 1_800_000_000_000,
mints: [{ mintUrl: "https://mint.example", reachable: true, latencyMs: 12 }],
unpaidQuotes: [],
paidUnissued: [],
stuckMelts: [],
uncheckedQuotes: 0,
};
const diagnoseWallet = mock(async () => report);
const res = await get("/wallet/doctor", { diagnoseWallet });
expect(res.status).toBe(200);
expect(JSON.parse(res.body)).toEqual({ output: report });
expect(diagnoseWallet).toHaveBeenCalledTimes(1);
});
it("returns 501 when the wallet client has no doctor support", async () => {
const res = await get("/wallet/doctor", {});
expect(res.status).toBe(501);
expect(JSON.parse(res.body).error).toContain("not supported");
});
it("surfaces client failures as errors, not a fake-healthy report", async () => {
const diagnoseWallet = mock(async () => {
throw new Error("coco repositories unavailable");
});
const res = await get("/wallet/doctor", { diagnoseWallet });
expect(res.status).toBe(500);
expect(JSON.parse(res.body).error).toContain(
"coco repositories unavailable",
);
});
});
+214
View File
@@ -21,6 +21,7 @@ import {
isZombieProcess,
reopenFailedMintOperation,
runMintQuoteRecovery,
runWalletDoctor,
settleExpiredMintQuotes,
settlePendingMintQuotes,
stopLegacyCocod,
@@ -28,6 +29,7 @@ import {
type MintQuoteRecoverySource,
type PendingMintQuoteSource,
type PendingMintSweepState,
type WalletDoctorSource,
} from "./coco-client";
import { OperationInProgressError } from "@cashu/coco-core";
import { logger } from "../../utils/logger";
@@ -1728,3 +1730,215 @@ describe("createRecoveryGate", () => {
await waiting;
});
});
describe("runWalletDoctor", () => {
const NOW = Date.now();
function doctorMintOp(overrides: Record<string, unknown> = {}) {
return {
id: "op-1",
mintUrl: "https://mint.example",
quoteId: "quote-1",
state: "pending",
amount: 500,
expiry: Math.floor(NOW / 1000) + 600,
createdAt: NOW - 5 * 60_000,
updatedAt: NOW - 60_000,
...overrides,
};
}
function doctorMeltOp(overrides: Record<string, unknown> = {}) {
return {
id: "melt-1",
mintUrl: "https://mint.example",
quoteId: "melt-quote-1",
state: "prepared",
amount: 2_100,
fee_reserve: 12,
inputProofSecrets: ["secret-a"],
createdAt: NOW - 3 * 24 * 3_600_000,
updatedAt: NOW - 3 * 24 * 3_600_000,
...overrides,
};
}
function fakeDoctorSource(overrides: Partial<WalletDoctorSource> = {}) {
const fetchQuoteState = mock(async (_mintUrl: string, _quoteId: string) => "UNPAID");
const source: WalletDoctorSource = {
listTrustedMintUrls: async () => ["https://mint.example"],
listPendingMintOps: async () => [],
listFailedMintOps: async () => [],
listPreparedMelts: async () => [],
listInFlightMelts: async () => [],
listFailedMelts: async () => [],
getInflightProofSecrets: async () => [],
probeMint: async (mintUrl: string) => ({
mintUrl,
reachable: true,
latencyMs: 5,
}),
fetchQuoteState,
...overrides,
};
return { source, fetchQuoteState };
}
it("reports a healthy wallet as all-green", async () => {
const { source } = fakeDoctorSource();
const report = await runWalletDoctor(source);
expect(report.mints).toEqual([
{ mintUrl: "https://mint.example", reachable: true, latencyMs: 5 },
]);
expect(report.unpaidQuotes).toEqual([]);
expect(report.paidUnissued).toEqual([]);
expect(report.stuckMelts).toEqual([]);
expect(report.uncheckedQuotes).toBe(0);
});
it("lists a recent quote the mint still reports UNPAID", async () => {
const { source } = fakeDoctorSource({
listPendingMintOps: async () => [doctorMintOp() as never],
});
const report = await runWalletDoctor(source);
expect(report.unpaidQuotes).toHaveLength(1);
expect(report.unpaidQuotes[0]).toMatchObject({
operationId: "op-1",
amount: 500,
});
expect(report.paidUnissued).toEqual([]);
});
it("flags a pending quote the mint reports PAID with the recover command", async () => {
const { source } = fakeDoctorSource({
listPendingMintOps: async () => [doctorMintOp() as never],
fetchQuoteState: async () => "PAID",
});
const report = await runWalletDoctor(source);
expect(report.unpaidQuotes).toEqual([]);
expect(report.paidUnissued).toHaveLength(1);
expect(report.paidUnissued[0]).toMatchObject({
operationId: "op-1",
remoteState: "PAID",
remediation: "routstrd wallet recover --op op-1",
});
});
it("flags a failed quote last seen PAID with --include-failed", async () => {
const { source } = fakeDoctorSource({
listFailedMintOps: async () => [
doctorMintOp({
id: "op-9",
state: "failed",
lastObservedRemoteState: "PAID",
}) as never,
],
fetchQuoteState: async () => "PAID",
});
const report = await runWalletDoctor(source);
expect(report.paidUnissued).toHaveLength(1);
expect(report.paidUnissued[0]?.remediation).toBe(
"routstrd wallet recover --op op-9 --include-failed",
);
});
it("does not probe failed quotes last seen UNPAID", async () => {
const { source, fetchQuoteState } = fakeDoctorSource({
listFailedMintOps: async () => [
doctorMintOp({
state: "failed",
lastObservedRemoteState: "UNPAID",
}) as never,
],
});
const report = await runWalletDoctor(source);
expect(fetchQuoteState).not.toHaveBeenCalled();
expect(report.paidUnissued).toEqual([]);
expect(report.uncheckedQuotes).toBe(0);
});
it("probes each target quote only once", async () => {
const { source, fetchQuoteState } = fakeDoctorSource({
listPendingMintOps: async () => [doctorMintOp() as never],
});
fetchQuoteState.mockImplementation(async () => "PAID");
await runWalletDoctor(source);
expect(fetchQuoteState).toHaveBeenCalledTimes(1);
expect(fetchQuoteState).toHaveBeenCalledWith(
"https://mint.example",
"quote-1",
);
});
it("counts probe failures as unchecked instead of dropping them", async () => {
const { source } = fakeDoctorSource({
listPendingMintOps: async () => [doctorMintOp() as never],
fetchQuoteState: async () => {
throw new Error("mint unreachable");
},
});
const report = await runWalletDoctor(source);
expect(report.uncheckedQuotes).toBe(1);
expect(report.unpaidQuotes).toEqual([]);
expect(report.paidUnissued).toEqual([]);
});
it("counts quotes the probe budget never reached as unchecked", async () => {
const { source, fetchQuoteState } = fakeDoctorSource({
listPendingMintOps: async () => [
doctorMintOp() as never,
doctorMintOp({ id: "op-2", quoteId: "quote-2" }) as never,
],
quoteProbeBudgetMs: 0,
});
const report = await runWalletDoctor(source);
expect(fetchQuoteState).not.toHaveBeenCalled();
expect(report.uncheckedQuotes).toBe(2);
});
it("passes unreachable mint probes through", async () => {
const { source } = fakeDoctorSource({
probeMint: async (mintUrl: string) => ({
mintUrl,
reachable: false,
error: "fetch failed",
}),
});
const report = await runWalletDoctor(source);
expect(report.mints).toEqual([
{ mintUrl: "https://mint.example", reachable: false, error: "fetch failed" },
]);
});
it("flags old prepared melts and failed melts with locked proofs", async () => {
const { source } = fakeDoctorSource({
listPreparedMelts: async () => [doctorMeltOp()],
listFailedMelts: async () => [
doctorMeltOp({ id: "melt-2", state: "failed" }),
],
getInflightProofSecrets: async () => ["secret-a"],
});
const report = await runWalletDoctor(source);
expect(report.stuckMelts).toHaveLength(2);
const byKind = new Map(report.stuckMelts.map((melt) => [melt.kind, melt]));
expect(byKind.get("prepared")).toMatchObject({
operationId: "melt-1",
feeReserve: 12,
});
expect(byKind.get("failed-locked")).toMatchObject({
operationId: "melt-2",
lockedSecrets: 1,
});
});
it("ignores failed melts whose proofs were released", async () => {
const { source } = fakeDoctorSource({
listFailedMelts: async () => [
doctorMeltOp({ id: "melt-2", state: "failed" }),
],
getInflightProofSecrets: async () => [],
});
const report = await runWalletDoctor(source);
expect(report.stuckMelts).toEqual([]);
});
});
+304
View File
@@ -39,6 +39,18 @@ import type {
WalletRecoveryProgress,
} from "./wallet-client";
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
import {
classifyPaidUnissued,
classifyStuckMelt,
mintQuoteStateToCategory,
selectPaidUnissuedCandidates,
selectRecentMintQuotes,
toUnpaidQuote,
type DoctorMeltOperation,
type DoctorMintOperation,
type DoctorMintProbe,
type WalletDoctorReport,
} from "./doctor";
import {
classifyMintQuoteObservation,
selectMintQuotesForRecovery,
@@ -731,6 +743,31 @@ async function enableCocoManager(coco: Manager): Promise<void> {
*/
const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000;
/** Per-request timeout for the doctor's NUT-06/NUT-04 mint probes. */
const DOCTOR_MINT_PROBE_TIMEOUT_MS = 4_000;
/** Overall budget for the doctor's quote-state probe round. */
const DOCTOR_QUOTE_PROBE_BUDGET_MS = 15_000;
/** How many doctor quote-state probes run at once. */
const DOCTOR_QUOTE_PROBE_CONCURRENCY = 4;
/**
* Private coco repositories the doctor reads failed operations and locked
* proofs through. Read-only access only; resolved fail-closed (see
* doctorRepositories) so a coco upgrade that renames them surfaces
* immediately instead of silently skipping health checks.
*/
interface MintOperationRepositoryDoctor {
getByState(state: string): Promise<DoctorMintOperation[]>;
}
interface MeltOperationRepositoryDoctor {
getByState(state: string): Promise<Array<Record<string, unknown>>>;
}
interface ProofRepositoryDoctor {
getInflightProofs(mintUrls?: string[]): Promise<Array<{ secret: string }>>;
}
/** Rejects when `timeoutMs` elapses before `promise` settles. */
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
if (timeoutMs === Infinity) return promise;
@@ -1810,6 +1847,240 @@ export async function runWalletRecovery(
}
}
/**
* Structural source for the wallet doctor, so the orchestration is testable
* without a wallet database or network access (mirrors
* MintQuoteRecoverySource). Every method is a READ: the doctor never
* observes-and-persists, finalizes, or fails operations.
*/
export interface WalletDoctorSource {
listTrustedMintUrls(): Promise<string[]>;
/** Pending and executing mint operations. */
listPendingMintOps(): Promise<DoctorMintOperation[]>;
/** Terminally failed mint operations (via coco's private repository). */
listFailedMintOps(): Promise<DoctorMintOperation[]>;
listPreparedMelts(): Promise<Array<Record<string, unknown>>>;
listInFlightMelts(): Promise<Array<Record<string, unknown>>>;
listFailedMelts(): Promise<Array<Record<string, unknown>>>;
/** Secrets of proofs currently locked (`inflight`). */
getInflightProofSecrets(): Promise<string[]>;
/** NUT-06 reachability probe for one mint. */
probeMint(mintUrl: string): Promise<DoctorMintProbe>;
/** Read-only NUT-04 quote state fetch (no persistence). */
fetchQuoteState(mintUrl: string, quoteId: string): Promise<string>;
/** Probe budget/concurrency overrides for tests. */
quoteProbeBudgetMs?: number;
quoteProbeConcurrency?: number;
}
/**
* Run the read-only wallet health checks behind `routstrd wallet doctor`.
*
* Four checks, all driven by the pure helpers in doctor.ts:
*
* 1. Mint reachability: every trusted mint gets a NUT-06 probe, in parallel.
* 2. Recent unpaid quotes: pending quotes created in the last hour whose
* mint still reports UNPAID - informational, an invoice awaiting payment.
* 3. Paid but not issued: the stuck scenario `wallet recover` remediates.
* Candidates share recovery's selection rules; the remote state is
* classified by the same helper recovery uses, so doctor and recovery
* never disagree about what is claimable.
* 4. Stuck melts: prepared melts holding reserved proofs, in-flight melts,
* and failed melts whose input proofs were never released.
*
* Quote-state probes are read-only NUT-04 fetches (never coco's
* observe-and-persist path), bounded by an overall budget and a concurrency
* limit so a slow mint cannot stall the whole report. Quotes that could not
* be checked are counted in `uncheckedQuotes` rather than silently dropped.
*/
export async function runWalletDoctor(
source: WalletDoctorSource,
): Promise<WalletDoctorReport> {
const nowMs = Date.now();
const report: WalletDoctorReport = {
generatedAt: nowMs,
mints: [],
unpaidQuotes: [],
paidUnissued: [],
stuckMelts: [],
uncheckedQuotes: 0,
};
const trustedMintUrls = await source.listTrustedMintUrls();
report.mints = await Promise.all(
trustedMintUrls.map((mintUrl) => source.probeMint(mintUrl)),
);
const [pendingMintOps, failedMintOps] = await Promise.all([
source.listPendingMintOps(),
source.listFailedMintOps(),
]);
const recentQuotes = selectRecentMintQuotes(pendingMintOps, nowMs);
const recentIds = new Set(recentQuotes.map((op) => op.id));
const candidates = selectPaidUnissuedCandidates([
...pendingMintOps,
...failedMintOps,
]);
const targets = new Map<string, DoctorMintOperation>();
for (const op of [...recentQuotes, ...candidates]) {
if (!targets.has(op.id)) targets.set(op.id, op);
}
const queue = [...targets.values()];
const budgetMs = source.quoteProbeBudgetMs ?? DOCTOR_QUOTE_PROBE_BUDGET_MS;
const concurrency =
source.quoteProbeConcurrency ?? DOCTOR_QUOTE_PROBE_CONCURRENCY;
const deadlineAt = nowMs + budgetMs;
let next = 0;
await Promise.all(
Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
while (next < queue.length && Date.now() < deadlineAt) {
const op = queue[next++];
if (!op) break;
try {
const remoteState = await source.fetchQuoteState(
op.mintUrl,
op.quoteId as string,
);
if (
recentIds.has(op.id) &&
mintQuoteStateToCategory(remoteState) === "waiting"
) {
report.unpaidQuotes.push(toUnpaidQuote(op, nowMs));
}
const finding = classifyPaidUnissued(op, remoteState);
if (finding) report.paidUnissued.push(finding);
} catch {
report.uncheckedQuotes++;
}
}
}),
);
report.uncheckedQuotes += queue.length - next;
// Concurrent workers push in completion order; sort for a stable report.
report.unpaidQuotes.sort((a, b) => a.operationId.localeCompare(b.operationId));
report.paidUnissued.sort((a, b) => a.operationId.localeCompare(b.operationId));
const [preparedMelts, inFlightMelts, failedMelts, inflightSecretsList] =
await Promise.all([
source.listPreparedMelts(),
source.listInFlightMelts(),
source.listFailedMelts(),
source.getInflightProofSecrets(),
]);
const inflightSecrets = new Set(inflightSecretsList);
for (const raw of [...preparedMelts, ...inFlightMelts, ...failedMelts]) {
const finding = classifyStuckMelt(toDoctorMeltOperation(raw), {
nowMs,
inflightSecrets,
});
if (finding) report.stuckMelts.push(finding);
}
report.stuckMelts.sort((a, b) => a.operationId.localeCompare(b.operationId));
return report;
}
/** Map coco's melt operation rows onto the doctor's structural subset. */
function toDoctorMeltOperation(
raw: Record<string, unknown>,
): DoctorMeltOperation {
return {
id: String(raw.id),
mintUrl: String(raw.mintUrl),
quoteId: typeof raw.quoteId === "string" ? raw.quoteId : undefined,
state: String(raw.state),
amount: Number(raw.amount ?? 0),
feeReserve: Number(raw.fee_reserve ?? 0),
inputProofSecrets: Array.isArray(raw.inputProofSecrets)
? raw.inputProofSecrets.map((secret) => String(secret))
: [],
createdAt: Number(raw.createdAt ?? 0),
updatedAt: Number(raw.updatedAt ?? 0),
error: typeof raw.error === "string" ? raw.error : undefined,
};
}
/**
* Resolve coco's private repositories for the doctor. Fails closed - a coco
* upgrade that renames them must surface here and in the doctor tests, not
* silently skip the failed-operation and locked-proof checks.
*/
function doctorRepositories(coco: Manager): {
mintOps: MintOperationRepositoryDoctor;
meltOps: MeltOperationRepositoryDoctor;
proofs: ProofRepositoryDoctor;
} {
const inner = coco as unknown as {
mintOperationRepository?: MintOperationRepositoryDoctor;
meltOperationRepository?: MeltOperationRepositoryDoctor;
proofRepository?: ProofRepositoryDoctor;
};
const checks: Array<[string, object | undefined, string]> = [
["mintOperationRepository", inner.mintOperationRepository, "getByState"],
["meltOperationRepository", inner.meltOperationRepository, "getByState"],
["proofRepository", inner.proofRepository, "getInflightProofs"],
];
for (const [name, repo, method] of checks) {
if (
!repo ||
typeof (repo as Record<string, unknown>)[method] !== "function"
) {
throw new Error(
`coco ${name}.${method} is unavailable; refusing to run wallet doctor with partial health checks`,
);
}
}
return {
mintOps: inner.mintOperationRepository as MintOperationRepositoryDoctor,
meltOps: inner.meltOperationRepository as MeltOperationRepositoryDoctor,
proofs: inner.proofRepository as ProofRepositoryDoctor,
};
}
/** Read-only NUT-06 reachability probe for one mint. */
export async function probeMintInfo(mintUrl: string): Promise<DoctorMintProbe> {
const startedAt = Date.now();
try {
const response = await fetch(`${mintUrl.replace(/\/+$/, "")}/v1/info`, {
signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS),
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
return {
mintUrl,
reachable: true,
latencyMs: Date.now() - startedAt,
};
} catch (error) {
return {
mintUrl,
reachable: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
/**
* Read-only NUT-04 quote state fetch. Unlike coco's
* observePendingOperation this persists nothing, which is what keeps the
* doctor free of side effects.
*/
export async function fetchMintQuoteState(
mintUrl: string,
quoteId: string,
): Promise<string> {
const response = await fetch(
`${mintUrl.replace(/\/+$/, "")}/v1/mint/quote/bolt11/${encodeURIComponent(quoteId)}`,
{ signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS) },
);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const body = (await response.json()) as { state?: unknown };
if (typeof body.state !== "string") {
throw new Error("mint response missing quote state");
}
return body.state;
}
export async function createCocoClient(
options: CreateCocoClientOptions = {},
): Promise<WalletClient> {
@@ -2602,6 +2873,39 @@ export async function createCocoClient(
};
},
async diagnoseWallet() {
const repos = doctorRepositories(coco);
return runWalletDoctor({
listTrustedMintUrls: async () =>
(await coco.mint.getAllTrustedMints()).map((mint) => mint.mintUrl),
listPendingMintOps: async () => {
// listInFlight can overlap with listPending; dedupe by id.
const byId = new Map<string, DoctorMintOperation>();
for (const op of [
...(await coco.ops.mint.listPending()),
...(await coco.ops.mint.listInFlight()),
] as unknown as DoctorMintOperation[]) {
byId.set(op.id, op);
}
return [...byId.values()];
},
listFailedMintOps: () => repos.mintOps.getByState("failed"),
listPreparedMelts: async () =>
coco.ops.melt.listPrepared() as unknown as Array<
Record<string, unknown>
>,
listInFlightMelts: async () =>
coco.ops.melt.listInFlight() as unknown as Array<
Record<string, unknown>
>,
listFailedMelts: () => repos.meltOps.getByState("failed"),
getInflightProofSecrets: async () =>
(await repos.proofs.getInflightProofs()).map((proof) => proof.secret),
probeMint: probeMintInfo,
fetchQuoteState: fetchMintQuoteState,
});
},
async recoverMintQuotes(options, onProgress) {
await waitForRecovery();
const service = (
+1 -1
View File
@@ -142,7 +142,7 @@ export interface WalletDoctorReport {
paidUnissued: DoctorPaidUnissuedQuote[];
/** Melt operations holding locked proofs. */
stuckMelts: DoctorStuckMelt[];
/** Quotes skipped because the probe budget ran out. */
/** Quotes whose remote state could not be checked (probe failure or budget exhausted). */
uncheckedQuotes: number;
}
@@ -15,7 +15,15 @@ import { join } from "node:path";
import { Manager } from "@cashu/coco-core";
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint";
import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client";
import {
fetchMintQuoteState,
probeMintInfo,
reopenFailedMintOperation,
runMintQuoteRecovery,
runWalletDoctor,
type WalletDoctorSource,
} from "./coco-client";
import type { DoctorMintOperation } from "./doctor";
type AnyRecord = Record<string, unknown>;
@@ -384,3 +392,101 @@ describe("PAID mint quote recovery with a real Manager and mint", () => {
expect(await booted.spendable()).toBe(21_000);
});
});
describe("wallet doctor with a real Manager and mint", () => {
function doctorSource(booted: Booted): WalletDoctorSource {
const manager = booted.manager;
const repos = manager as unknown as {
mintOperationRepository: {
getByState(state: string): Promise<DoctorMintOperation[]>;
};
meltOperationRepository: {
getByState(state: string): Promise<AnyRecord[]>;
};
proofRepository: {
getInflightProofs(): Promise<Array<{ secret: string }>>;
};
};
return {
listTrustedMintUrls: async () =>
(await manager.mint.getAllTrustedMints()).map((mint) => mint.mintUrl),
listPendingMintOps: async () => {
const byId = new Map<string, DoctorMintOperation>();
for (const op of [
...(await manager.ops.mint.listPending()),
...(await manager.ops.mint.listInFlight()),
] as unknown as DoctorMintOperation[]) {
byId.set(op.id, op);
}
return [...byId.values()];
},
listFailedMintOps: () => repos.mintOperationRepository.getByState("failed"),
listPreparedMelts: async () =>
manager.ops.melt.listPrepared() as unknown as AnyRecord[],
listInFlightMelts: async () =>
manager.ops.melt.listInFlight() as unknown as AnyRecord[],
listFailedMelts: () => repos.meltOperationRepository.getByState("failed"),
getInflightProofSecrets: async () =>
(await repos.proofRepository.getInflightProofs()).map(
(proof) => proof.secret,
),
probeMint: probeMintInfo,
fetchQuoteState: fetchMintQuoteState,
};
}
it("surfaces unpaid, then paid-unissued, then a clean bill after recovery", async () => {
booted = await boot({ quoteExpiry: null });
const op = await prepareQuote(booted, 100);
// Fresh quote, unpaid: the doctor reaches the mint and lists the quote.
const unpaid = await runWalletDoctor(doctorSource(booted));
expect(unpaid.mints).toHaveLength(1);
expect(unpaid.mints[0]?.reachable).toBe(true);
expect(unpaid.paidUnissued).toEqual([]);
expect(unpaid.uncheckedQuotes).toBe(0);
expect(unpaid.unpaidQuotes).toHaveLength(1);
expect(unpaid.unpaidQuotes[0]).toMatchObject({
operationId: op.id as string,
amount: 100,
});
// The quote gets paid while the daemon is "down": the doctor surfaces
// exactly the stuck scenario recovery exists for, with the recover hint.
booted.mint.markPaid(op.quoteId as string);
const stuck = await runWalletDoctor(doctorSource(booted));
expect(stuck.unpaidQuotes).toEqual([]);
expect(stuck.paidUnissued).toHaveLength(1);
expect(stuck.paidUnissued[0]).toMatchObject({
operationId: op.id as string,
remoteState: "PAID",
amount: 100,
remediation: `routstrd wallet recover --op ${op.id as string}`,
});
// Recovery claims the sats; the doctor goes back to all-green.
const recovered = (await runMintQuoteRecovery(
booted.source() as never,
)) as unknown as Record<string, number>;
expect(recovered).toMatchObject({ recovered: 1 });
const clean = await runWalletDoctor(doctorSource(booted));
expect(clean.unpaidQuotes).toEqual([]);
expect(clean.paidUnissued).toEqual([]);
expect(clean.stuckMelts).toEqual([]);
expect(clean.uncheckedQuotes).toBe(0);
});
it("reports unreachable mints without failing the other checks", async () => {
booted = await boot({ quoteExpiry: null });
await prepareQuote(booted, 100);
booted.mint.stop();
const report = await runWalletDoctor(doctorSource(booted));
expect(report.mints[0]?.reachable).toBe(false);
// The quote probe fails too, and is counted rather than dropped.
expect(report.uncheckedQuotes).toBe(1);
expect(report.unpaidQuotes).toEqual([]);
expect(report.paidUnissued).toEqual([]);
booted.mint.start();
});
});