mirror of
https://github.com/Routstr/routstrd.git
synced 2026-10-05 12:28:23 +00:00
feat(wallet): add live health checks to wallet doctor
wire the doctor helpers from the previous commit into a read-only health check surfaced by the daemon and the CLI: - coco-client: runWalletDoctor orchestrates the four checks against an injected structural source (unit-testable like runMintQuoteRecovery); diagnoseWallet() wires it to coco - trusted mints, pending/in-flight mint ops, and failed ops + inflight proofs via coco's private repositories, resolved fail-closed so a coco upgrade that renames them surfaces immediately. Quote checks are plain NUT-04 reads (never coco's observe-and-persist path), bounded by an overall budget and a concurrency limit; uncheckable quotes are counted, not dropped. - http: GET /wallet/doctor returns the report (501 when the wallet client has no doctor support). - cli: wallet doctor now prints the health section first (mint reachability, recent unpaid quotes, paid-but-not-issued with recover remediation, stuck melts), then the existing offline migration diagnosis. NO_COLOR/TTY-aware colors, --json for scripting, exit 1 when money is provably at risk or the wallets conflict. When the daemon is down the live section is skipped rather than auto-starting the daemon. - tests: HTTP route tests, runWalletDoctor unit tests with a fake source, and real-Manager + fake-mint integration tests covering the full unpaid -> paid-unissued -> recovered -> clean arc and unreachable-mint degradation. - docs: wallet doctor section in wallet-mint-recovery.md.
This commit is contained in:
@@ -46,6 +46,27 @@ and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate
|
|||||||
with coco's watcher/processor. It needs its own integration tests before handling
|
with coco's watcher/processor. It needs its own integration tests before handling
|
||||||
real funds.
|
real funds.
|
||||||
|
|
||||||
|
## Wallet doctor
|
||||||
|
|
||||||
|
`routstrd wallet doctor` runs read-only health checks before the legacy
|
||||||
|
migration diagnosis:
|
||||||
|
|
||||||
|
1. **Mint reachability** — a NUT-06 probe against every trusted mint.
|
||||||
|
2. **Recent unpaid quotes** — pending quotes from the last hour the mint
|
||||||
|
still reports UNPAID (an invoice awaiting payment; informational).
|
||||||
|
3. **Paid but not issued** — the stuck scenario this recovery feature
|
||||||
|
fixes. Each finding prints its `wallet recover --op <id>` remediation,
|
||||||
|
with `--include-failed` when the operation must be re-opened first.
|
||||||
|
4. **Stuck melts** — prepared melts holding reserved proofs, in-flight
|
||||||
|
melts, and failed melts whose input proofs were never released.
|
||||||
|
|
||||||
|
The doctor never mutates wallet state: quote checks are plain NUT-04 reads,
|
||||||
|
not coco's observe-and-persist path, and remediation is always left to the
|
||||||
|
operator. It exits non-zero when money is provably at risk (unreachable
|
||||||
|
mint, paid-but-unissued quote, failed melt with locked proofs) or the
|
||||||
|
migration diagnosis finds a conflict, so it can gate scripts. When the
|
||||||
|
daemon is down only the offline migration section runs.
|
||||||
|
|
||||||
## Cleanup preview and force
|
## Cleanup preview and force
|
||||||
|
|
||||||
`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not
|
`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
import { describe, expect, it } from "bun:test";
|
||||||
|
import { renderWalletHealth } from "./cli";
|
||||||
|
import type { WalletDoctorReport } from "./daemon/wallet/doctor";
|
||||||
|
|
||||||
|
const NOW = 1_800_000_000_000;
|
||||||
|
|
||||||
|
function report(overrides: Partial<WalletDoctorReport> = {}): WalletDoctorReport {
|
||||||
|
return {
|
||||||
|
generatedAt: NOW,
|
||||||
|
mints: [],
|
||||||
|
unpaidQuotes: [],
|
||||||
|
paidUnissued: [],
|
||||||
|
stuckMelts: [],
|
||||||
|
uncheckedQuotes: 0,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("renderWalletHealth", () => {
|
||||||
|
it("renders an all-green report", () => {
|
||||||
|
const output = renderWalletHealth(
|
||||||
|
report({
|
||||||
|
mints: [
|
||||||
|
{ mintUrl: "https://mint.example", reachable: true, latencyMs: 212 },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(output).toContain("Wallet health");
|
||||||
|
expect(output).toContain("✓ https://mint.example — reachable (212 ms)");
|
||||||
|
expect(output).toContain("none unpaid");
|
||||||
|
expect(output).toContain("none stuck");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders unreachable mints with their error", () => {
|
||||||
|
const output = renderWalletHealth(
|
||||||
|
report({
|
||||||
|
mints: [
|
||||||
|
{ mintUrl: "https://mint.down", reachable: false, error: "fetch failed" },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(output).toContain("✗ https://mint.down — unreachable: fetch failed");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders unpaid quotes with age and expiry", () => {
|
||||||
|
const output = renderWalletHealth(
|
||||||
|
report({
|
||||||
|
unpaidQuotes: [
|
||||||
|
{
|
||||||
|
operationId: "op-1234567890abcdef",
|
||||||
|
quoteId: "quote-abcdef1234567890",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
amount: 500,
|
||||||
|
ageMs: 5 * 60_000,
|
||||||
|
expiresInMs: 41 * 60_000,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(output).toContain("⚠ quote-ab… at https://mint.example — 500 sat, UNPAID");
|
||||||
|
expect(output).toContain("created 5m ago, expires in 41m");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders paid-unissued findings with the recover remediation", () => {
|
||||||
|
const output = renderWalletHealth(
|
||||||
|
report({
|
||||||
|
paidUnissued: [
|
||||||
|
{
|
||||||
|
operationId: "op-1234567890abcdef",
|
||||||
|
quoteId: "quote-abcdef1234567890",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
amount: 1_000,
|
||||||
|
localState: "failed",
|
||||||
|
remoteState: "PAID",
|
||||||
|
error: "keyset id inactive.",
|
||||||
|
remediation: "routstrd wallet recover --op op-1234567890abcdef --include-failed",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(output).toContain("✗ op-12345… (quote quote-ab…)");
|
||||||
|
expect(output).toContain("1,000 sat PAID at mint, local state failed");
|
||||||
|
expect(output).toContain("mint error: keyset id inactive.");
|
||||||
|
expect(output).toContain(
|
||||||
|
"→ routstrd wallet recover --op op-1234567890abcdef --include-failed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders stuck melts with locked totals and remediation", () => {
|
||||||
|
const output = renderWalletHealth(
|
||||||
|
report({
|
||||||
|
stuckMelts: [
|
||||||
|
{
|
||||||
|
operationId: "melt-1234567890",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
amount: 2_100,
|
||||||
|
feeReserve: 12,
|
||||||
|
ageMs: 3 * 86_400_000,
|
||||||
|
kind: "prepared",
|
||||||
|
lockedSecrets: 4,
|
||||||
|
remediation: "routstrd wallet cleanup",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
operationId: "melt-abcdef",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
amount: 50,
|
||||||
|
feeReserve: 2,
|
||||||
|
ageMs: 90_000,
|
||||||
|
kind: "failed-locked",
|
||||||
|
lockedSecrets: 1,
|
||||||
|
remediation: "restart the daemon",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(output).toContain(
|
||||||
|
"⚠ melt-123… at https://mint.example — 2,112 sat locked (4 proof(s)), prepared, payment never attempted, 3d old",
|
||||||
|
);
|
||||||
|
expect(output).toContain("✗ melt-abcdef at");
|
||||||
|
expect(output).toContain("failed but proofs still locked");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("notes quotes that could not be checked", () => {
|
||||||
|
const output = renderWalletHealth(report({ uncheckedQuotes: 3 }));
|
||||||
|
expect(output).toContain("3 quote(s) could not be checked with their mint");
|
||||||
|
});
|
||||||
|
});
|
||||||
+150
-3
@@ -49,6 +49,11 @@ import {
|
|||||||
summarizeWalletDirectory,
|
summarizeWalletDirectory,
|
||||||
WalletMigrationConflictError,
|
WalletMigrationConflictError,
|
||||||
} from "./daemon/wallet/diagnostics";
|
} from "./daemon/wallet/diagnostics";
|
||||||
|
import {
|
||||||
|
doctorReportSeverity,
|
||||||
|
DOCTOR_RECENT_QUOTE_WINDOW_MS,
|
||||||
|
type WalletDoctorReport,
|
||||||
|
} from "./daemon/wallet/doctor";
|
||||||
import {
|
import {
|
||||||
legacyCocodDir,
|
legacyCocodDir,
|
||||||
legacyCocodPidPath,
|
legacyCocodPidPath,
|
||||||
@@ -2062,14 +2067,156 @@ walletCmd
|
|||||||
await handleDaemonCommand("/wallet/status");
|
await handleDaemonCommand("/wallet/status");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const doctorUseColor = !process.env.NO_COLOR && process.stdout.isTTY === true;
|
||||||
|
const doctorPaint = (code: string, text: string): string =>
|
||||||
|
doctorUseColor ? `\x1b[${code}m${text}\x1b[0m` : text;
|
||||||
|
const doctorGreen = (text: string) => doctorPaint("32", text);
|
||||||
|
const doctorRed = (text: string) => doctorPaint("31", text);
|
||||||
|
const doctorYellow = (text: string) => doctorPaint("33", text);
|
||||||
|
|
||||||
|
function shortenOperationId(id: string): string {
|
||||||
|
return id.length > 12 ? `${id.slice(0, 8)}…` : id;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSats(amount: number): string {
|
||||||
|
return amount.toLocaleString("en-US");
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDoctorDuration(ms: number): string {
|
||||||
|
const abs = Math.abs(ms);
|
||||||
|
if (abs < 60_000) return `${Math.max(1, Math.round(abs / 1000))}s`;
|
||||||
|
if (abs < 3_600_000) return `${Math.round(abs / 60_000)}m`;
|
||||||
|
if (abs < 86_400_000) return `${Math.round(abs / 3_600_000)}h`;
|
||||||
|
return `${Math.round(abs / 86_400_000)}d`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Render the daemon's live doctor report; colors are NO_COLOR/TTY aware. */
|
||||||
|
export function renderWalletHealth(report: WalletDoctorReport): string {
|
||||||
|
const lines: string[] = ["Wallet health", "=============", ""];
|
||||||
|
|
||||||
|
lines.push("Mints");
|
||||||
|
if (report.mints.length === 0) {
|
||||||
|
lines.push(" (no trusted mints)");
|
||||||
|
}
|
||||||
|
for (const probe of report.mints) {
|
||||||
|
if (probe.reachable) {
|
||||||
|
const latency =
|
||||||
|
probe.latencyMs !== undefined ? ` (${probe.latencyMs} ms)` : "";
|
||||||
|
lines.push(` ${doctorGreen("✓")} ${probe.mintUrl} — reachable${latency}`);
|
||||||
|
} else {
|
||||||
|
lines.push(
|
||||||
|
` ${doctorRed("✗")} ${probe.mintUrl} — unreachable${probe.error ? `: ${probe.error}` : ""}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
"",
|
||||||
|
`Mint quotes (last ${formatDoctorDuration(DOCTOR_RECENT_QUOTE_WINDOW_MS)})`,
|
||||||
|
);
|
||||||
|
if (report.unpaidQuotes.length === 0) {
|
||||||
|
lines.push(" none unpaid");
|
||||||
|
}
|
||||||
|
for (const quote of report.unpaidQuotes) {
|
||||||
|
const expiry =
|
||||||
|
quote.expiresInMs === undefined
|
||||||
|
? ""
|
||||||
|
: quote.expiresInMs >= 0
|
||||||
|
? `, expires in ${formatDoctorDuration(quote.expiresInMs)}`
|
||||||
|
: `, expired ${formatDoctorDuration(quote.expiresInMs)} ago`;
|
||||||
|
lines.push(
|
||||||
|
` ${doctorYellow("⚠")} ${shortenOperationId(quote.quoteId ?? quote.operationId)} at ${quote.mintUrl} — ${formatSats(quote.amount)} sat, UNPAID, created ${formatDoctorDuration(quote.ageMs)} ago${expiry}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push("", "Paid but not issued");
|
||||||
|
if (report.paidUnissued.length === 0) {
|
||||||
|
lines.push(" none");
|
||||||
|
}
|
||||||
|
for (const finding of report.paidUnissued) {
|
||||||
|
lines.push(
|
||||||
|
` ${doctorRed("✗")} ${shortenOperationId(finding.operationId)} (quote ${shortenOperationId(finding.quoteId ?? "?")}) at ${finding.mintUrl} — ${formatSats(finding.amount)} sat ${finding.remoteState} at mint, local state ${finding.localState}`,
|
||||||
|
);
|
||||||
|
if (finding.error) lines.push(` mint error: ${finding.error}`);
|
||||||
|
lines.push(` → ${finding.remediation}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push("", "Melts");
|
||||||
|
if (report.stuckMelts.length === 0) {
|
||||||
|
lines.push(" none stuck");
|
||||||
|
}
|
||||||
|
for (const melt of report.stuckMelts) {
|
||||||
|
const icon = melt.kind === "failed-locked" ? doctorRed("✗") : doctorYellow("⚠");
|
||||||
|
const what =
|
||||||
|
melt.kind === "prepared"
|
||||||
|
? "prepared, payment never attempted"
|
||||||
|
: melt.kind === "in-flight"
|
||||||
|
? "payment in flight"
|
||||||
|
: "failed but proofs still locked";
|
||||||
|
lines.push(
|
||||||
|
` ${icon} ${shortenOperationId(melt.operationId)} at ${melt.mintUrl} — ${formatSats(melt.amount + melt.feeReserve)} sat locked (${melt.lockedSecrets} proof(s)), ${what}, ${formatDoctorDuration(melt.ageMs)} old`,
|
||||||
|
);
|
||||||
|
lines.push(` → ${melt.remediation}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (report.uncheckedQuotes > 0) {
|
||||||
|
lines.push(
|
||||||
|
"",
|
||||||
|
` ${doctorYellow("⚠")} ${report.uncheckedQuotes} quote(s) could not be checked with their mint`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return lines.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
walletCmd
|
walletCmd
|
||||||
.command("doctor")
|
.command("doctor")
|
||||||
.description("Diagnose conflicting wallets (current routstrd wallet vs legacy cocod)")
|
.description(
|
||||||
.action(async () => {
|
"Wallet health checks (mint reachability, unpaid/stuck quotes, locked melts), then legacy wallet migration diagnosis",
|
||||||
|
)
|
||||||
|
.option("--json", "Print the raw health report as JSON", false)
|
||||||
|
.action(async (options: { json: boolean }) => {
|
||||||
const target = summarizeWalletDirectory(defaultWalletDir(), "canonical");
|
const target = summarizeWalletDirectory(defaultWalletDir(), "canonical");
|
||||||
const source = summarizeWalletDirectory(legacyCocodDir(), "legacy");
|
const source = summarizeWalletDirectory(legacyCocodDir(), "legacy");
|
||||||
|
const migrationConflict = diagnoseWallets(target, source).conflict;
|
||||||
|
|
||||||
|
// Live health checks need the daemon; when it is down the offline
|
||||||
|
// migration diagnosis still runs, because a broken startup is exactly
|
||||||
|
// when the doctor gets invoked.
|
||||||
|
let report: WalletDoctorReport | undefined;
|
||||||
|
try {
|
||||||
|
const result = await callDaemon("/wallet/doctor");
|
||||||
|
if (result.error) throw new Error(result.error);
|
||||||
|
report = result.output as WalletDoctorReport | undefined;
|
||||||
|
} catch {
|
||||||
|
report = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.json) {
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{ health: report ?? null, migrationConflict },
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
if (report) {
|
||||||
|
console.log(renderWalletHealth(report));
|
||||||
|
} else {
|
||||||
|
console.log("Wallet health");
|
||||||
|
console.log("=============");
|
||||||
|
console.log("");
|
||||||
|
console.log(
|
||||||
|
" skipped: daemon is not running; live checks need it (routstrd start)",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
console.log("");
|
||||||
console.log(renderWalletDoctor(target, source));
|
console.log(renderWalletDoctor(target, source));
|
||||||
if (diagnoseWallets(target, source).conflict) process.exit(1);
|
}
|
||||||
|
|
||||||
|
const healthCritical =
|
||||||
|
report !== undefined && doctorReportSeverity(report) === "critical";
|
||||||
|
if (healthCritical || migrationConflict) process.exit(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
walletCmd
|
walletCmd
|
||||||
|
|||||||
@@ -549,6 +549,19 @@ export function createDaemonRequestHandler(deps: {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/wallet/doctor") {
|
||||||
|
await respond(res, async () => {
|
||||||
|
if (!deps.walletClient.diagnoseWallet) {
|
||||||
|
throw new WalletHttpError(
|
||||||
|
501,
|
||||||
|
"Wallet doctor is not supported by this wallet client.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { output: await deps.walletClient.diagnoseWallet() };
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "POST" && url.pathname === "/wallet/recover") {
|
if (req.method === "POST" && url.pathname === "/wallet/recover") {
|
||||||
await respond(res, async () => {
|
await respond(res, async () => {
|
||||||
if (!deps.walletClient.recoverMintQuotes) {
|
if (!deps.walletClient.recoverMintQuotes) {
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
import { describe, expect, it, mock } from "bun:test";
|
||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import { createDaemonRequestHandler } from "./index";
|
||||||
|
|
||||||
|
function get(path: string, walletClient: unknown) {
|
||||||
|
const handler = createDaemonRequestHandler({ walletClient } as never);
|
||||||
|
const req = new EventEmitter() as any;
|
||||||
|
Object.assign(req, {
|
||||||
|
method: "GET",
|
||||||
|
url: path,
|
||||||
|
headers: { host: "localhost" },
|
||||||
|
});
|
||||||
|
const res = {
|
||||||
|
status: 0,
|
||||||
|
body: "",
|
||||||
|
writeHead(status: number) {
|
||||||
|
this.status = status;
|
||||||
|
},
|
||||||
|
end(chunk: string) {
|
||||||
|
this.body = chunk;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
setImmediate(() => req.emit("end"));
|
||||||
|
return handler(req, res as never).then(() => res);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("GET /wallet/doctor", () => {
|
||||||
|
it("returns the client's doctor report", async () => {
|
||||||
|
const report = {
|
||||||
|
generatedAt: 1_800_000_000_000,
|
||||||
|
mints: [{ mintUrl: "https://mint.example", reachable: true, latencyMs: 12 }],
|
||||||
|
unpaidQuotes: [],
|
||||||
|
paidUnissued: [],
|
||||||
|
stuckMelts: [],
|
||||||
|
uncheckedQuotes: 0,
|
||||||
|
};
|
||||||
|
const diagnoseWallet = mock(async () => report);
|
||||||
|
const res = await get("/wallet/doctor", { diagnoseWallet });
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(JSON.parse(res.body)).toEqual({ output: report });
|
||||||
|
expect(diagnoseWallet).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 501 when the wallet client has no doctor support", async () => {
|
||||||
|
const res = await get("/wallet/doctor", {});
|
||||||
|
expect(res.status).toBe(501);
|
||||||
|
expect(JSON.parse(res.body).error).toContain("not supported");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces client failures as errors, not a fake-healthy report", async () => {
|
||||||
|
const diagnoseWallet = mock(async () => {
|
||||||
|
throw new Error("coco repositories unavailable");
|
||||||
|
});
|
||||||
|
const res = await get("/wallet/doctor", { diagnoseWallet });
|
||||||
|
expect(res.status).toBe(500);
|
||||||
|
expect(JSON.parse(res.body).error).toContain(
|
||||||
|
"coco repositories unavailable",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -21,6 +21,7 @@ import {
|
|||||||
isZombieProcess,
|
isZombieProcess,
|
||||||
reopenFailedMintOperation,
|
reopenFailedMintOperation,
|
||||||
runMintQuoteRecovery,
|
runMintQuoteRecovery,
|
||||||
|
runWalletDoctor,
|
||||||
settleExpiredMintQuotes,
|
settleExpiredMintQuotes,
|
||||||
settlePendingMintQuotes,
|
settlePendingMintQuotes,
|
||||||
stopLegacyCocod,
|
stopLegacyCocod,
|
||||||
@@ -28,6 +29,7 @@ import {
|
|||||||
type MintQuoteRecoverySource,
|
type MintQuoteRecoverySource,
|
||||||
type PendingMintQuoteSource,
|
type PendingMintQuoteSource,
|
||||||
type PendingMintSweepState,
|
type PendingMintSweepState,
|
||||||
|
type WalletDoctorSource,
|
||||||
} from "./coco-client";
|
} from "./coco-client";
|
||||||
import { OperationInProgressError } from "@cashu/coco-core";
|
import { OperationInProgressError } from "@cashu/coco-core";
|
||||||
import { logger } from "../../utils/logger";
|
import { logger } from "../../utils/logger";
|
||||||
@@ -1728,3 +1730,215 @@ describe("createRecoveryGate", () => {
|
|||||||
await waiting;
|
await waiting;
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("runWalletDoctor", () => {
|
||||||
|
const NOW = Date.now();
|
||||||
|
|
||||||
|
function doctorMintOp(overrides: Record<string, unknown> = {}) {
|
||||||
|
return {
|
||||||
|
id: "op-1",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
quoteId: "quote-1",
|
||||||
|
state: "pending",
|
||||||
|
amount: 500,
|
||||||
|
expiry: Math.floor(NOW / 1000) + 600,
|
||||||
|
createdAt: NOW - 5 * 60_000,
|
||||||
|
updatedAt: NOW - 60_000,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function doctorMeltOp(overrides: Record<string, unknown> = {}) {
|
||||||
|
return {
|
||||||
|
id: "melt-1",
|
||||||
|
mintUrl: "https://mint.example",
|
||||||
|
quoteId: "melt-quote-1",
|
||||||
|
state: "prepared",
|
||||||
|
amount: 2_100,
|
||||||
|
fee_reserve: 12,
|
||||||
|
inputProofSecrets: ["secret-a"],
|
||||||
|
createdAt: NOW - 3 * 24 * 3_600_000,
|
||||||
|
updatedAt: NOW - 3 * 24 * 3_600_000,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeDoctorSource(overrides: Partial<WalletDoctorSource> = {}) {
|
||||||
|
const fetchQuoteState = mock(async (_mintUrl: string, _quoteId: string) => "UNPAID");
|
||||||
|
const source: WalletDoctorSource = {
|
||||||
|
listTrustedMintUrls: async () => ["https://mint.example"],
|
||||||
|
listPendingMintOps: async () => [],
|
||||||
|
listFailedMintOps: async () => [],
|
||||||
|
listPreparedMelts: async () => [],
|
||||||
|
listInFlightMelts: async () => [],
|
||||||
|
listFailedMelts: async () => [],
|
||||||
|
getInflightProofSecrets: async () => [],
|
||||||
|
probeMint: async (mintUrl: string) => ({
|
||||||
|
mintUrl,
|
||||||
|
reachable: true,
|
||||||
|
latencyMs: 5,
|
||||||
|
}),
|
||||||
|
fetchQuoteState,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
return { source, fetchQuoteState };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("reports a healthy wallet as all-green", async () => {
|
||||||
|
const { source } = fakeDoctorSource();
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.mints).toEqual([
|
||||||
|
{ mintUrl: "https://mint.example", reachable: true, latencyMs: 5 },
|
||||||
|
]);
|
||||||
|
expect(report.unpaidQuotes).toEqual([]);
|
||||||
|
expect(report.paidUnissued).toEqual([]);
|
||||||
|
expect(report.stuckMelts).toEqual([]);
|
||||||
|
expect(report.uncheckedQuotes).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lists a recent quote the mint still reports UNPAID", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listPendingMintOps: async () => [doctorMintOp() as never],
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.unpaidQuotes).toHaveLength(1);
|
||||||
|
expect(report.unpaidQuotes[0]).toMatchObject({
|
||||||
|
operationId: "op-1",
|
||||||
|
amount: 500,
|
||||||
|
});
|
||||||
|
expect(report.paidUnissued).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("flags a pending quote the mint reports PAID with the recover command", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listPendingMintOps: async () => [doctorMintOp() as never],
|
||||||
|
fetchQuoteState: async () => "PAID",
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.unpaidQuotes).toEqual([]);
|
||||||
|
expect(report.paidUnissued).toHaveLength(1);
|
||||||
|
expect(report.paidUnissued[0]).toMatchObject({
|
||||||
|
operationId: "op-1",
|
||||||
|
remoteState: "PAID",
|
||||||
|
remediation: "routstrd wallet recover --op op-1",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("flags a failed quote last seen PAID with --include-failed", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listFailedMintOps: async () => [
|
||||||
|
doctorMintOp({
|
||||||
|
id: "op-9",
|
||||||
|
state: "failed",
|
||||||
|
lastObservedRemoteState: "PAID",
|
||||||
|
}) as never,
|
||||||
|
],
|
||||||
|
fetchQuoteState: async () => "PAID",
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.paidUnissued).toHaveLength(1);
|
||||||
|
expect(report.paidUnissued[0]?.remediation).toBe(
|
||||||
|
"routstrd wallet recover --op op-9 --include-failed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not probe failed quotes last seen UNPAID", async () => {
|
||||||
|
const { source, fetchQuoteState } = fakeDoctorSource({
|
||||||
|
listFailedMintOps: async () => [
|
||||||
|
doctorMintOp({
|
||||||
|
state: "failed",
|
||||||
|
lastObservedRemoteState: "UNPAID",
|
||||||
|
}) as never,
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(fetchQuoteState).not.toHaveBeenCalled();
|
||||||
|
expect(report.paidUnissued).toEqual([]);
|
||||||
|
expect(report.uncheckedQuotes).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes each target quote only once", async () => {
|
||||||
|
const { source, fetchQuoteState } = fakeDoctorSource({
|
||||||
|
listPendingMintOps: async () => [doctorMintOp() as never],
|
||||||
|
});
|
||||||
|
fetchQuoteState.mockImplementation(async () => "PAID");
|
||||||
|
await runWalletDoctor(source);
|
||||||
|
expect(fetchQuoteState).toHaveBeenCalledTimes(1);
|
||||||
|
expect(fetchQuoteState).toHaveBeenCalledWith(
|
||||||
|
"https://mint.example",
|
||||||
|
"quote-1",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("counts probe failures as unchecked instead of dropping them", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listPendingMintOps: async () => [doctorMintOp() as never],
|
||||||
|
fetchQuoteState: async () => {
|
||||||
|
throw new Error("mint unreachable");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.uncheckedQuotes).toBe(1);
|
||||||
|
expect(report.unpaidQuotes).toEqual([]);
|
||||||
|
expect(report.paidUnissued).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("counts quotes the probe budget never reached as unchecked", async () => {
|
||||||
|
const { source, fetchQuoteState } = fakeDoctorSource({
|
||||||
|
listPendingMintOps: async () => [
|
||||||
|
doctorMintOp() as never,
|
||||||
|
doctorMintOp({ id: "op-2", quoteId: "quote-2" }) as never,
|
||||||
|
],
|
||||||
|
quoteProbeBudgetMs: 0,
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(fetchQuoteState).not.toHaveBeenCalled();
|
||||||
|
expect(report.uncheckedQuotes).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes unreachable mint probes through", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
probeMint: async (mintUrl: string) => ({
|
||||||
|
mintUrl,
|
||||||
|
reachable: false,
|
||||||
|
error: "fetch failed",
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.mints).toEqual([
|
||||||
|
{ mintUrl: "https://mint.example", reachable: false, error: "fetch failed" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("flags old prepared melts and failed melts with locked proofs", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listPreparedMelts: async () => [doctorMeltOp()],
|
||||||
|
listFailedMelts: async () => [
|
||||||
|
doctorMeltOp({ id: "melt-2", state: "failed" }),
|
||||||
|
],
|
||||||
|
getInflightProofSecrets: async () => ["secret-a"],
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.stuckMelts).toHaveLength(2);
|
||||||
|
const byKind = new Map(report.stuckMelts.map((melt) => [melt.kind, melt]));
|
||||||
|
expect(byKind.get("prepared")).toMatchObject({
|
||||||
|
operationId: "melt-1",
|
||||||
|
feeReserve: 12,
|
||||||
|
});
|
||||||
|
expect(byKind.get("failed-locked")).toMatchObject({
|
||||||
|
operationId: "melt-2",
|
||||||
|
lockedSecrets: 1,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores failed melts whose proofs were released", async () => {
|
||||||
|
const { source } = fakeDoctorSource({
|
||||||
|
listFailedMelts: async () => [
|
||||||
|
doctorMeltOp({ id: "melt-2", state: "failed" }),
|
||||||
|
],
|
||||||
|
getInflightProofSecrets: async () => [],
|
||||||
|
});
|
||||||
|
const report = await runWalletDoctor(source);
|
||||||
|
expect(report.stuckMelts).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -39,6 +39,18 @@ import type {
|
|||||||
WalletRecoveryProgress,
|
WalletRecoveryProgress,
|
||||||
} from "./wallet-client";
|
} from "./wallet-client";
|
||||||
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
|
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
|
||||||
|
import {
|
||||||
|
classifyPaidUnissued,
|
||||||
|
classifyStuckMelt,
|
||||||
|
mintQuoteStateToCategory,
|
||||||
|
selectPaidUnissuedCandidates,
|
||||||
|
selectRecentMintQuotes,
|
||||||
|
toUnpaidQuote,
|
||||||
|
type DoctorMeltOperation,
|
||||||
|
type DoctorMintOperation,
|
||||||
|
type DoctorMintProbe,
|
||||||
|
type WalletDoctorReport,
|
||||||
|
} from "./doctor";
|
||||||
import {
|
import {
|
||||||
classifyMintQuoteObservation,
|
classifyMintQuoteObservation,
|
||||||
selectMintQuotesForRecovery,
|
selectMintQuotesForRecovery,
|
||||||
@@ -731,6 +743,31 @@ async function enableCocoManager(coco: Manager): Promise<void> {
|
|||||||
*/
|
*/
|
||||||
const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000;
|
const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000;
|
||||||
|
|
||||||
|
/** Per-request timeout for the doctor's NUT-06/NUT-04 mint probes. */
|
||||||
|
const DOCTOR_MINT_PROBE_TIMEOUT_MS = 4_000;
|
||||||
|
/** Overall budget for the doctor's quote-state probe round. */
|
||||||
|
const DOCTOR_QUOTE_PROBE_BUDGET_MS = 15_000;
|
||||||
|
/** How many doctor quote-state probes run at once. */
|
||||||
|
const DOCTOR_QUOTE_PROBE_CONCURRENCY = 4;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Private coco repositories the doctor reads failed operations and locked
|
||||||
|
* proofs through. Read-only access only; resolved fail-closed (see
|
||||||
|
* doctorRepositories) so a coco upgrade that renames them surfaces
|
||||||
|
* immediately instead of silently skipping health checks.
|
||||||
|
*/
|
||||||
|
interface MintOperationRepositoryDoctor {
|
||||||
|
getByState(state: string): Promise<DoctorMintOperation[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MeltOperationRepositoryDoctor {
|
||||||
|
getByState(state: string): Promise<Array<Record<string, unknown>>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProofRepositoryDoctor {
|
||||||
|
getInflightProofs(mintUrls?: string[]): Promise<Array<{ secret: string }>>;
|
||||||
|
}
|
||||||
|
|
||||||
/** Rejects when `timeoutMs` elapses before `promise` settles. */
|
/** Rejects when `timeoutMs` elapses before `promise` settles. */
|
||||||
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
||||||
if (timeoutMs === Infinity) return promise;
|
if (timeoutMs === Infinity) return promise;
|
||||||
@@ -1810,6 +1847,240 @@ export async function runWalletRecovery(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Structural source for the wallet doctor, so the orchestration is testable
|
||||||
|
* without a wallet database or network access (mirrors
|
||||||
|
* MintQuoteRecoverySource). Every method is a READ: the doctor never
|
||||||
|
* observes-and-persists, finalizes, or fails operations.
|
||||||
|
*/
|
||||||
|
export interface WalletDoctorSource {
|
||||||
|
listTrustedMintUrls(): Promise<string[]>;
|
||||||
|
/** Pending and executing mint operations. */
|
||||||
|
listPendingMintOps(): Promise<DoctorMintOperation[]>;
|
||||||
|
/** Terminally failed mint operations (via coco's private repository). */
|
||||||
|
listFailedMintOps(): Promise<DoctorMintOperation[]>;
|
||||||
|
listPreparedMelts(): Promise<Array<Record<string, unknown>>>;
|
||||||
|
listInFlightMelts(): Promise<Array<Record<string, unknown>>>;
|
||||||
|
listFailedMelts(): Promise<Array<Record<string, unknown>>>;
|
||||||
|
/** Secrets of proofs currently locked (`inflight`). */
|
||||||
|
getInflightProofSecrets(): Promise<string[]>;
|
||||||
|
/** NUT-06 reachability probe for one mint. */
|
||||||
|
probeMint(mintUrl: string): Promise<DoctorMintProbe>;
|
||||||
|
/** Read-only NUT-04 quote state fetch (no persistence). */
|
||||||
|
fetchQuoteState(mintUrl: string, quoteId: string): Promise<string>;
|
||||||
|
/** Probe budget/concurrency overrides for tests. */
|
||||||
|
quoteProbeBudgetMs?: number;
|
||||||
|
quoteProbeConcurrency?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the read-only wallet health checks behind `routstrd wallet doctor`.
|
||||||
|
*
|
||||||
|
* Four checks, all driven by the pure helpers in doctor.ts:
|
||||||
|
*
|
||||||
|
* 1. Mint reachability: every trusted mint gets a NUT-06 probe, in parallel.
|
||||||
|
* 2. Recent unpaid quotes: pending quotes created in the last hour whose
|
||||||
|
* mint still reports UNPAID - informational, an invoice awaiting payment.
|
||||||
|
* 3. Paid but not issued: the stuck scenario `wallet recover` remediates.
|
||||||
|
* Candidates share recovery's selection rules; the remote state is
|
||||||
|
* classified by the same helper recovery uses, so doctor and recovery
|
||||||
|
* never disagree about what is claimable.
|
||||||
|
* 4. Stuck melts: prepared melts holding reserved proofs, in-flight melts,
|
||||||
|
* and failed melts whose input proofs were never released.
|
||||||
|
*
|
||||||
|
* Quote-state probes are read-only NUT-04 fetches (never coco's
|
||||||
|
* observe-and-persist path), bounded by an overall budget and a concurrency
|
||||||
|
* limit so a slow mint cannot stall the whole report. Quotes that could not
|
||||||
|
* be checked are counted in `uncheckedQuotes` rather than silently dropped.
|
||||||
|
*/
|
||||||
|
export async function runWalletDoctor(
|
||||||
|
source: WalletDoctorSource,
|
||||||
|
): Promise<WalletDoctorReport> {
|
||||||
|
const nowMs = Date.now();
|
||||||
|
const report: WalletDoctorReport = {
|
||||||
|
generatedAt: nowMs,
|
||||||
|
mints: [],
|
||||||
|
unpaidQuotes: [],
|
||||||
|
paidUnissued: [],
|
||||||
|
stuckMelts: [],
|
||||||
|
uncheckedQuotes: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const trustedMintUrls = await source.listTrustedMintUrls();
|
||||||
|
report.mints = await Promise.all(
|
||||||
|
trustedMintUrls.map((mintUrl) => source.probeMint(mintUrl)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const [pendingMintOps, failedMintOps] = await Promise.all([
|
||||||
|
source.listPendingMintOps(),
|
||||||
|
source.listFailedMintOps(),
|
||||||
|
]);
|
||||||
|
const recentQuotes = selectRecentMintQuotes(pendingMintOps, nowMs);
|
||||||
|
const recentIds = new Set(recentQuotes.map((op) => op.id));
|
||||||
|
const candidates = selectPaidUnissuedCandidates([
|
||||||
|
...pendingMintOps,
|
||||||
|
...failedMintOps,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const targets = new Map<string, DoctorMintOperation>();
|
||||||
|
for (const op of [...recentQuotes, ...candidates]) {
|
||||||
|
if (!targets.has(op.id)) targets.set(op.id, op);
|
||||||
|
}
|
||||||
|
const queue = [...targets.values()];
|
||||||
|
const budgetMs = source.quoteProbeBudgetMs ?? DOCTOR_QUOTE_PROBE_BUDGET_MS;
|
||||||
|
const concurrency =
|
||||||
|
source.quoteProbeConcurrency ?? DOCTOR_QUOTE_PROBE_CONCURRENCY;
|
||||||
|
const deadlineAt = nowMs + budgetMs;
|
||||||
|
let next = 0;
|
||||||
|
await Promise.all(
|
||||||
|
Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
|
||||||
|
while (next < queue.length && Date.now() < deadlineAt) {
|
||||||
|
const op = queue[next++];
|
||||||
|
if (!op) break;
|
||||||
|
try {
|
||||||
|
const remoteState = await source.fetchQuoteState(
|
||||||
|
op.mintUrl,
|
||||||
|
op.quoteId as string,
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
recentIds.has(op.id) &&
|
||||||
|
mintQuoteStateToCategory(remoteState) === "waiting"
|
||||||
|
) {
|
||||||
|
report.unpaidQuotes.push(toUnpaidQuote(op, nowMs));
|
||||||
|
}
|
||||||
|
const finding = classifyPaidUnissued(op, remoteState);
|
||||||
|
if (finding) report.paidUnissued.push(finding);
|
||||||
|
} catch {
|
||||||
|
report.uncheckedQuotes++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
report.uncheckedQuotes += queue.length - next;
|
||||||
|
// Concurrent workers push in completion order; sort for a stable report.
|
||||||
|
report.unpaidQuotes.sort((a, b) => a.operationId.localeCompare(b.operationId));
|
||||||
|
report.paidUnissued.sort((a, b) => a.operationId.localeCompare(b.operationId));
|
||||||
|
|
||||||
|
const [preparedMelts, inFlightMelts, failedMelts, inflightSecretsList] =
|
||||||
|
await Promise.all([
|
||||||
|
source.listPreparedMelts(),
|
||||||
|
source.listInFlightMelts(),
|
||||||
|
source.listFailedMelts(),
|
||||||
|
source.getInflightProofSecrets(),
|
||||||
|
]);
|
||||||
|
const inflightSecrets = new Set(inflightSecretsList);
|
||||||
|
for (const raw of [...preparedMelts, ...inFlightMelts, ...failedMelts]) {
|
||||||
|
const finding = classifyStuckMelt(toDoctorMeltOperation(raw), {
|
||||||
|
nowMs,
|
||||||
|
inflightSecrets,
|
||||||
|
});
|
||||||
|
if (finding) report.stuckMelts.push(finding);
|
||||||
|
}
|
||||||
|
report.stuckMelts.sort((a, b) => a.operationId.localeCompare(b.operationId));
|
||||||
|
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map coco's melt operation rows onto the doctor's structural subset. */
|
||||||
|
function toDoctorMeltOperation(
|
||||||
|
raw: Record<string, unknown>,
|
||||||
|
): DoctorMeltOperation {
|
||||||
|
return {
|
||||||
|
id: String(raw.id),
|
||||||
|
mintUrl: String(raw.mintUrl),
|
||||||
|
quoteId: typeof raw.quoteId === "string" ? raw.quoteId : undefined,
|
||||||
|
state: String(raw.state),
|
||||||
|
amount: Number(raw.amount ?? 0),
|
||||||
|
feeReserve: Number(raw.fee_reserve ?? 0),
|
||||||
|
inputProofSecrets: Array.isArray(raw.inputProofSecrets)
|
||||||
|
? raw.inputProofSecrets.map((secret) => String(secret))
|
||||||
|
: [],
|
||||||
|
createdAt: Number(raw.createdAt ?? 0),
|
||||||
|
updatedAt: Number(raw.updatedAt ?? 0),
|
||||||
|
error: typeof raw.error === "string" ? raw.error : undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve coco's private repositories for the doctor. Fails closed - a coco
|
||||||
|
* upgrade that renames them must surface here and in the doctor tests, not
|
||||||
|
* silently skip the failed-operation and locked-proof checks.
|
||||||
|
*/
|
||||||
|
function doctorRepositories(coco: Manager): {
|
||||||
|
mintOps: MintOperationRepositoryDoctor;
|
||||||
|
meltOps: MeltOperationRepositoryDoctor;
|
||||||
|
proofs: ProofRepositoryDoctor;
|
||||||
|
} {
|
||||||
|
const inner = coco as unknown as {
|
||||||
|
mintOperationRepository?: MintOperationRepositoryDoctor;
|
||||||
|
meltOperationRepository?: MeltOperationRepositoryDoctor;
|
||||||
|
proofRepository?: ProofRepositoryDoctor;
|
||||||
|
};
|
||||||
|
const checks: Array<[string, object | undefined, string]> = [
|
||||||
|
["mintOperationRepository", inner.mintOperationRepository, "getByState"],
|
||||||
|
["meltOperationRepository", inner.meltOperationRepository, "getByState"],
|
||||||
|
["proofRepository", inner.proofRepository, "getInflightProofs"],
|
||||||
|
];
|
||||||
|
for (const [name, repo, method] of checks) {
|
||||||
|
if (
|
||||||
|
!repo ||
|
||||||
|
typeof (repo as Record<string, unknown>)[method] !== "function"
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`coco ${name}.${method} is unavailable; refusing to run wallet doctor with partial health checks`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
mintOps: inner.mintOperationRepository as MintOperationRepositoryDoctor,
|
||||||
|
meltOps: inner.meltOperationRepository as MeltOperationRepositoryDoctor,
|
||||||
|
proofs: inner.proofRepository as ProofRepositoryDoctor,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read-only NUT-06 reachability probe for one mint. */
|
||||||
|
export async function probeMintInfo(mintUrl: string): Promise<DoctorMintProbe> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
try {
|
||||||
|
const response = await fetch(`${mintUrl.replace(/\/+$/, "")}/v1/info`, {
|
||||||
|
signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
|
return {
|
||||||
|
mintUrl,
|
||||||
|
reachable: true,
|
||||||
|
latencyMs: Date.now() - startedAt,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
mintUrl,
|
||||||
|
reachable: false,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read-only NUT-04 quote state fetch. Unlike coco's
|
||||||
|
* observePendingOperation this persists nothing, which is what keeps the
|
||||||
|
* doctor free of side effects.
|
||||||
|
*/
|
||||||
|
export async function fetchMintQuoteState(
|
||||||
|
mintUrl: string,
|
||||||
|
quoteId: string,
|
||||||
|
): Promise<string> {
|
||||||
|
const response = await fetch(
|
||||||
|
`${mintUrl.replace(/\/+$/, "")}/v1/mint/quote/bolt11/${encodeURIComponent(quoteId)}`,
|
||||||
|
{ signal: AbortSignal.timeout(DOCTOR_MINT_PROBE_TIMEOUT_MS) },
|
||||||
|
);
|
||||||
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
|
const body = (await response.json()) as { state?: unknown };
|
||||||
|
if (typeof body.state !== "string") {
|
||||||
|
throw new Error("mint response missing quote state");
|
||||||
|
}
|
||||||
|
return body.state;
|
||||||
|
}
|
||||||
|
|
||||||
export async function createCocoClient(
|
export async function createCocoClient(
|
||||||
options: CreateCocoClientOptions = {},
|
options: CreateCocoClientOptions = {},
|
||||||
): Promise<WalletClient> {
|
): Promise<WalletClient> {
|
||||||
@@ -2602,6 +2873,39 @@ export async function createCocoClient(
|
|||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|
||||||
|
async diagnoseWallet() {
|
||||||
|
const repos = doctorRepositories(coco);
|
||||||
|
return runWalletDoctor({
|
||||||
|
listTrustedMintUrls: async () =>
|
||||||
|
(await coco.mint.getAllTrustedMints()).map((mint) => mint.mintUrl),
|
||||||
|
listPendingMintOps: async () => {
|
||||||
|
// listInFlight can overlap with listPending; dedupe by id.
|
||||||
|
const byId = new Map<string, DoctorMintOperation>();
|
||||||
|
for (const op of [
|
||||||
|
...(await coco.ops.mint.listPending()),
|
||||||
|
...(await coco.ops.mint.listInFlight()),
|
||||||
|
] as unknown as DoctorMintOperation[]) {
|
||||||
|
byId.set(op.id, op);
|
||||||
|
}
|
||||||
|
return [...byId.values()];
|
||||||
|
},
|
||||||
|
listFailedMintOps: () => repos.mintOps.getByState("failed"),
|
||||||
|
listPreparedMelts: async () =>
|
||||||
|
coco.ops.melt.listPrepared() as unknown as Array<
|
||||||
|
Record<string, unknown>
|
||||||
|
>,
|
||||||
|
listInFlightMelts: async () =>
|
||||||
|
coco.ops.melt.listInFlight() as unknown as Array<
|
||||||
|
Record<string, unknown>
|
||||||
|
>,
|
||||||
|
listFailedMelts: () => repos.meltOps.getByState("failed"),
|
||||||
|
getInflightProofSecrets: async () =>
|
||||||
|
(await repos.proofs.getInflightProofs()).map((proof) => proof.secret),
|
||||||
|
probeMint: probeMintInfo,
|
||||||
|
fetchQuoteState: fetchMintQuoteState,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
async recoverMintQuotes(options, onProgress) {
|
async recoverMintQuotes(options, onProgress) {
|
||||||
await waitForRecovery();
|
await waitForRecovery();
|
||||||
const service = (
|
const service = (
|
||||||
|
|||||||
@@ -142,7 +142,7 @@ export interface WalletDoctorReport {
|
|||||||
paidUnissued: DoctorPaidUnissuedQuote[];
|
paidUnissued: DoctorPaidUnissuedQuote[];
|
||||||
/** Melt operations holding locked proofs. */
|
/** Melt operations holding locked proofs. */
|
||||||
stuckMelts: DoctorStuckMelt[];
|
stuckMelts: DoctorStuckMelt[];
|
||||||
/** Quotes skipped because the probe budget ran out. */
|
/** Quotes whose remote state could not be checked (probe failure or budget exhausted). */
|
||||||
uncheckedQuotes: number;
|
uncheckedQuotes: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,15 @@ import { join } from "node:path";
|
|||||||
import { Manager } from "@cashu/coco-core";
|
import { Manager } from "@cashu/coco-core";
|
||||||
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
||||||
import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint";
|
import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint";
|
||||||
import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client";
|
import {
|
||||||
|
fetchMintQuoteState,
|
||||||
|
probeMintInfo,
|
||||||
|
reopenFailedMintOperation,
|
||||||
|
runMintQuoteRecovery,
|
||||||
|
runWalletDoctor,
|
||||||
|
type WalletDoctorSource,
|
||||||
|
} from "./coco-client";
|
||||||
|
import type { DoctorMintOperation } from "./doctor";
|
||||||
|
|
||||||
type AnyRecord = Record<string, unknown>;
|
type AnyRecord = Record<string, unknown>;
|
||||||
|
|
||||||
@@ -384,3 +392,101 @@ describe("PAID mint quote recovery with a real Manager and mint", () => {
|
|||||||
expect(await booted.spendable()).toBe(21_000);
|
expect(await booted.spendable()).toBe(21_000);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("wallet doctor with a real Manager and mint", () => {
|
||||||
|
function doctorSource(booted: Booted): WalletDoctorSource {
|
||||||
|
const manager = booted.manager;
|
||||||
|
const repos = manager as unknown as {
|
||||||
|
mintOperationRepository: {
|
||||||
|
getByState(state: string): Promise<DoctorMintOperation[]>;
|
||||||
|
};
|
||||||
|
meltOperationRepository: {
|
||||||
|
getByState(state: string): Promise<AnyRecord[]>;
|
||||||
|
};
|
||||||
|
proofRepository: {
|
||||||
|
getInflightProofs(): Promise<Array<{ secret: string }>>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
listTrustedMintUrls: async () =>
|
||||||
|
(await manager.mint.getAllTrustedMints()).map((mint) => mint.mintUrl),
|
||||||
|
listPendingMintOps: async () => {
|
||||||
|
const byId = new Map<string, DoctorMintOperation>();
|
||||||
|
for (const op of [
|
||||||
|
...(await manager.ops.mint.listPending()),
|
||||||
|
...(await manager.ops.mint.listInFlight()),
|
||||||
|
] as unknown as DoctorMintOperation[]) {
|
||||||
|
byId.set(op.id, op);
|
||||||
|
}
|
||||||
|
return [...byId.values()];
|
||||||
|
},
|
||||||
|
listFailedMintOps: () => repos.mintOperationRepository.getByState("failed"),
|
||||||
|
listPreparedMelts: async () =>
|
||||||
|
manager.ops.melt.listPrepared() as unknown as AnyRecord[],
|
||||||
|
listInFlightMelts: async () =>
|
||||||
|
manager.ops.melt.listInFlight() as unknown as AnyRecord[],
|
||||||
|
listFailedMelts: () => repos.meltOperationRepository.getByState("failed"),
|
||||||
|
getInflightProofSecrets: async () =>
|
||||||
|
(await repos.proofRepository.getInflightProofs()).map(
|
||||||
|
(proof) => proof.secret,
|
||||||
|
),
|
||||||
|
probeMint: probeMintInfo,
|
||||||
|
fetchQuoteState: fetchMintQuoteState,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it("surfaces unpaid, then paid-unissued, then a clean bill after recovery", async () => {
|
||||||
|
booted = await boot({ quoteExpiry: null });
|
||||||
|
const op = await prepareQuote(booted, 100);
|
||||||
|
|
||||||
|
// Fresh quote, unpaid: the doctor reaches the mint and lists the quote.
|
||||||
|
const unpaid = await runWalletDoctor(doctorSource(booted));
|
||||||
|
expect(unpaid.mints).toHaveLength(1);
|
||||||
|
expect(unpaid.mints[0]?.reachable).toBe(true);
|
||||||
|
expect(unpaid.paidUnissued).toEqual([]);
|
||||||
|
expect(unpaid.uncheckedQuotes).toBe(0);
|
||||||
|
expect(unpaid.unpaidQuotes).toHaveLength(1);
|
||||||
|
expect(unpaid.unpaidQuotes[0]).toMatchObject({
|
||||||
|
operationId: op.id as string,
|
||||||
|
amount: 100,
|
||||||
|
});
|
||||||
|
|
||||||
|
// The quote gets paid while the daemon is "down": the doctor surfaces
|
||||||
|
// exactly the stuck scenario recovery exists for, with the recover hint.
|
||||||
|
booted.mint.markPaid(op.quoteId as string);
|
||||||
|
const stuck = await runWalletDoctor(doctorSource(booted));
|
||||||
|
expect(stuck.unpaidQuotes).toEqual([]);
|
||||||
|
expect(stuck.paidUnissued).toHaveLength(1);
|
||||||
|
expect(stuck.paidUnissued[0]).toMatchObject({
|
||||||
|
operationId: op.id as string,
|
||||||
|
remoteState: "PAID",
|
||||||
|
amount: 100,
|
||||||
|
remediation: `routstrd wallet recover --op ${op.id as string}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Recovery claims the sats; the doctor goes back to all-green.
|
||||||
|
const recovered = (await runMintQuoteRecovery(
|
||||||
|
booted.source() as never,
|
||||||
|
)) as unknown as Record<string, number>;
|
||||||
|
expect(recovered).toMatchObject({ recovered: 1 });
|
||||||
|
const clean = await runWalletDoctor(doctorSource(booted));
|
||||||
|
expect(clean.unpaidQuotes).toEqual([]);
|
||||||
|
expect(clean.paidUnissued).toEqual([]);
|
||||||
|
expect(clean.stuckMelts).toEqual([]);
|
||||||
|
expect(clean.uncheckedQuotes).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports unreachable mints without failing the other checks", async () => {
|
||||||
|
booted = await boot({ quoteExpiry: null });
|
||||||
|
await prepareQuote(booted, 100);
|
||||||
|
booted.mint.stop();
|
||||||
|
|
||||||
|
const report = await runWalletDoctor(doctorSource(booted));
|
||||||
|
expect(report.mints[0]?.reachable).toBe(false);
|
||||||
|
// The quote probe fails too, and is counted rather than dropped.
|
||||||
|
expect(report.uncheckedQuotes).toBe(1);
|
||||||
|
expect(report.unpaidQuotes).toEqual([]);
|
||||||
|
expect(report.paidUnissued).toEqual([]);
|
||||||
|
booted.mint.start();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user