feat: add wallet cleanup command for stuck operations

Adds `routstrd wallet cleanup` to clear stuck wallet operations that
slow startup recovery:
- Fail expired pending mint quotes (bolt11 invoices that can never be paid)
- Reclaim pending sends older than the min-age threshold (default 1 week)
- Cancel prepared melts older than the threshold

Includes dry-run mode, per-mint filtering, confirmation prompt, and an
HTTP endpoint (POST /wallet/cleanup) backed by coco-core operation
services so state transitions emit normal events and release proof
reservations. Selection logic is pure and unit-tested.
This commit is contained in:
redshift
2026-08-16 22:32:55 +01:00
parent 530db89677
commit 850714656a
6 changed files with 502 additions and 0 deletions
+107
View File
@@ -1525,6 +1525,113 @@ walletCmd
await handleDaemonCommand("/wallet/balance");
});
walletCmd
.command("cleanup")
.description("Clear stuck pending/in-flight wallet operations")
.option("--mint-url <url>", "Only clean up operations for this mint URL")
.option(
"--min-age <hours>",
"Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)",
"168",
)
.option("--dry-run", "Report what would be cleaned without applying changes", false)
.option("-y, --yes", "Skip confirmation prompt", false)
.action(
async (options: {
mintUrl?: string;
minAge: string;
dryRun: boolean;
yes: boolean;
}) => {
const minAgeHours = Number.parseFloat(options.minAge);
if (!Number.isFinite(minAgeHours) || minAgeHours < 0) {
console.error(`Invalid --min-age value: ${options.minAge}`);
process.exit(1);
}
if (!options.dryRun && !options.yes) {
const rl = require("readline").createInterface({
input: process.stdin,
output: process.stdout,
});
const answer = await new Promise<string>((resolve) => {
rl.question(
"This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ",
(value: string) => {
rl.close();
resolve(value.trim().toLowerCase());
},
);
});
if (answer !== "y" && answer !== "yes") {
console.log("Aborted.");
return;
}
}
try {
await ensureDaemonRunning();
const result = await callDaemon("/wallet/cleanup", {
method: "POST",
body: {
mintUrl: options.mintUrl,
minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000),
dryRun: options.dryRun === true,
},
});
if (result.error) {
console.log(result.error);
process.exit(1);
}
const output = result.output as
| {
dryRun?: boolean;
failedMintQuotes?: number;
reclaimedSends?: number;
cancelledMelts?: number;
skipped?: number;
errors?: Array<{ operationId: string; error: string }>;
}
| undefined;
if (output) {
const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:";
console.log(prefix);
console.log(
` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`,
);
console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`);
console.log(
` Prepared melts cancelled: ${output.cancelledMelts ?? 0}`,
);
console.log(
` Skipped (still recent or already terminal): ${output.skipped ?? 0}`,
);
if (output.errors && output.errors.length > 0) {
console.log("\nErrors:");
for (const e of output.errors) {
console.log(` - ${e.operationId}: ${e.error}`);
}
}
}
} catch (error) {
const message = (error as Error).message;
if (
message?.includes("fetch failed") ||
message?.includes("Connection refused")
) {
console.error("Daemon is not running");
process.exit(1);
}
console.error(message);
process.exit(1);
}
},
);
const walletReceiveCmd = walletCmd
.command("receive")
.description("Wallet receive operations");
+23
View File
@@ -387,6 +387,29 @@ export function createDaemonRequestHandler(deps: {
return;
}
if (req.method === "POST" && url.pathname === "/wallet/cleanup") {
await respond(res, async () => {
if (!deps.walletClient.cleanupStuckOperations) {
throw new CocodHttpError(
501,
"Wallet cleanup is not supported by this wallet client.",
);
}
const body = await readJsonBody(req);
const result = await deps.walletClient.cleanupStuckOperations({
mintUrl: optionalStringField(body, "mintUrl"),
minAgeMs:
typeof body.minAgeMs === "number" && Number.isFinite(body.minAgeMs)
? body.minAgeMs
: undefined,
dryRun: body.dryRun === true,
});
return { output: result };
});
return;
}
if (req.method === "POST" && url.pathname === "/wallet/receive/cashu") {
await respond(res, async () => {
const body = await readJsonBody(req);
+143
View File
@@ -0,0 +1,143 @@
import { describe, expect, it } from "bun:test";
import { selectCleanupOperations } from "./cleanup";
const NOW_MS = 1_800_000_000_000;
const DAY_MS = 24 * 60 * 60 * 1000;
function mint(overrides: Record<string, unknown>) {
return {
id: "mint-1",
state: "pending",
expiry: NOW_MS / 1000 - 1000, // expired
updatedAt: NOW_MS - 2 * DAY_MS,
...overrides,
};
}
function send(overrides: Record<string, unknown>) {
return {
id: "send-1",
state: "pending",
updatedAt: NOW_MS - 2 * DAY_MS,
...overrides,
};
}
function melt(overrides: Record<string, unknown>) {
return {
id: "melt-1",
state: "prepared",
updatedAt: NOW_MS - 2 * DAY_MS,
...overrides,
};
}
describe("selectCleanupOperations", () => {
it("selects expired pending mint quotes that are old enough", () => {
const result = selectCleanupOperations({
mints: [mint({ id: "a" })],
sends: [],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.mintsToFail.map((op) => op.id)).toEqual(["a"]);
});
it("ignores mint quotes that have not expired", () => {
const result = selectCleanupOperations({
mints: [mint({ id: "a", expiry: NOW_MS / 1000 + 1000 })],
sends: [],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.mintsToFail).toEqual([]);
});
it("ignores mint quotes without an expiry", () => {
const result = selectCleanupOperations({
mints: [mint({ id: "a", expiry: 0 })],
sends: [],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.mintsToFail).toEqual([]);
});
it("selects expired mint quotes even when the watcher recently touched them", () => {
const result = selectCleanupOperations({
mints: [mint({ id: "a", updatedAt: NOW_MS - 60_000 })],
sends: [],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.mintsToFail.map((op) => op.id)).toEqual(["a"]);
});
it("selects stale pending sends for reclaim", () => {
const result = selectCleanupOperations({
mints: [],
sends: [send({ id: "s" })],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.sendsToReclaim.map((op) => op.id)).toEqual(["s"]);
});
it("ignores recent pending sends", () => {
const result = selectCleanupOperations({
mints: [],
sends: [send({ id: "s", updatedAt: NOW_MS - 60_000 })],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.sendsToReclaim).toEqual([]);
});
it("ignores non-pending sends even when old", () => {
const result = selectCleanupOperations({
mints: [],
sends: [send({ id: "s", state: "executing" })],
melts: [],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.sendsToReclaim).toEqual([]);
});
it("selects stale prepared melts for cancellation", () => {
const result = selectCleanupOperations({
mints: [],
sends: [],
melts: [melt({ id: "m" })],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.meltsToCancel.map((op) => op.id)).toEqual(["m"]);
});
it("ignores non-prepared melts", () => {
const result = selectCleanupOperations({
mints: [],
sends: [],
melts: [melt({ id: "m", state: "pending" })],
nowMs: NOW_MS,
minAgeMs: DAY_MS,
});
expect(result.meltsToCancel).toEqual([]);
});
});
+86
View File
@@ -0,0 +1,86 @@
/**
* Pure selection helpers for the wallet cleanup command.
*
* These helpers decide *which* stuck operations are safe to clear. The actual
* state transitions are applied by the in-process coco wallet client so that
* coco-core's operation services emit their normal events and release proof
* reservations. Keeping the selection logic here makes it easy to unit test
* without a wallet database or network access.
*/
export interface MintCleanupCandidate {
id: string;
state: string;
/** Quote expiry in epoch seconds. `0` means unknown/not applicable. */
expiry: number;
/** Last update time in epoch milliseconds. */
updatedAt: number;
}
export interface NonMintCleanupCandidate {
id: string;
state: string;
/** Last update time in epoch milliseconds. */
updatedAt: number;
}
export interface CleanupSelectionOptions<
TMint extends MintCleanupCandidate,
TSend extends NonMintCleanupCandidate,
TMelt extends NonMintCleanupCandidate,
> {
mints: TMint[];
sends: TSend[];
melts: TMelt[];
nowMs: number;
minAgeMs: number;
}
export interface CleanupSelection<
TMint extends MintCleanupCandidate,
TSend extends NonMintCleanupCandidate,
TMelt extends NonMintCleanupCandidate,
> {
mintsToFail: TMint[];
sendsToReclaim: TSend[];
meltsToCancel: TMelt[];
}
/**
* Select stuck operations that are old enough to be safe to clear.
*
* - Pending mint quotes are failed only when their bolt11 quote has expired
* (an expired Lightning invoice can never be paid).
* - Pending sends are reclaimed (rolled back) only when they are older than
* `minAgeMs`, so we never roll back a token that a receiver might still
* legitimately claim.
* - Prepared melts are cancelled under the same age guard.
*/
export function selectCleanupOperations<
TMint extends MintCleanupCandidate,
TSend extends NonMintCleanupCandidate,
TMelt extends NonMintCleanupCandidate,
>(
options: CleanupSelectionOptions<TMint, TSend, TMelt>,
): CleanupSelection<TMint, TSend, TMelt> {
const { mints, sends, melts, nowMs, minAgeMs } = options;
// Expiry alone is enough for mint quotes: once a bolt11 quote has expired it
// can never be paid, regardless of when the watcher last touched the row.
const mintsToFail = mints.filter(
(op) =>
op.state === "pending" &&
op.expiry > 0 &&
op.expiry * 1000 <= nowMs,
);
const sendsToReclaim = sends.filter(
(op) => op.state === "pending" && nowMs - op.updatedAt >= minAgeMs,
);
const meltsToCancel = melts.filter(
(op) => op.state === "prepared" && nowMs - op.updatedAt >= minAgeMs,
);
return { mintsToFail, sendsToReclaim, meltsToCancel };
}
+115
View File
@@ -30,7 +30,10 @@ import type {
CocodState,
NpcAddress,
NpcUsernameResult,
WalletCleanupOptions,
WalletCleanupResult,
} from "./cocod-client";
import { selectCleanupOperations } from "./cleanup";
import { cocoLogger, logger } from "../../utils/logger";
import {
legacyCocodPidPath,
@@ -512,6 +515,19 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
};
}
/**
* Minimal structural view of coco-core's MintOperationService.
* `failPendingOperation` is private on the exported class, so the in-process
* client reaches it through this narrow cast. The method only needs the
* operation id; it reloads the latest persisted row before mutating it.
*/
interface MintOperationServiceCleanup {
failPendingOperation(
op: { id: string },
terminalFailure: { reason: string; retryable?: boolean; observedAt: number },
): Promise<unknown>;
}
export interface CreateCocoClientOptions {
/** Override the canonical wallet data directory. */
walletDir?: string;
@@ -825,5 +841,104 @@ export async function createCocoClient(
async syncNpc(): Promise<void> {
await npcApi().sync();
},
async cleanupStuckOperations(
options: WalletCleanupOptions = {},
): Promise<WalletCleanupResult> {
const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000;
const dryRun = options.dryRun === true;
const nowMs = Date.now();
const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([
coco.ops.mint.listPending(),
coco.ops.send.listInFlight(),
coco.ops.melt.listPrepared(),
]);
const filteredMints = options.mintUrl
? pendingMints.filter((op) => op.mintUrl === options.mintUrl)
: pendingMints;
const filteredSends = options.mintUrl
? inFlightSends.filter((op) => op.mintUrl === options.mintUrl)
: inFlightSends;
const filteredMelts = options.mintUrl
? preparedMelts.filter((op) => op.mintUrl === options.mintUrl)
: preparedMelts;
const selection = selectCleanupOperations({
mints: filteredMints,
sends: filteredSends,
melts: filteredMelts,
nowMs,
minAgeMs,
});
const errors: WalletCleanupResult["errors"] = [];
if (!dryRun) {
const mintService = (
coco as unknown as {
mintOperationService: MintOperationServiceCleanup;
}
).mintOperationService;
for (const op of selection.mintsToFail) {
try {
await mintService.failPendingOperation(
{ id: op.id },
{
reason: "Expired unpaid mint quote cleaned up by routstrd",
retryable: false,
observedAt: nowMs,
},
);
} catch (error) {
errors.push({
operationId: op.id,
error: error instanceof Error ? error.message : String(error),
});
}
}
for (const op of selection.sendsToReclaim) {
try {
await coco.ops.send.reclaim(op.id);
} catch (error) {
errors.push({
operationId: op.id,
error: error instanceof Error ? error.message : String(error),
});
}
}
for (const op of selection.meltsToCancel) {
try {
await coco.ops.melt.cancel(op.id, "Cancelled by wallet cleanup");
} catch (error) {
errors.push({
operationId: op.id,
error: error instanceof Error ? error.message : String(error),
});
}
}
}
const actedOn =
selection.mintsToFail.length +
selection.sendsToReclaim.length +
selection.meltsToCancel.length;
const skipped =
filteredMints.length + filteredSends.length + filteredMelts.length -
actedOn;
return {
dryRun,
failedMintQuotes: selection.mintsToFail.length,
reclaimedSends: selection.sendsToReclaim.length,
cancelledMelts: selection.meltsToCancel.length,
skipped,
errors,
};
},
};
}
+28
View File
@@ -57,6 +57,30 @@ export interface NpcUsernameResult {
};
}
/** Options for the wallet cleanup command. */
export interface WalletCleanupOptions {
/** Only clean up operations for this mint URL. */
mintUrl?: string;
/** Minimum operation age in milliseconds (defaults to 7 days / 1 week). */
minAgeMs?: number;
/** Report what would be cleaned without applying changes. */
dryRun?: boolean;
}
/** Summary of a wallet cleanup run. */
export interface WalletCleanupResult {
dryRun: boolean;
/** Number of expired pending mint quotes marked as failed. */
failedMintQuotes: number;
/** Number of stale pending send operations reclaimed. */
reclaimedSends: number;
/** Number of stale prepared melt operations cancelled. */
cancelledMelts: number;
/** Number of in-flight operations that were left untouched. */
skipped: number;
errors: Array<{ operationId: string; error: string }>;
}
export class CocodHttpError extends Error {
status: number;
@@ -90,6 +114,10 @@ export interface CocodClient {
setNpcUsername(username: string, confirm?: boolean): Promise<NpcUsernameResult>;
/** Manually trigger an NPC quote sync into the wallet. */
syncNpc(): Promise<void>;
/** Clear stuck pending/in-flight wallet operations that are safe to resolve. */
cleanupStuckOperations?(
options?: WalletCleanupOptions,
): Promise<WalletCleanupResult>;
}
export function resolveCocodExecutable(cocodPath?: string | null): string {