mirror of
https://github.com/Routstr/routstrd.git
synced 2026-10-05 12:28:23 +00:00
feat: add wallet cleanup command for stuck operations
Adds `routstrd wallet cleanup` to clear stuck wallet operations that slow startup recovery: - Fail expired pending mint quotes (bolt11 invoices that can never be paid) - Reclaim pending sends older than the min-age threshold (default 1 week) - Cancel prepared melts older than the threshold Includes dry-run mode, per-mint filtering, confirmation prompt, and an HTTP endpoint (POST /wallet/cleanup) backed by coco-core operation services so state transitions emit normal events and release proof reservations. Selection logic is pure and unit-tested.
This commit is contained in:
+107
@@ -1525,6 +1525,113 @@ walletCmd
|
||||
await handleDaemonCommand("/wallet/balance");
|
||||
});
|
||||
|
||||
walletCmd
|
||||
.command("cleanup")
|
||||
.description("Clear stuck pending/in-flight wallet operations")
|
||||
.option("--mint-url <url>", "Only clean up operations for this mint URL")
|
||||
.option(
|
||||
"--min-age <hours>",
|
||||
"Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)",
|
||||
"168",
|
||||
)
|
||||
.option("--dry-run", "Report what would be cleaned without applying changes", false)
|
||||
.option("-y, --yes", "Skip confirmation prompt", false)
|
||||
.action(
|
||||
async (options: {
|
||||
mintUrl?: string;
|
||||
minAge: string;
|
||||
dryRun: boolean;
|
||||
yes: boolean;
|
||||
}) => {
|
||||
const minAgeHours = Number.parseFloat(options.minAge);
|
||||
if (!Number.isFinite(minAgeHours) || minAgeHours < 0) {
|
||||
console.error(`Invalid --min-age value: ${options.minAge}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!options.dryRun && !options.yes) {
|
||||
const rl = require("readline").createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
const answer = await new Promise<string>((resolve) => {
|
||||
rl.question(
|
||||
"This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ",
|
||||
(value: string) => {
|
||||
rl.close();
|
||||
resolve(value.trim().toLowerCase());
|
||||
},
|
||||
);
|
||||
});
|
||||
if (answer !== "y" && answer !== "yes") {
|
||||
console.log("Aborted.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureDaemonRunning();
|
||||
|
||||
const result = await callDaemon("/wallet/cleanup", {
|
||||
method: "POST",
|
||||
body: {
|
||||
mintUrl: options.mintUrl,
|
||||
minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000),
|
||||
dryRun: options.dryRun === true,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
console.log(result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const output = result.output as
|
||||
| {
|
||||
dryRun?: boolean;
|
||||
failedMintQuotes?: number;
|
||||
reclaimedSends?: number;
|
||||
cancelledMelts?: number;
|
||||
skipped?: number;
|
||||
errors?: Array<{ operationId: string; error: string }>;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
if (output) {
|
||||
const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:";
|
||||
console.log(prefix);
|
||||
console.log(
|
||||
` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`,
|
||||
);
|
||||
console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`);
|
||||
console.log(
|
||||
` Prepared melts cancelled: ${output.cancelledMelts ?? 0}`,
|
||||
);
|
||||
console.log(
|
||||
` Skipped (still recent or already terminal): ${output.skipped ?? 0}`,
|
||||
);
|
||||
if (output.errors && output.errors.length > 0) {
|
||||
console.log("\nErrors:");
|
||||
for (const e of output.errors) {
|
||||
console.log(` - ${e.operationId}: ${e.error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const message = (error as Error).message;
|
||||
if (
|
||||
message?.includes("fetch failed") ||
|
||||
message?.includes("Connection refused")
|
||||
) {
|
||||
console.error("Daemon is not running");
|
||||
process.exit(1);
|
||||
}
|
||||
console.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const walletReceiveCmd = walletCmd
|
||||
.command("receive")
|
||||
.description("Wallet receive operations");
|
||||
|
||||
@@ -387,6 +387,29 @@ export function createDaemonRequestHandler(deps: {
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && url.pathname === "/wallet/cleanup") {
|
||||
await respond(res, async () => {
|
||||
if (!deps.walletClient.cleanupStuckOperations) {
|
||||
throw new CocodHttpError(
|
||||
501,
|
||||
"Wallet cleanup is not supported by this wallet client.",
|
||||
);
|
||||
}
|
||||
|
||||
const body = await readJsonBody(req);
|
||||
const result = await deps.walletClient.cleanupStuckOperations({
|
||||
mintUrl: optionalStringField(body, "mintUrl"),
|
||||
minAgeMs:
|
||||
typeof body.minAgeMs === "number" && Number.isFinite(body.minAgeMs)
|
||||
? body.minAgeMs
|
||||
: undefined,
|
||||
dryRun: body.dryRun === true,
|
||||
});
|
||||
return { output: result };
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && url.pathname === "/wallet/receive/cashu") {
|
||||
await respond(res, async () => {
|
||||
const body = await readJsonBody(req);
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { selectCleanupOperations } from "./cleanup";
|
||||
|
||||
const NOW_MS = 1_800_000_000_000;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
function mint(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
id: "mint-1",
|
||||
state: "pending",
|
||||
expiry: NOW_MS / 1000 - 1000, // expired
|
||||
updatedAt: NOW_MS - 2 * DAY_MS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function send(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
id: "send-1",
|
||||
state: "pending",
|
||||
updatedAt: NOW_MS - 2 * DAY_MS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function melt(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
id: "melt-1",
|
||||
state: "prepared",
|
||||
updatedAt: NOW_MS - 2 * DAY_MS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("selectCleanupOperations", () => {
|
||||
it("selects expired pending mint quotes that are old enough", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [mint({ id: "a" })],
|
||||
sends: [],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.mintsToFail.map((op) => op.id)).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("ignores mint quotes that have not expired", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [mint({ id: "a", expiry: NOW_MS / 1000 + 1000 })],
|
||||
sends: [],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.mintsToFail).toEqual([]);
|
||||
});
|
||||
|
||||
it("ignores mint quotes without an expiry", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [mint({ id: "a", expiry: 0 })],
|
||||
sends: [],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.mintsToFail).toEqual([]);
|
||||
});
|
||||
|
||||
it("selects expired mint quotes even when the watcher recently touched them", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [mint({ id: "a", updatedAt: NOW_MS - 60_000 })],
|
||||
sends: [],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.mintsToFail.map((op) => op.id)).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("selects stale pending sends for reclaim", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [],
|
||||
sends: [send({ id: "s" })],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.sendsToReclaim.map((op) => op.id)).toEqual(["s"]);
|
||||
});
|
||||
|
||||
it("ignores recent pending sends", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [],
|
||||
sends: [send({ id: "s", updatedAt: NOW_MS - 60_000 })],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.sendsToReclaim).toEqual([]);
|
||||
});
|
||||
|
||||
it("ignores non-pending sends even when old", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [],
|
||||
sends: [send({ id: "s", state: "executing" })],
|
||||
melts: [],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.sendsToReclaim).toEqual([]);
|
||||
});
|
||||
|
||||
it("selects stale prepared melts for cancellation", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [],
|
||||
sends: [],
|
||||
melts: [melt({ id: "m" })],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.meltsToCancel.map((op) => op.id)).toEqual(["m"]);
|
||||
});
|
||||
|
||||
it("ignores non-prepared melts", () => {
|
||||
const result = selectCleanupOperations({
|
||||
mints: [],
|
||||
sends: [],
|
||||
melts: [melt({ id: "m", state: "pending" })],
|
||||
nowMs: NOW_MS,
|
||||
minAgeMs: DAY_MS,
|
||||
});
|
||||
|
||||
expect(result.meltsToCancel).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Pure selection helpers for the wallet cleanup command.
|
||||
*
|
||||
* These helpers decide *which* stuck operations are safe to clear. The actual
|
||||
* state transitions are applied by the in-process coco wallet client so that
|
||||
* coco-core's operation services emit their normal events and release proof
|
||||
* reservations. Keeping the selection logic here makes it easy to unit test
|
||||
* without a wallet database or network access.
|
||||
*/
|
||||
|
||||
export interface MintCleanupCandidate {
|
||||
id: string;
|
||||
state: string;
|
||||
/** Quote expiry in epoch seconds. `0` means unknown/not applicable. */
|
||||
expiry: number;
|
||||
/** Last update time in epoch milliseconds. */
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export interface NonMintCleanupCandidate {
|
||||
id: string;
|
||||
state: string;
|
||||
/** Last update time in epoch milliseconds. */
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export interface CleanupSelectionOptions<
|
||||
TMint extends MintCleanupCandidate,
|
||||
TSend extends NonMintCleanupCandidate,
|
||||
TMelt extends NonMintCleanupCandidate,
|
||||
> {
|
||||
mints: TMint[];
|
||||
sends: TSend[];
|
||||
melts: TMelt[];
|
||||
nowMs: number;
|
||||
minAgeMs: number;
|
||||
}
|
||||
|
||||
export interface CleanupSelection<
|
||||
TMint extends MintCleanupCandidate,
|
||||
TSend extends NonMintCleanupCandidate,
|
||||
TMelt extends NonMintCleanupCandidate,
|
||||
> {
|
||||
mintsToFail: TMint[];
|
||||
sendsToReclaim: TSend[];
|
||||
meltsToCancel: TMelt[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Select stuck operations that are old enough to be safe to clear.
|
||||
*
|
||||
* - Pending mint quotes are failed only when their bolt11 quote has expired
|
||||
* (an expired Lightning invoice can never be paid).
|
||||
* - Pending sends are reclaimed (rolled back) only when they are older than
|
||||
* `minAgeMs`, so we never roll back a token that a receiver might still
|
||||
* legitimately claim.
|
||||
* - Prepared melts are cancelled under the same age guard.
|
||||
*/
|
||||
export function selectCleanupOperations<
|
||||
TMint extends MintCleanupCandidate,
|
||||
TSend extends NonMintCleanupCandidate,
|
||||
TMelt extends NonMintCleanupCandidate,
|
||||
>(
|
||||
options: CleanupSelectionOptions<TMint, TSend, TMelt>,
|
||||
): CleanupSelection<TMint, TSend, TMelt> {
|
||||
const { mints, sends, melts, nowMs, minAgeMs } = options;
|
||||
|
||||
// Expiry alone is enough for mint quotes: once a bolt11 quote has expired it
|
||||
// can never be paid, regardless of when the watcher last touched the row.
|
||||
const mintsToFail = mints.filter(
|
||||
(op) =>
|
||||
op.state === "pending" &&
|
||||
op.expiry > 0 &&
|
||||
op.expiry * 1000 <= nowMs,
|
||||
);
|
||||
|
||||
const sendsToReclaim = sends.filter(
|
||||
(op) => op.state === "pending" && nowMs - op.updatedAt >= minAgeMs,
|
||||
);
|
||||
|
||||
const meltsToCancel = melts.filter(
|
||||
(op) => op.state === "prepared" && nowMs - op.updatedAt >= minAgeMs,
|
||||
);
|
||||
|
||||
return { mintsToFail, sendsToReclaim, meltsToCancel };
|
||||
}
|
||||
@@ -30,7 +30,10 @@ import type {
|
||||
CocodState,
|
||||
NpcAddress,
|
||||
NpcUsernameResult,
|
||||
WalletCleanupOptions,
|
||||
WalletCleanupResult,
|
||||
} from "./cocod-client";
|
||||
import { selectCleanupOperations } from "./cleanup";
|
||||
import { cocoLogger, logger } from "../../utils/logger";
|
||||
import {
|
||||
legacyCocodPidPath,
|
||||
@@ -512,6 +515,19 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal structural view of coco-core's MintOperationService.
|
||||
* `failPendingOperation` is private on the exported class, so the in-process
|
||||
* client reaches it through this narrow cast. The method only needs the
|
||||
* operation id; it reloads the latest persisted row before mutating it.
|
||||
*/
|
||||
interface MintOperationServiceCleanup {
|
||||
failPendingOperation(
|
||||
op: { id: string },
|
||||
terminalFailure: { reason: string; retryable?: boolean; observedAt: number },
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface CreateCocoClientOptions {
|
||||
/** Override the canonical wallet data directory. */
|
||||
walletDir?: string;
|
||||
@@ -825,5 +841,104 @@ export async function createCocoClient(
|
||||
async syncNpc(): Promise<void> {
|
||||
await npcApi().sync();
|
||||
},
|
||||
|
||||
async cleanupStuckOperations(
|
||||
options: WalletCleanupOptions = {},
|
||||
): Promise<WalletCleanupResult> {
|
||||
const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
const dryRun = options.dryRun === true;
|
||||
const nowMs = Date.now();
|
||||
|
||||
const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([
|
||||
coco.ops.mint.listPending(),
|
||||
coco.ops.send.listInFlight(),
|
||||
coco.ops.melt.listPrepared(),
|
||||
]);
|
||||
|
||||
const filteredMints = options.mintUrl
|
||||
? pendingMints.filter((op) => op.mintUrl === options.mintUrl)
|
||||
: pendingMints;
|
||||
const filteredSends = options.mintUrl
|
||||
? inFlightSends.filter((op) => op.mintUrl === options.mintUrl)
|
||||
: inFlightSends;
|
||||
const filteredMelts = options.mintUrl
|
||||
? preparedMelts.filter((op) => op.mintUrl === options.mintUrl)
|
||||
: preparedMelts;
|
||||
|
||||
const selection = selectCleanupOperations({
|
||||
mints: filteredMints,
|
||||
sends: filteredSends,
|
||||
melts: filteredMelts,
|
||||
nowMs,
|
||||
minAgeMs,
|
||||
});
|
||||
|
||||
const errors: WalletCleanupResult["errors"] = [];
|
||||
|
||||
if (!dryRun) {
|
||||
const mintService = (
|
||||
coco as unknown as {
|
||||
mintOperationService: MintOperationServiceCleanup;
|
||||
}
|
||||
).mintOperationService;
|
||||
|
||||
for (const op of selection.mintsToFail) {
|
||||
try {
|
||||
await mintService.failPendingOperation(
|
||||
{ id: op.id },
|
||||
{
|
||||
reason: "Expired unpaid mint quote cleaned up by routstrd",
|
||||
retryable: false,
|
||||
observedAt: nowMs,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const op of selection.sendsToReclaim) {
|
||||
try {
|
||||
await coco.ops.send.reclaim(op.id);
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const op of selection.meltsToCancel) {
|
||||
try {
|
||||
await coco.ops.melt.cancel(op.id, "Cancelled by wallet cleanup");
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const actedOn =
|
||||
selection.mintsToFail.length +
|
||||
selection.sendsToReclaim.length +
|
||||
selection.meltsToCancel.length;
|
||||
const skipped =
|
||||
filteredMints.length + filteredSends.length + filteredMelts.length -
|
||||
actedOn;
|
||||
|
||||
return {
|
||||
dryRun,
|
||||
failedMintQuotes: selection.mintsToFail.length,
|
||||
reclaimedSends: selection.sendsToReclaim.length,
|
||||
cancelledMelts: selection.meltsToCancel.length,
|
||||
skipped,
|
||||
errors,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -57,6 +57,30 @@ export interface NpcUsernameResult {
|
||||
};
|
||||
}
|
||||
|
||||
/** Options for the wallet cleanup command. */
|
||||
export interface WalletCleanupOptions {
|
||||
/** Only clean up operations for this mint URL. */
|
||||
mintUrl?: string;
|
||||
/** Minimum operation age in milliseconds (defaults to 7 days / 1 week). */
|
||||
minAgeMs?: number;
|
||||
/** Report what would be cleaned without applying changes. */
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
/** Summary of a wallet cleanup run. */
|
||||
export interface WalletCleanupResult {
|
||||
dryRun: boolean;
|
||||
/** Number of expired pending mint quotes marked as failed. */
|
||||
failedMintQuotes: number;
|
||||
/** Number of stale pending send operations reclaimed. */
|
||||
reclaimedSends: number;
|
||||
/** Number of stale prepared melt operations cancelled. */
|
||||
cancelledMelts: number;
|
||||
/** Number of in-flight operations that were left untouched. */
|
||||
skipped: number;
|
||||
errors: Array<{ operationId: string; error: string }>;
|
||||
}
|
||||
|
||||
export class CocodHttpError extends Error {
|
||||
status: number;
|
||||
|
||||
@@ -90,6 +114,10 @@ export interface CocodClient {
|
||||
setNpcUsername(username: string, confirm?: boolean): Promise<NpcUsernameResult>;
|
||||
/** Manually trigger an NPC quote sync into the wallet. */
|
||||
syncNpc(): Promise<void>;
|
||||
/** Clear stuck pending/in-flight wallet operations that are safe to resolve. */
|
||||
cleanupStuckOperations?(
|
||||
options?: WalletCleanupOptions,
|
||||
): Promise<WalletCleanupResult>;
|
||||
}
|
||||
|
||||
export function resolveCocodExecutable(cocodPath?: string | null): string {
|
||||
|
||||
Reference in New Issue
Block a user