mirror of
https://github.com/Routstr/routstrd.git
synced 2026-10-05 12:28:23 +00:00
Merge pull request #114 from Routstr/paid-mint-quote-recovery
feat(wallet): recover PAID mint quotes that were paid but never issued
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
# Mint quote recovery: scope and troubleshooting
|
||||
|
||||
`routstrd wallet recover` explicitly retries mint operations through coco using
|
||||
**their existing stored outputs**. It can restore signatures when a quote is
|
||||
already issued, and reopen failed operations when explicitly requested:
|
||||
|
||||
```sh
|
||||
routstrd history --json
|
||||
routstrd wallet recover --op <operationId> --include-failed
|
||||
```
|
||||
|
||||
Failed operations require explicit IDs over both HTTP and the CLI. A successful
|
||||
re-run on an already finalized operation is a no-op. Requests that exceed their
|
||||
wait budget are not cancelled; explicit retries skip the operation while the
|
||||
underlying work is outstanding.
|
||||
|
||||
## What this fixes—and what it does not
|
||||
|
||||
Coco already checks pending quotes on startup and the daemon periodically
|
||||
refreshes them. A quote paid while the daemon was offline does not, by itself,
|
||||
require a new issuance implementation.
|
||||
|
||||
This change makes normal cleanup confirm UNPAID with the mint before failing an
|
||||
expired quote. PAID, ISSUED and unverified quotes remain pending. It also gives
|
||||
operators a recovery path for operations previously marked failed.
|
||||
|
||||
It does **not** replace rejected outputs with fresh outputs on an active keyset.
|
||||
An inactive-keyset rejection can therefore remain retryable with zero recovery.
|
||||
Recovery reports coco's persisted mint error when available, rather than only a
|
||||
generic “remains pending” error.
|
||||
|
||||
Do not infer that the production incidents were caused by keyset retirement.
|
||||
Before claiming those incidents are fixed, collect:
|
||||
|
||||
- The affected operation IDs, quote IDs, state and persisted `error`.
|
||||
- A fresh remote quote state and, where provided, paid/issued amounts.
|
||||
- The keyset IDs in the stored outputs and the mint's current keyset metadata.
|
||||
- A reproduction showing existing recovery fails and the proposed fix succeeds.
|
||||
|
||||
Inspect persisted operation data through a read-only database copy; do not edit
|
||||
rows or run recovery scripts concurrently with a daemon against the same wallet.
|
||||
Never share the mnemonic, output secrets, or full wallet database in a PR.
|
||||
|
||||
A future fresh-output path must preserve original outputs for uncertain issuance
|
||||
and NUT-09 restore, allocate fresh deterministic counters safely, and coordinate
|
||||
with coco's watcher/processor. It needs its own integration tests before handling
|
||||
real funds.
|
||||
|
||||
## Cleanup preview and force
|
||||
|
||||
`wallet cleanup --dry-run` is local-only: it reports `mintQuoteCandidates`, not
|
||||
confirmed failures. `failedMintQuotes` and `leftForRecovery` are zero because no
|
||||
mint check or cleanup transition was performed. Send/melt counts remain planned
|
||||
cleanup counts in dry-run mode.
|
||||
|
||||
`--force` deliberately bypasses mint confirmation and can strand paid sats in a
|
||||
failed operation. Prefer normal cleanup. Forced operations can be retried with
|
||||
`--op <operationId> --include-failed`, but recovery still depends on the mint
|
||||
accepting their stored outputs or restoring their signatures.
|
||||
|
||||
## Integration and release notes
|
||||
|
||||
The reopen helper uses private coco-core 1.0.1 methods. Retain real-Manager and
|
||||
HTTP fake-mint coverage, use frozen dependency installs, and re-run integration
|
||||
tests on coco upgrades. A controlled low-value live-mint smoke test remains
|
||||
recommended before release.
|
||||
|
||||
PR #118 removes `cocod-client.ts`. When integrating that change, move recovery
|
||||
and cleanup contracts into its replacement `wallet-client.ts`, rename HTTP error
|
||||
references accordingly, and make recovery mandatory for the in-process client.
|
||||
This follow-up does not pull in #118's unrelated removal.
|
||||
+139
-3
@@ -2062,16 +2062,22 @@ walletCmd
|
||||
.option("--mint-url <url>", "Only clean up operations for this mint URL")
|
||||
.option(
|
||||
"--min-age <hours>",
|
||||
"Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are always failed)",
|
||||
"Minimum age for reclaiming sends/cancelling melts, in hours (default: 168, one week; expired mint quotes are checked with their mint)",
|
||||
"168",
|
||||
)
|
||||
.option("--dry-run", "Report what would be cleaned without applying changes", false)
|
||||
.option(
|
||||
"--force",
|
||||
"Fail expired mint quotes without confirming UNPAID with the mint (may strand paid quotes)",
|
||||
false,
|
||||
)
|
||||
.option("-y, --yes", "Skip confirmation prompt", false)
|
||||
.action(
|
||||
async (options: {
|
||||
mintUrl?: string;
|
||||
minAge: string;
|
||||
dryRun: boolean;
|
||||
force: boolean;
|
||||
yes: boolean;
|
||||
}) => {
|
||||
const minAgeHours = Number.parseFloat(options.minAge);
|
||||
@@ -2087,7 +2093,9 @@ walletCmd
|
||||
});
|
||||
const answer = await new Promise<string>((resolve) => {
|
||||
rl.question(
|
||||
"This will fail expired mint quotes, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ",
|
||||
options.force
|
||||
? "WARNING: --force fails expired mint quotes WITHOUT checking the mint and may strand paid sats. It also reclaims old pending sends and cancels prepared melts. Continue? [y/N] "
|
||||
: "This will fail expired mint quotes confirmed unpaid, reclaim old pending sends, and cancel prepared melts. Continue? [y/N] ",
|
||||
(value: string) => {
|
||||
rl.close();
|
||||
resolve(value.trim().toLowerCase());
|
||||
@@ -2109,6 +2117,7 @@ walletCmd
|
||||
mintUrl: options.mintUrl,
|
||||
minAgeMs: Math.round(minAgeHours * 60 * 60 * 1000),
|
||||
dryRun: options.dryRun === true,
|
||||
force: options.force === true,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -2121,6 +2130,8 @@ walletCmd
|
||||
| {
|
||||
dryRun?: boolean;
|
||||
failedMintQuotes?: number;
|
||||
mintQuoteCandidates?: number;
|
||||
leftForRecovery?: number;
|
||||
reclaimedSends?: number;
|
||||
cancelledMelts?: number;
|
||||
skipped?: number;
|
||||
@@ -2131,8 +2142,15 @@ walletCmd
|
||||
if (output) {
|
||||
const prefix = output.dryRun ? "Would clean up:" : "Cleaned up:";
|
||||
console.log(prefix);
|
||||
if (output.dryRun) {
|
||||
console.log(
|
||||
` Expired mint quote candidates (not checked with mint): ${output.mintQuoteCandidates ?? 0}`,
|
||||
);
|
||||
} else {
|
||||
console.log(` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`);
|
||||
}
|
||||
console.log(
|
||||
` Expired mint quotes failed: ${output.failedMintQuotes ?? 0}`,
|
||||
` Expired quotes kept for recovery (paid/issued/unverified): ${output.leftForRecovery ?? 0}`,
|
||||
);
|
||||
console.log(` Pending sends reclaimed: ${output.reclaimedSends ?? 0}`);
|
||||
console.log(
|
||||
@@ -2163,6 +2181,124 @@ walletCmd
|
||||
},
|
||||
);
|
||||
|
||||
walletCmd
|
||||
.command("recover")
|
||||
.description(
|
||||
"Retry mint quotes using their stored outputs (does not replace rejected outputs)",
|
||||
)
|
||||
.option(
|
||||
"--op <id>",
|
||||
"Recover this operation id (repeatable; find IDs with routstrd history --json)",
|
||||
(value: string, previous: string[]) => [...previous, value],
|
||||
[] as string[],
|
||||
)
|
||||
.option(
|
||||
"--include-failed",
|
||||
"Also re-open operations coco already gave up on (requires --op)",
|
||||
false,
|
||||
)
|
||||
.option("-y, --yes", "Skip confirmation prompt", false)
|
||||
.action(
|
||||
async (options: {
|
||||
op: string[];
|
||||
includeFailed: boolean;
|
||||
yes: boolean;
|
||||
}) => {
|
||||
const operationIds = options.op ?? [];
|
||||
if (options.includeFailed && operationIds.length === 0) {
|
||||
console.error(
|
||||
"--include-failed can only target operations named with --op",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!options.yes) {
|
||||
const rl = require("readline").createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
const prompt =
|
||||
operationIds.length > 0
|
||||
? `Recover ${operationIds.length} mint quote operation(s)? [y/N] `
|
||||
: "Check every pending mint quote with its mint and claim any paid sats? [y/N] ";
|
||||
const answer = await new Promise<string>((resolve) => {
|
||||
rl.question(prompt, (value: string) => {
|
||||
rl.close();
|
||||
resolve(value.trim().toLowerCase());
|
||||
});
|
||||
});
|
||||
if (answer !== "y" && answer !== "yes") {
|
||||
console.log("Aborted.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureDaemonRunning();
|
||||
|
||||
const result = await callDaemon("/wallet/recover", {
|
||||
method: "POST",
|
||||
body: {
|
||||
operationIds: operationIds.length > 0 ? operationIds : undefined,
|
||||
includeFailed: options.includeFailed === true,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
console.log(result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const output = result.output as
|
||||
| {
|
||||
checked?: number;
|
||||
recovered?: number;
|
||||
waiting?: number;
|
||||
terminal?: number;
|
||||
reopened?: number;
|
||||
retryable?: number;
|
||||
busy?: number;
|
||||
errors?: Array<{ operationId: string; error: string }>;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
if (output) {
|
||||
console.log("Mint quote recovery:");
|
||||
console.log(` Checked with mint: ${output.checked ?? 0}`);
|
||||
console.log(
|
||||
` Recovered (paid sats claimed): ${output.recovered ?? 0}`,
|
||||
);
|
||||
console.log(` Still unpaid: ${output.waiting ?? 0}`);
|
||||
console.log(` No longer issuable: ${output.terminal ?? 0}`);
|
||||
console.log(
|
||||
` Re-opened failed operations: ${output.reopened ?? 0}`,
|
||||
);
|
||||
console.log(` Left for a later run: ${output.retryable ?? 0}`);
|
||||
console.log(
|
||||
` Skipped (recovery still running): ${output.busy ?? 0}`,
|
||||
);
|
||||
if (output.errors && output.errors.length > 0) {
|
||||
console.log("\nErrors:");
|
||||
for (const e of output.errors) {
|
||||
console.log(` - ${e.operationId}: ${e.error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const message = (error as Error).message;
|
||||
if (
|
||||
message?.includes("fetch failed") ||
|
||||
message?.includes("Connection refused")
|
||||
) {
|
||||
console.error("Daemon is not running");
|
||||
process.exit(1);
|
||||
}
|
||||
console.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const walletReceiveCmd = walletCmd
|
||||
.command("receive")
|
||||
.description("Wallet receive operations");
|
||||
|
||||
@@ -283,6 +283,21 @@ function optionalStringField(
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function optionalStringArrayField(
|
||||
body: Record<string, unknown>,
|
||||
field: string,
|
||||
): string[] | undefined {
|
||||
const value = body[field];
|
||||
if (value === undefined) return undefined;
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.some((item) => typeof item !== "string" || !item.trim())
|
||||
) {
|
||||
throw new CocodHttpError(400, `'${field}' must be an array of non-empty strings.`);
|
||||
}
|
||||
return value.map((item: string) => item.trim());
|
||||
}
|
||||
|
||||
function getCurrentMode(deps: DaemonDeps): ClientMode {
|
||||
const stateMode = deps.store.getState()?.mode;
|
||||
return stateMode || deps.mode || "apikeys";
|
||||
@@ -513,6 +528,44 @@ export function createDaemonRequestHandler(deps: {
|
||||
? body.minAgeMs
|
||||
: undefined,
|
||||
dryRun: body.dryRun === true,
|
||||
force: body.force === true,
|
||||
});
|
||||
return { output: result };
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && url.pathname === "/wallet/recover") {
|
||||
await respond(res, async () => {
|
||||
if (!deps.walletClient.recoverMintQuotes) {
|
||||
throw new CocodHttpError(
|
||||
501,
|
||||
"Mint quote recovery is not supported by this wallet client.",
|
||||
);
|
||||
}
|
||||
|
||||
const body = await readJsonBody(req);
|
||||
const operationIds = optionalStringArrayField(body, "operationIds");
|
||||
if (body.includeFailed === true && !operationIds?.length) {
|
||||
throw new CocodHttpError(
|
||||
400,
|
||||
"'includeFailed' requires non-empty 'operationIds'.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
body.timeoutMs !== undefined &&
|
||||
(typeof body.timeoutMs !== "number" ||
|
||||
!Number.isFinite(body.timeoutMs) || body.timeoutMs <= 0)
|
||||
) {
|
||||
throw new CocodHttpError(
|
||||
400,
|
||||
"'timeoutMs' must be a positive finite number.",
|
||||
);
|
||||
}
|
||||
const result = await deps.walletClient.recoverMintQuotes({
|
||||
operationIds,
|
||||
includeFailed: body.includeFailed === true,
|
||||
timeoutMs: body.timeoutMs as number | undefined,
|
||||
});
|
||||
return { output: result };
|
||||
});
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it, mock } from "bun:test";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { createDaemonRequestHandler } from "./index";
|
||||
|
||||
async function recover(body: unknown) {
|
||||
const recoverMintQuotes = mock(async (_options: unknown) => ({ recovered: 0 }));
|
||||
const handler = createDaemonRequestHandler({ walletClient: { recoverMintQuotes } } as never);
|
||||
const req = new EventEmitter() as any;
|
||||
Object.assign(req, { method: "POST", url: "/wallet/recover", headers: { host: "localhost" } });
|
||||
const res = {
|
||||
status: 0, body: "",
|
||||
writeHead(status: number) { this.status = status; },
|
||||
end(chunk: string) { this.body = chunk; },
|
||||
};
|
||||
setImmediate(() => {
|
||||
req.emit("data", Buffer.from(JSON.stringify(body)));
|
||||
req.emit("end");
|
||||
});
|
||||
await handler(req, res as never);
|
||||
return { res, recoverMintQuotes };
|
||||
}
|
||||
|
||||
describe("POST /wallet/recover validation", () => {
|
||||
it.each([{}, { operationIds: [] }])("rejects includeFailed without explicit IDs: %j", async (body) => {
|
||||
const { res, recoverMintQuotes } = await recover({ ...body, includeFailed: true });
|
||||
expect(res.status).toBe(400);
|
||||
expect(recoverMintQuotes).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([[""], [" "], [42]])("rejects invalid operation IDs: %j", async (operationIds) => {
|
||||
const { res, recoverMintQuotes } = await recover({ operationIds });
|
||||
expect(res.status).toBe(400);
|
||||
expect(recoverMintQuotes).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([0, -1, "1000"])("rejects invalid timeout %j", async (timeoutMs) => {
|
||||
const { res, recoverMintQuotes } = await recover({ timeoutMs });
|
||||
expect(res.status).toBe(400);
|
||||
expect(recoverMintQuotes).not.toHaveBeenCalled();
|
||||
});
|
||||
it("passes normalized explicit IDs and a positive timeout", async () => {
|
||||
const { res, recoverMintQuotes } = await recover({ operationIds: [" op-1 "], includeFailed: true, timeoutMs: 1000 });
|
||||
expect(res.status).toBe(200);
|
||||
expect(recoverMintQuotes).toHaveBeenCalledWith({ operationIds: ["op-1"], includeFailed: true, timeoutMs: 1000 });
|
||||
});
|
||||
it("still permits checking pending quotes without IDs", async () => {
|
||||
const { res, recoverMintQuotes } = await recover({});
|
||||
expect(res.status).toBe(200);
|
||||
expect(recoverMintQuotes).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { selectCleanupOperations } from "./cleanup";
|
||||
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
|
||||
|
||||
const NOW_MS = 1_800_000_000_000;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
@@ -166,3 +166,14 @@ describe("selectCleanupOperations", () => {
|
||||
expect(result.meltsToCancel).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mint cleanup reporting", () => {
|
||||
it("reports dry-run candidates, not confirmed failures", () => {
|
||||
expect(summarizeMintCleanup({ dryRun: true, candidates: 3, failed: 0, leftForRecovery: 0 }))
|
||||
.toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 0, leftForRecovery: 0 });
|
||||
});
|
||||
it("reports only actual failures in a real run", () => {
|
||||
expect(summarizeMintCleanup({ dryRun: false, candidates: 3, failed: 1, leftForRecovery: 2 }))
|
||||
.toEqual({ mintQuoteCandidates: 3, failedMintQuotes: 1, leftForRecovery: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,8 +64,8 @@ export interface CleanupSelection<
|
||||
* can have happened before expiry while the daemon was down, leaving no
|
||||
* local observation. Callers that fail quotes automatically at startup must
|
||||
* therefore confirm UNPAID with the mint first (see
|
||||
* settleExpiredMintQuotes in coco-client.ts); only the explicit,
|
||||
* user-invoked cleanup command may fail candidates purely locally.
|
||||
* failExpiredMintQuoteIfUnpaid in coco-client.ts). Explicit cleanup follows
|
||||
* the same rule unless the operator opts into unsafe `--force` behaviour.
|
||||
* - Pending sends are reclaimed (rolled back) only when they are older than
|
||||
* `minAgeMs`, so we never roll back a token that a receiver might still
|
||||
* legitimately claim.
|
||||
@@ -102,3 +102,17 @@ export function selectCleanupOperations<
|
||||
|
||||
return { mintsToFail, sendsToReclaim, meltsToCancel };
|
||||
}
|
||||
|
||||
/** Keep a local-only dry-run preview distinct from mint-confirmed outcomes. */
|
||||
export function summarizeMintCleanup(input: {
|
||||
dryRun: boolean;
|
||||
candidates: number;
|
||||
failed: number;
|
||||
leftForRecovery: number;
|
||||
}) {
|
||||
return {
|
||||
mintQuoteCandidates: input.candidates,
|
||||
failedMintQuotes: input.dryRun ? 0 : input.failed,
|
||||
leftForRecovery: input.dryRun ? 0 : input.leftForRecovery,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,12 +14,17 @@ import {
|
||||
assertLegacyCocodNotRunning,
|
||||
claimLegacyCocodPidFile,
|
||||
createCocoClient,
|
||||
createRunQueue,
|
||||
DEFAULT_TRUSTED_MINT_URLS,
|
||||
failExpiredMintQuoteIfUnpaid,
|
||||
isZombieProcess,
|
||||
reopenFailedMintOperation,
|
||||
runMintQuoteRecovery,
|
||||
settleExpiredMintQuotes,
|
||||
settlePendingMintQuotes,
|
||||
stopLegacyCocod,
|
||||
type ExpiredMintQuoteSource,
|
||||
type MintQuoteRecoverySource,
|
||||
type PendingMintQuoteSource,
|
||||
type PendingMintSweepState,
|
||||
} from "./coco-client";
|
||||
@@ -901,3 +906,731 @@ describe("settlePendingMintQuotes", () => {
|
||||
expect(logged.mock.calls[0]?.[0]).toContain("21 sat minted");
|
||||
});
|
||||
});
|
||||
|
||||
describe("createRunQueue", () => {
|
||||
it("runs tasks strictly one after another", async () => {
|
||||
const enqueue = createRunQueue();
|
||||
const order: string[] = [];
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
const task = (name: string, delay: number) => async () => {
|
||||
active++;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
order.push(`${name}:start`);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
order.push(`${name}:end`);
|
||||
active--;
|
||||
return name;
|
||||
};
|
||||
|
||||
const results = await Promise.all([
|
||||
enqueue(task("a", 20)),
|
||||
enqueue(task("b", 1)),
|
||||
enqueue(task("c", 1)),
|
||||
]);
|
||||
|
||||
expect(results).toEqual(["a", "b", "c"]);
|
||||
expect(maxActive).toBe(1);
|
||||
expect(order).toEqual([
|
||||
"a:start",
|
||||
"a:end",
|
||||
"b:start",
|
||||
"b:end",
|
||||
"c:start",
|
||||
"c:end",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps the chain alive after a rejected task", async () => {
|
||||
const enqueue = createRunQueue();
|
||||
|
||||
const failed = enqueue(async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
const next = enqueue(async () => "ok");
|
||||
|
||||
await expect(failed).rejects.toThrow("boom");
|
||||
expect(await next).toBe("ok");
|
||||
});
|
||||
});
|
||||
|
||||
describe("failExpiredMintQuoteIfUnpaid", () => {
|
||||
function fakeMintService(observe: (id: string) => Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>) {
|
||||
const failPendingOperation = mock(
|
||||
async (
|
||||
_op: { id: string },
|
||||
_failure: { reason: string; retryable?: boolean; observedAt: number },
|
||||
) => ({}),
|
||||
);
|
||||
return {
|
||||
mintService: {
|
||||
observePendingOperation: mock(observe),
|
||||
failPendingOperation,
|
||||
},
|
||||
failPendingOperation,
|
||||
};
|
||||
}
|
||||
|
||||
it("fails a quote its mint confirms unpaid", async () => {
|
||||
const { mintService, failPendingOperation } = fakeMintService(async () => ({
|
||||
category: "waiting",
|
||||
}));
|
||||
|
||||
const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000);
|
||||
|
||||
expect(result.outcome).toBe("failed");
|
||||
expect(failPendingOperation).toHaveBeenCalledTimes(1);
|
||||
expect(failPendingOperation.mock.calls[0]?.[1]?.reason).toContain(
|
||||
"confirmed unpaid by mint",
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["ready", "completed", "terminal"] as const)(
|
||||
"leaves a quote observed as %s for recovery",
|
||||
async (category) => {
|
||||
const { mintService, failPendingOperation } = fakeMintService(
|
||||
async () => ({ category }),
|
||||
);
|
||||
|
||||
const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000);
|
||||
|
||||
expect(result).toEqual({ outcome: "leftForRecovery", category });
|
||||
expect(failPendingOperation).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("leaves a quote pending when the mint cannot be reached", async () => {
|
||||
const { mintService, failPendingOperation } = fakeMintService(async () => {
|
||||
throw new Error("Network request failed");
|
||||
});
|
||||
|
||||
const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 1000);
|
||||
|
||||
expect(result.outcome).toBe("unobserved");
|
||||
expect(failPendingOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("gives up waiting on a hung mint without failing the quote", async () => {
|
||||
const { mintService, failPendingOperation } = fakeMintService(
|
||||
() => new Promise(() => {}),
|
||||
);
|
||||
|
||||
const result = await failExpiredMintQuoteIfUnpaid(mintService, "op-1", 20);
|
||||
|
||||
expect(result.outcome).toBe("unobserved");
|
||||
expect(failPendingOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("reopenFailedMintOperation", () => {
|
||||
/**
|
||||
* Mirrors coco's OperationIdLock, which is fail-fast: acquiring an id that is
|
||||
* already locked throws OperationInProgressError instead of waiting.
|
||||
*/
|
||||
function makeLock() {
|
||||
let held = false;
|
||||
return {
|
||||
get held() {
|
||||
return held;
|
||||
},
|
||||
async acquire() {
|
||||
if (held) {
|
||||
const error = new Error("Operation op-1 is already in progress");
|
||||
error.name = "OperationInProgressError";
|
||||
throw error;
|
||||
}
|
||||
held = true;
|
||||
return () => {
|
||||
held = false;
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeService(
|
||||
current: Record<string, unknown> | null,
|
||||
hooks: {
|
||||
lock?: ReturnType<typeof makeLock>;
|
||||
onWrite?: (lock: ReturnType<typeof makeLock>) => void;
|
||||
} = {},
|
||||
) {
|
||||
const lock = hooks.lock ?? makeLock();
|
||||
const transitionToPending = mock(
|
||||
async (_op: Record<string, unknown>, _error?: string) => {
|
||||
hooks.onWrite?.(lock);
|
||||
return {};
|
||||
},
|
||||
);
|
||||
return {
|
||||
service: {
|
||||
acquireOperationLock: mock(async (_id: string) => lock.acquire()),
|
||||
getOperation: mock(async (_id: string) => current),
|
||||
transitionToPending,
|
||||
},
|
||||
transitionToPending,
|
||||
lock,
|
||||
};
|
||||
}
|
||||
|
||||
it("re-opens a failed operation with the full persisted row", async () => {
|
||||
// coco spreads whatever it is handed and the sqlite repository rewrites
|
||||
// every column, so a partial object would erase the stored outputs.
|
||||
const row = {
|
||||
id: "op-1",
|
||||
state: "failed",
|
||||
mintUrl: "https://mint.example.com",
|
||||
quoteId: "quote-1",
|
||||
method: "bolt11",
|
||||
amount: 210_000,
|
||||
unit: "sat",
|
||||
request: "lnbc...",
|
||||
expiry: 1_800_000_000,
|
||||
outputDataJson: "[{\"secret\":\"abc\"}]",
|
||||
terminalFailure: { reason: "expired" },
|
||||
};
|
||||
const { service, transitionToPending } = fakeService(row);
|
||||
|
||||
const reopened = await reopenFailedMintOperation(service, "op-1");
|
||||
|
||||
expect(reopened).toBe(true);
|
||||
expect(transitionToPending).toHaveBeenCalledTimes(1);
|
||||
const passed = transitionToPending.mock.calls[0]?.[0];
|
||||
expect(passed).toMatchObject({
|
||||
id: "op-1",
|
||||
quoteId: "quote-1",
|
||||
amount: 210_000,
|
||||
unit: "sat",
|
||||
outputDataJson: "[{\"secret\":\"abc\"}]",
|
||||
});
|
||||
// The stale terminal marker must not survive the re-open.
|
||||
expect(passed?.terminalFailure).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does nothing when the operation is no longer failed", async () => {
|
||||
const { service, transitionToPending, lock } = fakeService({
|
||||
id: "op-1",
|
||||
state: "finalized",
|
||||
});
|
||||
|
||||
expect(await reopenFailedMintOperation(service, "op-1")).toBe(false);
|
||||
expect(transitionToPending).not.toHaveBeenCalled();
|
||||
// The lock must be released even on the no-op path.
|
||||
expect(lock.held).toBe(false);
|
||||
});
|
||||
|
||||
it("throws when the operation is missing", async () => {
|
||||
const { service, lock } = fakeService(null);
|
||||
|
||||
await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow(
|
||||
"not found",
|
||||
);
|
||||
expect(lock.held).toBe(false);
|
||||
});
|
||||
|
||||
it("fails closed when coco no longer exposes the operation lock", async () => {
|
||||
const transitionToPending = mock(
|
||||
async (_op: Record<string, unknown>, _error?: string) => ({}),
|
||||
);
|
||||
const service = {
|
||||
getOperation: mock(async () => ({ id: "op-1", state: "failed" })),
|
||||
transitionToPending,
|
||||
} as unknown as Parameters<typeof reopenFailedMintOperation>[0];
|
||||
|
||||
await expect(
|
||||
reopenFailedMintOperation(service, "op-1"),
|
||||
).rejects.toThrow("acquireOperationLock");
|
||||
expect(transitionToPending).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("holds the operation lock across read-check-write", async () => {
|
||||
const lock = makeLock();
|
||||
const order: string[] = [];
|
||||
const service = {
|
||||
acquireOperationLock: mock(async (_id: string) => {
|
||||
order.push("lock");
|
||||
const release = await lock.acquire();
|
||||
return () => {
|
||||
order.push("unlock");
|
||||
release();
|
||||
};
|
||||
}),
|
||||
getOperation: mock(async (_id: string) => {
|
||||
expect(lock.held).toBe(true);
|
||||
order.push("read");
|
||||
return { id: "op-1", state: "failed", quoteId: "quote-1" };
|
||||
}),
|
||||
transitionToPending: mock(async () => {
|
||||
expect(lock.held).toBe(true);
|
||||
order.push("write");
|
||||
return {};
|
||||
}),
|
||||
};
|
||||
|
||||
const reopened = await reopenFailedMintOperation(service, "op-1");
|
||||
|
||||
expect(reopened).toBe(true);
|
||||
expect(order).toEqual(["lock", "read", "write", "unlock"]);
|
||||
expect(lock.held).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses to re-open while the operation lock is held elsewhere", async () => {
|
||||
const lock = makeLock();
|
||||
const release = await lock.acquire();
|
||||
const { service, transitionToPending } = fakeService(
|
||||
{ id: "op-1", state: "failed" },
|
||||
{ lock },
|
||||
);
|
||||
|
||||
await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow(
|
||||
/in progress/,
|
||||
);
|
||||
expect(transitionToPending).not.toHaveBeenCalled();
|
||||
release();
|
||||
});
|
||||
});
|
||||
|
||||
describe("runMintQuoteRecovery", () => {
|
||||
function mintOp(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: "op-1",
|
||||
mintUrl: "https://mint.example.com",
|
||||
quoteId: "quote-1",
|
||||
state: "pending",
|
||||
amount: 210_000,
|
||||
expiry: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeSource(
|
||||
ops: Array<Record<string, unknown>>,
|
||||
behavior: {
|
||||
observe?: (id: string) => Promise<{
|
||||
category: "waiting" | "ready" | "completed" | "terminal";
|
||||
}>;
|
||||
finalize?: (id: string) => Promise<unknown>;
|
||||
reopen?: (id: string) => Promise<boolean>;
|
||||
} = {},
|
||||
) {
|
||||
const finalize = mock(
|
||||
behavior.finalize ??
|
||||
(async (_id: string) => ({ state: "finalized" })),
|
||||
);
|
||||
const observePendingOperation = mock(
|
||||
behavior.observe ?? (async () => ({ category: "waiting" as const })),
|
||||
);
|
||||
const reopenFailedOperation = mock(
|
||||
behavior.reopen ?? (async (_id: string) => true),
|
||||
);
|
||||
const byId = new Map(ops.map((op) => [op.id as string, op]));
|
||||
const source = {
|
||||
ops: {
|
||||
mint: {
|
||||
listPending: async () =>
|
||||
ops.filter(
|
||||
(op) => op.state === "pending" || op.state === "executing",
|
||||
),
|
||||
get: async (id: string) => byId.get(id) ?? null,
|
||||
finalize,
|
||||
},
|
||||
},
|
||||
mintOperationService: { observePendingOperation },
|
||||
reopenFailedOperation,
|
||||
} as unknown as MintQuoteRecoverySource;
|
||||
return { source, finalize, observePendingOperation, reopenFailedOperation };
|
||||
}
|
||||
|
||||
it("mints the stored outputs for a quote the mint reports PAID", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ checked: 1, recovered: 1, waiting: 0 });
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
expect(finalize.mock.calls[0]?.[0]).toBe("op-1");
|
||||
});
|
||||
|
||||
it("restores proofs for a quote already issued at the mint", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "completed" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ recovered: 1 });
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("leaves an unpaid quote pending", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "waiting" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ checked: 1, recovered: 0, waiting: 1 });
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports a quote the mint can no longer issue", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "terminal" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ terminal: 1, recovered: 0 });
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries later when the mint is unreachable", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => {
|
||||
throw new Error("fetch failed");
|
||||
},
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not count a failed finalize as a recovery", async () => {
|
||||
// coco returns a terminal operation instead of throwing when the mint
|
||||
// refuses, so a fulfilled finalize is not evidence that sats were claimed.
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: async () => ({
|
||||
state: "failed",
|
||||
error: "Recovered: quote quote-1 expired while executing mint",
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 1 });
|
||||
expect(result.errors[0]?.error).toContain("expired");
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not count a finalized-with-error operation as a recovery", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "completed" }),
|
||||
finalize: async () => ({
|
||||
state: "finalized",
|
||||
error: "Recovered issued quote quote-1 but no proofs could be restored",
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 1 });
|
||||
expect(result.errors).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("falls back to the original failure when diagnostic lookup fails", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: async () => { throw new Error("original failure"); },
|
||||
});
|
||||
source.ops.mint.get = async () => { throw new Error("lookup failed"); };
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
expect(result).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]);
|
||||
});
|
||||
|
||||
it("bounds a hung diagnostic lookup after finalize fails", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: async () => { throw new Error("original failure"); },
|
||||
});
|
||||
source.ops.mint.get = () => new Promise(() => {});
|
||||
const result = await runMintQuoteRecovery(source, { timeoutMs: 20 });
|
||||
expect(result).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(result.errors).toEqual([{ operationId: "op-1", error: "original failure" }]);
|
||||
});
|
||||
|
||||
it("surfaces the persisted mint error even when the mint budget is nearly spent", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: async () => {
|
||||
// Consume almost the whole per-op mint budget before throwing: exactly
|
||||
// the slow-mint case where a diagnostic bound to remaining() would
|
||||
// starve and silently fall back to the generic message.
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
throw new Error("remains pending");
|
||||
},
|
||||
});
|
||||
source.ops.mint.get = async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
return {
|
||||
...mintOp(),
|
||||
state: "pending",
|
||||
error: "keyset id inactive.",
|
||||
};
|
||||
};
|
||||
const result = await runMintQuoteRecovery(source, { timeoutMs: 30 });
|
||||
expect(result).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(result.errors).toEqual([
|
||||
{ operationId: "op-1", error: "keyset id inactive." },
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds finalize so one hung mint cannot block recovery", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: () => new Promise(() => {}),
|
||||
});
|
||||
|
||||
const started = Date.now();
|
||||
const result = await runMintQuoteRecovery(source, { timeoutMs: 20 });
|
||||
|
||||
expect(Date.now() - started).toBeLessThan(5_000);
|
||||
expect(result).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
});
|
||||
|
||||
it("recovers an interrupted mint without re-checking the quote", async () => {
|
||||
const { source, finalize, observePendingOperation } = fakeSource([
|
||||
mintOp({ state: "executing" }),
|
||||
]);
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ recovered: 1 });
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
expect(observePendingOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips failed operations unless the caller opts in", async () => {
|
||||
const { source, reopenFailedOperation } = fakeSource([
|
||||
mintOp({ state: "failed", lastObservedRemoteState: "PAID" }),
|
||||
]);
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ checked: 0, recovered: 0, reopened: 0 });
|
||||
expect(reopenFailedOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("requires explicit IDs when including failed operations", async () => {
|
||||
const { source, reopenFailedOperation, observePendingOperation } = fakeSource([]);
|
||||
await expect(runMintQuoteRecovery(source, { includeFailed: true })).rejects.toThrow(
|
||||
"includeFailed requires explicit operationIds",
|
||||
);
|
||||
await expect(runMintQuoteRecovery(source, { includeFailed: true, operationIds: [] })).rejects.toThrow(
|
||||
"includeFailed requires explicit operationIds",
|
||||
);
|
||||
expect(reopenFailedOperation).not.toHaveBeenCalled();
|
||||
expect(observePendingOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([0, -1, NaN, Infinity])("rejects invalid recovery timeout %s", async (timeoutMs) => {
|
||||
const { source, observePendingOperation } = fakeSource([]);
|
||||
await expect(runMintQuoteRecovery(source, { timeoutMs })).rejects.toThrow(
|
||||
"timeoutMs must be a positive finite number",
|
||||
);
|
||||
expect(observePendingOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("re-opens a named failed operation, then mints it", async () => {
|
||||
const { source, reopenFailedOperation, finalize } = fakeSource(
|
||||
[mintOp({ state: "failed", lastObservedRemoteState: "PAID" })],
|
||||
{ observe: async () => ({ category: "ready" }) },
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
includeFailed: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ reopened: 1, recovered: 1 });
|
||||
expect(reopenFailedOperation).toHaveBeenCalledWith("op-1");
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("re-opens a named failed operation even without a PAID observation", async () => {
|
||||
// The old local-fail bug left quotes with a stale or missing observation,
|
||||
// which is exactly when an operator needs to retry them.
|
||||
const { source, reopenFailedOperation } = fakeSource(
|
||||
[mintOp({ state: "failed" })],
|
||||
{ observe: async () => ({ category: "ready" }) },
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
includeFailed: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ reopened: 1, recovered: 1 });
|
||||
expect(reopenFailedOperation).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips an operation that is no longer failed when re-opened", async () => {
|
||||
const { source, finalize } = fakeSource(
|
||||
[mintOp({ state: "failed" })],
|
||||
{ reopen: async () => false },
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
includeFailed: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ reopened: 0, checked: 0, recovered: 0 });
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports an unknown operation id instead of throwing", async () => {
|
||||
const { source } = fakeSource([]);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["missing"],
|
||||
});
|
||||
|
||||
expect(result.checked).toBe(0);
|
||||
expect(result.errors).toEqual([
|
||||
{ operationId: "missing", error: "operation not found" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("deduplicates repeated operation ids", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1", "op-1"],
|
||||
});
|
||||
|
||||
expect(result.checked).toBe(1);
|
||||
expect(finalize).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("ignores finalized operations even when targeted", async () => {
|
||||
const { source, finalize } = fakeSource([mintOp({ state: "finalized" })]);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
});
|
||||
|
||||
expect(result.checked).toBe(0);
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves a non-terminal finalize result for a later run, not terminal", async () => {
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: async () => ({ state: "pending" }),
|
||||
});
|
||||
|
||||
const result = await runMintQuoteRecovery(source);
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 });
|
||||
expect(result.errors[0]?.error).toContain("will retry");
|
||||
});
|
||||
|
||||
it("skips operations whose earlier recovery is still in flight", async () => {
|
||||
const outstanding = new Map<string, Promise<unknown>>([
|
||||
["op-1", new Promise(() => {})],
|
||||
]);
|
||||
const { source, finalize, observePendingOperation } = fakeSource(
|
||||
[mintOp()],
|
||||
{ observe: async () => ({ category: "ready" }) },
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, { outstanding });
|
||||
|
||||
expect(result).toMatchObject({ busy: 1, checked: 0, recovered: 0 });
|
||||
expect(observePendingOperation).not.toHaveBeenCalled();
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a timed-out finalize registered so a retry waits", async () => {
|
||||
const outstanding = new Map<string, Promise<unknown>>();
|
||||
const { source } = fakeSource([mintOp()], {
|
||||
observe: async () => ({ category: "ready" }),
|
||||
finalize: () => new Promise(() => {}),
|
||||
});
|
||||
|
||||
const first = await runMintQuoteRecovery(source, {
|
||||
timeoutMs: 20,
|
||||
outstanding,
|
||||
});
|
||||
const second = await runMintQuoteRecovery(source, {
|
||||
timeoutMs: 20,
|
||||
outstanding,
|
||||
});
|
||||
|
||||
expect(first).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(outstanding.has("op-1")).toBe(true);
|
||||
// The abandoned mint request must not be retried underneath.
|
||||
expect(second).toMatchObject({ busy: 1, checked: 0 });
|
||||
});
|
||||
|
||||
it("does not re-open a failed operation whose recovery is in flight", async () => {
|
||||
const outstanding = new Map<string, Promise<unknown>>([
|
||||
["op-1", new Promise(() => {})],
|
||||
]);
|
||||
const { source, reopenFailedOperation } = fakeSource(
|
||||
[mintOp({ state: "failed" })],
|
||||
{},
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
includeFailed: true,
|
||||
outstanding,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ busy: 1, reopened: 0 });
|
||||
expect(reopenFailedOperation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts an in-progress operation as busy rather than retryable", async () => {
|
||||
const { source } = fakeSource(
|
||||
[mintOp({ state: "failed" })],
|
||||
{
|
||||
reopen: async () => {
|
||||
const error = new Error("Operation op-1 is already in progress");
|
||||
error.name = "OperationInProgressError";
|
||||
throw error;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const result = await runMintQuoteRecovery(source, {
|
||||
operationIds: ["op-1"],
|
||||
includeFailed: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ busy: 1, retryable: 0, reopened: 0 });
|
||||
});
|
||||
|
||||
it("keeps a timed-out quote check registered so a retry waits", async () => {
|
||||
// observePendingOperation is not read-only, so a hung check must not be
|
||||
// retried underneath: it could persist a stale observation later.
|
||||
const outstanding = new Map<string, Promise<unknown>>();
|
||||
const { source, finalize } = fakeSource([mintOp()], {
|
||||
observe: () => new Promise(() => {}),
|
||||
});
|
||||
|
||||
const first = await runMintQuoteRecovery(source, {
|
||||
timeoutMs: 20,
|
||||
outstanding,
|
||||
});
|
||||
const second = await runMintQuoteRecovery(source, {
|
||||
timeoutMs: 20,
|
||||
outstanding,
|
||||
});
|
||||
|
||||
expect(first).toMatchObject({ retryable: 1, checked: 1 });
|
||||
expect(outstanding.has("op-1")).toBe(true);
|
||||
expect(second).toMatchObject({ busy: 1, checked: 0 });
|
||||
expect(finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -36,7 +36,12 @@ import type {
|
||||
WalletCleanupResult,
|
||||
WalletRecoveryProgress,
|
||||
} from "./cocod-client";
|
||||
import { selectCleanupOperations } from "./cleanup";
|
||||
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
|
||||
import {
|
||||
classifyMintQuoteObservation,
|
||||
selectMintQuotesForRecovery,
|
||||
type MintQuoteRecoveryCandidate,
|
||||
} from "./mint-quote-recovery";
|
||||
import {
|
||||
clearInterruptedReceiveReservations,
|
||||
deleteReceiveTokenReservation,
|
||||
@@ -579,6 +584,96 @@ interface MintOperationServiceCleanup {
|
||||
observePendingOperation(
|
||||
operationId: string,
|
||||
): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>;
|
||||
/**
|
||||
* Acquire coco's per-operation lock for `operationId` and return its release
|
||||
* function. coco's execute/finalize/recover paths take the same lock, so
|
||||
* holding it across a read-check-write makes the transition atomic with
|
||||
* respect to them.
|
||||
*/
|
||||
acquireOperationLock(operationId: string): Promise<() => void>;
|
||||
/** Reload a mint operation row, or null when it no longer exists. */
|
||||
getOperation(operationId: string): Promise<Record<string, unknown> | null>;
|
||||
/**
|
||||
* Put a terminally failed operation back into `pending`.
|
||||
*
|
||||
* coco keeps this private, and it spreads whatever it is handed into the row
|
||||
* it writes. The sqlite repository rewrites every column, so callers MUST
|
||||
* pass a freshly reloaded full row: a partial object such as `{ id }` would
|
||||
* erase `outputDataJson` and make the paid sats unrecoverable.
|
||||
*/
|
||||
transitionToPending(
|
||||
op: Record<string, unknown>,
|
||||
error?: string,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-open a terminally failed mint operation so recovery can retry it.
|
||||
*
|
||||
* Two details make this safe:
|
||||
*
|
||||
* - The persisted row is reloaded and handed to coco in full. coco spreads
|
||||
* whatever it is given and the sqlite repository rewrites every column, so a
|
||||
* partial object would be rejected by the NOT NULL schema or, on a more
|
||||
* permissive adapter, erase the stored outputs.
|
||||
* - The read-check-write runs under coco's per-operation lock, the same lock
|
||||
* coco's execute/finalize/recover paths take. Reloading alone only narrows
|
||||
* the race: without the lock two concurrent recoveries could both see
|
||||
* `failed` and the slower one would clobber a newer state.
|
||||
*
|
||||
* The lock is fail-fast rather than wait-based: coco's `OperationIdLock.acquire`
|
||||
* throws `OperationInProgressError` when the id is already locked. So either
|
||||
* this helper holds the lock - and coco's own execute/finalize/recover paths
|
||||
* cannot interleave, because acquiring would throw for them too - or it throws
|
||||
* and writes nothing. It never waits, and never writes without the lock, which
|
||||
* is why a stale `failed` snapshot cannot clobber a newer state.
|
||||
*
|
||||
* Scope of that lock, in this coco version: `recordPendingObservation` and
|
||||
* `failPendingOperation` write without taking it. The justified claim is
|
||||
* therefore narrow - a re-open cannot clobber a concurrent executing/recovery
|
||||
* pass - not a general guarantee against every watcher write.
|
||||
*
|
||||
* This is a compatibility shim over private coco internals, so it fails closed:
|
||||
* if any of the expected methods are missing it throws before writing. That
|
||||
* check only catches removals, not changed behaviour under the same name: it
|
||||
* was written against @cashu/coco-core 1.0.1, so any coco bump must re-run the
|
||||
* real-Manager and fake-mint integration tests. The long-term fix is an
|
||||
* upstream public `reopenFailedOperation(id)` that takes the same lock, reloads
|
||||
* the full row, preserves the outputs and emits the usual events.
|
||||
*/
|
||||
export async function reopenFailedMintOperation(
|
||||
service: Pick<
|
||||
MintOperationServiceCleanup,
|
||||
"acquireOperationLock" | "getOperation" | "transitionToPending"
|
||||
>,
|
||||
operationId: string,
|
||||
): Promise<boolean> {
|
||||
for (const method of [
|
||||
"acquireOperationLock",
|
||||
"getOperation",
|
||||
"transitionToPending",
|
||||
] as const) {
|
||||
if (typeof service[method] !== "function") {
|
||||
throw new Error(
|
||||
`coco mintOperationService.${method} is unavailable; refusing to re-open a failed mint operation`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const release = await service.acquireOperationLock(operationId);
|
||||
try {
|
||||
const current = await service.getOperation(operationId);
|
||||
if (!current) throw new Error(`Operation ${operationId} not found`);
|
||||
if (current.state !== "failed") return false;
|
||||
// Clearing the terminal-failure marker keeps the re-opened row from
|
||||
// looking terminally failed to readers that inspect it alongside `state`.
|
||||
await service.transitionToPending(
|
||||
{ ...current, terminalFailure: undefined },
|
||||
undefined,
|
||||
);
|
||||
return true;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateCocoClientOptions {
|
||||
@@ -671,6 +766,57 @@ export interface ExpiredMintSettlement {
|
||||
unobserved: number;
|
||||
}
|
||||
|
||||
/** Outcome of asking a mint about one expired pending quote. */
|
||||
export type ExpiredMintQuoteOutcome =
|
||||
| "failed"
|
||||
| "leftForRecovery"
|
||||
| "unobserved";
|
||||
|
||||
/**
|
||||
* Decide one expired pending quote's fate by asking the mint.
|
||||
*
|
||||
* Expiry alone does not prove the quote was never paid: the Lightning payment
|
||||
* can land just before expiry while the daemon is down, leaving no local
|
||||
* observation. A quote the mint still reports UNPAID can never be issued and
|
||||
* is safe to fail locally; anything else (PAID/ISSUED, or a mint that cannot
|
||||
* answer) stays pending so recovery can still claim the sats.
|
||||
*/
|
||||
export async function failExpiredMintQuoteIfUnpaid(
|
||||
mintService: Pick<
|
||||
MintOperationServiceCleanup,
|
||||
"observePendingOperation" | "failPendingOperation"
|
||||
>,
|
||||
operationId: string,
|
||||
timeoutMs: number,
|
||||
): Promise<{
|
||||
outcome: ExpiredMintQuoteOutcome;
|
||||
category?: "waiting" | "ready" | "completed" | "terminal";
|
||||
error?: unknown;
|
||||
}> {
|
||||
try {
|
||||
const observation = await withTimeout(
|
||||
mintService.observePendingOperation(operationId),
|
||||
timeoutMs,
|
||||
);
|
||||
if (observation.category !== "waiting") {
|
||||
return { outcome: "leftForRecovery", category: observation.category };
|
||||
}
|
||||
// The mint confirms the expired quote is still unpaid: it can never be
|
||||
// issued now, so failing it locally cannot strand funds.
|
||||
await mintService.failPendingOperation(
|
||||
{ id: operationId },
|
||||
{
|
||||
reason: "Expired mint quote confirmed unpaid by mint",
|
||||
retryable: false,
|
||||
observedAt: Date.now(),
|
||||
},
|
||||
);
|
||||
return { outcome: "failed", category: observation.category };
|
||||
} catch (error) {
|
||||
return { outcome: "unobserved", error };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Settle expired pending mint quotes before the mint recovery sweep runs.
|
||||
*
|
||||
@@ -726,45 +872,37 @@ export async function settleExpiredMintQuotes(
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await withTimeout(
|
||||
source.mintOperationService.observePendingOperation(op.id),
|
||||
remainingMs,
|
||||
const check = await failExpiredMintQuoteIfUnpaid(
|
||||
source.mintOperationService,
|
||||
op.id,
|
||||
remainingMs,
|
||||
);
|
||||
if (check.outcome === "failed") {
|
||||
settlement.failed++;
|
||||
} else if (check.outcome === "leftForRecovery") {
|
||||
// PAID/ISSUED (or terminally failed) at the mint: normal recovery
|
||||
// must see this quote so paid proofs get claimed.
|
||||
settlement.leftForRecovery++;
|
||||
const observed =
|
||||
check.category === "ready"
|
||||
? "was paid at the mint"
|
||||
: check.category === "completed"
|
||||
? "was already issued at the mint"
|
||||
: "failed terminally at the mint";
|
||||
startupProgress(
|
||||
`Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`,
|
||||
);
|
||||
if (result.category === "waiting") {
|
||||
// The mint confirms the expired quote is still unpaid: it can never
|
||||
// be issued now, so failing it locally cannot strand funds.
|
||||
await source.mintOperationService.failPendingOperation(
|
||||
{ id: op.id },
|
||||
{
|
||||
reason: "Expired mint quote confirmed unpaid by mint",
|
||||
retryable: false,
|
||||
observedAt: Date.now(),
|
||||
},
|
||||
);
|
||||
settlement.failed++;
|
||||
} else {
|
||||
// PAID/ISSUED (or terminally failed) at the mint: normal recovery
|
||||
// must see this quote so paid proofs get claimed.
|
||||
settlement.leftForRecovery++;
|
||||
const observed =
|
||||
result.category === "ready"
|
||||
? "was paid at the mint"
|
||||
: result.category === "completed"
|
||||
? "was already issued at the mint"
|
||||
: "failed terminally at the mint";
|
||||
startupProgress(
|
||||
`Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
} else {
|
||||
// Mint unreachable, too slow, or the quote unknown to it: leave the
|
||||
// operation pending so a later startup can still recover it.
|
||||
settlement.unobserved++;
|
||||
logger.warn("Could not check expired mint quote; leaving it pending", {
|
||||
operationId: op.id,
|
||||
mintUrl: op.mintUrl,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
error:
|
||||
check.error instanceof Error
|
||||
? check.error.message
|
||||
: String(check.error),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -772,6 +910,371 @@ export async function settleExpiredMintQuotes(
|
||||
return settlement;
|
||||
}
|
||||
|
||||
/**
|
||||
* Source for explicit PAID mint-quote recovery.
|
||||
*
|
||||
* Unlike the startup sweeps this also accepts caller-supplied operation ids so
|
||||
* an operator can target a quote coco already gave up on (state `failed`).
|
||||
*/
|
||||
export interface MintQuoteRecoverySource {
|
||||
ops: {
|
||||
mint: {
|
||||
listPending(): Promise<MintQuoteRecoveryCandidate[]>;
|
||||
get(operationId: string): Promise<MintQuoteRecoveryCandidate | null>;
|
||||
finalize(operationId: string): Promise<unknown>;
|
||||
};
|
||||
};
|
||||
mintOperationService: Pick<
|
||||
MintOperationServiceCleanup,
|
||||
"observePendingOperation"
|
||||
>;
|
||||
/**
|
||||
* Re-open a failed operation so it can be recovered; false when it is no
|
||||
* longer failed. Implementations must reload the full row (see
|
||||
* `reopenFailedMintOperation`).
|
||||
*/
|
||||
reopenFailedOperation(operationId: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface MintQuoteRecoveryOptions {
|
||||
/** Target only these operation ids (may include failed operations). */
|
||||
operationIds?: string[];
|
||||
/** Per-quote budget for observing the mint and finalizing the operation. */
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* Re-open failed operations instead of skipping them. Only applies to
|
||||
* operations named by `operationIds`: coco's pending listing never returns
|
||||
* failed operations, so they can only be recovered by explicit id.
|
||||
*/
|
||||
includeFailed?: boolean;
|
||||
/**
|
||||
* In-flight recovery work keyed by operation id, shared across runs.
|
||||
* withTimeout does not cancel the underlying request, so a timed-out quote
|
||||
* check or finalize must keep blocking a retry until it actually settles.
|
||||
*/
|
||||
outstanding?: Map<string, Promise<unknown>>;
|
||||
}
|
||||
|
||||
export interface MintQuoteRecoveryResult {
|
||||
/** Operations recovery acted on. */
|
||||
checked: number;
|
||||
/** Operations whose paid sats were minted or restored. */
|
||||
recovered: number;
|
||||
/** Quotes the mint still reports UNPAID; left pending. */
|
||||
waiting: number;
|
||||
/**
|
||||
* Quotes that ended terminally: the mint can no longer issue them, or coco
|
||||
* finalised them without recovering any proofs.
|
||||
*/
|
||||
terminal: number;
|
||||
/** Failed operations moved back to pending before checking. */
|
||||
reopened: number;
|
||||
/**
|
||||
* Operations left to a later run: the mint was unreachable, the per-quote
|
||||
* budget ran out, or the operation ended in a non-terminal state.
|
||||
*/
|
||||
retryable: number;
|
||||
/** Operations skipped because an earlier recovery of them is still running. */
|
||||
busy: number;
|
||||
errors: Array<{ operationId: string; error: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run async tasks strictly one after another.
|
||||
*
|
||||
* Used to serialize explicit wallet recovery: two concurrent requests must not
|
||||
* both snapshot the same failed operation, and a retry must not start
|
||||
* underneath work that outlived its timeout. A rejected task never breaks the
|
||||
* chain for the next one.
|
||||
*/
|
||||
export function createRunQueue(): <T>(run: () => Promise<T>) => Promise<T> {
|
||||
let tail: Promise<unknown> = Promise.resolve();
|
||||
return <T>(run: () => Promise<T>): Promise<T> => {
|
||||
const result = tail.then(run, run);
|
||||
tail = result.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
/** Per-quote budget for the mint round-trip during explicit recovery. */
|
||||
const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000;
|
||||
|
||||
/**
|
||||
* Bound for the local post-finalize diagnostic read. This is a database
|
||||
* lookup, not a mint round-trip, so it gets its own small budget: the per-op
|
||||
* mint budget is often already spent when finalize throws, and a starved
|
||||
* diagnostic would silently fall back to the generic error message.
|
||||
*/
|
||||
const DIAGNOSTIC_LOOKUP_TIMEOUT_MS = 250;
|
||||
|
||||
/**
|
||||
* Recover mint quotes whose sats are PAID at the mint but were never claimed.
|
||||
*
|
||||
* For every target the mint is asked for the current quote state, and only it
|
||||
* decides the outcome: PAID quotes have their stored outputs submitted, ISSUED
|
||||
* quotes have their signatures restored (NUT-09), UNPAID quotes are left
|
||||
* pending, and quotes the mint can no longer issue are reported rather than
|
||||
* silently dropped. Anything the mint cannot answer is retried later.
|
||||
*
|
||||
* `finalize()` does not throw when the mint refuses to issue or when an
|
||||
* already-issued quote's proofs cannot be restored: it returns a terminal
|
||||
* operation instead. Recovery therefore inspects the returned operation's
|
||||
* state and error and only counts a genuine finalized-without-error as
|
||||
* recovered.
|
||||
*
|
||||
* Failed operations are skipped unless `includeFailed` is set, and they can
|
||||
* only be targeted by explicit id because coco's pending listing never returns
|
||||
* them. Re-opening an operation is a mutation, so it happens only here, never
|
||||
* during startup recovery.
|
||||
*
|
||||
* Callers should serialize their own invocations and pass a shared
|
||||
* `outstanding` map: `timeoutMs` bounds the wait but does not cancel the
|
||||
* request behind it, so both a timed-out quote check and a timed-out finalize
|
||||
* keep blocking a retry until they actually settle.
|
||||
*/
|
||||
export async function runMintQuoteRecovery(
|
||||
source: MintQuoteRecoverySource,
|
||||
options: MintQuoteRecoveryOptions = {},
|
||||
onProgress?: (message: string) => void,
|
||||
): Promise<MintQuoteRecoveryResult> {
|
||||
if (options.includeFailed && !options.operationIds?.length) {
|
||||
throw new Error("includeFailed requires explicit operationIds");
|
||||
}
|
||||
const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("timeoutMs must be a positive finite number");
|
||||
}
|
||||
const outstanding =
|
||||
options.outstanding ?? new Map<string, Promise<unknown>>();
|
||||
const result: MintQuoteRecoveryResult = {
|
||||
checked: 0,
|
||||
recovered: 0,
|
||||
waiting: 0,
|
||||
terminal: 0,
|
||||
reopened: 0,
|
||||
retryable: 0,
|
||||
busy: 0,
|
||||
errors: [],
|
||||
};
|
||||
const messageOf = (error: unknown) =>
|
||||
error instanceof Error ? error.message : String(error);
|
||||
/** coco's fail-fast operation lock rejected the call: another holder exists. */
|
||||
const isInProgress = (error: unknown) =>
|
||||
error instanceof Error && error.name === "OperationInProgressError";
|
||||
/**
|
||||
* Register in-flight work for an operation. Entries are cleared only once the
|
||||
* work actually settles (withTimeout does not cancel the request behind it),
|
||||
* so a timed-out call keeps blocking a retry. The identity check stops a late
|
||||
* settlement from clearing a newer entry for the same operation.
|
||||
*/
|
||||
const track = (operationId: string, work: Promise<unknown>) => {
|
||||
outstanding.set(operationId, work);
|
||||
const clear = () => {
|
||||
if (outstanding.get(operationId) === work) {
|
||||
outstanding.delete(operationId);
|
||||
}
|
||||
};
|
||||
void work.then(clear, clear);
|
||||
};
|
||||
|
||||
let targets: MintQuoteRecoveryCandidate[];
|
||||
if (options.operationIds && options.operationIds.length > 0) {
|
||||
targets = [];
|
||||
const seen = new Set<string>();
|
||||
for (const operationId of options.operationIds) {
|
||||
if (seen.has(operationId)) continue;
|
||||
seen.add(operationId);
|
||||
try {
|
||||
const op = await source.ops.mint.get(operationId);
|
||||
if (!op) {
|
||||
result.errors.push({ operationId, error: "operation not found" });
|
||||
continue;
|
||||
}
|
||||
targets.push(op);
|
||||
} catch (error) {
|
||||
result.errors.push({ operationId, error: messageOf(error) });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
targets = await source.ops.mint.listPending();
|
||||
}
|
||||
|
||||
const { pending, failed } = selectMintQuotesForRecovery({
|
||||
mints: targets,
|
||||
includeFailed: options.includeFailed === true,
|
||||
});
|
||||
|
||||
for (const op of failed) {
|
||||
const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`;
|
||||
if (outstanding.has(op.id)) {
|
||||
result.busy++;
|
||||
onProgress?.(`${label}: an earlier recovery is still running; skipped`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (!(await source.reopenFailedOperation(op.id))) {
|
||||
onProgress?.(`${label}: no longer failed; skipped`);
|
||||
continue;
|
||||
}
|
||||
result.reopened++;
|
||||
onProgress?.(`${label}: re-opened failed operation for recovery`);
|
||||
} catch (error) {
|
||||
// coco's operation lock is fail-fast, so an in-progress error means a
|
||||
// processor or another recovery holds the operation right now.
|
||||
if (isInProgress(error)) {
|
||||
result.busy++;
|
||||
onProgress?.(`${label}: another recovery holds it; skipped`);
|
||||
} else {
|
||||
result.retryable++;
|
||||
onProgress?.(`${label}: could not re-open: ${messageOf(error)}`);
|
||||
}
|
||||
result.errors.push({ operationId: op.id, error: messageOf(error) });
|
||||
continue;
|
||||
}
|
||||
await recoverOne(op);
|
||||
}
|
||||
|
||||
for (const op of pending) await recoverOne(op);
|
||||
|
||||
return result;
|
||||
|
||||
async function recoverOne(op: MintQuoteRecoveryCandidate): Promise<void> {
|
||||
const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`;
|
||||
if (outstanding.has(op.id)) {
|
||||
result.busy++;
|
||||
onProgress?.(`${label}: an earlier recovery is still running; skipped`);
|
||||
return;
|
||||
}
|
||||
result.checked++;
|
||||
// One budget per operation, shared by the mint check and the finalize, so
|
||||
// a slow mint cannot silently double the documented per-quote wait. The
|
||||
// local post-finalize diagnostic read is exempt (DIAGNOSTIC_LOOKUP_TIMEOUT_MS).
|
||||
const deadlineAt = Date.now() + timeoutMs;
|
||||
const remaining = () => Math.max(1, deadlineAt - Date.now());
|
||||
|
||||
if (op.state === "executing") {
|
||||
// A crash mid-mint can leave outputs already signed at the mint;
|
||||
// finalize recovers them instead of minting a second time.
|
||||
await finalizeAndClassify(
|
||||
op.id,
|
||||
label,
|
||||
"recovered interrupted mint",
|
||||
remaining,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let observation: {
|
||||
category: "waiting" | "ready" | "completed" | "terminal";
|
||||
};
|
||||
// observePendingOperation is not read-only: it emits quote-state-changed,
|
||||
// persists the observation and can fail a terminal operation. Track it too,
|
||||
// so a timed-out check cannot be retried and then persist a stale read.
|
||||
const check = source.mintOperationService.observePendingOperation(op.id);
|
||||
track(op.id, check);
|
||||
try {
|
||||
observation = await withTimeout(check, remaining());
|
||||
} catch (error) {
|
||||
result.retryable++;
|
||||
result.errors.push({ operationId: op.id, error: messageOf(error) });
|
||||
onProgress?.(`${label}: could not check with mint: ${messageOf(error)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const decision = classifyMintQuoteObservation(observation.category);
|
||||
if (decision.action === "finalize") {
|
||||
await finalizeAndClassify(
|
||||
op.id,
|
||||
label,
|
||||
decision.observedRemoteState === "PAID"
|
||||
? `paid, minting proofs (${op.amount} sat)`
|
||||
: `already issued, restoring proofs (${op.amount} sat)`,
|
||||
remaining,
|
||||
);
|
||||
} else if (decision.action === "waiting") {
|
||||
result.waiting++;
|
||||
onProgress?.(`${label}: mint reports UNPAID; left pending`);
|
||||
} else {
|
||||
// coco records the mint's terminal verdict by failing the operation.
|
||||
result.terminal++;
|
||||
onProgress?.(`${label}: mint can no longer issue this quote`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run finalize and classify its result. coco returns a terminal operation
|
||||
* rather than throwing when the mint refuses (for example an expired quote)
|
||||
* or when an already-issued quote's proofs could not be restored, so a
|
||||
* fulfilled promise is not by itself evidence that sats were recovered.
|
||||
*/
|
||||
async function finalizeAndClassify(
|
||||
operationId: string,
|
||||
label: string,
|
||||
successMessage: string,
|
||||
remaining: () => number,
|
||||
): Promise<void> {
|
||||
const work = source.ops.mint.finalize(operationId);
|
||||
track(operationId, work);
|
||||
let terminal: { state?: string; error?: string } | null | undefined;
|
||||
try {
|
||||
terminal = (await withTimeout(work, remaining())) as
|
||||
| { state?: string; error?: string }
|
||||
| null
|
||||
| undefined;
|
||||
} catch (error) {
|
||||
if (isInProgress(error)) {
|
||||
result.busy++;
|
||||
onProgress?.(`${label}: another recovery is working on it; skipped`);
|
||||
} else {
|
||||
result.retryable++;
|
||||
// finalize can throw a generic "remains pending" error after coco has
|
||||
// persisted the actionable mint rejection (for example inactive keyset).
|
||||
const current = await withTimeout(
|
||||
source.ops.mint.get(operationId),
|
||||
Math.max(remaining(), DIAGNOSTIC_LOOKUP_TIMEOUT_MS),
|
||||
).catch(() => null);
|
||||
const detail = current?.state === "pending" && current.error
|
||||
? current.error
|
||||
: messageOf(error);
|
||||
result.errors.push({ operationId, error: detail });
|
||||
onProgress?.(`${label}: could not finish recovery: ${detail}`);
|
||||
return;
|
||||
}
|
||||
result.errors.push({ operationId, error: messageOf(error) });
|
||||
return;
|
||||
}
|
||||
if (terminal?.state === "finalized" && !terminal.error) {
|
||||
result.recovered++;
|
||||
onProgress?.(`${label}: ${successMessage}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
terminal?.state === "failed" ||
|
||||
(terminal?.state === "finalized" && terminal.error)
|
||||
) {
|
||||
result.terminal++;
|
||||
const detail =
|
||||
terminal.error ?? `left in state ${terminal.state ?? "unknown"}`;
|
||||
result.errors.push({ operationId, error: detail });
|
||||
onProgress?.(`${label}: not recovered: ${detail}`);
|
||||
return;
|
||||
}
|
||||
// Pending/executing/unknown: coco may still be working on the operation,
|
||||
// so leave it to a later run rather than calling it terminal.
|
||||
result.retryable++;
|
||||
result.errors.push({
|
||||
operationId,
|
||||
error: `left in state ${terminal?.state ?? "unknown"}; will retry`,
|
||||
});
|
||||
onProgress?.(
|
||||
`${label}: still ${terminal?.state ?? "unknown"}; left for a later run`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const PENDING_MINT_SWEEP_INTERVAL_MS = 15_000;
|
||||
/** Per-quote wait inside a sweep, so one stalled mint cannot starve the rest. */
|
||||
const PENDING_MINT_CHECK_TIMEOUT_MS = 10_000;
|
||||
@@ -1397,6 +1900,10 @@ export async function createCocoClient(
|
||||
};
|
||||
|
||||
let disposed = false;
|
||||
// Explicit recovery runs are serialized, and finalize work that outlives its
|
||||
// timeout stays in the map so a retry waits for it.
|
||||
const enqueueRecovery = createRunQueue();
|
||||
const recoveryOutstanding = new Map<string, Promise<unknown>>();
|
||||
|
||||
/**
|
||||
* Block a value-moving operation until background recovery has settled.
|
||||
@@ -1752,6 +2259,7 @@ export async function createCocoClient(
|
||||
await waitForRecovery();
|
||||
const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
const dryRun = options.dryRun === true;
|
||||
const force = options.force === true;
|
||||
const nowMs = Date.now();
|
||||
|
||||
const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([
|
||||
@@ -1779,6 +2287,8 @@ export async function createCocoClient(
|
||||
});
|
||||
|
||||
const errors: WalletCleanupResult["errors"] = [];
|
||||
let failedMintQuotes = 0;
|
||||
let leftForRecovery = 0;
|
||||
|
||||
if (!dryRun) {
|
||||
const mintService = (
|
||||
@@ -1788,20 +2298,49 @@ export async function createCocoClient(
|
||||
).mintOperationService;
|
||||
|
||||
for (const op of selection.mintsToFail) {
|
||||
try {
|
||||
await mintService.failPendingOperation(
|
||||
{ id: op.id },
|
||||
{
|
||||
reason: "Expired unpaid mint quote cleaned up by routstrd",
|
||||
retryable: false,
|
||||
observedAt: nowMs,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
if (force) {
|
||||
// Legacy behaviour: fail the quote locally without asking the mint.
|
||||
try {
|
||||
await mintService.failPendingOperation(
|
||||
{ id: op.id },
|
||||
{
|
||||
reason: "Expired mint quote cleaned up by routstrd (forced)",
|
||||
retryable: false,
|
||||
observedAt: nowMs,
|
||||
},
|
||||
);
|
||||
failedMintQuotes++;
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Expiry alone does not prove the quote was never paid: the
|
||||
// Lightning payment can land before expiry while the daemon is down.
|
||||
// Confirm UNPAID with the mint before failing, exactly as startup
|
||||
// recovery does; paid quotes are left for recovery to finalize.
|
||||
const check = await failExpiredMintQuoteIfUnpaid(
|
||||
mintService,
|
||||
op.id,
|
||||
EXPIRED_MINT_OBSERVATION_DEADLINE_MS,
|
||||
);
|
||||
if (check.outcome === "failed") {
|
||||
failedMintQuotes++;
|
||||
} else {
|
||||
leftForRecovery++;
|
||||
if (check.outcome === "unobserved") {
|
||||
errors.push({
|
||||
operationId: op.id,
|
||||
error: `could not confirm quote state with mint: ${
|
||||
check.error instanceof Error
|
||||
? check.error.message
|
||||
: String(check.error)
|
||||
}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1828,8 +2367,15 @@ export async function createCocoClient(
|
||||
}
|
||||
}
|
||||
|
||||
const mintSummary = summarizeMintCleanup({
|
||||
dryRun,
|
||||
candidates: selection.mintsToFail.length,
|
||||
failed: failedMintQuotes,
|
||||
leftForRecovery,
|
||||
});
|
||||
const actedOn =
|
||||
selection.mintsToFail.length +
|
||||
(dryRun ? mintSummary.mintQuoteCandidates : mintSummary.failedMintQuotes) +
|
||||
leftForRecovery +
|
||||
selection.sendsToReclaim.length +
|
||||
selection.meltsToCancel.length;
|
||||
const skipped =
|
||||
@@ -1838,12 +2384,36 @@ export async function createCocoClient(
|
||||
|
||||
return {
|
||||
dryRun,
|
||||
failedMintQuotes: selection.mintsToFail.length,
|
||||
...mintSummary,
|
||||
reclaimedSends: selection.sendsToReclaim.length,
|
||||
cancelledMelts: selection.meltsToCancel.length,
|
||||
skipped,
|
||||
errors,
|
||||
};
|
||||
},
|
||||
|
||||
async recoverMintQuotes(options, onProgress) {
|
||||
await waitForRecovery();
|
||||
const service = (
|
||||
coco as unknown as {
|
||||
mintOperationService: MintOperationServiceCleanup;
|
||||
}
|
||||
).mintOperationService;
|
||||
// Serialize explicit recovery: two concurrent requests must not both
|
||||
// snapshot the same failed operation, and a retry must not start
|
||||
// underneath a finalize that outlived its timeout.
|
||||
return enqueueRecovery(() =>
|
||||
runMintQuoteRecovery(
|
||||
{
|
||||
ops: coco.ops as unknown as MintQuoteRecoverySource["ops"],
|
||||
mintOperationService: service,
|
||||
reopenFailedOperation: (operationId) =>
|
||||
reopenFailedMintOperation(service, operationId),
|
||||
},
|
||||
{ ...options, outstanding: recoveryOutstanding },
|
||||
onProgress,
|
||||
),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -83,13 +83,23 @@ export interface WalletCleanupOptions {
|
||||
minAgeMs?: number;
|
||||
/** Report what would be cleaned without applying changes. */
|
||||
dryRun?: boolean;
|
||||
/**
|
||||
* Fail expired mint quotes without confirming UNPAID with the mint. Only for
|
||||
* operators who accept the risk of stranding a quote that was paid before
|
||||
* its invoice expired; recovery is the safe default.
|
||||
*/
|
||||
force?: boolean;
|
||||
}
|
||||
|
||||
/** Summary of a wallet cleanup run. */
|
||||
export interface WalletCleanupResult {
|
||||
dryRun: boolean;
|
||||
/** Number of expired pending mint quotes marked as failed. */
|
||||
/** Expired quotes selected for checking; dry runs do not contact the mint. */
|
||||
mintQuoteCandidates: number;
|
||||
/** Number actually marked failed (always zero in a dry run). */
|
||||
failedMintQuotes: number;
|
||||
/** Expired quotes kept pending because they are paid/issued or unverified. */
|
||||
leftForRecovery: number;
|
||||
/** Number of stale pending send operations reclaimed. */
|
||||
reclaimedSends: number;
|
||||
/** Number of stale prepared melt operations cancelled. */
|
||||
@@ -120,6 +130,35 @@ export interface MintQuoteStatus {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Options for explicit PAID mint-quote recovery. */
|
||||
export interface WalletMintQuoteRecoveryOptions {
|
||||
/** Target only these operation ids (may include failed operations). */
|
||||
operationIds?: string[];
|
||||
/** Re-open failed operations instead of skipping them. */
|
||||
includeFailed?: boolean;
|
||||
/** Per-quote mint timeout in milliseconds. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
/** Summary of a PAID mint-quote recovery run. */
|
||||
export interface WalletMintQuoteRecoveryResult {
|
||||
/** Operations whose quote state was checked with the mint. */
|
||||
checked: number;
|
||||
/** Operations whose paid sats were minted or restored. */
|
||||
recovered: number;
|
||||
/** Quotes the mint still reports UNPAID; left pending. */
|
||||
waiting: number;
|
||||
/** Quotes the mint can no longer issue. */
|
||||
terminal: number;
|
||||
/** Failed operations moved back to pending before checking. */
|
||||
reopened: number;
|
||||
/** Operations left to a later run (mint unreachable, budget spent, non-terminal). */
|
||||
retryable: number;
|
||||
/** Operations skipped because an earlier recovery of them is still running. */
|
||||
busy: number;
|
||||
errors: Array<{ operationId: string; error: string }>;
|
||||
}
|
||||
|
||||
export interface CocodClient {
|
||||
ping(): Promise<boolean>;
|
||||
getStatus(): Promise<CocodState>;
|
||||
@@ -154,6 +193,14 @@ export interface CocodClient {
|
||||
cleanupStuckOperations?(
|
||||
options?: WalletCleanupOptions,
|
||||
): Promise<WalletCleanupResult>;
|
||||
/**
|
||||
* Re-issue PAID mint quotes whose sats were never claimed, optionally
|
||||
* targeting specific operations (including ones coco already failed).
|
||||
*/
|
||||
recoverMintQuotes?(
|
||||
options?: WalletMintQuoteRecoveryOptions,
|
||||
onProgress?: (message: string) => void,
|
||||
): Promise<WalletMintQuoteRecoveryResult>;
|
||||
/** Report background wallet recovery progress, when the wallet supports it. */
|
||||
getRecoveryProgress?(): Promise<WalletRecoveryProgress>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Re-opening a failed mint operation is the one genuinely destructive step of
|
||||
* PAID-quote recovery, because coco's private `transitionToPending` spreads
|
||||
* whatever it is handed and `SqliteMintOperationRepository.update` rewrites
|
||||
* every column. A partial object such as `{ id }` is therefore rejected by the
|
||||
* NOT NULL schema, and on a more permissive adapter would overwrite `quoteId`,
|
||||
* `amount`, `request`, `lastObservedRemoteState` and `outputDataJson` with NULL,
|
||||
* destroying the material needed to claim the paid sats.
|
||||
*
|
||||
* These tests drive the production `reopenFailedMintOperation` helper against a
|
||||
* real coco sqlite repository through adapter-backed getOperation and
|
||||
* transitionToPending implementations, so a regression at the helper/service
|
||||
* boundary is caught rather than a mock standing in for it.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
||||
import { reopenFailedMintOperation } from "./coco-client";
|
||||
|
||||
const OUTPUT_DATA = [
|
||||
{
|
||||
blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" },
|
||||
blindingFactor: "1234567890",
|
||||
secret: "aabbccdd",
|
||||
},
|
||||
];
|
||||
|
||||
function failedRow(state = "failed") {
|
||||
return {
|
||||
id: "op-1",
|
||||
mintUrl: "https://mint.example.com",
|
||||
quoteId: "quote-1",
|
||||
state,
|
||||
createdAt: 1_000,
|
||||
updatedAt: 2_000,
|
||||
error: state === "failed" ? "expired" : undefined,
|
||||
method: "bolt11",
|
||||
methodData: { method: "bolt11", data: {} },
|
||||
amount: 210_000,
|
||||
unit: "sat",
|
||||
request: "lnbc1example",
|
||||
expiry: 1_800_000_000,
|
||||
pubkey: undefined,
|
||||
lastObservedRemoteState: "PAID",
|
||||
lastObservedRemoteStateAt: 3_000,
|
||||
terminalFailure:
|
||||
state === "failed" ? { reason: "expired", observedAt: 3_000 } : undefined,
|
||||
outputData: OUTPUT_DATA,
|
||||
};
|
||||
}
|
||||
|
||||
describe("reopenFailedMintOperation against real coco sqlite", () => {
|
||||
let dir: string | undefined;
|
||||
let database: Database | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
database?.close();
|
||||
database = undefined;
|
||||
if (dir) rmSync(dir, { recursive: true, force: true });
|
||||
dir = undefined;
|
||||
});
|
||||
|
||||
async function repos() {
|
||||
dir = mkdtempSync(join(tmpdir(), "routstrd-reopen-"));
|
||||
database = new Database(join(dir, "coco.db"));
|
||||
const repositories = new SqliteRepositories({ database });
|
||||
await repositories.init();
|
||||
return repositories;
|
||||
}
|
||||
|
||||
function readRow(repositories: SqliteRepositories, id: string) {
|
||||
return repositories.mintOperationRepository.getById(id) as unknown as Promise<
|
||||
Record<string, unknown> | null
|
||||
>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapter-backed stand-in for the private coco service methods the helper
|
||||
* uses: getOperation reads and transitionToPending mirrors coco's
|
||||
* spread-and-update implementation.
|
||||
*/
|
||||
function serviceOver(repositories: SqliteRepositories) {
|
||||
return {
|
||||
acquireOperationLock: async (_id: string) => () => {},
|
||||
getOperation: (id: string) => readRow(repositories, id),
|
||||
transitionToPending: async (
|
||||
op: Record<string, unknown>,
|
||||
error?: string,
|
||||
) => {
|
||||
await repositories.mintOperationRepository.update({
|
||||
...op,
|
||||
state: "pending",
|
||||
error,
|
||||
} as never);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("re-opens a failed row without losing quote metadata or stored outputs", async () => {
|
||||
const repositories = await repos();
|
||||
await repositories.mintOperationRepository.create(
|
||||
failedRow() as never,
|
||||
);
|
||||
|
||||
const reopened = await reopenFailedMintOperation(
|
||||
serviceOver(repositories),
|
||||
"op-1",
|
||||
);
|
||||
|
||||
expect(reopened).toBe(true);
|
||||
const row = await readRow(repositories, "op-1");
|
||||
expect(row?.state).toBe("pending");
|
||||
expect(row?.quoteId).toBe("quote-1");
|
||||
expect(row?.amount).toBe(210_000);
|
||||
expect(row?.unit).toBe("sat");
|
||||
expect(row?.request).toBe("lnbc1example");
|
||||
expect(row?.lastObservedRemoteState).toBe("PAID");
|
||||
expect(row?.outputData).toEqual(OUTPUT_DATA);
|
||||
expect(row?.terminalFailure ?? undefined).toBeUndefined();
|
||||
});
|
||||
|
||||
it("is a no-op when the operation is no longer failed", async () => {
|
||||
const repositories = await repos();
|
||||
await repositories.mintOperationRepository.create(
|
||||
failedRow("finalized") as never,
|
||||
);
|
||||
|
||||
const reopened = await reopenFailedMintOperation(
|
||||
serviceOver(repositories),
|
||||
"op-1",
|
||||
);
|
||||
|
||||
expect(reopened).toBe(false);
|
||||
const row = await readRow(repositories, "op-1");
|
||||
expect(row?.state).toBe("finalized");
|
||||
expect(row?.outputData).toEqual(OUTPUT_DATA);
|
||||
});
|
||||
|
||||
it("rejects a partial row, which is why the helper reloads in full", async () => {
|
||||
// Locks in the reason for reloading. If a future coco version accepts
|
||||
// partial updates this fails, and the helper can be simplified rather than
|
||||
// silently losing paid sats.
|
||||
const repositories = await repos();
|
||||
await repositories.mintOperationRepository.create(
|
||||
failedRow() as never,
|
||||
);
|
||||
|
||||
await expect(
|
||||
repositories.mintOperationRepository.update({
|
||||
id: "op-1",
|
||||
state: "pending",
|
||||
updatedAt: Date.now(),
|
||||
} as never),
|
||||
).rejects.toThrow();
|
||||
|
||||
const unchanged = await readRow(repositories, "op-1");
|
||||
expect(unchanged?.state).toBe("failed");
|
||||
expect(unchanged?.outputData).toEqual(OUTPUT_DATA);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* End-to-end PAID mint-quote recovery against a real coco Manager, real sqlite
|
||||
* and a real in-process mint that produces genuine blind signatures.
|
||||
*
|
||||
* Nothing here mocks the wallet: a quote is created through coco, the mint is
|
||||
* told what to report, and the production `runMintQuoteRecovery` drives the
|
||||
* outcome. These are the release-gating scenarios for the feature, and they are
|
||||
* the only tests that exercise issuance and NUT-09 restore over HTTP.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Manager } from "@cashu/coco-core";
|
||||
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
||||
import { QUOTE_EXPIRED, FakeMint } from "./testing/fake-mint";
|
||||
import { reopenFailedMintOperation, runMintQuoteRecovery } from "./coco-client";
|
||||
|
||||
type AnyRecord = Record<string, unknown>;
|
||||
|
||||
interface Booted {
|
||||
manager: Manager;
|
||||
repositories: SqliteRepositories;
|
||||
mint: FakeMint;
|
||||
/** Build the recovery source the production function expects. */
|
||||
source: () => AnyRecord;
|
||||
spendable: () => Promise<number>;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
async function boot(options: { quoteExpiry?: number | null } = {}) {
|
||||
const mint = new FakeMint();
|
||||
mint.quoteExpiry = options.quoteExpiry ?? null;
|
||||
mint.start();
|
||||
const dir = mkdtempSync(join(tmpdir(), "routstrd-fakemint-"));
|
||||
const database = new Database(join(dir, "coco.db"));
|
||||
const repositories = new SqliteRepositories({ database });
|
||||
await repositories.init();
|
||||
const manager = new Manager(repositories, async () => new Uint8Array(64).fill(7));
|
||||
await manager.mint.addMint(mint.url, { trusted: true });
|
||||
|
||||
const service = (manager as unknown as { mintOperationService: AnyRecord })
|
||||
.mintOperationService;
|
||||
|
||||
const booted: Booted = {
|
||||
manager,
|
||||
repositories,
|
||||
mint,
|
||||
spendable: async () => {
|
||||
const balances = (await manager.wallet.balances.byMint()) as Record<
|
||||
string,
|
||||
{ spendable: number }
|
||||
>;
|
||||
return balances[mint.url]?.spendable ?? 0;
|
||||
},
|
||||
source: () => ({
|
||||
ops: {
|
||||
mint: {
|
||||
listPending: () => manager.ops.mint.listPending(),
|
||||
get: (id: string) => manager.ops.mint.get(id),
|
||||
finalize: (id: string) => manager.ops.mint.finalize(id),
|
||||
},
|
||||
},
|
||||
mintOperationService: service,
|
||||
reopenFailedOperation: (id: string) =>
|
||||
reopenFailedMintOperation(service as never, id),
|
||||
}),
|
||||
close: async () => {
|
||||
await manager.dispose().catch(() => undefined);
|
||||
database.close();
|
||||
mint.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
|
||||
return booted;
|
||||
}
|
||||
|
||||
async function prepareQuote(booted: Booted, amount: number) {
|
||||
const op = (await booted.manager.ops.mint.prepare({
|
||||
mintUrl: booted.mint.url,
|
||||
amount,
|
||||
method: "bolt11",
|
||||
} as never)) as unknown as AnyRecord;
|
||||
return op;
|
||||
}
|
||||
|
||||
function outputsOf(op: AnyRecord) {
|
||||
// coco stores mint outputs as { keep, send }, like the on-disk output JSON.
|
||||
const outputData = (op.outputData as AnyRecord).keep as Array<{
|
||||
blindedMessage: { amount: unknown; id: string; B_: string };
|
||||
}>;
|
||||
return outputData.map((output) => ({
|
||||
amount: Number(String(output.blindedMessage.amount)),
|
||||
id: output.blindedMessage.id,
|
||||
B_: output.blindedMessage.B_,
|
||||
}));
|
||||
}
|
||||
|
||||
let booted: Booted | undefined;
|
||||
afterEach(async () => {
|
||||
await booted?.close();
|
||||
booted = undefined;
|
||||
});
|
||||
|
||||
describe("PAID mint quote recovery with a real Manager and mint", () => {
|
||||
it("issues an expired-but-PAID quote with the operation's own outputs, once", async () => {
|
||||
// The motivating case: the invoice expired, but the mint says PAID and has
|
||||
// issued nothing.
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 210_000);
|
||||
const expectedOutputs = outputsOf(op).map((o) => o.B_);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
|
||||
const result = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
|
||||
expect(result).toMatchObject({ checked: 1, recovered: 1, terminal: 0 });
|
||||
expect(await booted.spendable()).toBe(210_000);
|
||||
// Issuance used exactly the blinded outputs stored on the operation.
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual(
|
||||
[...expectedOutputs].sort(),
|
||||
);
|
||||
|
||||
// A second run must not mint again or double-credit.
|
||||
const again = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
expect(again).toMatchObject({ checked: 0, recovered: 0 });
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
expect(await booted.spendable()).toBe(210_000);
|
||||
});
|
||||
|
||||
it("existing coco recovery already issues expired paid pending quotes", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 100);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
await booted.manager.recoverPendingMintOperations();
|
||||
expect(await booted.spendable()).toBe(100);
|
||||
expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("ISSUED");
|
||||
});
|
||||
|
||||
it("keeps rejected stored outputs and reports the actionable mint error", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 100);
|
||||
const outputs = outputsOf(op);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
// Model the mint refusing the stored outputs, not invoice expiry. This is
|
||||
// not evidence that the production quotes used an inactive keyset.
|
||||
booted.mint.mintError = { code: 12001, detail: "keyset id inactive." };
|
||||
await booted.manager.recoverPendingMintOperations();
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
const result = await runMintQuoteRecovery(booted.source() as never, {
|
||||
operationIds: [op.id as string],
|
||||
});
|
||||
expect(result).toMatchObject({ recovered: 0, retryable: 1 });
|
||||
expect(result.errors.some((entry) => entry.error.includes("keyset id inactive"))).toBe(true);
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
expect(booted.mint.getQuote(op.quoteId as string)?.state).toBe("PAID");
|
||||
expect(outputsOf(await booted.manager.ops.mint.get(op.id as string) as unknown as AnyRecord)).toEqual(outputs);
|
||||
for (const request of booted.mint.requests) expect(request.outputs).toEqual(outputs);
|
||||
});
|
||||
|
||||
it("restores proofs for a quote already issued at the mint", async () => {
|
||||
booted = await boot({ quoteExpiry: null });
|
||||
const op = await prepareQuote(booted, 210_000);
|
||||
// Another wallet issued it: the signatures exist at the mint for the very
|
||||
// outputs this operation stored.
|
||||
booted.mint.signFor(op.quoteId as string, outputsOf(op));
|
||||
|
||||
const result = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
|
||||
expect(result).toMatchObject({ recovered: 1, terminal: 0, retryable: 0 });
|
||||
expect(await booted.spendable()).toBe(210_000);
|
||||
});
|
||||
|
||||
it("reports terminal without credit when the mint refuses issuance", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
booted.mint.mintError = { code: QUOTE_EXPIRED, detail: "quote expired" };
|
||||
|
||||
const result = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, unknown>;
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 1 });
|
||||
expect((result.errors as unknown[]).length).toBeGreaterThan(0);
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
});
|
||||
|
||||
it("reports terminal without credit when an issued quote cannot be restored", async () => {
|
||||
booted = await boot({ quoteExpiry: null });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
// Issued at the mint, but the signatures for our outputs are gone.
|
||||
booted.mint.markIssued(op.quoteId as string);
|
||||
|
||||
const result = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 1 });
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
});
|
||||
|
||||
it("does not credit a quote when the mint returns null NUT-09 signatures", async () => {
|
||||
// KNOWN INTEROP GAP, not a supported path. NUT-09 permits `null` in the
|
||||
// positional `signatures` array for outputs the mint never signed, but
|
||||
// cashu-ts 3.7.1 - which coco depends on - dereferences every entry while
|
||||
// normalising amounts, so the wallet throws instead of skipping the null.
|
||||
// Recovery therefore surfaces an error and credits nothing, leaving the
|
||||
// operation for a later run.
|
||||
//
|
||||
// This test pins the current behaviour so the gap cannot quietly disappear.
|
||||
// Revisit (and change this expectation) once coco's cashu-ts parses
|
||||
// positional nulls, and file/track it upstream in the meantime.
|
||||
booted = await boot({ quoteExpiry: null });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
// Issued at the mint with nothing signed for this operation's outputs.
|
||||
booted.mint.markIssued(op.quoteId as string);
|
||||
booted.mint.restoreIncludesNulls = true;
|
||||
|
||||
const result = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, terminal: 0, retryable: 1 });
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
});
|
||||
|
||||
it("recovers a failed operation whose stale history says UNPAID", async () => {
|
||||
// The composed path the feature exists for: a real prepared operation,
|
||||
// failed locally, with a stale UNPAID observation from the old local-fail
|
||||
// behaviour, recovered by explicit id and issued with its own outputs.
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
const storedOutputs = outputsOf(op).map((o) => o.B_);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
|
||||
const row = (await booted.repositories.mintOperationRepository.getById(
|
||||
op.id as string,
|
||||
)) as unknown as AnyRecord;
|
||||
expect(row.outputData).toBeDefined();
|
||||
await booted.repositories.mintOperationRepository.update({
|
||||
...row,
|
||||
state: "failed",
|
||||
lastObservedRemoteState: "UNPAID",
|
||||
error: "Expired unpaid mint quote cleaned up by routstrd",
|
||||
terminalFailure: { reason: "expired", observedAt: Date.now() },
|
||||
} as never);
|
||||
|
||||
// A purely local decision would skip this row; the mint has the last word.
|
||||
const result = (await runMintQuoteRecovery(booted.source() as never, {
|
||||
operationIds: [op.id as string],
|
||||
includeFailed: true,
|
||||
})) as unknown as Record<string, number>;
|
||||
|
||||
expect(result).toMatchObject({ reopened: 1, recovered: 1, terminal: 0 });
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
expect(booted.mint.requests[0]?.outputs.map((o) => o.B_).sort()).toEqual(
|
||||
[...storedOutputs].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("counts a locked operation as busy instead of minting underneath it", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
const service = (
|
||||
booted.manager as unknown as {
|
||||
mintOperationService: { acquireOperationLock(id: string): Promise<() => void> };
|
||||
}
|
||||
).mintOperationService;
|
||||
|
||||
// Hold the operation, as a processor or another recovery would.
|
||||
const release = await service.acquireOperationLock(op.id as string);
|
||||
const blocked = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
|
||||
expect(blocked).toMatchObject({ recovered: 0 });
|
||||
expect((blocked.busy ?? 0) + (blocked.retryable ?? 0)).toBeGreaterThan(0);
|
||||
expect(booted.mint.requests).toHaveLength(0);
|
||||
expect(await booted.spendable()).toBe(0);
|
||||
|
||||
release();
|
||||
const after = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
expect(after).toMatchObject({ recovered: 1 });
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not mint a second time while issuance is in flight", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
|
||||
// Hold the mint's response so the first recovery is visibly in flight.
|
||||
let releaseGate!: () => void;
|
||||
booted.mint.gate = new Promise<void>((resolve) => {
|
||||
releaseGate = resolve;
|
||||
});
|
||||
const first = runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
) as unknown as Promise<Record<string, number>>;
|
||||
|
||||
for (let i = 0; i < 400 && booted.mint.requests.length === 0; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
|
||||
// A concurrent run must not issue again while that request is outstanding.
|
||||
await runMintQuoteRecovery(booted.source() as never);
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
|
||||
releaseGate();
|
||||
expect(await first).toMatchObject({ recovered: 1 });
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
expect(booted.mint.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("coexists with coco's own mint operation watcher and processor", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
// The real background machinery coco uses to settle pending mint quotes.
|
||||
await booted.manager.enableMintOperationWatcher();
|
||||
await booted.manager.enableMintOperationProcessor();
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
|
||||
// Either our recovery or coco's processor may win; both are safe.
|
||||
await runMintQuoteRecovery(booted.source() as never);
|
||||
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
expect(booted.mint.requests.length).toBeLessThanOrEqual(1);
|
||||
|
||||
// Give the processor time to act and confirm nothing is credited twice.
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
expect(booted.mint.requests.length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("tracks a hung quote check so a later run waits instead of re-reading", async () => {
|
||||
booted = await boot({ quoteExpiry: -60 });
|
||||
const op = await prepareQuote(booted, 21_000);
|
||||
booted.mint.markPaid(op.quoteId as string);
|
||||
|
||||
let releaseObserve!: () => void;
|
||||
booted.mint.observeGate = new Promise<void>((resolve) => {
|
||||
releaseObserve = resolve;
|
||||
});
|
||||
const outstanding = new Map<string, Promise<unknown>>();
|
||||
|
||||
const first = (await runMintQuoteRecovery(booted.source() as never, {
|
||||
timeoutMs: 30,
|
||||
outstanding,
|
||||
})) as unknown as Record<string, number>;
|
||||
expect(first).toMatchObject({ retryable: 1, recovered: 0 });
|
||||
expect(outstanding.has(op.id as string)).toBe(true);
|
||||
|
||||
const second = (await runMintQuoteRecovery(booted.source() as never, {
|
||||
outstanding,
|
||||
})) as unknown as Record<string, number>;
|
||||
expect(second).toMatchObject({ checked: 0, busy: 1 });
|
||||
|
||||
// Once the held check settles the tracking drains, and recovery proceeds.
|
||||
releaseObserve();
|
||||
for (let i = 0; i < 400 && outstanding.size > 0; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
expect(outstanding.size).toBe(0);
|
||||
|
||||
const third = (await runMintQuoteRecovery(
|
||||
booted.source() as never,
|
||||
)) as unknown as Record<string, number>;
|
||||
expect(third).toMatchObject({ recovered: 1 });
|
||||
expect(await booted.spendable()).toBe(21_000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* Real-Manager integration for re-opening a failed mint operation.
|
||||
*
|
||||
* The unit tests exercise `reopenFailedMintOperation` against a hand-written
|
||||
* service double, which cannot catch a change in coco's own
|
||||
* `MintOperationService.transitionToPending` semantics or in its per-operation
|
||||
* lock. This drives the production helper against an actual coco `Manager`
|
||||
* backed by sqlite, so the private-service boundary that the helper depends on
|
||||
* is exercised for real: full-row preservation, the shared operation lock, and
|
||||
* the `mint-op:pending` event.
|
||||
*
|
||||
* No mint or network access is involved; nothing here enables the mint watcher
|
||||
* or processor, which the fake-mint integration covers.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Manager } from "@cashu/coco-core";
|
||||
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
||||
import { reopenFailedMintOperation } from "./coco-client";
|
||||
|
||||
const OUTPUT_DATA = [
|
||||
{
|
||||
blindedMessage: { amount: "210000", id: "00deadbeef", B_: "02deadbeef" },
|
||||
blindingFactor: "1234567890",
|
||||
secret: "aabbccdd",
|
||||
},
|
||||
];
|
||||
|
||||
function failedRow() {
|
||||
return {
|
||||
id: "op-1",
|
||||
mintUrl: "https://mint.invalid",
|
||||
quoteId: "quote-1",
|
||||
state: "failed",
|
||||
createdAt: 1_000,
|
||||
updatedAt: 2_000,
|
||||
error: "expired",
|
||||
method: "bolt11",
|
||||
methodData: { method: "bolt11", data: {} },
|
||||
amount: 210_000,
|
||||
unit: "sat",
|
||||
request: "lnbc1example",
|
||||
expiry: 1_800_000_000,
|
||||
pubkey: undefined,
|
||||
lastObservedRemoteState: "PAID",
|
||||
lastObservedRemoteStateAt: 3_000,
|
||||
terminalFailure: { reason: "expired", observedAt: 3_000 },
|
||||
outputData: OUTPUT_DATA,
|
||||
};
|
||||
}
|
||||
|
||||
describe("reopenFailedMintOperation with a real coco Manager", () => {
|
||||
let dir: string | undefined;
|
||||
let database: Database | undefined;
|
||||
let manager: Manager | undefined;
|
||||
let repositories: SqliteRepositories | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await manager?.dispose().catch(() => undefined);
|
||||
manager = undefined;
|
||||
database?.close();
|
||||
database = undefined;
|
||||
repositories = undefined;
|
||||
if (dir) rmSync(dir, { recursive: true, force: true });
|
||||
dir = undefined;
|
||||
});
|
||||
|
||||
async function boot() {
|
||||
dir = mkdtempSync(join(tmpdir(), "routstrd-manager-"));
|
||||
database = new Database(join(dir, "coco.db"));
|
||||
repositories = new SqliteRepositories({ database });
|
||||
await repositories.init();
|
||||
manager = new Manager(repositories, async () => new Uint8Array(64).fill(7));
|
||||
await repositories.mintOperationRepository.create(failedRow() as never);
|
||||
const service = (
|
||||
manager as unknown as {
|
||||
mintOperationService: {
|
||||
acquireOperationLock(id: string): Promise<() => void>;
|
||||
getOperation(id: string): Promise<Record<string, unknown> | null>;
|
||||
transitionToPending(
|
||||
op: Record<string, unknown>,
|
||||
error?: string,
|
||||
): Promise<unknown>;
|
||||
};
|
||||
}
|
||||
).mintOperationService;
|
||||
const eventBus = (
|
||||
manager as unknown as {
|
||||
eventBus: { on(event: string, handler: () => void): () => void };
|
||||
}
|
||||
).eventBus;
|
||||
return { service, eventBus };
|
||||
}
|
||||
|
||||
function readRow(id: string) {
|
||||
return repositories!.mintOperationRepository.getById(id) as unknown as Promise<
|
||||
Record<string, unknown> | null
|
||||
>;
|
||||
}
|
||||
|
||||
it("re-opens through the real service and emits mint-op:pending", async () => {
|
||||
const { service, eventBus } = await boot();
|
||||
const events: string[] = [];
|
||||
const off = eventBus.on("mint-op:pending", () => events.push("pending"));
|
||||
|
||||
const reopened = await reopenFailedMintOperation(service, "op-1");
|
||||
off();
|
||||
|
||||
expect(reopened).toBe(true);
|
||||
const row = await readRow("op-1");
|
||||
expect(row?.state).toBe("pending");
|
||||
expect(row?.quoteId).toBe("quote-1");
|
||||
expect(row?.amount).toBe(210_000);
|
||||
expect(row?.lastObservedRemoteState).toBe("PAID");
|
||||
expect(row?.outputData).toEqual(OUTPUT_DATA);
|
||||
expect(row?.terminalFailure ?? undefined).toBeUndefined();
|
||||
expect(events).toEqual(["pending"]);
|
||||
});
|
||||
|
||||
it("refuses to re-open while coco's operation lock is held", async () => {
|
||||
const { service } = await boot();
|
||||
// coco's OperationIdLock is fail-fast: a holder blocks the re-open by
|
||||
// making it throw, and nothing is written.
|
||||
const release = await service.acquireOperationLock("op-1");
|
||||
|
||||
await expect(reopenFailedMintOperation(service, "op-1")).rejects.toThrow(
|
||||
/already in progress/,
|
||||
);
|
||||
const blocked = await readRow("op-1");
|
||||
expect(blocked?.state).toBe("failed");
|
||||
expect(blocked?.outputData).toEqual(OUTPUT_DATA);
|
||||
|
||||
release();
|
||||
expect(await reopenFailedMintOperation(service, "op-1")).toBe(true);
|
||||
const reopened = await readRow("op-1");
|
||||
expect(reopened?.state).toBe("pending");
|
||||
expect(reopened?.outputData).toEqual(OUTPUT_DATA);
|
||||
});
|
||||
|
||||
it("leaves an operation a processor finalized alone", async () => {
|
||||
const { service } = await boot();
|
||||
// Emulate a processor winning the race while holding the same lock.
|
||||
const release = await service.acquireOperationLock("op-1");
|
||||
const row = await readRow("op-1");
|
||||
await repositories!.mintOperationRepository.update({
|
||||
...row,
|
||||
state: "finalized",
|
||||
} as never);
|
||||
release();
|
||||
|
||||
expect(await reopenFailedMintOperation(service, "op-1")).toBe(false);
|
||||
const after = await readRow("op-1");
|
||||
expect(after?.state).toBe("finalized");
|
||||
expect(after?.outputData).toEqual(OUTPUT_DATA);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
classifyMintQuoteObservation,
|
||||
selectMintQuotesForRecovery,
|
||||
type MintQuoteRecoveryCandidate,
|
||||
} from "./mint-quote-recovery";
|
||||
|
||||
function mint(
|
||||
overrides: Partial<MintQuoteRecoveryCandidate> = {},
|
||||
): MintQuoteRecoveryCandidate {
|
||||
return {
|
||||
id: "mint-1",
|
||||
mintUrl: "https://mint.example",
|
||||
quoteId: "quote-1",
|
||||
state: "pending",
|
||||
amount: 1000,
|
||||
expiry: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("classifyMintQuoteObservation", () => {
|
||||
it("finalizes a paid-but-unissued quote by minting its stored outputs", () => {
|
||||
expect(classifyMintQuoteObservation("ready")).toEqual({
|
||||
action: "finalize",
|
||||
observedRemoteState: "PAID",
|
||||
});
|
||||
});
|
||||
|
||||
it("finalizes an already-issued quote by restoring its proofs", () => {
|
||||
expect(classifyMintQuoteObservation("completed")).toEqual({
|
||||
action: "finalize",
|
||||
observedRemoteState: "ISSUED",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves an unpaid quote alone", () => {
|
||||
expect(classifyMintQuoteObservation("waiting")).toEqual({
|
||||
action: "waiting",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a quote the mint can no longer issue", () => {
|
||||
expect(classifyMintQuoteObservation("terminal")).toEqual({
|
||||
action: "terminal",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("selectMintQuotesForRecovery", () => {
|
||||
it("selects every pending quote because only the mint knows if it was paid", () => {
|
||||
const result = selectMintQuotesForRecovery({
|
||||
mints: [
|
||||
mint({ id: "expired", expiry: 1 }),
|
||||
mint({ id: "fresh" }),
|
||||
mint({ id: "observed-unpaid", lastObservedRemoteState: "UNPAID" }),
|
||||
mint({ id: "observed-paid", lastObservedRemoteState: "PAID" }),
|
||||
],
|
||||
});
|
||||
expect(result.pending.map((op) => op.id)).toEqual([
|
||||
"expired",
|
||||
"fresh",
|
||||
"observed-unpaid",
|
||||
"observed-paid",
|
||||
]);
|
||||
});
|
||||
|
||||
it("recovers executing operations left behind by a crash mid-mint", () => {
|
||||
const result = selectMintQuotesForRecovery({
|
||||
mints: [mint({ id: "executing", state: "executing" })],
|
||||
});
|
||||
expect(result.pending.map((op) => op.id)).toEqual(["executing"]);
|
||||
});
|
||||
|
||||
it("ignores finalized operations", () => {
|
||||
const result = selectMintQuotesForRecovery({
|
||||
mints: [mint({ id: "done", state: "finalized" })],
|
||||
});
|
||||
expect(result.pending).toEqual([]);
|
||||
expect(result.failed).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not re-open failed operations by default", () => {
|
||||
const result = selectMintQuotesForRecovery({
|
||||
mints: [mint({ id: "given-up", state: "failed" })],
|
||||
});
|
||||
expect(result.failed).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns failed operations when the caller opts in", () => {
|
||||
const result = selectMintQuotesForRecovery({
|
||||
mints: [mint({ id: "given-up", state: "failed" })],
|
||||
includeFailed: true,
|
||||
});
|
||||
expect(result.failed.map((op) => op.id)).toEqual(["given-up"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Pure helpers for PAID mint-quote recovery.
|
||||
*
|
||||
* A mint quote can be PAID at the mint while its local operation is still
|
||||
* `pending` (the Lightning payment landed before expiry while the daemon was
|
||||
* down, so no local observation was ever recorded) or even terminally
|
||||
* `failed` (coco gives up when the mint refuses to sign, for example after the
|
||||
* invoice expiry). Claimability still depends on the mint accepting issuance.
|
||||
* This feature retries the stored outputs or restores their signatures; it
|
||||
* does not regenerate outputs rejected by the mint (for example an inactive
|
||||
* keyset). coco already reconciles pending paid quotes at startup and in the
|
||||
* periodic sweep. The new capability is operator-targeted recovery, including
|
||||
* explicitly reopening failed operations, alongside safer cleanup.
|
||||
*
|
||||
* Recovery asks the mint what it thinks, then retries issuance or restore. These helpers decide *what* to do from a remote observation; the
|
||||
* actual state transitions are applied by the in-process coco wallet client
|
||||
* so coco-core's operation services emit their normal events and release
|
||||
* proof reservations. Keeping the decisions here makes them unit testable
|
||||
* without a wallet database or network access.
|
||||
*/
|
||||
|
||||
/** Subset of coco's mint operation rows that recovery needs. */
|
||||
export interface MintQuoteRecoveryCandidate {
|
||||
id: string;
|
||||
mintUrl: string;
|
||||
quoteId?: string;
|
||||
state: string;
|
||||
/** Quote amount in sats. */
|
||||
amount: number;
|
||||
/** Quote expiry in epoch seconds. `0` means unknown/not applicable. */
|
||||
expiry: number;
|
||||
/** Last quote state observed from the mint (UNPAID, PAID, ISSUED). */
|
||||
lastObservedRemoteState?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Coco's classification of a fresh remote quote check. */
|
||||
export type PendingMintCheckCategory =
|
||||
| "waiting"
|
||||
| "ready"
|
||||
| "completed"
|
||||
| "terminal";
|
||||
|
||||
/** What recovery should do with a quote after checking it with the mint. */
|
||||
export type MintQuoteRecoveryDecision =
|
||||
| { action: "finalize"; observedRemoteState: "PAID" | "ISSUED" }
|
||||
| { action: "waiting" }
|
||||
| { action: "terminal" };
|
||||
|
||||
/**
|
||||
* Map a remote quote check onto a recovery action.
|
||||
*
|
||||
* - `ready` means the mint reports the quote PAID but never issued: submit the
|
||||
* operation's stored outputs to claim the sats.
|
||||
* - `completed` means the mint already issued it: recover the signatures
|
||||
* (NUT-09) instead of minting again.
|
||||
* - `waiting` means the mint still reports the quote UNPAID: nothing is
|
||||
* claimable, so leave the operation alone.
|
||||
* - `terminal` means the quote can no longer be issued (for example the mint
|
||||
* refused an expired quote). coco persists that verdict as a failed
|
||||
* operation, so recovery must report it rather than treat it as progress.
|
||||
*/
|
||||
export function classifyMintQuoteObservation(
|
||||
category: PendingMintCheckCategory,
|
||||
): MintQuoteRecoveryDecision {
|
||||
switch (category) {
|
||||
case "ready":
|
||||
return { action: "finalize", observedRemoteState: "PAID" };
|
||||
case "completed":
|
||||
return { action: "finalize", observedRemoteState: "ISSUED" };
|
||||
case "waiting":
|
||||
return { action: "waiting" };
|
||||
case "terminal":
|
||||
return { action: "terminal" };
|
||||
}
|
||||
}
|
||||
|
||||
export interface MintQuoteRecoverySelectionOptions<
|
||||
T extends MintQuoteRecoveryCandidate,
|
||||
> {
|
||||
mints: T[];
|
||||
/**
|
||||
* Also consider terminally failed operations. Off by default: re-opening a
|
||||
* failed operation is a mutation, so only an explicit user-invoked recovery
|
||||
* may do it. Startup recovery must never resurrect quotes on its own.
|
||||
*/
|
||||
includeFailed?: boolean;
|
||||
}
|
||||
|
||||
export interface MintQuoteRecoverySelection<
|
||||
T extends MintQuoteRecoveryCandidate,
|
||||
> {
|
||||
/** Pending (or executing) operations that need a fresh mint observation. */
|
||||
pending: T[];
|
||||
/** Failed operations the caller may re-open and retry. */
|
||||
failed: T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Split operations into those recovery should check and those that were
|
||||
* already given up on.
|
||||
*
|
||||
* Every `pending` operation is selected: only the mint knows whether an
|
||||
* expired quote was paid before the local invoice ran out. `executing`
|
||||
* operations are recovered too, since a crash mid-mint leaves outputs that
|
||||
* may already be signed.
|
||||
*/
|
||||
export function selectMintQuotesForRecovery<
|
||||
T extends MintQuoteRecoveryCandidate,
|
||||
>(options: MintQuoteRecoverySelectionOptions<T>): MintQuoteRecoverySelection<T> {
|
||||
const { mints, includeFailed = false } = options;
|
||||
const pending = mints.filter(
|
||||
(op) => op.state === "pending" || op.state === "executing",
|
||||
);
|
||||
const failed = includeFailed
|
||||
? mints.filter((op) => op.state === "failed")
|
||||
: [];
|
||||
return { pending, failed };
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
/**
|
||||
* In-process Cashu mint for integration tests.
|
||||
*
|
||||
* Implements just enough of NUT-01/02/04/06/07/09 to drive a real coco
|
||||
* `Manager` against real HTTP: keyset publication, bolt11 mint quotes, minting
|
||||
* blinded outputs with a real secp256k1 blind signature, NUT-09 restore and
|
||||
* NUT-07 proof states. No Lightning, no network, no NPC.
|
||||
*
|
||||
* The signing keys and signatures are genuine (`@cashu/cashu-ts` mint-side
|
||||
* helpers), so a wallet that receives these signatures can unblind and verify
|
||||
* them exactly as with a production mint.
|
||||
*/
|
||||
import {
|
||||
createBlindSignature,
|
||||
createNewMintKeys,
|
||||
pointFromHex,
|
||||
} from "@cashu/cashu-ts";
|
||||
|
||||
/** NUT error codes used by the scenarios. */
|
||||
export const QUOTE_EXPIRED = 20007;
|
||||
export const ALREADY_ISSUED = 20002;
|
||||
|
||||
export type FakeQuoteState = "UNPAID" | "PAID" | "ISSUED";
|
||||
|
||||
export interface FakeMintQuote {
|
||||
quote: string;
|
||||
request: string;
|
||||
amount: number;
|
||||
unit: string;
|
||||
state: FakeQuoteState;
|
||||
/** Epoch seconds, or null for a quote that never expires. */
|
||||
expiry: number | null;
|
||||
amountPaid: number;
|
||||
amountIssued: number;
|
||||
pubkey: null;
|
||||
}
|
||||
|
||||
export interface FakeMintRequest {
|
||||
quote: string;
|
||||
outputs: Array<{ amount: number; id: string; B_: string }>;
|
||||
}
|
||||
|
||||
interface StoredSignature {
|
||||
amount: number;
|
||||
id: string;
|
||||
C_: string;
|
||||
}
|
||||
|
||||
const toHex = (bytes: Uint8Array) => Buffer.from(bytes).toString("hex");
|
||||
|
||||
export class FakeMint {
|
||||
readonly keysetId: string;
|
||||
readonly keysByAmount: Record<string, string>;
|
||||
readonly requests: FakeMintRequest[] = [];
|
||||
/** Every output the mint has ever signed, keyed by B_. */
|
||||
readonly signed = new Map<string, StoredSignature>();
|
||||
|
||||
/** When set, POST /v1/mint/bolt11 fails with this NUT error. */
|
||||
mintError: { code: number; detail: string } | null = null;
|
||||
/** When set, quote creation returns this expiry (epoch seconds). */
|
||||
quoteExpiry: number | null = 3_600;
|
||||
/**
|
||||
* Awaited before responding to a mint request, so a test can hold minting
|
||||
* open and interleave another recovery attempt.
|
||||
*/
|
||||
gate: Promise<void> | null = null;
|
||||
/** Awaited before answering a quote-state check, to hold observe open. */
|
||||
observeGate: Promise<void> | null = null;
|
||||
/**
|
||||
* When true, POST /v1/restore answers with NUT-09's spec-legal positional
|
||||
* arrays, including `null` for outputs the mint never signed.
|
||||
*
|
||||
* This is a known interop gap, not a supported path: cashu-ts 3.7.1 (which
|
||||
* coco depends on) dereferences every entry of `signatures` while normalising
|
||||
* amounts, so a `null` makes the wallet throw instead of skipping it. The
|
||||
* switch exists so tests keep that behaviour visible; see
|
||||
* mint-quote-recovery.fake-mint.test.ts.
|
||||
*/
|
||||
restoreIncludesNulls = false;
|
||||
|
||||
private readonly quotes = new Map<string, FakeMintQuote>();
|
||||
private counter = 0;
|
||||
private server?: ReturnType<typeof Bun.serve>;
|
||||
|
||||
constructor() {
|
||||
const pair = createNewMintKeys(20, new Uint8Array(32).fill(9));
|
||||
this.keysetId = pair.keysetId;
|
||||
this.keysByAmount = Object.fromEntries(
|
||||
Object.entries(pair.pubKeys).map(([amount, key]) => [
|
||||
amount,
|
||||
typeof key === "string" ? key : toHex(key),
|
||||
]),
|
||||
);
|
||||
this.privKeys = Object.fromEntries(
|
||||
Object.entries(pair.privKeys) as Array<[string, Uint8Array]>,
|
||||
);
|
||||
}
|
||||
|
||||
private readonly privKeys: Record<string, Uint8Array>;
|
||||
|
||||
get url(): string {
|
||||
if (!this.server) throw new Error("fake mint not started");
|
||||
return `http://127.0.0.1:${this.server.port}`;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
this.server = Bun.serve({
|
||||
hostname: "127.0.0.1",
|
||||
port: 0,
|
||||
fetch: (request) => this.handle(request),
|
||||
});
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.server?.stop(true);
|
||||
this.server = undefined;
|
||||
}
|
||||
|
||||
/** Test control: the mint sees the invoice as paid but has issued nothing. */
|
||||
markPaid(quoteId: string): void {
|
||||
const quote = this.quotes.get(quoteId);
|
||||
if (!quote) throw new Error(`unknown fake quote ${quoteId}`);
|
||||
quote.state = "PAID";
|
||||
quote.amountPaid = quote.amount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test control: mark the quote issued without going through the mint
|
||||
* endpoint, simulating a wallet that lost the signatures.
|
||||
*/
|
||||
markIssued(quoteId: string): void {
|
||||
const quote = this.quotes.get(quoteId);
|
||||
if (!quote) throw new Error(`unknown fake quote ${quoteId}`);
|
||||
quote.state = "ISSUED";
|
||||
quote.amountPaid = quote.amount;
|
||||
quote.amountIssued = quote.amount;
|
||||
}
|
||||
|
||||
/** Test control: sign outputs directly, as if another wallet had issued them. */
|
||||
signFor(quoteId: string, outputs: FakeMintRequest["outputs"]): void {
|
||||
for (const output of outputs) {
|
||||
this.signOutput(output);
|
||||
}
|
||||
this.markIssued(quoteId);
|
||||
}
|
||||
|
||||
getQuote(quoteId: string): FakeMintQuote | undefined {
|
||||
return this.quotes.get(quoteId);
|
||||
}
|
||||
|
||||
private quoteBody(quote: FakeMintQuote) {
|
||||
return {
|
||||
quote: quote.quote,
|
||||
request: quote.request,
|
||||
amount: quote.amount,
|
||||
unit: quote.unit,
|
||||
state: quote.state,
|
||||
expiry: quote.expiry,
|
||||
amount_paid: quote.amountPaid,
|
||||
amount_issued: quote.amountIssued,
|
||||
pubkey: quote.pubkey,
|
||||
};
|
||||
}
|
||||
|
||||
private signOutput(output: {
|
||||
amount: number;
|
||||
id: string;
|
||||
B_: string;
|
||||
}): StoredSignature {
|
||||
const privKey = this.privKeys[String(output.amount)];
|
||||
if (!privKey) {
|
||||
throw new Error(`fake mint has no key for amount ${output.amount}`);
|
||||
}
|
||||
const signature = createBlindSignature(
|
||||
pointFromHex(output.B_),
|
||||
privKey,
|
||||
this.keysetId,
|
||||
);
|
||||
const stored: StoredSignature = {
|
||||
amount: output.amount,
|
||||
id: this.keysetId,
|
||||
C_: toHex(signature.C_.toBytes(false)),
|
||||
};
|
||||
this.signed.set(output.B_, stored);
|
||||
return stored;
|
||||
}
|
||||
|
||||
private json(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
private error(code: number, detail: string, status = 400): Response {
|
||||
return this.json({ code, detail }, status);
|
||||
}
|
||||
|
||||
private async handle(request: Request): Promise<Response> {
|
||||
const { pathname } = new URL(request.url);
|
||||
const body = async () => {
|
||||
try {
|
||||
return (await request.json()) as Record<string, unknown>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
if (pathname === "/v1/info") {
|
||||
return this.json({
|
||||
name: "fake-mint",
|
||||
version: "0.0.1",
|
||||
nuts: {
|
||||
4: {
|
||||
methods: [
|
||||
{
|
||||
method: "bolt11",
|
||||
unit: "sat",
|
||||
min_amount: 1,
|
||||
max_amount: 1_000_000,
|
||||
},
|
||||
],
|
||||
},
|
||||
5: {
|
||||
methods: [
|
||||
{
|
||||
method: "bolt11",
|
||||
unit: "sat",
|
||||
min_amount: 1,
|
||||
max_amount: 1_000_000,
|
||||
},
|
||||
],
|
||||
},
|
||||
7: { supported: true },
|
||||
9: { supported: true },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (pathname === "/v1/keys" || pathname.startsWith("/v1/keys/")) {
|
||||
return this.json({
|
||||
keysets: [
|
||||
{ id: this.keysetId, unit: "sat", keys: this.keysByAmount },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
if (pathname === "/v1/keysets") {
|
||||
return this.json({
|
||||
keysets: [{ id: this.keysetId, unit: "sat", active: true }],
|
||||
});
|
||||
}
|
||||
|
||||
if (pathname === "/v1/mint/quote/bolt11" && request.method === "POST") {
|
||||
const input = await body();
|
||||
const amount = Number(input.amount);
|
||||
const unit = typeof input.unit === "string" ? input.unit : "sat";
|
||||
const quote: FakeMintQuote = {
|
||||
quote: `fake-quote-${++this.counter}`,
|
||||
request: `lnbcfake${this.counter}`,
|
||||
amount,
|
||||
unit,
|
||||
state: "UNPAID",
|
||||
expiry:
|
||||
this.quoteExpiry === null
|
||||
? null
|
||||
: Math.floor(Date.now() / 1000) + this.quoteExpiry,
|
||||
amountPaid: 0,
|
||||
amountIssued: 0,
|
||||
pubkey: null,
|
||||
};
|
||||
this.quotes.set(quote.quote, quote);
|
||||
return this.json(this.quoteBody(quote));
|
||||
}
|
||||
|
||||
const quoteMatch = pathname.match(/^\/v1\/mint\/quote\/bolt11\/(.+)$/);
|
||||
if (quoteMatch?.[1] && request.method === "GET") {
|
||||
if (this.observeGate) await this.observeGate;
|
||||
const quote = this.quotes.get(decodeURIComponent(quoteMatch[1]));
|
||||
if (!quote) return this.error(50000, "Unknown quote");
|
||||
return this.json(this.quoteBody(quote));
|
||||
}
|
||||
|
||||
if (pathname === "/v1/mint/bolt11" && request.method === "POST") {
|
||||
const input = await body();
|
||||
const quoteId = String(input.quote);
|
||||
const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"];
|
||||
this.requests.push({ quote: quoteId, outputs });
|
||||
if (this.gate) await this.gate;
|
||||
if (this.mintError) {
|
||||
return this.error(this.mintError.code, this.mintError.detail);
|
||||
}
|
||||
const quote = this.quotes.get(quoteId);
|
||||
if (!quote) return this.error(50000, "Unknown quote");
|
||||
if (quote.state === "ISSUED" || quote.amountIssued > 0) {
|
||||
return this.error(ALREADY_ISSUED, "Quote already issued");
|
||||
}
|
||||
if (quote.state !== "PAID") {
|
||||
return this.error(20001, "Quote is not paid");
|
||||
}
|
||||
const total = outputs.reduce((sum, o) => sum + Number(o.amount), 0);
|
||||
if (total > quote.amountPaid - quote.amountIssued) {
|
||||
return this.error(10002, "Outputs exceed the paid amount");
|
||||
}
|
||||
const signatures = outputs.map((output) => ({
|
||||
amount: output.amount,
|
||||
id: this.keysetId,
|
||||
C_: this.signOutput(output).C_,
|
||||
}));
|
||||
quote.state = "ISSUED";
|
||||
quote.amountIssued += total;
|
||||
return this.json({ signatures });
|
||||
}
|
||||
|
||||
if (pathname === "/v1/restore" && request.method === "POST") {
|
||||
const input = await body();
|
||||
const outputs = (input.outputs ?? []) as FakeMintRequest["outputs"];
|
||||
if (this.restoreIncludesNulls) {
|
||||
// Spec-legal NUT-09 shape, including nulls. Kept behind a switch
|
||||
// because it is currently unusable with coco's cashu-ts version.
|
||||
return this.json({
|
||||
outputs,
|
||||
signatures: outputs.map(
|
||||
(output) => this.signed.get(output.B_) ?? null,
|
||||
),
|
||||
});
|
||||
}
|
||||
// Return only the signed outputs; coco matches them by B_ and treats the
|
||||
// rest as "nothing to restore".
|
||||
const signed = outputs.filter((output) => this.signed.has(output.B_));
|
||||
return this.json({
|
||||
outputs: signed,
|
||||
signatures: signed.map((output) => this.signed.get(output.B_)),
|
||||
});
|
||||
}
|
||||
|
||||
if (pathname === "/v1/checkstate" && request.method === "POST") {
|
||||
const input = await body();
|
||||
const ys = (input.Ys ?? []) as string[];
|
||||
return this.json({
|
||||
states: ys.map((Y) => ({ Y, state: "UNSPENT", witness: null })),
|
||||
});
|
||||
}
|
||||
|
||||
return this.error(404, `fake mint has no route for ${pathname}`, 404);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user