mirror of
https://github.com/Routstr/routstrd.git
synced 2026-10-05 12:28:23 +00:00
redshift
2cdd8d745b
Add `routstrd wallet mints remove <url-or-index>` plus the daemon plumbing behind it: - GET /wallet/mints/removal-info reads local state (held sats including reserved proofs, pending mint quotes, in-flight melts, default flag) so the check works even when the mint is offline. - DELETE /wallet/mints removes a mint, refusing to remove the last one. - The CLI warns and asks for confirmation only when the mint still holds funds or pending quotes; an empty mint is removed without a prompt. - Removed mints are recorded in the wallet config and skipped by trusted-mint seeding, so a removed shipped mint does not reappear on restart. Adding it back clears the marker. - `wallet mints list` now renders a numbered list (with `--json` for the raw response). Proofs are intentionally left in place: deleting a mint never burns sats, and re-adding the mint restores access to them.
2831 lines
102 KiB
TypeScript
2831 lines
102 KiB
TypeScript
import { withTimeout as withRequestTimeout } from "../../utils/with-timeout";
|
|
import { recoveryKey, trackRecovery, drainRecoveryWork, waitForRecoveryWork, createRecoveryDisposer, type RecoveryWork } from "./recovery-work";
|
|
import {
|
|
Manager,
|
|
OperationInProgressError,
|
|
getEncodedToken,
|
|
normalizeMintUrl,
|
|
} from "@cashu/coco-core";
|
|
import type {
|
|
HistoryEntry,
|
|
ReceiveOperation,
|
|
Logger as CocoLogger,
|
|
Plugin as CocoPlugin,
|
|
} from "@cashu/coco-core";
|
|
import { SqliteRepositories } from "@cashu/coco-sqlite-bun";
|
|
import { Database } from "bun:sqlite";
|
|
import { NPCPlugin, type PluginApi as NpcPluginApi } from "coco-cashu-plugin-npc";
|
|
import { privateKeyFromSeedWords } from "nostr-tools/nip06";
|
|
import { finalizeEvent, nip19, type EventTemplate } from "nostr-tools";
|
|
import {
|
|
closeSync,
|
|
existsSync,
|
|
mkdirSync,
|
|
openSync,
|
|
readFileSync,
|
|
renameSync,
|
|
unlinkSync,
|
|
writeFileSync,
|
|
} from "fs";
|
|
import { dirname, join } from "path";
|
|
import { mnemonicToSeedSync } from "@scure/bip39";
|
|
import type {
|
|
WalletClient,
|
|
WalletRuntimeState,
|
|
NpcAddress,
|
|
NpcUsernameResult,
|
|
WalletCleanupOptions,
|
|
WalletCleanupResult,
|
|
WalletRecoveryProgress,
|
|
MintRemovalInfo,
|
|
} from "./wallet-client";
|
|
import { selectCleanupOperations, summarizeMintCleanup } from "./cleanup";
|
|
import {
|
|
classifyMintQuoteObservation,
|
|
selectMintQuotesForRecovery,
|
|
type MintQuoteRecoveryCandidate,
|
|
} from "./mint-quote-recovery";
|
|
import {
|
|
collectStuckOperations,
|
|
probeMintReachability,
|
|
runTargetedRecovery,
|
|
type SendRecoveryService,
|
|
type StuckOperation,
|
|
} from "./recovery-probe";
|
|
import {
|
|
clearInterruptedReceiveReservations,
|
|
deleteReceiveTokenReservation,
|
|
getReceiveReconcileBackup,
|
|
initReceiveDedupSchema,
|
|
listProcessingReceiveTokens,
|
|
receiveInputFingerprint,
|
|
reconcileExecutingReceives,
|
|
releaseReceiveToken,
|
|
reserveReceiveToken,
|
|
setReceiveReconcileBackup,
|
|
updateReceiveToken,
|
|
type ReceiveReconcileSource,
|
|
} from "./receive-dedup";
|
|
import { cocoLogger, logger } from "../../utils/logger";
|
|
import {
|
|
legacyCocodPidPath,
|
|
legacyCocodSocketPath,
|
|
walletDir as defaultWalletDir,
|
|
walletPidPath as defaultWalletPidPath,
|
|
} from "./paths";
|
|
import { DEFAULT_MINT_URL, seedTrustedMints } from "./trusted-mints";
|
|
|
|
export { DEFAULT_MINT_URL, DEFAULT_TRUSTED_MINT_URLS } from "./trusted-mints";
|
|
|
|
const NPC_DEFAULT_BASE_URL = "https://npubx.cash";
|
|
|
|
const STALE_SOCKET_ERROR_CODES = new Set([
|
|
"ECONNREFUSED",
|
|
"ENOENT",
|
|
// Bun's Unix-socket fetch error for an abandoned socket inode.
|
|
"FailedToOpenSocket",
|
|
]);
|
|
|
|
type UnixRequestInit = RequestInit & { unix: string };
|
|
type LegacyCocodFetch = (
|
|
input: string | URL | Request,
|
|
init: UnixRequestInit,
|
|
) => Promise<Response>;
|
|
|
|
export interface LegacyCocodGuardOptions {
|
|
socketPath?: string;
|
|
pathExists?: (path: string) => boolean;
|
|
fetchImpl?: LegacyCocodFetch;
|
|
timeoutMs?: number;
|
|
}
|
|
|
|
export interface LegacyCocodPidClaimOptions {
|
|
pidFilePath?: string;
|
|
pid?: number;
|
|
/** Human-readable lock name used in contention errors. */
|
|
label?: string;
|
|
openExclusive?: (path: string) => number;
|
|
writePid?: (fd: number, pid: number) => void;
|
|
closeFile?: (fd: number) => void;
|
|
readFile?: (path: string) => string;
|
|
removeFile?: (path: string) => void;
|
|
isProcessRunning?: (pid: number) => boolean;
|
|
}
|
|
|
|
export interface LegacyCocodStopOptions {
|
|
socketPath?: string;
|
|
pidFilePath?: string;
|
|
pathExists?: (path: string) => boolean;
|
|
readFile?: (path: string) => string;
|
|
isProcessRunning?: (pid: number) => boolean;
|
|
fetchImpl?: LegacyCocodFetch;
|
|
killProcess?: (pid: number, signal: NodeJS.Signals) => void;
|
|
/** Total time to wait for cocod to exit after SIGTERM. */
|
|
timeoutMs?: number;
|
|
/** Interval between exit checks. */
|
|
pollIntervalMs?: number;
|
|
/** Timeout for identifying cocod through its Unix socket. */
|
|
socketTimeoutMs?: number;
|
|
}
|
|
|
|
interface CocodConfig {
|
|
mnemonic: string;
|
|
encrypted: boolean;
|
|
defaultMintUrl?: string;
|
|
/**
|
|
* Mint URLs the user removed from the wallet. Trusted-mint seeding skips
|
|
* these so a removed shipped mint does not reappear on the next restart.
|
|
* Adding a mint again clears its entry.
|
|
*/
|
|
removedMintUrls?: string[];
|
|
}
|
|
|
|
const STARTUP_LOG_PREFIX = "[routstrd:start]";
|
|
|
|
function startupProgress(message: string): void {
|
|
logger.info(message);
|
|
// The daemon is detached and stdout is captured by start-daemon.ts. The
|
|
// prefix lets the CLI surface only safe, user-facing startup progress while
|
|
// the full diagnostic stream remains in the normal log file.
|
|
console.log(`${STARTUP_LOG_PREFIX} ${message}`);
|
|
}
|
|
|
|
// Set only while the background wallet recovery sweeps are running. While set,
|
|
// coco logger messages that indicate a stalled/failed per-mint check are
|
|
// forwarded to the startup stream (see createCocoLogger).
|
|
let surfacingRecoveryProgress = false;
|
|
|
|
// These fire once per operation when a mint is unreachable or otherwise fails
|
|
// to reconcile. They are the only per-mint signal coco emits, and they happen
|
|
// exactly when recovery is slow.
|
|
const RECOVERY_STALL_MARKERS = new Map<string, string>([
|
|
["SendOperationService\u0000Could not reach mint for recovery, will retry later", "Send recovery: mint unreachable"],
|
|
["ReceiveOperationService\u0000Could not reach mint for receive recovery, will retry later", "Receive recovery: mint unreachable"],
|
|
["MintOperationService\u0000Failed to reconcile stale pending mint operation", "Mint recovery: pending operation check failed"],
|
|
]);
|
|
|
|
const SAFE_COCO_LOG_FIELDS = new Set([
|
|
"module",
|
|
"mintUrl",
|
|
"operationId",
|
|
"quoteId",
|
|
"state",
|
|
"code",
|
|
"detail",
|
|
"status",
|
|
"count",
|
|
"total",
|
|
"filterCount",
|
|
"subId",
|
|
"initOperations",
|
|
"executingOperations",
|
|
"pendingOperations",
|
|
"rollingBackOperations",
|
|
"orphanedReservations",
|
|
]);
|
|
|
|
function safeCocoMetadata(values: unknown[]): Record<string, unknown> {
|
|
const safe: Record<string, unknown> = {};
|
|
for (const value of values) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
|
|
for (const [key, fieldValue] of Object.entries(value)) {
|
|
if (SAFE_COCO_LOG_FIELDS.has(key)) safe[key] = fieldValue;
|
|
}
|
|
}
|
|
return safe;
|
|
}
|
|
|
|
function createCocoLogger(bindings: Record<string, unknown> = {}): CocoLogger {
|
|
const write = (
|
|
level: "error" | "warn" | "info" | "debug",
|
|
message: string,
|
|
meta: unknown[],
|
|
) => {
|
|
// Coco diagnostics may contain proof secrets or encoded tokens. Keep only
|
|
// an explicit metadata allowlist; startup counts and operation IDs remain
|
|
// useful without copying wallet material into routstrd's logs. Written to
|
|
// ~/.routstrd/coco-logs/ so wallet-engine noise stays out of the main logs.
|
|
const metadata = safeCocoMetadata([bindings, ...meta]);
|
|
|
|
if (surfacingRecoveryProgress) {
|
|
const module = typeof bindings.module === "string" ? bindings.module : undefined;
|
|
if (module) {
|
|
const stall = RECOVERY_STALL_MARKERS.get(`${module}\u0000${message}`);
|
|
if (stall) {
|
|
const mintUrl =
|
|
typeof metadata.mintUrl === "string" ? metadata.mintUrl : undefined;
|
|
startupProgress(mintUrl ? `${stall} (${mintUrl})` : stall);
|
|
}
|
|
}
|
|
}
|
|
|
|
cocoLogger[level](
|
|
`[coco] ${message}`,
|
|
...(Object.keys(metadata).length > 0 ? [metadata] : []),
|
|
);
|
|
};
|
|
|
|
return {
|
|
error: (message, ...meta) => write("error", message, meta),
|
|
warn: (message, ...meta) => write("warn", message, meta),
|
|
info: (message, ...meta) => write("info", message, meta),
|
|
debug: (message, ...meta) => write("debug", message, meta),
|
|
log: (level, message, ...meta) => write(level, message, meta),
|
|
child: (childBindings) =>
|
|
createCocoLogger({ ...bindings, ...childBindings }),
|
|
};
|
|
}
|
|
|
|
function loadConfig(configFile: string): CocodConfig {
|
|
if (!existsSync(configFile)) {
|
|
throw new Error(
|
|
`Config file not found at ${configFile}. Run 'routstrd onboard' first.`,
|
|
);
|
|
}
|
|
const config = JSON.parse(readFileSync(configFile, "utf-8")) as CocodConfig;
|
|
if (config.encrypted) {
|
|
throw new Error(
|
|
"Encrypted wallets are not supported yet. Please use an unencrypted wallet.",
|
|
);
|
|
}
|
|
return config;
|
|
}
|
|
|
|
function saveConfig(config: CocodConfig, configFile: string): void {
|
|
const temporaryFile = `${configFile}.${process.pid}.tmp`;
|
|
try {
|
|
writeFileSync(temporaryFile, JSON.stringify(config, null, 2), {
|
|
mode: 0o600,
|
|
flag: "wx",
|
|
});
|
|
renameSync(temporaryFile, configFile);
|
|
} catch (error) {
|
|
try {
|
|
unlinkSync(temporaryFile);
|
|
} catch {
|
|
// The temporary file may not have been created.
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// Config mint URLs are written by routstrd, but a hand-edited file must not
|
|
// crash startup. Normalization only strips a default port and trailing slash.
|
|
function configMintUrl(mintUrl: string): string {
|
|
try {
|
|
return normalizeMintUrl(mintUrl);
|
|
} catch {
|
|
return mintUrl.trim();
|
|
}
|
|
}
|
|
|
|
/** Record a removed mint so trusted-mint seeding will not re-add it. */
|
|
function markMintRemoved(config: CocodConfig, mintUrl: string): boolean {
|
|
const url = configMintUrl(mintUrl);
|
|
const removed = new Set((config.removedMintUrls ?? []).map(configMintUrl));
|
|
if (removed.has(url)) return false;
|
|
removed.add(url);
|
|
config.removedMintUrls = [...removed];
|
|
return true;
|
|
}
|
|
|
|
/** Clear a mint's removed marker because the user added it back. */
|
|
function clearMintRemoved(config: CocodConfig, mintUrl: string): boolean {
|
|
const url = configMintUrl(mintUrl);
|
|
const current = config.removedMintUrls ?? [];
|
|
const remaining = current.filter((entry) => configMintUrl(entry) !== url);
|
|
if (remaining.length === current.length) return false;
|
|
config.removedMintUrls = remaining;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Remove a mint's record and keysets from the wallet database. The public
|
|
* `MintApi` only exposes trust changes, so reach the underlying `MintService`
|
|
* structurally, as this module already does for other coco internals. Stored
|
|
* proofs are intentionally left untouched: deleting them would burn sats, and
|
|
* re-adding the mint restores access to them.
|
|
*/
|
|
async function deleteMintFromWallet(coco: Manager, mintUrl: string): Promise<void> {
|
|
const service = (
|
|
coco as unknown as {
|
|
mintService?: { deleteMint?: (url: string) => Promise<void> };
|
|
}
|
|
).mintService;
|
|
if (!service?.deleteMint) {
|
|
throw new Error("Wallet backend does not support removing mints");
|
|
}
|
|
await service.deleteMint(mintUrl);
|
|
}
|
|
|
|
/** Pending top-up (mint) quotes for one mint, read from local state. */
|
|
async function countPendingMintQuotes(
|
|
coco: Manager,
|
|
mintUrl: string,
|
|
): Promise<number> {
|
|
try {
|
|
const pending = await coco.ops.mint.listPending();
|
|
return pending.filter((op) => configMintUrl(op.mintUrl) === mintUrl).length;
|
|
} catch (error) {
|
|
logger.warn("Could not read pending mint quotes while inspecting a mint", {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/** Prepared or in-flight outbound (melt) payments for one mint. */
|
|
async function countPendingMeltQuotes(
|
|
coco: Manager,
|
|
mintUrl: string,
|
|
): Promise<number> {
|
|
try {
|
|
const [prepared, inFlight] = await Promise.all([
|
|
coco.ops.melt.listPrepared(),
|
|
coco.ops.melt.listInFlight(),
|
|
]);
|
|
const seen = new Set<string>();
|
|
for (const op of [...prepared, ...inFlight]) {
|
|
if (configMintUrl(op.mintUrl) !== mintUrl) continue;
|
|
seen.add(op.id);
|
|
}
|
|
return seen.size;
|
|
} catch (error) {
|
|
logger.warn("Could not read pending melt quotes while inspecting a mint", {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
export function isZombieProcess(
|
|
pid: number,
|
|
readFile: (path: string) => string = (path) =>
|
|
readFileSync(path, "utf-8"),
|
|
): boolean {
|
|
try {
|
|
// Linux exposes zombie state as the character following the final `)` in
|
|
// /proc/<pid>/stat. Use the final parenthesis because process names may
|
|
// themselves contain spaces or parentheses. Other platforms simply fall
|
|
// back to process.kill(pid, 0) below.
|
|
const stat = readFile(`/proc/${pid}/stat`);
|
|
const commandEnd = stat.lastIndexOf(")");
|
|
return commandEnd >= 0 && stat.charAt(commandEnd + 2) === "Z";
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Test whether a PID is alive; dead-but-unreaped zombies count as dead. */
|
|
export function defaultIsProcessRunning(pid: number): boolean {
|
|
try {
|
|
process.kill(pid, 0);
|
|
} catch (error) {
|
|
return (error as NodeJS.ErrnoException).code === "EPERM";
|
|
}
|
|
|
|
// kill(pid, 0) also succeeds for dead-but-unreaped processes. Zombies hold
|
|
// no database or socket resources, so treating them as dead allows stale
|
|
// wallet locks to be reclaimed (notably under non-reaping Docker PID 1s).
|
|
return !isZombieProcess(pid);
|
|
}
|
|
|
|
function hasErrorCode(error: unknown, codes: Set<string>): boolean {
|
|
let current: unknown = error;
|
|
const visited = new Set<unknown>();
|
|
|
|
while (current && typeof current === "object" && !visited.has(current)) {
|
|
visited.add(current);
|
|
const candidate = current as { code?: unknown; cause?: unknown };
|
|
if (typeof candidate.code === "string" && codes.has(candidate.code)) {
|
|
return true;
|
|
}
|
|
current = candidate.cause;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Refuse to open coco.db while a daemon answers on legacy cocod's Unix socket.
|
|
* Two independent wallet engines must never operate on the same proof database.
|
|
*
|
|
* cocod.pid is deliberately shared: routstrd writes its own PID there while
|
|
* the in-process wallet is open, fencing old cocod binaries from starting. A
|
|
* live PID in that file therefore cannot identify cocod. Socket responsiveness
|
|
* is the authoritative identity check; the atomic PID-file claim below closes
|
|
* the race when cocod is still starting and has not opened its socket yet.
|
|
*
|
|
* A socket left behind after a crash is safe to ignore only when connecting
|
|
* fails with ENOENT or ECONNREFUSED. Other probe failures are treated as unsafe
|
|
* because they do not prove that cocod has stopped.
|
|
*/
|
|
export async function assertLegacyCocodNotRunning(
|
|
options: LegacyCocodGuardOptions = {},
|
|
): Promise<void> {
|
|
const socketPath = options.socketPath || legacyCocodSocketPath();
|
|
const pathExists = options.pathExists || existsSync;
|
|
|
|
if (!pathExists(socketPath)) return;
|
|
|
|
const fetchImpl = options.fetchImpl || (fetch as LegacyCocodFetch);
|
|
const timeoutMs = options.timeoutMs ?? 1_000;
|
|
|
|
try {
|
|
const response = await fetchImpl("http://localhost/ping", {
|
|
unix: socketPath,
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
});
|
|
await response.body?.cancel();
|
|
} catch (error) {
|
|
if (hasErrorCode(error, STALE_SOCKET_ERROR_CODES)) {
|
|
logger.debug(`Ignoring stale legacy cocod socket at ${socketPath}`);
|
|
return;
|
|
}
|
|
|
|
throw new Error(
|
|
`Cannot verify whether the legacy cocod daemon has stopped at ${socketPath}. ` +
|
|
"Refusing to open the wallet database to prevent concurrent access. " +
|
|
"Run 'cocod stop', verify the daemon has exited, and try again.",
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
throw new Error(
|
|
`Legacy cocod daemon is still running at ${socketPath}. ` +
|
|
"Refusing to open the wallet database because cocod and coco-core cannot safely use it at the same time. " +
|
|
"Run 'cocod stop' and try again.",
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Gracefully stop a legacy cocod daemon that is still running, so the new
|
|
* in-process coco wallet can safely open the shared database.
|
|
*
|
|
* Sends SIGTERM to the PID recorded in cocod's PID file, then polls until the
|
|
* process exits and the PID file is removed (cocod cleans up both on graceful
|
|
* shutdown). On timeout it refuses rather than escalating to SIGKILL, because
|
|
* killing a wallet engine mid-proof-recovery risks corrupting coco.db — the
|
|
* exact failure the guard exists to prevent.
|
|
*/
|
|
export async function stopLegacyCocod(
|
|
options: LegacyCocodStopOptions = {},
|
|
): Promise<void> {
|
|
const socketPath = options.socketPath || legacyCocodSocketPath();
|
|
const pidFilePath = options.pidFilePath || legacyCocodPidPath();
|
|
const pathExists = options.pathExists || existsSync;
|
|
const readFile =
|
|
options.readFile || ((path: string) => readFileSync(path, "utf-8"));
|
|
const isProcessRunning = options.isProcessRunning || defaultIsProcessRunning;
|
|
const fetchImpl = options.fetchImpl || (fetch as LegacyCocodFetch);
|
|
const killProcess =
|
|
options.killProcess || ((pid, signal) => process.kill(pid, signal));
|
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
const pollIntervalMs = options.pollIntervalMs ?? 500;
|
|
const socketTimeoutMs = options.socketTimeoutMs ?? 1_000;
|
|
|
|
const readPid = (): number | null => {
|
|
if (!pathExists(pidFilePath)) return null;
|
|
try {
|
|
const pid = Number.parseInt(readFile(pidFilePath).trim(), 10);
|
|
return Number.isInteger(pid) && pid > 0 && isProcessRunning(pid)
|
|
? pid
|
|
: null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
|
|
const pid = readPid();
|
|
if (pid === null) {
|
|
logger.debug(
|
|
"stopLegacyCocod: no running legacy cocod found, nothing to stop.",
|
|
);
|
|
return;
|
|
}
|
|
|
|
// routstrd intentionally writes its own PID to cocod.pid while the in-process
|
|
// wallet is open. Never identify the owner from the shared PID file alone:
|
|
// only a process responding through cocod's Unix socket is safe to terminate.
|
|
if (!pathExists(socketPath)) {
|
|
logger.debug(
|
|
`PID ${pid} owns ${pidFilePath}, but no legacy cocod socket exists; leaving it running.`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetchImpl("http://localhost/ping", {
|
|
unix: socketPath,
|
|
signal: AbortSignal.timeout(socketTimeoutMs),
|
|
});
|
|
await response.body?.cancel();
|
|
} catch (error) {
|
|
if (hasErrorCode(error, STALE_SOCKET_ERROR_CODES)) {
|
|
logger.debug(
|
|
`PID ${pid} owns ${pidFilePath}, but the legacy cocod socket is stale; leaving it running.`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
throw new Error(
|
|
`Cannot verify whether PID ${pid} is the legacy cocod daemon at ${socketPath}. ` +
|
|
"Refusing to stop an unidentified process.",
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
logger.log(`Stopping legacy cocod daemon (PID ${pid})…`);
|
|
killProcess(pid, "SIGTERM");
|
|
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
if (!isProcessRunning(pid) || readPid() !== pid) {
|
|
logger.log(`Legacy cocod daemon (PID ${pid}) stopped.`);
|
|
return;
|
|
}
|
|
}
|
|
|
|
throw new Error(
|
|
`Legacy cocod daemon (PID ${pid}) did not stop within ${Math.round(
|
|
timeoutMs / 1000,
|
|
)}s of SIGTERM. ` + `Run 'kill ${pid}' and try again.`,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Atomically claim cocod's PID file for the lifetime of the in-process wallet.
|
|
* Legacy cocod checks this same file before opening coco.db, so a live routstrd
|
|
* owner prevents cocod from starting after the initial socket/PID probe.
|
|
*/
|
|
export function claimLegacyCocodPidFile(
|
|
options: LegacyCocodPidClaimOptions = {},
|
|
): () => void {
|
|
return claimPidFile({
|
|
...options,
|
|
pidFilePath: options.pidFilePath || legacyCocodPidPath(),
|
|
label: options.label || "legacy cocod exclusion lock",
|
|
});
|
|
}
|
|
|
|
function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: string }): () => void {
|
|
const pidFilePath = options.pidFilePath;
|
|
const pid = options.pid ?? process.pid;
|
|
const label = options.label || "wallet process lock";
|
|
const openExclusive =
|
|
options.openExclusive || ((path: string) => openSync(path, "wx", 0o600));
|
|
const writePid =
|
|
options.writePid ||
|
|
((fd: number, ownerPid: number) => writeFileSync(fd, String(ownerPid)));
|
|
const closeFile = options.closeFile || closeSync;
|
|
const readFile =
|
|
options.readFile || ((path: string) => readFileSync(path, "utf-8"));
|
|
const removeFile = options.removeFile || unlinkSync;
|
|
const isProcessRunning = options.isProcessRunning || defaultIsProcessRunning;
|
|
|
|
let fd: number;
|
|
try {
|
|
fd = openExclusive(pidFilePath);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
|
|
// The earlier guard permits a dead PID file. Remove only a parseable,
|
|
// confirmed-dead owner; an empty/malformed file may belong to a process
|
|
// that has created the file but has not written its PID yet.
|
|
let stalePid: number;
|
|
try {
|
|
stalePid = Number.parseInt(readFile(pidFilePath).trim(), 10);
|
|
} catch {
|
|
throw new Error(
|
|
`Cannot claim the ${label} at ${pidFilePath}. ` +
|
|
"Another cocod or routstrd process may be starting. Stop it and try again.",
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
if (
|
|
!Number.isInteger(stalePid) ||
|
|
stalePid <= 0 ||
|
|
isProcessRunning(stalePid)
|
|
) {
|
|
const ownerMessage =
|
|
Number.isInteger(stalePid) && stalePid > 0
|
|
? `PID ${stalePid} is still running and holds it. ` +
|
|
`Stop that process first ('routstrd stop' or 'kill ${stalePid}').`
|
|
: "Another cocod or routstrd process may be starting. Stop it and try again.";
|
|
throw new Error(
|
|
`Cannot claim the ${label} at ${pidFilePath}: ${ownerMessage}`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
try {
|
|
removeFile(pidFilePath);
|
|
fd = openExclusive(pidFilePath);
|
|
} catch (retryError) {
|
|
throw new Error(
|
|
`Cannot claim the ${label} at ${pidFilePath}. ` +
|
|
"Another cocod or routstrd process may be starting. Stop it and try again.",
|
|
{ cause: retryError },
|
|
);
|
|
}
|
|
}
|
|
|
|
try {
|
|
writePid(fd, pid);
|
|
} catch (error) {
|
|
try {
|
|
removeFile(pidFilePath);
|
|
} catch {
|
|
// Preserve the original write failure.
|
|
}
|
|
throw error;
|
|
} finally {
|
|
closeFile(fd);
|
|
}
|
|
|
|
let released = false;
|
|
const release = () => {
|
|
if (released) return;
|
|
released = true;
|
|
process.removeListener("exit", release);
|
|
|
|
try {
|
|
if (readFile(pidFilePath).trim() === String(pid)) {
|
|
removeFile(pidFilePath);
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
logger.warn(
|
|
`Failed to release ${label} at ${pidFilePath}:`,
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
};
|
|
|
|
// process.exit() and natural shutdown still run synchronous exit handlers.
|
|
// This prevents migration or startup failures from stranding our PID files.
|
|
process.once("exit", release);
|
|
return release;
|
|
}
|
|
|
|
/**
|
|
* Minimal structural view of coco-core's MintOperationService.
|
|
* The service is private on the exported Manager class, so the in-process
|
|
* client reaches it through this narrow cast. Both methods reload the latest
|
|
* persisted row before mutating anything, so a bare operation id is enough.
|
|
*/
|
|
interface MintOperationServiceCleanup {
|
|
failPendingOperation(
|
|
op: { id: string },
|
|
terminalFailure: { reason: string; retryable?: boolean; observedAt: number },
|
|
): Promise<unknown>;
|
|
/**
|
|
* Ask the mint for a pending quote's current state and persist the
|
|
* observation. "waiting" means the mint still reports the quote as unpaid.
|
|
*/
|
|
observePendingOperation(
|
|
operationId: string,
|
|
): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>;
|
|
/**
|
|
* Acquire coco's per-operation lock for `operationId` and return its release
|
|
* function. coco's execute/finalize/recover paths take the same lock, so
|
|
* holding it across a read-check-write makes the transition atomic with
|
|
* respect to them.
|
|
*/
|
|
acquireOperationLock(operationId: string): Promise<() => void>;
|
|
/** Reload a mint operation row, or null when it no longer exists. */
|
|
getOperation(operationId: string): Promise<Record<string, unknown> | null>;
|
|
/**
|
|
* Put a terminally failed operation back into `pending`.
|
|
*
|
|
* coco keeps this private, and it spreads whatever it is handed into the row
|
|
* it writes. The sqlite repository rewrites every column, so callers MUST
|
|
* pass a freshly reloaded full row: a partial object such as `{ id }` would
|
|
* erase `outputDataJson` and make the paid sats unrecoverable.
|
|
*/
|
|
transitionToPending(
|
|
op: Record<string, unknown>,
|
|
error?: string,
|
|
): Promise<unknown>;
|
|
}
|
|
|
|
/**
|
|
* Re-open a terminally failed mint operation so recovery can retry it.
|
|
*
|
|
* Two details make this safe:
|
|
*
|
|
* - The persisted row is reloaded and handed to coco in full. coco spreads
|
|
* whatever it is given and the sqlite repository rewrites every column, so a
|
|
* partial object would be rejected by the NOT NULL schema or, on a more
|
|
* permissive adapter, erase the stored outputs.
|
|
* - The read-check-write runs under coco's per-operation lock, the same lock
|
|
* coco's execute/finalize/recover paths take. Reloading alone only narrows
|
|
* the race: without the lock two concurrent recoveries could both see
|
|
* `failed` and the slower one would clobber a newer state.
|
|
*
|
|
* The lock is fail-fast rather than wait-based: coco's `OperationIdLock.acquire`
|
|
* throws `OperationInProgressError` when the id is already locked. So either
|
|
* this helper holds the lock - and coco's own execute/finalize/recover paths
|
|
* cannot interleave, because acquiring would throw for them too - or it throws
|
|
* and writes nothing. It never waits, and never writes without the lock, which
|
|
* is why a stale `failed` snapshot cannot clobber a newer state.
|
|
*
|
|
* Scope of that lock, in this coco version: `recordPendingObservation` and
|
|
* `failPendingOperation` write without taking it. The justified claim is
|
|
* therefore narrow - a re-open cannot clobber a concurrent executing/recovery
|
|
* pass - not a general guarantee against every watcher write.
|
|
*
|
|
* This is a compatibility shim over private coco internals, so it fails closed:
|
|
* if any of the expected methods are missing it throws before writing. That
|
|
* check only catches removals, not changed behaviour under the same name: it
|
|
* was written against @cashu/coco-core 1.0.1, so any coco bump must re-run the
|
|
* real-Manager and fake-mint integration tests. The long-term fix is an
|
|
* upstream public `reopenFailedOperation(id)` that takes the same lock, reloads
|
|
* the full row, preserves the outputs and emits the usual events.
|
|
*/
|
|
export async function reopenFailedMintOperation(
|
|
service: Pick<
|
|
MintOperationServiceCleanup,
|
|
"acquireOperationLock" | "getOperation" | "transitionToPending"
|
|
>,
|
|
operationId: string,
|
|
): Promise<boolean> {
|
|
for (const method of [
|
|
"acquireOperationLock",
|
|
"getOperation",
|
|
"transitionToPending",
|
|
] as const) {
|
|
if (typeof service[method] !== "function") {
|
|
throw new Error(
|
|
`coco mintOperationService.${method} is unavailable; refusing to re-open a failed mint operation`,
|
|
);
|
|
}
|
|
}
|
|
const release = await service.acquireOperationLock(operationId);
|
|
try {
|
|
const current = await service.getOperation(operationId);
|
|
if (!current) throw new Error(`Operation ${operationId} not found`);
|
|
if (current.state !== "failed") return false;
|
|
// Clearing the terminal-failure marker keeps the re-opened row from
|
|
// looking terminally failed to readers that inspect it alongside `state`.
|
|
await service.transitionToPending(
|
|
{ ...current, terminalFailure: undefined },
|
|
undefined,
|
|
);
|
|
return true;
|
|
} finally {
|
|
release();
|
|
}
|
|
}
|
|
|
|
export interface CreateCocoClientOptions {
|
|
/** Override the canonical wallet data directory. */
|
|
walletDir?: string;
|
|
/** Deprecated alias retained for existing callers during migration. */
|
|
configDir?: string;
|
|
/** Override the in-process wallet lock path. */
|
|
walletPidPath?: string;
|
|
/** Override legacy external-cocod coordination paths. */
|
|
legacySocketPath?: string;
|
|
legacyPidPath?: string;
|
|
/** Set to false to skip NPC (npubx.cash) plugin registration. Default: true. */
|
|
enableNpc?: boolean;
|
|
/** NPC server base URL. Default: https://npubx.cash */
|
|
npcBaseUrl?: string;
|
|
}
|
|
|
|
/**
|
|
* Build a usable coco Manager without running the blocking recovery sweeps.
|
|
*
|
|
* This replicates `initializeCoco()` up to (but not including) the send/melt/
|
|
* receive/mint recovery passes, so the daemon can serve wallet reads while
|
|
* recovery proceeds in the background.
|
|
*/
|
|
function constructCocoManager(
|
|
repo: SqliteRepositories,
|
|
seed: Uint8Array,
|
|
): Manager {
|
|
return new Manager(repo, async () => seed, createCocoLogger());
|
|
}
|
|
|
|
async function enableCocoManager(coco: Manager): Promise<void> {
|
|
await coco.initPlugins();
|
|
await coco.reconcileLegacyMintQuotes();
|
|
await coco.enableMintOperationWatcher();
|
|
await coco.enableProofStateWatcher();
|
|
await coco.enableMintOperationProcessor();
|
|
}
|
|
|
|
/**
|
|
* Shared wall-clock budget for checking expired mint quotes with their mints
|
|
* during background recovery. coco-core issues mint requests without a
|
|
* timeout, so a hung mint could otherwise stall this phase (and with it the
|
|
* recovery promise that gates value-moving operations) far longer than this.
|
|
*/
|
|
const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000;
|
|
|
|
/** Rejects when `timeoutMs` elapses before `promise` settles. */
|
|
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
if (timeoutMs === Infinity) return promise;
|
|
return withRequestTimeout(promise, timeoutMs, "Timed out contacting mint");
|
|
}
|
|
|
|
/** Structural subset of coco's Manager used by expired-quote settlement. */
|
|
export interface ExpiredMintQuoteSource {
|
|
ops: {
|
|
mint: {
|
|
listPending(): Promise<
|
|
Array<{
|
|
id: string;
|
|
mintUrl: string;
|
|
quoteId?: string;
|
|
state: string;
|
|
/** Quote expiry in epoch seconds. */
|
|
expiry: number;
|
|
updatedAt: number;
|
|
lastObservedRemoteState?: string;
|
|
}>
|
|
>;
|
|
};
|
|
};
|
|
mintOperationService: MintOperationServiceCleanup;
|
|
}
|
|
|
|
export interface ExpiredMintSettlement {
|
|
/** Quotes their mint confirmed as UNPAID, failed locally. */
|
|
failed: number;
|
|
/** Quotes observed as PAID/ISSUED, left for mint recovery to finalize. */
|
|
leftForRecovery: number;
|
|
/** Quotes whose mint could not be checked in time, left pending. */
|
|
unobserved: number;
|
|
}
|
|
|
|
/** Outcome of asking a mint about one expired pending quote. */
|
|
export type ExpiredMintQuoteOutcome =
|
|
| "failed"
|
|
| "leftForRecovery"
|
|
| "unobserved";
|
|
|
|
/**
|
|
* Decide one expired pending quote's fate by asking the mint.
|
|
*
|
|
* Expiry alone does not prove the quote was never paid: the Lightning payment
|
|
* can land just before expiry while the daemon is down, leaving no local
|
|
* observation. A quote the mint still reports UNPAID can never be issued and
|
|
* is safe to fail locally; anything else (PAID/ISSUED, or a mint that cannot
|
|
* answer) stays pending so recovery can still claim the sats.
|
|
*/
|
|
export async function failExpiredMintQuoteIfUnpaid(
|
|
mintService: Pick<
|
|
MintOperationServiceCleanup,
|
|
"observePendingOperation" | "failPendingOperation"
|
|
>,
|
|
operationId: string,
|
|
timeoutMs: number,
|
|
): Promise<{
|
|
outcome: ExpiredMintQuoteOutcome;
|
|
category?: "waiting" | "ready" | "completed" | "terminal";
|
|
error?: unknown;
|
|
}> {
|
|
try {
|
|
const observation = await withTimeout(
|
|
mintService.observePendingOperation(operationId),
|
|
timeoutMs,
|
|
);
|
|
if (observation.category !== "waiting") {
|
|
return { outcome: "leftForRecovery", category: observation.category };
|
|
}
|
|
// The mint confirms the expired quote is still unpaid: it can never be
|
|
// issued now, so failing it locally cannot strand funds.
|
|
await mintService.failPendingOperation(
|
|
{ id: operationId },
|
|
{
|
|
reason: "Expired mint quote confirmed unpaid by mint",
|
|
retryable: false,
|
|
observedAt: Date.now(),
|
|
},
|
|
);
|
|
return { outcome: "failed", category: observation.category };
|
|
} catch (error) {
|
|
return { outcome: "unobserved", error };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Settle expired pending mint quotes before the mint recovery sweep runs.
|
|
*
|
|
* An expired bolt11 invoice can never be paid again, so a quote the mint
|
|
* still reports as UNPAID is guaranteed never to be issued and is failed
|
|
* locally. That local fail is what keeps coco-core's mint recovery sweep
|
|
* quick: the sweep treats UNPAID as "waiting" and would otherwise re-contact
|
|
* every dead quote's mint on every startup.
|
|
*
|
|
* The observation round is what makes the local fail safe: a quote can have
|
|
* been paid before expiry while the daemon was down, leaving no local
|
|
* observation behind. Failing such a quote without asking the mint would
|
|
* strand the paid funds, because failed operations are skipped by recovery.
|
|
* Asking the mint first closes that hole: PAID/ISSUED quotes are left for
|
|
* the sweep to finalize, and quotes whose mint is unreachable or too slow
|
|
* are left pending so a later startup can still recover them.
|
|
*/
|
|
export async function settleExpiredMintQuotes(
|
|
source: ExpiredMintQuoteSource,
|
|
nowMs: number,
|
|
deadlineMs: number = EXPIRED_MINT_OBSERVATION_DEADLINE_MS,
|
|
options: { unreachableMints?: Set<string>; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {},
|
|
): Promise<ExpiredMintSettlement> {
|
|
const pendingMints = await source.ops.mint.listPending();
|
|
const selection = selectCleanupOperations({
|
|
mints: pendingMints,
|
|
sends: [],
|
|
melts: [],
|
|
nowMs,
|
|
minAgeMs: 0,
|
|
});
|
|
|
|
const settlement: ExpiredMintSettlement = {
|
|
failed: 0,
|
|
leftForRecovery: 0,
|
|
unobserved: 0,
|
|
};
|
|
const candidates = selection.mintsToFail;
|
|
if (candidates.length === 0) return settlement;
|
|
|
|
const startedAt = Date.now();
|
|
for (const op of candidates) {
|
|
if (options.shouldStop?.() || options.outstanding?.has(recoveryKey("mint", op.id))) {
|
|
settlement.unobserved++; continue;
|
|
}
|
|
// A mint the startup probe already found unreachable cannot answer an
|
|
// observation either; skip it without spending the shared wall-clock
|
|
// budget, leaving the quote pending for a later startup.
|
|
if (options.unreachableMints) {
|
|
let mintUrl = op.mintUrl;
|
|
try {
|
|
mintUrl = normalizeMintUrl(op.mintUrl);
|
|
} catch {
|
|
// Malformed persisted URL: probe keys are raw for those, and the
|
|
// observation below would fail anyway, landing in `unobserved`.
|
|
}
|
|
if (options.unreachableMints.has(mintUrl)) {
|
|
settlement.unobserved++;
|
|
continue;
|
|
}
|
|
}
|
|
const remainingMs = deadlineMs - (Date.now() - startedAt);
|
|
if (remainingMs <= 0) {
|
|
const skipped =
|
|
candidates.length -
|
|
settlement.failed -
|
|
settlement.leftForRecovery -
|
|
settlement.unobserved;
|
|
settlement.unobserved += skipped;
|
|
startupProgress(
|
|
`Expired mint quote check budget exhausted; ${skipped} quote(s) left for mint recovery.`,
|
|
);
|
|
break;
|
|
}
|
|
|
|
// Track the complete observation/failure chain, not just its bounded wait.
|
|
const work = failExpiredMintQuoteIfUnpaid(source.mintOperationService, op.id, Infinity);
|
|
if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("mint", op.id), work);
|
|
const check = await waitForRecoveryWork(work, remainingMs).catch(error => ({
|
|
outcome: "unobserved" as const, error, category: undefined,
|
|
}));
|
|
if (check.outcome === "failed") {
|
|
settlement.failed++;
|
|
} else if (check.outcome === "leftForRecovery") {
|
|
// PAID/ISSUED (or terminally failed) at the mint: normal recovery
|
|
// must see this quote so paid proofs get claimed.
|
|
settlement.leftForRecovery++;
|
|
const observed =
|
|
check.category === "ready"
|
|
? "was paid at the mint"
|
|
: check.category === "completed"
|
|
? "was already issued at the mint"
|
|
: "failed terminally at the mint";
|
|
startupProgress(
|
|
`Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`,
|
|
);
|
|
} else {
|
|
// Mint unreachable, too slow, or the quote unknown to it: leave the
|
|
// operation pending so a later startup can still recover it.
|
|
settlement.unobserved++;
|
|
logger.warn("Could not check expired mint quote; leaving it pending", {
|
|
operationId: op.id,
|
|
mintUrl: op.mintUrl,
|
|
error:
|
|
check.error instanceof Error
|
|
? check.error.message
|
|
: String(check.error),
|
|
});
|
|
}
|
|
}
|
|
|
|
return settlement;
|
|
}
|
|
|
|
/**
|
|
* Source for explicit PAID mint-quote recovery.
|
|
*
|
|
* Unlike the startup sweeps this also accepts caller-supplied operation ids so
|
|
* an operator can target a quote coco already gave up on (state `failed`).
|
|
*/
|
|
export interface MintQuoteRecoverySource {
|
|
ops: {
|
|
mint: {
|
|
listPending(): Promise<MintQuoteRecoveryCandidate[]>;
|
|
get(operationId: string): Promise<MintQuoteRecoveryCandidate | null>;
|
|
finalize(operationId: string): Promise<unknown>;
|
|
};
|
|
};
|
|
mintOperationService: Pick<
|
|
MintOperationServiceCleanup,
|
|
"observePendingOperation"
|
|
>;
|
|
/**
|
|
* Re-open a failed operation so it can be recovered; false when it is no
|
|
* longer failed. Implementations must reload the full row (see
|
|
* `reopenFailedMintOperation`).
|
|
*/
|
|
reopenFailedOperation(operationId: string): Promise<boolean>;
|
|
}
|
|
|
|
export interface MintQuoteRecoveryOptions {
|
|
shouldStop?: () => boolean;
|
|
/** Target only these operation ids (may include failed operations). */
|
|
operationIds?: string[];
|
|
/** Per-quote budget for observing the mint and finalizing the operation. */
|
|
timeoutMs?: number;
|
|
/**
|
|
* Re-open failed operations instead of skipping them. Only applies to
|
|
* operations named by `operationIds`: coco's pending listing never returns
|
|
* failed operations, so they can only be recovered by explicit id.
|
|
*/
|
|
includeFailed?: boolean;
|
|
/**
|
|
* In-flight recovery work keyed by family:id (mint:<operation id>), shared across runs.
|
|
* withTimeout does not cancel the underlying request, so a timed-out quote
|
|
* check or finalize must keep blocking a retry until it actually settles.
|
|
*/
|
|
outstanding?: Map<string, Promise<unknown>>;
|
|
}
|
|
|
|
export interface MintQuoteRecoveryResult {
|
|
/** Operations recovery acted on. */
|
|
checked: number;
|
|
/** Operations whose paid sats were minted or restored. */
|
|
recovered: number;
|
|
/** Quotes the mint still reports UNPAID; left pending. */
|
|
waiting: number;
|
|
/**
|
|
* Quotes that ended terminally: the mint can no longer issue them, or coco
|
|
* finalised them without recovering any proofs.
|
|
*/
|
|
terminal: number;
|
|
/** Failed operations moved back to pending before checking. */
|
|
reopened: number;
|
|
/**
|
|
* Operations left to a later run: the mint was unreachable, the per-quote
|
|
* budget ran out, or the operation ended in a non-terminal state.
|
|
*/
|
|
retryable: number;
|
|
/** Operations skipped because an earlier recovery of them is still running. */
|
|
busy: number;
|
|
errors: Array<{ operationId: string; error: string }>;
|
|
}
|
|
|
|
/**
|
|
* Run async tasks strictly one after another.
|
|
*
|
|
* Used to serialize explicit wallet recovery: two concurrent requests must not
|
|
* both snapshot the same failed operation, and a retry must not start
|
|
* underneath work that outlived its timeout. A rejected task never breaks the
|
|
* chain for the next one.
|
|
*/
|
|
export function createRunQueue(): (<T>(run: () => Promise<T>) => Promise<T>) & { drain(): Promise<void> } {
|
|
let tail: Promise<unknown> = Promise.resolve();
|
|
const enqueue = <T>(run: () => Promise<T>): Promise<T> => {
|
|
const result = tail.then(run, run);
|
|
tail = result.then(
|
|
() => undefined,
|
|
() => undefined,
|
|
);
|
|
return result;
|
|
};
|
|
return Object.assign(enqueue, { drain: async () => { await tail; } });
|
|
}
|
|
|
|
/** Per-quote budget for the mint round-trip during explicit recovery. */
|
|
const MINT_QUOTE_RECOVERY_TIMEOUT_MS = 20_000;
|
|
|
|
/**
|
|
* Bound for the local post-finalize diagnostic read. This is a database
|
|
* lookup, not a mint round-trip, so it gets its own small budget: the per-op
|
|
* mint budget is often already spent when finalize throws, and a starved
|
|
* diagnostic would silently fall back to the generic error message.
|
|
*/
|
|
const DIAGNOSTIC_LOOKUP_TIMEOUT_MS = 250;
|
|
|
|
/**
|
|
* Recover mint quotes whose sats are PAID at the mint but were never claimed.
|
|
*
|
|
* For every target the mint is asked for the current quote state, and only it
|
|
* decides the outcome: PAID quotes have their stored outputs submitted, ISSUED
|
|
* quotes have their signatures restored (NUT-09), UNPAID quotes are left
|
|
* pending, and quotes the mint can no longer issue are reported rather than
|
|
* silently dropped. Anything the mint cannot answer is retried later.
|
|
*
|
|
* `finalize()` does not throw when the mint refuses to issue or when an
|
|
* already-issued quote's proofs cannot be restored: it returns a terminal
|
|
* operation instead. Recovery therefore inspects the returned operation's
|
|
* state and error and only counts a genuine finalized-without-error as
|
|
* recovered.
|
|
*
|
|
* Failed operations are skipped unless `includeFailed` is set, and they can
|
|
* only be targeted by explicit id because coco's pending listing never returns
|
|
* them. Re-opening an operation is a mutation, so it happens only here, never
|
|
* during startup recovery.
|
|
*
|
|
* Callers should serialize their own invocations and pass a shared
|
|
* `outstanding` map: `timeoutMs` bounds the wait but does not cancel the
|
|
* request behind it, so both a timed-out quote check and a timed-out finalize
|
|
* keep blocking a retry until they actually settle.
|
|
*/
|
|
export async function runMintQuoteRecovery(
|
|
source: MintQuoteRecoverySource,
|
|
options: MintQuoteRecoveryOptions = {},
|
|
onProgress?: (message: string) => void,
|
|
): Promise<MintQuoteRecoveryResult> {
|
|
if (options.includeFailed && !options.operationIds?.length) {
|
|
throw new Error("includeFailed requires explicit operationIds");
|
|
}
|
|
const timeoutMs = options.timeoutMs ?? MINT_QUOTE_RECOVERY_TIMEOUT_MS;
|
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
throw new Error("timeoutMs must be a positive finite number");
|
|
}
|
|
const outstanding =
|
|
options.outstanding ?? new Map<string, Promise<unknown>>();
|
|
const result: MintQuoteRecoveryResult = {
|
|
checked: 0,
|
|
recovered: 0,
|
|
waiting: 0,
|
|
terminal: 0,
|
|
reopened: 0,
|
|
retryable: 0,
|
|
busy: 0,
|
|
errors: [],
|
|
};
|
|
const messageOf = (error: unknown) =>
|
|
error instanceof Error ? error.message : String(error);
|
|
/** coco's fail-fast operation lock rejected the call: another holder exists. */
|
|
const isInProgress = (error: unknown) =>
|
|
error instanceof Error && error.name === "OperationInProgressError";
|
|
const track = (operationId: string, work: Promise<unknown>) =>
|
|
trackRecovery(outstanding, recoveryKey("mint", operationId), work);
|
|
|
|
let targets: MintQuoteRecoveryCandidate[];
|
|
if (options.operationIds && options.operationIds.length > 0) {
|
|
targets = [];
|
|
const seen = new Set<string>();
|
|
for (const operationId of options.operationIds) {
|
|
if (seen.has(operationId)) continue;
|
|
seen.add(operationId);
|
|
try {
|
|
const op = await source.ops.mint.get(operationId);
|
|
if (!op) {
|
|
result.errors.push({ operationId, error: "operation not found" });
|
|
continue;
|
|
}
|
|
targets.push(op);
|
|
} catch (error) {
|
|
result.errors.push({ operationId, error: messageOf(error) });
|
|
}
|
|
}
|
|
} else {
|
|
targets = await source.ops.mint.listPending();
|
|
}
|
|
|
|
const { pending, failed } = selectMintQuotesForRecovery({
|
|
mints: targets,
|
|
includeFailed: options.includeFailed === true,
|
|
});
|
|
|
|
for (const op of failed) {
|
|
if (options.shouldStop?.()) break;
|
|
const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`;
|
|
if (outstanding.has(recoveryKey("mint", op.id))) {
|
|
result.busy++;
|
|
onProgress?.(`${label}: an earlier recovery is still running; skipped`);
|
|
continue;
|
|
}
|
|
try {
|
|
if (!(await source.reopenFailedOperation(op.id))) {
|
|
onProgress?.(`${label}: no longer failed; skipped`);
|
|
continue;
|
|
}
|
|
result.reopened++;
|
|
onProgress?.(`${label}: re-opened failed operation for recovery`);
|
|
} catch (error) {
|
|
// coco's operation lock is fail-fast, so an in-progress error means a
|
|
// processor or another recovery holds the operation right now.
|
|
if (isInProgress(error)) {
|
|
result.busy++;
|
|
onProgress?.(`${label}: another recovery holds it; skipped`);
|
|
} else {
|
|
result.retryable++;
|
|
onProgress?.(`${label}: could not re-open: ${messageOf(error)}`);
|
|
}
|
|
result.errors.push({ operationId: op.id, error: messageOf(error) });
|
|
continue;
|
|
}
|
|
await recoverOne(op);
|
|
}
|
|
|
|
for (const op of pending) {
|
|
if (options.shouldStop?.()) break;
|
|
await recoverOne(op);
|
|
}
|
|
|
|
return result;
|
|
|
|
async function recoverOne(op: MintQuoteRecoveryCandidate): Promise<void> {
|
|
const label = `Mint quote ${op.quoteId ?? op.id} at ${op.mintUrl}`;
|
|
if (outstanding.has(recoveryKey("mint", op.id))) {
|
|
result.busy++;
|
|
onProgress?.(`${label}: an earlier recovery is still running; skipped`);
|
|
return;
|
|
}
|
|
result.checked++;
|
|
// One budget per operation, shared by the mint check and the finalize, so
|
|
// a slow mint cannot silently double the documented per-quote wait. The
|
|
// local post-finalize diagnostic read is exempt (DIAGNOSTIC_LOOKUP_TIMEOUT_MS).
|
|
const deadlineAt = Date.now() + timeoutMs;
|
|
const remaining = () => Math.max(1, deadlineAt - Date.now());
|
|
|
|
if (op.state === "executing") {
|
|
// A crash mid-mint can leave outputs already signed at the mint;
|
|
// finalize recovers them instead of minting a second time.
|
|
await finalizeAndClassify(
|
|
op.id,
|
|
label,
|
|
"recovered interrupted mint",
|
|
remaining,
|
|
);
|
|
return;
|
|
}
|
|
|
|
let observation: {
|
|
category: "waiting" | "ready" | "completed" | "terminal";
|
|
};
|
|
// observePendingOperation is not read-only: it emits quote-state-changed,
|
|
// persists the observation and can fail a terminal operation. Track it too,
|
|
// so a timed-out check cannot be retried and then persist a stale read.
|
|
const check = source.mintOperationService.observePendingOperation(op.id);
|
|
track(op.id, check);
|
|
try {
|
|
observation = await withTimeout(check, remaining());
|
|
} catch (error) {
|
|
result.retryable++;
|
|
result.errors.push({ operationId: op.id, error: messageOf(error) });
|
|
onProgress?.(`${label}: could not check with mint: ${messageOf(error)}`);
|
|
return;
|
|
}
|
|
|
|
const decision = classifyMintQuoteObservation(observation.category);
|
|
if (decision.action === "finalize") {
|
|
await finalizeAndClassify(
|
|
op.id,
|
|
label,
|
|
decision.observedRemoteState === "PAID"
|
|
? `paid, minting proofs (${op.amount} sat)`
|
|
: `already issued, restoring proofs (${op.amount} sat)`,
|
|
remaining,
|
|
);
|
|
} else if (decision.action === "waiting") {
|
|
result.waiting++;
|
|
onProgress?.(`${label}: mint reports UNPAID; left pending`);
|
|
} else {
|
|
// coco records the mint's terminal verdict by failing the operation.
|
|
result.terminal++;
|
|
onProgress?.(`${label}: mint can no longer issue this quote`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run finalize and classify its result. coco returns a terminal operation
|
|
* rather than throwing when the mint refuses (for example an expired quote)
|
|
* or when an already-issued quote's proofs could not be restored, so a
|
|
* fulfilled promise is not by itself evidence that sats were recovered.
|
|
*/
|
|
async function finalizeAndClassify(
|
|
operationId: string,
|
|
label: string,
|
|
successMessage: string,
|
|
remaining: () => number,
|
|
): Promise<void> {
|
|
const work = source.ops.mint.finalize(operationId);
|
|
track(operationId, work);
|
|
let terminal: { state?: string; error?: string } | null | undefined;
|
|
try {
|
|
terminal = (await withTimeout(work, remaining())) as
|
|
| { state?: string; error?: string }
|
|
| null
|
|
| undefined;
|
|
} catch (error) {
|
|
if (isInProgress(error)) {
|
|
result.busy++;
|
|
onProgress?.(`${label}: another recovery is working on it; skipped`);
|
|
} else {
|
|
result.retryable++;
|
|
// finalize can throw a generic "remains pending" error after coco has
|
|
// persisted the actionable mint rejection (for example inactive keyset).
|
|
const current = await withTimeout(
|
|
source.ops.mint.get(operationId),
|
|
Math.max(remaining(), DIAGNOSTIC_LOOKUP_TIMEOUT_MS),
|
|
).catch(() => null);
|
|
const detail = current?.state === "pending" && current.error
|
|
? current.error
|
|
: messageOf(error);
|
|
result.errors.push({ operationId, error: detail });
|
|
onProgress?.(`${label}: could not finish recovery: ${detail}`);
|
|
return;
|
|
}
|
|
result.errors.push({ operationId, error: messageOf(error) });
|
|
return;
|
|
}
|
|
if (terminal?.state === "finalized" && !terminal.error) {
|
|
result.recovered++;
|
|
onProgress?.(`${label}: ${successMessage}`);
|
|
return;
|
|
}
|
|
if (
|
|
terminal?.state === "failed" ||
|
|
(terminal?.state === "finalized" && terminal.error)
|
|
) {
|
|
result.terminal++;
|
|
const detail =
|
|
terminal.error ?? `left in state ${terminal.state ?? "unknown"}`;
|
|
result.errors.push({ operationId, error: detail });
|
|
onProgress?.(`${label}: not recovered: ${detail}`);
|
|
return;
|
|
}
|
|
// Pending/executing/unknown: coco may still be working on the operation,
|
|
// so leave it to a later run rather than calling it terminal.
|
|
result.retryable++;
|
|
result.errors.push({
|
|
operationId,
|
|
error: `left in state ${terminal?.state ?? "unknown"}; will retry`,
|
|
});
|
|
onProgress?.(
|
|
`${label}: still ${terminal?.state ?? "unknown"}; left for a later run`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const PENDING_MINT_SWEEP_INTERVAL_MS = 15_000;
|
|
/** Per-quote wait inside a sweep, so one stalled mint cannot starve the rest. */
|
|
const PENDING_MINT_CHECK_TIMEOUT_MS = 10_000;
|
|
const PENDING_MINT_SWEEP_DEADLINE_MS = 30_000;
|
|
const PENDING_MINT_STOP_DRAIN_MS = 10_000;
|
|
|
|
export interface PendingMintQuoteSource {
|
|
ops: { mint: Pick<Manager["ops"]["mint"], "listPending" | "refresh" | "get"> };
|
|
wallet: { balances: Pick<Manager["wallet"]["balances"], "byMint"> };
|
|
mintOperationService: Pick<MintOperationServiceCleanup, "failPendingOperation">;
|
|
}
|
|
|
|
export interface PendingMintSweepState {
|
|
/** Refreshes that outlived their wait; skipped until they settle. */
|
|
outstanding: Map<string, Promise<unknown>>;
|
|
/** Next sweep starts after this operation. */
|
|
after?: string;
|
|
}
|
|
|
|
export interface PendingMintSweepOptions {
|
|
deadlineMs?: number;
|
|
checkTimeoutMs?: number;
|
|
state?: PendingMintSweepState;
|
|
shouldStop?: () => boolean;
|
|
}
|
|
|
|
type PendingMintOutcome = "unreachable" | "other";
|
|
type PendingMintOp = Awaited<ReturnType<PendingMintQuoteSource["ops"]["mint"]["listPending"]>>[number];
|
|
|
|
/** Startup mint recovery, run while up: coco's live subscriptions can stop. */
|
|
export async function settlePendingMintQuotes(
|
|
source: PendingMintQuoteSource,
|
|
nowMs: number,
|
|
options: PendingMintSweepOptions = {},
|
|
): Promise<{ unreachable: number }> {
|
|
const deadlineMs = options.deadlineMs ?? PENDING_MINT_SWEEP_DEADLINE_MS;
|
|
const checkTimeoutMs = options.checkTimeoutMs ?? PENDING_MINT_CHECK_TIMEOUT_MS;
|
|
const state: PendingMintSweepState = options.state ?? { outstanding: new Map() };
|
|
let unreachable = 0;
|
|
|
|
const pending = await source.ops.mint.listPending();
|
|
const resumeAt = pending.findIndex((op) => op.id === state.after) + 1;
|
|
const ordered = [...pending.slice(resumeAt), ...pending.slice(0, resumeAt)];
|
|
const startedAt = Date.now();
|
|
for (const op of ordered) {
|
|
if (options.shouldStop?.()) break;
|
|
const remainingMs = deadlineMs - (Date.now() - startedAt);
|
|
if (state.outstanding.has(recoveryKey("mint", op.id)) || remainingMs <= 0) {
|
|
unreachable++;
|
|
continue;
|
|
}
|
|
state.after = op.id;
|
|
// Track refresh AND its late-result reporting mutations as one lifetime.
|
|
const settled = source.ops.mint
|
|
.refresh(op.id)
|
|
.then(
|
|
(result) => reportPendingMintRefresh(source, op, result, nowMs),
|
|
(error) => reportPendingMintRefreshError(source, op, error),
|
|
);
|
|
trackRecovery(state.outstanding, recoveryKey("mint", op.id), settled);
|
|
try {
|
|
const outcome = await withTimeout(settled, Math.min(checkTimeoutMs, remainingMs));
|
|
if (outcome === "unreachable") unreachable++;
|
|
} catch {
|
|
unreachable++;
|
|
}
|
|
}
|
|
return { unreachable };
|
|
}
|
|
|
|
async function reportPendingMintRefresh(
|
|
source: PendingMintQuoteSource,
|
|
op: PendingMintOp,
|
|
result: Awaited<ReturnType<PendingMintQuoteSource["ops"]["mint"]["refresh"]>>,
|
|
nowMs: number,
|
|
): Promise<PendingMintOutcome> {
|
|
const quote = `Mint quote ${op.quoteId} at ${op.mintUrl}`;
|
|
if (result.state === "finalized") {
|
|
if (result.error) {
|
|
// Already issued, proofs not restored: not a credit.
|
|
logger.warn(`${quote}: ${result.error}`);
|
|
return "other";
|
|
}
|
|
const balance = (await source.wallet.balances.byMint())[op.mintUrl]?.spendable;
|
|
logger.log(
|
|
`${quote}: paid, ${op.amount} sat minted` +
|
|
(balance === undefined ? "" : `, balance now ${balance} sat`),
|
|
);
|
|
return "other";
|
|
}
|
|
if (result.state === "failed") {
|
|
logger.warn(`${quote}: failed at the mint: ${result.error ?? "unknown reason"}`);
|
|
return "other";
|
|
}
|
|
// Paid-before-expiry is only known to the mint, so expired quotes are
|
|
// checked too; confirmed UNPAID after expiry is failed locally, as at startup.
|
|
const expired = op.expiry > 0 && op.expiry * 1000 <= nowMs;
|
|
if (expired && result.state === "pending" && result.lastObservedRemoteState === "UNPAID") {
|
|
await source.mintOperationService.failPendingOperation(
|
|
{ id: op.id },
|
|
{
|
|
reason: "Expired mint quote confirmed unpaid by mint",
|
|
retryable: false,
|
|
observedAt: Date.now(),
|
|
},
|
|
);
|
|
}
|
|
return "other";
|
|
}
|
|
|
|
async function reportPendingMintRefreshError(
|
|
source: PendingMintQuoteSource,
|
|
op: PendingMintOp,
|
|
error: unknown,
|
|
): Promise<PendingMintOutcome> {
|
|
// coco's own watcher is minting this quote right now; let it finish.
|
|
if (error instanceof OperationInProgressError) return "other";
|
|
// A failed mint attempt rejects after coco moved the operation back to pending.
|
|
const current = await source.ops.mint.get(op.id);
|
|
if (current?.state === "pending" && current.lastObservedRemoteState === "PAID") {
|
|
if (op.lastObservedRemoteState !== "PAID" || current.error !== op.error) {
|
|
logger.warn(
|
|
`Mint quote ${op.quoteId} at ${op.mintUrl}: paid (${op.amount} sat) but proofs not minted yet: ${current.error ?? String(error)}; will retry`,
|
|
);
|
|
}
|
|
return "other";
|
|
}
|
|
return "unreachable";
|
|
}
|
|
|
|
/**
|
|
* Stopping waits for the running sweep, then briefly for stragglers. Anything
|
|
* still running after that fails against the closed database and is picked
|
|
* up by startup recovery.
|
|
*/
|
|
function startPendingMintSweep(source: PendingMintQuoteSource, outstanding: RecoveryWork): () => Promise<void> {
|
|
let stopped = false;
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
let inFlight: Promise<void> = Promise.resolve();
|
|
let unreachableBefore = 0;
|
|
const state: PendingMintSweepState = { outstanding };
|
|
|
|
const tick = async () => {
|
|
if (stopped) return;
|
|
inFlight = settlePendingMintQuotes(source, Date.now(), { state, shouldStop: () => stopped }).then(
|
|
({ unreachable }) => {
|
|
// Report a mint becoming unreachable, or reachable again, once.
|
|
if (unreachable > 0 && unreachableBefore === 0) {
|
|
logger.warn(`Could not check ${unreachable} pending mint quote(s); will keep retrying`);
|
|
} else if (unreachable === 0 && unreachableBefore > 0) {
|
|
logger.log("Pending mint quote checks are reaching the mint again");
|
|
}
|
|
unreachableBefore = unreachable;
|
|
},
|
|
(error: unknown) => {
|
|
logger.warn(
|
|
`Pending mint quote sweep failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
},
|
|
);
|
|
await inFlight;
|
|
if (!stopped) timer = setTimeout(tick, PENDING_MINT_SWEEP_INTERVAL_MS);
|
|
};
|
|
timer = setTimeout(tick, PENDING_MINT_SWEEP_INTERVAL_MS);
|
|
|
|
return async () => {
|
|
stopped = true;
|
|
if (timer !== undefined) clearTimeout(timer);
|
|
await inFlight;
|
|
await withTimeout(
|
|
Promise.allSettled(state.outstanding.values()),
|
|
PENDING_MINT_STOP_DRAIN_MS,
|
|
).catch(() => {});
|
|
};
|
|
}
|
|
|
|
interface ReceiveRecoveryInternals {
|
|
receiveOperationService: {
|
|
checkProofStatesWithMint(
|
|
mintUrl: string,
|
|
proofs: Array<{ secret: string }>,
|
|
): Promise<Array<{ state: string }>>;
|
|
hasSavedOutputs(operation: unknown): Promise<boolean>;
|
|
markAsRolledBack(operation: unknown, error: string): Promise<unknown>;
|
|
};
|
|
mintAdapter: {
|
|
getCashuMint(mintUrl: string): {
|
|
restore(input: { outputs: Array<{ amount: number; id: string; B_: string }> }): Promise<{
|
|
outputs: Array<{ B_: string }>;
|
|
}>;
|
|
};
|
|
};
|
|
}
|
|
|
|
async function reconcileDuplicateReceiveOperations(
|
|
coco: Manager,
|
|
repo: SqliteRepositories,
|
|
): Promise<Awaited<ReturnType<typeof reconcileExecutingReceives>>> {
|
|
const internals = coco as unknown as ReceiveRecoveryInternals;
|
|
const unavailableMints = new Set<string>();
|
|
const deadline = Date.now() + 45_000;
|
|
const ensureMintBudget = (mintUrl: string): void => {
|
|
if (Date.now() >= deadline) throw new Error("Receive cleanup time budget exhausted");
|
|
if (unavailableMints.has(mintUrl)) throw new Error("Mint already failed receive cleanup");
|
|
};
|
|
const markMintFailure = (mintUrl: string, error: unknown): never => {
|
|
unavailableMints.add(mintUrl);
|
|
throw error;
|
|
};
|
|
const source: ReceiveReconcileSource = {
|
|
listExecuting: () => repo.receiveOperationRepository.getByState("executing"),
|
|
checkProofStates: async (operation) => {
|
|
ensureMintBudget(operation.mintUrl);
|
|
try {
|
|
return await withTimeout(
|
|
internals.receiveOperationService.checkProofStatesWithMint(
|
|
operation.mintUrl,
|
|
operation.inputProofs,
|
|
),
|
|
Math.min(15_000, Math.max(1, deadline - Date.now())),
|
|
);
|
|
} catch (error) {
|
|
return markMintFailure(operation.mintUrl, error);
|
|
}
|
|
},
|
|
restoreOutputs: async (mintUrl, outputs) => {
|
|
ensureMintBudget(mintUrl);
|
|
// Probe deterministic outputs in bounded batches. The Cashu restore
|
|
// response echoes only blinded messages with stored signatures, which
|
|
// identifies the owning operation. Coco later performs the real proof
|
|
// recovery for the retained operation.
|
|
const restored: Array<{ B_: string }> = [];
|
|
const mint = internals.mintAdapter.getCashuMint(mintUrl);
|
|
for (let index = 0; index < outputs.length; index += 300) {
|
|
try {
|
|
const response = await withTimeout(
|
|
mint.restore({ outputs: outputs.slice(index, index + 300) }),
|
|
Math.min(15_000, Math.max(1, deadline - Date.now())),
|
|
);
|
|
restored.push(...response.outputs);
|
|
} catch (error) {
|
|
return markMintFailure(mintUrl, error);
|
|
}
|
|
}
|
|
return restored;
|
|
},
|
|
hasSavedOutputs: (operation) =>
|
|
internals.receiveOperationService.hasSavedOutputs(operation),
|
|
rollBack: async (operation, reason) => {
|
|
await internals.receiveOperationService.markAsRolledBack(operation, reason);
|
|
},
|
|
};
|
|
return reconcileExecutingReceives(source);
|
|
}
|
|
|
|
interface RecoveryPhaseProgress {
|
|
phase: string;
|
|
failedMintQuotes: number;
|
|
}
|
|
|
|
/**
|
|
* Coco keeps per-operation recovery private on its services; routstrd already
|
|
* reaches into the Manager the same way for `mintOperationService`. Send is
|
|
* the only family whose public `refresh()` cannot recover executing ops.
|
|
*/
|
|
function sendRecoveryServiceOf(coco: Manager): SendRecoveryService {
|
|
return (coco as unknown as { sendOperationService: SendRecoveryService })
|
|
.sendOperationService;
|
|
}
|
|
|
|
/** Local-only crash cleanup, run before the degraded gate opens. */
|
|
export async function cleanupLocalRecoveryState(
|
|
coco: Manager,
|
|
repo: SqliteRepositories,
|
|
): Promise<void> {
|
|
// Coco 1.0.1 implements these as local repository/proof operations only.
|
|
// Keep this version-sensitive bridge together with the send recovery bridge.
|
|
const services = coco as unknown as Record<string, {
|
|
recoverInitOperation?(op: unknown): Promise<void>;
|
|
cleanupOrphanedReservations?(): Promise<number>;
|
|
} | undefined>;
|
|
const families = [
|
|
["send", repo.sendOperationRepository],
|
|
["melt", repo.meltOperationRepository],
|
|
["receive", repo.receiveOperationRepository],
|
|
["mint", repo.mintOperationRepository],
|
|
] as const;
|
|
// Fail closed the way reopenFailedMintOperation does: a coco bump that
|
|
// removes or renames these privates must stop recovery with a clear error
|
|
// before anything is written, not crash halfway through the loop with the
|
|
// cleanup half-applied.
|
|
for (const [kind] of families) {
|
|
if (typeof services[`${kind}OperationService`]?.recoverInitOperation !== "function") {
|
|
throw new Error(
|
|
`coco ${kind}OperationService.recoverInitOperation is unavailable; refusing local recovery cleanup`,
|
|
);
|
|
}
|
|
}
|
|
if (typeof services.sendOperationService?.cleanupOrphanedReservations !== "function") {
|
|
throw new Error(
|
|
"coco sendOperationService.cleanupOrphanedReservations is unavailable; refusing local recovery cleanup",
|
|
);
|
|
}
|
|
for (const [kind, repository] of families) {
|
|
for (const op of await repository.getByState("init")) {
|
|
await services[`${kind}OperationService`]!.recoverInitOperation!(op);
|
|
}
|
|
}
|
|
await services.sendOperationService!.cleanupOrphanedReservations!();
|
|
}
|
|
|
|
/**
|
|
* Gate for value-moving wallet operations while startup recovery runs.
|
|
*
|
|
* On degraded startup only, publishStuckMints opens the gate for callers
|
|
* whose target mint has no stuck operations after probing and local cleanup.
|
|
* A dead mint must not stall spends from a healthy one. On the happy path
|
|
* all callers wait until the global sweeps finish. Callers
|
|
* without a target mint, or whose mint has stuck operations, wait for the
|
|
* full sweep. fail() poisons every caller; reads are never gated.
|
|
*/
|
|
export interface RecoveryGate {
|
|
waitForRecovery(mintUrl?: string): Promise<void>;
|
|
publishStuckMints(mints: Set<string>): void;
|
|
complete(): void;
|
|
fail(error: string): void;
|
|
}
|
|
|
|
export function createRecoveryGate(): RecoveryGate {
|
|
let stuckMints: Set<string> | undefined;
|
|
let done = false;
|
|
let error: string | undefined;
|
|
let mintsResolve: (() => void) | undefined;
|
|
const mintsPromise = new Promise<void>((resolve) => {
|
|
mintsResolve = resolve;
|
|
});
|
|
let doneResolve: (() => void) | undefined;
|
|
const donePromise = new Promise<void>((resolve) => {
|
|
doneResolve = resolve;
|
|
});
|
|
|
|
return {
|
|
async waitForRecovery(mintUrl?: string): Promise<void> {
|
|
if (mintUrl) {
|
|
let normalized: string | undefined;
|
|
try {
|
|
normalized = normalizeMintUrl(mintUrl);
|
|
} catch {
|
|
// Unparseable URL falls back to the global gate.
|
|
normalized = undefined;
|
|
}
|
|
if (normalized) {
|
|
await mintsPromise;
|
|
if (!stuckMints?.has(normalized)) {
|
|
if (error) throw new Error(`Wallet is not ready: ${error}`);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
if (!done) await donePromise;
|
|
if (error) throw new Error(`Wallet is not ready: ${error}`);
|
|
},
|
|
publishStuckMints(mints: Set<string>): void {
|
|
if (stuckMints) return;
|
|
stuckMints = mints;
|
|
mintsResolve?.();
|
|
},
|
|
complete(): void {
|
|
done = true;
|
|
mintsResolve?.();
|
|
doneResolve?.();
|
|
},
|
|
fail(message: string): void {
|
|
done = true;
|
|
error = message;
|
|
mintsResolve?.();
|
|
doneResolve?.();
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Run the wallet recovery sweeps in order, reporting phase changes.
|
|
*
|
|
* Probe first, then settle expired mint quotes: quotes confirmed as unpaid
|
|
* are failed locally so `recoverPendingMintOperations()` skips them, while
|
|
* paid/issued and unreachable-mint quotes stay pending for the sweep.
|
|
*/
|
|
export async function runWalletRecovery(
|
|
coco: Manager,
|
|
onProgress: (progress: RecoveryPhaseProgress) => void,
|
|
receiveOperationIds?: string[],
|
|
onStuckMintsKnown?: (mints: Set<string>) => void,
|
|
options: { cleanupLocalState?: () => Promise<void>; fetchImpl?: typeof fetch; outstanding?: RecoveryWork; shouldStop?: () => boolean } = {},
|
|
): Promise<void> {
|
|
surfacingRecoveryProgress = true;
|
|
let failedMintQuotes = 0;
|
|
try {
|
|
// Probe every mint that has stuck operations once, FIRST, so a dead mint
|
|
// costs a single short probe instead of taxing settlement's observation
|
|
// budget plus a network timeout per operation per sweep. Healthy-mint
|
|
// operations are recovered per op; dead-mint operations stay parked
|
|
// exactly as coco's own "will retry later" path would leave them.
|
|
onProgress({ phase: "Probing mints", failedMintQuotes });
|
|
const stuckOperations = await collectStuckOperations(coco.ops);
|
|
const unreachableMints = await probeMintReachability(
|
|
[...new Set(stuckOperations.map((op) => op.mintUrl))],
|
|
{ fetchImpl: options.fetchImpl },
|
|
);
|
|
for (const mintUrl of unreachableMints) {
|
|
const count = stuckOperations.filter((op) => op.mintUrl === mintUrl).length;
|
|
startupProgress(
|
|
`Skipping recovery for unreachable mint: ${mintUrl} (${count} op${count === 1 ? "" : "s"})`,
|
|
);
|
|
}
|
|
const degraded = unreachableMints.size > 0;
|
|
if (degraded) {
|
|
// Local-only housekeeping must finish before any new operation is allowed.
|
|
await options.cleanupLocalState?.();
|
|
// Global sweeps enumerate fresh state and are unsafe beside live sends.
|
|
// Only the snapshot-based degraded path may open the per-mint gate.
|
|
onStuckMintsKnown?.(new Set(stuckOperations.map((op) => op.mintUrl)));
|
|
}
|
|
|
|
// Settlement runs after the gate opens and only spends its observation
|
|
// budget on mints the probe found reachable; dead-mint quotes stay
|
|
// pending untouched. It only reads and locally fails long-expired quotes,
|
|
// so it cannot conflict with live operations the gate just admitted.
|
|
onProgress({ phase: "Settling expired mint quotes", failedMintQuotes });
|
|
const settlement = await settleExpiredMintQuotes(
|
|
{
|
|
ops: coco.ops,
|
|
mintOperationService: (
|
|
coco as unknown as {
|
|
mintOperationService: MintOperationServiceCleanup;
|
|
}
|
|
).mintOperationService,
|
|
},
|
|
Date.now(),
|
|
undefined,
|
|
{ unreachableMints, outstanding: options.outstanding, shouldStop: options.shouldStop },
|
|
);
|
|
failedMintQuotes = settlement.failed;
|
|
if (settlement.leftForRecovery > 0 || settlement.unobserved > 0) {
|
|
startupProgress(
|
|
`Expired mint quotes: ${settlement.failed} failed locally, ` +
|
|
`${settlement.leftForRecovery} paid/issued (kept for recovery), ` +
|
|
`${settlement.unobserved} unverifiable (kept pending).`,
|
|
);
|
|
}
|
|
onProgress({ phase: "Settled expired mint quotes", failedMintQuotes });
|
|
const targeted = (kinds: Array<StuckOperation["kind"]>) =>
|
|
runTargetedRecovery(coco.ops, sendRecoveryServiceOf(coco), {
|
|
kinds,
|
|
outstanding: options.outstanding,
|
|
shouldStop: options.shouldStop,
|
|
stuckOperations,
|
|
unreachableMints,
|
|
});
|
|
|
|
// Happy path (every mint reachable) keeps coco's global sweeps: they also
|
|
// clean up init operations and orphaned proof reservations. Degraded
|
|
// startup runs that local housekeeping before opening its gate, and only
|
|
// drives the previously collected snapshot when a dead mint would
|
|
// otherwise tax every stuck op with a network timeout.
|
|
onProgress({ phase: "Send recovery", failedMintQuotes });
|
|
if (!degraded) await coco.ops.send.recovery.run();
|
|
else await targeted(["send"]);
|
|
|
|
onProgress({ phase: "Melt recovery", failedMintQuotes });
|
|
if (!degraded) await coco.ops.melt.recovery.run();
|
|
else await targeted(["melt"]);
|
|
|
|
onProgress({ phase: "Receive recovery", failedMintQuotes });
|
|
if (receiveOperationIds) {
|
|
// The pre-check already classified every executing receive by unique
|
|
// input set. Recover only the conclusive retained operations; unresolved
|
|
// groups stay untouched instead of falling back to Coco 1's expensive
|
|
// per-row sweep on this startup. Operations at mints the probe found
|
|
// unreachable are skipped rather than costing their 15s timeout each.
|
|
const mintByOperation = new Map(
|
|
stuckOperations
|
|
.filter((op) => op.kind === "receive")
|
|
.map((op) => [op.id, op.mintUrl]),
|
|
);
|
|
for (const operationId of receiveOperationIds) {
|
|
if (options.shouldStop?.()) break;
|
|
const mintUrl = mintByOperation.get(operationId);
|
|
if (mintUrl && unreachableMints.has(mintUrl)) continue;
|
|
try {
|
|
const work = coco.ops.receive.refresh(operationId);
|
|
if (options.outstanding) trackRecovery(options.outstanding, recoveryKey("receive", operationId), work);
|
|
await withTimeout(work, 15_000);
|
|
} catch (error) {
|
|
logger.warn("Targeted receive recovery did not complete", {
|
|
operationId,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
} else if (!degraded) {
|
|
await coco.ops.receive.recovery.run();
|
|
} else {
|
|
await targeted(["receive"]);
|
|
}
|
|
|
|
onProgress({ phase: "Mint recovery", failedMintQuotes });
|
|
// A settlement wait may have timed out while an unlocked observation
|
|
// still runs. Never let a fresh global mint sweep observe it again.
|
|
if (!degraded && ![...(options.outstanding?.keys() ?? [])].some(key => key.startsWith("mint:"))) {
|
|
await coco.recoverPendingMintOperations();
|
|
}
|
|
else await targeted(["mint"]);
|
|
|
|
onProgress({ phase: "done", failedMintQuotes });
|
|
} finally {
|
|
surfacingRecoveryProgress = false;
|
|
}
|
|
}
|
|
|
|
export async function createCocoClient(
|
|
options: CreateCocoClientOptions = {},
|
|
): Promise<WalletClient> {
|
|
const configDir = options.walletDir || options.configDir || defaultWalletDir();
|
|
const configFile = join(configDir, "config.json");
|
|
const dbPath = join(configDir, "coco.db");
|
|
const walletPidFile =
|
|
options.walletPidPath ||
|
|
(options.walletDir || options.configDir
|
|
? join(configDir, "wallet.pid")
|
|
: defaultWalletPidPath());
|
|
const legacySocket = options.legacySocketPath || legacyCocodSocketPath();
|
|
const legacyPidFile = options.legacyPidPath || legacyCocodPidPath();
|
|
const npcBaseUrl = options.npcBaseUrl || NPC_DEFAULT_BASE_URL;
|
|
const npcAddressDomain = new URL(npcBaseUrl).host;
|
|
|
|
await assertLegacyCocodNotRunning({ socketPath: legacySocket });
|
|
// The canonical wallet directory is created by initialization/migration.
|
|
// Keep a legacy PID claim as an exclusion fence for old cocod binaries.
|
|
mkdirSync(dirname(legacyPidFile), { recursive: true, mode: 0o700 });
|
|
const releaseWalletPidClaim = claimPidFile({
|
|
pidFilePath: walletPidFile,
|
|
label: "routstrd wallet lock",
|
|
});
|
|
let releaseLegacyPidClaim: () => void;
|
|
try {
|
|
releaseLegacyPidClaim = claimLegacyCocodPidFile({
|
|
pidFilePath: legacyPidFile,
|
|
});
|
|
} catch (error) {
|
|
releaseWalletPidClaim();
|
|
throw error;
|
|
}
|
|
|
|
let database: Database | undefined;
|
|
let coco: Manager | undefined;
|
|
let findFinalizedReceiveSibling: (
|
|
operation: ReceiveOperation | null,
|
|
) => Promise<string | null> = async () => null;
|
|
let walletConfig = loadConfig(configFile);
|
|
|
|
let recoveryPhase = "queued";
|
|
let recoveryFailedMintQuotes = 0;
|
|
let recoveryDone = false;
|
|
let recoveryError: string | undefined;
|
|
const recoveryCounts = {
|
|
pendingSends: 0,
|
|
inflightProofs: 0,
|
|
pendingMints: 0,
|
|
};
|
|
let recoveryResolve: (() => void) | undefined;
|
|
let stopPendingMintSweep: (() => Promise<void>) | undefined;
|
|
const recoveryPromise = new Promise<void>((resolve) => {
|
|
recoveryResolve = resolve;
|
|
});
|
|
const recoveryGate = createRecoveryGate();
|
|
let disposed = false;
|
|
const enqueueRecovery = createRunQueue();
|
|
const recoveryOutstanding: RecoveryWork = new Map();
|
|
|
|
try {
|
|
startupProgress("Opening Cashu wallet database...");
|
|
|
|
// Read and validate the existing cocod config during startup rather than
|
|
// deferring failure until coco-core first needs wallet key material.
|
|
const mnemonic = walletConfig.mnemonic;
|
|
const seed = mnemonicToSeedSync(mnemonic);
|
|
database = new Database(dbPath);
|
|
const repo = new SqliteRepositories({ database });
|
|
await repo.init();
|
|
initReceiveDedupSchema(database);
|
|
const interruptedReservations = clearInterruptedReceiveReservations(database);
|
|
if (interruptedReservations > 0) {
|
|
logger.warn("Cleared interrupted receive reservations with no Coco operation", {
|
|
count: interruptedReservations,
|
|
});
|
|
}
|
|
|
|
const [pendingSends, inflightProofs, pendingMints] = await Promise.all([
|
|
repo.sendOperationRepository.getPending(),
|
|
repo.proofRepository.getInflightProofs(),
|
|
repo.mintOperationRepository.getPending(),
|
|
]);
|
|
recoveryCounts.pendingSends = pendingSends.length;
|
|
recoveryCounts.inflightProofs = inflightProofs.length;
|
|
recoveryCounts.pendingMints = pendingMints.length;
|
|
const recoveryCount =
|
|
recoveryCounts.pendingSends +
|
|
recoveryCounts.inflightProofs +
|
|
recoveryCounts.pendingMints;
|
|
|
|
if (recoveryCount > 0) {
|
|
startupProgress(
|
|
`Recovering wallet state in background: ${recoveryCounts.pendingSends} pending sends, ` +
|
|
`${recoveryCounts.inflightProofs} in-flight proofs, ${recoveryCounts.pendingMints} pending mints.`,
|
|
);
|
|
} else {
|
|
startupProgress("Initializing Cashu wallet...");
|
|
}
|
|
|
|
// Construct Coco so the pre-recovery checker can reuse its mint adapter,
|
|
// but do not enable watchers/processors until the backup and cleanup finish.
|
|
coco = constructCocoManager(repo, seed);
|
|
|
|
const executingReceives = await repo.receiveOperationRepository.getByState("executing");
|
|
let receiveRecoveryOperationIds: string[] | undefined;
|
|
if (executingReceives.length > 0) {
|
|
startupProgress(
|
|
`Checking ${executingReceives.length} unfinished Cashu receive operation(s) for duplicates...`,
|
|
);
|
|
const recordedBackup = getReceiveReconcileBackup(database);
|
|
if (!recordedBackup || !existsSync(recordedBackup)) {
|
|
const backupPath = `${dbPath}.pre-receive-reconcile-${Date.now()}`;
|
|
database.exec(`VACUUM INTO '${backupPath.replaceAll("'", "''")}'`);
|
|
setReceiveReconcileBackup(database, backupPath);
|
|
startupProgress(`Created wallet backup before receive cleanup: ${backupPath}`);
|
|
}
|
|
const receiveReconcile = await reconcileDuplicateReceiveOperations(coco, repo);
|
|
receiveRecoveryOperationIds = receiveReconcile.recoveryOperationIds;
|
|
startupProgress(
|
|
`Receive cleanup: ${receiveReconcile.executing} operation(s), ` +
|
|
`${receiveReconcile.uniqueGroups} unique input set(s), ` +
|
|
`${receiveReconcile.rolledBack} stale duplicate(s) retired, ` +
|
|
`${receiveReconcile.unresolved} unresolved.`,
|
|
);
|
|
}
|
|
|
|
await enableCocoManager(coco);
|
|
const openDatabase = database;
|
|
|
|
findFinalizedReceiveSibling = async (
|
|
operation: ReceiveOperation | null,
|
|
): Promise<string | null> => {
|
|
if (!operation) return null;
|
|
const fingerprint = receiveInputFingerprint(operation);
|
|
const siblings = await repo.receiveOperationRepository.getByMintUrl(operation.mintUrl);
|
|
return (
|
|
siblings.find(
|
|
(candidate) =>
|
|
candidate.state === "finalized" &&
|
|
receiveInputFingerprint(candidate) === fingerprint,
|
|
)?.id ?? null
|
|
);
|
|
};
|
|
|
|
const syncReceiveReservations = async (): Promise<void> => {
|
|
for (const reservation of listProcessingReceiveTokens(openDatabase)) {
|
|
if (!reservation.operationId) continue;
|
|
try {
|
|
const operation = await coco!.ops.receive.get(reservation.operationId);
|
|
if (operation?.state === "finalized") {
|
|
updateReceiveToken(openDatabase, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: operation.id,
|
|
});
|
|
} else if (operation?.state === "rolled_back") {
|
|
const finalizedSibling = await findFinalizedReceiveSibling(operation);
|
|
updateReceiveToken(openDatabase, reservation.tokenHash, finalizedSibling
|
|
? {
|
|
state: "succeeded",
|
|
operationId: finalizedSibling,
|
|
}
|
|
: {
|
|
state: "failed",
|
|
operationId: operation.id,
|
|
error: operation.error || "Token receive was rolled back",
|
|
});
|
|
} else if (operation?.state === "prepared") {
|
|
// A crash before execute had no mint side effect. Cancel the stale
|
|
// prepared operation and permit a fresh exact-token attempt.
|
|
await coco!.ops.receive.cancel(
|
|
operation.id,
|
|
"Cancelled interrupted receive before execution",
|
|
);
|
|
deleteReceiveTokenReservation(openDatabase, reservation.tokenHash);
|
|
} else if (!operation) {
|
|
deleteReceiveTokenReservation(openDatabase, reservation.tokenHash);
|
|
}
|
|
} catch (error) {
|
|
// One damaged/stale reservation must never block daemon startup or
|
|
// make every wallet write fail after recovery.
|
|
logger.warn("Could not reconcile receive token reservation", {
|
|
operationId: reservation.operationId,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
};
|
|
await syncReceiveReservations();
|
|
|
|
const trustedMints = await coco.mint.getAllTrustedMints();
|
|
const configuredDefault = walletConfig.defaultMintUrl;
|
|
const defaultMintUrl = normalizeMintUrl(
|
|
configuredDefault || trustedMints[0]?.mintUrl || DEFAULT_MINT_URL,
|
|
);
|
|
|
|
// Seeds the mints we ship as trusted. The default mint is strict (see
|
|
// seedTrustedMints); extra seeds only warn, so an unreachable mint that is
|
|
// not the default cannot stop the daemon from starting.
|
|
await seedTrustedMints(
|
|
{
|
|
trustedMints: trustedMints.map((mint) => mint.mintUrl),
|
|
addMint: (mintUrl) => coco!.mint.addMint(mintUrl, { trusted: true }),
|
|
},
|
|
defaultMintUrl,
|
|
{
|
|
skipMints: walletConfig.removedMintUrls,
|
|
onProgress: startupProgress,
|
|
onError: (message, error) =>
|
|
logger.warn(message, {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
}),
|
|
},
|
|
);
|
|
|
|
// Persist only after the mint was successfully fetched and trusted. A failed
|
|
// network request must not leave config pointing at an unusable default.
|
|
walletConfig.defaultMintUrl = defaultMintUrl;
|
|
if (configuredDefault !== defaultMintUrl) {
|
|
saveConfig(walletConfig, configFile);
|
|
}
|
|
|
|
if (options.enableNpc !== false) {
|
|
startupProgress("Registering NPC (npubx.cash) plugin...");
|
|
// NPC authenticates with a Nostr key derived from the same wallet seed
|
|
// (NIP-06). The signer only produces JWT auth events for the NPC
|
|
// server; it never signs anything that moves funds by itself.
|
|
const npcSecretKey = privateKeyFromSeedWords(mnemonic);
|
|
const npcSigner = async (template: EventTemplate) =>
|
|
finalizeEvent(template, npcSecretKey);
|
|
const npcPlugin = new NPCPlugin(npcBaseUrl, npcSigner, {
|
|
useWebsocket: true,
|
|
logger: createCocoLogger({ module: "npc" }),
|
|
});
|
|
// coco-cashu-plugin-npc implements the plugin contract from the
|
|
// coco-cashu-core package while routstrd runs the equivalent
|
|
// @cashu/coco-core build. The plugin host API is structurally identical
|
|
// in both (verified: mintService.addMintByUrl,
|
|
// mintOperationService.importQuote/getOperationByQuote), so this cast
|
|
// only bridges the duplicate package names, not a real API gap.
|
|
coco.use(npcPlugin as unknown as CocoPlugin);
|
|
}
|
|
|
|
startupProgress("Cashu wallet ready.");
|
|
|
|
// Recovery runs in the background so the daemon can serve wallet reads
|
|
// immediately. Value-moving operations await the same promise below.
|
|
runWalletRecovery(
|
|
coco,
|
|
(progress) => {
|
|
recoveryPhase = progress.phase;
|
|
recoveryFailedMintQuotes = progress.failedMintQuotes;
|
|
if (progress.phase !== "done") {
|
|
startupProgress(`Wallet recovery: ${progress.phase}...`);
|
|
}
|
|
},
|
|
receiveRecoveryOperationIds,
|
|
(mints) => recoveryGate.publishStuckMints(mints),
|
|
{ cleanupLocalState: () => cleanupLocalRecoveryState(coco!, repo), outstanding: recoveryOutstanding, shouldStop: () => disposed },
|
|
)
|
|
.then(async () => {
|
|
await syncReceiveReservations();
|
|
recoveryDone = true;
|
|
recoveryPhase = "done";
|
|
recoveryGate.complete();
|
|
recoveryResolve?.();
|
|
startupProgress("Wallet recovery complete.");
|
|
stopPendingMintSweep = startPendingMintSweep({
|
|
ops: coco!.ops,
|
|
wallet: coco!.wallet,
|
|
mintOperationService: (
|
|
coco as unknown as { mintOperationService: MintOperationServiceCleanup }
|
|
).mintOperationService,
|
|
}, recoveryOutstanding);
|
|
})
|
|
.catch((error) => {
|
|
recoveryDone = true;
|
|
recoveryPhase = "error";
|
|
recoveryError = error instanceof Error ? error.message : String(error);
|
|
recoveryGate.fail(recoveryError);
|
|
recoveryResolve?.();
|
|
startupProgress(`Wallet recovery failed: ${recoveryError}`);
|
|
});
|
|
} catch (error) {
|
|
database?.close();
|
|
releaseLegacyPidClaim();
|
|
releaseWalletPidClaim();
|
|
throw error;
|
|
}
|
|
|
|
const npcApi = (): NpcPluginApi => {
|
|
// The plugin augments coco-cashu-core's PluginExtensions; the equivalent
|
|
// registration lives on manager.ext here. Guard for enableNpc=false.
|
|
const api = coco
|
|
? (coco.ext as { npc?: NpcPluginApi }).npc
|
|
: undefined;
|
|
if (!api) {
|
|
throw new Error("NPC plugin is not enabled for this wallet.");
|
|
}
|
|
return api;
|
|
};
|
|
|
|
const assertOpen = () => { if (disposed) throw new Error("Wallet is shutting down"); };
|
|
|
|
// Block a value-moving operation until background recovery has settled for
|
|
// its target mint (see createRecoveryGate). Reads stay ungated so the
|
|
// daemon can report balances/status immediately.
|
|
const waitForRecovery = async (mintUrl?: string): Promise<void> => {
|
|
assertOpen();
|
|
await recoveryGate.waitForRecovery(mintUrl);
|
|
assertOpen();
|
|
};
|
|
|
|
const disposeRecovery = createRecoveryDisposer(
|
|
() => { disposed = true; },
|
|
async () => {
|
|
await recoveryPromise;
|
|
await stopPendingMintSweep?.();
|
|
await enqueueRecovery.drain();
|
|
await drainRecoveryWork(recoveryOutstanding);
|
|
},
|
|
async () => {
|
|
await coco.dispose();
|
|
database.close();
|
|
releaseLegacyPidClaim();
|
|
releaseWalletPidClaim();
|
|
},
|
|
);
|
|
|
|
return {
|
|
async ping(): Promise<boolean> {
|
|
try {
|
|
await coco.wallet.balances.total();
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
},
|
|
|
|
async getStatus(): Promise<WalletRuntimeState> {
|
|
if (recoveryError) return "ERROR";
|
|
if (!recoveryDone) return "RECOVERING";
|
|
try {
|
|
await coco.wallet.balances.total();
|
|
return "UNLOCKED";
|
|
} catch {
|
|
return "ERROR";
|
|
}
|
|
},
|
|
|
|
async getRecoveryProgress(): Promise<WalletRecoveryProgress> {
|
|
return {
|
|
state: recoveryError ? "ERROR" : recoveryDone ? "UNLOCKED" : "RECOVERING",
|
|
phase: recoveryPhase,
|
|
pendingSends: recoveryCounts.pendingSends,
|
|
inflightProofs: recoveryCounts.inflightProofs,
|
|
pendingMints: recoveryCounts.pendingMints,
|
|
failedMintQuotes: recoveryFailedMintQuotes,
|
|
...(recoveryError ? { error: recoveryError } : {}),
|
|
};
|
|
},
|
|
|
|
async unlock(_passphrase: string): Promise<string> {
|
|
// coco-core does not support passphrase locking.
|
|
// Wallet access is controlled by ~/.routstrd/wallet/config.json.
|
|
return "wallet does not require unlocking";
|
|
},
|
|
|
|
async getBalances(): Promise<Record<string, number>> {
|
|
const byMint = await coco.wallet.balances.byMint();
|
|
return Object.fromEntries(
|
|
Object.entries(byMint).map(([mintUrl, snapshot]) => [
|
|
mintUrl,
|
|
snapshot.spendable,
|
|
]),
|
|
);
|
|
},
|
|
|
|
async receiveCashu(token: string): Promise<string> {
|
|
const reservation = reserveReceiveToken(database, token);
|
|
if (!reservation.acquired) {
|
|
if (reservation.existing?.state === "succeeded") {
|
|
return "Token already received successfully";
|
|
}
|
|
if (
|
|
reservation.existing?.state === "processing" &&
|
|
reservation.existing.operationId
|
|
) {
|
|
// A prior request may have stopped after the mint call became
|
|
// uncertain. Re-drive that one Coco operation instead of creating a
|
|
// duplicate. Refresh is idempotent and uses its stored output data.
|
|
try {
|
|
const operation = await withTimeout(
|
|
coco.ops.receive.refresh(reservation.existing.operationId),
|
|
15_000,
|
|
);
|
|
if (operation.state === "finalized") {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: operation.id,
|
|
});
|
|
return "Token received successfully";
|
|
}
|
|
if (operation.state === "rolled_back") {
|
|
const finalizedSibling = await findFinalizedReceiveSibling(operation);
|
|
if (finalizedSibling) {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: finalizedSibling,
|
|
});
|
|
return "Token already received successfully";
|
|
}
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "failed",
|
|
operationId: operation.id,
|
|
error: operation.error || "Token receive was rolled back",
|
|
});
|
|
throw new Error(operation.error || "Token receive was rolled back");
|
|
}
|
|
} catch (error) {
|
|
const latest = await coco.ops.receive.get(reservation.existing.operationId);
|
|
if (latest?.state === "finalized") {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: latest.id,
|
|
});
|
|
return "Token received successfully";
|
|
}
|
|
throw error;
|
|
}
|
|
throw new Error("Token receive is still unresolved");
|
|
}
|
|
if (reservation.existing?.state === "processing") {
|
|
throw new Error("Token receive is already in progress");
|
|
}
|
|
throw new Error(reservation.existing?.error || "Token receive previously failed");
|
|
}
|
|
|
|
let preparedOperationId: string | undefined;
|
|
try {
|
|
await waitForRecovery();
|
|
const prepared = await coco.ops.receive.prepare({ token });
|
|
preparedOperationId = prepared.id;
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "processing",
|
|
operationId: prepared.id,
|
|
});
|
|
await coco.ops.receive.execute(prepared.id);
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: prepared.id,
|
|
});
|
|
return "Token received successfully";
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (preparedOperationId) {
|
|
let latest: Awaited<ReturnType<typeof coco.ops.receive.get>> = null;
|
|
try {
|
|
latest = await coco.ops.receive.get(preparedOperationId);
|
|
} catch (lookupError) {
|
|
logger.warn("Could not inspect failed receive operation", {
|
|
operationId: preparedOperationId,
|
|
error:
|
|
lookupError instanceof Error ? lookupError.message : String(lookupError),
|
|
});
|
|
}
|
|
if (latest?.state === "finalized") {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: preparedOperationId,
|
|
});
|
|
return "Token received successfully";
|
|
}
|
|
if (latest?.state === "executing") {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "processing",
|
|
operationId: preparedOperationId,
|
|
error: message,
|
|
});
|
|
} else if (latest?.state === "rolled_back") {
|
|
const finalizedSibling = await findFinalizedReceiveSibling(latest);
|
|
if (finalizedSibling) {
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "succeeded",
|
|
operationId: finalizedSibling,
|
|
});
|
|
return "Token already received successfully";
|
|
}
|
|
updateReceiveToken(database, reservation.tokenHash, {
|
|
state: "failed",
|
|
operationId: preparedOperationId,
|
|
error: latest.error || message,
|
|
});
|
|
} else {
|
|
// A prepared or missing operation had no known mint side effect.
|
|
deleteReceiveTokenReservation(database, reservation.tokenHash);
|
|
}
|
|
} else {
|
|
// Decode/validation failed before coco created an operation. Do not
|
|
// permanently reserve malformed input or transient mint-fetch errors.
|
|
releaseReceiveToken(database, reservation.tokenHash);
|
|
}
|
|
throw error;
|
|
}
|
|
},
|
|
|
|
async receiveBolt11(amount: number, mintUrl?: string) {
|
|
const targetMint = mintUrl
|
|
? normalizeMintUrl(mintUrl)
|
|
: walletConfig.defaultMintUrl;
|
|
if (!targetMint) {
|
|
throw new Error("No trusted mint available for Lightning invoice");
|
|
}
|
|
await waitForRecovery(targetMint);
|
|
const op = await coco.ops.mint.prepare({
|
|
mintUrl: targetMint,
|
|
amount,
|
|
method: "bolt11",
|
|
});
|
|
if (!("request" in op)) {
|
|
throw new Error("mint prepare did not return a payment request");
|
|
}
|
|
return { invoice: op.request as string, operationId: op.id };
|
|
},
|
|
|
|
async getMintQuote(operationId: string) {
|
|
const op = await coco.ops.mint.get(operationId);
|
|
if (!op || op.state === "init") return null;
|
|
return {
|
|
operationId: op.id,
|
|
state: op.state,
|
|
mintState: op.lastObservedRemoteState,
|
|
amount: op.amount,
|
|
mintUrl: op.mintUrl,
|
|
error: op.error,
|
|
};
|
|
},
|
|
|
|
async sendCashu(amount: number, mintUrl?: string): Promise<string> {
|
|
const targetMint = mintUrl
|
|
? normalizeMintUrl(mintUrl)
|
|
: walletConfig.defaultMintUrl;
|
|
if (!targetMint) {
|
|
throw new Error("No trusted mint available for sending");
|
|
}
|
|
await waitForRecovery(targetMint);
|
|
const prepared = await coco.ops.send.prepare({
|
|
mintUrl: targetMint,
|
|
amount,
|
|
});
|
|
const { token } = await coco.ops.send.execute(prepared.id);
|
|
return getEncodedToken(token);
|
|
},
|
|
|
|
async sendBolt11(invoice: string, mintUrl?: string): Promise<string> {
|
|
const targetMint = mintUrl
|
|
? normalizeMintUrl(mintUrl)
|
|
: walletConfig.defaultMintUrl;
|
|
if (!targetMint) {
|
|
throw new Error("No trusted mint available for Lightning payment");
|
|
}
|
|
await waitForRecovery(targetMint);
|
|
const prepared = await coco.ops.melt.prepare({
|
|
mintUrl: targetMint,
|
|
method: "bolt11",
|
|
methodData: { invoice },
|
|
});
|
|
await coco.ops.melt.execute(prepared.id);
|
|
return "Payment sent successfully";
|
|
},
|
|
|
|
async listMints(): Promise<string[]> {
|
|
const mints = await coco.mint.getAllTrustedMints();
|
|
return mints.map((m) => m.mintUrl);
|
|
},
|
|
|
|
async addMint(url: string): Promise<string> {
|
|
await waitForRecovery();
|
|
const mintUrl = normalizeMintUrl(url);
|
|
await coco.mint.addMint(mintUrl, { trusted: true });
|
|
// A mint the user adds back is wanted again, so forget any removal
|
|
// marker that would make trusted-mint seeding skip it on restart.
|
|
if (clearMintRemoved(walletConfig, mintUrl)) {
|
|
saveConfig(walletConfig, configFile);
|
|
}
|
|
return `Mint ${mintUrl} added successfully`;
|
|
},
|
|
|
|
async removeMint(url: string): Promise<string> {
|
|
await waitForRecovery();
|
|
const mintUrl = normalizeMintUrl(url);
|
|
const trustedMints = await coco.mint.getAllTrustedMints();
|
|
if (!trustedMints.some((mint) => mint.mintUrl === mintUrl)) {
|
|
throw new Error(`Mint ${mintUrl} is not in the wallet mint list`);
|
|
}
|
|
if (trustedMints.length <= 1) {
|
|
throw new Error(
|
|
"Cannot remove the last mint in the wallet; add another mint first",
|
|
);
|
|
}
|
|
|
|
const remaining = trustedMints.filter((mint) => mint.mintUrl !== mintUrl);
|
|
await deleteMintFromWallet(coco, mintUrl);
|
|
|
|
let message = `Mint ${mintUrl} removed from the wallet`;
|
|
const wasDefault =
|
|
!!walletConfig.defaultMintUrl &&
|
|
configMintUrl(walletConfig.defaultMintUrl) === mintUrl;
|
|
if (wasDefault) {
|
|
const nextDefault = normalizeMintUrl(remaining[0]!.mintUrl);
|
|
walletConfig.defaultMintUrl = nextDefault;
|
|
message += `; default mint switched to ${nextDefault}`;
|
|
}
|
|
markMintRemoved(walletConfig, mintUrl);
|
|
saveConfig(walletConfig, configFile);
|
|
return message;
|
|
},
|
|
|
|
async getMintRemovalInfo(url: string): Promise<MintRemovalInfo> {
|
|
const mintUrl = normalizeMintUrl(url);
|
|
const trustedMints = await coco.mint.getAllTrustedMints();
|
|
if (!trustedMints.some((mint) => mint.mintUrl === mintUrl)) {
|
|
throw new Error(`Mint ${mintUrl} is not in the wallet mint list`);
|
|
}
|
|
|
|
// Balances come from stored proofs, so this stays local even when the
|
|
// mint is offline. `reserved` covers sats locked in in-flight sends.
|
|
const balances = await coco.wallet.balances.byMint({
|
|
mintUrls: [mintUrl],
|
|
});
|
|
const snapshot = balances[mintUrl] ?? {
|
|
spendable: 0,
|
|
reserved: 0,
|
|
total: 0,
|
|
};
|
|
const [pendingMintQuotes, pendingMeltQuotes] = await Promise.all([
|
|
countPendingMintQuotes(coco, mintUrl),
|
|
countPendingMeltQuotes(coco, mintUrl),
|
|
]);
|
|
const isDefault =
|
|
!!walletConfig.defaultMintUrl &&
|
|
configMintUrl(walletConfig.defaultMintUrl) === mintUrl;
|
|
|
|
return {
|
|
url: mintUrl,
|
|
spendable: snapshot.spendable,
|
|
reserved: snapshot.reserved,
|
|
total: snapshot.total,
|
|
pendingMintQuotes,
|
|
pendingMeltQuotes,
|
|
isDefault,
|
|
mintCount: trustedMints.length,
|
|
hasAssets:
|
|
snapshot.total > 0 || pendingMintQuotes > 0 || pendingMeltQuotes > 0,
|
|
};
|
|
},
|
|
|
|
async getMintInfo(url: string): Promise<unknown> {
|
|
return coco.mint.getMintInfo(normalizeMintUrl(url));
|
|
},
|
|
|
|
async getDefaultMint(): Promise<string | null> {
|
|
return walletConfig.defaultMintUrl || null;
|
|
},
|
|
|
|
async setDefaultMint(url: string): Promise<string> {
|
|
await waitForRecovery();
|
|
const mintUrl = normalizeMintUrl(url);
|
|
const trustedMints = await coco.mint.getAllTrustedMints();
|
|
if (!trustedMints.some((mint) => mint.mintUrl === mintUrl)) {
|
|
await coco.mint.addMint(mintUrl, { trusted: true });
|
|
// A mint the user adds back is wanted again, so forget any removal
|
|
// marker that would make trusted-mint seeding skip it on restart.
|
|
clearMintRemoved(walletConfig, mintUrl);
|
|
}
|
|
|
|
walletConfig.defaultMintUrl = mintUrl;
|
|
saveConfig(walletConfig, configFile);
|
|
return `Default mint set to ${mintUrl}`;
|
|
},
|
|
|
|
async dispose(): Promise<void> {
|
|
await disposeRecovery().catch(error => {
|
|
logger.warn("Wallet shutdown incomplete; database and ownership retained until recovery settles");
|
|
throw error;
|
|
});
|
|
},
|
|
|
|
async getHistory(offset?: number, limit?: number): Promise<HistoryEntry[]> {
|
|
return coco.history.getPaginatedHistory(offset, limit);
|
|
},
|
|
|
|
async getHistoryEntryById(id: string): Promise<HistoryEntry | null> {
|
|
return coco.history.getHistoryEntryById(id);
|
|
},
|
|
|
|
async getNpcAddress(): Promise<NpcAddress> {
|
|
const info = await npcApi().getInfo();
|
|
const name =
|
|
typeof info?.name === "string" && info.name.trim()
|
|
? info.name.trim()
|
|
: undefined;
|
|
const localPart = name ?? nip19.npubEncode(info.pubkey);
|
|
return {
|
|
address: `${localPart}@${npcAddressDomain}`,
|
|
...(name ? { name } : {}),
|
|
pubkey: info.pubkey,
|
|
};
|
|
},
|
|
|
|
async setNpcUsername(
|
|
username: string,
|
|
confirm?: boolean,
|
|
): Promise<NpcUsernameResult> {
|
|
await waitForRecovery();
|
|
const result = await npcApi().setUsername(username, confirm === true);
|
|
if (result.success) {
|
|
return { success: true };
|
|
}
|
|
return {
|
|
success: false,
|
|
paymentRequest:
|
|
result.pr as NpcUsernameResult["paymentRequest"],
|
|
};
|
|
},
|
|
|
|
async syncNpc(): Promise<void> {
|
|
await waitForRecovery();
|
|
await npcApi().sync();
|
|
},
|
|
|
|
async cleanupStuckOperations(
|
|
options: WalletCleanupOptions = {},
|
|
): Promise<WalletCleanupResult> {
|
|
await waitForRecovery();
|
|
const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000;
|
|
const dryRun = options.dryRun === true;
|
|
const force = options.force === true;
|
|
const nowMs = Date.now();
|
|
|
|
const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([
|
|
coco.ops.mint.listPending(),
|
|
coco.ops.send.listInFlight(),
|
|
coco.ops.melt.listPrepared(),
|
|
]);
|
|
|
|
const filteredMints = options.mintUrl
|
|
? pendingMints.filter((op) => op.mintUrl === options.mintUrl)
|
|
: pendingMints;
|
|
const filteredSends = options.mintUrl
|
|
? inFlightSends.filter((op) => op.mintUrl === options.mintUrl)
|
|
: inFlightSends;
|
|
const filteredMelts = options.mintUrl
|
|
? preparedMelts.filter((op) => op.mintUrl === options.mintUrl)
|
|
: preparedMelts;
|
|
|
|
const selection = selectCleanupOperations({
|
|
mints: filteredMints,
|
|
sends: filteredSends,
|
|
melts: filteredMelts,
|
|
nowMs,
|
|
minAgeMs,
|
|
});
|
|
|
|
const errors: WalletCleanupResult["errors"] = [];
|
|
let failedMintQuotes = 0;
|
|
let leftForRecovery = 0;
|
|
|
|
if (!dryRun) {
|
|
const mintService = (
|
|
coco as unknown as {
|
|
mintOperationService: MintOperationServiceCleanup;
|
|
}
|
|
).mintOperationService;
|
|
|
|
for (const op of selection.mintsToFail) {
|
|
if (force) {
|
|
// Legacy behaviour: fail the quote locally without asking the mint.
|
|
try {
|
|
await mintService.failPendingOperation(
|
|
{ id: op.id },
|
|
{
|
|
reason: "Expired mint quote cleaned up by routstrd (forced)",
|
|
retryable: false,
|
|
observedAt: nowMs,
|
|
},
|
|
);
|
|
failedMintQuotes++;
|
|
} catch (error) {
|
|
errors.push({
|
|
operationId: op.id,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
// Expiry alone does not prove the quote was never paid: the
|
|
// Lightning payment can land before expiry while the daemon is down.
|
|
// Confirm UNPAID with the mint before failing, exactly as startup
|
|
// recovery does; paid quotes are left for recovery to finalize.
|
|
const check = await failExpiredMintQuoteIfUnpaid(
|
|
mintService,
|
|
op.id,
|
|
EXPIRED_MINT_OBSERVATION_DEADLINE_MS,
|
|
);
|
|
if (check.outcome === "failed") {
|
|
failedMintQuotes++;
|
|
} else {
|
|
leftForRecovery++;
|
|
if (check.outcome === "unobserved") {
|
|
errors.push({
|
|
operationId: op.id,
|
|
error: `could not confirm quote state with mint: ${
|
|
check.error instanceof Error
|
|
? check.error.message
|
|
: String(check.error)
|
|
}`,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const op of selection.sendsToReclaim) {
|
|
try {
|
|
await coco.ops.send.reclaim(op.id);
|
|
} catch (error) {
|
|
errors.push({
|
|
operationId: op.id,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
for (const op of selection.meltsToCancel) {
|
|
try {
|
|
await coco.ops.melt.cancel(op.id, "Cancelled by wallet cleanup");
|
|
} catch (error) {
|
|
errors.push({
|
|
operationId: op.id,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
const mintSummary = summarizeMintCleanup({
|
|
dryRun,
|
|
candidates: selection.mintsToFail.length,
|
|
failed: failedMintQuotes,
|
|
leftForRecovery,
|
|
});
|
|
const actedOn =
|
|
(dryRun ? mintSummary.mintQuoteCandidates : mintSummary.failedMintQuotes) +
|
|
leftForRecovery +
|
|
selection.sendsToReclaim.length +
|
|
selection.meltsToCancel.length;
|
|
const skipped =
|
|
filteredMints.length + filteredSends.length + filteredMelts.length -
|
|
actedOn;
|
|
|
|
return {
|
|
dryRun,
|
|
...mintSummary,
|
|
reclaimedSends: selection.sendsToReclaim.length,
|
|
cancelledMelts: selection.meltsToCancel.length,
|
|
skipped,
|
|
errors,
|
|
};
|
|
},
|
|
|
|
async recoverMintQuotes(options, onProgress) {
|
|
await waitForRecovery();
|
|
const service = (
|
|
coco as unknown as {
|
|
mintOperationService: MintOperationServiceCleanup;
|
|
}
|
|
).mintOperationService;
|
|
// Serialize explicit recovery: two concurrent requests must not both
|
|
// snapshot the same failed operation, and a retry must not start
|
|
// underneath a finalize that outlived its timeout.
|
|
return enqueueRecovery(() => {
|
|
assertOpen();
|
|
return runMintQuoteRecovery(
|
|
{
|
|
ops: coco.ops as unknown as MintQuoteRecoverySource["ops"],
|
|
mintOperationService: service,
|
|
reopenFailedOperation: (operationId) =>
|
|
reopenFailedMintOperation(service, operationId),
|
|
},
|
|
{ ...options, outstanding: recoveryOutstanding, shouldStop: () => disposed },
|
|
onProgress,
|
|
);
|
|
});
|
|
},
|
|
|
|
async recoverStuckOperations() {
|
|
await waitForRecovery();
|
|
// Serialized against explicit mint-quote recovery (and itself) through
|
|
// the same queue and lifetime tracker, so timed-out passes cannot retry the same
|
|
// operation. Receive stays startup-only: recovering competing receives
|
|
// safely requires the startup dedup classification (receive-dedup.ts).
|
|
// Operations a live execute holds come back as busy via coco's
|
|
// fail-fast operation lock, never driven underneath it.
|
|
const result = await enqueueRecovery(() => {
|
|
assertOpen();
|
|
return runTargetedRecovery(coco!.ops, sendRecoveryServiceOf(coco!), {
|
|
kinds: ["send", "melt", "mint"],
|
|
outstanding: recoveryOutstanding,
|
|
shouldStop: () => disposed,
|
|
});
|
|
});
|
|
return {
|
|
timedOut: result.timedOut,
|
|
attempted: result.attempted,
|
|
busy: result.busy,
|
|
skipped: result.skipped,
|
|
failed: result.failed,
|
|
skippedMints: Object.fromEntries(result.skippedMints),
|
|
};
|
|
},
|
|
};
|
|
}
|