From b21f8677ceded01df4cf855b1c6bcb3faa1f86df Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 17:30:33 +0000 Subject: [PATCH 1/3] fix(marmot-interop): clean up daemons on Ctrl+C / SIGTERM / SIGHUP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous trap only fired on the EXIT pseudo-signal, which doesn't always run cleanly when the script is killed mid-flight — leaving the nohup'd wnd daemons (and the local relay) alive and still holding the port. Route SIGINT/SIGTERM/SIGHUP through an explicit `exit`, use a single idempotent cleanup handler, and preserve the exit code so the harness still reports failure when killed. --- tools/marmot-interop/marmot-interop-headless.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/marmot-interop/marmot-interop-headless.sh b/tools/marmot-interop/marmot-interop-headless.sh index bd6ac542d8..df2bd83081 100755 --- a/tools/marmot-interop/marmot-interop-headless.sh +++ b/tools/marmot-interop/marmot-interop-headless.sh @@ -80,7 +80,22 @@ source "$SCRIPT_DIR/headless/tests-manage.sh" # shellcheck source=headless/tests-extras.sh source "$SCRIPT_DIR/headless/tests-extras.sh" -trap 'stop_daemons; stop_local_relay; print_summary' EXIT +# Make sure Ctrl+C / SIGTERM / SIGHUP all run the full cleanup path — +# otherwise wnd is nohup'd and keeps running after the script dies, +# and the next run's `ss` check then complains the port is in use. +# Trap INT/TERM/HUP forces `exit`, which triggers the EXIT handler once. +cleanup() { + local rc=$? + trap - EXIT INT TERM HUP + stop_daemons + stop_local_relay + print_summary + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +trap 'exit 129' HUP banner "Marmot headless interop harness ($RUN_TS)" preflight From ac72262accdfa76a56de88028a92dd1cc3b342f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 17:33:27 +0000 Subject: [PATCH 2/3] fix(marmot-interop): clean up daemons on Ctrl+C in interactive harness Apply the same SIGINT/SIGTERM/SIGHUP trap hardening to the interactive harness that just landed on the headless one. Without it, killing the script mid-run leaves the nohup'd wn daemons alive and the next run fails in preflight because the sockets are still held. --- tools/marmot-interop/marmot-interop.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/marmot-interop/marmot-interop.sh b/tools/marmot-interop/marmot-interop.sh index ec999d87f5..a150f3c97d 100755 --- a/tools/marmot-interop/marmot-interop.sh +++ b/tools/marmot-interop/marmot-interop.sh @@ -925,7 +925,20 @@ test_14_push_notifications() { # ==== main =================================================================== main() { - trap 'stop_daemons; print_summary' EXIT + # Make sure Ctrl+C / SIGTERM / SIGHUP all run the full cleanup path — + # otherwise wnd is nohup'd and keeps running after the script dies. + # Trap INT/TERM/HUP forces `exit`, which triggers the EXIT handler once. + cleanup() { + local rc=$? + trap - EXIT INT TERM HUP + stop_daemons + print_summary + exit "$rc" + } + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + trap 'exit 129' HUP banner "Amethyst <-> whitenoise-rs interop harness ($RUN_TS)" preflight From c85848968cfa17882152fad788cfc064572bfd1e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 17:38:08 +0000 Subject: [PATCH 3/3] fix(marmot-interop): recover from stale MLS DB in interactive harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The interactive harness was failing with the same KeyringEntryMissingForExistingDatabase error as the headless one — the MLS SQLite DB on disk references a keyring entry that no longer exists (keychain entry pruned, data dir restored out of band, or a previous run used the mock keyring). wnd can't open the DB in that state, so the daemon exits before its socket appears. Unlike the headless harness (which wipes unconditionally because its mock keyring is always ephemeral), the interactive harness uses the real OS keyring and benefits from preserving B/C identities across runs. So: try once, and only wipe + retry when stderr actually shows the keyring-missing error. Also dump stderr/stdout tails inline on final failure so the operator doesn't have to chase a log path. --- tools/marmot-interop/marmot-interop.sh | 58 ++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tools/marmot-interop/marmot-interop.sh b/tools/marmot-interop/marmot-interop.sh index a150f3c97d..876572cd6f 100755 --- a/tools/marmot-interop/marmot-interop.sh +++ b/tools/marmot-interop/marmot-interop.sh @@ -103,13 +103,10 @@ preflight() { } # --- daemons ----------------------------------------------------------------- -start_daemon() { +# One attempt at bringing wnd up. Returns 0 on ready, 1 on crash/timeout. +# Caller inspects $data_dir/logs/stderr.log to decide whether to retry. +_start_daemon_attempt() { local name="$1" data_dir="$2" socket="$3" - step "starting $name daemon" - if [[ -S "$socket" ]] && "$WN_BIN" --socket "$socket" whoami >/dev/null 2>&1; then - info "$name daemon already running" - return 0 - fi rm -f "$socket" mkdir -p "$data_dir/logs" "$data_dir/release" # wnd puts its socket at {data_dir}/release/wnd.sock (release build) — we @@ -125,9 +122,56 @@ start_daemon() { info "$name ready" return 0 fi + # Exit early if wnd already crashed — no point waiting the full 30s. + if ! kill -0 "$pid" 2>/dev/null; then + return 1 + fi sleep 1 done - fail_msg "$name daemon failed to start (see $data_dir/logs/stderr.log)" + # Hung rather than crashed — kill it so the retry path has a clean slot. + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi + return 1 +} + +start_daemon() { + local name="$1" data_dir="$2" socket="$3" + step "starting $name daemon" + if [[ -S "$socket" ]] && "$WN_BIN" --socket "$socket" whoami >/dev/null 2>&1; then + info "$name daemon already running" + return 0 + fi + if _start_daemon_attempt "$name" "$data_dir" "$socket"; then + return 0 + fi + # Recover from a stale MLS SQLite DB whose keyring entry has gone + # missing (e.g. the keychain entry was pruned, the data dir was + # restored without the keyring, or a previous run used the mock + # keyring). wnd can't open the DB in that state, but the identity is + # disposable — wipe the data dir and let ensure_identity recreate it. + if [[ -s "$data_dir/logs/stderr.log" ]] && \ + grep -q 'KeyringEntryMissingForExistingDatabase' "$data_dir/logs/stderr.log"; then + warn "$name: stale MLS DB detected (keyring entry missing) — wiping $data_dir and retrying" + find "$data_dir" -mindepth 1 -maxdepth 1 ! -name 'logs' \ + -exec rm -rf {} + 2>/dev/null || true + if _start_daemon_attempt "$name" "$data_dir" "$socket"; then + return 0 + fi + fi + fail_msg "$name daemon failed to start" + if [[ -s "$data_dir/logs/stderr.log" ]]; then + printf ' --- last 40 lines of %s ---\n' "$data_dir/logs/stderr.log" >&2 + tail -n 40 "$data_dir/logs/stderr.log" | sed 's/^/ /' >&2 + printf ' --- end stderr ---\n' >&2 + fi + if [[ -s "$data_dir/logs/stdout.log" ]]; then + printf ' --- last 20 lines of %s ---\n' "$data_dir/logs/stdout.log" >&2 + tail -n 20 "$data_dir/logs/stdout.log" | sed 's/^/ /' >&2 + printf ' --- end stdout ---\n' >&2 + fi exit 1 }