From c90851610a89afb3dea1cd0ea66123f3d7cc18d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 14:42:35 +0000 Subject: [PATCH] test(marmot): point the interop reference at mdk and generate current-profile vectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 0 of the Marmot resync: get a live reference back, so the later stages are written against real bytes instead of a careful reading of the spec. The vector generator pinned stock crates.io openmls 0.8. MDK builds against erskingardner/openmls with the `extensions-draft` feature, and the whole current Marmot profile is expressed in terms of what that feature adds — app_data_dictionary (0x0006), app_components (0x0001), safe_aad (0x0002), app_data_update (0x0008). Vectors from the published crate cannot reach any of it. Pinned to MDK's exact rev instead. Adds `marmot-profile-gen`, which builds a group the way cgka-engine does: required capabilities of extension 0x0006 plus proposal 0x0008; GroupContext dictionary carrying the required-component list, group profile, admin policy, Nostr routing and lifecycle; per-leaf dictionaries carrying the supported list, an empty safe_aad list and the 104-byte account-identity-proof v2 component; last resort as the empty-data 0x0004 component in the KeyPackage dictionary, not an extension type; PublicMessage handshakes. It emits the Add commit, the Welcome, and exporter KATs for both group-event and the conformance commitment. The identity-proof encoder is hand-rolled from the spec rather than lifted from MDK, and asserts itself against the fixture published in account-identity-proof-v2.md before emitting anything — so if the generator runs at all, the kind-450 canonical serialization, its id, the BIP-340 signature and the component layout are known to match. The interop harness cloned marmot-protocol/whitenoise-rs, which was archived on 2026-08-05 pinned to mdk-core 0.8.0: it was testing us against a frozen MIP-era client, which is part of how the drift went unnoticed. Repointed at marmot-protocol/mdk, building -p wn-cli. Both source patches are dropped — mock-keyring is replaced by MDK's native --secret-store file, and skip-unprocessable-retry targeted a path MDK does not have. The daemon socket is now pinned via wnd --socket rather than guessed from a derived default. The harness changes are read off MDK's DaemonArgs and wn-cli manifest, not off a passing run; building MDK's workspace needs its pinned toolchain and a local relay. A human run of marmot-interop-headless.sh is the acceptance test. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016kCuA6tc4JQzHPCDd39GHq --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- cli/tests/README.md | 34 +- cli/tests/marmot/marmot-interop-headless.sh | 8 +- cli/tests/marmot/marmot-interop.sh | 52 +- .../patches/whitenoise-mock-keyring.patch | 17 - .../whitenoise-skip-unprocessable-retry.patch | 26 - cli/tests/marmot/setup.sh | 105 ++-- quartz/plans/2026-09-08-marmot-spec-resync.md | 66 +- quartz/plans/README.md | 2 +- .../resources/mls/marmot-current-profile.json | 116 ++++ quartz/tools/mdk-vector-gen/Cargo.toml | 24 +- quartz/tools/mdk-vector-gen/README.md | 97 ++- .../mdk-vector-gen/src/emit_joiner_kp.rs | 2 +- quartz/tools/mdk-vector-gen/src/main.rs | 18 +- .../mdk-vector-gen/src/marmot_profile_gen.rs | 577 ++++++++++++++++++ .../mdk-vector-gen/src/verify_amethyst.rs | 2 +- 16 files changed, 934 insertions(+), 214 deletions(-) delete mode 100644 cli/tests/marmot/patches/whitenoise-mock-keyring.patch delete mode 100644 cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch create mode 100644 quartz/src/commonTest/resources/mls/marmot-current-profile.json create mode 100644 quartz/tools/mdk-vector-gen/src/marmot_profile_gen.rs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d7a136344b..835ebef259 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,7 +40,7 @@ locally and tick the box. If your change can't possibly affect them (docs-only, UI-only on unrelated screens, etc.), tick "N/A". --> - [ ] N/A — change can't affect wire bytes / decoded audio / MLS state / DM envelopes -- [ ] Marmot / MLS — `cli/tests/marmot/marmot-interop-headless.sh` (NIP-EE / `whitenoise-rs`) +- [ ] Marmot / MLS — `cli/tests/marmot/marmot-interop-headless.sh` (Marmot / MDK `wn`) - [ ] NIP-17 DM — `cli/tests/dm/dm-interop-headless.sh` - [ ] Audio rooms manual — `cli/tests/nests/nests-interop.sh` (Amethyst ↔ nostrnests.com) - [ ] MoQ-lite hang-tier — `:nestsClient:jvmTest -DnestsHangInterop=true` diff --git a/cli/tests/README.md b/cli/tests/README.md index 19d9098f1f..564f899a53 100644 --- a/cli/tests/README.md +++ b/cli/tests/README.md @@ -28,7 +28,6 @@ cli/tests/ │ ├── tests-create.sh # tests 01–05 │ ├── tests-manage.sh # tests 06–08, 11 │ ├── tests-extras.sh # tests 09, 10, 12, 13 -│ └── patches/ # whitenoise-rs harness patches ├── nests/ # Audio-rooms interop (Amethyst ↔ nostrnests.com) │ ├── nests-interop.sh # 47-test manual harness │ └── README.md # operator brief + per-test matrix @@ -96,7 +95,7 @@ A third, slimmer harness covers the NIP-17 DM surface: - **`dm/dm-interop-headless.sh`** — two `amy` processes (Identity A and Identity D) exchange NIP-17 DMs through the loopback nostr-rs-relay. - No whitenoise-rs required — only `amy` and the relay binary (which + No MDK required — only `amy` and the relay binary (which is shared with the Marmot harness's checkout at `marmot/state-headless/nostr-rs-relay/`). @@ -123,11 +122,17 @@ A fourth harness covers audio rooms (NIP-53 + moq-lite): background audio. See `nests/README.md` for the full matrix and prereqs. -Both Marmot harnesses validate Amethyst against **whitenoise-rs** -(https://github.com/marmot-protocol/whitenoise-rs), the reference Rust -implementation that powers the White Noise Flutter app. Every test records a -pass/fail/skip result into a tab-separated log, and the summary is printed at -the end of the run. +Both Marmot harnesses validate Amethyst against **MDK** +(https://github.com/marmot-protocol/mdk), the reference Rust implementation of +the Marmot protocol, via its `wn` / `wnd` binaries (the `wn-cli` package). +Every test records a pass/fail/skip result into a tab-separated log, and the +summary is printed at the end of the run. + +> These harnesses previously targeted `marmot-protocol/whitenoise-rs`, which was +> archived on 2026-08-05 pinned to `mdk-core 0.8.0`. Testing against it meant +> testing against a frozen MIP-era client. The reference moved into `mdk`, and +> so did we — see `quartz/plans/2026-09-08-marmot-spec-resync.md` for what that +> change exposed. ## What gets tested @@ -197,9 +202,10 @@ cd tools/marmot-interop The script will, in order: 1. Verify `jq`, `git`, `cargo` etc. are present. -2. Clone `whitenoise-rs` into `state/whitenoise-rs/` and build `wn`/`wnd` - (release, `--features cli`). First build takes ~5 minutes; subsequent runs - reuse the binaries. +2. Clone `mdk` into `state/mdk/` and build `wn`/`wnd` + (`cargo build --release -p wn-cli`). First build takes ~5 minutes; + subsequent runs reuse the binaries. MDK pins its own Rust toolchain in + `rust-toolchain.toml`, so rustup may fetch a toolchain on the first run. 3. Launch two `wnd` daemons (one for Identity B, one for Identity C). 4. Create Nostr identities for B and C, persist their npubs in `state/run.env`. 5. Ask you to paste **your Amethyst account npub** (Identity A). This is @@ -219,7 +225,7 @@ The script will, in order: ``` --local-relays Use ws://localhost:8080 instead of the default public relays. Required if the public relays reject kinds 444/445/30443. - Run 'just docker-up' inside whitenoise-rs first. + Run 'just docker-up' inside the mdk checkout first. --transponder Run Test 14 (push notifications via the transponder service). --no-build Fail instead of rebuilding wn/wnd. Useful when iterating. -h, --help Show help. @@ -228,7 +234,7 @@ The script will, in order: Environment overrides: ``` -WN_REPO=/some/path/whitenoise-rs # use an existing checkout +WN_REPO=/some/path/mdk # use an existing checkout ``` ## Default relays @@ -245,7 +251,7 @@ that B just published — the harness warns you and continues. In that case re-run with `--local-relays` after starting the Docker stack: ```bash -cd state/whitenoise-rs +cd state/mdk just docker-up cd ../.. ./marmot-interop.sh --local-relays @@ -324,6 +330,6 @@ for B/C if this matters to you. - `marmot-interop.sh` — main entry point; orchestrates preflight, daemons, identities, relays, and runs the 13 tests in sequence. - `lib.sh` — helpers (logging, prompts, polling, jq wrappers, result table). -- `state/` — runtime directory, gitignored. Contains `whitenoise-rs/` source +- `state/` — runtime directory, gitignored. Contains the `mdk/` source checkout, per-daemon data/log dirs, the session `run.env`, logs, and results TSVs. diff --git a/cli/tests/marmot/marmot-interop-headless.sh b/cli/tests/marmot/marmot-interop-headless.sh index 15a49f79a2..59bd501fa3 100755 --- a/cli/tests/marmot/marmot-interop-headless.sh +++ b/cli/tests/marmot/marmot-interop-headless.sh @@ -3,7 +3,7 @@ # marmot-interop-headless.sh — zero-prompt, zero-internet interop harness. # # Drives Identity A via the `amy` CLI (./gradlew :cli:installDist) and -# Identities B/C via whitenoise-rs `wn`/`wnd`. Spins up a local +# Identities B/C via MDK's `wn`/`wnd`. Spins up a local # nostr-rs-relay on ws://127.0.0.1:$RELAY_PORT so nothing ever leaves the # machine. Matches the 13 test scenarios in marmot-interop.sh but without # any human prompts — all checks run to completion and the exit code @@ -24,14 +24,14 @@ LOG_DIR="$STATE_DIR/logs" A_DIR="$STATE_DIR/.amy/A" B_DIR="$STATE_DIR/B" C_DIR="$STATE_DIR/C" -B_SOCKET="$B_DIR/release/wnd.sock" -C_SOCKET="$C_DIR/release/wnd.sock" +B_SOCKET="$B_DIR/wnd.sock" +C_SOCKET="$C_DIR/wnd.sock" RUN_TS="$(date +%Y%m%d-%H%M%S)" LOG_FILE="$LOG_DIR/run-$RUN_TS.log" RESULTS_FILE="$STATE_DIR/results-$RUN_TS.tsv" -WN_REPO="${WN_REPO:-$SCRIPT_DIR/state/whitenoise-rs}" +WN_REPO="${WN_REPO:-$SCRIPT_DIR/state/mdk}" WN_BIN="$WN_REPO/target/release/wn" WND_BIN="$WN_REPO/target/release/wnd" AMY_BIN="$REPO_ROOT/cli/build/install/amy/bin/amy" diff --git a/cli/tests/marmot/marmot-interop.sh b/cli/tests/marmot/marmot-interop.sh index d901a69c78..b7b9a44d25 100755 --- a/cli/tests/marmot/marmot-interop.sh +++ b/cli/tests/marmot/marmot-interop.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# marmot-interop.sh — interop test harness: Amethyst <-> whitenoise-rs (wn/wnd) +# marmot-interop.sh — interop test harness: Amethyst <-> MDK (wn/wnd) # # Sequential, all-or-nothing. Script drives the `wn` side automatically and # prompts the human operator at each step that requires Amethyst UI action. @@ -15,17 +15,16 @@ STATE_DIR="$SCRIPT_DIR/state" LOG_DIR="$STATE_DIR/logs" B_DIR="$STATE_DIR/B" C_DIR="$STATE_DIR/C" -# wnd derives its socket path as "{data_dir}/release/wnd.sock" for release -# builds (and ".../dev/wnd.sock" for debug); our preflight always uses -# --release, so we hardcode the "release" suffix here. -B_SOCKET="$B_DIR/release/wnd.sock" -C_SOCKET="$C_DIR/release/wnd.sock" +# The harness pins the daemon socket explicitly via wnd's --socket flag, so +# these paths are our choice rather than a guess at wnd's derived default. +B_SOCKET="$B_DIR/wnd.sock" +C_SOCKET="$C_DIR/wnd.sock" RUN_TS="$(date +%Y%m%d-%H%M%S)" LOG_FILE="$LOG_DIR/run-$RUN_TS.log" RESULTS_FILE="$STATE_DIR/results-$RUN_TS.tsv" -WN_REPO="${WN_REPO:-$STATE_DIR/whitenoise-rs}" +WN_REPO="${WN_REPO:-$STATE_DIR/mdk}" WN_BIN="" WND_BIN="" B_NPUB="" @@ -48,7 +47,7 @@ NO_BUILD=0 usage() { cat < whitenoise-rs interop harness +marmot-interop.sh — Amethyst <-> MDK interop harness Options: --local-relays Use ws://localhost:8080 instead of public relays (requires 'just docker-up') @@ -57,7 +56,7 @@ Options: -h, --help Show this help Environment: - WN_REPO Path to whitenoise-rs checkout (default: state/whitenoise-rs) + WN_REPO Path to the mdk checkout (default: state/mdk) EOF } @@ -97,12 +96,14 @@ preflight() { fail_msg "wn/wnd not found and --no-build set: $WN_BIN"; exit 1 fi if [[ ! -d "$WN_REPO/.git" ]]; then - step "cloning whitenoise-rs into $WN_REPO" - git clone --depth 1 https://github.com/marmot-protocol/whitenoise-rs.git "$WN_REPO" \ + # marmot-protocol/whitenoise-rs was archived on 2026-08-05; wn/wnd now + # ship from marmot-protocol/mdk as the `wn-cli` package. + step "cloning mdk into $WN_REPO" + git clone --depth 1 https://github.com/marmot-protocol/mdk.git "$WN_REPO" \ 2>&1 | tee -a "$LOG_FILE" fi - step "building wn + wnd (cargo build --release --features cli) — ~5 min first run" - ( cd "$WN_REPO" && cargo build --release --features cli --bin wn --bin wnd ) \ + step "building wn + wnd (cargo build --release -p wn-cli) — ~5 min first run" + ( cd "$WN_REPO" && cargo build --release -p wn-cli --bin wn --bin wnd ) \ 2>&1 | tee -a "$LOG_FILE" fi printf ' wn: %s\n wnd: %s\n' "$WN_BIN" "$WND_BIN" >>"$LOG_FILE" @@ -114,10 +115,13 @@ preflight() { _start_daemon_attempt() { local name="$1" data_dir="$2" socket="$3" rm -f "$socket" - mkdir -p "$data_dir/logs" "$data_dir/release" - # wnd puts its socket at {data_dir}/release/wnd.sock (release build) — we - # don't pass --socket because the daemon doesn't accept that flag. + mkdir -p "$data_dir/logs" + # MDK's wnd accepts an explicit --socket, so the harness pins the listen + # path instead of guessing at the derived one ({home}/dev/wnd.sock today). + # --secret-store file keeps account secrets out of the OS keychain, which + # is what lets this run in a container. nohup "$WND_BIN" --data-dir "$data_dir" --logs-dir "$data_dir/logs" \ + --socket "$socket" --secret-store file \ >"$data_dir/logs/stdout.log" 2>"$data_dir/logs/stderr.log" & local pid=$! echo "$pid" > "$data_dir/pid" @@ -153,11 +157,11 @@ start_daemon() { if _start_daemon_attempt "$name" "$data_dir" "$socket"; then return 0 fi - # Recover from a stale MLS SQLite DB whose keyring entry has gone - # missing (e.g. the keychain entry was pruned, the data dir was - # restored without the keyring, or a previous run used the mock - # keyring). wnd can't open the DB in that state, but the identity is - # disposable — wipe the data dir and let ensure_identity recreate it. + # Recover from a stale MLS SQLite DB whose secret has gone missing (the + # data dir was restored without its secret store, or an earlier run used a + # different --secret-store). wnd can't open the DB in that state, but the + # identity is disposable — wipe the data dir and let ensure_identity + # recreate it. if [[ -s "$data_dir/logs/stderr.log" ]] && \ grep -q 'KeyringEntryMissingForExistingDatabase' "$data_dir/logs/stderr.log"; then warn "$name: stale MLS DB detected (keyring entry missing) — wiping $data_dir and retrying" @@ -428,7 +432,7 @@ configure_relays() { local who="$1" wnfn if [[ "$who" == "B" ]]; then wnfn=wn_b; else wnfn=wn_c; fi local name="marmot-interop $who" - local about="Scripted wn identity for Amethyst<->whitenoise-rs interop harness" + local about="Scripted wn identity for Amethyst<->MDK interop harness" local out if out=$("$wnfn" profile update --name "$name" --about "$about" 2>&1); then printf '%s profile update ok: %s\n' "$who" "$out" >>"$LOG_FILE" @@ -530,7 +534,7 @@ configure_relays() { wn_b groups leave "$sanity_gid" >/dev/null 2>&1 || true else warn "kind:10050/1059 failed — C never received welcome; relays likely dropping gift wraps or inbox lists" - warn "Consider rerunning with --local-relays (requires 'just docker-up' in whitenoise-rs)." + warn "Consider rerunning with --local-relays (requires 'just docker-up' in the mdk checkout)." fi fi } @@ -1304,7 +1308,7 @@ main() { trap 'exit 130' INT trap 'exit 143' TERM trap 'exit 129' HUP - banner "Amethyst <-> whitenoise-rs interop harness ($RUN_TS)" + banner "Amethyst <-> MDK interop harness ($RUN_TS)" preflight start_daemon B "$B_DIR" "$B_SOCKET" diff --git a/cli/tests/marmot/patches/whitenoise-mock-keyring.patch b/cli/tests/marmot/patches/whitenoise-mock-keyring.patch deleted file mode 100644 index ed729fcf91..0000000000 --- a/cli/tests/marmot/patches/whitenoise-mock-keyring.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- a/crates/whitenoise-cli/src/bin/wnd.rs -+++ b/crates/whitenoise-cli/src/bin/wnd.rs -@@ -44,6 +44,14 @@ async fn main() -> whitenoise_cli::Result<()> { - let args = Args::parse(); - let config = Config::resolve(args.data_dir.as_ref(), args.logs_dir.as_ref()); - -+ // marmot-interop-headless patch: allow sandboxes / CI without a real kernel -+ // keyring to fall back to the integration-tests mock keyring store by setting -+ // $WHITENOISE_MOCK_KEYRING=1. Requires building the binaries with -+ // `--features whitenoise/integration-tests` (the harness does). -+ if std::env::var("WHITENOISE_MOCK_KEYRING").is_ok() { -+ Whitenoise::initialize_mock_keyring_store(); -+ } -+ - let mut wn_config = - WhitenoiseConfig::new(&config.data_dir, &config.logs_dir, KEYRING_SERVICE_ID); - if !args.discovery_relays.is_empty() { diff --git a/cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch b/cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch deleted file mode 100644 index 875666fc5d..0000000000 --- a/cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch +++ /dev/null @@ -1,26 +0,0 @@ ---- a/src/whitenoise/event_processor/account_event_processor.rs -+++ b/src/whitenoise/event_processor/account_event_processor.rs -@@ -178,7 +178,22 @@ - } - Err(e) => { - // Handle retry logic for actual processing errors -- if retry_info.should_retry() { -+ // marmot-interop-headless patch: MLS errors that come from -+ // mdk are ALREADY terminal — mdk doesn't retry internally, so -+ // any Err it returns (Unprocessable, PreviouslyFailed, decrypt -+ // failure, group-not-found, etc.) is provably permanent. -+ // Retrying those 10 times with exponential backoff (total -+ // ~17 min) just blocks later decryptable commits behind a -+ // queue of doomed retries, so every later join / rename / -+ // leave propagation races the test timeout. Treat them all -+ // as one-shot: log once, move on. -+ let is_terminal = matches!( -+ e, -+ WhitenoiseError::MlsMessageUnprocessable(_) -+ | WhitenoiseError::MlsMessagePreviouslyFailed -+ | WhitenoiseError::MdkCoreError(_), -+ ); -+ if !is_terminal && retry_info.should_retry() { - self.schedule_retry(event, source, retry_info, e); - } else { - tracing::error!( diff --git a/cli/tests/marmot/setup.sh b/cli/tests/marmot/setup.sh index 6d03eb44fd..0c06556b9c 100644 --- a/cli/tests/marmot/setup.sh +++ b/cli/tests/marmot/setup.sh @@ -6,12 +6,11 @@ # --- preflight --------------------------------------------------------------- preflight() { banner "Preflight" - for cmd in jq git curl cargo protoc patch; do + for cmd in jq git curl cargo protoc; do if ! command -v "$cmd" >/dev/null 2>&1; then fail_msg "missing required tool: $cmd" case "$cmd" in protoc) info "hint: apt-get install protobuf-compiler (or brew install protobuf on macOS)" ;; - patch) info "hint: apt-get install patch" ;; esac exit 1 fi @@ -42,61 +41,36 @@ preflight() { [[ -x "$AMY_BIN" ]] || { fail_msg "amy still missing after build"; exit 1; } info "amy: $AMY_BIN" - # Clone/build whitenoise-rs if needed (shared between both harnesses). + # Clone/build the MDK reference client if needed (shared between both + # harnesses). + # + # This used to point at marmot-protocol/whitenoise-rs. That repository was + # archived on 2026-08-05 ("This repository is obsolete and is no longer + # updated") pinned to mdk-core 0.8.0, and wn/wnd moved into + # marmot-protocol/mdk as the `wn-cli` package. Pointing the harness at the + # dead repo tested us against a frozen MIP-era client, which is exactly the + # blind spot that let our implementation drift off the adopted spec. if [[ ! -d "$WN_REPO/.git" ]]; then if [[ "$NO_BUILD" -eq 1 ]]; then - fail_msg "whitenoise-rs checkout missing at $WN_REPO and --no-build set"; exit 1 + fail_msg "mdk checkout missing at $WN_REPO and --no-build set"; exit 1 fi - step "cloning whitenoise-rs into $WN_REPO" - git clone --depth 1 https://github.com/marmot-protocol/whitenoise-rs.git "$WN_REPO" \ + step "cloning mdk into $WN_REPO" + git clone --depth 1 https://github.com/marmot-protocol/mdk.git "$WN_REPO" \ 2>&1 | tee -a "$LOG_FILE" fi - # Two harness-only patches to whitenoise-rs so it runs in sandboxes that - # block the kernel keyring: - # 1. mock-keyring: honour $WHITENOISE_MOCK_KEYRING so wnd uses the - # integration-tests mock keyring store when the kernel keyutils - # syscalls are blocked (common in containers / CI). Compiled in via - # `--features whitenoise/integration-tests` on the build below. - # 2. skip-unprocessable-retry: when mdk-core returns a terminal MLS - # error (MlsMessageUnprocessable / PreviouslyFailed / MdkCoreError) - # the message is provably undecryptable — retrying it ten times with - # exponential backoff (~17 min) just blocks later decryptable commits - # behind a queue of doomed retries, which in the harness manifests as - # "A already left" / "name unchanged" timeouts. The patch treats those - # errors as terminal. + # No source patches. The harness used to carry two against whitenoise-rs: + # + # 1. mock-keyring, so wnd could run where the kernel keyring is blocked. + # MDK replaces this with a native flag: `--secret-store file` keeps + # account secrets in files under the data dir instead of the OS + # keychain. start_daemon passes it. + # 2. skip-unprocessable-retry, which made terminal MLS errors stop + # retrying. That patched `src/whitenoise/event_processor/`, a path MDK + # does not have. If MDK's retry behaviour turns out to stall this + # harness the same way, that is a fresh diagnosis against MDK's own + # code, not a patch to port. # - # The relay-override patches this harness used to carry (discovery-env / - # defaults-env) are gone: upstream wnd now takes native --discovery-relays - # and --default-account-relays flags (passed in start_daemon), which do the - # same job without patching. wn/wnd also moved into the crates/whitenoise-cli - # workspace member — the mock-keyring patch targets that path. - local -a patches=( - "whitenoise-mock-keyring.patch" - "whitenoise-skip-unprocessable-retry.patch" - ) - # Apply each patch with a real exit-code check. The previous version - # swallowed patch's exit status via `| tee`, which meant a miscounted - # hunk header silently left the marker touched and the binary unpatched - # — the resulting wn retried provably-doomed MLS messages for ~17min - # and every later test flapped or timed out. Fail fast instead. - for name in "${patches[@]}"; do - local marker="$WN_REPO/.headless-patched-${name%.patch}" - if [[ ! -f "$marker" ]]; then - step "patching whitenoise-rs: $name" - if ( cd "$WN_REPO" && patch -p1 --forward --reject-file=- \ - <"$SCRIPT_DIR/patches/$name" >>"$LOG_FILE" 2>&1 ); then - touch "$marker" - # Invalidate the previous build so the patched source is picked up. - rm -f "$WN_BIN" "$WND_BIN" - else - fail_msg "patch $name failed — see $LOG_FILE" - tail -n 30 "$LOG_FILE" | sed 's/^/ /' >&2 - exit 1 - fi - fi - done - # cargo's transitive deps (rustup, crates.io) both return 503 on cold # caches often enough that a single attempt fails ~30% of the time. # Retry each cargo build until the binary actually exists or we've @@ -109,8 +83,7 @@ preflight() { for attempt in $(seq 1 $max); do step "building wn + wnd (attempt $attempt/$max, ~5 min first run)" ( cd "$WN_REPO" && \ - cargo build --release -p whitenoise-cli \ - --features whitenoise/integration-tests --bin wn --bin wnd ) \ + cargo build --release -p wn-cli --bin wn --bin wnd ) \ 2>&1 | tee -a "$LOG_FILE" [[ -x "$WN_BIN" && -x "$WND_BIN" ]] && break [[ "$attempt" -lt "$max" ]] && warn "wn/wnd build failed (likely transient 503 from rustup or crates.io) — retrying" @@ -226,29 +199,31 @@ start_daemon() { info "$name daemon already running"; return 0 fi rm -f "$socket" - # The mock keyring (WHITENOISE_MOCK_KEYRING=1) is in-memory only and - # resets to empty on every wnd restart, but the SQLite databases that - # wnd writes under $data_dir persist across runs and reference keys that - # no longer exist — wnd then bails with KeyringEntryMissingForExistingDatabase - # before it can even open a socket. Wipe the keyring-dependent state on - # each start so the daemon always comes up cold and consistent. Logs - # and the pid file are preserved for post-mortem. + # Start every daemon from a cold data dir. A stale SQLite database whose + # matching secret is gone leaves wnd unable to open its store, and it then + # bails before it can even create the socket. The identities here are + # disposable, so wiping is always the right move. Logs and the pid file are + # preserved for post-mortem. if [[ -d "$data_dir" ]]; then find "$data_dir" -mindepth 1 -maxdepth 1 \ ! -name 'logs' ! -name 'pid' \ -exec rm -rf {} + 2>/dev/null || true fi - mkdir -p "$data_dir/logs" "$data_dir/release" + mkdir -p "$data_dir/logs" # --discovery-relays / --default-account-relays are native wnd flags that # force both the discovery plane and freshly-created accounts' NIP-65 / inbox # / key-package lists onto our loopback relay (kills the "can't reach nos.lol" # exit path and stops accounts from carrying unreachable public relays). # - # WHITENOISE_MOCK_KEYRING=1 is consumed by the mock-keyring patch: it swaps in - # the integration-tests mock secret store so wnd doesn't fall over when the - # kernel blocks keyutils syscalls. Harmless on a real host with a real keyring. - WHITENOISE_MOCK_KEYRING=1 \ - nohup "$WND_BIN" --data-dir "$data_dir" --logs-dir "$data_dir/logs" \ + # --socket pins the listen path instead of letting wnd derive it. MDK derives + # it as {home}/dev/wnd.sock, whitenoise-rs used {data_dir}/{profile}/wnd.sock; + # passing it explicitly makes the harness independent of that choice. + # + # --secret-store file replaces the old mock-keyring source patch: account + # secrets live in files under the data dir, so the daemon comes up in + # containers and CI where the kernel keyring is unavailable. + nohup "$WND_BIN" --data-dir "$data_dir" --logs-dir "$data_dir/logs" \ + --socket "$socket" --secret-store file \ --discovery-relays "$RELAY_URL" --default-account-relays "$RELAY_URL" \ >"$data_dir/logs/stdout.log" 2>"$data_dir/logs/stderr.log" & local pid=$! diff --git a/quartz/plans/2026-09-08-marmot-spec-resync.md b/quartz/plans/2026-09-08-marmot-spec-resync.md index ab217c1081..69a812522d 100644 --- a/quartz/plans/2026-09-08-marmot-spec-resync.md +++ b/quartz/plans/2026-09-08-marmot-spec-resync.md @@ -1,6 +1,6 @@ # Marmot: resync against the adopted spec and current MDK -Status: analysis + staged plan. Nothing implemented yet. +Status: Stage 0 done. Stages 1-7 open. Sources checked on 2026-09-08: @@ -230,16 +230,16 @@ lineage. Kinds 446–449 exist for us. Missing: the kind `451` push **owner proof** (spec'd 2026-07-23) and the token-record `relay_hint` publish-target rules. -### 4.11 Test/interop infrastructure — BLOCKER for verification +### 4.11 Test/interop infrastructure — was a BLOCKER, addressed in Stage 0 -- `cli/tests/marmot/marmot-interop.sh:101` clones the archived `whitenoise-rs`. The `wn`/`wnd` - binaries moved to `mdk/crates/cli`. -- `quartz/tools/mdk-vector-gen` pins plain openmls 0.8; MDK now uses the `extensions-draft` - fork. Regenerating against the fork is what gives us `app_data_dictionary` fixtures. -- Our MIP tests (`MarmotMipComplianceTest`, `MarmotMipBehaviorTest`) assert the deprecated - rules — e.g. `MarmotMipBehaviorTest.kt:793` asserts `RequiredCapabilities == [0xF2EE]`. - They pin us to the old profile and must be re-pointed, not deleted (the legacy bytes still - matter for reading our own stored groups). +- ~~`cli/tests/marmot/` clones the archived `whitenoise-rs`~~ — repointed at + `marmot-protocol/mdk` (`-p wn-cli`). +- ~~`quartz/tools/mdk-vector-gen` pins plain openmls 0.8~~ — repointed at the + `extensions-draft` fork, and `marmot-profile-gen` now emits current-profile fixtures. +- Still open: our MIP tests (`MarmotMipComplianceTest`, `MarmotMipBehaviorTest`) assert the + deprecated rules — e.g. `MarmotMipBehaviorTest.kt:793` asserts + `RequiredCapabilities == [0xF2EE]`. They pin us to the old profile and must be re-pointed, + not deleted (the legacy bytes still matter for reading our own stored groups). ## 5. Interop verdict today @@ -255,10 +255,26 @@ and the token-record `relay_hint` publish-target rules. Each stage is independently shippable and independently testable. -**Stage 0 — re-establish a live reference (small).** -Repoint `marmot-interop.sh` at `marmot-protocol/mdk` and its `wn`/`wnd`; regenerate -`mdk-vector-gen` against the `erskingardner/openmls` `extensions-draft` fork. Without this we -are guessing at bytes. Do this first regardless of what else gets scoped. +**Stage 0 — re-establish a live reference. DONE.** + +- `quartz/tools/mdk-vector-gen` now pins `erskingardner/openmls` at the exact rev MDK's root + `Cargo.toml` names, with the `extensions-draft` feature. Verified: it builds and runs. +- New generator `marmot-profile-gen` emits + `quartz/src/commonTest/resources/mls/marmot-current-profile.json` — a real current-profile + group with `app_data_dictionary` state at every location, PublicMessage handshakes, the Add + commit, the Welcome, and exporter KATs. It self-checks the account-identity-proof v2 + construction against the spec's published fixture at startup, so the emitted proofs are known + to match byte-for-byte. +- The interop harness (`cli/tests/marmot/`) now clones `marmot-protocol/mdk` and builds + `-p wn-cli` instead of the archived `whitenoise-rs`. Both source patches are gone: the + mock-keyring patch is replaced by MDK's native `--secret-store file`, and the + skip-unprocessable-retry patch targeted a path MDK does not have. The daemon socket is now + pinned with `wnd --socket` rather than guessed from a derived default. + +**Not yet run end-to-end.** The harness changes are derived from reading MDK's `DaemonArgs` and +`wn-cli` manifest, not from a passing run — building MDK's full workspace needs its pinned +toolchain and a local relay. A human run of `marmot-interop-headless.sh` is the acceptance test, +and is likely to surface at least the retry behaviour the old patch used to paper over. **Stage 1 — MLS extensions draft in Quartz (large, foundational).** `AppDataDictionary` / `ComponentData` TLS codecs; `app_components` (`0x0001`) and `safe_aad` @@ -293,11 +309,19 @@ withdrawal. Delete `CommitOrdering`'s transport-metadata tiebreak at this point, **Stage 7 — durability/restart conformance, app payload kinds (1009/1210), encrypted-media v2, push owner proof.** -### Open question for scoping +### Settled: Quartz keeps its own MLS -Stages 1–6 are a protocol rewrite, not a patch. The alternative worth naming: our Kotlin MLS -stack is ~7,900 lines and now has to chase a moving IETF draft that upstream tracks via a -fork of OpenMLS. If cross-client Marmot interop is a hard requirement, it may be cheaper to -decide *now* whether Quartz keeps its own MLS or binds MDK (which ships `marmot-c` and -`marmot-uniffi`) on Android/JVM. MDK is MIT, so licensing is clear; the cost is JNI/uniffi packaging per -target and losing our pure-Kotlin iOS/Linux reach. +Decided 2026-09-08. We do not bind `marmot-c` / `marmot-uniffi`; the pure-Kotlin stack stays, +and full MDK interoperability is the target. + +Consequences to plan around, since they are now ours to carry: + +- Stage 1 means implementing draft-ietf-mls-extensions-10's `app_data_dictionary` (`0x0006`), + `app_components` (`0x0001`), `safe_aad` (`0x0002`) and the `app_data_update` proposal + (`0x0008`) in `quartz/.../marmot/mls/`, against a draft upstream tracks through a fork of + OpenMLS rather than a released crate. +- The OpenMLS rev pinned in `mdk-vector-gen/Cargo.toml` is a version we now track deliberately. + When MDK bumps it, regenerate the vectors in the same change and diff them — a silent bump is + how we would drift again. +- Byte-level conformance is the only thing that keeps us honest, so every stage below lands with + vectors from `marmot-profile-gen`, not just unit tests written against our own reading. diff --git a/quartz/plans/README.md b/quartz/plans/README.md index 8016c5c8fd..0c4e9c9809 100644 --- a/quartz/plans/README.md +++ b/quartz/plans/README.md @@ -10,7 +10,7 @@ _Audited 2026-09-08. 12 plans: 7 shipped (archived), 0 in-progress, 4 queued, 1 | [2026-07-03-incremental-negentropy-storage.md](2026-07-03-incremental-negentropy-storage.md) | Always-current (created_at, id) index so cold NEG-OPENs stop paying a full scan + seal (~340 ms at 50k vs strfry's ~21 ms). | | [2026-07-04-small-req-floor.md](2026-07-04-small-req-floor.md) | Small-REQ dispatch floor: decomposed, inline fast path tried and reverted (no wire-level win); floor is transport-side. | | [2026-08-13-gpu-pow-mining.md](2026-08-13-gpu-pow-mining.md) | GPU NIP-13 mining declined (ARMv8 has SHA-256 in silicon, mobile GPUs do not). Midstate is ~3x on JVM targets; Android hinges on Conscrypt per-digest JNI cost, still unmeasured. created_at refresh while mining shipped. | -| [2026-09-08-marmot-spec-resync.md](2026-09-08-marmot-spec-resync.md) | Marmot moved off the MIP-era spec (2026-07-02): group state split into `app_data_dictionary` components, account identity proof v2, and a convergence engine. Current MDK rejects our groups outright. Gap analysis + 8-stage plan. | +| [2026-09-08-marmot-spec-resync.md](2026-09-08-marmot-spec-resync.md) | Marmot moved off the MIP-era spec (2026-07-02): group state split into `app_data_dictionary` components, account identity proof v2, and a convergence engine. Current MDK rejects our groups outright. Gap analysis + 8-stage plan; Stage 0 (interop reference repointed at mdk, current-profile vector generator) done. | ## Archived (shipped) | Plan | Summary | diff --git a/quartz/src/commonTest/resources/mls/marmot-current-profile.json b/quartz/src/commonTest/resources/mls/marmot-current-profile.json new file mode 100644 index 0000000000..990effea03 --- /dev/null +++ b/quartz/src/commonTest/resources/mls/marmot-current-profile.json @@ -0,0 +1,116 @@ +{ + "add_commit_public_message": "000100011058b3ff7a7dcea415fd50f694576fe3e500000000000000000100000000000341c101000100010001201d8b71e8aec7159699367f9207331aa22066acaa81159c8b124de4a31ad8a37f20b267d00662eb4bf0f18df0e94d5729a71d67a062b18d16bf8150b9bb3bf9f45a203eb99cd422e76ed22ed3a815d4f02405198c762ceeb9e63dacd54a402f455f1f0001201be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc1102000102000102000602000802000101000000006aa00f7b000000006b0edb8b408600064082408000010d0c00018001800380048009800c00020100800940681be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11000000006553f10009d497513391541c32fcaf208440252d654c0f67aa8bcc3a58ff9cc6fa0dc9cb9691fbed684e0958b21d2057418e3e70d2c11d17f14278bf82fefa9228b00c6e40400600d62704ab3b3b5361dbf8d5df4f967d265ace3124515acbe4039a901461f503fca8778455bc7641dc12b556cefff19bc424ea2b888e87c96153abb7aa2806070006040300040040401b03b36e266b3584d63b77257a84968b91c81476861d67cc4a12de4107846612edcf4ad9879a7d372ea6aa9d1ea9209206ff47950b760567f0a95a476fbdf00b01205b6f5f83aa775e03e9d483dbd3ba388557e90885676fe8f76a290e99ed6fb02a20d73d15dffb68fea7694378e9e38748e43d11ec5e011e3e981282bbd197d748f9000120defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a3402000102000102000602000802000103203190b1ec03d9b1098905686760b3c3d8106fc77b885a2405408042270894bdda408600064082408000010d0c00018001800380048009800c0002010080094068defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34000000006553f100725ed9e4a85cc98dd5963095a941d29f4d6de79ab486a196fbe137eea10ad5f9678748b25045d9ecd67e621f4ffc6d42c1d1247e62c3a4f391f105f3d72028944040b34b2073656c6cc38edbbfd94f63d5c89843a6816c980916a839c162200f20a9eaf604e119657a78e7859b72c2259a1861417c42898c09ef353027f5ee72e30a222044de425a5fd03f8a248ad21f5e4689bd1e0f922277cdf5fd48afc1298f1bc83c004040533b06dffddb875a91d0632496c51b7d769724cb98aa2bb15aafaed2bef9a5bfe30f6314dc9af00266d170ce66ba7eb73f64aaf52a5b727e540106e5421e5c0c20242aec20e9523416b9b864e376b91f2acee0d2ed0a637076ab1142c2166a5790209a02103deb0e453262559d75bb30c4ad05cd466f3a1912fef9569fa8d338693d", + "app_messages_alice_to_bob": [ + { + "plaintext": "48656c6c6f20426f6221", + "private_message": "000100021058b3ff7a7dcea415fd50f694576fe3e5000000000000000101001c12c465a57743d0504b0628eb8f4f482bf7eb6c790feed11c25e92457405d2c8fc2a14b0ecfa1117fdb858ba1633847267a556f509fabeef97e22074e299b26286ecc18f52da7a065e046292804a517cd27e9c8ce7e4faa154aedfc065a0c9e24d21187bace68a0ade9bf7b317552a2ee83f31af094f36ab155e357" + }, + { + "plaintext": "5365636f6e64206d65737361676520696e207468652073616d652065706f63682e", + "private_message": "000100021058b3ff7a7dcea415fd50f694576fe3e5000000000000000101001ce42f14d203be2d20c6cb94aa51a0394e4233e7af52df9707e0cfe0fa4074d155bb71e0f30beec4b2aa64f2afb45e2376bc05a4c935f6033d6200be8f42c5b81178238c0db0f925f9a0abeb62fee44d7fbd1bebe14980775d9bcc8566657811f9150c28f86155a2e895a3cced47e6f0b3162c87e588761b029edfe58274e8e09cc0a32bfd9e9fce8498b75a9fd970b71bf4c4" + } + ], + "cipher_suite": 1, + "committer": { + "account_identity_proof": { + "component": "defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34000000006553f100725ed9e4a85cc98dd5963095a941d29f4d6de79ab486a196fbe137eea10ad5f9678748b25045d9ecd67e621f4ffc6d42c1d1247e62c3a4f391f105f3d7202894", + "created_at": 1700000000, + "event_id": "8915534faaa884893c2b09d411c4f83232026a1468687002d95073468d800cf0", + "event_json": "[0,\"defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34\",1700000000,450,[[\"d\",\"marmot.account-identity-proof.v2\"],[\"component\",\"0x8009\"],[\"ciphersuite\",\"0x0001\"],[\"signature_scheme\",\"0x0807\"],[\"mls_signature_key\",\"d73d15dffb68fea7694378e9e38748e43d11ec5e011e3e981282bbd197d748f9\"]],\"Authorize this MLS leaf key for my Marmot account\"]", + "signature": "725ed9e4a85cc98dd5963095a941d29f4d6de79ab486a196fbe137eea10ad5f9678748b25045d9ecd67e621f4ffc6d42c1d1247e62c3a4f391f105f3d7202894" + }, + "account_pubkey": "defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34", + "leaf_dictionary": { + "0x0001": "0c00018001800380048009800c", + "0x0002": "00", + "0x8009": "defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34000000006553f100725ed9e4a85cc98dd5963095a941d29f4d6de79ab486a196fbe137eea10ad5f9678748b25045d9ecd67e621f4ffc6d42c1d1247e62c3a4f391f105f3d7202894" + }, + "signer_pub": "d73d15dffb68fea7694378e9e38748e43d11ec5e011e3e981282bbd197d748f9" + }, + "component_ids": { + "account_identity_proof_v2": "0x8009", + "admin_policy_v1": "0x8003", + "app_components": "0x0001", + "group_lifecycle_v1": "0x800c", + "group_profile_v1": "0x8001", + "last_resort_key_package": "0x0004", + "nostr_routing_v1": "0x8004", + "safe_aad": "0x0002" + }, + "description": "Current-profile Marmot group (app_data_dictionary + account-identity-proof v2), built on the OpenMLS extensions-draft fork MDK pins.", + "exporters": { + "convergence_conformance_v1": { + "commitment": "33c6245e99921e4d122e819d26d6a7d6d1c4888fdb3a560d6d30781afbb78f70", + "context": "636f6e76657267656e63652d636f6e666f726d616e63652d7631", + "label": "marmot", + "length": 32 + }, + "group_event": { + "context": "67726f75702d6576656e74", + "label": "marmot", + "length": 32, + "secret": "b75b3ceac7a9a9439ee146cbdf84a1305805acefec8d6ffd41c4690d6f9ea574" + } + }, + "group_state": { + "admins": [ + "defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34" + ], + "description": "current-profile fixture", + "epoch0_group_context_dictionary": { + "0x0001": "0a8001800380048009800c", + "0x8001": "0e4d61726d6f7420696e7465726f701763757272656e742d70726f66696c652066697874757265", + "0x8003": "20defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34", + "0x8004": "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a230d7773733a2f2f6e6f732e6c6f6c147773733a2f2f72656c61792e64616d75732e696f", + "0x800c": "00" + }, + "epoch1_group_context_dictionary": { + "0x0001": "0a8001800380048009800c", + "0x8001": "0e4d61726d6f7420696e7465726f701763757272656e742d70726f66696c652066697874757265", + "0x8003": "20defdea4cdb677750a420fee807eacf21eb9898ae79b9768766e4faa04a2d4a34", + "0x8004": "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a230d7773733a2f2f6e6f732e6c6f6c147773733a2f2f72656c61792e64616d75732e696f", + "0x800c": "00" + }, + "name": "Marmot interop", + "nostr_group_id": "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "relays": [ + "wss://nos.lol", + "wss://relay.damus.io" + ] + }, + "handshake_wire_format": "public_message", + "joiner": { + "account_identity_proof": { + "component": "1be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11000000006553f10009d497513391541c32fcaf208440252d654c0f67aa8bcc3a58ff9cc6fa0dc9cb9691fbed684e0958b21d2057418e3e70d2c11d17f14278bf82fefa9228b00c6e", + "created_at": 1700000000, + "event_id": "81e5ab834204d79cbeab9d475d7c1f0f74bdbf1e55f321bd4e8bbcb79da4db95", + "event_json": "[0,\"1be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11\",1700000000,450,[[\"d\",\"marmot.account-identity-proof.v2\"],[\"component\",\"0x8009\"],[\"ciphersuite\",\"0x0001\"],[\"signature_scheme\",\"0x0807\"],[\"mls_signature_key\",\"3eb99cd422e76ed22ed3a815d4f02405198c762ceeb9e63dacd54a402f455f1f\"]],\"Authorize this MLS leaf key for my Marmot account\"]", + "signature": "09d497513391541c32fcaf208440252d654c0f67aa8bcc3a58ff9cc6fa0dc9cb9691fbed684e0958b21d2057418e3e70d2c11d17f14278bf82fefa9228b00c6e" + }, + "account_pubkey": "1be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11", + "encryption_priv": "698980352ccbedd93b48c0dfa8570e25e5910eb5d5990e862fd78e1df7f1c320", + "init_priv": "1c4da405915323129e5d493c780735b836042d57b22704cc5fb7081c4cf1cc1a", + "key_package": "0001000500010001201d8b71e8aec7159699367f9207331aa22066acaa81159c8b124de4a31ad8a37f20b267d00662eb4bf0f18df0e94d5729a71d67a062b18d16bf8150b9bb3bf9f45a203eb99cd422e76ed22ed3a815d4f02405198c762ceeb9e63dacd54a402f455f1f0001201be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc1102000102000102000602000802000101000000006aa00f7b000000006b0edb8b408600064082408000010d0c00018001800380048009800c00020100800940681be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11000000006553f10009d497513391541c32fcaf208440252d654c0f67aa8bcc3a58ff9cc6fa0dc9cb9691fbed684e0958b21d2057418e3e70d2c11d17f14278bf82fefa9228b00c6e40400600d62704ab3b3b5361dbf8d5df4f967d265ace3124515acbe4039a901461f503fca8778455bc7641dc12b556cefff19bc424ea2b888e87c96153abb7aa2806070006040300040040401b03b36e266b3584d63b77257a84968b91c81476861d67cc4a12de4107846612edcf4ad9879a7d372ea6aa9d1ea9209206ff47950b760567f0a95a476fbdf00b", + "key_package_dictionary": { + "0x0004": "" + }, + "leaf_dictionary": { + "0x0001": "0c00018001800380048009800c", + "0x0002": "00", + "0x8009": "1be68a5a028f2601d0e80d468c344ba331d611b96c358b6032e8b4da0547fc11000000006553f10009d497513391541c32fcaf208440252d654c0f67aa8bcc3a58ff9cc6fa0dc9cb9691fbed684e0958b21d2057418e3e70d2c11d17f14278bf82fefa9228b00c6e" + }, + "signature_priv": "262dcc11eb77d04576b435a4eca5aa1116f7c025587f917684b2ad94166a5007", + "signature_pub": "3eb99cd422e76ed22ed3a815d4f02405198c762ceeb9e63dacd54a402f455f1f" + }, + "profile": "current", + "required_capabilities": { + "extensions": [ + "0x0006" + ], + "proposals": [ + "0x0008" + ] + }, + "signature_scheme": 2055, + "welcome": "0001000300014098202b385bd9cb7ae93aef0a66f277de147980248243dd008bd5db3477bdb6d79f6a20e7214b81b521f0443d6a688ac1699974b2e50e74acaf0c48ed53635b1d38b173405400b0a4835f6f23bd61579c0ab06687e0c9b6fa8a83c60a77bc3217b0e395026f28bf7f9af4ba5a2e51202956243471cd9968f0416382cc2690a09fecbdcbe17010d120f31df82c6315bba95327c3b6998945a87c44704262452a784e6c4fda66b73fb3c4a536877a0d210d1e88516290d9989015c98432b671bb587976ecb0884c8c942ac3844441c2cd43a1e3cb258c56aff41c10187900b56f46bac2ddfc3ba02f5f81cbecc2c55811976e7033696c474351afddd9301b42cd040112b7760a1cbac8cd59b1b81ae2e46de715cc6920527dabf08c2d2a44b920cb5bf7c6b25c8949463a47aeb9645bc1f957c192daf875d0bbf4ca55d285931a97d43937206889aea6f540741d78b850b8f21db3481be98510ab70b5639c73b7a9d884f3dfa4fbbcdbc871937c887622e69052880d7702d374635190151a4bfbf5839e4491b75880a55337f60bcbabc4446e849717f1e36f53e29d3bd5b9c5031c353bb833ee8f61b7578650782c9c8bf121fd2a9ff1d39798ffdfa0d2e7a890d746ad4ab9416c415441b54e2274cfdf793e3381dd1c765421674e0a315395508a71c3a1dc16d2b2d880a462b278da06c3f0ae3c9e21fc067eb002d3502c3cbf4c0f9d23ac83fba24659c8d87400395048a7def733212d2354c23d4f73ba749dd64ab714218233424f1917bc9ef160a4a0ef58c99d70343c4358c50123dce3d6172352266ecca64741dafc3350299cf775a6db1d0be273fd0581271e54178f51514c493cdf64534e22027cdcf33ea5083fc9d6e20da7e11f2802d89b49a188abc2cdcb5cc06f2f78a2d7539f4294ce53e8e6715e9165fbe2543dd6ffa5b0f0fe7aadbd21e240a9468a43075a3229a974cb1ea30a6cacbbe595532e569b8454792eb37e005b06f5df5cd6c09f48fb44d64370e8f453821a02ce1f16b0ef018c6e800f4947107c7d82ae045659a9b505c359d695ba54228d7801dd1b6b4be2ad71e8a2e272f5fe170413463655150bc826309bf5d1ab44def010179c8e0dcd06066911fd40ca33e7aaad0206f1456abd0e59a77ce9882152f4e4dd05956ade1f42eeae9af62df56dc9d448f80d07bd7c63e9f07ebc779f2c6f137ed622a88b77531e018fa3d54c3be49d27947a3aa61d8f883e550716fda5748505bdbe2151cb73767b092c3d3ba1df53e78d91048ed095f6b8a328e7bdf099cbc31e9ef1106e6b5d6bf5542d55bb9818c4f32a65989f3cd4c21903ac010351436e1dab1564bf2d91cc9afd7577c8920da06584d7f6259b57e7c49cf1df0b4d09743bf15dcc106c131154b2aae7780bebf2163eb0281aeb617974df8762a201ba6ab2d55adb5cef310a9aa2d3c0d0935f1bcb9b85adb60339a3d953c725c543e40a59f01a5bca133248028dcc3b565a101d12124e18b528f2e5a57c044be0cdb88fdb1999d04f81135ee40d4c256f239a8b43a3df62ccafee2e052b5ae2721572f54d3886d9cf16033e0a836a9ac9bb1903ef8083228938702c9b40772031998994c39c7ff081d9e7bcdad82b079ddfe4952ccb17be8ee2d9796530edbc24206fdcc5936c415de287b80c52e743a7dc392d6c0949ab23e4a622c500c51207869c4d24966a9a66dbcf29ae11f9226a5772cc9cddde35f1581a8713fb18d319c4d945af2a2ae42c151dca743b3dc77f6e76d5c76c59ef729b364b6b587eb3e0102036cc781ae67f632e1df0ae5206b0f0f588feae" +} diff --git a/quartz/tools/mdk-vector-gen/Cargo.toml b/quartz/tools/mdk-vector-gen/Cargo.toml index 853d15b0ea..623de2e975 100644 --- a/quartz/tools/mdk-vector-gen/Cargo.toml +++ b/quartz/tools/mdk-vector-gen/Cargo.toml @@ -4,22 +4,34 @@ version = "0.1.0" edition = "2021" [dependencies] -openmls = { version = "0.8.1", features = ["test-utils"] } -openmls_rust_crypto = "0.5.1" -openmls_basic_credential = { version = "0.5", features = ["test-utils"] } -openmls_memory_storage = { version = "0.5", features = ["persistence"] } -openmls_traits = "0.5" -tls_codec = "0.4" +# Pinned to the exact OpenMLS fork + rev MDK builds against, with the +# `extensions-draft` feature that carries draft-ietf-mls-extensions +# `app_data_dictionary` / `app_components` / `app_data_update`. Marmot's current +# profile is defined in those terms, so vectors generated from stock crates.io +# openmls cannot exercise it. Keep this rev in lockstep with mdk's root +# Cargo.toml -- a drift here silently generates vectors for the wrong wire shape. +openmls = { git = "https://github.com/erskingardner/openmls.git", rev = "59e7d3b27a7e95237879dd5478de1fd90eff7ada", features = ["test-utils", "extensions-draft"] } +openmls_rust_crypto = { git = "https://github.com/erskingardner/openmls.git", rev = "59e7d3b27a7e95237879dd5478de1fd90eff7ada" } +openmls_basic_credential = { git = "https://github.com/erskingardner/openmls.git", rev = "59e7d3b27a7e95237879dd5478de1fd90eff7ada", features = ["test-utils"] } +openmls_memory_storage = { git = "https://github.com/erskingardner/openmls.git", rev = "59e7d3b27a7e95237879dd5478de1fd90eff7ada", features = ["persistence", "extensions-draft"] } +openmls_traits = { git = "https://github.com/erskingardner/openmls.git", rev = "59e7d3b27a7e95237879dd5478de1fd90eff7ada", features = ["extensions-draft"] } +tls_codec = "0.5" hex = "0.4" serde_json = "1" serde = { version = "1", features = ["derive"] } base64 = "0.22" env_logger = "0.11" +secp256k1 = { version = "0.31", features = ["std", "global-context"] } +sha2 = "0.10" [[bin]] name = "mdk-vector-gen" path = "src/main.rs" +[[bin]] +name = "marmot-profile-gen" +path = "src/marmot_profile_gen.rs" + [[bin]] name = "emit-joiner-kp" path = "src/emit_joiner_kp.rs" diff --git a/quartz/tools/mdk-vector-gen/README.md b/quartz/tools/mdk-vector-gen/README.md index 51a35eb964..f2fcfc1db2 100644 --- a/quartz/tools/mdk-vector-gen/README.md +++ b/quartz/tools/mdk-vector-gen/README.md @@ -1,36 +1,85 @@ # mdk-vector-gen -Rust helper that emits MLS interop test vectors from the same openmls 0.8 -backend MDK/whitenoise uses. Produces `quartz/src/commonTest/resources/mls/mdk-welcome.json`, -which [`MdkWelcomeInteropTest`] consumes to prove Amethyst can parse and -decrypt a Welcome + application messages authored by the Rust side. +Rust helpers that emit MLS and Marmot interop test vectors from the same +OpenMLS backend MDK builds against. -## What the vector contains +**The dependency pin is the point of this tool.** `Cargo.toml` tracks +`erskingardner/openmls` at the exact rev MDK's root `Cargo.toml` names, built +with the `extensions-draft` feature. Stock crates.io `openmls` does not carry +`app_data_dictionary` / `app_components` / `app_data_update`, and the current +Marmot profile is defined entirely in those terms — so vectors generated from +the published crate cannot exercise it. When MDK bumps its OpenMLS rev, bump it +here in the same change. + +## Binaries + +### `marmot-profile-gen` → `marmot-current-profile.json` + +The current-profile Marmot vector: a group built the way MDK's `cgka-engine` +builds one. + +- `RequiredCapabilities` = extension `0x0006` (`app_data_dictionary`) + + proposal `0x0008` (`app_data_update`). +- GroupContext `app_data_dictionary` carrying the required-component list + (`0x0001`) plus `marmot.group.profile.v1` (`0x8001`), + `marmot.group.admin-policy.v1` (`0x8003`), + `marmot.transport.nostr.routing.v1` (`0x8004`) and + `marmot.group.lifecycle.v1` (`0x800c`). Component `0x8009` is *required* but + leaf-only, so it deliberately has no GroupContext data. +- Every member LeafNode dictionary carrying the supported-component list, an + empty `safe_aad` list, and the 104-byte + `marmot.member.account-identity-proof.v2` component. +- The KeyPackage-level dictionary carrying the empty-data + `last_resort_key_package` component (`0x0004`) — last resort is not an MLS + extension type in this profile. +- Handshake messages as `PublicMessage`, matching Marmot's pinned wire format; + the Add commit is emitted so the peeler has a real one to authenticate. +- Exporter KATs for both `MLS-Exporter("marmot", "group-event", 32)` and the + conformance commitment from `foundation/conformance.md`. + +The identity-proof encoder is hand-rolled from the spec text rather than pulled +from MDK, and `assert_spec_proof_vector()` checks it against the fixed vector +published in `app-components/account-identity-proof-v2.md` before anything else +runs. If the generator starts up at all, the canonical kind-450 event +serialization, its id, the BIP-340 signature and the 104-byte component layout +all match the spec byte-for-byte. + +``` +cd quartz/tools/mdk-vector-gen +cargo run --release --bin marmot-profile-gen \ + > ../../src/commonTest/resources/mls/marmot-current-profile.json +``` + +### `mdk-vector-gen` → `mdk-welcome.json` + +The MLS-core vector, unchanged in intent: it proves Amethyst can parse and +decrypt a Welcome plus application messages authored by the Rust side. It +builds a plain OpenMLS group with no Marmot profile state, which is exactly +what makes it a clean test of the key schedule alone. - `joiner.init_priv` / `encryption_priv` / `signature_priv` / `signature_pub` — - all the private key material the joiner needs to drive - `MlsGroup.processWelcome`. + the private key material the joiner needs to drive `MlsGroup.processWelcome`. - `joiner.key_package` (MlsMessage-wrapped) and `key_package_raw`. - `welcome` (MlsMessage-wrapped) — Alice's Welcome for Bob. - `committer.signer_pub` — Alice's Ed25519 signature public key. - `exporter.{label,context,length,secret}` — `MLS-Exporter("marmot", - "group-event", 32)` derived from Bob's post-join state. This is the - exporter Marmot uses to derive the outer ChaCha20 key for kind:445 - events, so a match here means Amethyst's whole post-join key schedule - agrees with openmls byte-for-byte. -- `app_messages_alice_to_bob[]` — Alice-sent PrivateMessage bytes plus - the expected plaintext. (Amethyst decrypt currently skips - `PrivateMessageContent` framing; the matching test is `@Ignore`d - until that is fixed.) + "group-event", 32)` derived from Bob's post-join state. A match here means + Amethyst's whole post-join key schedule agrees with OpenMLS byte-for-byte. +- `app_messages_alice_to_bob[]` — Alice-sent PrivateMessage bytes plus the + expected plaintext. + +``` +cargo run --release --bin mdk-vector-gen \ + > ../../src/commonTest/resources/mls/mdk-welcome.json +``` + +### `emit-joiner-kp`, `verify-amethyst` + +Debug helpers for driving one side of a join by hand. ## Regenerating -``` -cd quartz/tools/mdk-vector-gen -cargo run --release > ../../src/commonTest/resources/mls/mdk-welcome.json -``` - -The generator uses fresh randomness each run, so the committed vector -changes on regeneration — that is fine because the Kotlin test only -asserts round-trip correctness against whatever is in the JSON. Commit -the regenerated file if you change the generator. +Both generators use fresh randomness each run, so the committed vectors change +on regeneration — that is fine, because the Kotlin tests assert round-trip +correctness against whatever is in the JSON rather than against fixed bytes. +Commit the regenerated file if you change a generator. diff --git a/quartz/tools/mdk-vector-gen/src/emit_joiner_kp.rs b/quartz/tools/mdk-vector-gen/src/emit_joiner_kp.rs index 6abee9d3c2..4443510d68 100644 --- a/quartz/tools/mdk-vector-gen/src/emit_joiner_kp.rs +++ b/quartz/tools/mdk-vector-gen/src/emit_joiner_kp.rs @@ -22,11 +22,11 @@ use std::env; use std::fs::File; use std::process; +use ::tls_codec::Serialize; use openmls::prelude::*; use openmls_basic_credential::SignatureKeyPair; use openmls_rust_crypto::OpenMlsRustCrypto; use openmls_traits::OpenMlsProvider; -use tls_codec::Serialize; const CS: Ciphersuite = Ciphersuite::MLS_128_DHKEMX25519_AES128GCM_SHA256_Ed25519; diff --git a/quartz/tools/mdk-vector-gen/src/main.rs b/quartz/tools/mdk-vector-gen/src/main.rs index a1b5615936..a19d042288 100644 --- a/quartz/tools/mdk-vector-gen/src/main.rs +++ b/quartz/tools/mdk-vector-gen/src/main.rs @@ -13,11 +13,11 @@ // public half to the committer, then keep the three private keys so the // vector is fully-self-decryptable. +use ::tls_codec::{Deserialize, Serialize}; use openmls::prelude::*; use openmls_basic_credential::SignatureKeyPair; use openmls_rust_crypto::OpenMlsRustCrypto; use openmls_traits::OpenMlsProvider; -use tls_codec::{Deserialize, Serialize}; const CS: Ciphersuite = Ciphersuite::MLS_128_DHKEMX25519_AES128GCM_SHA256_Ed25519; @@ -64,13 +64,8 @@ fn main() { .use_ratchet_tree_extension(true) .build(); - let mut alice_group = MlsGroup::new( - &provider_a, - &alice_sig, - &group_cfg, - alice_cwk.clone(), - ) - .unwrap(); + let mut alice_group = + MlsGroup::new(&provider_a, &alice_sig, &group_cfg, alice_cwk.clone()).unwrap(); let (_commit_out, welcome_out, _group_info) = alice_group .add_members(&provider_a, &alice_sig, &[bob_kp.clone()]) @@ -95,7 +90,12 @@ fn main() { let exporter_context = b"group-event"; let exporter_length: usize = 32; let exporter_secret = bob_group - .export_secret(provider_b.crypto(), exporter_label, exporter_context, exporter_length) + .export_secret( + provider_b.crypto(), + exporter_label, + exporter_context, + exporter_length, + ) .unwrap(); // Alice sends three application messages to the group. Bob will replay diff --git a/quartz/tools/mdk-vector-gen/src/marmot_profile_gen.rs b/quartz/tools/mdk-vector-gen/src/marmot_profile_gen.rs new file mode 100644 index 0000000000..e896ab8ab1 --- /dev/null +++ b/quartz/tools/mdk-vector-gen/src/marmot_profile_gen.rs @@ -0,0 +1,577 @@ +// Generate a *current-profile Marmot* interop vector for the Amethyst Marmot module. +// +// `main.rs` proves our MLS core against stock OpenMLS: Welcome unwrap, key +// schedule, exporter. This binary proves the layer above it — the Marmot +// profile the adopted spec actually defines — by building a group the same way +// MDK's `cgka-engine` does: +// +// * handshake wire format is PublicMessage (`foundation/mls-protocol.md`, +// "Handshake wire format"); OpenMLS's WireFormatPolicy governs handshakes +// only, application messages stay PrivateMessage per RFC 9420; +// * `RequiredCapabilities` = extension `0x0006` app_data_dictionary + +// proposal `0x0008` app_data_update; +// * GroupContext carries an `app_data_dictionary` with the required-component +// list (`0x0001`) plus profile / admin-policy / nostr-routing / lifecycle; +// * every member LeafNode carries its own `app_data_dictionary` with the +// supported-component list, an empty `safe_aad` list, and the 104-byte +// `marmot.member.account-identity-proof.v2` component (`0x8009`); +// * the KeyPackage-level dictionary carries the empty-data +// `last_resort_key_package` component (`0x0004`) — last resort is NOT an +// MLS extension type in this profile. +// +// Everything emitted here is the byte shape our Kotlin side has to produce and +// parse. The component payload encoders below are deliberately hand-rolled from +// the spec text rather than pulled from MDK: if the hand-rolled bytes and +// OpenMLS's framing agree with MDK, the spec was read correctly. + +use ::tls_codec::{Deserialize, Serialize}; +use openmls::extensions::{AppDataDictionary, AppDataDictionaryExtension}; +use openmls::prelude::*; +use openmls_basic_credential::SignatureKeyPair; +use openmls_rust_crypto::OpenMlsRustCrypto; +use openmls_traits::OpenMlsProvider; +use secp256k1::{Keypair, Secp256k1, SecretKey, XOnlyPublicKey}; +use sha2::{Digest, Sha256}; + +const CS: Ciphersuite = Ciphersuite::MLS_128_DHKEMX25519_AES128GCM_SHA256_Ed25519; + +// foundation/registries.md — upstream MLS extensions draft ids. +const COMPONENT_APP_COMPONENTS: u16 = 0x0001; +const COMPONENT_SAFE_AAD: u16 = 0x0002; +const COMPONENT_LAST_RESORT: u16 = 0x0004; +// foundation/registries.md — Marmot private-range component ids. +const COMPONENT_GROUP_PROFILE: u16 = 0x8001; +const COMPONENT_ADMIN_POLICY: u16 = 0x8003; +const COMPONENT_NOSTR_ROUTING: u16 = 0x8004; +const COMPONENT_ACCOUNT_IDENTITY_PROOF: u16 = 0x8009; +const COMPONENT_GROUP_LIFECYCLE: u16 = 0x800c; + +const PROOF_EVENT_KIND: u16 = 450; +const PROOF_DOMAIN: &str = "marmot.account-identity-proof.v2"; +const PROOF_CONTENT: &str = "Authorize this MLS leaf key for my Marmot account"; + +// ---------------------------------------------------------------- encoders + +/// QUIC variable-length integer, `foundation/canonical-encoding.md`. +fn put_varint(value: u64, out: &mut Vec) { + if value < 64 { + out.push(value as u8); + } else if value < 16_384 { + out.extend_from_slice(&(0x4000_u16 | value as u16).to_be_bytes()); + } else if value < 1_073_741_824 { + out.extend_from_slice(&(0x8000_0000_u32 | value as u32).to_be_bytes()); + } else { + out.extend_from_slice(&(0xC000_0000_0000_0000_u64 | value).to_be_bytes()); + } +} + +fn put_var_bytes(bytes: &[u8], out: &mut Vec) { + put_varint(bytes.len() as u64, out); + out.extend_from_slice(bytes); +} + +/// `ComponentsList { ComponentID component_ids; }` — ids ascending, no dups. +fn encode_components_list(ids: &[u16]) -> Vec { + let mut sorted = ids.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let mut out = Vec::new(); + put_varint((sorted.len() * 2) as u64, &mut out); + for id in sorted { + out.extend_from_slice(&id.to_be_bytes()); + } + out +} + +/// `marmot.group.profile.v1` — two var-byte UTF-8 fields. +fn encode_group_profile(name: &str, description: &str) -> Vec { + let mut out = Vec::new(); + put_var_bytes(name.as_bytes(), &mut out); + put_var_bytes(description.as_bytes(), &mut out); + out +} + +/// `marmot.group.admin-policy.v1` — one var-byte vector of concatenated +/// 32-byte x-only account keys, sorted and de-duplicated. +fn encode_admin_policy(admins: &[[u8; 32]]) -> Vec { + let mut sorted = admins.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let mut flat = Vec::with_capacity(sorted.len() * 32); + for admin in &sorted { + flat.extend_from_slice(admin); + } + let mut out = Vec::new(); + put_var_bytes(&flat, &mut out); + out +} + +/// `marmot.transport.nostr.routing.v1` — raw 32-byte routing id followed by a +/// var-byte vector of var-byte relay URLs, sorted lexicographically. +fn encode_nostr_routing(nostr_group_id: &[u8; 32], relays: &[&str]) -> Vec { + let mut sorted = relays.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let mut entries = Vec::new(); + for relay in &sorted { + put_var_bytes(relay.as_bytes(), &mut entries); + } + let mut out = Vec::with_capacity(32 + entries.len() + 8); + out.extend_from_slice(nostr_group_id); + put_var_bytes(&entries, &mut out); + out +} + +/// `marmot.group.lifecycle.v1` — exactly one byte; 0x00 active, 0x01 disbanded. +fn encode_group_lifecycle_active() -> Vec { + vec![0x00] +} + +// ------------------------------------------------- account identity proof v2 + +struct ProofMaterial { + component: Vec, + event_json: String, + event_id: [u8; 32], + signature: [u8; 64], + created_at: u64, +} + +/// Build the kind-450 signing template from `app-components/account-identity-proof-v2.md` +/// and return its NIP-01 canonical serialization plus id. +fn proof_event( + account_pubkey: &XOnlyPublicKey, + mls_signature_key: &[u8], + created_at: u64, +) -> (String, [u8; 32]) { + let tags = serde_json::json!([ + ["d", PROOF_DOMAIN], + [ + "component", + format!("0x{COMPONENT_ACCOUNT_IDENTITY_PROOF:04x}") + ], + ["ciphersuite", format!("0x{:04x}", u16::from(CS))], + [ + "signature_scheme", + format!("0x{:04x}", CS.signature_algorithm() as u16) + ], + ["mls_signature_key", hex::encode(mls_signature_key)], + ]); + // NIP-01 canonical form: [0, pubkey, created_at, kind, tags, content]. + // serde_json applies the exact escaping rules the id is defined over. + let canonical = serde_json::json!([ + 0, + hex::encode(account_pubkey.serialize()), + created_at, + PROOF_EVENT_KIND, + tags, + PROOF_CONTENT, + ]); + let serialized = serde_json::to_string(&canonical).unwrap(); + let id: [u8; 32] = Sha256::digest(serialized.as_bytes()).into(); + (serialized, id) +} + +fn build_proof(account: &Keypair, mls_signature_key: &[u8], created_at: u64) -> ProofMaterial { + let secp = Secp256k1::new(); + let (xonly, _parity) = account.x_only_public_key(); + let (event_json, event_id) = proof_event(&xonly, mls_signature_key, created_at); + + // The 32-byte event id is itself the BIP-340 message; Marmot does not + // re-hash it (`account-identity-proof-v2.md`, "Signing event"). + let signature = secp + .sign_schnorr_no_aux_rand(&event_id, account) + .to_byte_array(); + + let mut component = Vec::with_capacity(104); + component.extend_from_slice(&xonly.serialize()); + component.extend_from_slice(&created_at.to_be_bytes()); + component.extend_from_slice(&signature); + assert_eq!(component.len(), 104, "proof component must be 104 bytes"); + + ProofMaterial { + component, + event_json, + event_id, + signature, + created_at, + } +} + +/// Self-check the hand-rolled proof construction against the fixed vector +/// published in `app-components/account-identity-proof-v2.md`. If this trips, +/// the generator is emitting proofs no MDK client will accept — and every +/// downstream vector in this file is worthless. +fn assert_spec_proof_vector() { + let secp = Secp256k1::new(); + let mut sk_bytes = [0_u8; 32]; + sk_bytes[31] = 3; + let account = Keypair::from_secret_key(&secp, &SecretKey::from_byte_array(sk_bytes).unwrap()); + let mls_signature_key: Vec = (0_u8..32).collect(); + let (xonly, _) = account.x_only_public_key(); + assert_eq!( + hex::encode(xonly.serialize()), + "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9", + "spec fixture pubkey mismatch" + ); + + let (json, id) = proof_event(&xonly, &mls_signature_key, 1_700_000_000); + assert_eq!( + json, + r#"[0,"f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9",1700000000,450,[["d","marmot.account-identity-proof.v2"],["component","0x8009"],["ciphersuite","0x0001"],["signature_scheme","0x0807"],["mls_signature_key","000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"]],"Authorize this MLS leaf key for my Marmot account"]"#, + "canonical proof-event serialization does not match the spec fixture" + ); + assert_eq!( + hex::encode(id), + "b7e9a15dd85990fb0f49c33db3cc9875f73986207b038404ceb6b7fec4e0af6b", + "proof event id does not match the spec fixture" + ); + + let expected_component = "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9\ + 000000006553f100\ + c5315d3c85b9d4907cb03395a2a97b3ba2eab393f8e45b13a5d5233acedac60a\ + 51d2a295e1b1b5ee372d18a49bdb8041a7dba9dedce722c7c6f712f78bbdfb5d" + .replace([' ', '\n'], ""); + let proof = build_proof(&account, &mls_signature_key, 1_700_000_000); + assert_eq!( + hex::encode(&proof.component), + expected_component, + "104-byte proof component does not match the spec fixture" + ); +} + +// ------------------------------------------------------------------- member + +struct Member { + account: Keypair, + account_xonly: [u8; 32], + signer: SignatureKeyPair, + credential: CredentialWithKey, +} + +fn new_member(secret_byte: u8) -> Member { + let secp = Secp256k1::new(); + let mut sk_bytes = [0_u8; 32]; + sk_bytes[31] = secret_byte; + let account = Keypair::from_secret_key(&secp, &SecretKey::from_byte_array(sk_bytes).unwrap()); + let (xonly, _) = account.x_only_public_key(); + let account_xonly = xonly.serialize(); + + // foundation/identity.md: the MLS BasicCredential identity is the raw + // 32-byte x-only account key — not hex text, not an npub. + let signer = SignatureKeyPair::new(CS.signature_algorithm()).unwrap(); + let credential = CredentialWithKey { + credential: BasicCredential::new(account_xonly.to_vec()).into(), + signature_key: signer.public().into(), + }; + Member { + account, + account_xonly, + signer, + credential, + } +} + +/// Component ids this generator claims to support, advertised in every leaf. +fn supported_components() -> Vec { + vec![ + COMPONENT_APP_COMPONENTS, + COMPONENT_GROUP_PROFILE, + COMPONENT_ADMIN_POLICY, + COMPONENT_NOSTR_ROUTING, + COMPONENT_ACCOUNT_IDENTITY_PROOF, + COMPONENT_GROUP_LIFECYCLE, + ] +} + +/// Component ids a current-profile group requires. +fn required_components() -> Vec { + vec![ + COMPONENT_GROUP_PROFILE, + COMPONENT_ADMIN_POLICY, + COMPONENT_NOSTR_ROUTING, + COMPONENT_ACCOUNT_IDENTITY_PROOF, + COMPONENT_GROUP_LIFECYCLE, + ] +} + +fn leaf_capabilities() -> Capabilities { + // RFC 9420 section 7.2 forbids advertising default extension types, so + // only the draft app_data_dictionary extension and app_data_update + // proposal appear here. + Capabilities::new( + None, + Some(&[CS]), + Some(&[ExtensionType::AppDataDictionary]), + Some(&[ProposalType::AppDataUpdate]), + None, + ) +} + +fn leaf_extensions(proof_component: &[u8]) -> Extensions { + let mut dict = AppDataDictionary::new(); + dict.insert( + COMPONENT_APP_COMPONENTS, + encode_components_list(&supported_components()), + ); + // Advertising safe_aad support with an empty component list: understood, + // nothing contributed. Required of anyone advertising app_data_dictionary. + dict.insert(COMPONENT_SAFE_AAD, encode_components_list(&[])); + dict.insert(COMPONENT_ACCOUNT_IDENTITY_PROOF, proof_component.to_vec()); + Extensions::single(Extension::AppDataDictionary( + AppDataDictionaryExtension::new(dict), + )) + .unwrap() +} + +fn group_context_extensions( + admins: &[[u8; 32]], + nostr_group_id: &[u8; 32], + relays: &[&str], + name: &str, + description: &str, +) -> Extensions { + let mut dict = AppDataDictionary::new(); + dict.insert( + COMPONENT_APP_COMPONENTS, + encode_components_list(&required_components()), + ); + dict.insert( + COMPONENT_GROUP_PROFILE, + encode_group_profile(name, description), + ); + dict.insert(COMPONENT_ADMIN_POLICY, encode_admin_policy(admins)); + dict.insert( + COMPONENT_NOSTR_ROUTING, + encode_nostr_routing(nostr_group_id, relays), + ); + dict.insert(COMPONENT_GROUP_LIFECYCLE, encode_group_lifecycle_active()); + // 0x8009 is required but leaf-only: its data must NOT appear here. + + Extensions::from_vec(vec![ + Extension::RequiredCapabilities(RequiredCapabilitiesExtension::new( + &[ExtensionType::AppDataDictionary], + &[ProposalType::AppDataUpdate], + &[], + )), + Extension::AppDataDictionary(AppDataDictionaryExtension::new(dict)), + ]) + .unwrap() +} + +fn dictionary_entries_json(dictionary: Option<&AppDataDictionaryExtension>) -> serde_json::Value { + let mut entries = serde_json::Map::new(); + if let Some(ext) = dictionary { + for entry in ext.dictionary().entries() { + entries.insert( + format!("0x{:04x}", entry.id()), + serde_json::Value::String(hex::encode(entry.data())), + ); + } + } + serde_json::Value::Object(entries) +} + +fn main() { + assert_spec_proof_vector(); + + let provider_a = OpenMlsRustCrypto::default(); + let provider_b = OpenMlsRustCrypto::default(); + + let alice = new_member(0x11); + alice.signer.store(provider_a.storage()).unwrap(); + let bob = new_member(0x22); + bob.signer.store(provider_b.storage()).unwrap(); + + let created_at = 1_700_000_000_u64; + let alice_proof = build_proof(&alice.account, alice.signer.public(), created_at); + let bob_proof = build_proof(&bob.account, bob.signer.public(), created_at); + + // Bob publishes a last-resort KeyPackage in the current profile. + let bob_kp_bundle = KeyPackage::builder() + .leaf_node_capabilities(leaf_capabilities()) + .leaf_node_extensions(leaf_extensions(&bob_proof.component)) + .mark_as_last_resort() + .build(CS, &provider_b, &bob.signer, bob.credential.clone()) + .unwrap(); + let bob_kp = bob_kp_bundle.key_package().clone(); + let bob_kp_msg: MlsMessageOut = MlsMessageOut::from(bob_kp.clone()); + let bob_kp_msg_bytes = bob_kp_msg.tls_serialize_detached().unwrap(); + let bob_init_priv: Vec = (**bob_kp_bundle.init_private_key()).to_vec(); + let bob_enc_priv: Vec = (**bob_kp_bundle.encryption_private_key()).to_vec(); + let bob_kp_dict = dictionary_entries_json(bob_kp.extensions().app_data_dictionary()); + let bob_leaf_dict = + dictionary_entries_json(bob_kp.leaf_node().extensions().app_data_dictionary()); + + let nostr_group_id = [0x5a_u8; 32]; + let relays = ["wss://nos.lol", "wss://relay.damus.io"]; + let gc_exts = group_context_extensions( + &[alice.account_xonly], + &nostr_group_id, + &relays, + "Marmot interop", + "current-profile fixture", + ); + + let group_cfg = MlsGroupCreateConfig::builder() + .ciphersuite(CS) + .capabilities(leaf_capabilities()) + .with_leaf_node_extensions(leaf_extensions(&alice_proof.component)) + .unwrap() + .wire_format_policy(openmls::group::PURE_PLAINTEXT_WIRE_FORMAT_POLICY) + .with_group_context_extensions(gc_exts) + .use_ratchet_tree_extension(true) + .build(); + + let mut alice_group = MlsGroup::new( + &provider_a, + &alice.signer, + &group_cfg, + alice.credential.clone(), + ) + .unwrap(); + + let epoch0_gc_dict = dictionary_entries_json(alice_group.extensions().app_data_dictionary()); + let alice_leaf_dict = dictionary_entries_json( + alice_group + .own_leaf_node() + .unwrap() + .extensions() + .app_data_dictionary(), + ); + + let (commit_out, welcome_out, _group_info) = alice_group + .add_members(&provider_a, &alice.signer, &[bob_kp.clone()]) + .unwrap(); + alice_group.merge_pending_commit(&provider_a).unwrap(); + + // The Add commit is a PublicMessage under the Marmot handshake profile — + // the exact shape our peeler has to authenticate and replay. + let add_commit_bytes = commit_out.tls_serialize_detached().unwrap(); + let welcome_bytes = welcome_out.tls_serialize_detached().unwrap(); + + let welcome_in = MlsMessageIn::tls_deserialize(&mut welcome_bytes.as_slice()).unwrap(); + let welcome = match welcome_in.extract() { + MlsMessageBodyIn::Welcome(w) => w, + other => panic!("expected Welcome, got {other:?}"), + }; + let join_cfg = MlsGroupJoinConfig::builder().build(); + let staged = StagedWelcome::new_from_welcome(&provider_b, &join_cfg, welcome, None).unwrap(); + let bob_group = staged.into_group(&provider_b).unwrap(); + + let group_event_secret = bob_group + .export_secret(provider_b.crypto(), "marmot", b"group-event", 32) + .unwrap(); + // foundation/conformance.md — the synthetic state-commitment exporter. + let conformance_secret = bob_group + .export_secret( + provider_b.crypto(), + "marmot", + b"convergence-conformance-v1", + 32, + ) + .unwrap(); + let conformance_commitment: [u8; 32] = { + let mut hasher = Sha256::new(); + hasher.update(b"marmot-convergence-conformance-v1"); + hasher.update([0x00]); + hasher.update(&conformance_secret); + hasher.finalize().into() + }; + + let plaintexts: Vec<&[u8]> = vec![ + b"Hello Bob!".as_ref(), + b"Second message in the same epoch.".as_ref(), + ]; + let mut app_messages = Vec::new(); + for pt in &plaintexts { + let out = alice_group + .create_message(&provider_a, &alice.signer, pt) + .unwrap(); + app_messages.push(serde_json::json!({ + "plaintext": hex::encode(pt), + "private_message": hex::encode(out.tls_serialize_detached().unwrap()), + })); + } + + let vector = serde_json::json!({ + "cipher_suite": u16::from(CS), + "signature_scheme": CS.signature_algorithm() as u16, + "description": + "Current-profile Marmot group (app_data_dictionary + account-identity-proof v2), \ + built on the OpenMLS extensions-draft fork MDK pins.", + "profile": "current", + "handshake_wire_format": "public_message", + "required_capabilities": { + "extensions": ["0x0006"], + "proposals": ["0x0008"], + }, + "component_ids": { + "app_components": format!("0x{COMPONENT_APP_COMPONENTS:04x}"), + "safe_aad": format!("0x{COMPONENT_SAFE_AAD:04x}"), + "last_resort_key_package": format!("0x{COMPONENT_LAST_RESORT:04x}"), + "group_profile_v1": format!("0x{COMPONENT_GROUP_PROFILE:04x}"), + "admin_policy_v1": format!("0x{COMPONENT_ADMIN_POLICY:04x}"), + "nostr_routing_v1": format!("0x{COMPONENT_NOSTR_ROUTING:04x}"), + "account_identity_proof_v2": format!("0x{COMPONENT_ACCOUNT_IDENTITY_PROOF:04x}"), + "group_lifecycle_v1": format!("0x{COMPONENT_GROUP_LIFECYCLE:04x}"), + }, + "group_state": { + "nostr_group_id": hex::encode(nostr_group_id), + "relays": relays, + "name": "Marmot interop", + "description": "current-profile fixture", + "admins": [hex::encode(alice.account_xonly)], + "epoch0_group_context_dictionary": epoch0_gc_dict, + "epoch1_group_context_dictionary": + dictionary_entries_json(alice_group.extensions().app_data_dictionary()), + }, + "committer": { + "account_pubkey": hex::encode(alice.account_xonly), + "signer_pub": hex::encode(alice.signer.public()), + "leaf_dictionary": alice_leaf_dict, + "account_identity_proof": { + "component": hex::encode(&alice_proof.component), + "created_at": alice_proof.created_at, + "event_json": alice_proof.event_json, + "event_id": hex::encode(alice_proof.event_id), + "signature": hex::encode(alice_proof.signature), + }, + }, + "joiner": { + "account_pubkey": hex::encode(bob.account_xonly), + "init_priv": hex::encode(&bob_init_priv), + "encryption_priv": hex::encode(&bob_enc_priv), + "signature_priv": hex::encode(bob.signer.private()), + "signature_pub": hex::encode(bob.signer.public()), + "key_package": hex::encode(&bob_kp_msg_bytes), + "key_package_dictionary": bob_kp_dict, + "leaf_dictionary": bob_leaf_dict, + "account_identity_proof": { + "component": hex::encode(&bob_proof.component), + "created_at": bob_proof.created_at, + "event_json": bob_proof.event_json, + "event_id": hex::encode(bob_proof.event_id), + "signature": hex::encode(bob_proof.signature), + }, + }, + "add_commit_public_message": hex::encode(&add_commit_bytes), + "welcome": hex::encode(&welcome_bytes), + "exporters": { + "group_event": { + "label": "marmot", + "context": hex::encode(b"group-event"), + "length": 32, + "secret": hex::encode(&group_event_secret), + }, + "convergence_conformance_v1": { + "label": "marmot", + "context": hex::encode(b"convergence-conformance-v1"), + "length": 32, + "commitment": hex::encode(conformance_commitment), + }, + }, + "app_messages_alice_to_bob": app_messages, + }); + println!("{}", serde_json::to_string_pretty(&vector).unwrap()); +} diff --git a/quartz/tools/mdk-vector-gen/src/verify_amethyst.rs b/quartz/tools/mdk-vector-gen/src/verify_amethyst.rs index d9e90f8e65..6b353e60a6 100644 --- a/quartz/tools/mdk-vector-gen/src/verify_amethyst.rs +++ b/quartz/tools/mdk-vector-gen/src/verify_amethyst.rs @@ -14,13 +14,13 @@ use std::env; use std::fs::{self, File}; use std::process; +use ::tls_codec::Deserialize as TlsDeserialize; use base64::Engine; use openmls::prelude::*; use openmls_rust_crypto::OpenMlsRustCrypto; use openmls_traits::OpenMlsProvider; use serde::Deserialize; use std::collections::HashMap; -use tls_codec::Deserialize as TlsDeserialize; #[derive(Deserialize)] struct Handoff {