diff --git a/testing/mesh-lab/README.md b/testing/mesh-lab/README.md index f036363..565911a 100644 --- a/testing/mesh-lab/README.md +++ b/testing/mesh-lab/README.md @@ -135,6 +135,18 @@ each rep does, set them in the invoking shell: containers; the sidecar fills that gap. Only applies to the `nat-lan` suite; other suites ignore it. +- **`FIPS_MESH_LAB_TRACE_TREE`** — when set to any non-empty value, + layers `compose-trace-tree.yml` over the rekey-family compose stack + to bump `RUST_LOG` to trace level on `fips::node::tree`, + `fips::tree`, `fips::node::handlers::mmp`, and + `fips::node::handlers::handshake`. Distinct from + `FIPS_MESH_LAB_TRACE` (rekey/forwarding/session/encrypted at trace); + targeted at tree-partition race investigation during multi-peer + startup. Mutually exclusive with `FIPS_MESH_LAB_TRACE` in practice — + both env vars layer their overlay, but the second one's per-service + environment replaces the first's. Only applies to the rekey-family + suites. + - **`FIPS_MESH_LAB_NO_RESOURCE_LIMITS`** — when set to any non-empty value, omits the `compose-resource-limits.yml` overlay for rekey-family runs. Default behaviour keeps the overlay engaged so rekey-family lab diff --git a/testing/mesh-lab/compose-trace-tree.yml b/testing/mesh-lab/compose-trace-tree.yml new file mode 100644 index 0000000..e6a3a20 --- /dev/null +++ b/testing/mesh-lab/compose-trace-tree.yml @@ -0,0 +1,71 @@ +# Compose override that bumps RUST_LOG to trace level on the modules +# relevant to tree-partition / parent-re-eval investigations: +# +# - fips::node::tree — handle_tree_announce, +# evaluate_parent decisions, +# send_tree_announce_to_all +# recipient enumeration +# - fips::node::handlers::mmp — MMP receiver-report processing, +# first-RTT trigger, SRTT updates +# - fips::node::handlers::handshake — peer promotion timing +# (correlate with tree-announce +# broadcast window) +# - fips::tree — TreeState::evaluate_parent +# internal predicate flow +# +# Distinct from compose-trace.yml (rekey-class flake) — that overlay +# pulls in rekey / forwarding / session / encrypted at trace level +# which would drown the tree-convergence signal. This overlay is +# scoped to the tree-partition mechanism only. +# +# Service-name layout matches compose-trace.yml: per-profile services +# named `-` for the three rekey-family profiles. +# All 15 rekey-family services get the same trace RUST_LOG. +# +# Include this override alongside the base compose via: +# docker compose -f testing/static/docker-compose.yml \ +# -f testing/mesh-lab/compose-trace-tree.yml up -d +# +# The run-loop.sh harness includes it automatically when the mesh-lab +# environment variable FIPS_MESH_LAB_TRACE_TREE=1 is set. Mutually +# exclusive with FIPS_MESH_LAB_TRACE — set one or the other, not both. + +x-trace-tree-rust-log: &trace-tree-rust-log + RUST_LOG: "info,fips::node::tree=trace,fips::tree=trace,fips::node::handlers::mmp=trace,fips::node::handlers::handshake=trace" + +services: + # rekey profile + rekey-a: + environment: *trace-tree-rust-log + rekey-b: + environment: *trace-tree-rust-log + rekey-c: + environment: *trace-tree-rust-log + rekey-d: + environment: *trace-tree-rust-log + rekey-e: + environment: *trace-tree-rust-log + + # rekey-accept-off profile + rekey-accept-off-a: + environment: *trace-tree-rust-log + rekey-accept-off-b: + environment: *trace-tree-rust-log + rekey-accept-off-c: + environment: *trace-tree-rust-log + rekey-accept-off-d: + environment: *trace-tree-rust-log + rekey-accept-off-e: + environment: *trace-tree-rust-log + + # rekey-outbound-only profile + rekey-outbound-only-a: + environment: *trace-tree-rust-log + rekey-outbound-only-b: + environment: *trace-tree-rust-log + rekey-outbound-only-c: + environment: *trace-tree-rust-log + rekey-outbound-only-d: + environment: *trace-tree-rust-log + rekey-outbound-only-e: + environment: *trace-tree-rust-log diff --git a/testing/mesh-lab/run-loop.sh b/testing/mesh-lab/run-loop.sh index 2061bf3..9fa9487 100755 --- a/testing/mesh-lab/run-loop.sh +++ b/testing/mesh-lab/run-loop.sh @@ -25,6 +25,14 @@ # of the base + resource-limits stack to # bump RUST_LOG to trace on rekey/handshake/ # forwarding/session/encrypted/mmp modules. +# FIPS_MESH_LAB_TRACE_TREE +# when set, layers compose-trace-tree.yml to +# bump RUST_LOG to trace on tree/mmp/handshake +# modules. Targeted at tree-partition race +# investigation during multi-peer startup. +# Mutually exclusive with FIPS_MESH_LAB_TRACE in +# practice — both apply but the second overlay +# replaces the first's per-service environment. # FIPS_MESH_LAB_NO_RESOURCE_LIMITS # when set, omits the compose-resource-limits.yml # overlay for rekey-family runs. Use for @@ -187,6 +195,9 @@ run_rekey_family() { if [ -n "${FIPS_MESH_LAB_TRACE:-}" ]; then compose_args+=(-f testing/mesh-lab/compose-trace.yml) fi + if [ -n "${FIPS_MESH_LAB_TRACE_TREE:-}" ]; then + compose_args+=(-f testing/mesh-lab/compose-trace-tree.yml) + fi compose_args+=(--profile "$compose_profile") ( @@ -361,16 +372,20 @@ mechanism_match_rekey() { echo "false" return fi - local pairs phase6 p1_status p1_passed + local pairs phase6 p1_status pairs=$(jq -r '.phase5_failing_pairs' "$sig") phase6=$(jq -r '.phase6_log_analysis' "$sig") p1_status=$(jq -r '.phase1_status' "$sig") - p1_passed=$(jq -r '.phase1_baseline_passed' "$sig") if [ -n "$pairs" ] && [ "$phase6" = "all-green" ]; then echo "true" return fi - if [ "$p1_status" = "timeout" ] && [ "$p1_passed" = "12" ]; then + # Any Phase 1 baseline-convergence timeout — the tree-partition + # surfaces with a variable count of failing multi-hop pairs depending + # on which node misses its parent re-evaluation (e.g., node-c orphan + # → 12/20 PASS, node-b orphan → 14/20). The exact count remains in + # signature.json for cross-rep analysis. + if [ "$p1_status" = "timeout" ]; then echo "true" return fi