mesh-lab: tree/mmp-targeted trace overlay (compose-trace-tree.yml + env-var gate)

Adds a tree/mmp-targeted compose overlay (compose-trace-tree.yml) and
a new FIPS_MESH_LAB_TRACE_TREE env-var gate in run_rekey_family,
layered independently of the existing FIPS_MESH_LAB_TRACE rekey-class
overlay. Trace targets: fips::node::tree, fips::tree,
fips::node::handlers::mmp, fips::node::handlers::handshake.

Used for tree-partition race investigations during multi-peer startup
where evaluate_parent inputs, send_tree_announce_to_all recipient
enumeration, and process_receiver_report first-RTT triggers all need
to be visible together to bracket the loss window.

testing/mesh-lab/README.md

@@ -135,6 +135,18 @@ each rep does, set them in the invoking shell:
   containers; the sidecar fills that gap. Only applies to the
   `nat-lan` suite; other suites ignore it.
 
+- **`FIPS_MESH_LAB_TRACE_TREE`** — when set to any non-empty value,
+  layers `compose-trace-tree.yml` over the rekey-family compose stack
+  to bump `RUST_LOG` to trace level on `fips::node::tree`,
+  `fips::tree`, `fips::node::handlers::mmp`, and
+  `fips::node::handlers::handshake`. Distinct from
+  `FIPS_MESH_LAB_TRACE` (rekey/forwarding/session/encrypted at trace);
+  targeted at tree-partition race investigation during multi-peer
+  startup. Mutually exclusive with `FIPS_MESH_LAB_TRACE` in practice —
+  both env vars layer their overlay, but the second one's per-service
+  environment replaces the first's. Only applies to the rekey-family
+  suites.
+
 - **`FIPS_MESH_LAB_NO_RESOURCE_LIMITS`** — when set to any non-empty
   value, omits the `compose-resource-limits.yml` overlay for rekey-family
   runs. Default behaviour keeps the overlay engaged so rekey-family lab

testing/mesh-lab/compose-trace-tree.yml

@@ -0,0 +1,71 @@
+# Compose override that bumps RUST_LOG to trace level on the modules
+# relevant to tree-partition / parent-re-eval investigations:
+#
+#   - fips::node::tree                  — handle_tree_announce,
+#                                          evaluate_parent decisions,
+#                                          send_tree_announce_to_all
+#                                          recipient enumeration
+#   - fips::node::handlers::mmp         — MMP receiver-report processing,
+#                                          first-RTT trigger, SRTT updates
+#   - fips::node::handlers::handshake   — peer promotion timing
+#                                          (correlate with tree-announce
+#                                          broadcast window)
+#   - fips::tree                        — TreeState::evaluate_parent
+#                                          internal predicate flow
+#
+# Distinct from compose-trace.yml (rekey-class flake) — that overlay
+# pulls in rekey / forwarding / session / encrypted at trace level
+# which would drown the tree-convergence signal. This overlay is
+# scoped to the tree-partition mechanism only.
+#
+# Service-name layout matches compose-trace.yml: per-profile services
+# named `<profile>-<node-letter>` for the three rekey-family profiles.
+# All 15 rekey-family services get the same trace RUST_LOG.
+#
+# Include this override alongside the base compose via:
+#   docker compose -f testing/static/docker-compose.yml \
+#                  -f testing/mesh-lab/compose-trace-tree.yml up -d
+#
+# The run-loop.sh harness includes it automatically when the mesh-lab
+# environment variable FIPS_MESH_LAB_TRACE_TREE=1 is set. Mutually
+# exclusive with FIPS_MESH_LAB_TRACE — set one or the other, not both.
+
+x-trace-tree-rust-log: &trace-tree-rust-log
+  RUST_LOG: "info,fips::node::tree=trace,fips::tree=trace,fips::node::handlers::mmp=trace,fips::node::handlers::handshake=trace"
+
+services:
+  # rekey profile
+  rekey-a:
+    environment: *trace-tree-rust-log
+  rekey-b:
+    environment: *trace-tree-rust-log
+  rekey-c:
+    environment: *trace-tree-rust-log
+  rekey-d:
+    environment: *trace-tree-rust-log
+  rekey-e:
+    environment: *trace-tree-rust-log
+
+  # rekey-accept-off profile
+  rekey-accept-off-a:
+    environment: *trace-tree-rust-log
+  rekey-accept-off-b:
+    environment: *trace-tree-rust-log
+  rekey-accept-off-c:
+    environment: *trace-tree-rust-log
+  rekey-accept-off-d:
+    environment: *trace-tree-rust-log
+  rekey-accept-off-e:
+    environment: *trace-tree-rust-log
+
+  # rekey-outbound-only profile
+  rekey-outbound-only-a:
+    environment: *trace-tree-rust-log
+  rekey-outbound-only-b:
+    environment: *trace-tree-rust-log
+  rekey-outbound-only-c:
+    environment: *trace-tree-rust-log
+  rekey-outbound-only-d:
+    environment: *trace-tree-rust-log
+  rekey-outbound-only-e:
+    environment: *trace-tree-rust-log

testing/mesh-lab/run-loop.sh

@@ -25,6 +25,14 @@
 #                           of the base + resource-limits stack to
 #                           bump RUST_LOG to trace on rekey/handshake/
 #                           forwarding/session/encrypted/mmp modules.
+#   FIPS_MESH_LAB_TRACE_TREE
+#                           when set, layers compose-trace-tree.yml to
+#                           bump RUST_LOG to trace on tree/mmp/handshake
+#                           modules. Targeted at tree-partition race
+#                           investigation during multi-peer startup.
+#                           Mutually exclusive with FIPS_MESH_LAB_TRACE in
+#                           practice — both apply but the second overlay
+#                           replaces the first's per-service environment.
 #   FIPS_MESH_LAB_NO_RESOURCE_LIMITS
 #                           when set, omits the compose-resource-limits.yml
 #                           overlay for rekey-family runs. Use for
@@ -187,6 +195,9 @@ run_rekey_family() {
     if [ -n "${FIPS_MESH_LAB_TRACE:-}" ]; then
         compose_args+=(-f testing/mesh-lab/compose-trace.yml)
     fi
+    if [ -n "${FIPS_MESH_LAB_TRACE_TREE:-}" ]; then
+        compose_args+=(-f testing/mesh-lab/compose-trace-tree.yml)
+    fi
     compose_args+=(--profile "$compose_profile")
 
     (
@@ -361,16 +372,20 @@ mechanism_match_rekey() {
         echo "false"
         return
     fi
-    local pairs phase6 p1_status p1_passed
+    local pairs phase6 p1_status
     pairs=$(jq -r '.phase5_failing_pairs' "$sig")
     phase6=$(jq -r '.phase6_log_analysis' "$sig")
     p1_status=$(jq -r '.phase1_status' "$sig")
-    p1_passed=$(jq -r '.phase1_baseline_passed' "$sig")
     if [ -n "$pairs" ] && [ "$phase6" = "all-green" ]; then
         echo "true"
         return
     fi
-    if [ "$p1_status" = "timeout" ] && [ "$p1_passed" = "12" ]; then
+    # Any Phase 1 baseline-convergence timeout — the tree-partition
+    # surfaces with a variable count of failing multi-hop pairs depending
+    # on which node misses its parent re-evaluation (e.g., node-c orphan
+    # → 12/20 PASS, node-b orphan → 14/20). The exact count remains in
+    # signature.json for cross-rep analysis.
+    if [ "$p1_status" = "timeout" ]; then
         echo "true"
         return
     fi