mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 00:04:54 +00:00
Admit rekey msg1 from established peers when addr forms differ
Companion to the ethernet `accept_connections: false` rekey-deadlock fix from earlier this release: the same dual-init failure mode shows up over UDP when peers register by hostname, and the existing addr_to_link-only carve-out in `should_admit_msg1` doesn't cover it. The carve-out's first predicate keys `addr_to_link` by the literal `TransportAddr` that `initiate_connection` inserted, which is the hostname-form when a peer config carries a hostname (e.g., `core-vm.tail65015.ts.net:2121`). Inbound packets always arrive with numeric source addrs because `udp_receive_loop` builds the `TransportAddr` from the `SocketAddr` the kernel reports via `recvfrom`. `TransportAddr` equality is byte-exact, so the two forms don't match and the lookup misses. With `udp.accept_connections: false` (or `udp.outbound_only: true`, which forces it false) the gate then rejects the rekey msg1 from an established peer. The dual-init tie-breaker stalls because the loser side never produces msg2; both sides retry indefinitely and the winner side keeps logging "Dual rekey initiation: we win, dropping their msg1" at 1Hz. The earlier ethernet fix didn't generalize to this variant because ethernet TransportAddrs are always numeric MAC bytes — both the config-time form and the inbound-arrival form match identically. Add a second predicate to `should_admit_msg1`: an active peer's `current_addr()` matching `(transport_id, remote_addr)`. `current_addr` is updated and refreshed from inbound encrypted-frame source addrs (`handlers/encrypted.rs`), which are always numeric `SocketAddr`-form, so this catches the established peer regardless of how its `addr_to_link` key was originally inserted. The fast `addr_to_link` check stays first; the iteration over peers is bounded by peer count and only runs when the first predicate misses. Regression coverage in this commit: - Unit test `test_should_admit_msg1_admits_rekey_when_addr_form_differs` in `src/node/tests/handshake.rs`. Constructs the failing scenario in-process: `addr_to_link` populated with hostname-form key, peer's `current_addr` at the resolved numeric form, query with numeric form. Without the new predicate this fails immediately. - New integration topology `rekey-outbound-only` plus matching docker-compose profile. Same 5-node mesh shape as `rekey-accept-off` but `inject-config` sets `udp.outbound_only: true` on node-b and rewrites node-b's peer-c address from the numeric docker IP to the docker hostname (`node-c:2121`), reproducing the production hostname-vs-numeric mismatch. The test asserts no sustained "Dual rekey initiation: we win" log lines on any node (>10 = bug) and the existing rekey health checks catch the connectivity loss the loop produces. - `testing/ci-local.sh` and `.github/workflows/ci.yml` extended to run the new variant in the local sweep and the GitHub CI integration matrix alongside `rekey` and `rekey-accept-off`. Verified locally: full `bash testing/ci-local.sh` sweep passes 29/29 suites (23m 12s) with the new variant green; 1084 unit tests pass.
This commit is contained in:
@@ -32,6 +32,16 @@ NODES="a b c d e"
|
||||
# initiation" log lines appear on the affected node.
|
||||
REKEY_ACCEPT_OFF_NODES="${REKEY_ACCEPT_OFF_NODES:-}"
|
||||
|
||||
# Comma-separated list of node IDs to set udp.outbound_only=true on
|
||||
# during inject-config. For each such node, peer addresses are also
|
||||
# rewritten from numeric docker IPs to docker hostnames (e.g.
|
||||
# 172.20.0.12:2121 → node-c:2121). This reproduces the production
|
||||
# scenario where peer configs carry hostnames so the `addr_to_link`
|
||||
# key is hostname-form while inbound packet source addrs are numeric,
|
||||
# making the should_admit_msg1 carve-out's `addr_to_link.contains_key`
|
||||
# check miss.
|
||||
REKEY_OUTBOUND_ONLY_NODES="${REKEY_OUTBOUND_ONLY_NODES:-}"
|
||||
|
||||
# Rekey timing configuration
|
||||
REKEY_AFTER_SECS=35
|
||||
|
||||
@@ -43,6 +53,9 @@ if [ "${1:-}" = "inject-config" ]; then
|
||||
if [ -n "$REKEY_ACCEPT_OFF_NODES" ]; then
|
||||
echo " Setting udp.accept_connections=false on nodes: $REKEY_ACCEPT_OFF_NODES"
|
||||
fi
|
||||
if [ -n "$REKEY_OUTBOUND_ONLY_NODES" ]; then
|
||||
echo " Setting udp.outbound_only=true + rewriting peer addrs to docker hostnames on nodes: $REKEY_OUTBOUND_ONLY_NODES"
|
||||
fi
|
||||
for node in $NODES; do
|
||||
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-$node.yaml"
|
||||
if [ ! -f "$cfg" ]; then
|
||||
@@ -57,6 +70,14 @@ if [ "${1:-}" = "inject-config" ]; then
|
||||
fi
|
||||
done
|
||||
fi
|
||||
outbound_only="false"
|
||||
if [ -n "$REKEY_OUTBOUND_ONLY_NODES" ]; then
|
||||
for oo_node in ${REKEY_OUTBOUND_ONLY_NODES//,/ }; do
|
||||
if [ "$oo_node" = "$node" ]; then
|
||||
outbound_only="true"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
python3 -c "
|
||||
import yaml
|
||||
with open('$cfg') as f:
|
||||
@@ -74,14 +95,48 @@ if '$accept_off' == 'true':
|
||||
transports['udp'] = udp
|
||||
if isinstance(udp, dict):
|
||||
udp['accept_connections'] = False
|
||||
if '$outbound_only' == 'true':
|
||||
transports = cfg.setdefault('transports', {})
|
||||
udp = transports.get('udp')
|
||||
if udp is None:
|
||||
udp = {}
|
||||
transports['udp'] = udp
|
||||
if isinstance(udp, dict):
|
||||
udp['outbound_only'] = True
|
||||
# Rewrite peer addrs to docker hostnames so the addr_to_link key
|
||||
# is hostname-form (mirroring production peer configs that carry
|
||||
# hostnames). Without this, peer addrs are numeric and the
|
||||
# carve-out's addr_to_link lookup matches inbound numeric source
|
||||
# addrs, masking the bug.
|
||||
ip_to_host = {
|
||||
'172.20.0.10': 'node-a',
|
||||
'172.20.0.11': 'node-b',
|
||||
'172.20.0.12': 'node-c',
|
||||
'172.20.0.13': 'node-d',
|
||||
'172.20.0.14': 'node-e',
|
||||
}
|
||||
for peer in cfg.get('peers', []) or []:
|
||||
for addr in peer.get('addresses', []) or []:
|
||||
t = addr.get('transport')
|
||||
if t is not None and t != 'udp':
|
||||
continue
|
||||
a = addr.get('addr', '')
|
||||
for ip, host in ip_to_host.items():
|
||||
if a.startswith(ip + ':'):
|
||||
port = a.split(':', 1)[1]
|
||||
addr['addr'] = f'{host}:{port}'
|
||||
break
|
||||
with open('$cfg', 'w') as f:
|
||||
yaml.dump(cfg, f, default_flow_style=False, sort_keys=False)
|
||||
"
|
||||
suffix=""
|
||||
if [ "$accept_off" = "true" ]; then
|
||||
echo " ✓ node-$node (accept_connections=false)"
|
||||
else
|
||||
echo " ✓ node-$node"
|
||||
suffix=" (accept_connections=false)"
|
||||
fi
|
||||
if [ "$outbound_only" = "true" ]; then
|
||||
suffix=" (outbound_only=true, hostname peer addrs)"
|
||||
fi
|
||||
echo " ✓ node-$node$suffix"
|
||||
done
|
||||
echo "✓ Config injection complete"
|
||||
exit 0
|
||||
@@ -368,6 +423,29 @@ if [ -n "$REKEY_ACCEPT_OFF_NODES" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# Variant-specific: udp.outbound_only=true. The pre-fix bug fired the
|
||||
# dual-init loop on the OTHER side (the peer of the outbound-only node)
|
||||
# because the outbound-only side rejects the inbound rekey msg1 due to
|
||||
# the addr_to_link hostname-vs-numeric mismatch, leaving the peer's
|
||||
# rekey state in a 1Hz retry loop that the outbound-only side keeps
|
||||
# dropping. The exact node that emits "we win" depends on which side
|
||||
# has the smaller NodeAddr, so check all five nodes for the sustained-
|
||||
# loop signature.
|
||||
if [ -n "$REKEY_OUTBOUND_ONLY_NODES" ]; then
|
||||
DUAL_INIT_THRESHOLD=10
|
||||
for n in $NODES; do
|
||||
count=$(docker logs "fips-node-$n" 2>&1 \
|
||||
| grep -cE "Dual rekey initiation: we win" || true)
|
||||
if [ "${count:-0}" -le "$DUAL_INIT_THRESHOLD" ]; then
|
||||
echo " PASS: node-$n dual-init drops below threshold ($count <= $DUAL_INIT_THRESHOLD)"
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
echo " FAIL: node-$n sustained dual-init drops ($count > $DUAL_INIT_THRESHOLD)"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
phase_result "Log analysis"
|
||||
echo ""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user