diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28d29130..ea094f14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -618,6 +618,9 @@ jobs: # ── Firewall baseline (fips0 nftables default-deny) ──────────── - suite: firewall type: firewall + # ── Dynamic interface binding (absent → present → absent) ────── + - suite: iface-binding + type: iface-binding # ── Outbound LAN gateway integration test ────────────────────── - suite: gateway type: gateway @@ -633,6 +636,9 @@ jobs: - suite: ethernet-only type: chaos scenario: ethernet-only + - suite: ethernet-churn + type: chaos + scenario: ethernet-churn - suite: tcp-mesh type: chaos scenario: tcp-mesh @@ -768,6 +774,22 @@ jobs: run: | docker compose -f testing/firewall/docker-compose.yml down --volumes --remove-orphans + # ── Dynamic interface binding integration test ───────────────────────── + - name: Run interface binding integration test + if: matrix.type == 'iface-binding' + run: bash testing/iface-binding/test.sh --skip-build --keep-up + + - name: Collect logs on failure (iface-binding) + if: matrix.type == 'iface-binding' && failure() + run: | + docker compose -f testing/iface-binding/docker-compose.yml logs --no-color + docker exec fips-ifb-node-a fipsctl show transports || true + + - name: Stop containers (iface-binding) + if: matrix.type == 'iface-binding' && always() + run: | + docker compose -f testing/iface-binding/docker-compose.yml down --volumes --remove-orphans + # ── Chaos simulation ─────────────────────────────────────────────────── - name: Install Python deps (chaos) if: matrix.type == 'chaos' diff --git a/src/transport/ethernet/mod.rs b/src/transport/ethernet/mod.rs index 6e459a01..ef4521dc 100644 --- a/src/transport/ethernet/mod.rs +++ b/src/transport/ethernet/mod.rs @@ -1849,8 +1849,18 @@ mod tests { assert!(panicked.is_err(), "the helper thread was supposed to panic"); assert!(eth.binding.tasks.is_poisoned()); - // Still answerable, and teardown still runs to completion. - let _ = eth.binding.tasks_alive(); + // Still answerable, and — the part that matters — it answers the way + // that keeps the transport recoverable. Discarding this, which the + // test used to do, left the whole point unasserted: treating a + // poisoned lock as "tasks are alive" would make the binder believe a + // dead binding is healthy and never rebind it, and treating it as an + // error would strand the transport instead. `false` is what routes it + // back through detach and rebind. + assert!( + !eth.binding.tasks_alive(), + "a poisoned task lock must read as a dead binding, so the binder \ + rebinds rather than either believing it healthy or giving up" + ); eth.stop_async().await.expect("stop"); assert_eq!(eth.state(), TransportState::Down); } @@ -2109,9 +2119,27 @@ mod tests { local_pubkey: None, presence_tx: None, }); + // Assert the *reason*, not merely that it errored. This transport's + // interface does not exist, so `bind_and_spawn` refuses at its + // presence probe — before it ever reaches the post-store shutdown + // check. Asserting `is_err()` alone therefore proved nothing about + // the stop flag: the assertion passed on absence, and would still + // pass with the shutdown check deleted outright. + // + // What is covered here is the observable half of the race — stop + // raises the flag before tearing down, and teardown leaves no socket + // and no loops. The post-store check itself needs a bind that + // *succeeds*, which needs a real bindable interface and the privilege + // to open a raw socket on it; that path is exercised only under the + // docker suite, and no unit test can reach it unprivileged. + let err = bind_and_spawn(&ctx) + .await + .expect_err("a bind must not complete for a stopped transport"); assert!( - bind_and_spawn(&ctx).await.is_err(), - "a bind must not complete for a stopped transport" + matches!(err, TransportError::InterfaceUnavailable { .. }), + "expected the absence refusal, got {err:?} — if this ever becomes \ + a shutdown refusal, this test has started covering the race it \ + is named for and the comment above is stale" ); assert!(eth.binding.socket().is_none()); } diff --git a/src/transport/ethernet/watcher.rs b/src/transport/ethernet/watcher.rs index 5e00b6bf..4311f98c 100644 --- a/src/transport/ethernet/watcher.rs +++ b/src/transport/ethernet/watcher.rs @@ -264,9 +264,34 @@ mod tests { #[tokio::test] async fn watcher_constructs_and_reports_its_backing() { let w = LinkWatcher::new(); - // Both answers are legitimate — a sandbox may refuse the socket — so - // this pins that asking is safe, not which answer comes back. - let _ = w.is_event_driven(); + + // On Linux the source is a plain `AF_NETLINK` socket in the + // `RTNLGRP_LINK` group, which needs no capability and no privilege — + // so on this platform "a sandbox might refuse it" is not a licence to + // accept either answer. Discarding the result, which this test used + // to do, meant nothing anywhere asserted that the event path exists: + // the 1 s poll is a complete fallback, so the entire suite passed with + // the source unavailable and no test could tell. + #[cfg(target_os = "linux")] + assert!( + w.is_event_driven(), + "the netlink link-event source must open on Linux; \ + falling back to the poll here is a silent loss of the fast path" + ); + + // Elsewhere both answers are legitimate, so pin only that asking is + // safe and that a watcher with no source parks rather than fires. + #[cfg(not(target_os = "linux"))] + { + let backed = w.is_event_driven(); + assert!( + backed + || tokio::time::timeout(Duration::from_millis(50), w.changed()) + .await + .is_err(), + "a watcher with no source must never resolve" + ); + } } /// A descriptor whose `recv` always fails must not become a busy loop. diff --git a/testing/README.md b/testing/README.md index 03ff2384..36e8b69e 100644 --- a/testing/README.md +++ b/testing/README.md @@ -85,6 +85,15 @@ End-to-end exercise of the production `fips0` nftables baseline at `packaging/common/fips.nft`, covering the default-deny, conntrack and drop-in semantics. +### [iface-binding/](iface-binding/) -- Dynamic Interface Binding + +Two nodes whose only transports are interface-bound, started before the +interface they name exists. Asserts the boot race (the daemon comes up +`Degraded` and binds when the interface appears, with no restart), the flap +(down/up in both directions), destroy-and-recreate, that an `optional` +interface's absence never moves node health, and that absence is logged once +on the edge rather than once per retry. + ### [acl-allowlist/](acl-allowlist/) -- Peer ACL Enforcement Six nodes with per-node allowlist files mounted at the runtime ACL @@ -146,6 +155,23 @@ matrix; a divergence fails the run. GitHub runs the same check as its own `ci-parity` job. `--check-parity` runs it alone (see [check-ci-parity.sh](check-ci-parity.sh)). +Note that `ci-local.sh` covers the integration suites and the glibc unit +tests. GitHub additionally runs the library tests on macOS, Windows and +**musl** (built for the musl target and run natively), and a `--features +profiling` pass; the musl +leg exists because interface presence is built on `getifaddrs`/`ifa_flags`, +which musl reimplements independently, and OpenWrt is a musl target. A local +green run does not certify those four. + +The Linux and musl legs also create an address-less dummy interface +(`fips-probe0`) and pass its name to the tests as +`FIPS_TEST_ADDRLESS_IFACE`. That is the one assumption the interface-binding +mechanism rests on that no ordinary test can reach: loopback has addresses, so +probing it asks whether `getifaddrs` works rather than whether it reports an +interface that has none — which is exactly what `fips-mesh0` and `fips-ap0` +are on OpenWrt. Set the variable by hand to run the assertion locally against +an interface you have created; leave it unset and the assertion does not run. + ### Per-run isolation and the `FIPS_CI_RUN_ID` override Every invocation derives a **run id** and scopes all of its Docker diff --git a/testing/chaos/scenarios/ethernet-churn.yaml b/testing/chaos/scenarios/ethernet-churn.yaml new file mode 100644 index 00000000..ca9b3cde --- /dev/null +++ b/testing/chaos/scenarios/ethernet-churn.yaml @@ -0,0 +1,128 @@ +# Ethernet rebind under active traffic +# +# The one case dynamic interface binding has no coverage for anywhere: +# traffic crossing an Ethernet link while the interface underneath it goes +# away and comes back. +# +# The other Ethernet scenarios cannot reach it. `ethernet-only` and +# `ethernet-mesh` both run with `traffic.enabled: false`, so no datagram +# crosses an Ethernet link in either — framing, the length field that trims +# NIC minimum-frame padding, and AEAD over Ethernet are all control-plane +# assumptions there. And `link_flaps` cannot produce a rebind whatever it is +# pointed at: it simulates a down link with netem 100% loss, so the interface +# stays IFF_UP and the presence machine never sees an edge. +# +# `node_churn` is what actually moves an interface. Stopping a container +# destroys its network namespace, which deletes every veth in it — and +# deleting one end of a veth deletes its peer — so a *surviving* node watches +# its Ethernet interface disappear outright. On restart the harness recreates +# the pair (`NodeChurnManager._start_node`), and the survivor watches it come +# back. That is a real detach and a real rebind, driven from outside the +# daemon, with iperf3 running across the mesh throughout. +# +# Topology: a 4-node ring, so removing any single node leaves the remaining +# three connected in a line and `protect_connectivity` has something to +# protect. +# +# n01 ---eth--- n02 +# | | +# eth eth +# | | +# n04 ---eth--- n03 + +scenario: + name: "ethernet-churn" + seed: 42 + duration_secs: 240 + +topology: + algorithm: explicit + num_nodes: 4 + default_transport: ethernet + params: + adjacency: + - [n01, n02] + - [n02, n03] + - [n03, n04] + - [n04, n01] + +# Mild, and deliberately so. The variable under test is the interface going +# away, not the link being bad while it is there; heavy loss here would make a +# traffic shortfall ambiguous between the two. +netem: + enabled: true + default_policy: + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + +# Off on purpose. A netem-simulated down link would add outage that never +# reaches the presence machine, which is the opposite of what this isolates. +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 2 + interval_secs: {min: 10, max: 20} + duration_secs: {min: 15, max: 25} + parallel_streams: 2 + +# The mechanism. One node down at a time, long enough to outlast the +# ten-second bring-up window so the absence is a real one rather than a race, +# and to give traffic time to run against the reduced mesh before it returns. +node_churn: + enabled: true + interval_secs: {min: 45, max: 60} + max_down_nodes: 1 + down_duration_secs: {min: 20, max: 35} + protect_connectivity: true + +assertions: + # The mesh re-forms after each interface comes back. + # + # Calibrated 2026-09-02 against four runs, all at this file's fixed seed 42 + # and therefore an identical churn schedule, so the spread is container + # timing rather than differing scenarios: + # + # nodes answering 4 in all four runs + # distinct roots 2 in all four runs + # nodes parented 2 in all four runs + # traffic 3-5 sessions, 206-388 MB + # + # `max_roots: 1` was wrong and failed every run: the harness restores + # stopped nodes immediately before the final snapshot, so a node that has + # just restarted has not re-parented yet and is briefly its own root. That + # is the scenario working, not failing. + # + # The ceiling is 3, one step beyond the observed maximum of 2, with the + # parented floor at its complement. A mesh that genuinely collapsed — every + # node islanded — still fails, which is all this assertion is for. Do not + # read a pass as convergence. + # + # READ BEFORE RETUNING: four runs is a thin sample. `churn-mixed` documents + # what happens when a threshold is set one step outside a small one — it + # ends up inside the real distribution and reddens runs whatever the daemon + # does. Widen on evidence; tighten only against a much larger sample. + baseline: + min_nodes_reporting: 3 + max_roots: 3 + min_nodes_parented: 2 + + # The point of the scenario. Traffic must actually have crossed Ethernet + # links while interfaces were being taken away underneath it — a green tree + # with zero bytes moved is the failure this catches, and is exactly what a + # control-plane-only assertion would have called a pass. + min_traffic: + min_sessions_ok: 2 + + # Chaos Ethernet transports are `optional: true` (see config_gen), precisely + # because a neighbour's interface disappearing is the scenario here rather + # than a fault. So absence must stay silent: any ERROR means something other + # than the churn went wrong. + max_errors: + max_total: 0 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/sim/assertions.py b/testing/chaos/sim/assertions.py index 1fc636aa..4652198f 100644 --- a/testing/chaos/sim/assertions.py +++ b/testing/chaos/sim/assertions.py @@ -19,6 +19,7 @@ from dataclasses import dataclass from .control import snapshot_all_bloom from .scenario import ( + MinTrafficAssertion, BaselineAssertion, BloomSendRateAssertion, CongestionSignalsAssertion, @@ -442,3 +443,61 @@ def evaluate_min_parent_switches( f"node win the root election?" ), ) + + +def _session_bytes(result: dict) -> int: + """Bytes actually received in one iperf3 session, or 0 if it failed. + + iperf3 reports a failed run as a top-level ``error`` string with no + ``end`` block, and a killed-but-partial run still carries whatever it + managed. Both are handled by reading the received total and treating + anything missing as zero, so a session only counts when it moved bytes. + """ + if not isinstance(result, dict) or result.get("error"): + return 0 + end = result.get("end") + if not isinstance(end, dict): + return 0 + summary = end.get("sum_received") or end.get("sum_sent") + if not isinstance(summary, dict): + return 0 + value = summary.get("bytes", 0) + return value if isinstance(value, int) and value > 0 else 0 + + +def evaluate_min_traffic( + cfg: MinTrafficAssertion, + results: list[dict], +) -> AssertionOutcome: + """Floor on iperf3 sessions that actually carried data. + + Without this the traffic generator is decoration: the results were + written to disk and never read, so a scenario whose every session + failed still passed on a healthy control plane. A rebind under load is + exactly the case a tree snapshot cannot see. + """ + per_session = [_session_bytes(r) for r in results] + ok = [b for b in per_session if b > 0] + total = sum(ok) + + if len(ok) >= cfg.min_sessions_ok and total >= cfg.min_bytes_total: + return AssertionOutcome( + name="min_traffic", + passed=True, + detail=( + f"PASS min_traffic: {len(ok)}/{len(results)} session(s) moved " + f"data (need {cfg.min_sessions_ok}); {total} byte(s) total " + f"(need {cfg.min_bytes_total})" + ), + ) + + return AssertionOutcome( + name="min_traffic", + passed=False, + detail=( + f"FAIL min_traffic: {len(ok)}/{len(results)} session(s) moved data " + f"(need {cfg.min_sessions_ok}); {total} byte(s) total (need " + f"{cfg.min_bytes_total}). A green control plane with no traffic " + f"means the data path did not survive what the scenario did to it." + ), + ) diff --git a/testing/chaos/sim/config_gen.py b/testing/chaos/sim/config_gen.py index 1b301444..39463456 100644 --- a/testing/chaos/sim/config_gen.py +++ b/testing/chaos/sim/config_gen.py @@ -64,7 +64,22 @@ def generate_peers_block( def _build_ethernet_config(iface: str) -> dict: - """Build an Ethernet transport config dict for a single interface.""" + """Build an Ethernet transport config dict for a single interface. + + ``optional: True`` because in this harness a neighbour's interface + disappearing is the scenario, not a fault. ``node_churn`` stops a + container, which destroys its netns and with it both ends of every veth + pair it held (see ``nodes.py``: the veths are recreated on restart), so a + surviving node watches a *required* interface vanish for the 30-90s the + neighbour is down -- once per churn event, on every neighbour. The daemon + reports a required interface absent past its bring-up window at ERROR, + which is correct for a deployment and wrong for a harness that tears the + interface down on purpose; the mesh-wide zero-ERROR ceiling would fail on + injected chaos rather than on a defect. + + Absence behaviour itself is asserted in ``testing/iface-binding/``, which + exists for it and drives both policies deliberately. + """ return { "interface": iface, "listen": True, @@ -72,6 +87,7 @@ def _build_ethernet_config(iface: str) -> dict: "auto_connect": True, "accept_connections": True, "beacon_interval_secs": 10, + "optional": True, } diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 75993411..502febe6 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -20,6 +20,7 @@ from .assertions import ( evaluate_max_errors, evaluate_max_parent_switches, evaluate_min_parent_switches, + evaluate_min_traffic, evaluate_tree_parents, ) from .compose import generate_compose @@ -699,6 +700,7 @@ class SimRunner: self.node_mgr.restore_all() # Collect iperf3 throughput results before containers stop + iperf_results: list[dict] = [] if self.traffic_mgr: iperf_results = self.traffic_mgr.collect_results() if iperf_results: @@ -758,6 +760,13 @@ class SimRunner: if err_cfg is not None: outcome = evaluate_max_errors(err_cfg, result.errors) self.assertion_outcomes.append(outcome) + + # Traffic. Evaluated even when no session completed, because + # "nothing ran" is the failure this exists to catch. + traffic_cfg = self.scenario.assertions.min_traffic + if traffic_cfg is not None: + outcome = evaluate_min_traffic(traffic_cfg, iperf_results) + self.assertion_outcomes.append(outcome) if outcome.passed: log.info("%s", outcome.detail) else: diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 56c27947..19e59954 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -323,6 +323,27 @@ class MaxErrorsAssertion: max_total: int = 0 +@dataclass +class MinTrafficAssertion: + """Floor on how much iperf3 traffic actually completed. + + Traffic has always been generated and its results saved to + ``iperf3-results.json``, but nothing read them: a scenario could carry + ``traffic.enabled: true``, have every single session fail, and still + exit 0 on a green control plane. That gap matters most for the + scenarios where traffic is the point — a datagram crossing an Ethernet + link while its interface rebinds underneath is not observable in the + tree snapshot at all. + + ``min_sessions_ok`` counts sessions that finished with bytes actually + received. ``min_bytes_total`` is the aggregate floor across them; 0 + disables it and leaves the session count as the only gate. + """ + + min_sessions_ok: int = 1 + min_bytes_total: int = 0 + + @dataclass class AssertionsConfig: """Optional post-run assertions evaluated against control-socket data.""" @@ -334,6 +355,7 @@ class AssertionsConfig: congestion_signals: CongestionSignalsAssertion | None = None tree_parents: TreeParentsAssertion | None = None baseline: BaselineAssertion | None = None + min_traffic: MinTrafficAssertion | None = None @dataclass @@ -414,6 +436,7 @@ _SECTION_KEYS = { "assertions": { "bloom_send_rate", "min_parent_switches", "max_parent_switches", "max_errors", "congestion_signals", "tree_parents", "baseline", + "min_traffic", }, "logging": {"rust_log", "output_dir"}, } @@ -422,6 +445,7 @@ _ASSERTION_KEYS = { "min_parent_switches": {"min_total"}, "max_parent_switches": {"max_total", "node"}, "max_errors": {"max_total"}, + "min_traffic": {"min_sessions_ok", "min_bytes_total"}, "congestion_signals": { "min_nodes_detected", "min_nodes_ce_forwarded", "min_nodes_ce_received", }, @@ -739,6 +763,27 @@ def load_scenario(path: str) -> Scenario: f"got {err_total}" ) s.assertions.max_errors = MaxErrorsAssertion(max_total=err_total) + + if "min_traffic" in asrt: + mt = asrt["min_traffic"] + _reject_unknown( + mt, _ASSERTION_KEYS["min_traffic"], "assertions.min_traffic", + ) + sessions = mt.get("min_sessions_ok", 1) + if not isinstance(sessions, int) or isinstance(sessions, bool) or sessions < 1: + raise ValueError( + "assertions.min_traffic: min_sessions_ok must be a positive " + f"integer, got {sessions!r} — a floor of zero asserts nothing" + ) + min_bytes = mt.get("min_bytes_total", 0) + if not isinstance(min_bytes, int) or isinstance(min_bytes, bool) or min_bytes < 0: + raise ValueError( + "assertions.min_traffic: min_bytes_total must be a " + f"non-negative integer, got {min_bytes!r}" + ) + s.assertions.min_traffic = MinTrafficAssertion( + min_sessions_ok=sessions, min_bytes_total=min_bytes, + ) else: # Default-on. See MaxErrorsAssertion for why this one assertion is # applied without being asked for: it is the floor on what a green diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 0f7a2226..8c3759f1 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -30,7 +30,8 @@ # firewall, nat-cone, nat-symmetric, # nat-lan, nostr-publish-consume, stun-faults, # chaos-churn-mixed-10, chaos-ethernet-mesh, -# chaos-ethernet-only, chaos-tcp-mesh, chaos-congestion-stress, +# chaos-ethernet-only, chaos-ethernet-churn, chaos-tcp-mesh, +# chaos-congestion-stress, # sidecar, dns-resolver, deb-install, medium-change # # Opt-in (require --with-tor; depend on live Tor network): @@ -149,6 +150,7 @@ CHAOS_SUITES=( "churn-mixed-10 churn-mixed --nodes 10 --duration 120" "ethernet-mesh ethernet-mesh" "ethernet-only ethernet-only" + "ethernet-churn ethernet-churn" "tcp-mesh tcp-mesh" "congestion-stress congestion-stress" ) @@ -194,6 +196,7 @@ CHAOS_SUITES=( GATEWAY_SUITES=(gateway) SIDECAR_SUITES=(sidecar) FIREWALL_SUITES=(firewall) +IFACE_BINDING_SUITES=(iface-binding) NAT_SUITES=(cone symmetric lan) NOSTR_RELAY_SUITES=(nostr-publish-consume) STUN_FAULTS_SUITES=(stun-faults) @@ -233,6 +236,9 @@ list_suites() { echo " Firewall baseline:" for s in "${FIREWALL_SUITES[@]}"; do echo " $s"; done echo "" + echo " Dynamic interface binding:" + for s in "${IFACE_BINDING_SUITES[@]}"; do echo " $s"; done + echo "" echo " NAT scenarios:" for s in "${NAT_SUITES[@]}"; do echo " nat-$s"; done echo "" @@ -782,6 +788,22 @@ run_sidecar() { record "sidecar" $rc } +# Run the dynamic interface binding integration test. +# +# Creates a veth pair from the host namespace after the daemons are up, so it +# needs the same privileged ip(8) helper the chaos harness uses. Scoped by +# COMPOSE_PROJECT_NAME like every other suite; the veth names carry the run +# suffix themselves (see testing/iface-binding/test.sh). +run_iface_binding() { + export COMPOSE_PROJECT_NAME="$(ci_project iface_binding)" + info "[iface-binding] Running integration test" + if bash testing/iface-binding/test.sh --skip-build 2>&1; then + record "iface-binding" 0 + else + record "iface-binding" 1 + fi +} + # Run firewall baseline integration test run_firewall() { export COMPOSE_PROJECT_NAME="$(ci_project firewall)" @@ -1248,6 +1270,9 @@ run_integration() { # Firewall baseline run_firewall + # Dynamic interface binding + run_iface_binding + # NAT scenarios (sequential — each owns its compose project) for scenario in "${NAT_SUITES[@]}"; do run_nat "$scenario" @@ -1361,6 +1386,8 @@ run_suite() { run_gateway ;; firewall) run_firewall ;; + iface-binding) + run_iface_binding ;; nat-cone|nat-symmetric|nat-lan) run_nat "${suite#nat-}" ;; nostr-publish-consume) diff --git a/testing/iface-binding/README.md b/testing/iface-binding/README.md new file mode 100644 index 00000000..eae8328f --- /dev/null +++ b/testing/iface-binding/README.md @@ -0,0 +1,57 @@ +# Dynamic Interface Binding + +Two FIPS daemons whose **only** transports are bound to network interfaces, +exercised against a veth pair the harness creates, downs, deletes and recreates +underneath them while they run. + +``` +node-a node-b + lab ve-lab0 required ── veth ── ve-lab0 required + dock fips-dock0 optional fips-dock0 optional +``` + +`ve-lab0` does not exist when the daemons start. `fips-dock0` never exists at +all, on any host, ever — it is the negative control for `optional: true`. + +## What it asserts + +| | Behavior | +| - | -------- | +| (a) | A daemon whose only interface is missing **starts**, reports the transport `absent`, and reports `Degraded` — it does not exit on `NoTransports`, and it does not skip the transport for the life of the process | +| (b) | The interface appears; both daemons bind it with no restart, `Degraded` clears, and they discover and peer over it | +| (c) | The interface goes down and comes back; presence and health follow it in **both** directions, and the rebind is counted | +| (d) | The interface is deleted outright and recreated; both daemons rebind and re-peer — the case the old ENXIO beacon-socket reopen half-covered | +| (e) | An `optional` interface that never appears logs at `info` and never moves node health | +| | Absence is logged **once on the edge**, not once per retry | + +Health is asserted through `fipsctl show status` (`state`), presence through +`fipsctl show transports` (the per-transport `interface` block: `presence`, +`policy`, `binds`, `since_secs`). + +## Running + +```sh +./test.sh # builds the image first +./test.sh --skip-build # reuse an existing image +./test.sh --keep-up # leave the containers running for inspection +``` + +Via the local CI runner: + +```sh +./testing/ci-local.sh --only iface-binding +``` + +## Notes + +The containers run under `FIPS_TEST_MODE=default`, **not** `chaos`. The chaos +entrypoint waits up to 30 s for every configured Ethernet interface before +starting the daemon — which is exactly the workaround this mechanism retires. +The daemon has to do its own waiting here or the suite proves nothing. + +Every `ip link` operation on the host network stack runs inside a short-lived +privileged container sharing the host network and PID namespaces, for the +reason [chaos/sim/veth.py](../chaos/sim/veth.py) documents: on macOS the +containers live in the Docker VM, so ip(8) run on the macOS host could never +reach them, while on Linux the shared namespaces make it identical to running +ip(8) directly. diff --git a/testing/iface-binding/docker-compose.yml b/testing/iface-binding/docker-compose.yml new file mode 100644 index 00000000..fd87ba6b --- /dev/null +++ b/testing/iface-binding/docker-compose.yml @@ -0,0 +1,80 @@ +networks: + # Management bridge only. The FIPS transport under test is raw Ethernet on a + # veth pair the harness creates *after* the daemons are already running — + # that is the whole point of the suite — so no FIPS traffic crosses this + # network. No subnet is requested, so two concurrent runs cannot collide on + # one address range. + # + # The compose project name is still fixed, so two runs that do not set + # COMPOSE_PROJECT_NAME share a project and the second `up` recreates the + # first's containers. The local CI runner scopes it externally + # (run_iface_binding in ci-local.sh); a bare hand run does not. + ifb-net: + driver: bridge + labels: + - "com.corganlabs.fips-ci=1" + +x-fips-common: &fips-common + build: + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} + image: ${FIPS_TEST_IMAGE:-fips-test:latest} + entrypoint: ["/usr/local/bin/entrypoint.sh"] + cap_add: + - NET_ADMIN + - NET_RAW + restart: "no" + environment: + # `default`, deliberately — NOT `chaos`. The chaos entrypoint waits up to + # 30 s for every configured Ethernet interface to appear before it starts + # the daemon, which is precisely the workaround this mechanism retires. The + # daemon must do its own waiting here or the suite proves nothing. + - FIPS_TEST_MODE=default + - RUST_LOG=info,fips::transport::ethernet=debug,fips::node=debug + networks: + - ifb-net + +services: + node-a: + <<: *fips-common + container_name: fips-ifb-node-a${FIPS_CI_NAME_SUFFIX:-} + hostname: host-a + volumes: + - ../docker/resolv.conf:/etc/resolv.conf:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.yaml:/etc/fips/fips.yaml:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.key:/etc/fips/fips.key:ro + + node-b: + <<: *fips-common + container_name: fips-ifb-node-b${FIPS_CI_NAME_SUFFIX:-} + hostname: host-b + volumes: + - ../docker/resolv.conf:/etc/resolv.conf:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.yaml:/etc/fips/fips.yaml:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.key:/etc/fips/fips.key:ro + + # The clean-start case. Its interface exists before its daemon does, which is + # the ordinary state of a booted router and the one ordering node-a and + # node-b cannot produce: their interface is created after they are already + # running, so they can only ever bind through the binder loop. + # + # The gate is what buys that ordering. The harness needs a running container + # to have a netns to move a veth into, but the daemon must not start until + # after the move — so the container comes up, parks on this file, and the + # harness releases it once the interface is in place. + node-c: + <<: *fips-common + container_name: fips-ifb-node-c${FIPS_CI_NAME_SUFFIX:-} + hostname: host-c + entrypoint: ["/bin/sh", "-c"] + command: + - | + while [ ! -e /tmp/fips-go ]; do sleep 0.2; done + exec /usr/local/bin/entrypoint.sh + volumes: + - ../docker/resolv.conf:/etc/resolv.conf:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/fips.yaml:/etc/fips/fips.yaml:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/fips.key:/etc/fips/fips.key:ro diff --git a/testing/iface-binding/generate-configs.sh b/testing/iface-binding/generate-configs.sh new file mode 100755 index 00000000..1aab0a76 --- /dev/null +++ b/testing/iface-binding/generate-configs.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Generate fixtures for the dynamic interface binding integration test. +# +# Two FIPS nodes, each with two Ethernet transports and nothing else: +# +# lab ve-lab0 required — does not exist when the daemon starts; the +# harness creates the veth pair afterwards +# dock fips-dock0 optional — never exists, on any host, ever +# +# Plus a third node whose single required interface exists *before* its daemon +# starts — the one ordering the other two cannot produce, and the one that +# `start_async`'s inline bind takes. See node-c in test.sh case (f). +# +# There is deliberately no UDP transport. A node whose only transports are +# interface-bound is the case that used to be unrecoverable: every transport +# skipped at start, nothing retried, and the node up and deaf. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Scoped by the per-run suffix because this directory is wiped and rewritten +# below: two runs sharing one output directory would delete each other's +# fixtures out from under running containers. +GENERATED_DIR="$SCRIPT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}" + +# Deterministic test identities (mirrors the firewall/acl-allowlist style). +KEY_A="0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" +KEY_B="b102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fb0" +KEY_C="c102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fc0" + +write_file() { + local path="$1" + mkdir -p "$(dirname "$path")" + cat > "$path" +} + +# Peers are found by beacon, not configured: a MAC address that does not exist +# until the harness creates the veth pair cannot be written into a config file +# ahead of time. Discovery over the late-bound interface is part of what the +# suite asserts. +write_node_config() { + write_file "$GENERATED_DIR/$1/fips.yaml" <&2; exit 1 ;; + esac +done + +log() { echo "=== $*"; } +pass() { echo "PASS: $*"; } +fail() { echo "FAIL: $*" >&2; dump_diagnostics; exit 1; } + +dump_diagnostics() { + echo "--- node-a transports ---" >&2 + docker exec "$NODE_A" fipsctl show transports >&2 2>&1 || true + echo "--- node-a status ---" >&2 + docker exec "$NODE_A" fipsctl show status >&2 2>&1 || true + echo "--- node-a log (tail) ---" >&2 + docker logs --tail 80 "$NODE_A" >&2 2>&1 || true + echo "--- node-b log (tail) ---" >&2 + docker logs --tail 80 "$NODE_B" >&2 2>&1 || true + return 0 +} + +cleanup() { + # Remove the veth pair wherever it survived: inside a container if the move + # succeeded, on the host if the run died between creation and the move. + docker exec "$NODE_A" ip link del "$LAB_IFACE" >/dev/null 2>&1 || true + ip_host "ip link del $HOST_VETH_A" >/dev/null 2>&1 || true + docker exec "$NODE_C" ip link del "$BOOT_IFACE" >/dev/null 2>&1 || true + ip_host "ip link del $HOST_VETH_C" >/dev/null 2>&1 || true + if [ "$KEEP_UP" = false ]; then + docker compose -f "$COMPOSE_FILE" down --volumes --remove-orphans >/dev/null 2>&1 || true + fi + return 0 +} + +# ── Host-namespace ip(8) ───────────────────────────────────────────────── +# +# Every `ip link` operation that touches the host network stack runs inside a +# short-lived privileged container sharing the host network and PID namespaces, +# for the reason testing/chaos/sim/veth.py documents at length: on macOS the +# containers live in the Docker VM, so running ip(8) on the macOS host could +# never reach them, while on Linux the shared namespaces make it identical to +# running ip(8) directly. +ip_host() { + docker run --rm --privileged --network host --pid host \ + --entrypoint /bin/sh "$IMAGE" -c "$1" +} + +# Docker's own view of a container, so the gate case can wait for a netns +# without implying the daemon inside it has started. +container_state() { + docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || true +} + +container_pid() { + docker inspect -f '{{.State.Pid}}' "$1" +} + +# Create the veth pair and move one end into each container. +create_veth() { + local pid_a pid_b + pid_a="$(container_pid "$NODE_A")" + pid_b="$(container_pid "$NODE_B")" + + # One invocation, not three. The host-side names exist only between the + # `add` and the two `netns` moves, and ci-cleanup.sh's host-veth sweep is + # deliberately shaped to the chaos simulation's names and does not cover + # these — so the window in which a hard kill could strand them is kept to + # a single command, with the EXIT trap covering the rest. + ip_host "set -e + ip link add $HOST_VETH_A type veth peer name $HOST_VETH_B + ip link set $HOST_VETH_A netns $pid_a name $LAB_IFACE + ip link set $HOST_VETH_B netns $pid_b name $LAB_IFACE" >/dev/null + + # A moved link arrives down. Presence is IFF_UP, so the daemon correctly + # does not bind until this runs — which is also why (c) can flap it with + # nothing but `ip link set down`. + docker exec "$NODE_A" ip link set "$LAB_IFACE" up + docker exec "$NODE_B" ip link set "$LAB_IFACE" up + return 0 +} + +# ── Daemon introspection ───────────────────────────────────────────────── + +# Field of a named transport's `interface` block, or "" if the transport, the +# block, or the daemon is not there. +iface_field() { + docker exec "$1" fipsctl show transports 2>/dev/null | python3 -c ' +import json, sys +try: + data = json.load(sys.stdin) +except Exception: + print(""); raise SystemExit +for t in data.get("transports", []): + if t.get("name") == sys.argv[1]: + print(t.get("interface", {}).get(sys.argv[2], "")) + break +else: + print("") +' "$2" "$3" +} + +node_state() { + docker exec "$1" fipsctl show status 2>/dev/null | python3 -c ' +import json, sys +try: + print(json.load(sys.stdin).get("state", "")) +except Exception: + print("") +' +} + +peer_count() { + docker exec "$1" fipsctl show peers 2>/dev/null | python3 -c ' +import json, sys +try: + data = json.load(sys.stdin) +except Exception: + print(0); raise SystemExit +print(len(data.get("peers", []))) +' +} + +# Count of a literal in a container log. Used for the log-hygiene assertion. +log_count() { + docker logs "$1" 2>&1 | grep -c -- "$2" || true +} + +# Poll `expr` until it prints `want`, up to `timeout` seconds. +# Usage: wait_for +wait_for() { + local timeout="$1" want="$2"; shift 2 + local i got + for i in $(seq 1 "$timeout"); do + got="$("$@" || true)" + if [ "$got" = "$want" ]; then + return 0 + fi + sleep 1 + done + echo " (last value: '${got:-}', wanted '$want')" >&2 + return 1 +} + +# Poll until the command prints a value that is at least `want`. +wait_for_at_least() { + local timeout="$1" want="$2"; shift 2 + local i got + for i in $(seq 1 "$timeout"); do + got="$("$@" || true)" + if [ -n "$got" ] && [ "$got" -ge "$want" ] 2>/dev/null; then + return 0 + fi + sleep 1 + done + echo " (last value: '${got:-}', wanted >= '$want')" >&2 + return 1 +} + +# ── Run ────────────────────────────────────────────────────────────────── + +IMAGE="${FIPS_TEST_IMAGE:-fips-test:latest}" + +trap cleanup EXIT + +if [ "$SKIP_BUILD" = false ]; then + log "Building test image" + bash "$TESTING_DIR/scripts/build.sh" +fi + +log "Generating fixtures" +LAB_IFACE="$LAB_IFACE" DOCK_IFACE="$DOCK_IFACE" BOOT_IFACE="$BOOT_IFACE" \ + bash "$SCRIPT_DIR/generate-configs.sh" + +log "Starting nodes with $LAB_IFACE absent" +docker compose -f "$COMPOSE_FILE" up -d + +# The daemon must reach a serving state without the interface. Waiting on the +# control socket answering at all is the first half of assertion (a): a daemon +# that exited on `NoTransports` never answers. +if ! wait_for 40 "degraded" node_state "$NODE_A"; then + fail "(a) node-a did not come up Degraded with $LAB_IFACE absent" +fi +pass "(a) daemon started and serves with its only interface absent" + +# ── (a) presence and policy are visible to an operator ─────────────────── + +[ "$(iface_field "$NODE_A" lab presence)" = "absent" ] \ + || fail "(a) lab transport is not reported ABSENT" +[ "$(iface_field "$NODE_A" lab policy)" = "required" ] \ + || fail "(a) lab transport is not reported required" +[ "$(iface_field "$NODE_A" lab name)" = "$LAB_IFACE" ] \ + || fail "(a) lab transport does not name $LAB_IFACE" +[ "$(iface_field "$NODE_A" dock presence)" = "absent" ] \ + || fail "(e) dock transport is not reported ABSENT" +[ "$(iface_field "$NODE_A" dock policy)" = "optional" ] \ + || fail "(e) dock transport is not reported optional" +pass "(a) presence and policy are visible in show_transports" + +# ── log hygiene, measured across the absence ───────────────────────────── +# +# The required interface's absence must be logged once, on the edge — not once +# per retry. Measured over an interval long enough for many retries (the binder +# polls every second). The 12 s also carries the next assertion past the 10 s +# bring-up window, so both the edge rule and the deadline are covered by one +# wait. + +# Matched on the edge line specifically. The deadline error below is a +# different line by design, and counting both here would read the deadline +# firing during the sleep as a repeated edge. +EDGE_LINE="Ethernet interface absent; waiting" + +absent_before="$(log_count "$NODE_A" "$EDGE_LINE")" +sleep 12 +absent_after="$(log_count "$NODE_A" "$EDGE_LINE")" +if [ "$absent_after" -ne "$absent_before" ]; then + fail "absence was logged $((absent_after - absent_before)) more times over 12s of retries; \ +the edge must be logged once, not per attempt" +fi +# Two edges total: one for the required lab interface, one for the optional +# dock interface. More would mean the edge is not an edge. +[ "$absent_before" -eq 2 ] \ + || fail "expected exactly 2 absence edges at start, saw $absent_before" +pass "absence is logged once on the edge, not per retry" + +# ── a sustained absence errors exactly once ────────────────────────────── +# +# The edge is not an error: an interface missing for a moment at boot and bound +# a moment later is the ordinary case this mechanism exists to absorb. Past the +# 10 s bring-up window it is no longer a race, and a *required* interface says +# so — once. The 12 s above put us on the far side of that window. +# +# Exactly one line, from lab. The optional dock interface has been absent just +# as long and must be silent, which is what `optional` means; two here would +# mean the policy is not being consulted. + +startup_errors="$(log_count "$NODE_A" " ERROR ")" +if [ "$startup_errors" -ne 1 ]; then + docker logs "$NODE_A" 2>&1 | grep -- " ERROR " >&2 || true + fail "expected exactly 1 ERROR line for the required interface past the \ +bring-up window, saw $startup_errors (an optional interface must contribute none)" +fi +[ "$(log_count "$NODE_A" "still missing past the bring-up window")" -eq 1 ] \ + || fail "the ERROR line is not the sustained-absence report" +pass "a required interface absent past the window errors, an optional one does not" + +# ...and does not keep saying it. Duration is state, published as +# interface.since_secs and as Degraded; re-announcing it on a timer is what +# the old 1 m / 10 m / 1 h ladder did. +sleep 12 +[ "$(log_count "$NODE_A" " ERROR ")" -eq 1 ] \ + || fail "the sustained-absence error repeated; it must be said once per episode" +pass "the sustained-absence error is said once, not on a schedule" + +# ── (b) late attach ────────────────────────────────────────────────────── + +log "Creating the veth pair" +create_veth + +if ! wait_for 30 "present" iface_field "$NODE_A" lab presence; then + fail "(b) node-a did not bind $LAB_IFACE after it appeared" +fi +if ! wait_for 30 "present" iface_field "$NODE_B" lab presence; then + fail "(b) node-b did not bind $LAB_IFACE after it appeared" +fi +pass "(b) both daemons bound the interface with no restart" + +# Health must clear. This is `Degraded` behaving as a level rather than a +# latch — the property that made the old monotonic failed-set wrong. +if ! wait_for 20 "running" node_state "$NODE_A"; then + fail "(b) node-a stayed Degraded after its interface returned" +fi +pass "(b) Degraded cleared when the interface came back" + +# The optional interface is still absent and must still not matter. +[ "$(iface_field "$NODE_A" dock presence)" = "absent" ] \ + || fail "(e) dock unexpectedly bound" +pass "(e) an absent optional interface does not degrade the node" + +# Discovery and peering over the late-bound interface: the point of binding at +# all. Without this the suite would prove the daemon can open a socket, not +# that traffic flows over it. +if ! wait_for_at_least 45 1 peer_count "$NODE_A"; then + fail "(b) node-a found no peer over the late-bound interface" +fi +if ! wait_for_at_least 45 1 peer_count "$NODE_B"; then + fail "(b) node-b found no peer over the late-bound interface" +fi +pass "(b) nodes discovered and peered over the late-bound interface" + +# ── (c) flap ───────────────────────────────────────────────────────────── + +errors_before_detach="$(log_count "$NODE_A" " ERROR ")" + +log "Taking $LAB_IFACE down on node-a" +docker exec "$NODE_A" ip link set "$LAB_IFACE" down + +detach_start=$SECONDS +if ! wait_for 20 "absent" iface_field "$NODE_A" lab presence; then + fail "(c) node-a did not notice the interface going down" +fi +detach_elapsed=$(( SECONDS - detach_start )) +if ! wait_for 20 "degraded" node_state "$NODE_A"; then + fail "(c) node-a did not report Degraded while its interface was down" +fi +pass "(c) a link going down is observed as absence and degrades health" + +# A detach is reported, at warn. The edge is not an error — a link coming and +# going is the weather in a mesh daemon, and the error is the 10 s deadline's +# to give, not the edge's. +if ! wait_for_at_least 10 1 log_count "$NODE_A" "Ethernet interface detached"; then + fail "(c) a runtime detach was not reported" +fi + +# Only meaningful while we are still inside the bring-up window. Detection is +# sub-second over netlink, so this is the ordinary path; if the runner was slow +# enough that the deadline could have fired, the check has nothing to say and +# says so rather than failing on the harness's own latency. +if [ "$detach_elapsed" -lt 8 ]; then + detach_errors="$(log_count "$NODE_A" " ERROR ")" + if [ "$detach_errors" -ne "$errors_before_detach" ]; then + docker logs "$NODE_A" 2>&1 | grep -- " ERROR " >&2 || true + fail "(c) the detach edge logged an ERROR after ${detach_elapsed}s; the \ +edge is a warn and only outlasting the window earns an error" + fi + pass "(c) a detach is reported without crying error" +else + echo " (skipped the edge-not-an-error check: detach took ${detach_elapsed}s," + echo " which is inside the deadline's reach)" +fi + +log "Bringing $LAB_IFACE back up on node-a" +docker exec "$NODE_A" ip link set "$LAB_IFACE" up + +if ! wait_for 30 "present" iface_field "$NODE_A" lab presence; then + fail "(c) node-a did not rebind after the interface came back" +fi +if ! wait_for 20 "running" node_state "$NODE_A"; then + fail "(c) node-a stayed Degraded after the interface came back" +fi +if ! wait_for_at_least 45 2 iface_field "$NODE_A" lab binds; then + fail "(c) the rebind was not counted" +fi +pass "(c) the interface flapped and the daemon followed it both ways" + +# ── (d) destroy and recreate ───────────────────────────────────────────── +# +# Deleting the netdev outright is the case the old ENXIO beacon-socket reopen +# half-covered: the veth is gone, the socket underneath is stale, and the name +# comes back a moment later. One presence machine now covers it. + +log "Deleting and recreating the veth pair" +docker exec "$NODE_A" ip link del "$LAB_IFACE" + +if ! wait_for 20 "absent" iface_field "$NODE_A" lab presence; then + fail "(d) node-a did not notice the interface being deleted" +fi +if ! wait_for 20 "absent" iface_field "$NODE_B" lab presence; then + fail "(d) node-b did not notice its end of the pair disappearing" +fi + +create_veth + +if ! wait_for 30 "present" iface_field "$NODE_A" lab presence; then + fail "(d) node-a did not rebind the recreated interface" +fi +if ! wait_for 30 "present" iface_field "$NODE_B" lab presence; then + fail "(d) node-b did not rebind the recreated interface" +fi +if ! wait_for 20 "running" node_state "$NODE_A"; then + fail "(d) node-a stayed Degraded after the interface was recreated" +fi +pass "(d) a destroyed and recreated interface is rebound" + +# Peering must re-establish over the new hardware. A recreated veth has a new +# MAC, so this also exercises the "same name, different hardware" path that +# drops cached neighbors instead of resuming onto them. +if ! wait_for_at_least 60 1 peer_count "$NODE_A"; then + fail "(d) node-a did not re-peer after the interface was recreated" +fi +pass "(d) peering re-established over the recreated interface" + +# ── (f) an interface present before the daemon starts ──────────────────── +# +# Everything above binds through the binder loop, because the interface does +# not exist until the harness makes it. The ordinary case on a booted router is +# the opposite one: the interface is already there and `start_async` binds it +# inline, before the loop is running. +# +# That path published its presence edge outside the churn guard, so the guard +# believed it had announced nothing and the *first* detach asked for no +# retraction. The node kept reporting Running with its only required interface +# gone, and stayed that way until a second detach happened to repair the guard. +# Nothing in cases (a)-(e) can reach it. +log "(f) starting node-c with its interface already present" + +docker compose -f "$COMPOSE_FILE" up -d node-c + +# The container parks on the gate, so this is the netns and not yet the daemon. +if ! wait_for 30 "running" container_state "$NODE_C"; then + fail "(f) node-c container did not start" +fi + +pid_c="$(container_pid "$NODE_C")" +ip_host "set -e + ip link add $HOST_VETH_C type veth peer name $HOST_VETH_D + ip link set $HOST_VETH_C netns $pid_c name $BOOT_IFACE + ip link set $HOST_VETH_D netns $pid_c name ${BOOT_IFACE}p" >/dev/null +docker exec "$NODE_C" ip link set "$BOOT_IFACE" up +docker exec "$NODE_C" ip link set "${BOOT_IFACE}p" up + +# Release the gate. The daemon now starts with the interface already up. +docker exec "$NODE_C" touch /tmp/fips-go + +if ! wait_for 40 "running" node_state "$NODE_C"; then + fail "(f) node-c did not come up Running with its interface present at start" +fi +[ "$(iface_field "$NODE_C" boot presence)" = "present" ] \ + || fail "(f) node-c did not bind $BOOT_IFACE inline at start" +pass "(f) an interface present at start is bound inline and reports Running" + +# The assertion. One detach, on a binding this loop did not create. +docker exec "$NODE_C" ip link set "$BOOT_IFACE" down + +if ! wait_for 30 "absent" iface_field "$NODE_C" boot presence; then + fail "(f) node-c did not notice $BOOT_IFACE going down" +fi +if ! wait_for 30 "degraded" node_state "$NODE_C"; then + fail "(f) node-c stayed Running after its only required interface went \ +away — the start-time bind never reached node health" +fi +pass "(f) the first detach after a clean start degrades the node" + +# And it is still a level, not a latch, on this path too. +docker exec "$NODE_C" ip link set "$BOOT_IFACE" up +if ! wait_for 30 "running" node_state "$NODE_C"; then + fail "(f) node-c stayed Degraded after its interface returned" +fi +pass "(f) health clears again when the interface returns" + +# ── (g) the link-event fast path is actually the one in use ────────────── +# +# The whole suite would pass with `open_link_socket()` hardcoded to Err: the +# 1 s poll is a complete fallback and covers every `wait_for` window here, so +# nothing else asserts that the netlink path exists, let alone that it is what +# detected anything. The binder says which backing it got at startup, so ask +# it directly rather than inferring from timing that the poll would also +# satisfy. +# `log_count`, not `grep -q`: under `set -o pipefail` a `grep -q` that exits on +# its first match closes the pipe, `docker logs` takes SIGPIPE, and the +# pipeline reports failure even though the line was found. `grep -c` reads the +# stream to the end. +if [ "$(log_count "$NODE_A" "event_driven=true")" -eq 0 ]; then + docker logs "$NODE_A" 2>&1 | grep -i "binder started" >&2 || true + fail "(g) the binder fell back to polling; the netlink link-event source \ +did not open, and every timing assertion in this suite would still pass" +fi +pass "(g) detection is driven by netlink events, not by the poll fallback" + +# ── (h) churn damping engages on a genuinely flapping interface ────────── +# +# This is load-bearing twice over. It is what stops a flapping interface +# logging a recovery per cycle, and — since the detach edge now withdraws the +# peers that interface carried — it is also the only thing bounding how often +# that withdrawal can fire. Nothing exercised it: every flap elsewhere in this +# suite is a single down/up with long settles either side, which is precisely +# the shape the damper ignores. +# +# Four bindings that each die well inside MIN_STABLE_BINDING (10 s). The +# streak crosses CHURN_THRESHOLD (3) on the third, which is the edge that +# announces itself. +log "(h) flapping $LAB_IFACE to drive the churn guard" +for _ in 1 2 3 4; do + docker exec "$NODE_A" ip link set "$LAB_IFACE" down + sleep 1 + docker exec "$NODE_A" ip link set "$LAB_IFACE" up + sleep 2 +done + +if ! wait_for_at_least 30 1 log_count "$NODE_A" "keeps dying immediately after binding"; then + docker logs "$NODE_A" 2>&1 | grep -i "ethernet" | tail -20 >&2 + fail "(h) four short-lived bindings did not engage the churn guard" +fi +pass "(h) a flapping interface engages churn damping" + +# Having engaged, the guard must hold health rather than announcing each bind. +# The failure this catches is a damper that counts but does not damp. +recoveries_during_churn="$(log_count "$NODE_A" "Ethernet interface recovered")" +if [ "$recoveries_during_churn" -gt 6 ]; then + fail "(h) node-a announced $recoveries_during_churn recoveries; the guard \ +counted the churn but kept announcing through it" +fi +pass "(h) churn suppressed the per-cycle recovery announcements" + +# And it is not a latch: once a binding lasts, the interface is announced +# again and the node returns to Running on its own. +log "(h) letting $LAB_IFACE settle" +docker exec "$NODE_A" ip link set "$LAB_IFACE" up >/dev/null 2>&1 || true +if ! wait_for 60 "present" iface_field "$NODE_A" lab presence; then + fail "(h) node-a did not rebind after the flapping stopped" +fi +if ! wait_for 60 "running" node_state "$NODE_A"; then + fail "(h) node-a stayed Degraded after the flapping stopped" +fi +pass "(h) a settled interface is announced again after churn" + +# ── final log hygiene ──────────────────────────────────────────────────── +# +# Four outages happened above (start, down, delete, and node-b's end of the +# delete). A generous ceiling still catches the failure mode that matters: a +# retry loop logging per attempt would be in the hundreds by now. +edges="$(log_count "$NODE_A" "Ethernet interface")" +[ "$edges" -lt 40 ] \ + || fail "node-a logged $edges interface lines; the edges are not edges" +pass "log volume stayed proportional to edges, not to retries" + +echo +echo "ALL PASSED"