From 5be7c6d0cb25aa1887c69823d9627943b72eeada Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Wed, 22 Jul 2026 06:05:40 +0000 Subject: [PATCH 1/5] Give each chaos scenario its own container names and config directory Chaos scenarios run four at a time under local CI, but every one of them claimed the container names fips-node-nNN and wrote its generated configs and compose file to the same generated-configs/sim directory. Container names are global in Docker and are not scoped by the compose project, so concurrent scenarios collided on both, and a scenario could start containers from a compose file another had overwritten. Thread the existing FIPS_CI_NAME_SUFFIX into the simulation. run_chaos narrows the run-wide suffix to the scenario, and the sim reads it once when the topology is built, applying it to the container names and to the config directory basename. The compose template renders the name through the topology accessor instead of duplicating the literal, so one expression produces every chaos container name. The suffix is empty when the variable is unset, so a bare chaos.sh run and the hosted CI jobs render byte-identical names and paths. Verified by rendering every scenario's compose file before and after with the variable unset and diffing. --- testing/chaos/sim/compose.py | 3 ++- testing/chaos/sim/naming.py | 17 +++++++++++++++++ testing/chaos/sim/runner.py | 10 +++++++++- testing/chaos/sim/topology.py | 15 +++++++++++++-- testing/ci-local.sh | 13 ++++++++++--- 5 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 testing/chaos/sim/naming.py diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py index 5eec26b..c27a153 100644 --- a/testing/chaos/sim/compose.py +++ b/testing/chaos/sim/compose.py @@ -47,7 +47,7 @@ services: {% for node in nodes %} {{ node.node_id }}: <<: *fips-common - container_name: fips-node-{{ node.node_id }} + container_name: {{ topology.container_name(node.node_id) }} hostname: {{ node.node_id }} volumes: - ./{{ node.node_id }}.yaml:/etc/fips/fips.yaml:ro @@ -82,6 +82,7 @@ def generate_compose( image=FIPS_SIM_IMAGE, nodes=nodes, resolv_conf=resolv_conf, + topology=topology, ) path = os.path.join(output_dir, "docker-compose.yml") diff --git a/testing/chaos/sim/naming.py b/testing/chaos/sim/naming.py new file mode 100644 index 0000000..d3418b1 --- /dev/null +++ b/testing/chaos/sim/naming.py @@ -0,0 +1,17 @@ +"""Scoping suffix for names that live in a global namespace.""" + +from __future__ import annotations + +import os + + +def name_suffix() -> str: + """Return the suffix appended to globally-scoped names. + + Docker container names and the generated-config directory are shared + across every simulation running on the host, so the harness exports + ``FIPS_CI_NAME_SUFFIX`` to keep concurrent runs and concurrent + scenarios apart. The suffix is empty when the variable is unset, so a + bare ``chaos.sh`` run renders exactly the same names as it always has. + """ + return os.environ.get("FIPS_CI_NAME_SUFFIX", "") diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index dc4960b..056f359 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -133,9 +133,17 @@ class SimRunner: log.info(" %s: peers=%s", nid, ",".join(peers)) # 2. Generate configs + # + # The directory carries the same suffix as the container names, so + # parallel scenarios cannot overwrite each other's compose file + # between writing it and starting containers from it. docker_network_dir = os.path.join(os.path.dirname(__file__), "..") config_dir = os.path.normpath( - os.path.join(docker_network_dir, "generated-configs", "sim") + os.path.join( + docker_network_dir, + "generated-configs", + f"sim{self.topology.name_suffix}", + ) ) # Select ephemeral identity nodes (if peer churn enabled) self._ephemeral_nodes: set[str] = set() diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py index e2e1ab6..c8b5d9c 100644 --- a/testing/chaos/sim/topology.py +++ b/testing/chaos/sim/topology.py @@ -8,6 +8,7 @@ from collections import deque from dataclasses import dataclass, field from .keys import derive +from .naming import name_suffix from .scenario import TopologyConfig @@ -28,6 +29,9 @@ class SimTopology: edges: set[tuple[str, str]] = field(default_factory=set) # Per-edge transport type; edges not in this dict default to "udp" edge_transport: dict[tuple[str, str], str] = field(default_factory=dict) + # Suffix scoping globally-visible names to this run and scenario; empty + # outside the CI harness, which keeps a bare run's names unchanged. + name_suffix: str = "" def transport_for_edge(self, a: str, b: str) -> str: """Get the transport type for an edge (defaults to 'udp').""" @@ -107,7 +111,7 @@ class SimTopology: return not connected def container_name(self, node_id: str) -> str: - return f"fips-node-{node_id}" + return f"fips-node-{node_id}{self.name_suffix}" def directed_outbound(self) -> dict[str, list[str]]: """Assign each static-config edge to exactly one node for outbound connection. @@ -219,7 +223,14 @@ def generate_topology( nodes[a].peers.append(b) nodes[b].peers.append(a) - topo = SimTopology(nodes=nodes, edges=edges, edge_transport=edge_transport) + # Read the environment once, here, so every name a run produces comes + # from the same value. + topo = SimTopology( + nodes=nodes, + edges=edges, + edge_transport=edge_transport, + name_suffix=name_suffix(), + ) # Connectivity check with retry if config.ensure_connected: diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 61917df..d639f74 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -516,9 +516,16 @@ run_chaos() { local name="$1" shift local rc=0 - # Distinct project per scenario (chaos children run in parallel). When - # invoked from a background subshell this export is local to that child. - export COMPOSE_PROJECT_NAME="$(ci_project "chaos-$name")" + # Distinct project per scenario (chaos children run in parallel). Scoped to + # this function so the --only path cannot leak it into a later suite. + local -x COMPOSE_PROJECT_NAME="$(ci_project "chaos-$name")" + + # Container names and the generated-config directory are GLOBAL and are not + # scoped by the compose project, so narrow the run-wide suffix to this + # scenario. Parallel children then cannot claim each other's names or + # overwrite each other's compose file. + local suffix="-${name}${FIPS_CI_NAME_SUFFIX:-}" + local -x FIPS_CI_NAME_SUFFIX="$suffix" info "[chaos/$name] Running simulation" if bash testing/chaos/scripts/chaos.sh "$@" 2>&1; then From 2ef36c0071ba0cd4f97a8a2a108e6725f609d066 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Wed, 22 Jul 2026 07:11:42 +0000 Subject: [PATCH 2/5] Scope chaos host interface names per scenario and reap orphaned pairs Ethernet edges get a veth pair created in the host namespace and then moved into the containers, named from the node ids alone. ethernet-mesh and ethernet-only both connect n01 to n04 and always run together, so one tore down the other's live link. Hash the scenario suffix to four hex characters and insert those after the vh prefix; 15 characters is far too few to carry the suffix itself. The names are now run-specific and never regenerated, so the delete-before-create no longer reclaims an interface an earlier run abandoned. ci-cleanup.sh takes over, deriving the names from the suffixes a run's teardown hands it. It is the first thing that script removes that is neither a docker object nor labelled, so an unscoped reap can sever a running bare simulation's links; testing/README.md and the --reap help say so. --- testing/README.md | 19 +++++ testing/chaos/sim/naming.py | 24 ++++++ testing/chaos/sim/topology.py | 31 +++++++- testing/chaos/sim/veth.py | 17 ++--- testing/ci-cleanup.sh | 140 +++++++++++++++++++++++++++++++++- testing/ci-local.sh | 50 ++++++++++-- 6 files changed, 261 insertions(+), 20 deletions(-) diff --git a/testing/README.md b/testing/README.md index 4a8c13d..2dd2760 100644 --- a/testing/README.md +++ b/testing/README.md @@ -144,3 +144,22 @@ and leaves resources behind, reap them with: label or a `fipsci_` compose-project prefix; it is safe to run when there is nothing to reap and safe to run repeatedly. Pass `--project-prefix` to scope the sweep to a single run. + +It also removes the chaos simulation's leftover host-namespace veth +interfaces (`vh…a`/`vh…b`), the one resource it touches that is neither a +docker object nor labelled — a host interface can carry neither a label +nor a compose project, so it is matched by name shape alone. That makes +the reach here asymmetric with everything above, and worth stating +plainly: + +- A bare `chaos.sh` run's **containers** survive a broad reap. Its + compose project is not `fipsci_`, and the simulation labels only the + network, not the services. +- A bare `chaos.sh` run's **veth interfaces do not.** An unscoped reap + deletes them while they are in use, severing the Ethernet links of a + live simulation and leaving its containers running. + +So do not run a broad `--reap` while a bare simulation is up. Scope the +interface sweep with `--veth-suffixes` (which is what `ci-local.sh`'s own +teardown passes) or wait for the simulation to finish. `--project-prefix` +does not help here: it scopes only the compose-project sweep. diff --git a/testing/chaos/sim/naming.py b/testing/chaos/sim/naming.py index d3418b1..61062d9 100644 --- a/testing/chaos/sim/naming.py +++ b/testing/chaos/sim/naming.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import os +import sys def name_suffix() -> str: @@ -15,3 +17,25 @@ def name_suffix() -> str: bare ``chaos.sh`` run renders exactly the same names as it always has. """ return os.environ.get("FIPS_CI_NAME_SUFFIX", "") + + +def veth_token(suffix: str) -> str: + """Shorten a name suffix to four hex characters. + + Host interface names have only 15 characters to work with, far fewer + than the suffix needs, so concurrent scenarios are told apart by a + hash of it instead. Empty for an empty suffix, so a bare run's + interface names are unchanged. + """ + if not suffix: + return "" + return hashlib.sha1(suffix.encode()).hexdigest()[:4] + + +if __name__ == "__main__": + # Print the token for each suffix given on the command line, one per + # line. ci-cleanup.sh reaps host interfaces by token and calls this + # rather than re-deriving the hash, so widening the token here cannot + # leave the reaper matching the old width. + for arg in sys.argv[1:]: + print(veth_token(arg)) diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py index c8b5d9c..69ea769 100644 --- a/testing/chaos/sim/topology.py +++ b/testing/chaos/sim/topology.py @@ -8,7 +8,7 @@ from collections import deque from dataclasses import dataclass, field from .keys import derive -from .naming import name_suffix +from .naming import name_suffix, veth_token from .scenario import TopologyConfig @@ -33,6 +33,15 @@ class SimTopology: # outside the CI harness, which keeps a bare run's names unchanged. name_suffix: str = "" + @property + def veth_token(self) -> str: + """Short stand-in for the suffix, for names bound by IFNAMSIZ. + + Derived rather than stored so no caller can build a topology whose + host names are scoped differently from its container names. + """ + return veth_token(self.name_suffix) + def transport_for_edge(self, a: str, b: str) -> str: """Get the transport type for an edge (defaults to 'udp').""" edge = _make_edge(a, b) @@ -113,6 +122,26 @@ class SimTopology: def container_name(self, node_id: str) -> str: return f"fips-node-{node_id}{self.name_suffix}" + def veth_host_name(self, node_a: str, node_b: str, end: str) -> str: + """Generate the host-namespace veth name for one end of an edge. + + Format: ``vh{token}{NN}{MM}{end}`` (max 15 chars for IFNAMSIZ). + Host interfaces are global, so the token keeps a scenario from + deleting a concurrent scenario's pair; it is empty outside the CI + harness, yielding the same "vh0104a" this has always produced. + + ``node_a`` and ``node_b`` must be in canonical edge order. Unlike + ``veth_interface_name()`` this is not symmetric: the far end is + ``end="b"`` on the same ordering, so swapping the arguments names + an interface that does not exist. + """ + nn_local = node_a.replace("n", "") + nn_peer = node_b.replace("n", "") + name = f"vh{self.veth_token}{nn_local}{nn_peer}{end}" + if len(name) > 15: + raise ValueError(f"veth host name too long: {name!r} ({len(name)} > 15)") + return name + def directed_outbound(self) -> dict[str, list[str]]: """Assign each static-config edge to exactly one node for outbound connection. diff --git a/testing/chaos/sim/veth.py b/testing/chaos/sim/veth.py index 95d1e96..13b7bd5 100644 --- a/testing/chaos/sim/veth.py +++ b/testing/chaos/sim/veth.py @@ -4,7 +4,8 @@ Creates veth pairs between Docker containers for Ethernet-transport edges. Each Ethernet edge gets a veth pair with one end moved into each container's network namespace. Naming: - Host (temporary): vh{NN}{MM}a / vh{NN}{MM}b + Host (temporary): vh{token}{NN}{MM}a / vh{token}{NN}{MM}b + (via SimTopology.veth_host_name()) Container: ve-{local}-{peer} (via veth_interface_name()) After creation, the container-side MAC addresses are queried and @@ -113,9 +114,7 @@ class VethManager: if a != node_id and b != node_id: continue # Remove existing pair if any (host-side might still exist) - nn_a = a.replace("n", "") - nn_b = b.replace("n", "") - host_a = f"vh{nn_a}{nn_b}a" + host_a = self.topology.veth_host_name(a, b, "a") _run_host(["ip", "link", "delete", host_a], image, check=False) # Re-create self._create_veth_pair(a, b, image) @@ -142,14 +141,14 @@ class VethManager: return # Generate names - nn_a = node_a.replace("n", "") - nn_b = node_b.replace("n", "") - host_a = f"vh{nn_a}{nn_b}a" - host_b = f"vh{nn_a}{nn_b}b" + host_a = self.topology.veth_host_name(node_a, node_b, "a") + host_b = self.topology.veth_host_name(node_a, node_b, "b") final_a = veth_interface_name(node_a, node_b) final_b = veth_interface_name(node_b, node_a) - # Clean up any stale pair + # Clean up a stale pair left by this scenario. The token makes the + # name unique to this run, so a pair orphaned by an earlier run is + # no longer reclaimed here — `ci-cleanup.sh` reaps those instead. _run_host(["ip", "link", "delete", host_a], image, check=False) # Create veth pair on host diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index 1231636..fbcb5ec 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Reap FIPS CI docker resources: containers, networks, volumes, images. +# Reap FIPS CI resources: containers, networks, volumes, images, veth pairs. # # Force-removes everything created by ci-local.sh that is still around — # whether a run finished cleanly, was preempted (SIGTERM/SIGKILL), OOM-killed, @@ -19,26 +19,56 @@ # the label sweep to one run; a run's own teardown always does. Without it the # label sweep stays broad, which is what a manual "reap everything" wants. # +# Host-namespace veth interfaces are the one non-docker resource reaped here. +# The chaos simulation creates each pair in the host namespace and then moves +# the two ends into container namespaces, so a run killed in between leaves the +# pair behind, and nothing else on the box removes it. Their names carry a +# short token derived from the scenario's name suffix, so --veth-suffixes takes +# the suffixes a run used and reaps only the tokens those could produce. Only a +# reap with neither --run-id nor --veth-suffixes matches every simulation veth +# name, in step with the broad label sweep. --project-prefix does not scope the +# veth sweep at all: a host interface carries no compose project. +# +# These interfaces are also the one resource reaped WITHOUT regard to the CI +# label — they cannot carry one — so an unscoped reap takes the host ends of a +# bare `chaos.sh` simulation's veth pairs too, even though it leaves that +# simulation's unlabelled containers running. Scope with --veth-suffixes (or +# just don't reap) while a bare simulation is up. +# # Usage: # ci-cleanup.sh Reap ALL fips-ci resources (any run) # ci-cleanup.sh --project-prefix P Restrict the compose-project sweep to # names starting with P (scopes the reap -# to a single run) +# to a single run). Does NOT scope the +# host veth sweep — see --veth-suffixes # ci-cleanup.sh --run-id ID Restrict the label sweep to the run # labelled ID (leaves other runs alone) # ci-cleanup.sh --label L Override the CI label (default above) # ci-cleanup.sh --images "a b,c" Also `docker rmi -f` these image tags # (space- or comma-separated) +# ci-cleanup.sh --veth-suffixes "a b" Restrict the host veth sweep to the +# name suffixes a single run used +# (space- or comma-separated). Without +# it the sweep reaches every simulation +# veth name on the host, including a +# bare `chaos.sh` run's live ones # # Safe to run when there is nothing to reap, and safe to run repeatedly. # Also reachable as `ci-local.sh --reap`. set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + LABEL="com.corganlabs.fips-ci=1" RUN_LABEL_KEY="com.corganlabs.fips-ci.run" PROJECT_PREFIX="fipsci_" # broad default: every CI run RUN_ID="" # broad default: every CI run IMAGES="" +VETH_SUFFIXES="" # empty AND no --run-id: every simulation veth name +# ip(8) runs inside this image, the same way the simulation creates the +# interfaces, so the reap works wherever the simulation does. The chaos +# simulation builds it, and it carries iproute2. +VETH_IMAGE="fips-test:latest" while [[ $# -gt 0 ]]; do case "$1" in @@ -46,6 +76,7 @@ while [[ $# -gt 0 ]]; do --project-prefix) PROJECT_PREFIX="$2"; shift 2 ;; --run-id) RUN_ID="$2"; shift 2 ;; --images) IMAGES="$2"; shift 2 ;; + --veth-suffixes) VETH_SUFFIXES="$2"; shift 2 ;; -h|--help) sed -n '2,/^set /{ /^set /d; s/^# \?//; p }' "$0"; exit 0 ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac @@ -107,6 +138,107 @@ reap_volumes() { | xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true } +# Every failure below leaves interfaces behind rather than widening the sweep, +# so each one is silent by construction. Say so on stderr instead, or a reap +# that reclaimed nothing looks exactly like a reap that had nothing to reclaim. +veth_warn() { echo "ci-cleanup: host veth sweep skipped: $*" >&2; } + +# One node id, exactly as chaos/sim/topology.py renders it (f"n{i+1:02d}"): +# zero-padded to two digits, and never zero-padded beyond that. Spelling it out +# keeps shapes the simulation cannot emit (vh01020a) out of the sweep. +VETH_NODE_ID='(0[0-9]|[1-9][0-9]+)' + +# Regex matching the host veth names to remove. With --veth-suffixes it covers +# only the tokens those suffixes hash to, so a concurrent run's interfaces — +# which carry a different token — cannot match. The token derivation is read +# from the simulation itself rather than repeated here, so widening it cannot +# leave this matching the old width. Empty output means "reap nothing". +veth_pattern() { + # vh{token}{NN}{MM}{a,b}: the token is 4 hex or wholly absent — never a + # part of one — and the two node ids follow. Anchored and shaped this + # tightly so the unscoped sweep cannot reach an interface the simulation + # never made. Broad only for an unscoped reap: once --run-id names a + # single run, no missing or empty suffix list may widen this back out to + # every run. + if [[ -z "$RUN_ID" && -z "$VETH_SUFFIXES" ]]; then + printf '^vh([0-9a-f]{4})?%s%s[ab]$' "$VETH_NODE_ID" "$VETH_NODE_ID" + return 0 + fi + if [[ -z "$VETH_SUFFIXES" ]]; then + veth_warn "--run-id given with no --veth-suffixes" + return 0 + fi + local sfx=() tok alt="" out rc + read -ra sfx <<< "${VETH_SUFFIXES//,/ }" + if [[ ${#sfx[@]} -eq 0 ]]; then + veth_warn "--veth-suffixes is empty" + return 0 + fi + if ! command -v python3 >/dev/null 2>&1; then + veth_warn "python3 not found, cannot derive interface tokens" + return 0 + fi + # SCRIPT_DIR comes from $0, so invoking this script through a symlink or a + # copy on $PATH points PYTHONPATH at a tree with no simulation in it. Say + # which tree was tried when the derivation fails, rather than no-opping. + out="$(PYTHONPATH="$SCRIPT_DIR/chaos" python3 -m sim.naming "${sfx[@]}" 2>&1)" + rc=$? + if [[ $rc -ne 0 ]]; then + veth_warn "token derivation failed under $SCRIPT_DIR/chaos: $out" + return 0 + fi + for tok in $out; do + [[ "$tok" =~ ^[0-9a-f]{4}$ ]] && alt="${alt:+$alt|}$tok" + done + if [[ -z "$alt" ]]; then + veth_warn "no interface tokens derived from: ${sfx[*]}" + return 0 + fi + printf '^vh(%s)%s%s[ab]$' "$alt" "$VETH_NODE_ID" "$VETH_NODE_ID" +} + +# ip(8) in a privileged --net=host container, matching how the simulation +# creates these interfaces (see chaos/sim/veth.py). +veth_ip() { + timeout "$TMO" docker run --rm --privileged --net=host \ + --entrypoint ip "$VETH_IMAGE" "$@" 2>/dev/null +} + +# The same, taking `ip -batch` commands on stdin so any number of deletes costs +# one container. -force keeps ip going past an interface that vanished under us. +veth_ip_batch() { + timeout "$TMO" docker run --rm -i --privileged --net=host \ + --entrypoint ip "$VETH_IMAGE" -force -batch - >/dev/null 2>&1 || true +} + +reap_veths() { + local pattern + pattern="$(veth_pattern)" + [[ -z "$pattern" ]] && return 0 + # Without the image there is no way to run ip(8). Orphans can outlive it — + # `docker image prune -a`, a build host that prunes between runs, or a run + # aborted before ci-local.sh retags :latest all remove it while interfaces + # are still up — so this is a real skip, not "nothing was ever run here". + if ! docker image inspect "$VETH_IMAGE" >/dev/null 2>&1; then + veth_warn "image $VETH_IMAGE not present, cannot run ip(8)" + return 0 + fi + # Both ends of a pair match, but deleting either removes both, so collapse + # each pair to one delete and issue the lot in a single container. The + # whole script runs under a caller-imposed timeout, and a container spawn + # per name would put a large orphan set at risk of exhausting it. `sort -u` + # orders `...a` before `...b`, so the surviving end of a pair the + # simulation was killed part-way through moving is still the one picked. + local names + names="$(veth_ip -o link show \ + | sed -E 's/^[0-9]+: ([^:@]+).*/\1/' \ + | grep -E "$pattern" | sort -u \ + | awk '{ k = substr($0, 1, length($0) - 1) + if (!(k in seen)) { seen[k] = 1; print } }')" + [[ -z "$names" ]] && return 0 + sed 's/^/link delete /' <<< "$names" | veth_ip_batch +} + reap_images() { [[ -z "$IMAGES" ]] && return 0 local imgs @@ -115,10 +247,12 @@ reap_images() { timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true } -# Order matters: containers reference networks/volumes, so drop them first. +# Order matters: containers reference networks/volumes, so drop them first, and +# the veth sweep needs an image to run ip(8) in, so it precedes the image reap. reap_containers reap_networks reap_volumes +reap_veths reap_images exit 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index d639f74..688608d 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -14,9 +14,14 @@ # --list List available integration suites # --check-parity Verify this suite set matches ci.yml's integration # matrix (see testing/check-ci-parity.sh), then exit -# --reap Force-remove all leftover FIPS CI docker resources +# --reap Force-remove all leftover FIPS CI resources # (containers/networks/volumes carrying the CI label or a -# fipsci_ compose project), then exit. See ci-cleanup.sh. +# fipsci_ compose project, plus every chaos-simulation +# host veth interface), then exit. See ci-cleanup.sh. +# Host interfaces carry no label, so they are matched by +# name shape: this reaps a bare chaos.sh run's live +# interfaces too, though not its containers. Don't run +# it while a bare simulation is up. # -h, --help Show this help # # Integration suites (default coverage): @@ -271,12 +276,25 @@ export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP" # container by name. Empty when unset, so a bare `docker compose up` outside # this harness still produces today's plain names. export FIPS_CI_NAME_SUFFIX="-${CI_RUN_ID}" +# run_chaos narrows that export to one scenario, and bash scopes a `local` +# dynamically: a function called while run_chaos is on the stack — the signal +# trap, on the `--only chaos-*` path where run_chaos is not a subshell — reads +# the narrowed value, not this one. Snapshot the run-wide value so the rebuild +# below always sees it, and make it readonly so no scope can shadow it either. +CI_RUN_NAME_SUFFIX="$FIPS_CI_NAME_SUFFIX" +readonly CI_RUN_NAME_SUFFIX # Per-suite compose project name: ${prefix}_. Keeps # today's intra-run distinctness (one project per compose file / chaos child) # while adding the cross-run prefix that scopes the reap. ci_project() { printf '%s_%s' "$CI_PROJECT_PREFIX" "$1"; } +# The name suffix one chaos scenario runs under. Its container names, its +# generated-config directory and the token in its host veth names all derive +# from this, so teardown recomputes it to know which interfaces are ours. Reads +# the snapshot rather than the export for the reason given above. +ci_chaos_suffix() { printf -- '-%s%s' "$1" "$CI_RUN_NAME_SUFFIX"; } + # PIDs of in-flight parallel chaos children (subshells). The trap signals these. CI_CHAOS_PIDS=() CI_CLEANED=0 @@ -301,13 +319,25 @@ ci_teardown() { fi # 2. Remove all compose projects + direct-run resources + per-run images - # for this run. ci-cleanup.sh wraps each docker op in `timeout`; bound - # the whole sweep too so the trap can never wedge. + # for this run, plus any host veth interface a chaos scenario was + # killed part-way through creating. Host interfaces carry no docker + # label, so hand over the suffixes this run's scenarios used and let + # the reap derive their names — a blind sweep would take a concurrent + # run's live interfaces with it. ci-cleanup.sh wraps each docker op in + # `timeout`; bound the whole sweep too so the trap can never wedge. + # Its stdout is routine progress and goes nowhere, but stderr carries + # only a skipped sweep or a bad option, and a sweep that quietly stops + # reaping is how interfaces would accumulate unnoticed. Let it through. + local _suffixes=() _entry + for _entry in "${CHAOS_SUITES[@]}"; do + _suffixes+=("$(ci_chaos_suffix "${_entry%% *}")") + done timeout 150 bash "$SCRIPT_DIR/ci-cleanup.sh" \ --label "$CI_LABEL" \ --run-id "$CI_RUN_ID" \ --project-prefix "$CI_PROJECT_PREFIX" \ - --images "$CI_IMAGE_TEST $CI_IMAGE_APP" >/dev/null 2>&1 || true + --images "$CI_IMAGE_TEST $CI_IMAGE_APP" \ + --veth-suffixes "${_suffixes[*]}" >/dev/null || true } on_signal() { @@ -524,7 +554,8 @@ run_chaos() { # scoped by the compose project, so narrow the run-wide suffix to this # scenario. Parallel children then cannot claim each other's names or # overwrite each other's compose file. - local suffix="-${name}${FIPS_CI_NAME_SUFFIX:-}" + local suffix + suffix="$(ci_chaos_suffix "$name")" local -x FIPS_CI_NAME_SUFFIX="$suffix" info "[chaos/$name] Running simulation" @@ -937,7 +968,12 @@ run_suite() { fi done if [[ "$found" != true ]]; then - # Fall back to using the name as the scenario directly + # Fall back to using the name as the scenario directly. Note + # teardown rebuilds veth suffixes from CHAOS_SUITES only, so a + # scenario named here but absent from that list is outside the + # host-interface reap. Harmless while every ethernet-transport + # scenario is declared there; add one that isn't and its + # orphaned pairs will need an unscoped `--reap` to clear. run_chaos "$chaos_name" "$chaos_name" fi ;; From 4f55a281ac9812c03799ff4007f1cad0790f0a23 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Wed, 22 Jul 2026 07:37:48 +0000 Subject: [PATCH 3/5] Let a failed chaos scenario fail the run The simulation caught every exception a scenario raised, logged it, and exited 0, so a scenario that died during setup was recorded as a pass. Local CI discarded the exit code in any case: run_chaos ends in a call whose own last statement is an echo, so it returned 0 whatever the scenario did, and the parallel launcher's wait therefore always succeeded and always recorded a pass. Between them no chaos failure of any kind could turn a local run red, and none could since the script was written. Carry the abort out of the run and give it its own exit code, ordered ahead of the two content codes so a run that never produced a mesh cannot report that mesh's panic and assertion counts. Return the scenario's status from run_chaos so the launcher sees it. The documented exit codes were stale before this change and are rewritten in full, including that the argument parser rejects a malformed command line with the same code that reports panics. Expect scenarios that have been reporting green to start reporting red. In the last recorded local run twelve of the thirteen scenarios never got past starting their containers, and all thirteen were counted as passes; those that now execute have not run for months, and their assertions have never been evaluated against a mesh of their own. --- testing/chaos/README.md | 16 +++++++++++++++- testing/chaos/sim/__main__.py | 5 +++++ testing/chaos/sim/runner.py | 10 ++++++++++ testing/ci-local.sh | 11 +++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/testing/chaos/README.md b/testing/chaos/README.md index 1a52191..7d21a1d 100644 --- a/testing/chaos/README.md +++ b/testing/chaos/README.md @@ -262,7 +262,21 @@ Results written to `sim-results/` (configurable via - `runner.log` -- Orchestration events (topology, netem, churn, traffic) with timestamps - `fips-node-nXX.log` -- Per-node log output -Exit code 0 on success, 2 if panics detected. +Exit codes: + +- `0` -- Ran to completion, no panics, every assertion passed +- `1` -- The scenario file could not be loaded, teardown itself raised, or a + second interrupt arrived while the first was being handled +- `2` -- Panics found in the collected node logs. Also what the argument + parser exits with when it rejects the command line, before any run starts +- `3` -- A post-run assertion failed +- `4` -- Setup, warmup or the simulation loop raised, so the run did not + complete; `runner.log` carries the traceback + +Codes 2 and 3 describe what a mesh that ran did. Code 4 says there is +nothing to describe, and takes precedence over both. Code 2 is dual-use: +a run that never started cannot have panicked, so read it together with +whether `runner.log` exists. ## Creating Custom Scenarios diff --git a/testing/chaos/sim/__main__.py b/testing/chaos/sim/__main__.py index ba4f2b8..ec1867f 100644 --- a/testing/chaos/sim/__main__.py +++ b/testing/chaos/sim/__main__.py @@ -60,6 +60,11 @@ def main(): runner = SimRunner(scenario) result = runner.run() + # Liveness before content. A simulation that raised on its way up, or part + # way through, has no panic count or assertion outcome worth reporting, and + # saying so is more useful than whatever partial content it did produce. + if runner.aborted: + sys.exit(4) if result and result.panics: sys.exit(2) if runner.assertions_failed: diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 056f359..61f2ce1 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -40,6 +40,11 @@ class SimRunner: self.output_dir: str = self._resolve_output_dir(scenario) self._interrupted = False + # Set when setup, warmup or the simulation loop raises. The exception + # is logged rather than propagated, so nothing else on the return path + # of run() carries the fact that the simulation did not complete. + self.aborted = False + # Shared set of currently-down node IDs (updated by NodeManager, # read by NetemManager, LinkManager, TrafficManager) self._down_nodes: set[str] = set() @@ -82,7 +87,12 @@ class SimRunner: self._warmup() self._simulation_loop() except Exception: + # Log rather than re-raise: the default excepthook writes to stderr + # and would not reach the file handler installed in _setup(), so a + # re-raise would lose the traceback from runner.log, which is the + # artifact that survives into CI. The flag carries the abort out. log.exception("Simulation failed") + self.aborted = True finally: result = self._teardown() diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 688608d..ffdb60d 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -566,6 +566,17 @@ run_chaos() { fi record "chaos-$name" $rc + + # record() ends in pass()/fail(), which are echoes, so it returns 0 for any + # rc — and without this return so does run_chaos. On the parallel path the + # function's status is the child subshell's status, and that is the only + # thing the waiting parent can see: the child's own RESULTS entry and its + # OVERALL=1 die with its process, and its FAIL line goes to a logfile only + # the unreachable branch reads. So a bare record() left every scenario + # recorded as a pass whatever the simulation reported. On the --only path + # record() already wrote the true rc into the parent's own RESULTS, and + # returning it as well is inert: this script does not set -e. + return $rc } # Run gateway integration test From 226c6994d3b25810120040d8a823c6346dab9931 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Wed, 22 Jul 2026 08:02:38 +0000 Subject: [PATCH 4/5] Harvest chaos results only from a mesh that actually started Teardown runs from a finally, so it also runs after a failed setup, and it was guarded only on the topology and the compose file, both of which are set well before any container exists. A scenario that died bringing its containers up therefore ran the whole harvest anyway: final snapshots, docker logs, the analysis, the assertions and the metadata, all addressing containers by names that are global to the host. What it left behind was not an empty directory an investigator would notice but a full and plausible one, in the recorded case ten node logs and an analysis reporting two promotions and two parent switches, every byte of it from a different scenario's mesh. Gate the harvest on whether the containers ever started, and leave the compose down outside that gate so a partly successful start still gets cleaned up. Record the outcome in a status file in every result directory, naming the run as completed, interrupted, aborted, setup-failed or teardown-failed, alongside the scenario, the seed and the container names it used. With the harvest gated, the presence of an analysis file is now itself proof that the scenario's own mesh existed, and the status file says which of the several ways a run can end applies. A run cut short by a signal keeps the existing exit codes rather than gaining one of its own: what it collected before stopping is real and still worth reporting, the status file records that it was truncated, and every wrapper already reports a Ctrl-C of its own. Log collection never looked at the status of docker logs. It concatenated stdout and stderr unconditionally, so collecting from containers that were not there wrote the daemon's "No such container" reply into each node log and analysed the result as a mesh with no panics, no errors and no sessions, which reads exactly like a clean run and exits zero. Check the return code, and treat a harvest that cannot read every container, or that reads none, as a failed teardown. A teardown that raises now also reports on the same footing as a run that never started, instead of escaping past the exit codes as a bare traceback and being read as a malformed command line. --- testing/chaos/README.md | 33 +++++++++++++++-- testing/chaos/sim/logs.py | 49 ++++++++++++++++++------ testing/chaos/sim/runner.py | 74 +++++++++++++++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 19 deletions(-) diff --git a/testing/chaos/README.md b/testing/chaos/README.md index 7d21a1d..401c40e 100644 --- a/testing/chaos/README.md +++ b/testing/chaos/README.md @@ -257,27 +257,52 @@ since they use beacon discovery. Results written to `sim-results/` (configurable via `logging.output_dir`): +- `status.txt` -- How the run ended, plus the scenario, the seed and the + container names it used; one `key=value` per line - `analysis.txt` -- Summary: panics, errors, sessions, metrics - `metadata.txt` -- Seed, node count, edges, adjacency list - `runner.log` -- Orchestration events (topology, netem, churn, traffic) with timestamps - `fips-node-nXX.log` -- Per-node log output +The `status` field reads: + +- `completed` -- ran for its configured duration +- `interrupted` -- a signal cut the run short, so the artifacts are real + but describe less time than the scenario asked for +- `aborted` -- the run raised part way through; same caveat, and + `runner.log` carries the traceback +- `setup-failed` -- the containers never started +- `teardown-failed` -- the mesh ran but its logs or analysis could not be + produced + +A `setup-failed` directory holds `runner.log` and `status.txt` and nothing +else. Nothing is harvested, because container names are global to the host +and reading them after a failed setup describes whichever run holds them +now. So `analysis.txt` in a result directory is proof that this scenario's +own mesh existed. A directory with no `status.txt` was written before this +was the case and says nothing either way. + Exit codes: - `0` -- Ran to completion, no panics, every assertion passed -- `1` -- The scenario file could not be loaded, teardown itself raised, or a - second interrupt arrived while the first was being handled +- `1` -- The scenario file could not be loaded, or a second interrupt + arrived while the first was being handled - `2` -- Panics found in the collected node logs. Also what the argument parser exits with when it rejects the command line, before any run starts - `3` -- A post-run assertion failed -- `4` -- Setup, warmup or the simulation loop raised, so the run did not - complete; `runner.log` carries the traceback +- `4` -- Setup, warmup, the simulation loop or teardown raised, so the run + did not complete; `runner.log` carries the traceback Codes 2 and 3 describe what a mesh that ran did. Code 4 says there is nothing to describe, and takes precedence over both. Code 2 is dual-use: a run that never started cannot have panicked, so read it together with whether `runner.log` exists. +A run stopped by a signal exits on this same ladder rather than one of its +own: what it collected before stopping is still worth reporting, and +`status.txt` says it was cut short. `chaos.sh` reports 130 for a Ctrl-C of +its own accord. + ## Creating Custom Scenarios 1. Copy an existing scenario from `scenarios/`. diff --git a/testing/chaos/sim/logs.py b/testing/chaos/sim/logs.py index 0033635..e698002 100644 --- a/testing/chaos/sim/logs.py +++ b/testing/chaos/sim/logs.py @@ -29,9 +29,18 @@ __all__ = ["AnalysisResult", "analyze_logs", "collect_logs", "write_sim_metadata def collect_logs(container_names: list[str], output_dir: str) -> dict[str, str]: - """Collect all output (stdout + stderr) from all containers.""" + """Collect all output (stdout + stderr) from all containers. + + Raises RuntimeError if any container's output could not be read, or if + there was nothing to read. `docker logs` writes its own failures to + stderr and exits non-zero, so without the returncode check the daemon's + "No such container" reply was stored as though it were the node's log + and analysed as a mesh with no panics, no errors and no sessions -- + which reads exactly like a clean run. + """ os.makedirs(output_dir, exist_ok=True) logs = {} + failed = [] for name in container_names: try: @@ -41,17 +50,35 @@ def collect_logs(container_names: list[str], output_dir: str) -> dict[str, str]: text=True, timeout=30, ) - raw = result.stdout + result.stderr - log_text = strip_ansi(raw) - logs[name] = log_text - - path = os.path.join(output_dir, f"{name}.log") - with open(path, "w") as f: - f.write(log_text) - - except (subprocess.TimeoutExpired, Exception) as e: + except Exception as e: log.warning("Failed to collect logs from %s: %s", name, e) - logs[name] = "" + failed.append(name) + continue + + if result.returncode != 0: + log.warning( + "docker logs %s exited %d: %s", + name, result.returncode, result.stderr.strip(), + ) + failed.append(name) + continue + + # A container's own stderr comes back on our stderr, so both + # streams are log content on the success path. + log_text = strip_ansi(result.stdout + result.stderr) + logs[name] = log_text + + path = os.path.join(output_dir, f"{name}.log") + with open(path, "w") as f: + f.write(log_text) + + if failed: + raise RuntimeError( + f"could not collect logs from {len(failed)}/{len(container_names)} " + f"containers: {', '.join(failed)}" + ) + if not logs: + raise RuntimeError("no container logs were collected") return logs diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 61f2ce1..cd34cd0 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -20,6 +20,7 @@ from .docker_exec import docker_compose from .link_swap import LinkSwapManager from .links import LinkManager from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata +from .naming import name_suffix from .netem import NetemManager from .nodes import NodeManager from .peer_churn import PeerChurnManager @@ -45,6 +46,13 @@ class SimRunner: # of run() carries the fact that the simulation did not complete. self.aborted = False + # Whether this run ever owned containers. Teardown runs from a + # finally, so it runs after a failed setup too, and container names + # are global: without this it would harvest whatever currently + # answers to them. topology and compose_file are both set well + # before the containers start and so cannot stand in for it. + self._containers_started = False + # Shared set of currently-down node IDs (updated by NodeManager, # read by NetemManager, LinkManager, TrafficManager) self._down_nodes: set[str] = set() @@ -190,6 +198,7 @@ class SimRunner: # 5. Start containers log.info("Starting %d containers...", len(self.topology.nodes)) docker_compose(self.compose_file, ["up", "-d"]) + self._containers_started = True # 6. Set up veth pairs for Ethernet edges (before netem) # @@ -427,11 +436,57 @@ class SimRunner: def assertions_failed(self) -> bool: return any(not o.passed for o in self.assertion_outcomes) + def _run_status(self) -> str: + """Name for how the run itself ended, before teardown is considered.""" + if self.aborted: + return "aborted" + if self._interrupted: + return "interrupted" + return "completed" + + def _write_status(self, status: str) -> None: + """Record how the run ended, for a reader who has only this directory. + + One field per line so it can be grepped. The container names are + included because they are the handle on everything else the run + touched, and because a directory that names them cannot be mistaken + for a directory assembled from some other scenario's mesh. + + Never raises. Both callers run this immediately before stopping the + containers, so letting a full disk or a vanished output directory + out of here would leak the mesh it was about to tear down. + """ + try: + os.makedirs(self.output_dir, exist_ok=True) + path = os.path.join(self.output_dir, "status.txt") + with open(path, "w") as f: + f.write(f"status={status}\n") + f.write(f"scenario={self.scenario.name}\n") + f.write(f"seed={self.scenario.seed}\n") + f.write(f"containers=fips-node-nNN{name_suffix()}\n") + except Exception: + log.exception("Could not write status file") + def _teardown(self) -> AnalysisResult | None: """Stop dynamic elements, collect logs, analyze, stop containers.""" - result = None + if not self._containers_started: + # There is no mesh to harvest. Snapshots, logs, analysis and + # assertions all address containers by a name that is global to + # the host, so running them here would describe whoever holds + # those names now and leave a plausible report of a run that + # never happened. Leaving them out makes the presence of + # analysis.txt proof that this scenario's own mesh existed. + self._write_status("setup-failed") + if self.compose_file: + # `up -d` can fail part way through, so this run may own + # containers or a network even with no mesh to speak of. + log.info("Stopping containers...") + docker_compose(self.compose_file, ["down"], check=False) + return None - if self.topology and self.compose_file: + result = None + status = self._run_status() + try: # Evaluate post-run assertions before doing any teardown so # control sockets are still reachable. self._evaluate_assertions() @@ -516,8 +571,19 @@ class SimRunner: if self.veth_mgr: log.info("Cleaning up veth pairs...") self.veth_mgr.teardown_all() - - # Stop containers + except Exception: + # Same reasoning as run(): logging keeps the traceback in + # runner.log, where re-raising would send it to stderr instead + # and past the exit ladder, which would report a harvest that + # fell over as a bad command line. + log.exception("Teardown failed") + self.aborted = True + status = "teardown-failed" + result = None + finally: + # Status first: it is the one artifact that must exist whatever + # else happens, and stopping containers can still time out. + self._write_status(status) log.info("Stopping containers...") docker_compose( self.compose_file, From ab750f60b3eaafa7fa4832a5859e879e4e6a2676 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Wed, 22 Jul 2026 08:26:59 +0000 Subject: [PATCH 5/5] Record why a chaos docker command failed Compose commands run with their output captured and with check set, so a failure raised a CalledProcessError and nothing ever looked at what the daemon had said. That exception reports the argv and the exit status and nothing else, so every scenario that died bringing its containers up left a runner log saying only that docker compose returned non-zero exit status 1, and the reason was captured and then discarded. In the run that prompted this work twelve scenarios failed that way and why they failed can no longer be established at all. Log the captured stderr and stdout before the error goes anywhere, keeping the tail of each so a verbose failure cannot fill the log file. A command that times out raises before its exit status is ever read, so that path records whatever it had emitted too, rather than leaving the same hole one shape over. Reproduced against both of the shapes this is meant to catch: an unusable network prefix now records the daemon's parse error, and a name already held by another container now records the conflict and names the container holding it. The success path logs nothing new, and a completed scenario's runner log is unchanged line for line. The raising behaviour is deliberately untouched, since the abort flag and the exit code that came with it depend on it. The check is applied here rather than by subprocess so that a caller who passed check false gets the same record: both such callers are the compose down in teardown, which today can leave a mesh behind without saying so. The same omission was costing the veth path its reasons. Failing ip commands had their stderr logged at debug, which the runner does not emit unless asked for verbose output, so a failed pair creation reached the log as nothing more than the names of the two interfaces it could not create. Raise that to a warning. The deletes issued to clear a stale pair are expected to fail and still say nothing. --- testing/chaos/sim/docker_exec.py | 78 ++++++++++++++++++++++++++++---- testing/chaos/sim/veth.py | 6 ++- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/testing/chaos/sim/docker_exec.py b/testing/chaos/sim/docker_exec.py index 2d8781b..fcd328c 100644 --- a/testing/chaos/sim/docker_exec.py +++ b/testing/chaos/sim/docker_exec.py @@ -7,6 +7,34 @@ import logging log = logging.getLogger(__name__) +# `docker compose` renders its progress UI on stderr, so a failed command can +# have hundreds of lines of captured output behind it. Log the tail: whatever +# the daemon refused is the last thing it wrote. +OUTPUT_TAIL_CHARS = 2000 + + +def _tail(text: str) -> str: + """Trim captured output to its last few KB for logging.""" + text = text.strip() + if not text: + return "(empty)" + if len(text) <= OUTPUT_TAIL_CHARS: + return text + return "...\n" + text[-OUTPUT_TAIL_CHARS:] + + +def _decode(output) -> str: + """Render captured output as text. + + A timed-out command hands back what it had written as bytes, even when + the call asked for text, so the timeout path cannot assume either. + """ + if output is None: + return "" + if isinstance(output, bytes): + return output.decode(errors="replace") + return output + class DockerExecError(Exception): def __init__(self, container, cmd, returncode, stderr): @@ -47,16 +75,50 @@ def docker_compose( timeout: int = 300, check: bool = True, ) -> subprocess.CompletedProcess: - """Run a docker compose command with the given compose file.""" + """Run a docker compose command with the given compose file. + + Output is captured, so a failure's stderr is logged here before anything + else sees it. `CalledProcessError` reports only the argv and the exit + status, and a `check=False` caller reads neither, so without this the one + place the daemon says what it objected to -- a container name already in + use, an unusable subnet, a missing image -- is captured and then thrown + away. Nothing about the success path changes. + """ cmd = ["docker", "compose", "-f", compose_file] + args log.info("Running: %s", " ".join(cmd)) - return subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - check=check, - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as e: + # A timeout raises from inside communicate(), so the return-code + # branch below never runs and this is the only chance to say what + # the command had managed to emit. The partials arrive as bytes + # even under text=True. + log.error( + "%s timed out after %ds\nstderr: %s\nstdout: %s", + " ".join(cmd), + timeout, + _tail(_decode(e.stderr)), + _tail(_decode(e.stdout)), + ) + raise + if result.returncode != 0: + log.error( + "%s exited %d\nstderr: %s\nstdout: %s", + " ".join(cmd), + result.returncode, + _tail(result.stderr), + _tail(result.stdout), + ) + if check: + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result def is_container_running(container: str) -> bool: diff --git a/testing/chaos/sim/veth.py b/testing/chaos/sim/veth.py index 13b7bd5..a60a25f 100644 --- a/testing/chaos/sim/veth.py +++ b/testing/chaos/sim/veth.py @@ -252,7 +252,11 @@ def _run_host(cmd: list[str], image: str, check: bool = True) -> bool: timeout=30, ) if check and result.returncode != 0: - log.debug( + # Warning, not debug: the runner logs at INFO unless asked for + # -v, so at debug this never reached runner.log and the callers + # below report only that a pair could not be created. The + # check=False deletes are expected to fail and stay silent. + log.warning( "ip cmd failed: %s -> %s", " ".join(cmd), result.stderr.strip(),