diff --git a/testing/README.md b/testing/README.md index 4a8c13d..2dd2760 100644 --- a/testing/README.md +++ b/testing/README.md @@ -144,3 +144,22 @@ and leaves resources behind, reap them with: label or a `fipsci_` compose-project prefix; it is safe to run when there is nothing to reap and safe to run repeatedly. Pass `--project-prefix` to scope the sweep to a single run. + +It also removes the chaos simulation's leftover host-namespace veth +interfaces (`vh…a`/`vh…b`), the one resource it touches that is neither a +docker object nor labelled — a host interface can carry neither a label +nor a compose project, so it is matched by name shape alone. That makes +the reach here asymmetric with everything above, and worth stating +plainly: + +- A bare `chaos.sh` run's **containers** survive a broad reap. Its + compose project is not `fipsci_`, and the simulation labels only the + network, not the services. +- A bare `chaos.sh` run's **veth interfaces do not.** An unscoped reap + deletes them while they are in use, severing the Ethernet links of a + live simulation and leaving its containers running. + +So do not run a broad `--reap` while a bare simulation is up. Scope the +interface sweep with `--veth-suffixes` (which is what `ci-local.sh`'s own +teardown passes) or wait for the simulation to finish. `--project-prefix` +does not help here: it scopes only the compose-project sweep. diff --git a/testing/chaos/sim/naming.py b/testing/chaos/sim/naming.py index d3418b1..61062d9 100644 --- a/testing/chaos/sim/naming.py +++ b/testing/chaos/sim/naming.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import os +import sys def name_suffix() -> str: @@ -15,3 +17,25 @@ def name_suffix() -> str: bare ``chaos.sh`` run renders exactly the same names as it always has. """ return os.environ.get("FIPS_CI_NAME_SUFFIX", "") + + +def veth_token(suffix: str) -> str: + """Shorten a name suffix to four hex characters. + + Host interface names have only 15 characters to work with, far fewer + than the suffix needs, so concurrent scenarios are told apart by a + hash of it instead. Empty for an empty suffix, so a bare run's + interface names are unchanged. + """ + if not suffix: + return "" + return hashlib.sha1(suffix.encode()).hexdigest()[:4] + + +if __name__ == "__main__": + # Print the token for each suffix given on the command line, one per + # line. ci-cleanup.sh reaps host interfaces by token and calls this + # rather than re-deriving the hash, so widening the token here cannot + # leave the reaper matching the old width. + for arg in sys.argv[1:]: + print(veth_token(arg)) diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py index c8b5d9c..69ea769 100644 --- a/testing/chaos/sim/topology.py +++ b/testing/chaos/sim/topology.py @@ -8,7 +8,7 @@ from collections import deque from dataclasses import dataclass, field from .keys import derive -from .naming import name_suffix +from .naming import name_suffix, veth_token from .scenario import TopologyConfig @@ -33,6 +33,15 @@ class SimTopology: # outside the CI harness, which keeps a bare run's names unchanged. name_suffix: str = "" + @property + def veth_token(self) -> str: + """Short stand-in for the suffix, for names bound by IFNAMSIZ. + + Derived rather than stored so no caller can build a topology whose + host names are scoped differently from its container names. + """ + return veth_token(self.name_suffix) + def transport_for_edge(self, a: str, b: str) -> str: """Get the transport type for an edge (defaults to 'udp').""" edge = _make_edge(a, b) @@ -113,6 +122,26 @@ class SimTopology: def container_name(self, node_id: str) -> str: return f"fips-node-{node_id}{self.name_suffix}" + def veth_host_name(self, node_a: str, node_b: str, end: str) -> str: + """Generate the host-namespace veth name for one end of an edge. + + Format: ``vh{token}{NN}{MM}{end}`` (max 15 chars for IFNAMSIZ). + Host interfaces are global, so the token keeps a scenario from + deleting a concurrent scenario's pair; it is empty outside the CI + harness, yielding the same "vh0104a" this has always produced. + + ``node_a`` and ``node_b`` must be in canonical edge order. Unlike + ``veth_interface_name()`` this is not symmetric: the far end is + ``end="b"`` on the same ordering, so swapping the arguments names + an interface that does not exist. + """ + nn_local = node_a.replace("n", "") + nn_peer = node_b.replace("n", "") + name = f"vh{self.veth_token}{nn_local}{nn_peer}{end}" + if len(name) > 15: + raise ValueError(f"veth host name too long: {name!r} ({len(name)} > 15)") + return name + def directed_outbound(self) -> dict[str, list[str]]: """Assign each static-config edge to exactly one node for outbound connection. diff --git a/testing/chaos/sim/veth.py b/testing/chaos/sim/veth.py index 95d1e96..13b7bd5 100644 --- a/testing/chaos/sim/veth.py +++ b/testing/chaos/sim/veth.py @@ -4,7 +4,8 @@ Creates veth pairs between Docker containers for Ethernet-transport edges. Each Ethernet edge gets a veth pair with one end moved into each container's network namespace. Naming: - Host (temporary): vh{NN}{MM}a / vh{NN}{MM}b + Host (temporary): vh{token}{NN}{MM}a / vh{token}{NN}{MM}b + (via SimTopology.veth_host_name()) Container: ve-{local}-{peer} (via veth_interface_name()) After creation, the container-side MAC addresses are queried and @@ -113,9 +114,7 @@ class VethManager: if a != node_id and b != node_id: continue # Remove existing pair if any (host-side might still exist) - nn_a = a.replace("n", "") - nn_b = b.replace("n", "") - host_a = f"vh{nn_a}{nn_b}a" + host_a = self.topology.veth_host_name(a, b, "a") _run_host(["ip", "link", "delete", host_a], image, check=False) # Re-create self._create_veth_pair(a, b, image) @@ -142,14 +141,14 @@ class VethManager: return # Generate names - nn_a = node_a.replace("n", "") - nn_b = node_b.replace("n", "") - host_a = f"vh{nn_a}{nn_b}a" - host_b = f"vh{nn_a}{nn_b}b" + host_a = self.topology.veth_host_name(node_a, node_b, "a") + host_b = self.topology.veth_host_name(node_a, node_b, "b") final_a = veth_interface_name(node_a, node_b) final_b = veth_interface_name(node_b, node_a) - # Clean up any stale pair + # Clean up a stale pair left by this scenario. The token makes the + # name unique to this run, so a pair orphaned by an earlier run is + # no longer reclaimed here — `ci-cleanup.sh` reaps those instead. _run_host(["ip", "link", "delete", host_a], image, check=False) # Create veth pair on host diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index 1231636..fbcb5ec 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Reap FIPS CI docker resources: containers, networks, volumes, images. +# Reap FIPS CI resources: containers, networks, volumes, images, veth pairs. # # Force-removes everything created by ci-local.sh that is still around — # whether a run finished cleanly, was preempted (SIGTERM/SIGKILL), OOM-killed, @@ -19,26 +19,56 @@ # the label sweep to one run; a run's own teardown always does. Without it the # label sweep stays broad, which is what a manual "reap everything" wants. # +# Host-namespace veth interfaces are the one non-docker resource reaped here. +# The chaos simulation creates each pair in the host namespace and then moves +# the two ends into container namespaces, so a run killed in between leaves the +# pair behind, and nothing else on the box removes it. Their names carry a +# short token derived from the scenario's name suffix, so --veth-suffixes takes +# the suffixes a run used and reaps only the tokens those could produce. Only a +# reap with neither --run-id nor --veth-suffixes matches every simulation veth +# name, in step with the broad label sweep. --project-prefix does not scope the +# veth sweep at all: a host interface carries no compose project. +# +# These interfaces are also the one resource reaped WITHOUT regard to the CI +# label — they cannot carry one — so an unscoped reap takes the host ends of a +# bare `chaos.sh` simulation's veth pairs too, even though it leaves that +# simulation's unlabelled containers running. Scope with --veth-suffixes (or +# just don't reap) while a bare simulation is up. +# # Usage: # ci-cleanup.sh Reap ALL fips-ci resources (any run) # ci-cleanup.sh --project-prefix P Restrict the compose-project sweep to # names starting with P (scopes the reap -# to a single run) +# to a single run). Does NOT scope the +# host veth sweep — see --veth-suffixes # ci-cleanup.sh --run-id ID Restrict the label sweep to the run # labelled ID (leaves other runs alone) # ci-cleanup.sh --label L Override the CI label (default above) # ci-cleanup.sh --images "a b,c" Also `docker rmi -f` these image tags # (space- or comma-separated) +# ci-cleanup.sh --veth-suffixes "a b" Restrict the host veth sweep to the +# name suffixes a single run used +# (space- or comma-separated). Without +# it the sweep reaches every simulation +# veth name on the host, including a +# bare `chaos.sh` run's live ones # # Safe to run when there is nothing to reap, and safe to run repeatedly. # Also reachable as `ci-local.sh --reap`. set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + LABEL="com.corganlabs.fips-ci=1" RUN_LABEL_KEY="com.corganlabs.fips-ci.run" PROJECT_PREFIX="fipsci_" # broad default: every CI run RUN_ID="" # broad default: every CI run IMAGES="" +VETH_SUFFIXES="" # empty AND no --run-id: every simulation veth name +# ip(8) runs inside this image, the same way the simulation creates the +# interfaces, so the reap works wherever the simulation does. The chaos +# simulation builds it, and it carries iproute2. +VETH_IMAGE="fips-test:latest" while [[ $# -gt 0 ]]; do case "$1" in @@ -46,6 +76,7 @@ while [[ $# -gt 0 ]]; do --project-prefix) PROJECT_PREFIX="$2"; shift 2 ;; --run-id) RUN_ID="$2"; shift 2 ;; --images) IMAGES="$2"; shift 2 ;; + --veth-suffixes) VETH_SUFFIXES="$2"; shift 2 ;; -h|--help) sed -n '2,/^set /{ /^set /d; s/^# \?//; p }' "$0"; exit 0 ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac @@ -107,6 +138,107 @@ reap_volumes() { | xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true } +# Every failure below leaves interfaces behind rather than widening the sweep, +# so each one is silent by construction. Say so on stderr instead, or a reap +# that reclaimed nothing looks exactly like a reap that had nothing to reclaim. +veth_warn() { echo "ci-cleanup: host veth sweep skipped: $*" >&2; } + +# One node id, exactly as chaos/sim/topology.py renders it (f"n{i+1:02d}"): +# zero-padded to two digits, and never zero-padded beyond that. Spelling it out +# keeps shapes the simulation cannot emit (vh01020a) out of the sweep. +VETH_NODE_ID='(0[0-9]|[1-9][0-9]+)' + +# Regex matching the host veth names to remove. With --veth-suffixes it covers +# only the tokens those suffixes hash to, so a concurrent run's interfaces — +# which carry a different token — cannot match. The token derivation is read +# from the simulation itself rather than repeated here, so widening it cannot +# leave this matching the old width. Empty output means "reap nothing". +veth_pattern() { + # vh{token}{NN}{MM}{a,b}: the token is 4 hex or wholly absent — never a + # part of one — and the two node ids follow. Anchored and shaped this + # tightly so the unscoped sweep cannot reach an interface the simulation + # never made. Broad only for an unscoped reap: once --run-id names a + # single run, no missing or empty suffix list may widen this back out to + # every run. + if [[ -z "$RUN_ID" && -z "$VETH_SUFFIXES" ]]; then + printf '^vh([0-9a-f]{4})?%s%s[ab]$' "$VETH_NODE_ID" "$VETH_NODE_ID" + return 0 + fi + if [[ -z "$VETH_SUFFIXES" ]]; then + veth_warn "--run-id given with no --veth-suffixes" + return 0 + fi + local sfx=() tok alt="" out rc + read -ra sfx <<< "${VETH_SUFFIXES//,/ }" + if [[ ${#sfx[@]} -eq 0 ]]; then + veth_warn "--veth-suffixes is empty" + return 0 + fi + if ! command -v python3 >/dev/null 2>&1; then + veth_warn "python3 not found, cannot derive interface tokens" + return 0 + fi + # SCRIPT_DIR comes from $0, so invoking this script through a symlink or a + # copy on $PATH points PYTHONPATH at a tree with no simulation in it. Say + # which tree was tried when the derivation fails, rather than no-opping. + out="$(PYTHONPATH="$SCRIPT_DIR/chaos" python3 -m sim.naming "${sfx[@]}" 2>&1)" + rc=$? + if [[ $rc -ne 0 ]]; then + veth_warn "token derivation failed under $SCRIPT_DIR/chaos: $out" + return 0 + fi + for tok in $out; do + [[ "$tok" =~ ^[0-9a-f]{4}$ ]] && alt="${alt:+$alt|}$tok" + done + if [[ -z "$alt" ]]; then + veth_warn "no interface tokens derived from: ${sfx[*]}" + return 0 + fi + printf '^vh(%s)%s%s[ab]$' "$alt" "$VETH_NODE_ID" "$VETH_NODE_ID" +} + +# ip(8) in a privileged --net=host container, matching how the simulation +# creates these interfaces (see chaos/sim/veth.py). +veth_ip() { + timeout "$TMO" docker run --rm --privileged --net=host \ + --entrypoint ip "$VETH_IMAGE" "$@" 2>/dev/null +} + +# The same, taking `ip -batch` commands on stdin so any number of deletes costs +# one container. -force keeps ip going past an interface that vanished under us. +veth_ip_batch() { + timeout "$TMO" docker run --rm -i --privileged --net=host \ + --entrypoint ip "$VETH_IMAGE" -force -batch - >/dev/null 2>&1 || true +} + +reap_veths() { + local pattern + pattern="$(veth_pattern)" + [[ -z "$pattern" ]] && return 0 + # Without the image there is no way to run ip(8). Orphans can outlive it — + # `docker image prune -a`, a build host that prunes between runs, or a run + # aborted before ci-local.sh retags :latest all remove it while interfaces + # are still up — so this is a real skip, not "nothing was ever run here". + if ! docker image inspect "$VETH_IMAGE" >/dev/null 2>&1; then + veth_warn "image $VETH_IMAGE not present, cannot run ip(8)" + return 0 + fi + # Both ends of a pair match, but deleting either removes both, so collapse + # each pair to one delete and issue the lot in a single container. The + # whole script runs under a caller-imposed timeout, and a container spawn + # per name would put a large orphan set at risk of exhausting it. `sort -u` + # orders `...a` before `...b`, so the surviving end of a pair the + # simulation was killed part-way through moving is still the one picked. + local names + names="$(veth_ip -o link show \ + | sed -E 's/^[0-9]+: ([^:@]+).*/\1/' \ + | grep -E "$pattern" | sort -u \ + | awk '{ k = substr($0, 1, length($0) - 1) + if (!(k in seen)) { seen[k] = 1; print } }')" + [[ -z "$names" ]] && return 0 + sed 's/^/link delete /' <<< "$names" | veth_ip_batch +} + reap_images() { [[ -z "$IMAGES" ]] && return 0 local imgs @@ -115,10 +247,12 @@ reap_images() { timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true } -# Order matters: containers reference networks/volumes, so drop them first. +# Order matters: containers reference networks/volumes, so drop them first, and +# the veth sweep needs an image to run ip(8) in, so it precedes the image reap. reap_containers reap_networks reap_volumes +reap_veths reap_images exit 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index d639f74..688608d 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -14,9 +14,14 @@ # --list List available integration suites # --check-parity Verify this suite set matches ci.yml's integration # matrix (see testing/check-ci-parity.sh), then exit -# --reap Force-remove all leftover FIPS CI docker resources +# --reap Force-remove all leftover FIPS CI resources # (containers/networks/volumes carrying the CI label or a -# fipsci_ compose project), then exit. See ci-cleanup.sh. +# fipsci_ compose project, plus every chaos-simulation +# host veth interface), then exit. See ci-cleanup.sh. +# Host interfaces carry no label, so they are matched by +# name shape: this reaps a bare chaos.sh run's live +# interfaces too, though not its containers. Don't run +# it while a bare simulation is up. # -h, --help Show this help # # Integration suites (default coverage): @@ -271,12 +276,25 @@ export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP" # container by name. Empty when unset, so a bare `docker compose up` outside # this harness still produces today's plain names. export FIPS_CI_NAME_SUFFIX="-${CI_RUN_ID}" +# run_chaos narrows that export to one scenario, and bash scopes a `local` +# dynamically: a function called while run_chaos is on the stack — the signal +# trap, on the `--only chaos-*` path where run_chaos is not a subshell — reads +# the narrowed value, not this one. Snapshot the run-wide value so the rebuild +# below always sees it, and make it readonly so no scope can shadow it either. +CI_RUN_NAME_SUFFIX="$FIPS_CI_NAME_SUFFIX" +readonly CI_RUN_NAME_SUFFIX # Per-suite compose project name: ${prefix}_. Keeps # today's intra-run distinctness (one project per compose file / chaos child) # while adding the cross-run prefix that scopes the reap. ci_project() { printf '%s_%s' "$CI_PROJECT_PREFIX" "$1"; } +# The name suffix one chaos scenario runs under. Its container names, its +# generated-config directory and the token in its host veth names all derive +# from this, so teardown recomputes it to know which interfaces are ours. Reads +# the snapshot rather than the export for the reason given above. +ci_chaos_suffix() { printf -- '-%s%s' "$1" "$CI_RUN_NAME_SUFFIX"; } + # PIDs of in-flight parallel chaos children (subshells). The trap signals these. CI_CHAOS_PIDS=() CI_CLEANED=0 @@ -301,13 +319,25 @@ ci_teardown() { fi # 2. Remove all compose projects + direct-run resources + per-run images - # for this run. ci-cleanup.sh wraps each docker op in `timeout`; bound - # the whole sweep too so the trap can never wedge. + # for this run, plus any host veth interface a chaos scenario was + # killed part-way through creating. Host interfaces carry no docker + # label, so hand over the suffixes this run's scenarios used and let + # the reap derive their names — a blind sweep would take a concurrent + # run's live interfaces with it. ci-cleanup.sh wraps each docker op in + # `timeout`; bound the whole sweep too so the trap can never wedge. + # Its stdout is routine progress and goes nowhere, but stderr carries + # only a skipped sweep or a bad option, and a sweep that quietly stops + # reaping is how interfaces would accumulate unnoticed. Let it through. + local _suffixes=() _entry + for _entry in "${CHAOS_SUITES[@]}"; do + _suffixes+=("$(ci_chaos_suffix "${_entry%% *}")") + done timeout 150 bash "$SCRIPT_DIR/ci-cleanup.sh" \ --label "$CI_LABEL" \ --run-id "$CI_RUN_ID" \ --project-prefix "$CI_PROJECT_PREFIX" \ - --images "$CI_IMAGE_TEST $CI_IMAGE_APP" >/dev/null 2>&1 || true + --images "$CI_IMAGE_TEST $CI_IMAGE_APP" \ + --veth-suffixes "${_suffixes[*]}" >/dev/null || true } on_signal() { @@ -524,7 +554,8 @@ run_chaos() { # scoped by the compose project, so narrow the run-wide suffix to this # scenario. Parallel children then cannot claim each other's names or # overwrite each other's compose file. - local suffix="-${name}${FIPS_CI_NAME_SUFFIX:-}" + local suffix + suffix="$(ci_chaos_suffix "$name")" local -x FIPS_CI_NAME_SUFFIX="$suffix" info "[chaos/$name] Running simulation" @@ -937,7 +968,12 @@ run_suite() { fi done if [[ "$found" != true ]]; then - # Fall back to using the name as the scenario directly + # Fall back to using the name as the scenario directly. Note + # teardown rebuilds veth suffixes from CHAOS_SUITES only, so a + # scenario named here but absent from that list is outside the + # host-interface reap. Harmless while every ethernet-transport + # scenario is declared there; add one that isn't and its + # orphaned pairs will need an unscoped `--reap` to clear. run_chaos "$chaos_name" "$chaos_name" fi ;;