Close the remaining "caught" and "produces red" harness holes

Four sites where a check could report a verdict it had not established, or
detect a failure and then not turn it red.

assert_no_panic in both NAT suites read `docker logs ... || true`, so a
container that could not be read produced empty output, matched no panic
pattern, and returned success. The assertion's failure mode was
indistinguishable from its success condition. It now reports that absence of
panics is not established.

deb-install's apt capture discarded the exit status, so a failed docker exec
gave an empty capture that matched neither error pattern and reached the pass
branch. The status is now kept and checked before the output is inspected, with
the output still printed on failure.

wait_for_systemd printed a warning and returned success on timeout, so every
check after it read a system that may not have started its units. It now returns
non-zero and the caller abandons that distro leg rather than testing an
unstarted system.

interop's copy of count_log_pattern carried the same defect fixed in rekey: a
node whose logs could not be read contributed zero to eight expect-zero
assertions. Fixed the same way, and its consumer now reports the unreadable case
rather than comparing a sentinel against zero.

Each validated by breaking what it guards and by confirming the healthy path is
unchanged: an absent container now fails each check where it previously passed,
a readable panic-free container still passes, a real panic is still caught, and
a genuinely successful install still passes.
This commit is contained in:
Johnathan Corgan
2026-07-23 15:36:59 +00:00
parent c8a0ac5fca
commit 34a2561af7
4 changed files with 58 additions and 10 deletions
+22 -5
View File
@@ -82,8 +82,11 @@ wait_for_systemd() {
fi
sleep 1
done
echo " WARNING: systemd did not reach running state in ${BOOT_TIMEOUT}s (may still work)"
return 0
# A boot that never reached `running` or `degraded` is not a warning: every
# check after this point reads a system that may not have started its units,
# and returning 0 here made the timeout indistinguishable from a clean boot.
echo " ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2
return 1
}
wait_for_service_active() {
@@ -229,18 +232,32 @@ DOCKERFILE
rm -f "$CACHE_DIR/deb-for-image"
start_systemd_container_with_tun "$name" "$image"
wait_for_systemd "$name"
wait_for_systemd "$name" || {
fail "systemd did not boot in $name; the install checks below would read an unstarted system"
cleanup_container "$name"
return
}
# Install the .deb. apt handles dependencies (libc6, systemd,
# libdbus-1-3) and runs the maintainer scripts (postinst →
# systemctl enable fips.service; fips-dns.service starts and
# runs fips-dns-setup).
log "Installing .deb (apt install /opt/fips-deb/${deb_basename})"
local install_output
# `|| true` here used to discard apt's exit status, and an empty capture (a
# failed `docker exec`) matches neither error pattern below, so a install
# that never ran reached `pass "apt install completed"`. Keep the capture on
# failure so the diagnostics below can print it, but remember the status.
local install_output install_rc=0
install_output=$(docker exec "$name" bash -c "
apt-get update >/dev/null 2>&1
cd /opt/fips-deb && apt-get install -y --no-install-recommends ./${deb_basename} 2>&1
") || true
") || install_rc=$?
if [ "$install_rc" -ne 0 ]; then
fail "apt install exited $install_rc"
echo "$install_output" | tail -20
cleanup_container "$name"
return
fi
if echo "$install_output" | grep -qE "^E:|errors? were encountered"; then
fail "apt install reported errors"
echo "$install_output" | tail -20
+19 -3
View File
@@ -476,13 +476,24 @@ phase_result() {
}
# Count a pattern across all node logs.
#
# A node whose logs cannot be read makes the whole count unusable rather than
# contributing 0. The previous form ended each read `| grep -cE … || true`, so a
# failed `docker logs` yielded 0 for that node and the eight expect-zero
# assertions in the GLOBAL_PATTERNS loop read a clean result from a node that was
# never consulted. Same defect and same fix as rekey-test.sh.
count_log_pattern() {
local pattern="$1" total=0 n count
local pattern="$1" total=0 n logs count
for n in "${NODES[@]}"; do
count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true)
if ! logs=$(docker logs "${CONTAINER[$n]}" 2>&1); then
echo "unreadable:${CONTAINER[$n]}"
return 1
fi
count=$(grep -cE "$pattern" <<<"$logs" || true)
total=$((total + count))
done
echo "$total"
return 0
}
# Per-node count of a pattern.
@@ -854,7 +865,12 @@ declare -A GLOBAL_PATTERNS=(
for pat in "${!GLOBAL_PATTERNS[@]}"; do
desc="${GLOBAL_PATTERNS[$pat]}"
total="$(count_log_pattern "$pat")"
if ! total="$(count_log_pattern "$pat")"; then
echo " FAIL $desc: node logs unreadable ($total), zero not established"
FAILED=$((FAILED + 1))
INTEROP_FAILURES+=("[log] $desc: node logs unreadable ($total)")
continue
fi
if [ "$total" -eq 0 ]; then
echo " PASS $desc: 0"
PASSED=$((PASSED + 1))
+8 -1
View File
@@ -291,14 +291,21 @@ assert_process_alive() {
return 0
}
# A container whose logs cannot be read has not been shown to be panic-free.
# See the companion note in stun-faults-test.sh: the previous `|| true` made
# this assertion's failure mode indistinguishable from its success condition.
assert_no_panic() {
local container="$1"
local logs
logs="$(docker logs "$container" 2>&1 || true)"
if ! logs="$(docker logs "$container" 2>&1)"; then
echo "could not read logs from $container; absence of panics is not established" >&2
return 1
fi
if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then
echo "panic detected in $container logs" >&2
return 1
fi
return 0
}
run_test() {
+9 -1
View File
@@ -143,13 +143,21 @@ assert_process_alive() {
return 0
}
# A container whose logs cannot be read has not been shown to be panic-free.
# The previous form read `docker logs … || true`, so an unreadable container
# yielded empty output, matched no panic pattern, and returned success — the
# assertion's failure mode was indistinguishable from its success condition.
assert_no_panic() {
local logs
logs="$(docker logs "$NODE" 2>&1 || true)"
if ! logs="$(docker logs "$NODE" 2>&1)"; then
echo "could not read logs from $NODE; absence of panics is not established" >&2
return 1
fi
if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then
echo "panic detected in $NODE logs" >&2
return 1
fi
return 0
}
# Look for STUN-related fault evidence in the daemon's logs. The