diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 7a4ad345..d6923301 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -999,13 +999,25 @@ run_dns_resolver() { } # Run deb-install harness (multi-distro real-package install) +# +# Bounded because this worker is what gates artifact publication: a suite that +# hangs stops every branch publishing, which is worse than a red. The harness +# bounds its own service starts now, so this is the backstop for anything else +# that wedges -- a docker daemon that stops answering, a container that never +# boots. 40 minutes is well above the observed cold-cache cost of a full +# five-distro run and is not a performance budget. +DEB_INSTALL_TIMEOUT=${DEB_INSTALL_TIMEOUT:-2400} run_deb_install() { info "[deb-install] Running multi-distro test (slow — builds .deb + per-distro install)" - if bash testing/deb-install/test.sh 2>&1; then - record "deb-install" 0 - else - record "deb-install" 1 + local rc=0 + timeout "$DEB_INSTALL_TIMEOUT" bash testing/deb-install/test.sh 2>&1 || rc=$? + if [[ $rc -eq 124 ]]; then + # Say so explicitly. A bare red here reads as a failed assertion, and + # the difference matters: a timeout means no assertion was reached. + echo " ERROR: deb-install exceeded ${DEB_INSTALL_TIMEOUT}s and was killed;" >&2 + echo " no verdict was reached, so this is not an assertion failure." >&2 fi + record "deb-install" "$rc" } # Run Tor SOCKS5 outbound test (live Tor network) diff --git a/testing/deb-install/test.sh b/testing/deb-install/test.sh index 0b542744..81f4de65 100755 --- a/testing/deb-install/test.sh +++ b/testing/deb-install/test.sh @@ -36,6 +36,16 @@ DEB_CACHE_DIR="$CACHE_DIR/deb" BOOT_TIMEOUT=30 SERVICE_TIMEOUT=20 DAEMON_TIMEOUT=15 +# Bounds on a single `systemctl start`, which is not a wait loop and needs its +# own limit. See start_unit() for why an unbounded one can never return. +UNIT_START_TIMEOUT=30 +# fips-gateway.service's ExecStartPre waits up to 30s for fips0 to appear, by +# design, so its start legitimately takes longer than any other. +GATEWAY_START_TIMEOUT=60 +# The gateway-enable block restarts fips.service inside the container. Above +# systemd's default TimeoutStopSec of 90s, so a wedged stop trips this rather +# than this cutting a healthy stop short. +CONFIG_RESTART_TIMEOUT=120 PASS=0 FAIL=0 @@ -97,6 +107,41 @@ wait_for_systemd() { return 1 } +# Start a unit without waiting for its start job to finish. +# +# Start a unit and wait for its start job, under a bound. +# +# Blocking is the right default and the call returning is what synchronises the +# checks after it: `fips-gateway.service` in particular has an ExecStartPre that +# waits up to 30s for fips0, so a caller that does not wait races it. What the +# old code lacked was the bound, not the wait. +start_unit() { + local name="$1" unit="$2" limit="${3:-$UNIT_START_TIMEOUT}" + timeout "$limit" docker exec "$name" systemctl start "$unit" 2>&1 +} + +# Queue a unit's start job and return without waiting for it. +# +# For `fips-dns.service` only, and the reason is specific rather than general. +# It is Type=oneshot with Requires=fips.service, so its start job waits on a +# dependency that a broken daemon never satisfies: fips.service restarts every +# 5s for ever and the oneshot's job is never dispatched. `systemctl start` then +# never returns. That is the whole class of fault this suite exists to find, and +# the suite answered it by hanging -- no FAIL, no Results line, no exit status, +# observed at 21 minutes against a package whose binaries could not load. +# +# Queueing moves the verdict onto the wait_for_service_active call that follows, +# which carries a timeout and dumps the journal when it fails. RemainAfterExit=yes +# on that unit makes `is-active` a correct readiness test for a oneshot. +# +# This bounds these call sites, not every `docker exec` in the file. The backstop +# for the rest is the caller's own limit: ci-local.sh bounds the whole suite, and +# the GitHub leg carries timeout-minutes. +start_unit_queued() { + local name="$1" unit="$2" + timeout "$UNIT_START_TIMEOUT" docker exec "$name" systemctl start --no-block "$unit" 2>&1 +} + wait_for_service_active() { local name="$1" service="$2" timeout="${3:-$SERVICE_TIMEOUT}" for _i in $(seq 1 "$timeout"); do @@ -349,8 +394,8 @@ DOCKERFILE # Start the services as a simulated boot. (On a real system, # they'd come up on next reboot.) - docker exec "$name" systemctl start fips.service 2>&1 || true - docker exec "$name" systemctl start fips-dns.service 2>&1 || true + start_unit "$name" fips.service || true + start_unit_queued "$name" fips-dns.service || true if wait_for_service_active "$name" fips.service; then pass "fips.service active after explicit start" @@ -381,10 +426,21 @@ DOCKERFILE return fi - # Wait for fips-dns.service. This should have run fips-dns-setup - # which configures the resolver backend and writes - # /run/fips/dns-backend. - sleep 2 + # Wait for fips-dns.service to finish. Its start job is queued rather than + # waited on above, so this is what makes /run/fips/dns-backend safe to read: + # fips-dns-setup waits up to 30s for fips0 and restarts systemd-resolved + # before it writes that file, so reading it on a timer races the setup. + # RemainAfterExit=yes makes is-active correct for this oneshot. + if wait_for_service_active "$name" fips-dns.service; then + pass "fips-dns.service completed" + else + fail "fips-dns.service did not complete in ${SERVICE_TIMEOUT}s" + echo " --- fips-dns.service journal ---" + docker exec "$name" journalctl -u fips-dns.service --no-pager 2>&1 | tail -20 + cleanup_container "$name" + return + fi + local backend backend=$(docker exec "$name" cat /run/fips/dns-backend 2>/dev/null || echo "(missing)") local ver @@ -441,7 +497,7 @@ DOCKERFILE # default preset) and ipv6 forwarding (gateway checks before # the DNS upstream check). docker exec "$name" sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null 2>&1 || true - docker exec "$name" bash -c ' + timeout "$CONFIG_RESTART_TIMEOUT" docker exec "$name" bash -c ' systemctl unmask fips-gateway.service 2>/dev/null # Patch in a minimal gateway config since the shipped fips.yaml # has gateway disabled by default. @@ -482,7 +538,7 @@ EOF fail "non-root fips group member cannot reach control socket after restart" fi - docker exec "$name" systemctl start fips-gateway.service >/dev/null 2>&1 || true + start_unit "$name" fips-gateway.service "$GATEWAY_START_TIMEOUT" >/dev/null 2>&1 || true sleep 3 if docker exec "$name" journalctl -u fips-gateway.service --no-pager 2>/dev/null \ | grep -q "DNS upstream is reachable"; then