mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-05 19:18:25 +00:00
Johnathan Corgan
c5d62a6387
install.sh stopped fips before checking fips-dns. Both fips-dns and fips-gateway require fips.service, so stopping fips had already stopped them: the script saw fips-dns inactive and restarted only fips. .fips resolution stayed down and the gateway stayed stopped until started by hand. The script now records which of the three units were active before stopping any, and starts each of those again. A new tarball-install suite runs install.sh as an upgrade under real systemd, with stub binaries, in both local and GitHub CI.
368 lines
13 KiB
Bash
Executable File
368 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# Test the systemd tarball's install.sh as an upgrade, under real systemd.
|
|
#
|
|
# fips-dns.service and fips-gateway.service carry Requires=fips.service, so
|
|
# stopping fips stops both. An upgrade that checks which units are running
|
|
# only after stopping fips sees the other two as already stopped, and starts
|
|
# fips alone again. This suite installs the tarball once, puts the units in a
|
|
# known state, runs install.sh a second time (the upgrade), and checks that
|
|
# exactly the units that were running before are running after.
|
|
#
|
|
# The tarball is staged from the real packaging/systemd/install.sh and unit
|
|
# files, and the real packaging/common configuration, the same file set
|
|
# build-tarball.sh ships. The binaries and DNS helpers are stubs: the fips and
|
|
# fips-gateway stubs sleep, so their units stay active, and the DNS helpers
|
|
# exit 0. What is under test is install.sh against real units and a real
|
|
# systemd, which carries the Requires= stop propagation; nothing here depends
|
|
# on what the binaries do.
|
|
#
|
|
# Scenarios, each on a fresh container:
|
|
# all-active fips, fips-dns and fips-gateway running before the upgrade.
|
|
# All three must be running after it, and fips must have been
|
|
# restarted (its InvocationID changes).
|
|
# fips-only only fips running. fips must be running after, and the other
|
|
# two must still be stopped.
|
|
# none-active nothing running. Nothing may be running after.
|
|
#
|
|
# Usage: ./test.sh [--install-sh PATH] [scenario ...]
|
|
# No scenarios = run all of them.
|
|
# --install-sh PATH runs another install.sh in place of the tree's, so the
|
|
# suite can be pointed at an older script without editing the tree.
|
|
#
|
|
# Requirements: Docker able to grant SYS_ADMIN and NET_ADMIN and an unconfined
|
|
# AppArmor profile (the container is not privileged; see
|
|
# testing/lib/systemd-container.sh), and /dev/net/tun on the host.
|
|
#
|
|
# Exit 0 only when every check passed. A container that cannot boot or be
|
|
# inspected is a FAIL, never a skip.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck source=SCRIPTDIR/../lib/systemd-container.sh
|
|
source "$SCRIPT_DIR/../lib/systemd-container.sh"
|
|
# shellcheck source=SCRIPTDIR/../lib/image-build.sh
|
|
source "$SCRIPT_DIR/../lib/image-build.sh"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
IMAGE="fips-tarball-test:debian12"
|
|
BOOT_TIMEOUT=60
|
|
# Bounds one install.sh run. It starts at most three units, and the gateway's
|
|
# ExecStartPre returns at once because fips0 exists before the first start.
|
|
INSTALL_TIMEOUT=120
|
|
UNIT_START_TIMEOUT=60
|
|
TARBALL_DIR=/opt/fips-tarball
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
INSTALL_SH="$REPO_ROOT/packaging/systemd/install.sh"
|
|
STAGE=""
|
|
|
|
log() { echo "=== $*"; }
|
|
pass() { echo " PASS: $*"; PASS=$((PASS + 1)); }
|
|
fail() { echo " FAIL: $*"; FAIL=$((FAIL + 1)); }
|
|
|
|
# Remove a container, ignoring one that does not exist.
|
|
cleanup_container() {
|
|
local name="$1"
|
|
docker rm -f "$name" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Remove every scenario's container and the staged tarball.
|
|
cleanup_all() {
|
|
local s
|
|
for s in $ALL_SCENARIOS; do
|
|
cleanup_container "$(container_name "$s")"
|
|
done
|
|
[ -n "$STAGE" ] && rm -rf "$STAGE"
|
|
}
|
|
|
|
# The container name for a scenario, scoped to this CI run.
|
|
container_name() {
|
|
echo "fips-tarball-test-$1${FIPS_CI_NAME_SUFFIX:-}"
|
|
return 0
|
|
}
|
|
|
|
# Build the systemd image. The build context is an empty directory: the
|
|
# tarball is copied into the container after it starts, not baked in.
|
|
build_image() {
|
|
local ctx rc=0
|
|
ctx=$(mktemp -d)
|
|
retry_build "docker build -t $IMAGE" build_inline "$IMAGE" "$(cat <<'DOCKERFILE'
|
|
FROM debian:12
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
systemd iproute2 procps && \
|
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
CMD ["/lib/systemd/systemd"]
|
|
DOCKERFILE
|
|
)" "$ctx" || rc=$?
|
|
rm -rf "$ctx"
|
|
return "$rc"
|
|
}
|
|
|
|
# Stage a tarball directory: the real install script, units and configuration,
|
|
# with stub binaries and DNS helpers.
|
|
stage_tarball() {
|
|
local f
|
|
STAGE=$(mktemp -d)
|
|
cp "$INSTALL_SH" "$STAGE/install.sh" || return 1
|
|
for f in fips.service fips-dns.service fips-gateway.service fips-firewall.service; do
|
|
cp "$REPO_ROOT/packaging/systemd/$f" "$STAGE/" || return 1
|
|
done
|
|
for f in fips.yaml hosts fips.nft; do
|
|
cp "$REPO_ROOT/packaging/common/$f" "$STAGE/" || return 1
|
|
done
|
|
printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips"
|
|
printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips-gateway"
|
|
printf '#!/bin/sh\nexit 0\n' > "$STAGE/fipsctl"
|
|
printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-setup"
|
|
printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-teardown"
|
|
chmod 0755 "$STAGE"/install.sh "$STAGE"/fips "$STAGE"/fips-gateway \
|
|
"$STAGE"/fipsctl "$STAGE"/fips-dns-setup "$STAGE"/fips-dns-teardown
|
|
return 0
|
|
}
|
|
|
|
# Start the scenario's systemd container. Not privileged: see
|
|
# testing/lib/systemd-container.sh for the flags and why. The TUN device is
|
|
# passed so the scenario can create the fips0 that fips-gateway.service's
|
|
# ExecStartPre waits for.
|
|
start_container() {
|
|
local name="$1"
|
|
cleanup_container "$name"
|
|
run_quiet "docker run $name" \
|
|
docker run -d --name "$name" \
|
|
--label com.corganlabs.fips-ci=1 \
|
|
"${SYSTEMD_CAPS[@]}" \
|
|
--cgroupns=host \
|
|
--device /dev/net/tun \
|
|
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--tmpfs /run --tmpfs /run/lock \
|
|
"$IMAGE" || return
|
|
check_isolation "$name"
|
|
return
|
|
}
|
|
|
|
# Wait for systemd to finish booting. `degraded` counts as booted: units such
|
|
# as systemd-modules-load cannot succeed in a container, and the system still
|
|
# finished starting. The state string is tested directly rather than piped, so
|
|
# pipefail cannot turn a degraded boot into a failure.
|
|
wait_for_systemd() {
|
|
local name="$1" state
|
|
for _i in $(seq 1 "$BOOT_TIMEOUT"); do
|
|
state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true)
|
|
case "$state" in
|
|
running | degraded) return 0 ;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
echo " ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2
|
|
return 1
|
|
}
|
|
|
|
# Run the staged install.sh in the container, under a bound.
|
|
run_install() {
|
|
local name="$1" label="$2" out rc=0
|
|
out=$(timeout "$INSTALL_TIMEOUT" docker exec "$name" "$TARBALL_DIR/install.sh" 2>&1) || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
fail "$label: install.sh exited $rc"
|
|
echo "$out" | sed 's/^/ /'
|
|
return 1
|
|
fi
|
|
pass "$label: install.sh exits 0"
|
|
return 0
|
|
}
|
|
|
|
# Print a unit's active state ("unknown" when the container cannot be asked).
|
|
unit_state() {
|
|
local name="$1" unit="$2" state
|
|
state=$(docker exec "$name" systemctl is-active "$unit" 2>/dev/null)
|
|
echo "${state:-unknown}"
|
|
return 0
|
|
}
|
|
|
|
# Print a unit's InvocationID, which changes on every start.
|
|
invocation_id() {
|
|
docker exec "$1" systemctl show -p InvocationID --value "$2" 2>/dev/null
|
|
return
|
|
}
|
|
|
|
# Record a PASS when a unit is in the wanted state, a FAIL otherwise.
|
|
expect_state() {
|
|
local name="$1" label="$2" unit="$3" want="$4" got
|
|
got=$(unit_state "$name" "$unit")
|
|
if [ "$got" = "$want" ]; then
|
|
pass "$label: $unit is $got"
|
|
else
|
|
fail "$label: $unit is $got (want $want)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Start a unit under a bound, recording a FAIL when it does not start.
|
|
start_unit() {
|
|
local name="$1" unit="$2" out
|
|
if ! out=$(timeout "$UNIT_START_TIMEOUT" docker exec "$name" systemctl start "$unit" 2>&1); then
|
|
fail "setup: could not start $unit: ${out:-no output}"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Boot a fresh container, copy the tarball in and run the first install.
|
|
# Returns 1, having recorded a FAIL, when any of that does not happen.
|
|
prepare() {
|
|
local name="$1" label="$2"
|
|
start_container "$name" || { fail "$label: container did not start"; return 1; }
|
|
wait_for_systemd "$name" || {
|
|
fail "$label: systemd did not boot; the checks would read an unstarted system"
|
|
return 1
|
|
}
|
|
if ! docker cp "$STAGE/." "$name:$TARBALL_DIR" >/dev/null; then
|
|
fail "$label: could not copy the tarball into $name"
|
|
return 1
|
|
fi
|
|
if ! docker exec "$name" ip tuntap add fips0 mode tun >/dev/null 2>&1; then
|
|
fail "$label: could not create fips0 in $name"
|
|
return 1
|
|
fi
|
|
run_install "$name" "$label: first install" || return 1
|
|
return 0
|
|
}
|
|
|
|
scenario_all_active() {
|
|
local label="all-active" name before after
|
|
name=$(container_name "$label")
|
|
log "$label: fips, fips-dns and fips-gateway running before the upgrade"
|
|
if prepare "$name" "$label" \
|
|
&& start_unit "$name" fips.service \
|
|
&& start_unit "$name" fips-dns.service \
|
|
&& start_unit "$name" fips-gateway.service; then
|
|
expect_state "$name" "$label: before" fips.service active
|
|
expect_state "$name" "$label: before" fips-dns.service active
|
|
expect_state "$name" "$label: before" fips-gateway.service active
|
|
before=$(invocation_id "$name" fips.service)
|
|
if run_install "$name" "$label: upgrade"; then
|
|
expect_state "$name" "$label: after" fips.service active
|
|
expect_state "$name" "$label: after" fips-dns.service active
|
|
expect_state "$name" "$label: after" fips-gateway.service active
|
|
after=$(invocation_id "$name" fips.service)
|
|
if [ -z "$before" ] || [ -z "$after" ]; then
|
|
fail "$label: could not read fips.service's InvocationID (before '${before}', after '${after}')"
|
|
elif [ "$before" != "$after" ]; then
|
|
pass "$label: the upgrade restarted fips.service"
|
|
else
|
|
fail "$label: fips.service kept InvocationID $before, so the upgrade did not stop and restart it"
|
|
fi
|
|
fi
|
|
fi
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
scenario_fips_only() {
|
|
local label="fips-only" name
|
|
name=$(container_name "$label")
|
|
log "$label: only fips running before the upgrade"
|
|
if prepare "$name" "$label" && start_unit "$name" fips.service; then
|
|
expect_state "$name" "$label: before" fips.service active
|
|
expect_state "$name" "$label: before" fips-dns.service inactive
|
|
expect_state "$name" "$label: before" fips-gateway.service inactive
|
|
if run_install "$name" "$label: upgrade"; then
|
|
expect_state "$name" "$label: after" fips.service active
|
|
expect_state "$name" "$label: after" fips-dns.service inactive
|
|
expect_state "$name" "$label: after" fips-gateway.service inactive
|
|
fi
|
|
fi
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
scenario_none_active() {
|
|
local label="none-active" name
|
|
name=$(container_name "$label")
|
|
log "$label: nothing running before the upgrade"
|
|
if prepare "$name" "$label"; then
|
|
expect_state "$name" "$label: before" fips.service inactive
|
|
if run_install "$name" "$label: upgrade"; then
|
|
expect_state "$name" "$label: after" fips.service inactive
|
|
expect_state "$name" "$label: after" fips-dns.service inactive
|
|
expect_state "$name" "$label: after" fips-gateway.service inactive
|
|
fi
|
|
fi
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
ALL_SCENARIOS="all-active fips-only none-active"
|
|
|
|
_args=()
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--install-sh)
|
|
INSTALL_SH="${2:?--install-sh requires a path}"
|
|
shift 2
|
|
;;
|
|
-h|--help)
|
|
echo "usage: test.sh [--install-sh PATH] [scenario ...]"
|
|
echo "scenarios: $ALL_SCENARIOS"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
echo "Unknown option: $1" >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
_args+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
set -- ${_args[@]+"${_args[@]}"}
|
|
|
|
if [ $# -eq 0 ]; then
|
|
scenarios="$ALL_SCENARIOS"
|
|
else
|
|
scenarios="$*"
|
|
fi
|
|
|
|
for scenario in $scenarios; do
|
|
case " $ALL_SCENARIOS " in
|
|
*" $scenario "*) ;;
|
|
*)
|
|
echo "Unknown scenario: $scenario (available: $ALL_SCENARIOS)" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ ! -f "$INSTALL_SH" ]; then
|
|
echo "install script not found: $INSTALL_SH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
trap cleanup_all EXIT
|
|
|
|
log "Building $IMAGE"
|
|
if ! build_image; then
|
|
fail "image build failed"
|
|
elif ! stage_tarball; then
|
|
fail "could not stage the tarball"
|
|
else
|
|
for scenario in $scenarios; do
|
|
case "$scenario" in
|
|
all-active) scenario_all_active ;;
|
|
fips-only) scenario_fips_only ;;
|
|
none-active) scenario_none_active ;;
|
|
esac
|
|
echo
|
|
done
|
|
fi
|
|
|
|
echo "═══════════════════════════════════════"
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
echo "═══════════════════════════════════════"
|
|
|
|
[ "$FAIL" -eq 0 ] && [ "$PASS" -gt 0 ]
|