diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b60ae603..f936aead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -474,6 +474,36 @@ jobs: timeout-minutes: 5 run: bash testing/openwrt/maintainer-scripts-test.sh +# ───────────────────────────────────────────────────────────────────────────── +# Job 2f – systemd tarball upgrade scenarios +# +# Runs the tarball's install.sh twice in a systemd container, the second time +# as an upgrade, with fips, fips-dns and fips-gateway in a known state, and +# checks that the units running before the upgrade are running after it. The +# binaries are stubs, so like the OpenWrt job it needs no FIPS build and no +# shared test image, and has a job of its own. +# +# The leg keeps `suite:` because testing/check-ci-parity.sh matches it against +# TARBALL_INSTALL_SUITES in ci-local.sh; the step below does not read it. +# ───────────────────────────────────────────────────────────────────────────── + tarball-install: + name: systemd tarball (${{ matrix.suite }}) + runs-on: ubuntu-latest + if: ${{ !inputs.skip_integration }} + + strategy: + fail-fast: false + matrix: + include: + - suite: tarball-install + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Run the systemd tarball upgrade scenarios + timeout-minutes: 10 + run: bash testing/tarball-install/test.sh + # ───────────────────────────────────────────────────────────────────────────── # Job 3 – Integration tests (static mesh + chaos simulation) # diff --git a/packaging/systemd/install.sh b/packaging/systemd/install.sh index f71513b7..89e78111 100755 --- a/packaging/systemd/install.sh +++ b/packaging/systemd/install.sh @@ -112,19 +112,20 @@ fi # --- Install systemd units --- -was_active=false -if systemctl is-active --quiet fips.service 2>/dev/null; then - was_active=true - echo "Stopping running fips service..." - systemctl stop fips.service -fi - -dns_was_active=false -if systemctl is-active --quiet fips-dns.service 2>/dev/null; then - dns_was_active=true - echo "Stopping running fips-dns service..." - systemctl stop fips-dns.service -fi +# fips-dns and fips-gateway carry Requires=fips.service, so stopping fips +# stops them too. Record all three before stopping any of them. +declare -A was_active=() +for unit in fips fips-dns fips-gateway; do + if systemctl is-active --quiet "${unit}.service" 2>/dev/null; then + was_active[$unit]=true + fi +done +for unit in fips-gateway fips-dns fips; do + if [ "${was_active[$unit]:-}" = true ]; then + echo "Stopping running ${unit} service..." + systemctl stop "${unit}.service" + fi +done install -m 0644 "${SCRIPT_DIR}/fips.service" "${SYSTEMD_DIR}/fips.service" install -m 0644 "${SCRIPT_DIR}/fips-dns.service" "${SYSTEMD_DIR}/fips-dns.service" @@ -164,15 +165,13 @@ systemctl enable fips.service systemctl enable fips-dns.service echo "Services enabled (will start on boot)." -# Restart if they were running before -if $was_active; then - echo "Restarting fips service..." - systemctl start fips.service -fi -if $dns_was_active; then - echo "Restarting fips-dns service..." - systemctl start fips-dns.service -fi +# Restart each unit that was running before +for unit in fips fips-dns fips-gateway; do + if [ "${was_active[$unit]:-}" = true ]; then + echo "Restarting ${unit} service..." + systemctl start "${unit}.service" + fi +done echo "" echo "=== Installation complete ===" diff --git a/testing/README.md b/testing/README.md index 99772520..13d176ca 100644 --- a/testing/README.md +++ b/testing/README.md @@ -101,6 +101,14 @@ what the real `build-apk.sh` and `build-ipk.sh` package. No router, opkg or apk-tools is involved. Part of both CI runners as `openwrt-scripts`. +### [tarball-install/](tarball-install/) -- systemd Tarball Upgrade + +Runs the systemd tarball's `install.sh` twice in a Debian 12 systemd +container, the second time as an upgrade, with stub binaries. Checks +that the units running before the upgrade (fips, fips-dns and +fips-gateway, in three combinations) are the ones running after it. +Part of both CI runners as `tarball-install`. + ### [native-api/](native-api/) -- Native Datagram API Checks the experimental native datagram API: a client process opens a diff --git a/testing/ci-local.sh b/testing/ci-local.sh index cdaa413d..7ff15dc8 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -208,6 +208,7 @@ CHAOS_SUITES=( # testing/chaos/scripts/chaos.sh bloom-storm. GATEWAY_SUITES=(gateway) OPENWRT_SUITES=(openwrt-scripts) +TARBALL_INSTALL_SUITES=(tarball-install) SIDECAR_SUITES=(sidecar) FIREWALL_SUITES=(firewall) NAT_SUITES=(cone symmetric lan) @@ -248,6 +249,9 @@ list_suites() { echo " OpenWrt packaging:" for s in "${OPENWRT_SUITES[@]}"; do echo " $s"; done echo "" + echo " systemd tarball:" + for s in "${TARBALL_INSTALL_SUITES[@]}"; do echo " $s"; done + echo "" echo " Firewall baseline:" for s in "${FIREWALL_SUITES[@]}"; do echo " $s"; done echo "" @@ -1207,6 +1211,15 @@ run_integration() { return fi + # Also ahead of the build context, for the same reason: the tarball + # install scenarios use stub binaries and their own systemd image. + if [[ -z "$ONLY_SUITE" ]]; then + run_tarball_install + elif [[ "$ONLY_SUITE" == "tarball-install" ]]; then + run_tarball_install + return + fi + # Populate THIS run's build context, then install the binaries into it. # Everything but the binaries is copied from the tracked context directory; # the binaries are installed fresh, and a previous run's are deliberately @@ -1370,6 +1383,8 @@ run_suite() { run_gateway ;; openwrt-scripts) run_openwrt_scripts ;; + tarball-install) + run_tarball_install ;; firewall) run_firewall ;; nat-cone|nat-symmetric|nat-lan) @@ -1466,6 +1481,18 @@ run_openwrt_scripts() { return $rc } +# The systemd tarball's install.sh, run as an upgrade under real systemd with +# stub binaries: the units that were running before the upgrade must be the +# ones running after it, including fips-dns and fips-gateway, which stop with +# fips through Requires=. +run_tarball_install() { + local rc=0 + info "[tarball-install] Running the systemd tarball upgrade scenarios" + bash "$SCRIPT_DIR/tarball-install/test.sh" || rc=$? + record "tarball-install" $rc + return $rc +} + run_ci_parity() { local rc=0 info "[ci-parity] Comparing the local suite set against the GitHub matrix" diff --git a/testing/tarball-install/test.sh b/testing/tarball-install/test.sh new file mode 100755 index 00000000..e3474095 --- /dev/null +++ b/testing/tarball-install/test.sh @@ -0,0 +1,367 @@ +#!/bin/bash +# Test the systemd tarball's install.sh as an upgrade, under real systemd. +# +# fips-dns.service and fips-gateway.service carry Requires=fips.service, so +# stopping fips stops both. An upgrade that checks which units are running +# only after stopping fips sees the other two as already stopped, and starts +# fips alone again. This suite installs the tarball once, puts the units in a +# known state, runs install.sh a second time (the upgrade), and checks that +# exactly the units that were running before are running after. +# +# The tarball is staged from the real packaging/systemd/install.sh and unit +# files, and the real packaging/common configuration, the same file set +# build-tarball.sh ships. The binaries and DNS helpers are stubs: the fips and +# fips-gateway stubs sleep, so their units stay active, and the DNS helpers +# exit 0. What is under test is install.sh against real units and a real +# systemd, which carries the Requires= stop propagation; nothing here depends +# on what the binaries do. +# +# Scenarios, each on a fresh container: +# all-active fips, fips-dns and fips-gateway running before the upgrade. +# All three must be running after it, and fips must have been +# restarted (its InvocationID changes). +# fips-only only fips running. fips must be running after, and the other +# two must still be stopped. +# none-active nothing running. Nothing may be running after. +# +# Usage: ./test.sh [--install-sh PATH] [scenario ...] +# No scenarios = run all of them. +# --install-sh PATH runs another install.sh in place of the tree's, so the +# suite can be pointed at an older script without editing the tree. +# +# Requirements: Docker able to grant SYS_ADMIN and NET_ADMIN and an unconfined +# AppArmor profile (the container is not privileged; see +# testing/lib/systemd-container.sh), and /dev/net/tun on the host. +# +# Exit 0 only when every check passed. A container that cannot boot or be +# inspected is a FAIL, never a skip. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=SCRIPTDIR/../lib/systemd-container.sh +source "$SCRIPT_DIR/../lib/systemd-container.sh" +# shellcheck source=SCRIPTDIR/../lib/image-build.sh +source "$SCRIPT_DIR/../lib/image-build.sh" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +IMAGE="fips-tarball-test:debian12" +BOOT_TIMEOUT=60 +# Bounds one install.sh run. It starts at most three units, and the gateway's +# ExecStartPre returns at once because fips0 exists before the first start. +INSTALL_TIMEOUT=120 +UNIT_START_TIMEOUT=60 +TARBALL_DIR=/opt/fips-tarball + +PASS=0 +FAIL=0 + +INSTALL_SH="$REPO_ROOT/packaging/systemd/install.sh" +STAGE="" + +log() { echo "=== $*"; } +pass() { echo " PASS: $*"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $*"; FAIL=$((FAIL + 1)); } + +# Remove a container, ignoring one that does not exist. +cleanup_container() { + local name="$1" + docker rm -f "$name" >/dev/null 2>&1 || true +} + +# Remove every scenario's container and the staged tarball. +cleanup_all() { + local s + for s in $ALL_SCENARIOS; do + cleanup_container "$(container_name "$s")" + done + [ -n "$STAGE" ] && rm -rf "$STAGE" +} + +# The container name for a scenario, scoped to this CI run. +container_name() { + echo "fips-tarball-test-$1${FIPS_CI_NAME_SUFFIX:-}" + return 0 +} + +# Build the systemd image. The build context is an empty directory: the +# tarball is copied into the container after it starts, not baked in. +build_image() { + local ctx rc=0 + ctx=$(mktemp -d) + retry_build "docker build -t $IMAGE" build_inline "$IMAGE" "$(cat <<'DOCKERFILE' +FROM debian:12 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + systemd iproute2 procps && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +CMD ["/lib/systemd/systemd"] +DOCKERFILE + )" "$ctx" || rc=$? + rm -rf "$ctx" + return "$rc" +} + +# Stage a tarball directory: the real install script, units and configuration, +# with stub binaries and DNS helpers. +stage_tarball() { + local f + STAGE=$(mktemp -d) + cp "$INSTALL_SH" "$STAGE/install.sh" || return 1 + for f in fips.service fips-dns.service fips-gateway.service fips-firewall.service; do + cp "$REPO_ROOT/packaging/systemd/$f" "$STAGE/" || return 1 + done + for f in fips.yaml hosts fips.nft; do + cp "$REPO_ROOT/packaging/common/$f" "$STAGE/" || return 1 + done + printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips" + printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips-gateway" + printf '#!/bin/sh\nexit 0\n' > "$STAGE/fipsctl" + printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-setup" + printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-teardown" + chmod 0755 "$STAGE"/install.sh "$STAGE"/fips "$STAGE"/fips-gateway \ + "$STAGE"/fipsctl "$STAGE"/fips-dns-setup "$STAGE"/fips-dns-teardown + return 0 +} + +# Start the scenario's systemd container. Not privileged: see +# testing/lib/systemd-container.sh for the flags and why. The TUN device is +# passed so the scenario can create the fips0 that fips-gateway.service's +# ExecStartPre waits for. +start_container() { + local name="$1" + cleanup_container "$name" + run_quiet "docker run $name" \ + docker run -d --name "$name" \ + --label com.corganlabs.fips-ci=1 \ + "${SYSTEMD_CAPS[@]}" \ + --cgroupns=host \ + --device /dev/net/tun \ + -v /sys/fs/cgroup:/sys/fs/cgroup:rw \ + --tmpfs /run --tmpfs /run/lock \ + "$IMAGE" || return + check_isolation "$name" + return +} + +# Wait for systemd to finish booting. `degraded` counts as booted: units such +# as systemd-modules-load cannot succeed in a container, and the system still +# finished starting. The state string is tested directly rather than piped, so +# pipefail cannot turn a degraded boot into a failure. +wait_for_systemd() { + local name="$1" state + for _i in $(seq 1 "$BOOT_TIMEOUT"); do + state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true) + case "$state" in + running | degraded) return 0 ;; + esac + sleep 1 + done + echo " ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2 + return 1 +} + +# Run the staged install.sh in the container, under a bound. +run_install() { + local name="$1" label="$2" out rc=0 + out=$(timeout "$INSTALL_TIMEOUT" docker exec "$name" "$TARBALL_DIR/install.sh" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + fail "$label: install.sh exited $rc" + echo "$out" | sed 's/^/ /' + return 1 + fi + pass "$label: install.sh exits 0" + return 0 +} + +# Print a unit's active state ("unknown" when the container cannot be asked). +unit_state() { + local name="$1" unit="$2" state + state=$(docker exec "$name" systemctl is-active "$unit" 2>/dev/null) + echo "${state:-unknown}" + return 0 +} + +# Print a unit's InvocationID, which changes on every start. +invocation_id() { + docker exec "$1" systemctl show -p InvocationID --value "$2" 2>/dev/null + return +} + +# Record a PASS when a unit is in the wanted state, a FAIL otherwise. +expect_state() { + local name="$1" label="$2" unit="$3" want="$4" got + got=$(unit_state "$name" "$unit") + if [ "$got" = "$want" ]; then + pass "$label: $unit is $got" + else + fail "$label: $unit is $got (want $want)" + fi + return 0 +} + +# Start a unit under a bound, recording a FAIL when it does not start. +start_unit() { + local name="$1" unit="$2" out + if ! out=$(timeout "$UNIT_START_TIMEOUT" docker exec "$name" systemctl start "$unit" 2>&1); then + fail "setup: could not start $unit: ${out:-no output}" + return 1 + fi + return 0 +} + +# Boot a fresh container, copy the tarball in and run the first install. +# Returns 1, having recorded a FAIL, when any of that does not happen. +prepare() { + local name="$1" label="$2" + start_container "$name" || { fail "$label: container did not start"; return 1; } + wait_for_systemd "$name" || { + fail "$label: systemd did not boot; the checks would read an unstarted system" + return 1 + } + if ! docker cp "$STAGE/." "$name:$TARBALL_DIR" >/dev/null; then + fail "$label: could not copy the tarball into $name" + return 1 + fi + if ! docker exec "$name" ip tuntap add fips0 mode tun >/dev/null 2>&1; then + fail "$label: could not create fips0 in $name" + return 1 + fi + run_install "$name" "$label: first install" || return 1 + return 0 +} + +scenario_all_active() { + local label="all-active" name before after + name=$(container_name "$label") + log "$label: fips, fips-dns and fips-gateway running before the upgrade" + if prepare "$name" "$label" \ + && start_unit "$name" fips.service \ + && start_unit "$name" fips-dns.service \ + && start_unit "$name" fips-gateway.service; then + expect_state "$name" "$label: before" fips.service active + expect_state "$name" "$label: before" fips-dns.service active + expect_state "$name" "$label: before" fips-gateway.service active + before=$(invocation_id "$name" fips.service) + if run_install "$name" "$label: upgrade"; then + expect_state "$name" "$label: after" fips.service active + expect_state "$name" "$label: after" fips-dns.service active + expect_state "$name" "$label: after" fips-gateway.service active + after=$(invocation_id "$name" fips.service) + if [ -z "$before" ] || [ -z "$after" ]; then + fail "$label: could not read fips.service's InvocationID (before '${before}', after '${after}')" + elif [ "$before" != "$after" ]; then + pass "$label: the upgrade restarted fips.service" + else + fail "$label: fips.service kept InvocationID $before, so the upgrade did not stop and restart it" + fi + fi + fi + cleanup_container "$name" + return 0 +} + +scenario_fips_only() { + local label="fips-only" name + name=$(container_name "$label") + log "$label: only fips running before the upgrade" + if prepare "$name" "$label" && start_unit "$name" fips.service; then + expect_state "$name" "$label: before" fips.service active + expect_state "$name" "$label: before" fips-dns.service inactive + expect_state "$name" "$label: before" fips-gateway.service inactive + if run_install "$name" "$label: upgrade"; then + expect_state "$name" "$label: after" fips.service active + expect_state "$name" "$label: after" fips-dns.service inactive + expect_state "$name" "$label: after" fips-gateway.service inactive + fi + fi + cleanup_container "$name" + return 0 +} + +scenario_none_active() { + local label="none-active" name + name=$(container_name "$label") + log "$label: nothing running before the upgrade" + if prepare "$name" "$label"; then + expect_state "$name" "$label: before" fips.service inactive + if run_install "$name" "$label: upgrade"; then + expect_state "$name" "$label: after" fips.service inactive + expect_state "$name" "$label: after" fips-dns.service inactive + expect_state "$name" "$label: after" fips-gateway.service inactive + fi + fi + cleanup_container "$name" + return 0 +} + +ALL_SCENARIOS="all-active fips-only none-active" + +_args=() +while [ $# -gt 0 ]; do + case "$1" in + --install-sh) + INSTALL_SH="${2:?--install-sh requires a path}" + shift 2 + ;; + -h|--help) + echo "usage: test.sh [--install-sh PATH] [scenario ...]" + echo "scenarios: $ALL_SCENARIOS" + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + exit 1 + ;; + *) + _args+=("$1") + shift + ;; + esac +done +set -- ${_args[@]+"${_args[@]}"} + +if [ $# -eq 0 ]; then + scenarios="$ALL_SCENARIOS" +else + scenarios="$*" +fi + +for scenario in $scenarios; do + case " $ALL_SCENARIOS " in + *" $scenario "*) ;; + *) + echo "Unknown scenario: $scenario (available: $ALL_SCENARIOS)" >&2 + exit 1 + ;; + esac +done + +if [ ! -f "$INSTALL_SH" ]; then + echo "install script not found: $INSTALL_SH" >&2 + exit 1 +fi + +trap cleanup_all EXIT + +log "Building $IMAGE" +if ! build_image; then + fail "image build failed" +elif ! stage_tarball; then + fail "could not stage the tarball" +else + for scenario in $scenarios; do + case "$scenario" in + all-active) scenario_all_active ;; + fips-only) scenario_fips_only ;; + none-active) scenario_none_active ;; + esac + echo + done +fi + +echo "═══════════════════════════════════════" +echo "Results: $PASS passed, $FAIL failed" +echo "═══════════════════════════════════════" + +[ "$FAIL" -eq 0 ] && [ "$PASS" -gt 0 ]