Restart fips-dns and fips-gateway after a tarball upgrade

install.sh stopped fips before checking fips-dns. Both fips-dns and
fips-gateway require fips.service, so stopping fips had already stopped
them: the script saw fips-dns inactive and restarted only fips. .fips
resolution stayed down and the gateway stayed stopped until started by
hand. The script now records which of the three units were active
before stopping any, and starts each of those again.

A new tarball-install suite runs install.sh as an upgrade under real
systemd, with stub binaries, in both local and GitHub CI.

.github/workflows/ci.yml

@@ -474,6 +474,36 @@ jobs:
         timeout-minutes: 5
         run: bash testing/openwrt/maintainer-scripts-test.sh
 
+# ─────────────────────────────────────────────────────────────────────────────
+# Job 2f – systemd tarball upgrade scenarios
+#
+# Runs the tarball's install.sh twice in a systemd container, the second time
+# as an upgrade, with fips, fips-dns and fips-gateway in a known state, and
+# checks that the units running before the upgrade are running after it. The
+# binaries are stubs, so like the OpenWrt job it needs no FIPS build and no
+# shared test image, and has a job of its own.
+#
+# The leg keeps `suite:` because testing/check-ci-parity.sh matches it against
+# TARBALL_INSTALL_SUITES in ci-local.sh; the step below does not read it.
+# ─────────────────────────────────────────────────────────────────────────────
+  tarball-install:
+    name: systemd tarball (${{ matrix.suite }})
+    runs-on: ubuntu-latest
+    if: ${{ !inputs.skip_integration }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: tarball-install
+
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+
+      - name: Run the systemd tarball upgrade scenarios
+        timeout-minutes: 10
+        run: bash testing/tarball-install/test.sh
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Job 3 – Integration tests (static mesh + chaos simulation)
 #

packaging/systemd/install.sh

@@ -112,19 +112,20 @@ fi
 
 # --- Install systemd units ---
 
-was_active=false
+# fips-dns and fips-gateway carry Requires=fips.service, so stopping fips
-if systemctl is-active --quiet fips.service 2>/dev/null; then
+# stops them too. Record all three before stopping any of them.
-    was_active=true
+declare -A was_active=()
-    echo "Stopping running fips service..."
+for unit in fips fips-dns fips-gateway; do
-    systemctl stop fips.service
+    if systemctl is-active --quiet "${unit}.service" 2>/dev/null; then
+        was_active[$unit]=true
     fi
-
+done
-dns_was_active=false
+for unit in fips-gateway fips-dns fips; do
-if systemctl is-active --quiet fips-dns.service 2>/dev/null; then
+    if [ "${was_active[$unit]:-}" = true ]; then
-    dns_was_active=true
+        echo "Stopping running ${unit} service..."
-    echo "Stopping running fips-dns service..."
+        systemctl stop "${unit}.service"
-    systemctl stop fips-dns.service
     fi
+done
 
 install -m 0644 "${SCRIPT_DIR}/fips.service" "${SYSTEMD_DIR}/fips.service"
 install -m 0644 "${SCRIPT_DIR}/fips-dns.service" "${SYSTEMD_DIR}/fips-dns.service"
@@ -164,15 +165,13 @@ systemctl enable fips.service
 systemctl enable fips-dns.service
 echo "Services enabled (will start on boot)."
 
-# Restart if they were running before
+# Restart each unit that was running before
-if $was_active; then
+for unit in fips fips-dns fips-gateway; do
-    echo "Restarting fips service..."
+    if [ "${was_active[$unit]:-}" = true ]; then
-    systemctl start fips.service
+        echo "Restarting ${unit} service..."
-fi
+        systemctl start "${unit}.service"
-if $dns_was_active; then
-    echo "Restarting fips-dns service..."
-    systemctl start fips-dns.service
     fi
+done
 
 echo ""
 echo "=== Installation complete ==="

testing/README.md

@@ -101,6 +101,14 @@ what the real `build-apk.sh` and `build-ipk.sh` package. No router,
 opkg or apk-tools is involved. Part of both CI runners as
 `openwrt-scripts`.
 
+### [tarball-install/](tarball-install/) -- systemd Tarball Upgrade
+
+Runs the systemd tarball's `install.sh` twice in a Debian 12 systemd
+container, the second time as an upgrade, with stub binaries. Checks
+that the units running before the upgrade (fips, fips-dns and
+fips-gateway, in three combinations) are the ones running after it.
+Part of both CI runners as `tarball-install`.
+
 ### [native-api/](native-api/) -- Native Datagram API
 
 Checks the experimental native datagram API: a client process opens a

testing/ci-local.sh

@@ -208,6 +208,7 @@ CHAOS_SUITES=(
 #     testing/chaos/scripts/chaos.sh bloom-storm.
 GATEWAY_SUITES=(gateway)
 OPENWRT_SUITES=(openwrt-scripts)
+TARBALL_INSTALL_SUITES=(tarball-install)
 SIDECAR_SUITES=(sidecar)
 FIREWALL_SUITES=(firewall)
 NAT_SUITES=(cone symmetric lan)
@@ -248,6 +249,9 @@ list_suites() {
     echo "  OpenWrt packaging:"
     for s in "${OPENWRT_SUITES[@]}"; do echo "    $s"; done
     echo ""
+    echo "  systemd tarball:"
+    for s in "${TARBALL_INSTALL_SUITES[@]}"; do echo "    $s"; done
+    echo ""
     echo "  Firewall baseline:"
     for s in "${FIREWALL_SUITES[@]}"; do echo "    $s"; done
     echo ""
@@ -1207,6 +1211,15 @@ run_integration() {
         return
     fi
 
+    # Also ahead of the build context, for the same reason: the tarball
+    # install scenarios use stub binaries and their own systemd image.
+    if [[ -z "$ONLY_SUITE" ]]; then
+        run_tarball_install
+    elif [[ "$ONLY_SUITE" == "tarball-install" ]]; then
+        run_tarball_install
+        return
+    fi
+
     # Populate THIS run's build context, then install the binaries into it.
     # Everything but the binaries is copied from the tracked context directory;
     # the binaries are installed fresh, and a previous run's are deliberately
@@ -1370,6 +1383,8 @@ run_suite() {
             run_gateway ;;
         openwrt-scripts)
             run_openwrt_scripts ;;
+        tarball-install)
+            run_tarball_install ;;
         firewall)
             run_firewall ;;
         nat-cone|nat-symmetric|nat-lan)
@@ -1466,6 +1481,18 @@ run_openwrt_scripts() {
     return $rc
 }
 
+# The systemd tarball's install.sh, run as an upgrade under real systemd with
+# stub binaries: the units that were running before the upgrade must be the
+# ones running after it, including fips-dns and fips-gateway, which stop with
+# fips through Requires=.
+run_tarball_install() {
+    local rc=0
+    info "[tarball-install] Running the systemd tarball upgrade scenarios"
+    bash "$SCRIPT_DIR/tarball-install/test.sh" || rc=$?
+    record "tarball-install" $rc
+    return $rc
+}
+
 run_ci_parity() {
     local rc=0
     info "[ci-parity] Comparing the local suite set against the GitHub matrix"

testing/tarball-install/test.sh

@@ -0,0 +1,367 @@
+#!/bin/bash
+# Test the systemd tarball's install.sh as an upgrade, under real systemd.
+#
+# fips-dns.service and fips-gateway.service carry Requires=fips.service, so
+# stopping fips stops both. An upgrade that checks which units are running
+# only after stopping fips sees the other two as already stopped, and starts
+# fips alone again. This suite installs the tarball once, puts the units in a
+# known state, runs install.sh a second time (the upgrade), and checks that
+# exactly the units that were running before are running after.
+#
+# The tarball is staged from the real packaging/systemd/install.sh and unit
+# files, and the real packaging/common configuration, the same file set
+# build-tarball.sh ships. The binaries and DNS helpers are stubs: the fips and
+# fips-gateway stubs sleep, so their units stay active, and the DNS helpers
+# exit 0. What is under test is install.sh against real units and a real
+# systemd, which carries the Requires= stop propagation; nothing here depends
+# on what the binaries do.
+#
+# Scenarios, each on a fresh container:
+#   all-active  fips, fips-dns and fips-gateway running before the upgrade.
+#               All three must be running after it, and fips must have been
+#               restarted (its InvocationID changes).
+#   fips-only   only fips running. fips must be running after, and the other
+#               two must still be stopped.
+#   none-active nothing running. Nothing may be running after.
+#
+# Usage: ./test.sh [--install-sh PATH] [scenario ...]
+#   No scenarios = run all of them.
+#   --install-sh PATH runs another install.sh in place of the tree's, so the
+#   suite can be pointed at an older script without editing the tree.
+#
+# Requirements: Docker able to grant SYS_ADMIN and NET_ADMIN and an unconfined
+# AppArmor profile (the container is not privileged; see
+# testing/lib/systemd-container.sh), and /dev/net/tun on the host.
+#
+# Exit 0 only when every check passed. A container that cannot boot or be
+# inspected is a FAIL, never a skip.
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=SCRIPTDIR/../lib/systemd-container.sh
+source "$SCRIPT_DIR/../lib/systemd-container.sh"
+# shellcheck source=SCRIPTDIR/../lib/image-build.sh
+source "$SCRIPT_DIR/../lib/image-build.sh"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+IMAGE="fips-tarball-test:debian12"
+BOOT_TIMEOUT=60
+# Bounds one install.sh run. It starts at most three units, and the gateway's
+# ExecStartPre returns at once because fips0 exists before the first start.
+INSTALL_TIMEOUT=120
+UNIT_START_TIMEOUT=60
+TARBALL_DIR=/opt/fips-tarball
+
+PASS=0
+FAIL=0
+
+INSTALL_SH="$REPO_ROOT/packaging/systemd/install.sh"
+STAGE=""
+
+log()  { echo "=== $*"; }
+pass() { echo "  PASS: $*"; PASS=$((PASS + 1)); }
+fail() { echo "  FAIL: $*"; FAIL=$((FAIL + 1)); }
+
+# Remove a container, ignoring one that does not exist.
+cleanup_container() {
+    local name="$1"
+    docker rm -f "$name" >/dev/null 2>&1 || true
+}
+
+# Remove every scenario's container and the staged tarball.
+cleanup_all() {
+    local s
+    for s in $ALL_SCENARIOS; do
+        cleanup_container "$(container_name "$s")"
+    done
+    [ -n "$STAGE" ] && rm -rf "$STAGE"
+}
+
+# The container name for a scenario, scoped to this CI run.
+container_name() {
+    echo "fips-tarball-test-$1${FIPS_CI_NAME_SUFFIX:-}"
+    return 0
+}
+
+# Build the systemd image. The build context is an empty directory: the
+# tarball is copied into the container after it starts, not baked in.
+build_image() {
+    local ctx rc=0
+    ctx=$(mktemp -d)
+    retry_build "docker build -t $IMAGE" build_inline "$IMAGE" "$(cat <<'DOCKERFILE'
+FROM debian:12
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    systemd iproute2 procps && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+CMD ["/lib/systemd/systemd"]
+DOCKERFILE
+    )" "$ctx" || rc=$?
+    rm -rf "$ctx"
+    return "$rc"
+}
+
+# Stage a tarball directory: the real install script, units and configuration,
+# with stub binaries and DNS helpers.
+stage_tarball() {
+    local f
+    STAGE=$(mktemp -d)
+    cp "$INSTALL_SH" "$STAGE/install.sh" || return 1
+    for f in fips.service fips-dns.service fips-gateway.service fips-firewall.service; do
+        cp "$REPO_ROOT/packaging/systemd/$f" "$STAGE/" || return 1
+    done
+    for f in fips.yaml hosts fips.nft; do
+        cp "$REPO_ROOT/packaging/common/$f" "$STAGE/" || return 1
+    done
+    printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips"
+    printf '#!/bin/sh\nexec sleep infinity\n' > "$STAGE/fips-gateway"
+    printf '#!/bin/sh\nexit 0\n' > "$STAGE/fipsctl"
+    printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-setup"
+    printf '#!/bin/sh\nexit 0\n' > "$STAGE/fips-dns-teardown"
+    chmod 0755 "$STAGE"/install.sh "$STAGE"/fips "$STAGE"/fips-gateway \
+        "$STAGE"/fipsctl "$STAGE"/fips-dns-setup "$STAGE"/fips-dns-teardown
+    return 0
+}
+
+# Start the scenario's systemd container. Not privileged: see
+# testing/lib/systemd-container.sh for the flags and why. The TUN device is
+# passed so the scenario can create the fips0 that fips-gateway.service's
+# ExecStartPre waits for.
+start_container() {
+    local name="$1"
+    cleanup_container "$name"
+    run_quiet "docker run $name" \
+        docker run -d --name "$name" \
+        --label com.corganlabs.fips-ci=1 \
+        "${SYSTEMD_CAPS[@]}" \
+        --cgroupns=host \
+        --device /dev/net/tun \
+        -v /sys/fs/cgroup:/sys/fs/cgroup:rw \
+        --tmpfs /run --tmpfs /run/lock \
+        "$IMAGE" || return
+    check_isolation "$name"
+    return
+}
+
+# Wait for systemd to finish booting. `degraded` counts as booted: units such
+# as systemd-modules-load cannot succeed in a container, and the system still
+# finished starting. The state string is tested directly rather than piped, so
+# pipefail cannot turn a degraded boot into a failure.
+wait_for_systemd() {
+    local name="$1" state
+    for _i in $(seq 1 "$BOOT_TIMEOUT"); do
+        state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true)
+        case "$state" in
+            running | degraded) return 0 ;;
+        esac
+        sleep 1
+    done
+    echo "  ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2
+    return 1
+}
+
+# Run the staged install.sh in the container, under a bound.
+run_install() {
+    local name="$1" label="$2" out rc=0
+    out=$(timeout "$INSTALL_TIMEOUT" docker exec "$name" "$TARBALL_DIR/install.sh" 2>&1) || rc=$?
+    if [ "$rc" -ne 0 ]; then
+        fail "$label: install.sh exited $rc"
+        echo "$out" | sed 's/^/    /'
+        return 1
+    fi
+    pass "$label: install.sh exits 0"
+    return 0
+}
+
+# Print a unit's active state ("unknown" when the container cannot be asked).
+unit_state() {
+    local name="$1" unit="$2" state
+    state=$(docker exec "$name" systemctl is-active "$unit" 2>/dev/null)
+    echo "${state:-unknown}"
+    return 0
+}
+
+# Print a unit's InvocationID, which changes on every start.
+invocation_id() {
+    docker exec "$1" systemctl show -p InvocationID --value "$2" 2>/dev/null
+    return
+}
+
+# Record a PASS when a unit is in the wanted state, a FAIL otherwise.
+expect_state() {
+    local name="$1" label="$2" unit="$3" want="$4" got
+    got=$(unit_state "$name" "$unit")
+    if [ "$got" = "$want" ]; then
+        pass "$label: $unit is $got"
+    else
+        fail "$label: $unit is $got (want $want)"
+    fi
+    return 0
+}
+
+# Start a unit under a bound, recording a FAIL when it does not start.
+start_unit() {
+    local name="$1" unit="$2" out
+    if ! out=$(timeout "$UNIT_START_TIMEOUT" docker exec "$name" systemctl start "$unit" 2>&1); then
+        fail "setup: could not start $unit: ${out:-no output}"
+        return 1
+    fi
+    return 0
+}
+
+# Boot a fresh container, copy the tarball in and run the first install.
+# Returns 1, having recorded a FAIL, when any of that does not happen.
+prepare() {
+    local name="$1" label="$2"
+    start_container "$name" || { fail "$label: container did not start"; return 1; }
+    wait_for_systemd "$name" || {
+        fail "$label: systemd did not boot; the checks would read an unstarted system"
+        return 1
+    }
+    if ! docker cp "$STAGE/." "$name:$TARBALL_DIR" >/dev/null; then
+        fail "$label: could not copy the tarball into $name"
+        return 1
+    fi
+    if ! docker exec "$name" ip tuntap add fips0 mode tun >/dev/null 2>&1; then
+        fail "$label: could not create fips0 in $name"
+        return 1
+    fi
+    run_install "$name" "$label: first install" || return 1
+    return 0
+}
+
+scenario_all_active() {
+    local label="all-active" name before after
+    name=$(container_name "$label")
+    log "$label: fips, fips-dns and fips-gateway running before the upgrade"
+    if prepare "$name" "$label" \
+        && start_unit "$name" fips.service \
+        && start_unit "$name" fips-dns.service \
+        && start_unit "$name" fips-gateway.service; then
+        expect_state "$name" "$label: before" fips.service active
+        expect_state "$name" "$label: before" fips-dns.service active
+        expect_state "$name" "$label: before" fips-gateway.service active
+        before=$(invocation_id "$name" fips.service)
+        if run_install "$name" "$label: upgrade"; then
+            expect_state "$name" "$label: after" fips.service active
+            expect_state "$name" "$label: after" fips-dns.service active
+            expect_state "$name" "$label: after" fips-gateway.service active
+            after=$(invocation_id "$name" fips.service)
+            if [ -z "$before" ] || [ -z "$after" ]; then
+                fail "$label: could not read fips.service's InvocationID (before '${before}', after '${after}')"
+            elif [ "$before" != "$after" ]; then
+                pass "$label: the upgrade restarted fips.service"
+            else
+                fail "$label: fips.service kept InvocationID $before, so the upgrade did not stop and restart it"
+            fi
+        fi
+    fi
+    cleanup_container "$name"
+    return 0
+}
+
+scenario_fips_only() {
+    local label="fips-only" name
+    name=$(container_name "$label")
+    log "$label: only fips running before the upgrade"
+    if prepare "$name" "$label" && start_unit "$name" fips.service; then
+        expect_state "$name" "$label: before" fips.service active
+        expect_state "$name" "$label: before" fips-dns.service inactive
+        expect_state "$name" "$label: before" fips-gateway.service inactive
+        if run_install "$name" "$label: upgrade"; then
+            expect_state "$name" "$label: after" fips.service active
+            expect_state "$name" "$label: after" fips-dns.service inactive
+            expect_state "$name" "$label: after" fips-gateway.service inactive
+        fi
+    fi
+    cleanup_container "$name"
+    return 0
+}
+
+scenario_none_active() {
+    local label="none-active" name
+    name=$(container_name "$label")
+    log "$label: nothing running before the upgrade"
+    if prepare "$name" "$label"; then
+        expect_state "$name" "$label: before" fips.service inactive
+        if run_install "$name" "$label: upgrade"; then
+            expect_state "$name" "$label: after" fips.service inactive
+            expect_state "$name" "$label: after" fips-dns.service inactive
+            expect_state "$name" "$label: after" fips-gateway.service inactive
+        fi
+    fi
+    cleanup_container "$name"
+    return 0
+}
+
+ALL_SCENARIOS="all-active fips-only none-active"
+
+_args=()
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --install-sh)
+            INSTALL_SH="${2:?--install-sh requires a path}"
+            shift 2
+            ;;
+        -h|--help)
+            echo "usage: test.sh [--install-sh PATH] [scenario ...]"
+            echo "scenarios: $ALL_SCENARIOS"
+            exit 0
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            _args+=("$1")
+            shift
+            ;;
+    esac
+done
+set -- ${_args[@]+"${_args[@]}"}
+
+if [ $# -eq 0 ]; then
+    scenarios="$ALL_SCENARIOS"
+else
+    scenarios="$*"
+fi
+
+for scenario in $scenarios; do
+    case " $ALL_SCENARIOS " in
+        *" $scenario "*) ;;
+        *)
+            echo "Unknown scenario: $scenario (available: $ALL_SCENARIOS)" >&2
+            exit 1
+            ;;
+    esac
+done
+
+if [ ! -f "$INSTALL_SH" ]; then
+    echo "install script not found: $INSTALL_SH" >&2
+    exit 1
+fi
+
+trap cleanup_all EXIT
+
+log "Building $IMAGE"
+if ! build_image; then
+    fail "image build failed"
+elif ! stage_tarball; then
+    fail "could not stage the tarball"
+else
+    for scenario in $scenarios; do
+        case "$scenario" in
+            all-active) scenario_all_active ;;
+            fips-only) scenario_fips_only ;;
+            none-active) scenario_none_active ;;
+        esac
+        echo
+    done
+fi
+
+echo "═══════════════════════════════════════"
+echo "Results: $PASS passed, $FAIL failed"
+echo "═══════════════════════════════════════"
+
+[ "$FAIL" -eq 0 ] && [ "$PASS" -gt 0 ]