From 0c3c0c79c06d030f727f26110449ded535ebc006 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sat, 19 Sep 2026 13:38:47 +0000 Subject: [PATCH] Retry test, package and NAT lab image builds across transient registry failures Image builds in CI pull base images from Docker Hub and ghcr.io and fetch packages from distribution mirrors, and each has failed a leg for a few seconds at a time: a registry dial timeout, a 502, an apt file truncated mid-fetch. A single failed build failed the leg, and a rerun of the whole workflow was the only recovery. Add retry_build to testing/lib/image-build.sh: up to three attempts with 10 s and then 20 s between them, whole-build so the base is resolved again and every package-fetching RUN step runs again. It is quiet on a first-attempt success, prints each failed attempt and a recovery to stderr, and on GitHub Actions also raises a warning on the run summary when a build recovered, so recovered failures remain countable. It never wraps a test or a container start. Use it for the shared test images in the integration job, the deb-install and dns-resolver runtime images, and the package builder image. Each deb-install and dns-resolver attempt prints its captured output on failure, so a recovered failure still shows its cause. The NAT, nostr publish-consume and STUN fault suites built their lab images inside `docker compose up --build`: the relay image from ghcr.io and Alpine, the STUN server from the Python image, and the routers from Debian with apt. Build the profile's images first through retry_build, then bring the lab up with --no-build. The start is not retried, since a container that fails to start is a test result; only the build is. --- .github/workflows/ci.yml | 6 ++- packaging/debian/build-deb-container.sh | 6 ++- testing/deb-install/test.sh | 5 ++- testing/dns-resolver/test.sh | 5 ++- testing/lib/image-build.sh | 59 +++++++++++++++++++++++-- testing/nat/scripts/nat-test.sh | 18 ++++++-- testing/nat/scripts/nostr-relay-test.sh | 8 +++- testing/nat/scripts/stun-faults-test.sh | 8 +++- 8 files changed, 100 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e360605d..3f5f1fdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -587,8 +587,10 @@ jobs: chmod +x _bin/native-echo _bin/native-surface cp _bin/native-echo testing/docker/native-echo cp _bin/native-surface testing/docker/native-surface - docker build -t fips-test:latest testing/docker - docker build -t fips-test-app:latest -f testing/docker/Dockerfile.app testing/docker + # Retried: both builds pull from Docker Hub and the Debian mirrors. + source testing/lib/image-build.sh + retry_build "docker build fips-test" docker build -t fips-test:latest testing/docker + retry_build "docker build fips-test-app" docker build -t fips-test-app:latest -f testing/docker/Dockerfile.app testing/docker # ── Static topology ──────────────────────────────────────────────────── - name: Generate configs (static) diff --git a/packaging/debian/build-deb-container.sh b/packaging/debian/build-deb-container.sh index 29f75f59..ca19ddc9 100755 --- a/packaging/debian/build-deb-container.sh +++ b/packaging/debian/build-deb-container.sh @@ -31,6 +31,8 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" # shellcheck source=../build-floor.env . "$REPO_ROOT/packaging/build-floor.env" +# shellcheck source=SCRIPTDIR/../../testing/lib/image-build.sh +. "$REPO_ROOT/testing/lib/image-build.sh" DEST_DIR="$REPO_ROOT/deploy" VERSION="" @@ -115,7 +117,9 @@ fi if [ "$BUILT_IMAGE" -eq 1 ]; then echo "=== Building $IMAGE_TAG from $FIPS_BUILD_IMAGE with Rust $RUST_TOOLCHAIN ===" >&2 - docker build \ + # Retried because the build pulls the floor image and fetches apt packages, + # rustup and crates, any of which can fail for a few seconds at a time. + retry_build "docker build $IMAGE_TAG" docker build \ --build-arg "BASE=$FIPS_BUILD_IMAGE" \ --build-arg "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" \ -t "$IMAGE_TAG" \ diff --git a/testing/deb-install/test.sh b/testing/deb-install/test.sh index 5e47606c..31f421a2 100755 --- a/testing/deb-install/test.sh +++ b/testing/deb-install/test.sh @@ -83,8 +83,9 @@ cleanup_container() { build_image() { local tag="$1" shift - echo "$@" | run_quiet "docker build -t $tag" \ - docker build -t "$tag" -f - "$REPO_ROOT" + local dockerfile="$*" + retry_build "docker build -t $tag" build_inline "$tag" "$dockerfile" "$REPO_ROOT" || return + return 0 } # Start the scenario's systemd container. Not privileged: see diff --git a/testing/dns-resolver/test.sh b/testing/dns-resolver/test.sh index c9b347e4..038a1190 100755 --- a/testing/dns-resolver/test.sh +++ b/testing/dns-resolver/test.sh @@ -73,8 +73,9 @@ cleanup_container() { build_image() { local tag="$1" shift - echo "$@" | run_quiet "docker build -t $tag" \ - docker build -t "$tag" -f - "$REPO_ROOT" + local dockerfile="$*" + retry_build "docker build -t $tag" build_inline "$tag" "$dockerfile" "$REPO_ROOT" || return + return 0 } # Start a systemd container in the background. Not privileged: see diff --git a/testing/lib/image-build.sh b/testing/lib/image-build.sh index 0cf38435..f7367a7c 100644 --- a/testing/lib/image-build.sh +++ b/testing/lib/image-build.sh @@ -1,15 +1,22 @@ #!/bin/bash # Shared helpers for building test images and starting test containers. # -# Source this file to get dump_output() and run_quiet(): +# Source this file to get dump_output(), run_quiet(), build_inline() and +# retry_build(): # source "$SCRIPT_DIR/../lib/image-build.sh" # echo "$dockerfile" | run_quiet "docker build -t $tag" \ # docker build -t "$tag" -f - "$REPO_ROOT" +# retry_build "docker build $tag" docker build -t "$tag" "$context" # # A build or a container start that fails for a reason outside the project, # such as a registry timeout, is indistinguishable from one the project caused -# unless its output survives. These helpers keep a command quiet when it -# succeeds and print everything it said when it fails. +# unless its output survives. dump_output and run_quiet keep a command quiet +# when it succeeds and print everything it said when it fails. +# +# Image builds reach Docker Hub, ghcr.io and distribution mirrors, and every one +# of those has timed out or served a truncated file in CI for a few seconds at a +# time. retry_build runs a whole build again after such a failure, so the base +# image is resolved again and each RUN step that fetched packages runs again. # Emit a captured output file to stderr, delimited and labelled with the # command it came from. Callers use this only on failure: a command that @@ -40,3 +47,49 @@ run_quiet() { rm -f "$out" return "$rc" } + +# Build TAG from the Dockerfile text DOCKERFILE with context CONTEXT, output +# captured by run_quiet. The text is an argument rather than stdin, so that +# retry_build can run the build again: a pipe can be read only once. +build_inline() { + local tag="$1" dockerfile="$2" context="$3" + echo "$dockerfile" | run_quiet "docker build -t $tag" \ + docker build -t "$tag" -f - "$context" +} + +# Run an image build, and run it again after a failure, up to three attempts +# with 10 s and then 20 s between them. Use it for builds only: a test or a +# container start that fails is a result, and running it again would hide it. +# +# Every message goes to stderr, because some callers promise that their stdout +# holds only their result. A first-attempt success prints nothing. A success +# after a failure says so, and on GitHub Actions also raises a warning on the +# run summary, so a recovered failure is still counted. Returns the last +# attempt's exit status. +retry_build() { + local label="$1" + shift + local attempts=3 attempt=1 rc=0 wait + while true; do + rc=0 + if "$@"; then + if [ "$attempt" -gt 1 ]; then + echo "image-build: $label recovered on attempt $attempt of $attempts" >&2 + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::warning::image-build: $label recovered on attempt $attempt of $attempts" >&2 + fi + fi + return 0 + else + rc=$? + fi + if [ "$attempt" -ge "$attempts" ]; then + echo "image-build: $label failed on all $attempts attempts (last exit $rc)" >&2 + return "$rc" + fi + wait=$((attempt * 10)) + echo "image-build: $label failed on attempt $attempt of $attempts (exit $rc); retrying in ${wait}s" >&2 + sleep "$wait" + attempt=$((attempt + 1)) + done +} diff --git a/testing/nat/scripts/nat-test.sh b/testing/nat/scripts/nat-test.sh index 5e9c040e..76df689f 100755 --- a/testing/nat/scripts/nat-test.sh +++ b/testing/nat/scripts/nat-test.sh @@ -10,6 +10,7 @@ GENERATE_SCRIPT="$SCRIPT_DIR/generate-configs.sh" TOPOLOGY_SCRIPT="$SCRIPT_DIR/setup-topology.sh" WAIT_LIB="$ROOT_DIR/testing/lib/wait-converge.sh" RELAY_LIB="$ROOT_DIR/testing/lib/relay-verdict.sh" +IMAGE_LIB="$ROOT_DIR/testing/lib/image-build.sh" # Must track generate-configs.sh's OUTPUT_DIR and the compose bind-mounts: the # npubs are read back here after the containers are up, so reading a different # directory than the one the generator wrote pings an npub no node owns. @@ -45,6 +46,8 @@ fi source "$WAIT_LIB" # shellcheck disable=SC1090 source "$RELAY_LIB" +# shellcheck disable=SC1090 +source "$IMAGE_LIB" RELAY_CONTAINER="fips-nat-relay${FIPS_CI_NAME_SUFFIX:-}" @@ -500,7 +503,10 @@ run_cone() { echo "=== NAT lab: cone ===" cleanup "$GENERATE_SCRIPT" cone - "${COMPOSE[@]}" --profile cone up -d --build --force-recreate + # Build first, with retries, because the build pulls from registries that + # time out now and then; the start is not retried, since it is the test. + retry_build "compose build (cone)" "${COMPOSE[@]}" --profile cone build + "${COMPOSE[@]}" --profile cone up -d --no-build --force-recreate "$TOPOLOGY_SCRIPT" cone wait_for_peers fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} 1 45 || { dump_cone_diagnostics @@ -548,7 +554,10 @@ run_symmetric() { echo "=== NAT lab: symmetric fallback ===" cleanup NAT_MODE_A=symmetric NAT_MODE_B=symmetric "$GENERATE_SCRIPT" symmetric - NAT_MODE_A=symmetric NAT_MODE_B=symmetric "${COMPOSE[@]}" --profile symmetric up -d --build --force-recreate + # Build first, with retries, because the build pulls from registries that + # time out now and then; the start is not retried, since it is the test. + NAT_MODE_A=symmetric NAT_MODE_B=symmetric retry_build "compose build (symmetric)" "${COMPOSE[@]}" --profile symmetric build + NAT_MODE_A=symmetric NAT_MODE_B=symmetric "${COMPOSE[@]}" --profile symmetric up -d --no-build --force-recreate "$TOPOLOGY_SCRIPT" symmetric wait_for_peers fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} 1 60 || { dump_symmetric_diagnostics @@ -598,7 +607,10 @@ run_lan() { echo "=== NAT lab: lan preference ===" cleanup "$GENERATE_SCRIPT" lan - "${COMPOSE[@]}" --profile lan up -d --build --force-recreate + # Build first, with retries, because the build pulls from registries that + # time out now and then; the start is not retried, since it is the test. + retry_build "compose build (lan)" "${COMPOSE[@]}" --profile lan build + "${COMPOSE[@]}" --profile lan up -d --no-build --force-recreate wait_for_peers fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} 1 45 || { dump_lan_diagnostics return 1 diff --git a/testing/nat/scripts/nostr-relay-test.sh b/testing/nat/scripts/nostr-relay-test.sh index df6bdfff..e6b01795 100755 --- a/testing/nat/scripts/nostr-relay-test.sh +++ b/testing/nat/scripts/nostr-relay-test.sh @@ -22,6 +22,7 @@ BUILD_SCRIPT="$ROOT_DIR/testing/scripts/build.sh" GENERATE_SCRIPT="$SCRIPT_DIR/generate-configs.sh" WAIT_LIB="$ROOT_DIR/testing/lib/wait-converge.sh" RELAY_LIB="$ROOT_DIR/testing/lib/relay-verdict.sh" +IMAGE_LIB="$ROOT_DIR/testing/lib/image-build.sh" # Must track generate-configs.sh's OUTPUT_DIR and the compose bind-mounts. CONFIG_DIR="$NAT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}" @@ -55,6 +56,8 @@ RELAY_CONTAINER="fips-nat-relay${FIPS_CI_NAME_SUFFIX:-}" source "$WAIT_LIB" # shellcheck disable=SC1090 source "$RELAY_LIB" +# shellcheck disable=SC1090 +source "$IMAGE_LIB" cleanup() { "${COMPOSE[@]}" --profile "$PROFILE" down -v --remove-orphans \ @@ -430,7 +433,10 @@ run_test() { cleanup "$GENERATE_SCRIPT" "$SCENARIO" - "${COMPOSE[@]}" --profile "$PROFILE" up -d --build --force-recreate + # Build first, with retries, because the build pulls from registries that + # time out now and then; the start is not retried, since it is the test. + retry_build "compose build ($PROFILE)" "${COMPOSE[@]}" --profile "$PROFILE" build + "${COMPOSE[@]}" --profile "$PROFILE" up -d --no-build --force-recreate # Phase 1 + Phase 2 together: each side publishes its own advert, # subscribes for the other's, then dials. Bidirectional success diff --git a/testing/nat/scripts/stun-faults-test.sh b/testing/nat/scripts/stun-faults-test.sh index 149d5b33..57552681 100755 --- a/testing/nat/scripts/stun-faults-test.sh +++ b/testing/nat/scripts/stun-faults-test.sh @@ -30,6 +30,7 @@ ROOT_DIR="$(cd "$NAT_DIR/../.." && pwd)" BUILD_SCRIPT="$ROOT_DIR/testing/scripts/build.sh" GENERATE_SCRIPT="$SCRIPT_DIR/generate-configs.sh" RELAY_LIB="$ROOT_DIR/testing/lib/relay-verdict.sh" +IMAGE_LIB="$ROOT_DIR/testing/lib/image-build.sh" PROFILE="stun-faults" SCENARIO="$PROFILE" @@ -73,6 +74,8 @@ cleanup() { # shellcheck disable=SC1090 source "$RELAY_LIB" +# shellcheck disable=SC1090 +source "$IMAGE_LIB" trap 'echo ""; echo "stun-faults-test interrupted"; cleanup; exit 130' INT TERM @@ -269,7 +272,10 @@ run_test() { echo "=== stun-faults-test: setup ===" cleanup "$GENERATE_SCRIPT" "$SCENARIO" - "${COMPOSE[@]}" --profile "$PROFILE" up -d --build --force-recreate + # Build first, with retries, because the build pulls from registries that + # time out now and then; the start is not retried, since it is the test. + retry_build "compose build ($PROFILE)" "${COMPOSE[@]}" --profile "$PROFILE" build + "${COMPOSE[@]}" --profile "$PROFILE" up -d --no-build --force-recreate # Give the daemons time to come up. Both fault-node and fault-peer # need to start, publish their adverts to the relay, and discover