From 49163befd57d0cf52153082ae392233906d4c9a8 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:42:48 +0000 Subject: [PATCH] Give each local CI run its own docker build context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoping the image tag was never sufficient on its own. Every build read one unscoped directory in the working tree, into which each run copies the binaries it just built, so two concurrent runs raced on the contents of the context as well as on the name of the result — and a run could produce a correctly-per-run-tagged image built from the other run's binaries. The run now copies the context's tracked files into its own directory, installs its binaries there, and builds from it, exporting the path so every other consumer follows. Deliberately not carried over: a previous run's binaries, since inheriting them is the failure this prevents. The path is absolute because compose resolves a relative build context against the compose file's own directory rather than the working directory, which was measured rather than assumed. The chaos entry script gated the whole simulation on a binary in the shared directory by literal path, so it moves in this same commit: left behind, it would have failed on a clean checkout and, worse on a host with leftovers, passed while reading a binary that was not the one under test. Teardown removes the directory on red runs as well as green, since it holds only reproducible content and is never the evidence of a failure. The worker's SIGKILL runs no trap, so the cleanup script also sweeps contexts left by a preempted run, and the ignore rule keeps a concurrent run from showing up as untracked working-tree noise. --- .gitignore | 5 +++ testing/acl-allowlist/docker-compose.yml | 6 ++- testing/chaos/scripts/chaos.sh | 6 ++- testing/ci-cleanup.sh | 22 +++++++++++ testing/ci-local.sh | 48 ++++++++++++++++++++---- testing/firewall/docker-compose.yml | 6 ++- testing/sidecar/scripts/test-sidecar.sh | 2 +- 7 files changed, 84 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ce9b98e..d3bbb5e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,11 @@ __pycache__/ *.egg-info/ *.egg +# Per-run build contexts created by testing/ci-local.sh. Its teardown normally +# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a +# preempted run; ci-cleanup.sh sweeps the survivors. +/testing/docker-*/ + # Runtime artifacts from running fips in-tree during local testing. # Root-anchored so legitimately-tracked fips.yaml under packaging/ and # examples/ stays included. diff --git a/testing/acl-allowlist/docker-compose.yml b/testing/acl-allowlist/docker-compose.yml index 9bcb02e..a20428a 100644 --- a/testing/acl-allowlist/docker-compose.yml +++ b/testing/acl-allowlist/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/chaos/scripts/chaos.sh b/testing/chaos/scripts/chaos.sh index 0ffdbb9..2a22d2b 100755 --- a/testing/chaos/scripts/chaos.sh +++ b/testing/chaos/scripts/chaos.sh @@ -125,7 +125,11 @@ if ! docker info &> /dev/null; then exit 1 fi -DOCKER_DIR="$CHAOS_DIR/../docker" +# A harness that scopes its build context per run passes it in +# FIPS_BUILD_CONTEXT and stops writing to the shared directory, so checking the +# shared one would either fail on a clean checkout or, worse, pass while +# reading a stale binary that is not the one under test. +DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$CHAOS_DIR/../docker}" if [ ! -f "$DOCKER_DIR/fips" ]; then echo "Error: FIPS binary not found at $DOCKER_DIR/fips" >&2 echo "Run testing/scripts/build.sh first" >&2 diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index fbcb5ec..0f25243 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -247,6 +247,27 @@ reap_images() { timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true } +# Per-run build contexts left in the working tree. ci-local.sh removes its own +# from the EXIT trap, but the CI worker sends SIGKILL after SIGTERM and a +# SIGKILL runs no trap, so a preempted run can leave an 18 MB directory behind +# with nothing else that would ever notice it. +# +# Scoped mode takes only the named run's. Broad mode cannot tell a live run's +# context from an abandoned one by name, so it goes by age instead: a run lasts +# well under an hour, and a day is far outside that. +reap_build_contexts() { + local dir + if [[ -n "$RUN_ID" ]]; then + dir="$SCRIPT_DIR/docker-$RUN_ID" + [[ -d "$dir" ]] && rm -rf "$dir" + return 0 + fi + while IFS= read -r dir; do + [[ -n "$dir" ]] && rm -rf "$dir" + done < <(find "$SCRIPT_DIR" -maxdepth 1 -type d -name 'docker-*' -mtime +0 2>/dev/null) + return 0 +} + # Order matters: containers reference networks/volumes, so drop them first, and # the veth sweep needs an image to run ip(8) in, so it precedes the image reap. reap_containers @@ -254,5 +275,6 @@ reap_networks reap_volumes reap_veths reap_images +reap_build_contexts exit 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 3fe7796..670795e 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -318,6 +318,14 @@ CI_LABEL_RUN="com.corganlabs.fips-ci.run=${CI_RUN_ID}" export FIPS_CI_RUN_ID="$CI_RUN_ID" export FIPS_TEST_IMAGE="$CI_IMAGE_TEST" export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP" +# The build context is this run's too. testing/docker/ is a single directory in +# the working tree, so two runs racing on its CONTENTS produce a per-run-tagged +# image built from the other run's binaries — scoping the tag alone does not +# close that. Absolute, and it has to be: compose interpolates this into +# build.context but resolves a relative result against the compose FILE's +# directory, not the working directory. +CI_BUILD_CONTEXT="$SCRIPT_DIR/docker-${CI_RUN_ID}" +export FIPS_BUILD_CONTEXT="$CI_BUILD_CONTEXT" # Docker container names are GLOBAL — a compose project name does not scope # them — so the suite compose files append this suffix to every explicit # container_name, and the suite scripts append it wherever they address a @@ -405,6 +413,15 @@ ci_teardown() { rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}" rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}" fi + + # 4. This run's build context. Unlike the generated configs it is removed + # on a red run too: it holds the binaries and the Dockerfiles, both + # reproducible from the commit, so it is never the evidence of a + # failure. Guarded on the path having been derived at all, so an early + # exit cannot turn this into `rm -rf $SCRIPT_DIR/docker-`. + if [[ -n "${CI_BUILD_CONTEXT:-}" && "$CI_BUILD_CONTEXT" != "$SCRIPT_DIR/docker-" ]]; then + rm -rf "$CI_BUILD_CONTEXT" + fi } on_signal() { @@ -871,22 +888,39 @@ run_tor_directory() { run_integration() { stage "Stage 3: Integration Tests" - # Install binaries to shared docker context + # Populate THIS run's build context, then install the binaries into it. + # Everything but the binaries is copied from the tracked context directory; + # the binaries are installed fresh, and a previous run's are deliberately + # not carried over, since inheriting them is the failure this scoping + # exists to prevent. + info "Preparing build context $CI_BUILD_CONTEXT" + rm -rf "$CI_BUILD_CONTEXT" + mkdir -p "$CI_BUILD_CONTEXT" || { record "docker-build" 1; return; } + local _f + for _f in "$SCRIPT_DIR"/docker/*; do + case "$(basename "$_f")" in + fips|fipsctl|fipstop|fips-gateway) continue ;; + esac + cp -a "$_f" "$CI_BUILD_CONTEXT/" || { record "docker-build" 1; return; } + done + info "Installing release binaries" - install_binaries testing/docker + install_binaries "$CI_BUILD_CONTEXT" # Build unified test image once (used by all harnesses). Tag per-run # (fips-test:${run}) so a build killed mid-flight never wedges the next # run's rebuild, and concurrent runs never clobber each other's image. - # Then retag :latest for the compose files / harness scripts that still - # reference fips-test:latest directly; the retag happens only after BOTH - # builds succeed, so :latest never points at a half-built image. info "Building $CI_IMAGE_TEST Docker image" - docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" testing/docker --quiet \ + docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build" 1; return; } docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ - -f testing/docker/Dockerfile.app testing/docker --quiet \ + -f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build-app" 1; return; } + # The remaining bridge back to the shared mutable tag, for any consumer not + # yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the + # step that makes a missed consumer fail loudly instead of silently + # resolving another run's binaries. Both builds have succeeded by here, so + # :latest never points at a half-built image. docker tag "$CI_IMAGE_TEST" fips-test:latest docker tag "$CI_IMAGE_APP" fips-test-app:latest diff --git a/testing/firewall/docker-compose.yml b/testing/firewall/docker-compose.yml index 9cedf41..fe65773 100644 --- a/testing/firewall/docker-compose.yml +++ b/testing/firewall/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/sidecar/scripts/test-sidecar.sh b/testing/sidecar/scripts/test-sidecar.sh index 2cbd114..975973f 100755 --- a/testing/sidecar/scripts/test-sidecar.sh +++ b/testing/sidecar/scripts/test-sidecar.sh @@ -115,7 +115,7 @@ trap cleanup EXIT if [[ "${1:-}" != "--skip-build" ]]; then log "Building test images..." - DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)" + DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$(cd "$SIDECAR_DIR/../docker" && pwd)}" docker build -t fips-test:latest "$DOCKER_DIR" docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR" fi