Give each local CI run its own docker build context

Scoping the image tag was never sufficient on its own. Every build read one
unscoped directory in the working tree, into which each run copies the
binaries it just built, so two concurrent runs raced on the contents of the
context as well as on the name of the result — and a run could produce a
correctly-per-run-tagged image built from the other run's binaries.

The run now copies the context's tracked files into its own directory,
installs its binaries there, and builds from it, exporting the path so every
other consumer follows. Deliberately not carried over: a previous run's
binaries, since inheriting them is the failure this prevents. The path is
absolute because compose resolves a relative build context against the
compose file's own directory rather than the working directory, which was
measured rather than assumed.

The chaos entry script gated the whole simulation on a binary in the shared
directory by literal path, so it moves in this same commit: left behind, it
would have failed on a clean checkout and, worse on a host with leftovers,
passed while reading a binary that was not the one under test.

Teardown removes the directory on red runs as well as green, since it holds
only reproducible content and is never the evidence of a failure. The
worker's SIGKILL runs no trap, so the cleanup script also sweeps contexts
left by a preempted run, and the ignore rule keeps a concurrent run from
showing up as untracked working-tree noise.
This commit is contained in:
Johnathan Corgan
2026-07-26 15:42:48 +00:00
parent af847c68b5
commit 49163befd5
7 changed files with 84 additions and 11 deletions
+41 -7
View File
@@ -318,6 +318,14 @@ CI_LABEL_RUN="com.corganlabs.fips-ci.run=${CI_RUN_ID}"
export FIPS_CI_RUN_ID="$CI_RUN_ID"
export FIPS_TEST_IMAGE="$CI_IMAGE_TEST"
export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP"
# The build context is this run's too. testing/docker/ is a single directory in
# the working tree, so two runs racing on its CONTENTS produce a per-run-tagged
# image built from the other run's binaries — scoping the tag alone does not
# close that. Absolute, and it has to be: compose interpolates this into
# build.context but resolves a relative result against the compose FILE's
# directory, not the working directory.
CI_BUILD_CONTEXT="$SCRIPT_DIR/docker-${CI_RUN_ID}"
export FIPS_BUILD_CONTEXT="$CI_BUILD_CONTEXT"
# Docker container names are GLOBAL — a compose project name does not scope
# them — so the suite compose files append this suffix to every explicit
# container_name, and the suite scripts append it wherever they address a
@@ -405,6 +413,15 @@ ci_teardown() {
rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}"
rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}"
fi
# 4. This run's build context. Unlike the generated configs it is removed
# on a red run too: it holds the binaries and the Dockerfiles, both
# reproducible from the commit, so it is never the evidence of a
# failure. Guarded on the path having been derived at all, so an early
# exit cannot turn this into `rm -rf $SCRIPT_DIR/docker-`.
if [[ -n "${CI_BUILD_CONTEXT:-}" && "$CI_BUILD_CONTEXT" != "$SCRIPT_DIR/docker-" ]]; then
rm -rf "$CI_BUILD_CONTEXT"
fi
}
on_signal() {
@@ -871,22 +888,39 @@ run_tor_directory() {
run_integration() {
stage "Stage 3: Integration Tests"
# Install binaries to shared docker context
# Populate THIS run's build context, then install the binaries into it.
# Everything but the binaries is copied from the tracked context directory;
# the binaries are installed fresh, and a previous run's are deliberately
# not carried over, since inheriting them is the failure this scoping
# exists to prevent.
info "Preparing build context $CI_BUILD_CONTEXT"
rm -rf "$CI_BUILD_CONTEXT"
mkdir -p "$CI_BUILD_CONTEXT" || { record "docker-build" 1; return; }
local _f
for _f in "$SCRIPT_DIR"/docker/*; do
case "$(basename "$_f")" in
fips|fipsctl|fipstop|fips-gateway) continue ;;
esac
cp -a "$_f" "$CI_BUILD_CONTEXT/" || { record "docker-build" 1; return; }
done
info "Installing release binaries"
install_binaries testing/docker
install_binaries "$CI_BUILD_CONTEXT"
# Build unified test image once (used by all harnesses). Tag per-run
# (fips-test:${run}) so a build killed mid-flight never wedges the next
# run's rebuild, and concurrent runs never clobber each other's image.
# Then retag :latest for the compose files / harness scripts that still
# reference fips-test:latest directly; the retag happens only after BOTH
# builds succeed, so :latest never points at a half-built image.
info "Building $CI_IMAGE_TEST Docker image"
docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" testing/docker --quiet \
docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" "$CI_BUILD_CONTEXT" --quiet \
|| { record "docker-build" 1; return; }
docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \
-f testing/docker/Dockerfile.app testing/docker --quiet \
-f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \
|| { record "docker-build-app" 1; return; }
# The remaining bridge back to the shared mutable tag, for any consumer not
# yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the
# step that makes a missed consumer fail loudly instead of silently
# resolving another run's binaries. Both builds have succeeded by here, so
# :latest never points at a half-built image.
docker tag "$CI_IMAGE_TEST" fips-test:latest
docker tag "$CI_IMAGE_APP" fips-test-app:latest