Give each local CI run its own docker build context

Scoping the image tag was never sufficient on its own. Every build read one
unscoped directory in the working tree, into which each run copies the
binaries it just built, so two concurrent runs raced on the contents of the
context as well as on the name of the result — and a run could produce a
correctly-per-run-tagged image built from the other run's binaries.

The run now copies the context's tracked files into its own directory,
installs its binaries there, and builds from it, exporting the path so every
other consumer follows. Deliberately not carried over: a previous run's
binaries, since inheriting them is the failure this prevents. The path is
absolute because compose resolves a relative build context against the
compose file's own directory rather than the working directory, which was
measured rather than assumed.

The chaos entry script gated the whole simulation on a binary in the shared
directory by literal path, so it moves in this same commit: left behind, it
would have failed on a clean checkout and, worse on a host with leftovers,
passed while reading a binary that was not the one under test.

Teardown removes the directory on red runs as well as green, since it holds
only reproducible content and is never the evidence of a failure. The
worker's SIGKILL runs no trap, so the cleanup script also sweeps contexts
left by a preempted run, and the ignore rule keeps a concurrent run from
showing up as untracked working-tree noise.

.gitignore

@@ -33,6 +33,11 @@ __pycache__/
 *.egg-info/
 *.egg
 
+# Per-run build contexts created by testing/ci-local.sh. Its teardown normally
+# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a
+# preempted run; ci-cleanup.sh sweeps the survivors.
+/testing/docker-*/
+
 # Runtime artifacts from running fips in-tree during local testing.
 # Root-anchored so legitimately-tracked fips.yaml under packaging/ and
 # examples/ stays included.

testing/acl-allowlist/docker-compose.yml

@@ -19,7 +19,11 @@ networks:
 
 x-fips-common: &fips-common
   build:
-    context: ../docker
+    # The harness scopes its build context per run and passes it here; the
+    # shared directory is the hand-run default. Compose resolves a relative
+    # value against THIS file's directory, so the harness must export an
+    # absolute path.
+    context: ${FIPS_BUILD_CONTEXT:-../docker}
   image: ${FIPS_TEST_IMAGE:-fips-test:latest}
   entrypoint: ["/usr/local/bin/entrypoint.sh"]
   cap_add:

testing/chaos/scripts/chaos.sh

@@ -125,7 +125,11 @@ if ! docker info &> /dev/null; then
     exit 1
 fi
 
-DOCKER_DIR="$CHAOS_DIR/../docker"
+# A harness that scopes its build context per run passes it in
+# FIPS_BUILD_CONTEXT and stops writing to the shared directory, so checking the
+# shared one would either fail on a clean checkout or, worse, pass while
+# reading a stale binary that is not the one under test.
+DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$CHAOS_DIR/../docker}"
 if [ ! -f "$DOCKER_DIR/fips" ]; then
     echo "Error: FIPS binary not found at $DOCKER_DIR/fips" >&2
     echo "Run testing/scripts/build.sh first" >&2

testing/ci-cleanup.sh

@@ -247,6 +247,27 @@ reap_images() {
     timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true
 }
 
+# Per-run build contexts left in the working tree. ci-local.sh removes its own
+# from the EXIT trap, but the CI worker sends SIGKILL after SIGTERM and a
+# SIGKILL runs no trap, so a preempted run can leave an 18 MB directory behind
+# with nothing else that would ever notice it.
+#
+# Scoped mode takes only the named run's. Broad mode cannot tell a live run's
+# context from an abandoned one by name, so it goes by age instead: a run lasts
+# well under an hour, and a day is far outside that.
+reap_build_contexts() {
+    local dir
+    if [[ -n "$RUN_ID" ]]; then
+        dir="$SCRIPT_DIR/docker-$RUN_ID"
+        [[ -d "$dir" ]] && rm -rf "$dir"
+        return 0
+    fi
+    while IFS= read -r dir; do
+        [[ -n "$dir" ]] && rm -rf "$dir"
+    done < <(find "$SCRIPT_DIR" -maxdepth 1 -type d -name 'docker-*' -mtime +0 2>/dev/null)
+    return 0
+}
+
 # Order matters: containers reference networks/volumes, so drop them first, and
 # the veth sweep needs an image to run ip(8) in, so it precedes the image reap.
 reap_containers
@@ -254,5 +275,6 @@ reap_networks
 reap_volumes
 reap_veths
 reap_images
+reap_build_contexts
 
 exit 0

testing/ci-local.sh

@@ -318,6 +318,14 @@ CI_LABEL_RUN="com.corganlabs.fips-ci.run=${CI_RUN_ID}"
 export FIPS_CI_RUN_ID="$CI_RUN_ID"
 export FIPS_TEST_IMAGE="$CI_IMAGE_TEST"
 export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP"
+# The build context is this run's too. testing/docker/ is a single directory in
+# the working tree, so two runs racing on its CONTENTS produce a per-run-tagged
+# image built from the other run's binaries — scoping the tag alone does not
+# close that. Absolute, and it has to be: compose interpolates this into
+# build.context but resolves a relative result against the compose FILE's
+# directory, not the working directory.
+CI_BUILD_CONTEXT="$SCRIPT_DIR/docker-${CI_RUN_ID}"
+export FIPS_BUILD_CONTEXT="$CI_BUILD_CONTEXT"
 # Docker container names are GLOBAL — a compose project name does not scope
 # them — so the suite compose files append this suffix to every explicit
 # container_name, and the suite scripts append it wherever they address a
@@ -405,6 +413,15 @@ ci_teardown() {
         rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}"
         rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}"
     fi
+
+    # 4. This run's build context. Unlike the generated configs it is removed
+    #    on a red run too: it holds the binaries and the Dockerfiles, both
+    #    reproducible from the commit, so it is never the evidence of a
+    #    failure. Guarded on the path having been derived at all, so an early
+    #    exit cannot turn this into `rm -rf $SCRIPT_DIR/docker-`.
+    if [[ -n "${CI_BUILD_CONTEXT:-}" && "$CI_BUILD_CONTEXT" != "$SCRIPT_DIR/docker-" ]]; then
+        rm -rf "$CI_BUILD_CONTEXT"
+    fi
 }
 
 on_signal() {
@@ -871,22 +888,39 @@ run_tor_directory() {
 run_integration() {
     stage "Stage 3: Integration Tests"
 
-    # Install binaries to shared docker context
+    # Populate THIS run's build context, then install the binaries into it.
+    # Everything but the binaries is copied from the tracked context directory;
+    # the binaries are installed fresh, and a previous run's are deliberately
+    # not carried over, since inheriting them is the failure this scoping
+    # exists to prevent.
+    info "Preparing build context $CI_BUILD_CONTEXT"
+    rm -rf "$CI_BUILD_CONTEXT"
+    mkdir -p "$CI_BUILD_CONTEXT" || { record "docker-build" 1; return; }
+    local _f
+    for _f in "$SCRIPT_DIR"/docker/*; do
+        case "$(basename "$_f")" in
+            fips|fipsctl|fipstop|fips-gateway) continue ;;
+        esac
+        cp -a "$_f" "$CI_BUILD_CONTEXT/" || { record "docker-build" 1; return; }
+    done
+
     info "Installing release binaries"
-    install_binaries testing/docker
+    install_binaries "$CI_BUILD_CONTEXT"
 
     # Build unified test image once (used by all harnesses). Tag per-run
     # (fips-test:${run}) so a build killed mid-flight never wedges the next
     # run's rebuild, and concurrent runs never clobber each other's image.
-    # Then retag :latest for the compose files / harness scripts that still
-    # reference fips-test:latest directly; the retag happens only after BOTH
-    # builds succeed, so :latest never points at a half-built image.
     info "Building $CI_IMAGE_TEST Docker image"
-    docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" testing/docker --quiet \
+    docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" "$CI_BUILD_CONTEXT" --quiet \
         || { record "docker-build" 1; return; }
     docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \
-        -f testing/docker/Dockerfile.app testing/docker --quiet \
+        -f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \
         || { record "docker-build-app" 1; return; }
+    # The remaining bridge back to the shared mutable tag, for any consumer not
+    # yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the
+    # step that makes a missed consumer fail loudly instead of silently
+    # resolving another run's binaries. Both builds have succeeded by here, so
+    # :latest never points at a half-built image.
     docker tag "$CI_IMAGE_TEST" fips-test:latest
     docker tag "$CI_IMAGE_APP" fips-test-app:latest
 

testing/firewall/docker-compose.yml

@@ -19,7 +19,11 @@ networks:
 
 x-fips-common: &fips-common
   build:
-    context: ../docker
+    # The harness scopes its build context per run and passes it here; the
+    # shared directory is the hand-run default. Compose resolves a relative
+    # value against THIS file's directory, so the harness must export an
+    # absolute path.
+    context: ${FIPS_BUILD_CONTEXT:-../docker}
   image: ${FIPS_TEST_IMAGE:-fips-test:latest}
   entrypoint: ["/usr/local/bin/entrypoint.sh"]
   cap_add:

testing/sidecar/scripts/test-sidecar.sh

@@ -115,7 +115,7 @@ trap cleanup EXIT
 
 if [[ "${1:-}" != "--skip-build" ]]; then
     log "Building test images..."
-    DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)"
+    DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$(cd "$SIDECAR_DIR/../docker" && pwd)}"
     docker build -t fips-test:latest "$DOCKER_DIR"
     docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR"
 fi