diff --git a/.gitignore b/.gitignore index ce9b98e..d3bbb5e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,11 @@ __pycache__/ *.egg-info/ *.egg +# Per-run build contexts created by testing/ci-local.sh. Its teardown normally +# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a +# preempted run; ci-cleanup.sh sweeps the survivors. +/testing/docker-*/ + # Runtime artifacts from running fips in-tree during local testing. # Root-anchored so legitimately-tracked fips.yaml under packaging/ and # examples/ stays included. diff --git a/testing/acl-allowlist/docker-compose.yml b/testing/acl-allowlist/docker-compose.yml index 9bcb02e..a20428a 100644 --- a/testing/acl-allowlist/docker-compose.yml +++ b/testing/acl-allowlist/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/chaos/scripts/chaos.sh b/testing/chaos/scripts/chaos.sh index 0ffdbb9..2a22d2b 100755 --- a/testing/chaos/scripts/chaos.sh +++ b/testing/chaos/scripts/chaos.sh @@ -125,7 +125,11 @@ if ! docker info &> /dev/null; then exit 1 fi -DOCKER_DIR="$CHAOS_DIR/../docker" +# A harness that scopes its build context per run passes it in +# FIPS_BUILD_CONTEXT and stops writing to the shared directory, so checking the +# shared one would either fail on a clean checkout or, worse, pass while +# reading a stale binary that is not the one under test. +DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$CHAOS_DIR/../docker}" if [ ! -f "$DOCKER_DIR/fips" ]; then echo "Error: FIPS binary not found at $DOCKER_DIR/fips" >&2 echo "Run testing/scripts/build.sh first" >&2 diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index fbcb5ec..0f25243 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -247,6 +247,27 @@ reap_images() { timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true } +# Per-run build contexts left in the working tree. ci-local.sh removes its own +# from the EXIT trap, but the CI worker sends SIGKILL after SIGTERM and a +# SIGKILL runs no trap, so a preempted run can leave an 18 MB directory behind +# with nothing else that would ever notice it. +# +# Scoped mode takes only the named run's. Broad mode cannot tell a live run's +# context from an abandoned one by name, so it goes by age instead: a run lasts +# well under an hour, and a day is far outside that. +reap_build_contexts() { + local dir + if [[ -n "$RUN_ID" ]]; then + dir="$SCRIPT_DIR/docker-$RUN_ID" + [[ -d "$dir" ]] && rm -rf "$dir" + return 0 + fi + while IFS= read -r dir; do + [[ -n "$dir" ]] && rm -rf "$dir" + done < <(find "$SCRIPT_DIR" -maxdepth 1 -type d -name 'docker-*' -mtime +0 2>/dev/null) + return 0 +} + # Order matters: containers reference networks/volumes, so drop them first, and # the veth sweep needs an image to run ip(8) in, so it precedes the image reap. reap_containers @@ -254,5 +275,6 @@ reap_networks reap_volumes reap_veths reap_images +reap_build_contexts exit 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 3fe7796..670795e 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -318,6 +318,14 @@ CI_LABEL_RUN="com.corganlabs.fips-ci.run=${CI_RUN_ID}" export FIPS_CI_RUN_ID="$CI_RUN_ID" export FIPS_TEST_IMAGE="$CI_IMAGE_TEST" export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP" +# The build context is this run's too. testing/docker/ is a single directory in +# the working tree, so two runs racing on its CONTENTS produce a per-run-tagged +# image built from the other run's binaries — scoping the tag alone does not +# close that. Absolute, and it has to be: compose interpolates this into +# build.context but resolves a relative result against the compose FILE's +# directory, not the working directory. +CI_BUILD_CONTEXT="$SCRIPT_DIR/docker-${CI_RUN_ID}" +export FIPS_BUILD_CONTEXT="$CI_BUILD_CONTEXT" # Docker container names are GLOBAL — a compose project name does not scope # them — so the suite compose files append this suffix to every explicit # container_name, and the suite scripts append it wherever they address a @@ -405,6 +413,15 @@ ci_teardown() { rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}" rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}" fi + + # 4. This run's build context. Unlike the generated configs it is removed + # on a red run too: it holds the binaries and the Dockerfiles, both + # reproducible from the commit, so it is never the evidence of a + # failure. Guarded on the path having been derived at all, so an early + # exit cannot turn this into `rm -rf $SCRIPT_DIR/docker-`. + if [[ -n "${CI_BUILD_CONTEXT:-}" && "$CI_BUILD_CONTEXT" != "$SCRIPT_DIR/docker-" ]]; then + rm -rf "$CI_BUILD_CONTEXT" + fi } on_signal() { @@ -871,22 +888,39 @@ run_tor_directory() { run_integration() { stage "Stage 3: Integration Tests" - # Install binaries to shared docker context + # Populate THIS run's build context, then install the binaries into it. + # Everything but the binaries is copied from the tracked context directory; + # the binaries are installed fresh, and a previous run's are deliberately + # not carried over, since inheriting them is the failure this scoping + # exists to prevent. + info "Preparing build context $CI_BUILD_CONTEXT" + rm -rf "$CI_BUILD_CONTEXT" + mkdir -p "$CI_BUILD_CONTEXT" || { record "docker-build" 1; return; } + local _f + for _f in "$SCRIPT_DIR"/docker/*; do + case "$(basename "$_f")" in + fips|fipsctl|fipstop|fips-gateway) continue ;; + esac + cp -a "$_f" "$CI_BUILD_CONTEXT/" || { record "docker-build" 1; return; } + done + info "Installing release binaries" - install_binaries testing/docker + install_binaries "$CI_BUILD_CONTEXT" # Build unified test image once (used by all harnesses). Tag per-run # (fips-test:${run}) so a build killed mid-flight never wedges the next # run's rebuild, and concurrent runs never clobber each other's image. - # Then retag :latest for the compose files / harness scripts that still - # reference fips-test:latest directly; the retag happens only after BOTH - # builds succeed, so :latest never points at a half-built image. info "Building $CI_IMAGE_TEST Docker image" - docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" testing/docker --quiet \ + docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build" 1; return; } docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ - -f testing/docker/Dockerfile.app testing/docker --quiet \ + -f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build-app" 1; return; } + # The remaining bridge back to the shared mutable tag, for any consumer not + # yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the + # step that makes a missed consumer fail loudly instead of silently + # resolving another run's binaries. Both builds have succeeded by here, so + # :latest never points at a half-built image. docker tag "$CI_IMAGE_TEST" fips-test:latest docker tag "$CI_IMAGE_APP" fips-test-app:latest diff --git a/testing/firewall/docker-compose.yml b/testing/firewall/docker-compose.yml index 9cedf41..fe65773 100644 --- a/testing/firewall/docker-compose.yml +++ b/testing/firewall/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/sidecar/scripts/test-sidecar.sh b/testing/sidecar/scripts/test-sidecar.sh index 2cbd114..975973f 100755 --- a/testing/sidecar/scripts/test-sidecar.sh +++ b/testing/sidecar/scripts/test-sidecar.sh @@ -115,7 +115,7 @@ trap cleanup EXIT if [[ "${1:-}" != "--skip-build" ]]; then log "Building test images..." - DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)" + DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$(cd "$SIDECAR_DIR/../docker" && pwd)}" docker build -t fips-test:latest "$DOCKER_DIR" docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR" fi