From af847c68b55b2d48a04721284a9be9b6f9e6bbe8 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:40:03 +0000 Subject: [PATCH] Stop the chaos simulation rebuilding the shared test image per scenario The runner rebuilt the shared tag from the shared build context at the start of every scenario, unconditionally. Local CI runs chaos scenarios in parallel, so that was several concurrent builds into one name inside a single run, before any second run is considered, and it silently replaced whatever image the harness had already built and handed over. It now uses the caller's image when one is named, and asserts the image is present rather than building a substitute, because under a harness a miss means something upstream is broken and building would hide it. A bare run still builds the shared tag exactly as before, reading the run's own build context when one is set. --- testing/chaos/sim/compose.py | 9 +++++++-- testing/chaos/sim/runner.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py index 3594ead..461212e 100644 --- a/testing/chaos/sim/compose.py +++ b/testing/chaos/sim/compose.py @@ -10,8 +10,13 @@ from .scenario import Scenario from .topology import SimTopology # Image name for the pre-built FIPS test image. -# The runner builds this once before starting containers. -FIPS_SIM_IMAGE = "fips-test:latest" +# +# A harness that has already built an image passes it in FIPS_TEST_IMAGE, and +# it is then the caller's image: the runner uses it and must not rebuild it. +# Unset means a bare run, where the shared tag is the right name and the runner +# still builds it. Read at import, which is safe because the simulation always +# starts as a child process with the environment already set. +FIPS_SIM_IMAGE = os.environ.get("FIPS_TEST_IMAGE", "fips-test:latest") # Jinja2 template for the compose file. # Uses a pre-built image instead of per-service build to support large topologies. diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index ee9c8c6..8f661ca 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -268,14 +268,34 @@ class SimRunner: ) log.info("Wrote %s", self.compose_file) - # 4. Build the test image once (avoids per-service build at scale) - log.info("Building Docker image...") + # 4. Obtain the test image (once, rather than per-service at scale). + # + # Building it is right for a bare run and wrong under a harness. When + # FIPS_TEST_IMAGE is set the image belongs to the caller, and every + # scenario of a parallel run would otherwise rebuild it into one shared + # name from one shared context — so assert it exists and fail loudly if + # it does not, rather than manufacture a substitute nobody asked for. from .compose import FIPS_SIM_IMAGE - docker_dir = os.path.join(os.path.dirname(__file__), "..", "..", "docker") - subprocess.run( - ["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir], - check=True, - ) + if os.environ.get("FIPS_TEST_IMAGE"): + log.info("Using caller-supplied image %s", FIPS_SIM_IMAGE) + probe = subprocess.run( + ["docker", "image", "inspect", FIPS_SIM_IMAGE], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + if probe.returncode != 0: + raise RuntimeError( + f"FIPS_TEST_IMAGE names {FIPS_SIM_IMAGE}, which is not present. " + "The harness that set it is expected to have built it." + ) + else: + log.info("Building Docker image...") + docker_dir = os.environ.get("FIPS_BUILD_CONTEXT") or os.path.join( + os.path.dirname(__file__), "..", "..", "docker" + ) + subprocess.run( + ["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir], + check=True, + ) # 5. Start containers log.info("Starting %d containers...", len(self.topology.nodes))