From aa53da061f40a7072b7e6ca15b30e208e944eff0 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 9 Mar 2026 03:14:58 +0000 Subject: [PATCH] Add local CI runner script and gitignore fipstop in test dirs --- testing/chaos/.gitignore | 1 + testing/ci-local.sh | 445 ++++++++++++++++++++++++++++++++++++++ testing/static/.gitignore | 1 + 3 files changed, 447 insertions(+) create mode 100755 testing/ci-local.sh diff --git a/testing/chaos/.gitignore b/testing/chaos/.gitignore index 8c71f36..642b69c 100644 --- a/testing/chaos/.gitignore +++ b/testing/chaos/.gitignore @@ -1,5 +1,6 @@ fips fipsctl +fipstop generated-configs __pycache__ sim-results diff --git a/testing/ci-local.sh b/testing/ci-local.sh new file mode 100755 index 0000000..dd2b0e4 --- /dev/null +++ b/testing/ci-local.sh @@ -0,0 +1,445 @@ +#!/bin/bash +# Run the CI pipeline locally: build, unit tests, integration tests. +# +# Usage: ./ci-local.sh [options] +# +# Options: +# --build-only Only run build + clippy +# --test-only Only run unit tests (skip build, skip integration) +# --skip-integration Skip integration tests +# --skip-chaos Skip chaos scenarios (run static + rekey + sidecar only) +# --only Run a single integration suite +# -j, --jobs Max parallel chaos scenarios (default: 4) +# --list List available integration suites +# -h, --help Show this help +# +# Integration suites: +# static-mesh, static-chain, rekey, +# chaos-smoke-10, chaos-10, ethernet-mesh, ethernet-only, +# bottleneck-parent, cost-avoidance, cost-mixed-7node, +# cost-reeval, cost-stability, depth-vs-cost, mixed-technology, +# sidecar +# +# Exit codes: +# 0 — all stages passed +# 1 — one or more stages failed +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [[ ! -f "$PROJECT_ROOT/Cargo.toml" ]]; then + echo "Error: Cannot find Cargo.toml at $PROJECT_ROOT" >&2 + exit 1 +fi + +cd "$PROJECT_ROOT" || exit 1 + +# ── Configuration ────────────────────────────────────────────────────────── + +PARALLEL_JOBS=4 +BUILD_ONLY=false +TEST_ONLY=false +SKIP_INTEGRATION=false +SKIP_CHAOS=false +ONLY_SUITE="" + +# All integration suites matching ci.yml +STATIC_SUITES=(static-mesh static-chain) +REKEY_SUITES=(rekey) +CHAOS_SUITES=( + chaos-smoke-10 chaos-10 + ethernet-mesh ethernet-only + bottleneck-parent cost-avoidance cost-mixed-7node + cost-reeval cost-stability depth-vs-cost mixed-technology +) +SIDECAR_SUITES=(sidecar) + +# ── Colors ───────────────────────────────────────────────────────────────── + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +RESET='\033[0m' + +# ── Helpers ──────────────────────────────────────────────────────────────── + +stamp() { date '+%H:%M:%S'; } + +info() { echo -e "${CYAN}[$(stamp)]${RESET} $*"; } +pass() { echo -e "${GREEN}[$(stamp)] PASS${RESET} $*"; } +fail() { echo -e "${RED}[$(stamp)] FAIL${RESET} $*"; } +stage() { echo -e "\n${BOLD}${YELLOW}═══ $* ═══${RESET}\n"; } + +list_suites() { + echo "Available integration suites:" + echo "" + echo " Static topologies:" + for s in "${STATIC_SUITES[@]}"; do echo " $s"; done + echo "" + echo " Rekey:" + for s in "${REKEY_SUITES[@]}"; do echo " $s"; done + echo "" + echo " Chaos scenarios:" + for s in "${CHAOS_SUITES[@]}"; do echo " $s"; done + echo "" + echo " Sidecar:" + for s in "${SIDECAR_SUITES[@]}"; do echo " $s"; done + exit 0 +} + +usage() { + sed -n '2,/^$/{ s/^# \?//; p }' "$0" + exit 0 +} + +# ── Parse arguments ──────────────────────────────────────────────────────── + +while [[ $# -gt 0 ]]; do + case "$1" in + --build-only) BUILD_ONLY=true; shift ;; + --test-only) TEST_ONLY=true; shift ;; + --skip-integration) SKIP_INTEGRATION=true; shift ;; + --skip-chaos) SKIP_CHAOS=true; shift ;; + --only) ONLY_SUITE="$2"; shift 2 ;; + -j|--jobs) PARALLEL_JOBS="$2"; shift 2 ;; + --list) list_suites ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +# ── Results tracking ────────────────────────────────────────────────────── + +declare -A RESULTS +OVERALL=0 + +record() { + local name="$1" rc="$2" + RESULTS["$name"]=$rc + if [[ $rc -ne 0 ]]; then + OVERALL=1 + fail "$name" + else + pass "$name" + fi +} + +# ── Stage 1: Build ───────────────────────────────────────────────────────── + +run_build() { + stage "Stage 1: Build" + + info "cargo build --release" + if cargo build --release 2>&1; then + record "build" 0 + else + record "build" 1 + return 1 + fi + + info "cargo clippy --all -- -D warnings" + if cargo clippy --all -- -D warnings 2>&1; then + record "clippy" 0 + else + record "clippy" 1 + return 1 + fi +} + +# ── Stage 2: Unit Tests ─────────────────────────────────────────────────── + +run_tests() { + stage "Stage 2: Unit Tests" + + local cmd + if command -v cargo-nextest &>/dev/null; then + cmd="cargo nextest run --all" + info "$cmd" + if $cmd 2>&1; then + record "unit-tests" 0 + else + record "unit-tests" 1 + fi + else + cmd="cargo test --all" + info "$cmd (nextest not found, using cargo test)" + if $cmd 2>&1; then + record "unit-tests" 0 + else + record "unit-tests" 1 + fi + fi +} + +# ── Stage 3: Integration Tests ───────────────────────────────────────────── + +# Copy release binaries into a testing subdirectory +install_binaries() { + local dest="$1" + cp target/release/fips "$dest/fips" + cp target/release/fipsctl "$dest/fipsctl" + [[ -f target/release/fipstop ]] && cp target/release/fipstop "$dest/fipstop" || true + chmod +x "$dest/fips" "$dest/fipsctl" + [[ -f "$dest/fipstop" ]] && chmod +x "$dest/fipstop" || true +} + +# Run a static topology test (mesh, chain) +run_static() { + local topology="$1" + local compose="testing/static/docker-compose.yml" + local rc=0 + + info "[$topology] Generating configs" + bash testing/static/scripts/generate-configs.sh "$topology" || { record "static-$topology" 1; return; } + + info "[$topology] Building Docker images" + docker compose -f "$compose" --profile "$topology" build --quiet || { record "static-$topology" 1; return; } + + info "[$topology] Starting containers" + docker compose -f "$compose" --profile "$topology" up -d || { record "static-$topology" 1; return; } + + info "[$topology] Running ping test" + if bash testing/static/scripts/ping-test.sh "$topology"; then + rc=0 + else + rc=1 + info "[$topology] Collecting failure logs" + docker compose -f "$compose" --profile "$topology" logs --no-color 2>&1 | tail -100 + fi + + docker compose -f "$compose" --profile "$topology" down --volumes --remove-orphans 2>/dev/null + record "static-$topology" $rc +} + +# Run the rekey integration test +run_rekey() { + local compose="testing/static/docker-compose.yml" + local rc=0 + + info "[rekey] Generating configs" + bash testing/static/scripts/generate-configs.sh rekey || { record "rekey" 1; return; } + bash testing/static/scripts/rekey-test.sh inject-config || { record "rekey" 1; return; } + + info "[rekey] Building Docker images" + docker compose -f "$compose" --profile rekey build --quiet || { record "rekey" 1; return; } + + info "[rekey] Starting containers" + docker compose -f "$compose" --profile rekey up -d || { record "rekey" 1; return; } + + info "[rekey] Running rekey test" + if bash testing/static/scripts/rekey-test.sh; then + rc=0 + else + rc=1 + info "[rekey] Collecting failure logs" + docker compose -f "$compose" --profile rekey logs --no-color 2>&1 | tail -100 + fi + + docker compose -f "$compose" --profile rekey down --volumes --remove-orphans 2>/dev/null + record "rekey" $rc +} + +# Run a chaos scenario +run_chaos() { + local scenario="$1" + local rc=0 + + info "[chaos/$scenario] Running simulation" + if bash testing/chaos/scripts/chaos.sh "$scenario" 2>&1; then + rc=0 + else + rc=1 + fi + + record "chaos-$scenario" $rc +} + +# Run sidecar test +run_sidecar() { + local rc=0 + + info "[sidecar] Running integration test" + if bash testing/sidecar/scripts/test-sidecar.sh 2>&1; then + rc=0 + else + rc=1 + fi + + record "sidecar" $rc +} + +# Determine which suites to run and execute them +run_integration() { + stage "Stage 3: Integration Tests" + + # Install binaries to test directories + info "Installing release binaries to test directories" + install_binaries testing/static + install_binaries testing/chaos + install_binaries testing/sidecar + + # Build chaos Docker image once (shared by all chaos scenarios) + local need_chaos=false + if [[ -z "$ONLY_SUITE" && "$SKIP_CHAOS" != true ]]; then + need_chaos=true + elif [[ "$ONLY_SUITE" == chaos-* ]]; then + need_chaos=true + fi + + if [[ "$need_chaos" == true ]]; then + info "Building chaos Docker image" + docker build -t fips-chaos:latest testing/chaos --quiet || { record "chaos-build" 1; return; } + fi + + # Single suite mode + if [[ -n "$ONLY_SUITE" ]]; then + run_suite "$ONLY_SUITE" + return + fi + + # Static topologies (sequential — they share the docker-compose) + for topo in "${STATIC_SUITES[@]}"; do + local topology="${topo#static-}" + run_static "$topology" + done + + # Rekey + run_rekey + + # Chaos scenarios (parallel, throttled) + if [[ "$SKIP_CHAOS" != true ]]; then + info "Running ${#CHAOS_SUITES[@]} chaos scenarios (max $PARALLEL_JOBS parallel)" + local pids=() + local suite_names=() + local running=0 + + for suite in "${CHAOS_SUITES[@]}"; do + local scenario="${suite#chaos-}" + + # Throttle: wait for a slot + while [[ $running -ge $PARALLEL_JOBS ]]; do + wait -n -p done_pid 2>/dev/null || true + running=$((running - 1)) + done + + # Run in background, capture output to temp file + local logfile + logfile=$(mktemp "/tmp/ci-chaos-${scenario}.XXXXXX") + ( + run_chaos "$scenario" >"$logfile" 2>&1 + ) & + pids+=($!) + suite_names+=("$scenario:$logfile") + running=$((running + 1)) + done + + # Wait for all and collect results + for i in "${!pids[@]}"; do + local pid="${pids[$i]}" + local entry="${suite_names[$i]}" + local scenario="${entry%%:*}" + local logfile="${entry#*:}" + + if wait "$pid" 2>/dev/null; then + record "chaos-$scenario" 0 + else + record "chaos-$scenario" 1 + # Show tail of failure log + echo "--- chaos-$scenario output (last 20 lines) ---" + tail -20 "$logfile" 2>/dev/null || true + echo "---" + fi + rm -f "$logfile" + done + fi + + # Sidecar + run_sidecar +} + +# Run a single named suite +run_suite() { + local suite="$1" + case "$suite" in + static-mesh|static-chain) + run_static "${suite#static-}" ;; + rekey) + run_rekey ;; + chaos-*) + run_chaos "${suite#chaos-}" ;; + sidecar) + run_sidecar ;; + *) + fail "Unknown suite: $suite" + record "$suite" 1 ;; + esac +} + +# ── Summary ──────────────────────────────────────────────────────────────── + +print_summary() { + stage "Summary" + + local passed=0 failed=0 total=0 + for name in $(echo "${!RESULTS[@]}" | tr ' ' '\n' | sort); do + local rc="${RESULTS[$name]}" + total=$((total + 1)) + if [[ $rc -eq 0 ]]; then + passed=$((passed + 1)) + echo -e " ${GREEN}✓${RESET} $name" + else + failed=$((failed + 1)) + echo -e " ${RED}✗${RESET} $name" + fi + done + + echo "" + echo -e " ${BOLD}Total: $total Passed: $passed Failed: $failed${RESET}" + echo "" + + if [[ $OVERALL -eq 0 ]]; then + echo -e " ${GREEN}${BOLD}ALL PASSED${RESET}" + else + echo -e " ${RED}${BOLD}FAILED${RESET}" + fi + echo "" +} + +# ── Main ─────────────────────────────────────────────────────────────────── + +main() { + local start_time=$SECONDS + + stage "FIPS Local CI" + info "Project root: $PROJECT_ROOT" + + if [[ "$TEST_ONLY" == true ]]; then + run_tests + elif [[ "$BUILD_ONLY" == true ]]; then + run_build + else + run_build + if [[ "${RESULTS[build]:-1}" -ne 0 ]]; then + fail "Build failed, skipping remaining stages" + else + run_tests + if [[ "$SKIP_INTEGRATION" != true ]]; then + run_integration + fi + fi + fi + + print_summary + + local elapsed=$(( SECONDS - start_time )) + local mins=$(( elapsed / 60 )) + local secs=$(( elapsed % 60 )) + info "Total time: ${mins}m ${secs}s" + + exit $OVERALL +} + +main diff --git a/testing/static/.gitignore b/testing/static/.gitignore index 99e5c83..6f7fff8 100644 --- a/testing/static/.gitignore +++ b/testing/static/.gitignore @@ -1,3 +1,4 @@ fips fipsctl +fipstop generated-configs