diff --git a/.github/workflows/aur-publish.yml b/.github/workflows/aur-publish.yml index 50eadd1b..d82f8192 100644 --- a/.github/workflows/aur-publish.yml +++ b/.github/workflows/aur-publish.yml @@ -39,10 +39,16 @@ jobs: - name: Install build and lint tooling run: | set -euo pipefail - pacman -Sy --noconfirm --needed base-devel namcap git curl + pacman -Sy --noconfirm --needed base-devel namcap git curl jq - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - name: Test the publish gate + # Fixture tests for the script the publish job below waits with. They + # run here, on every trigger, so a change to the gate is exercised + # before a release tag depends on it. + run: bash packaging/aur/test-await-package-runs.sh + - name: Resolve package version id: ver env: @@ -136,11 +142,21 @@ jobs: # or a manual dispatch (packaging-only republish with explicit tag + pkgrel). # Branch pushes and pull requests build+lint above but never reach this job. # Gated on aur-build so a package that fails to build/lint is never published. + # It also waits for every package-*.yml run on the tag to succeed before it + # pushes: the AUR must not point at a tag whose release assets are still + # uploading or failed, and once it does, withdrawing the tag breaks the AUR + # package (its b2sum pins the tag's source archive). # ─────────────────────────────────────────────────────────────────────────── aur-publish-fips: name: Publish fips to AUR needs: aur-build runs-on: ubuntu-latest + # Above the gate's own 60-minute budget, so the gate reports a timeout + # rather than the runner killing it. + timeout-minutes: 90 + permissions: + contents: read + actions: read if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'push' @@ -180,6 +196,26 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # The gate script comes from this workflow's own revision, not the tag: + # a dispatch republishing a tag cut before the gate existed would not + # find it in the tag's tree. The workflows it waits on still come from + # the tag's tree, which is what the tag push triggered. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + path: gate-src + sparse-checkout: packaging/aur + + - name: Wait for the tag's package workflows + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + SHA=$(git rev-parse 'HEAD^{commit}') + echo "Tag $TAG is at $SHA" + SHA="$SHA" WORKFLOW_DIR=.github/workflows \ + bash gate-src/packaging/aur/await-package-runs.sh + - name: Patch PKGBUILD with pkgver, pkgrel, conflicts, and b2sums env: TAG: ${{ steps.tag.outputs.tag }} diff --git a/packaging/aur/README.md b/packaging/aur/README.md index ed5a2991..8017349f 100644 --- a/packaging/aur/README.md +++ b/packaging/aur/README.md @@ -20,6 +20,8 @@ This directory contains Arch Linux packaging files for two AUR packages: | `fips-dns.service` | Symlink to `../debian/fips-dns.service` | | `build-aur.sh` | Local `fips-git` build plus namcap validation (run by `make aur`) | | `patch-pkgbuild.sh` | Rewrites `pkgver`, `pkgrel`, `conflicts`, `options`, and `b2sums` in the PKGBUILD at publish time | +| `await-package-runs.sh` | Holds the AUR publish until every `package-*.yml` run for the release tag has succeeded | +| `test-await-package-runs.sh` | Fixture tests for `await-package-runs.sh`, run by the `aur-build` job | Both PKGBUILDs reference files from `packaging/debian/` (service files) and `packaging/common/` (config files) at build time. These are pulled from the diff --git a/packaging/aur/await-package-runs.sh b/packaging/aur/await-package-runs.sh new file mode 100755 index 00000000..dc2fcecd --- /dev/null +++ b/packaging/aur/await-package-runs.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Wait until every package workflow run for a release tag has succeeded. +# +# The AUR publish job runs this before it pushes a new pkgver, so the AUR never +# points at a tag whose release assets are still uploading or failed to build. +# Each package-*.yml workflow uploads its assets from a `release` job inside +# the same run, so a successful tag run means that workflow's assets are up. +# +# The population is discovered from the checked-out tree rather than listed +# here: the package-*.yml files at the tag are the workflows the tag push +# triggered. Finding none is a failure, never a pass. +# +# A tag push and the branch push of the same commit each start a run with the +# same head_sha; only the tag run carries the tag name in head_branch, so runs +# are matched on both. If the tag was re-pushed at the same commit, the newest +# matching run decides. +# +# Required environment variables: +# GITHUB_REPOSITORY - owner/repo +# TAG - release tag (e.g. v0.5.1) +# SHA - the commit the tag points at +# GH_TOKEN - token with actions:read (read by gh; not checked here) +# Optional: +# WORKFLOW_DIR - where to discover package-*.yml (default .github/workflows) +# AWAIT_POLLS - number of polls before giving up (default 60) +# AWAIT_INTERVAL - seconds between polls (default 60) +# DISCOVER_ONLY - if set to 1, print the discovered workflows and exit +# +# Exit status: 0 when every discovered workflow has a successful tag run; +# 1 at once when a tag run concludes anything but success; 1 when the poll +# budget runs out with any workflow unobserved, not finished, or unreadable. + +set -euo pipefail + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${TAG:?TAG must be set}" +: "${SHA:?SHA must be set}" +WORKFLOW_DIR="${WORKFLOW_DIR:-.github/workflows}" +AWAIT_POLLS="${AWAIT_POLLS:-60}" +AWAIT_INTERVAL="${AWAIT_INTERVAL:-60}" + +# How to recover once the cause of a failure is fixed. A workflow_dispatch run +# of a package workflow is not a push run of the tag and never satisfies this +# gate; re-running the failed jobs of the tag's own run keeps it one. +RECOVERY="If a run failed: use \"Re-run failed jobs\" on the package workflow's run for $TAG \ +(a new workflow_dispatch run does not count), then re-run the AUR Publish job for $TAG." + +# Print the basenames of the package workflows found in WORKFLOW_DIR. +discover() { + local f + for f in "$WORKFLOW_DIR"/package-*.yml; do + [ -e "$f" ] && basename "$f" + done + return 0 +} + +# Print " " for the newest push run of TAG at +# SHA for one workflow, or "none" if there is no such run yet. On an API or +# parse failure, print the error text and return nonzero. +latest() { + local wf=$1 body + if ! body=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/$wf/runs?head_sha=$SHA&event=push&per_page=100" 2>"$ERRS"); then + tr '\n' ' ' < "$ERRS" + return 1 + fi + printf '%s\n' "$body" | jq -er --arg tag "$TAG" --arg sha "$SHA" ' + [.workflow_runs[] + | select(.head_branch == $tag and .head_sha == $sha and .event == "push")] + | sort_by(.created_at) + | if length == 0 then "none" + else last | "\(.status) \(.conclusion // "none") \(.html_url)" end' 2>&1 +} + +mapfile -t pending < <(discover) +if [ "${#pending[@]}" -eq 0 ]; then + echo "No package-*.yml workflows found in $WORKFLOW_DIR; refusing to treat an empty set as done" >&2 + exit 1 +fi +if [ "${DISCOVER_ONLY:-}" = 1 ]; then + printf '%s\n' "${pending[@]}" + exit 0 +fi + +ERRS=$(mktemp) +trap 'rm -f "$ERRS"' EXIT + +echo "Waiting for $TAG ($SHA) runs of: ${pending[*]}" +declare -A seen +for ((poll = 1; poll <= AWAIT_POLLS; poll++)); do + still=() + for wf in "${pending[@]}"; do + if ! state=$(latest "$wf"); then + seen[$wf]="API error: $state" + still+=("$wf") + continue + fi + seen[$wf]=$state + read -r status conclusion url <<< "$state" + case "$status" in + none) + # The tag run has not been created yet. + still+=("$wf") ;; + completed) + if [ "$conclusion" = success ]; then + echo "$wf: succeeded ($url)" + else + echo "$wf: run for $TAG concluded '$conclusion' ($url); not publishing to the AUR" >&2 + echo "$RECOVERY" >&2 + exit 1 + fi ;; + *) + # queued, in_progress, waiting, requested or pending. + still+=("$wf") ;; + esac + done + pending=("${still[@]}") + if [ "${#pending[@]}" -eq 0 ]; then + echo "Every package workflow run for $TAG has succeeded" + exit 0 + fi + echo "Poll $poll/$AWAIT_POLLS: waiting on ${pending[*]}" + if [ "$poll" -lt "$AWAIT_POLLS" ]; then sleep "$AWAIT_INTERVAL"; fi +done + +echo "Gave up after $AWAIT_POLLS polls; not publishing to the AUR. Still unfinished:" >&2 +for wf in "${pending[@]}"; do + echo " $wf: ${seen[$wf]}" >&2 +done +echo "If a run is still going, re-run the AUR Publish job for $TAG once it has succeeded." >&2 +echo "$RECOVERY" >&2 +exit 1 diff --git a/packaging/aur/test-await-package-runs.sh b/packaging/aur/test-await-package-runs.sh new file mode 100755 index 00000000..42fa2813 --- /dev/null +++ b/packaging/aur/test-await-package-runs.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Fixture tests for await-package-runs.sh, the gate that holds the AUR publish +# until every package workflow run for the release tag has succeeded. +# +# Each case writes GitHub API responses into a fixture directory and puts a +# stub `gh` first on PATH. The stub serves ..json for the Nth +# call about a workflow, falls back to .json, and exits 1 when the +# fixture holds a file named `gh-fails`. It counts calls per workflow so a +# case can assert that the gate polled again rather than stopping early. +# +# Needs bash and jq. Exits nonzero if any case fails or if fewer cases ran +# than are defined. +# +# Usage: bash packaging/aur/test-await-package-runs.sh + +set -euo pipefail + +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +GATE="${GATE:-$HERE/await-package-runs.sh}" +REPO_ROOT=$(cd "$HERE/../.." && pwd) + +TAG=v9.9.9 +SHA=0123456789abcdef0123456789abcdef01234567 +WORKFLOWS=(package-freebsd.yml package-linux.yml package-macos.yml + package-openwrt.yml package-windows.yml) + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# The stub gh. It answers `gh api ` only, which is all the gate uses. +mkdir -p "$WORK/bin" +cat > "$WORK/bin/gh" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[ "${1:-}" = "api" ] || { echo "stub gh: unexpected args: $*" >&2; exit 2; } +[ -e "$STUB_FIXTURE/gh-fails" ] && { echo "stub gh: simulated API error" >&2; exit 1; } +wf=$(printf '%s\n' "$2" | sed -E 's|.*/actions/workflows/([^/]+)/runs.*|\1|') +count_file="$STUB_CALLS/$wf" +n=$(( $(cat "$count_file" 2>/dev/null || echo 0) + 1 )) +echo "$n" > "$count_file" +if [ -f "$STUB_FIXTURE/$wf.$n.json" ]; then + cat "$STUB_FIXTURE/$wf.$n.json" +elif [ -f "$STUB_FIXTURE/$wf.json" ]; then + cat "$STUB_FIXTURE/$wf.json" +else + echo '{"total_count":0,"workflow_runs":[]}' +fi +STUB +chmod +x "$WORK/bin/gh" + +# A workflow directory holding the five package workflows as empty files: the +# gate discovers the population by file name only. +mkdir -p "$WORK/workflows" "$WORK/empty-workflows" +for wf in "${WORKFLOWS[@]}"; do : > "$WORK/workflows/$wf"; done + +# Print one run object in the shape of the v0.5.1 API response. +# Args: head_branch status conclusion created_at id +run() { + local conclusion=null + [ "$3" = null ] || conclusion="\"$3\"" + printf '{"id":%s,"name":"Package","head_branch":"%s","head_sha":"%s","event":"push","status":"%s","conclusion":%s,"created_at":"%s","updated_at":"%s","html_url":"https://github.com/example/fips/actions/runs/%s"}' \ + "$5" "$1" "$SHA" "$2" "$conclusion" "$4" "$4" "$5" +} + +# Wrap run objects, given as arguments in the order the API returns them, in +# the list response envelope. +runs() { + local IFS=, + printf '{"total_count":%d,"workflow_runs":[%s]}\n' "$#" "$*" +} + +# Write the healthy v0.5.1 shape for every workflow into a fixture: a tag run +# and a branch run of the same commit, newest first, both successful. +all_green() { + local dir=$1 wf + for wf in "${WORKFLOWS[@]}"; do + runs "$(run "$TAG" completed success 2026-09-06T20:31:48Z 2)" \ + "$(run maint completed success 2026-09-06T20:31:10Z 1)" > "$dir/$wf.json" + done +} + +CASES_DEFINED=0 +CASES_RAN=0 +FAILED=0 + +# Run the gate against a fixture and check its exit status and, when a +# pattern is given, that its output states the expected reason (so a red +# caused by a crash in the gate does not pass as the intended red). +# Args: name fixture-dir expect(zero|nonzero) [pattern] [workflow-dir] +# Sets LAST_OUT to the gate's combined output. +check() { + local name=$1 fixture=$2 expect=$3 pattern=${4:-} wfdir=${5:-$WORK/workflows} rc=0 + CASES_RAN=$((CASES_RAN + 1)) + rm -rf "$WORK/calls"; mkdir -p "$WORK/calls" + LAST_OUT=$(PATH="$WORK/bin:$PATH" STUB_FIXTURE="$fixture" STUB_CALLS="$WORK/calls" \ + GITHUB_REPOSITORY=example/fips TAG="$TAG" SHA="$SHA" WORKFLOW_DIR="$wfdir" \ + AWAIT_POLLS=3 AWAIT_INTERVAL=0 bash "$GATE" 2>&1) || rc=$? + if { [ "$expect" = zero ] && [ "$rc" -eq 0 ]; } || + { [ "$expect" = nonzero ] && [ "$rc" -ne 0 ]; }; then + if [ -z "$pattern" ] || printf '%s\n' "$LAST_OUT" | grep -qE -- "$pattern"; then + echo "PASS $name (exit $rc)" + return 0 + fi + echo "FAIL $name: exit $rc as expected, but output lacks /$pattern/" + else + echo "FAIL $name: expected $expect exit, got $rc" + fi + printf '%s\n' "$LAST_OUT" | sed 's/^/ /' + FAILED=$((FAILED + 1)) + return 1 +} + +# Record an extra assertion's failure against the case that just ran. +fail() { + echo "FAIL $1" + FAILED=$((FAILED + 1)) +} + +# Make a fresh fixture directory for a case and print its path. +fixture() { + local dir="$WORK/fx/$1" + mkdir -p "$dir" + echo "$dir" +} + +# --- cases ------------------------------------------------------------------- + +# 1: every package workflow has a successful tag run. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 1); all_green "$fx" +check "1 all tag runs succeeded" "$fx" zero "has succeeded$" || true + +# 2: one tag run failed; the gate must name that workflow. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 2); all_green "$fx" +runs "$(run "$TAG" completed failure 2026-09-06T20:31:48Z 2)" \ + "$(run maint completed success 2026-09-06T20:31:10Z 1)" > "$fx/package-macos.yml.json" +if check "2 one tag run failed" "$fx" nonzero "concluded 'failure'"; then + printf '%s\n' "$LAST_OUT" | grep -q 'package-macos.yml' || + fail "2 one tag run failed: output does not name package-macos.yml" +fi + +# 3: one tag run was cancelled. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 3); all_green "$fx" +runs "$(run "$TAG" completed cancelled 2026-09-06T20:31:48Z 2)" > "$fx/package-openwrt.yml.json" +check "3 one tag run cancelled" "$fx" nonzero "package-openwrt.yml: run for .* concluded 'cancelled'" || true + +# 4: one workflow has only the branch run of the tag's commit (the v0.5.1 +# shape a SHA-only filter would accept). +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 4); all_green "$fx" +runs "$(run maint completed success 2026-09-06T20:31:10Z 1)" > "$fx/package-freebsd.yml.json" +check "4 only a branch run, no tag run" "$fx" nonzero "package-freebsd.yml: none$" || true + +# 5: one tag run is in progress on the first poll and succeeds on the second. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 5); all_green "$fx" +runs "$(run "$TAG" in_progress null 2026-09-06T20:31:48Z 2)" > "$fx/package-freebsd.yml.1.json" +if check "5 in progress, then succeeded" "$fx" zero "has succeeded$"; then + calls=$(cat "$WORK/calls/package-freebsd.yml" 2>/dev/null || echo 0) + [ "$calls" -ge 2 ] || + fail "5 in progress, then succeeded: gate queried package-freebsd.yml $calls time(s), expected at least 2" +fi + +# 6: one tag run stays in progress for the whole budget. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 6); all_green "$fx" +runs "$(run "$TAG" in_progress null 2026-09-06T20:31:48Z 2)" > "$fx/package-freebsd.yml.json" +check "6 in progress for the whole budget" "$fx" nonzero "package-freebsd.yml: in_progress " || true + +# 7: no package workflows discovered. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 7); all_green "$fx" +check "7 empty workflow population" "$fx" nonzero "No package-\*\.yml workflows found" \ + "$WORK/empty-workflows" || true + +# 8a-8d: the tag was re-pushed at the same commit, so two tag runs exist. The +# newest decides. The API lists newest first; 8b and 8c list oldest first so +# that "take the first match" and "take the newest" disagree. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 8a); all_green "$fx" +runs "$(run "$TAG" completed success 2026-09-06T21:00:00Z 3)" \ + "$(run "$TAG" completed failure 2026-09-06T20:31:48Z 2)" > "$fx/package-linux.yml.json" +check "8a older failure, newer success, newest first" "$fx" zero "has succeeded$" || true + +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 8b); all_green "$fx" +runs "$(run "$TAG" completed success 2026-09-06T20:31:48Z 2)" \ + "$(run "$TAG" completed failure 2026-09-06T21:00:00Z 3)" > "$fx/package-linux.yml.json" +check "8b older success, newer failure, oldest first" "$fx" nonzero \ + "package-linux.yml: run for .* concluded 'failure'" || true + +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 8c); all_green "$fx" +runs "$(run "$TAG" completed failure 2026-09-06T20:31:48Z 2)" \ + "$(run "$TAG" completed success 2026-09-06T21:00:00Z 3)" > "$fx/package-linux.yml.json" +check "8c older failure, newer success, oldest first" "$fx" zero "has succeeded$" || true + +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 8d); all_green "$fx" +runs "$(run "$TAG" completed failure 2026-09-06T21:00:00Z 3)" \ + "$(run "$TAG" completed success 2026-09-06T20:31:48Z 2)" > "$fx/package-linux.yml.json" +check "8d older success, newer failure, newest first" "$fx" nonzero \ + "package-linux.yml: run for .* concluded 'failure'" || true + +# 9: every API call fails. +CASES_DEFINED=$((CASES_DEFINED + 1)) +fx=$(fixture 9); all_green "$fx"; : > "$fx/gh-fails" +check "9 gh fails on every call" "$fx" nonzero "package-linux.yml: API error" || true + +# 10: discovery against the repository's real workflow directory. +CASES_DEFINED=$((CASES_DEFINED + 1)) +CASES_RAN=$((CASES_RAN + 1)) +rc=0 +found=$(DISCOVER_ONLY=1 WORKFLOW_DIR="$REPO_ROOT/.github/workflows" \ + GITHUB_REPOSITORY=example/fips TAG="$TAG" SHA="$SHA" bash "$GATE" 2>&1) || rc=$? +if [ "$rc" -eq 0 ] && [ -n "$found" ] && + printf '%s\n' "$found" | grep -qx 'package-linux.yml'; then + echo "PASS 10 real workflow discovery: $(printf '%s\n' "$found" | tr '\n' ' ')" +else + echo "FAIL 10 real workflow discovery: exit $rc, found: $found" + FAILED=$((FAILED + 1)) +fi + +# ----------------------------------------------------------------------------- + +echo "cases defined: $CASES_DEFINED, ran: $CASES_RAN, failed: $FAILED" +if [ "$CASES_RAN" -ne "$CASES_DEFINED" ]; then + echo "FAIL: not every defined case ran" + exit 1 +fi +[ "$FAILED" -eq 0 ]