diff --git a/.github/workflows/package-freebsd.yml b/.github/workflows/package-freebsd.yml index 4541657c..222c0f83 100644 --- a/.github/workflows/package-freebsd.yml +++ b/.github/workflows/package-freebsd.yml @@ -229,19 +229,22 @@ jobs: pfsense: name: Build and check the pfSense package (x86_64) - # A job of its own, and deliberately NOT a dependency of `release`. A - # pfSense-only failure — a new key in the common dns: block, a - # dependency that stops linking statically, a checker regression — reds - # this check and nothing else; it can never hide behind the FreeBSD - # job's result, and it cannot block the FreeBSD release asset. The - # pfSense package is therefore never a release asset. It is published - # here as a workflow artifact (30-day retention) for anyone to test, - # until someone has installed it on a real pfSense box; see - # packaging/pfsense/README.md. + # A job of its own, so a pfSense-only failure — a new key in the common + # dns: block, a dependency that stops linking statically, a checker + # regression — reds this check by name and can never hide behind the + # FreeBSD job's result. `release` needs both jobs: the packages this + # job builds are release assets next to the FreeBSD one, after being run + # on pfSense Plus 26.03.1 and 26.07 (see packaging/pfsense/README.md), + # so a pfSense failure holds the release the way a FreeBSD failure + # does. On every other ref the artifact is kept 30 days for anyone to + # test. # - # This VM is FreeBSD 15.1, so the package it produces is FreeBSD:15:amd64 - # (pfSense CE 2.8.1). CE 2.9 and Plus 26.x are FreeBSD 16 and need a - # FreeBSD 16 host this workflow does not have. No aarch64 package is built + # This VM is FreeBSD 15.1. One build yields static FreeBSD 15 binaries, + # packaged twice: as FreeBSD:15:amd64 for pfSense CE 2.8.1, and relabelled + # as FreeBSD:16:amd64 for CE 2.9 and Plus 26.x on Intel. Older binaries on + # a newer kernel is the direction FreeBSD's binary compatibility supports; + # the relabelled package has been run on Plus 26.03.1 and 26.07 (see + # packaging/pfsense/README.md). No aarch64 package is built # here or published anywhere: rustup ships no toolchain for # aarch64-unknown-freebsd, so such a build cannot honour the # rust-toolchain.toml pin every published artifact is built with. ARM is @@ -260,6 +263,16 @@ jobs: - name: Set SOURCE_DATE_EPOCH from git run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV" + - name: Lint the install smoke test + # Same arrangement as package-openwrt.yml's install-nak.sh lint: the + # script does not ship in the package, so the guards for shipped sh + # scripts do not apply. It is plain sh because pfSense has no bash. + run: | + if ! command -v shellcheck >/dev/null 2>&1; then + sudo apt-get install -y --no-install-recommends shellcheck + fi + shellcheck --shell=sh testing/pfsense-install-smoke.sh + - name: Build and check the pfSense package in a FreeBSD VM uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1 env: @@ -285,14 +298,29 @@ jobs: # them; on a FreeBSD 15.1 VM this yields the FreeBSD:15:amd64 # package for pfSense CE 2.8.1. packaging/pfsense/build-pkg.sh --version "$FREEBSD_PACKAGE_VERSION" + PKG15=$(ls deploy/fips-*-pfsense-ce2.8-amd64.pkg) - PKG=$(ls deploy/fips-*-pfsense-*.pkg) - testing/check-pfsense-pkg.sh "$PKG" + # The same binaries again, labelled for FreeBSD 16 (pfSense CE 2.9 + # and Plus 26.x on Intel). No FreeBSD 16 host is needed for that: + # static binaries from 15.1 run on a 16 kernel, the direction + # FreeBSD supports, and pkg only checks the label. + packaging/pfsense/build-pkg.sh --no-build --abi FreeBSD:16:amd64 \ + --version "$FREEBSD_PACKAGE_VERSION" + PKG16=$(ls deploy/fips-*-pfsense-ce2.9-plus26-amd64.pkg) + + for PKG in "$PKG15" "$PKG16"; do + testing/check-pfsense-pkg.sh "$PKG" + ( cd deploy && sha256 -q "$(basename "$PKG")" \ + | { read -r h; printf '%s %s\n' "$h" "$(basename "$PKG")"; } \ + > "$(basename "$PKG").sha256" ) + done php -l packaging/pfsense/fips-unbound-custom.php - - ( cd deploy && sha256 -q "$(basename "$PKG")" \ - | { read -r h; printf '%s %s\n' "$h" "$(basename "$PKG")"; } \ - > "$(basename "$PKG").sha256" ) + # Install the FreeBSD 15 package and run the daemon through the + # boot script's life on this FreeBSD 15 kernel; see the script + # header for what that does and does not prove about pfSense + # itself. pkg refuses the FreeBSD 16 package on this host (ABI + # major mismatch); its binaries are the same bytes. + testing/pfsense-install-smoke.sh "$PKG15" rm -rf target @@ -303,27 +331,33 @@ jobs: : ${GITHUB_OUTPUT:=/tmp/github_output} set -euo pipefail VER="${{ needs.determine-versioning.outputs.freebsd_pkg_file_version }}" - PKG=$(ls deploy/fips-${VER}-pfsense-*.pkg) - if [[ ! -f "$PKG" ]]; then - echo "No pfSense package was produced" >&2 - ls -la deploy >&2 || true - exit 1 - fi - echo "pkg=$PKG" >> "$GITHUB_OUTPUT" + PKG15="deploy/fips-${VER}-pfsense-ce2.8-amd64.pkg" + PKG16="deploy/fips-${VER}-pfsense-ce2.9-plus26-amd64.pkg" + for p in "$PKG15" "$PKG16"; do + if [[ ! -f "$p" || ! -f "$p.sha256" ]]; then + echo "pfSense package or its checksum is missing: $p" >&2 + ls -la deploy >&2 || true + exit 1 + fi + done + echo "pkg15=$PKG15" >> "$GITHUB_OUTPUT" + echo "pkg16=$PKG16" >> "$GITHUB_OUTPUT" - name: Upload pfSense artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: fips_${{ needs.determine-versioning.outputs.freebsd_package_version }}_x86_64_pfsense path: | - ${{ steps.pfsense-asset.outputs.pkg }} - ${{ steps.pfsense-asset.outputs.pkg }}.sha256 + ${{ steps.pfsense-asset.outputs.pkg15 }} + ${{ steps.pfsense-asset.outputs.pkg15 }}.sha256 + ${{ steps.pfsense-asset.outputs.pkg16 }} + ${{ steps.pfsense-asset.outputs.pkg16 }}.sha256 retention-days: 30 release: name: Publish FreeBSD assets to GitHub Release runs-on: ubuntu-latest - needs: build + needs: [build, pfsense] if: startsWith(github.ref, 'refs/tags/') permissions: contents: write @@ -332,28 +366,36 @@ jobs: - name: Download FreeBSD artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - # Only the FreeBSD artifact. Without a pattern this action downloads - # every artifact in the run — including the pfSense package from the - # `pfsense` job, which is a workflow artifact by design and must never - # reach a release. `needs` only orders jobs; it does not scope this. + # One named pattern per job: without a pattern this action downloads + # every artifact in the run, and `needs` only orders jobs; it does + # not scope this. pattern: fips_*_x86_64_freebsd path: dist merge-multiple: true + - name: Download pfSense artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: fips_*_x86_64_pfsense + path: dist + merge-multiple: true + - name: Validate .pkg bytes before publishing shell: bash run: | set -euo pipefail cd dist - # The pfSense package is never a release asset (see the `pfsense` - # job). The download step is scoped to the FreeBSD artifact; this - # keeps the rule if that artifact is ever renamed or a new one added. - if compgen -G '*-pfsense-*' >/dev/null; then - echo "FAIL: a pfSense package reached the release stage; it must stay a workflow artifact:" >&2 - ls -la ./*-pfsense-* >&2 || true - exit 1 - fi + # Three packages are expected: the FreeBSD one and the two pfSense + # ones (each job wrote the sidecars inside its own VM). Missing one + # is a failure, not a smaller release. + for want in '*-freebsd-*.pkg' '*-pfsense-ce2.8-amd64.pkg' '*-pfsense-ce2.9-plus26-amd64.pkg'; do + if ! compgen -G "$want" >/dev/null; then + echo "FAIL: no package matching $want was downloaded" >&2 + ls -la . >&2 || true + exit 1 + fi + done pkgs=$(find . -maxdepth 1 -type f -name '*.pkg' | LC_ALL=C sort) if [[ -z "$pkgs" ]]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 508863ea..5213c829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,12 +263,19 @@ with v0.5.x or earlier peers. fault at `posix_spawn`. Mechanics shared with the FreeBSD builder live in `packaging/common/pkg-lib.sh`, which both source; the FreeBSD package is byte-identical before and after. The pfSense package is - built and checked in its own CI job and published as a workflow - artifact, not attached to a release, until it has been installed on a - real pfSense box; CI produces the CE 2.8.1 (`FreeBSD:15:amd64`) - package, while CE 2.9 and Plus 26.x on Intel need a FreeBSD 16 build - host the CI does not have, and ARM stays build-it-yourself because - rustup ships no toolchain for it. See `packaging/pfsense/README.md`. + built and checked in its own CI job, then installed in the same VM and + run through the boot script's start, re-entrant start, a forced + `newsyslog` rotation, restart, stop and `pkg delete` + (`testing/pfsense-install-smoke.sh`, also the first thing to run on a + real box), and attached to each release next to the FreeBSD package, + after being run on pfSense Plus 26.03.1 and 26.07 with mesh traffic + between them. CI produces both Intel packages from that one FreeBSD 15.1 build: + the CE 2.8.1 (`FreeBSD:15:amd64`) package, and the CE 2.9 / Plus 26.x + (`FreeBSD:16:amd64`) package as the same static binaries relabelled, + which is the direction FreeBSD's binary compatibility supports and has + been run on pfSense Plus 26.03.1 and 26.07. ARM stays + build-it-yourself because rustup ships no toolchain for it. See + `packaging/pfsense/README.md`. ### Changed diff --git a/README.md b/README.md index 9933f558..6d4d0fd6 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,14 @@ and a ❌ there means the platform ships none. Windows is the odd one: its ZIP is an archive you unpack yourself rather than a package an installer consumes, and there is no MSI. +pfSense has no column of its own: it is the FreeBSD package with +pfSense's boot script and DNS Resolver integration, under +[`packaging/pfsense/`](packaging/pfsense/). CI builds it for CE 2.8.1 +(FreeBSD 15) and for CE 2.9.0 and Plus 26.x on x86_64 (FreeBSD 16, the +same static binaries relabelled) and attaches both to each release next +to the FreeBSD package; ARM is build-it-yourself. What each package has +been run on is in that directory's README. + Five of these columns are Linux: Debian/Ubuntu, Arch, NixOS, OpenWrt and Android. Linux is not one target. Debian, Ubuntu, Arch and NixOS are the same glibc build, and what diff --git a/packaging/README.md b/packaging/README.md index 87f4f675..64045732 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -240,12 +240,16 @@ daemon dies the first time it shells out. ARM builds must pass `--dynamic`, and then `ldd` on the appliance is the check that the base drift is not real. -The build host's architecture and FreeBSD major must still match the -target's: pfSense CE 2.8.1 is FreeBSD 15 amd64; CE 2.9.0 and Plus 26.x are -FreeBSD 16 (amd64, plus aarch64 for Plus on ARM appliances), and `pkg` refuses a -mismatched ABI. No aarch64 package is published: rustup ships no -toolchain for aarch64 FreeBSD, so such a build cannot honour the -`rust-toolchain.toml` pin. It is build-it-yourself. +The build host's architecture must match the target's, and `pkg` +refuses a mismatched ABI major, so the package carries the target's: +pfSense CE 2.8.1 is FreeBSD 15 amd64; CE 2.9.0 and Plus 26.x are +FreeBSD 16 (amd64, plus aarch64 for Plus on ARM appliances). The +FreeBSD 16 amd64 package is the FreeBSD 15.1 build relabelled +(`--no-build --abi FreeBSD:16:amd64`): static binaries from an older +release on a newer kernel is the direction FreeBSD supports, and it has +been run on Plus 26.03.1 and 26.07. No aarch64 package is published: +rustup ships no toolchain for aarch64 FreeBSD, so such a build cannot +honour the `rust-toolchain.toml` pin. It is build-it-yourself. ```sh # Build (on FreeBSD; this Makefile needs GNU make — pkg install gmake) diff --git a/packaging/pfsense/README.md b/packaging/pfsense/README.md index f8835c6e..a27449e3 100644 --- a/packaging/pfsense/README.md +++ b/packaging/pfsense/README.md @@ -62,13 +62,19 @@ The supported releases, from [Netgate's version table](https://docs.netgate.com/pfsense/en/latest/releases/versions.html) as of September 2026: -| Release | FreeBSD base | pkg ABI | Build host needed | +| Release | FreeBSD base | pkg ABI | Build host | |---|---|---|---| | pfSense CE 2.8.1 | 15.0-CURRENT | `FreeBSD:15:amd64` | FreeBSD 15, amd64 | -| pfSense CE 2.9.0 | 16.0-CURRENT | `FreeBSD:16:amd64` | FreeBSD 16, amd64 | -| pfSense Plus 26.03.1 / 26.07, Intel | 16.0-CURRENT | `FreeBSD:16:amd64` | FreeBSD 16, amd64 | +| pfSense CE 2.9.0 | 16.0-CURRENT | `FreeBSD:16:amd64` | FreeBSD 15 build, relabelled (see below) | +| pfSense Plus 26.03.1 / 26.07, Intel | 16.0-CURRENT | `FreeBSD:16:amd64` | FreeBSD 15 build, relabelled (see below) | | pfSense Plus 26.03.1 / 26.07, ARM | 16.0-CURRENT | `FreeBSD:16:aarch64` | FreeBSD 16, **aarch64** | +CE 2.8.1 can no longer be installed: the 2.8 line shipped only through +the Netgate installer, which offers the current release, and the public +mirror stops at the 2.7.2 ISOs. Its package serves existing 2.8.1 +installs and can only be tested on plain FreeBSD 15. Every pfSense a new +user can install runs FreeBSD 16. + CE has only ever shipped for amd64; Netgate has said there are no plans for an ARM CE image. Plus 24.x and 25.x are end-of-life and deliberately not in the build's table: a package named for an unsupported release @@ -112,24 +118,42 @@ difference decides which may be published: | Artifact | linkage | toolchain pin | CI | |---|---|---|---| -| `…-pfsense-ce2.8-amd64.pkg` | static | honoured | built + checked, workflow artifact | -| `…-pfsense-ce2.9-plus26-amd64.pkg` | static | honoured | not built — CI has no FreeBSD 16 host | +| `…-pfsense-ce2.8-amd64.pkg` | static | honoured | built, checked, install-smoked; release asset | +| `…-pfsense-ce2.9-plus26-amd64.pkg` | static | honoured | built (the CE 2.8 binaries, relabelled), checked; release asset | | `…-pfsense-plus26-aarch64.pkg` | dynamic | **not** honoured | not built — build it yourself | -No pfSense package is attached to a release. It is built and checked in -its own CI job (so a pfSense-only failure reds that job without blocking -the FreeBSD asset) and kept as a 30-day workflow artifact, until one has -been installed on a real pfSense box. +Both Intel packages are release assets: each tagged release carries +them next to the FreeBSD package, with their SHA-256 in +`checksums-freebsd.txt`. They are built and checked in their own CI job, +so a pfSense-only failure reds that job by name, and the release job +needs it, so such a failure holds the release rather than shipping +without them. On every other ref the job's output is a 30-day workflow +artifact for anyone to test. "Install-smoked" means +`testing/pfsense-install-smoke.sh` ran it on the plain FreeBSD VM of the +same major: `pkg add`, the boot script's start, re-entrant start, restart +and stop with the real daemon answering `fipsctl` and DNS queries, a +forced `newsyslog` rotation on the shipped entry with the daemon's open +log following to the new `/var/log/fips.log`, then `pkg delete`. That is +the same script to run first on a real box; its header says what a +plain-FreeBSD pass does not prove. -The two absences are not the same. The FreeBSD 16 Intel package builds -cleanly with the pinned compiler and links statically, so it is -releasable in principle and waits only on a FreeBSD 16 amd64 builder; -the CI VM is 15.1 and `vmactions/freebsd-vm` offers nothing newer, and -FreeBSD 16 is not released, so such a builder means a moving -16.0-CURRENT snapshot. Until then, CI builds and checks only the package -for the *older* supported CE release, as a workflow artifact. ARM cannot -honour the pin at all, so it -stays build-it-yourself regardless of infrastructure. +The FreeBSD 16 Intel package is the FreeBSD 15 package's binaries under +a `FreeBSD:16:amd64` label: the CI job builds once on FreeBSD 15.1 and +runs `build-pkg.sh --no-build --abi FreeBSD:16:amd64` for the second +package. That is the direction FreeBSD's binary compatibility runs, +older binaries on a newer kernel, so no FreeBSD 16 build host is needed +while 16 has no release (the CI VM is 15.1 and `vmactions/freebsd-vm` +offers nothing newer; a 16.0-CURRENT snapshot would also be *newer* +than any Netgate base, the direction that is not promised). The +relabelled package has been run on pfSense Plus 26.03.1 and 26.07; see +the test record at the end. What CI cannot do with it is `pkg add`, since +pkg refuses a package whose ABI major differs from the host's: the +install smoke runs on the FreeBSD 15 package, which carries the same +bytes, and the checker verifies the label against the binaries from the +outside. This holds as long as the code builds on 15.1 without needing +something only 16 provides; if that changes, a FreeBSD 16 build host is +needed again. ARM cannot honour the pin at all, so it stays +build-it-yourself regardless of infrastructure. ### There is no cross-compiling out of this @@ -204,18 +228,22 @@ annotations, and flags `pin_honoured: no` in its output. ### FreeBSD 16 is not released pfSense CE 2.9 and Plus 26.x are built from FreeBSD **16.0-CURRENT**, a development -branch; 16.0-RELEASE does not exist yet. So a FreeBSD 16 builder means a -[16.0-CURRENT snapshot](https://download.freebsd.org/snapshots/), not a -release image — and `vmactions/freebsd-vm`, which this repo's CI uses, -only goes up to 15.1. +branch; 16.0-RELEASE does not exist yet. A FreeBSD 16 build host would +therefore be a [16.0-CURRENT +snapshot](https://download.freebsd.org/snapshots/), not a release image, +and one that is months *newer* than any Netgate base (their releases +track a `main` commit from several months earlier). Binaries built there +would run on the appliance in the direction FreeBSD does not promise. +That is why the FreeBSD 16 amd64 package is not built on 16 at all but is +the FreeBSD 15.1 static build relabelled, which runs in the promised +direction and has been verified on Plus 26.03.1 and 26.07. -That makes base-library drift a real risk rather than a theoretical one: -Netgate's `16.0-CURRENT@` and a FreeBSD snapshot from another date -are different trees, and a binary can reference a symbol the appliance's -`libc` does not export. It installs and then fails to start. If `fips` -exits immediately with a linker error, that is this. Build from a -snapshot close to the appliance's base, and check what the binary -actually needs: +The drift is real for a `--dynamic` build, on either major: Netgate's +`16.0-CURRENT@` and a FreeBSD tree from another date are different +trees, and a binary can reference a symbol the appliance's `libc` does +not export. It installs and then fails to start. If `fips` exits +immediately with a linker error, that is this. Build from a base no newer +than the appliance's, and check what the binary actually needs: ```sh pkg info -F | grep -A5 "Shared Libs" # on the build host @@ -228,6 +256,7 @@ ldd /usr/local/bin/fips # on the appliance gmake -C packaging pfsense # or: ./packaging/pfsense/build-pkg.sh # cargo build --release + pkg create ./packaging/pfsense/build-pkg.sh --no-build # package existing release binaries +./packaging/pfsense/build-pkg.sh --no-build --abi FreeBSD:16:amd64 # the same binaries, labelled for FreeBSD 16 ./packaging/pfsense/build-pkg.sh --dynamic # link against libc.so.7 (see below) ``` @@ -252,8 +281,10 @@ appliance's, which is the direction that breaks: the binary references a versioned symbol the appliance does not export, installs cleanly, and then will not start. A dynamic package needs `libc.so.7`, `libm.so.5`, `libthr.so.3` and `libgcc_s.so.1` to agree with it; a static one -declares no shared libraries at all. What is left is the kernel syscall -ABI, which is stable within a FreeBSD major. +declares no shared libraries at all. What is left is the kernel's +binary compatibility, which FreeBSD promises in one direction only: +binaries from an older release run on a newer kernel. Build on a base no +newer than the appliance's. That is also why a static package survives a pfSense firmware upgrade's change of base, where a dynamic one is pinned to the image it was built @@ -485,8 +516,9 @@ pfctl -ss | grep tun # mesh state entries ``` `ifconfig ` prints `Opened by PID ` for the process holding -a tun device. The interface is destroyed automatically when the daemon -exits. +a tun device. After the daemon exits the interface stays listed, down, +without an address and with nobody holding it (observed on Plus 26.03.1 +and 26.07); the next start opens it again. ## What is and is not tested @@ -506,20 +538,49 @@ gate — nothing re-checks it when this code changes. Known still-unexercised paths, from that same run: `fips-dns-setup`'s refusal path (it has only ever run against a responder that was already -answering), its DNS Forwarder branch, and `pkg delete`. +answering) and its DNS Forwarder branch. (`fips-dns-teardown` has since been run on the same box and restored `custom_options` byte for byte.) -**No amd64 package has ever been installed.** The CE 2.8.1 (FreeBSD 15) -and the CE 2.9 / Plus 26.x (FreeBSD 16) amd64 packages are built and -pass the checker, and nothing more. The one hardware run was aarch64; -the amd64 packages share every script here and have had none of that -exposure, so read a passing check as "the package is well-formed", not -"it works". +**amd64 on pfSense.** The FreeBSD 16 amd64 package has been run on +pfSense Plus in KVM virtual machines installed with the Netgate +installer, so on Netgate's kernel. First a package built on the +16.0-CURRENT 20260907 snapshot, on Plus 26.07; that run found that +`fips-dns-setup` never restarted a running unbound. Then the package CI +now produces, the FreeBSD 15.1 build relabelled, on both Plus 26.03.1 +(`plus-RELENG_26_03_1-n256546-1d1bfd578383`, `kern.osreldate` 1600011) +and Plus 26.07 (`plus-RELENG_26_07-n256584-8183aef9d019`, 1600018), with +the same result on each: `testing/pfsense-install-smoke.sh` (45 checks), +the TUN interface up with its mesh address, the responder answering the +node's own name directly, `fips-dns-setup` writing the block and `.fips` +resolving through unbound once the resolver was restarted (that package +predates the fix that makes `fips-dns-setup` do it), `pfSctl -c 'service +reload packages'` leaving the running daemon alone, a reboot bringing up +exactly one daemon with the DNS Resolver block regenerated and resolving, +`fips-dns-teardown` leaving `custom_options` as it was, and `pkg delete` +against the running daemon. The two boxes were then peered with each +other over UDP (a pass-in rule on the tun interface loaded into pf's +`userrules` anchor, the "accept inbound deliberately" posture above): +the link authenticated, `ping6` across the mesh ran 200 packets of 56 +bytes and 100 of 1100 bytes each way with no loss, and 10 MiB by TCP each +way arrived byte-exact. Packets above the daemon's effective MTU (1203 +bytes over the 1280-byte UDP transport) are answered with ICMPv6 Packet +Too Big and TCP is MSS-clamped, as the no-fragmentation policy in +`docs/design/fips-mtu.md` says, so a fixed-size `ping6 -s 1160` or larger +shows loss by design on every platform. What the VMs did not cover: +physical hardware, and CE 2.9.0 (no installer at hand). +The CE 2.8.1 (FreeBSD 15) package carries the same binaries and is +install-smoked on plain FreeBSD 15.1 in CI; it cannot be run on pfSense +because CE 2.8.1 media no longer exists. -That matters because the aarch64 run found several defects, every one in -this packaging rather than the daemon — a boot script whose pid check -never succeeded, a DNS setup that reported success while nothing was -listening, and a static build that faulted at `posix_spawn`. The daemon -itself needed no changes. An untested path in the amd64 packages is -exactly where the next one would sit. +Left behind by `pkg delete`, by design or as known gaps: +`/usr/local/etc/fips/fips.key` if the daemon generated one (it may be the +node's identity), `/var/log/fips.log`, and the newsyslog entry under +`/var/etc`, which a RAM-disk `/var` drops at the next boot anyway. + +The hardware and VM runs found several defects, every one in this +packaging rather than the daemon — a boot script whose pid check never +succeeded, a DNS setup that reported success while nothing was +listening, a static build that faulted at `posix_spawn`, and the +resolver restart above. The daemon itself needed no changes. An untested +path is exactly where the next one would sit. diff --git a/packaging/pfsense/build-pkg.sh b/packaging/pfsense/build-pkg.sh index 165a9f3f..37efbbda 100755 --- a/packaging/pfsense/build-pkg.sh +++ b/packaging/pfsense/build-pkg.sh @@ -172,8 +172,12 @@ fi # appliance does not export, install cleanly, and then refuse to start. # # Linking statically removes the negotiation entirely — there is no -# libc.so.7 to disagree with. What is left is the kernel syscall ABI, -# which is stable within a FreeBSD major. +# libc.so.7 to disagree with. What is left is the kernel's binary +# compatibility, which FreeBSD promises in one direction only: binaries +# from an older release run on a newer kernel. So build on a base no +# newer than the appliance's, never on a snapshot ahead of it. That is +# why the FreeBSD 16 package is the 15.1 build relabelled (--no-build +# --abi FreeBSD:16:amd64) rather than a build on a 16.0-CURRENT host. # # Verified viable on this codebase: no dlopen/libloading anywhere, and # FreeBSD builds files+dns resolution into libc, so a static binary diff --git a/packaging/pfsense/fips-unbound-custom.php b/packaging/pfsense/fips-unbound-custom.php index 978d50d4..c01a292d 100644 --- a/packaging/pfsense/fips-unbound-custom.php +++ b/packaging/pfsense/fips-unbound-custom.php @@ -29,6 +29,7 @@ if (!getenv('FIPS_UNBOUND_HELPER_TEST')) { require_once("config.inc"); require_once("util.inc"); require_once("unbound.inc"); + require_once("services.inc"); } define('FIPS_BEGIN', '# BEGIN FIPS - managed by fips-dns-setup, do not edit this block'); @@ -218,9 +219,13 @@ if (!fips_resolver_enabled()) { } /* Regenerate /var/unbound/unbound.conf from config.xml and restart the - * resolver. Nothing shorter works: the file is generated wholesale, so - * a reload alone would re-read the config we have not rewritten yet. */ -sync_unbound_service(); + * resolver, the way the GUI's Apply does (services_unbound.php). It has to + * be this function: sync_unbound_service() regenerates the file too, but + * then only *starts* unbound, which is a no-op while one is running, so + * the running resolver kept its old forward set and answered NXDOMAIN for + * .fips (found on Plus 26.07 amd64). services_unbound_configure() TERMs + * the running instance first. */ +services_unbound_configure(); fips_log("DNS Resolver updated and restarted"); exit(0); diff --git a/testing/check-pfsense-pkg.sh b/testing/check-pfsense-pkg.sh index 943ef3db..bf18d115 100755 --- a/testing/check-pfsense-pkg.sh +++ b/testing/check-pfsense-pkg.sh @@ -527,6 +527,20 @@ fi # ── 6. PHP helper ─────────────────────────────────────────────────────────── echo "-- php helper" HELPER="$PAYLOAD/libexec/fips/fips-unbound-custom.php" +# After editing config.xml the helper must restart the running resolver the +# way the GUI's Apply does, services_unbound_configure(): it TERMs unbound +# and starts it on the regenerated unbound.conf. sync_unbound_service() +# regenerates the file too but then only *starts* unbound, a no-op while one +# is running, so the resolver kept its old forward set and answered NXDOMAIN +# for .fips while setup reported success (Plus 26.07 amd64). A static check, +# since the call needs pfSense's includes to run. +if grep -qE '^[^*/#]*services_unbound_configure\(' "$HELPER" \ + && ! grep -qE '^[^*/#]*sync_unbound_service\(' "$HELPER"; then + pass "helper restarts unbound with services_unbound_configure()" +else + fail "helper does not restart unbound with services_unbound_configure()" \ + "sync_unbound_service() leaves a running unbound on its old config." +fi if ! command -v php >/dev/null 2>&1; then skip "php -l and the fips_strip_block unit test (no php on this host)" else diff --git a/testing/pfsense-install-smoke.sh b/testing/pfsense-install-smoke.sh new file mode 100755 index 00000000..9b4c92b8 --- /dev/null +++ b/testing/pfsense-install-smoke.sh @@ -0,0 +1,273 @@ +#!/bin/sh +# ── pfSense package install smoke test ────────────────────────────────────── +# Installs a built pfSense .pkg on the FreeBSD system this runs on and drives +# it through the life it has on a firewall: pkg add, the boot script's start +# (twice, the second being what pfSense's rc.start_packages does on every WAN +# address change), the daemon answering on its control socket and its DNS +# port, a forced newsyslog rotation with the daemon following its log to the +# new inode, restart, stop, and pkg delete taking everything back out. +# +# check-pfsense-pkg.sh reads the package from the outside and exercises the +# boot script against a stub; this runs the real binary under the real +# daemon(8) on a real kernel of the target's major. The two cover different +# failures: a package can pass every structural check and still ship a +# daemon that dies at TUN setup, a post-install that leaves fips.yaml +# world-readable, or a pre-deinstall that lets pkg delete pull the binary +# out from under a running process. +# +# Runs on a plain FreeBSD host of the package's ABI major (CI: the 15.1 VM +# of the pfsense job), and on pfSense itself, where it is meant to be the +# first thing run against a new package. What it does NOT reach on plain FreeBSD, +# stated so a pass is not read as more than it is: the config.xml edit in +# fips-unbound-custom.php (needs pfSense's PHP includes), unbound forwarding +# .fips, /var as a RAM disk, and pf. On pfSense the DNS integration is still +# a manual step, because installing a test package must not rewrite the +# firewall's config.xml. +# +# It changes the system it runs on: installs and removes the package, starts +# and stops the daemon, creates a fips group. Run it on a throwaway VM or a +# box you are about to install on anyway, as root. Plain sh, because pfSense +# ships no bash and a test package should not need one installed to be tried. +# +# Usage: testing/pfsense-install-smoke.sh +# +# Exit 0 = every check passed. Exit 1 = at least one failed. Exit 2 = the +# test could not run; never treated as a pass. +# ───────────────────────────────────────────────────────────────────────────── +# daemon_answers, no_daemon and cleanup are called through wait_for and the +# EXIT trap, which shellcheck cannot follow: SC2317 in 0.9, SC2329 in 0.10+. +# shellcheck disable=SC2317,SC2329 +set -u + +failures=0 +checks=0 +skipped=0 + +pass() { checks=$((checks + 1)); printf ' PASS %s\n' "$1"; } +fail() { + checks=$((checks + 1)) + failures=$((failures + 1)) + printf ' FAIL %s\n' "$1" + [ $# -gt 1 ] && printf ' %s\n' "$2" + return 0 +} +skip() { skipped=$((skipped + 1)); printf ' SKIP %s\n' "$1"; } +bail() { printf 'pfsense-install-smoke: %s\n' "$1" >&2; exit 2; } + +PKG="${1:-}" +if [ -z "$PKG" ] || [ ! -f "$PKG" ]; then bail "usage: $0 "; fi +[ "$(id -u)" -eq 0 ] || bail "must run as root: it installs the package and starts the daemon" +if ! { command -v pkg >/dev/null 2>&1 && pkg config abi >/dev/null 2>&1; }; then + bail "needs pkg(8) on a FreeBSD-based host" +fi +pkg info fips >/dev/null 2>&1 && bail "fips is already installed here; this test needs a clean host" + +RC=/usr/local/etc/rc.d/fips.sh +CONF_DIR=/usr/local/etc/fips +RUN_DIR=/var/run/fips +LOG=/var/log/fips.log +NEWSYSLOG_ENTRY=/usr/local/etc/fips/fips.newsyslog +DNS_PORT=5354 + +# fipsctl and the daemon agree on the control socket under /var/run/fips +# once that directory exists; fips.sh creates it before starting the daemon. +daemon_answers() { fipsctl show status >/dev/null 2>&1; } +daemon_pids() { pgrep -x fips 2>/dev/null | wc -l | tr -d ' '; } +no_daemon() { [ "$(daemon_pids)" = "0" ]; } + +# Wait up to $1 seconds for a command to succeed. +wait_for() { + _seconds="$1"; shift + _i=0 + while [ "$_i" -lt "$_seconds" ]; do + "$@" && return 0 + sleep 1 + _i=$((_i + 1)) + done + return 1 +} + +# fips.sh start leaves daemon(8)'s supervisor behind, and the supervisor +# inherits whatever stdout it was given. Captured in a command substitution +# that pipe never closes and the substitution blocks for the daemon's whole +# life (fips.sh itself says so). So the boot script's output goes to a file +# and is read back afterwards. +OUT=$(mktemp "${TMPDIR:-/tmp}/fips-smoke.XXXXXX") || bail "mktemp failed" +run_rc() { + "$RC" "$@" >"$OUT" 2>&1 +} + +on_pfsense=0 +[ -f /etc/inc/config.inc ] && on_pfsense=1 + +# Leave nothing behind on a failure part-way: the daemon stopped and the +# package removed, so a re-run starts clean. Runs from the EXIT trap. +cleanup() { + if pkg info fips >/dev/null 2>&1; then + "$RC" stop >/dev/null 2>&1 || true + pkg delete -y fips >/dev/null 2>&1 || true + fi + rm -f "$OUT" +} +trap cleanup EXIT + +ABI=$(pkg config abi) +where="$ABI" +[ "$on_pfsense" = 1 ] && where="$ABI (pfSense)" +echo "==> install smoke test of $(basename "$PKG") on $where" + +# ── 1. Install ────────────────────────────────────────────────────────────── +echo "-- pkg add" +if out=$(pkg add "$PKG" 2>&1); then + pass "pkg add" +else + fail "pkg add" "$(printf '%s' "$out" | tail -3)" + echo "==> cannot continue without the package installed" >&2 + exit 1 +fi + +for bin in fips fipsctl fipstop; do + if [ -x "/usr/local/bin/$bin" ]; then pass "/usr/local/bin/$bin installed" + else fail "/usr/local/bin/$bin missing"; fi +done +# The boot script must carry the .sh suffix, or pfSense never runs it. +if [ -x "$RC" ]; then pass "boot script is $RC"; else fail "boot script $RC missing or not executable"; fi + +# post-install seeds the configs from the samples; fips.yaml may hold a +# private key, so it must not be world-readable. +for f in fips.yaml hosts fips.conf; do + if [ -f "$CONF_DIR/$f" ]; then pass "$CONF_DIR/$f seeded from its sample" + else fail "$CONF_DIR/$f was not seeded by post-install"; fi +done +mode=$(stat -f %Lp "$CONF_DIR/fips.yaml" 2>/dev/null || echo "?") +if [ "$mode" = "600" ]; then pass "fips.yaml is 0600"; else fail "fips.yaml mode is $mode, expected 600"; fi +if pw groupshow fips >/dev/null 2>&1; then pass "fips group exists"; else fail "post-install did not create the fips group"; fi +if grep -q 'bind_addr: "127.0.0.1"' "$CONF_DIR/fips.yaml"; then pass "shipped config binds the responder on 127.0.0.1" +else fail "shipped fips.yaml does not bind the DNS responder on 127.0.0.1"; fi + +# ── 2. Lifecycle ──────────────────────────────────────────────────────────── +echo "-- start" +if run_rc start; then pass "fips.sh start exits 0"; else fail "fips.sh start exited non-zero" "$(cat "$OUT")"; fi +if [ -s "$RUN_DIR/fips.pid" ]; then pass "pidfile written"; else fail "no pidfile at $RUN_DIR/fips.pid"; fi +if "$RC" status >/dev/null 2>&1; then pass "fips.sh status reports running"; else fail "fips.sh status says not running after start"; fi +if wait_for 30 daemon_answers; then + pass "daemon answers fipsctl show status" +else + fail "daemon did not answer on its control socket within 30s" "$(tail -n 5 "$LOG" 2>/dev/null)" +fi +n=$(daemon_pids) +if [ "$n" = "1" ]; then pass "exactly one fips process"; else fail "expected one fips process, found $n"; fi + +# The responder itself, before unbound. drill(1) is in FreeBSD base; any +# answer, including NXDOMAIN, shows the port is served by our daemon. +if command -v drill >/dev/null 2>&1; then + if wait_for 10 sh -c "drill -p $DNS_PORT smoke.fips @127.0.0.1 AAAA 2>/dev/null | grep -q '>>HEADER<<'"; then + pass "DNS responder answers on 127.0.0.1:$DNS_PORT" + else + fail "no DNS answer from 127.0.0.1:$DNS_PORT" "$(sockstat -4l 2>/dev/null | grep ":$DNS_PORT" || echo 'nothing listening on the port')" + fi +else + skip "DNS responder query (drill not installed)" +fi +if [ -s "$LOG" ]; then pass "daemon log $LOG is being written"; else fail "$LOG is empty or missing"; fi +if [ "$on_pfsense" = 1 ]; then + if [ -f /var/etc/newsyslog.conf.d/fips.conf ]; then pass "newsyslog entry placed for /var/etc" + else fail "start did not place /var/etc/newsyslog.conf.d/fips.conf"; fi +fi + +# Rotation. The shipped entry names the daemon(8) supervisor's pidfile, and +# the supervisor runs with -H, so the SIGHUP newsyslog sends after renaming +# the log must make it close the rotated inode and open the new +# /var/log/fips.log. Without that it keeps writing into the old inode, which +# the compression step unlinks: on a RAM-disk /var the space is never +# reclaimed and the log operators read stays empty. fips.sh copies the entry +# to /var/etc/newsyslog.conf.d on pfSense only, so drive newsyslog from the +# shipped file directly, which is the same text. +echo "-- log rotation" +sup_pid=$(cat "$RUN_DIR/daemon.pid" 2>/dev/null) +log_inode() { stat -f %i "$LOG" 2>/dev/null; } +# Inode numbers of the files a process holds open, from fstat's INUM column. +open_inodes() { fstat -p "$1" 2>/dev/null | awk 'NR > 1 && $6 ~ /^[0-9]+$/ { print $6 }'; } +holds_inode() { open_inodes "$1" | grep -qx "$2"; } +# The newest rotated generation, whichever suffix newsyslog gave it. +rotated_generation() { for _g in "$LOG".0*; do [ -e "$_g" ] && { echo "$_g"; return 0; }; done; return 1; } +# -F rotates regardless of size or age; -f reads the shipped entry alone. +force_rotation() { newsyslog -F -f "$NEWSYSLOG_ENTRY" 2>&1; } +before=$(log_inode) +if [ -n "$sup_pid" ] && [ -n "$before" ] && holds_inode "$sup_pid" "$before"; then + pass "supervisor (pid $sup_pid) holds $LOG open, inode $before" +else + fail "supervisor (pid ${sup_pid:-none}) does not hold $LOG open" "$(fstat -p "${sup_pid:-0}" 2>&1 | tail -n +2)" +fi +if [ -r "$NEWSYSLOG_ENTRY" ]; then + if out=$(force_rotation); then pass "newsyslog -F on the shipped entry exits 0" + else fail "newsyslog -F on the shipped entry failed" "$out"; fi + after=$(log_inode) + if [ -n "$after" ] && [ "$after" != "$before" ]; then pass "$LOG rotated: new inode $after" + else fail "$LOG was not rotated" "$(ls -li "$LOG"* 2>&1)"; fi + if gen=$(rotated_generation); then pass "rotated generation kept: $gen" + else fail "no rotated generation $LOG.0*" "$(ls -l "$LOG"* 2>&1)"; fi + if [ -n "$sup_pid" ] && wait_for 10 holds_inode "$sup_pid" "$after"; then + pass "supervisor reopened $LOG on SIGHUP, inode $after" + else + fail "supervisor did not reopen $LOG after rotation (holds $(open_inodes "${sup_pid:-0}" | paste -sd , -), wants $after)" + fi + if [ -n "$sup_pid" ] && holds_inode "$sup_pid" "$before"; then fail "supervisor still holds the rotated inode $before open" + else pass "rotated inode $before released"; fi + if wait_for 5 daemon_answers; then pass "daemon still answers after rotation"; else fail "daemon stopped answering after rotation"; fi +else + fail "shipped newsyslog entry $NEWSYSLOG_ENTRY is not installed" +fi + +# What rc.start_packages does on every WAN address change: start again +# while running. Must exit 0, say so, and not fork a second daemon. +echo "-- start while running" +if run_rc start; then pass "re-entrant start exits 0"; else fail "re-entrant start exited non-zero" "$(cat "$OUT")"; fi +if grep -q 'already running' "$OUT"; then pass "re-entrant start reports already running" +else fail "re-entrant start did not say 'already running'" "$(cat "$OUT")"; fi +n=$(daemon_pids) +if [ "$n" = "1" ]; then pass "still exactly one fips process"; else fail "re-entrant start left $n fips processes"; fi + +echo "-- restart" +old_pid=$(cat "$RUN_DIR/fips.pid" 2>/dev/null) +if run_rc restart; then pass "fips.sh restart exits 0"; else fail "fips.sh restart failed" "$(cat "$OUT")"; fi +new_pid=$(cat "$RUN_DIR/fips.pid" 2>/dev/null) +if [ -n "$new_pid" ] && [ "$new_pid" != "$old_pid" ]; then pass "restart produced a new pid ($old_pid -> $new_pid)" +else fail "restart did not replace the daemon (pid $old_pid -> ${new_pid:-none})"; fi +if wait_for 30 daemon_answers; then pass "daemon answers after restart"; else fail "daemon not answering after restart"; fi + +echo "-- stop" +if run_rc stop; then pass "fips.sh stop exits 0"; else fail "fips.sh stop failed" "$(cat "$OUT")"; fi +if wait_for 15 no_daemon; then pass "no fips process after stop" +else fail "fips still running after stop"; fi +if "$RC" status >/dev/null 2>&1; then fail "fips.sh status still reports running after stop"; else pass "fips.sh status reports not running"; fi +if [ ! -e "$RUN_DIR/fips.pid" ]; then pass "pidfile removed"; else fail "pidfile left behind after stop"; fi + +# ── 3. Remove ─────────────────────────────────────────────────────────────── +# pre-deinstall stops a running daemon first, so start it again and let pkg +# delete do the stopping. On plain FreeBSD fips-dns-teardown reports there +# is no config.xml to edit and pre-deinstall prints the hint; that is +# expected here and not a failure. +echo "-- pkg delete" +run_rc start || true +if out=$(pkg delete -y fips 2>&1); then pass "pkg delete"; else fail "pkg delete failed" "$(printf '%s' "$out" | tail -3)"; fi +if wait_for 15 no_daemon; then pass "pre-deinstall stopped the running daemon" +else fail "daemon still running after pkg delete"; fi +for bin in fips fipsctl fipstop; do + if [ ! -e "/usr/local/bin/$bin" ]; then pass "/usr/local/bin/$bin removed"; else fail "/usr/local/bin/$bin left behind"; fi +done +if [ ! -e "$RC" ]; then pass "boot script removed"; else fail "$RC left behind"; fi +# Unmodified configs go with the package; an edited fips.yaml (and the key +# it may hold) would stay, which is not tested here since none was edited. +for f in fips.yaml hosts fips.conf; do + if [ ! -e "$CONF_DIR/$f" ]; then pass "unmodified $f purged"; else fail "$CONF_DIR/$f left behind although unmodified"; fi +done + +# ── Result ────────────────────────────────────────────────────────────────── +echo +if [ "$failures" -eq 0 ]; then + echo "==> pfSense install smoke test PASSED (${checks} checks, ${skipped} skipped)" + exit 0 +fi +echo "==> pfSense install smoke test FAILED (${failures} of ${checks})" >&2 +exit 1