From 86eb122f988f8817d8f95491aa3eea19c5eb8e6c Mon Sep 17 00:00:00 2001 From: Laan Tungir Date: Fri, 8 May 2026 10:54:03 -0400 Subject: [PATCH] Added more debugging --- fips_debug.md | 100 ++++++++++++++++++++++++++++++++++++++ start_py_http80.sh | 74 ++++++++++++++++++++++++++++ test_fips.sh | 46 ++++++++++++++++++ test_qube_pair.sh | 79 ++++++++++++++++++++++++++++-- update_and_deploy_fips.sh | 66 +++++++++++++++++++++++-- 5 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 start_py_http80.sh diff --git a/fips_debug.md b/fips_debug.md index a425850..c699e5a 100644 --- a/fips_debug.md +++ b/fips_debug.md @@ -132,3 +132,103 @@ A backup of the prior local file was created before modification. ## Final status The target `.fips` web page is reachable from this local node after lowering local UDP MTU to `1280` and restarting `fips.service`. + +--- + +## Additional root cause discovered later (boot-time route collision) + +A subsequent incident showed another failure mode that can present as the same `curl: (7) Failed to connect` symptom: + +1. A stale kernel route existed: `fd00::/8 via fe80::... dev eth0`. +2. During startup, `fips.service` attempted to install `fd00::/8 dev fips0`. +3. Kernel returned `EEXIST`, so TUN initialization failed (`tun_state: failed`, no `fips0`). +4. Without TUN, `.fips` reachability failed even if service looked partially alive. + +### Key diagnostic indicators + +- `sudo fipsctl show status` reports: + - `tun_state: failed` + - missing/empty `tun_name` +- `ip -6 route show fd00::/8` points to non-`fips0` device (for example `eth0`) +- `ip -o link show dev fips0` fails + +### Immediate runtime recovery + +```bash +sudo ip -6 route del fd00::/8 +sudo systemctl restart fips.service +``` + +After recovery, verify: + +- `tun_state: active` +- `fips0` exists and is UP +- `ip -6 route show fd00::/8` -> `dev fips0` + +### Persistence/hardening + +To make this robust across reboots and service restarts: + +1. Keep config MTU safeguards: + - `tun.mtu: 1280` + - `transports.udp.mtu: 1280` +2. Ensure `fips.service` is enabled: + - `sudo systemctl enable fips.service` +3. Add a systemd drop-in `ExecStartPre` hook that removes stale non-`fips0` `fd00::/8` routes before daemon start. + +This repository now includes deployment-script updates to install that hook automatically. + +--- + +## Latest incident addendum (Qubes inter-qube `.fips` debug) + +### New findings + +1. **Mesh + DNS can be healthy while app check still fails** + - We observed successful `.fips` DNS resolution and ICMP reachability to the remote FIPS IPv6. + - Yet HTTP checks still failed due to listener/bind mismatch. + +2. **Qubes resolver path must include local dnsmasq for `.fips`** + - `dig @127.0.0.1 -p 5354 .fips AAAA` worked while `getent ahosts .fips` initially failed. + - Root cause: `/etc/resolv.conf` lacked `nameserver 127.0.0.1` on the local qube. + - Adding `127.0.0.1` restored system resolver `.fips` lookups. + +3. **Persistence in Qubes requires `/rw/config/rc.local` hygiene** + - Runtime fixes to `/etc/resolv.conf` are not enough. + - Persisting the resolver injection in `/rw/config/rc.local` is required for reboot durability. + - We also discovered/remediated a stale legacy block (`# === AI FIPS ROUTE START ===`) that forced `fd00::/8` via `eth0`; this must be removed to avoid breaking TUN routing on boot. + +4. **Remote service bind must match transport family used by `.fips` path** + - Remote python server bound to `0.0.0.0:80` (IPv4) was not reachable via `.fips` IPv6 destination. + - Rebinding server to remote FIPS IPv6 (`fd56:...:982f`) fixed HTTP connectivity immediately. + +### Practical runbook update for inter-qube checks + +Add these checks before concluding “FIPS is broken”: + +1. **Resolver path sanity** + - `getent ahosts .fips` + - If fail, check `/etc/resolv.conf` includes `nameserver 127.0.0.1` + - Confirm dnsmasq forwards `/fips/` to local FIPS DNS (`127.0.0.1#5354` or `::1#5354`) + +2. **Control/data plane split checks** + - DNS AAAA success + `ping6 .fips` success proves overlay path exists + - If HTTP fails, test TCP port explicitly (`/dev/tcp` or `curl -v -6`) + +3. **Listener-family verification on remote** + - `ss -lntp | grep ':80 '` + - Ensure remote listener is IPv6-capable for `.fips` access + +### Script updates from this incident + +- `test_qube_pair.sh` + - Added resolver hints when `.fips` name does not resolve + - Added IPv6 curl (`curl -6`) and configurable HTTP port via `REMOTE_HTTP_PORT` + - Added port probe diagnostics on HTTP failure +- `test_fips.sh` + - Added `/etc/resolv.conf` sanity reporting for `nameserver 127.0.0.1` +- `update_and_deploy_fips.sh` + - Persisted rc.local network overrides + - Added cleanup of legacy `# === AI FIPS ROUTE START/END ===` block when rewriting rc.local +- `start_py_http80.sh` + - Binds to detected local FIPS IPv6 by default (instead of IPv4-only bind) diff --git a/start_py_http80.sh b/start_py_http80.sh new file mode 100644 index 0000000..1dcd478 --- /dev/null +++ b/start_py_http80.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Start a simple Python HTTP server on port 80 for FIPS reachability tests. +# Run this script on the OTHER qube. +# +# Usage: +# ./start_py_http80.sh [serve_dir] +# +# Behavior: +# - Detects local FIPS IPv6 from fipsctl (if available) +# - Binds http.server to that FIPS IPv6 address (critical for .fips inbound tests) +# - Falls back to :: if detection fails +# - Prints listener info so you can confirm IPv6 bind +# +# Cross-reference: +# - test_qube_pair.sh expects HTTP over .fips on port 80 by default +# - If server binds only to 0.0.0.0 (IPv4), .fips curl from another qube may fail + +if ! command -v python3 >/dev/null 2>&1; then + echo "Error: python3 is required" >&2 + exit 1 +fi + +SERVE_DIR="${1:-$HOME/fips_http_root}" +mkdir -p "${SERVE_DIR}" + +if [[ ! -f "${SERVE_DIR}/index.html" ]]; then + cat > "${SERVE_DIR}/index.html" <<'HTML' + + + FIPS test server + +

FIPS test server is running on port 80

+

If you can read this over .fips, HTTP over FIPS is working.

+ + +HTML +fi + +BIND_ADDR="::" +if command -v fipsctl >/dev/null 2>&1; then + FIPS_IPV6="$(sudo fipsctl show status 2>/dev/null | python3 -c 'import json,sys +try: + d=json.loads(sys.stdin.read()) + print(d.get("ipv6_addr") or "") +except Exception: + print("")' || true)" + if [[ -n "${FIPS_IPV6}" ]]; then + BIND_ADDR="${FIPS_IPV6}" + fi +fi + +echo "Serving directory: ${SERVE_DIR}" +echo "Binding python HTTP server to [${BIND_ADDR}]:80 (sudo required)" + +echo "Stopping any existing python http.server processes..." +sudo pkill -f "python3 -m http.server" >/dev/null 2>&1 || true + +cd "${SERVE_DIR}" + +echo "Starting server..." +# Start in background briefly so we can validate listener, then wait on it. +sudo python3 -m http.server 80 --bind "${BIND_ADDR}" & +SERVER_PID=$! +sleep 1 + +echo "Listener check:" +sudo ss -lntp | grep ':80 ' || true + +echo "Server PID: ${SERVER_PID}" +echo "Press Ctrl+C to stop." +wait "${SERVER_PID}" + diff --git a/test_fips.sh b/test_fips.sh index a079808..4dbba7c 100755 --- a/test_fips.sh +++ b/test_fips.sh @@ -10,6 +10,11 @@ if ! command -v curl >/dev/null 2>&1; then exit 1 fi +if ! command -v ip >/dev/null 2>&1; then + echo "Error: iproute2 (ip command) is required." >&2 + exit 1 +fi + if ! command -v python3 >/dev/null 2>&1; then echo "Error: python3 is required." >&2 exit 1 @@ -110,6 +115,43 @@ FIPSCTL_VERSION="$(fipsctl --version | head -n1)" STATUS_JSON="$(get_cmd_output fipsctl show status)" PEERS_JSON="$(get_cmd_output fipsctl show peers)" +TUN_STATE="$(json_get "${STATUS_JSON}" tun_state)" +TUN_NAME="$(json_get "${STATUS_JSON}" tun_name)" +if [[ "${TUN_NAME}" == "n/a" || -z "${TUN_NAME}" ]]; then + TUN_NAME="fips0" +fi + +if grep -Eq '^[[:space:]]*nameserver[[:space:]]+127\.0\.0\.1([[:space:]]|$)' /etc/resolv.conf 2>/dev/null; then + RESOLV_127_STATUS="present" +else + RESOLV_127_STATUS="missing" +fi + +if [[ "${TUN_STATE}" != "active" ]]; then + echo "Error: expected tun_state=active, got ${TUN_STATE}." >&2 + exit 1 +fi + +if ! ip -o link show dev "${TUN_NAME}" >/dev/null 2>&1; then + echo "Error: expected TUN interface ${TUN_NAME} to exist." >&2 + exit 1 +fi + +FD00_ROUTE_LINE="$(ip -6 route show fd00::/8 2>/dev/null | head -n1 || true)" +if [[ -z "${FD00_ROUTE_LINE}" ]]; then + echo "Error: missing fd00::/8 route." >&2 + exit 1 +fi + +if [[ "${FD00_ROUTE_LINE}" != *"dev ${TUN_NAME}"* ]]; then + echo "Error: fd00::/8 route is not installed via ${TUN_NAME}: ${FD00_ROUTE_LINE}" >&2 + exit 1 +fi + +if [[ "${RESOLV_127_STATUS}" == "missing" ]]; then + echo "Warning: /etc/resolv.conf does not contain nameserver 127.0.0.1; .fips DNS may fail in some environments." >&2 +fi + echo "==> Testing .fips URL reachability" TMP_BODY="$(mktemp)" trap 'rm -f "${TMP_BODY}"' EXIT @@ -139,6 +181,10 @@ echo "7) peer_count: $(json_get "${STATUS_JSON}" peer_count)" echo "8) link_count: $(json_get "${STATUS_JSON}" link_count)" echo "9) session_count: $(json_get "${STATUS_JSON}" session_count)" echo "10) connected_peer_example: $(first_connected_peer "${PEERS_JSON}")" +echo "11) tun_state: ${TUN_STATE}" +echo "12) tun_name: ${TUN_NAME}" +echo "13) fd00_route: ${FD00_ROUTE_LINE}" +echo "14) resolv_conf_nameserver_127.0.0.1: ${RESOLV_127_STATUS}" echo "" echo "URL test: HTTP=${HTTP_CODE}, bytes=${BYTES}" diff --git a/test_qube_pair.sh b/test_qube_pair.sh index 6408f23..b48a060 100755 --- a/test_qube_pair.sh +++ b/test_qube_pair.sh @@ -6,6 +6,11 @@ set -euo pipefail # ./test_qube_pair.sh [remote_underlay_addr:port] # Example: # ./test_qube_pair.sh npub1... 10.137.0.22:2121 +# +# Optional env vars: +# REMOTE_HTTP_PORT=80 # target port for HTTP validation over .fips +# CURL_TIMEOUT=10 +# MIN_BYTES=200 if [[ $# -lt 1 ]]; then echo "Usage: $0 [remote_underlay_addr:port]" >&2 @@ -17,6 +22,13 @@ REMOTE_ADDR="${2:-}" REMOTE_FQDN="${REMOTE_NPUB}.fips" CURL_TIMEOUT="${CURL_TIMEOUT:-10}" MIN_BYTES="${MIN_BYTES:-200}" +REMOTE_HTTP_PORT="${REMOTE_HTTP_PORT:-80}" +REMOTE_URL="http://${REMOTE_FQDN}" +if [[ "${REMOTE_HTTP_PORT}" != "80" ]]; then + REMOTE_URL="http://${REMOTE_FQDN}:${REMOTE_HTTP_PORT}/" +else + REMOTE_URL="http://${REMOTE_FQDN}/" +fi need_cmd() { if ! command -v "$1" >/dev/null 2>&1; then @@ -29,6 +41,7 @@ need_cmd curl need_cmd python3 need_cmd getent need_cmd fipsctl +need_cmd ip echo "==> Local node" STATUS_JSON="$(sudo fipsctl show status)" @@ -44,6 +57,46 @@ print(f"peer_count={s.get('peer_count')}") print(f"session_count={s.get('session_count')}") PY +echo "" +echo "==> Local TUN/route preflight" +TUN_STATE="$(python3 - "${STATUS_JSON}" <<'PY' +import json +import sys +s = json.loads(sys.argv[1]) +print(s.get("tun_state", "n/a")) +PY +)" +TUN_NAME="$(python3 - "${STATUS_JSON}" <<'PY' +import json +import sys +s = json.loads(sys.argv[1]) +print(s.get("tun_name") or "fips0") +PY +)" + +if [[ "${TUN_STATE}" != "active" ]]; then + echo "Error: local tun_state is ${TUN_STATE} (expected active)." >&2 + exit 1 +fi + +if ! ip -o link show dev "${TUN_NAME}" >/dev/null 2>&1; then + echo "Error: local TUN interface ${TUN_NAME} is missing." >&2 + exit 1 +fi + +FD00_ROUTE_LINE="$(ip -6 route show fd00::/8 2>/dev/null | head -n1 || true)" +if [[ -z "${FD00_ROUTE_LINE}" ]]; then + echo "Error: missing local fd00::/8 route." >&2 + exit 1 +fi +if [[ "${FD00_ROUTE_LINE}" != *"dev ${TUN_NAME}"* ]]; then + echo "Error: local fd00::/8 route is not via ${TUN_NAME}: ${FD00_ROUTE_LINE}" >&2 + exit 1 +fi + +echo "tun_state=${TUN_STATE} tun_name=${TUN_NAME}" +echo "fd00_route=${FD00_ROUTE_LINE}" + echo "" echo "==> Attempting to resolve ${REMOTE_FQDN}" REMOTE_IPV6="" @@ -52,6 +105,11 @@ if RESOLVED="$(getent ahosts "${REMOTE_FQDN}" | awk '{print $1}' | head -n1)" && echo "resolved_ipv6=${REMOTE_IPV6}" else echo "not_resolved" + if grep -Eq '^[[:space:]]*nameserver[[:space:]]+127\.0\.0\.1([[:space:]]|$)' /etc/resolv.conf 2>/dev/null; then + echo "resolver_hint=127.0.0.1_present" + else + echo "resolver_hint=127.0.0.1_missing_in_/etc/resolv.conf" + fi fi if [[ -n "${REMOTE_ADDR}" ]]; then @@ -83,12 +141,21 @@ PY echo "" echo "==> HTTP check over .fips" +echo "remote_url=${REMOTE_URL}" TMP_BODY="$(mktemp)" trap 'rm -f "${TMP_BODY}"' EXIT -HTTP_CODE="$(curl -sS --max-time "${CURL_TIMEOUT}" -o "${TMP_BODY}" -w '%{http_code}' "http://${REMOTE_FQDN}/" || true)" +HTTP_CODE="$(curl -6 -sS --max-time "${CURL_TIMEOUT}" -o "${TMP_BODY}" -w '%{http_code}' "${REMOTE_URL}" || true)" BYTES="$(wc -c < "${TMP_BODY}" | tr -d '[:space:]')" echo "http_code=${HTTP_CODE:-000} bytes=${BYTES}" +if [[ "${HTTP_CODE:-000}" == "000" ]]; then + if timeout 2 bash -lc "cat /dev/tcp/${REMOTE_FQDN}/${REMOTE_HTTP_PORT}" >/dev/null 2>&1; then + echo "port_probe=tcp_open_but_http_failed" + else + echo "port_probe=tcp_closed_or_filtered" + fi +fi + if [[ "${HTTP_CODE:-000}" == "200" ]] && (( BYTES >= MIN_BYTES )); then echo "" echo "RESULT=PASS (inter-qube communication confirmed via .fips HTTP)" @@ -99,8 +166,10 @@ echo "" echo "RESULT=FAIL (communication not yet proven)" echo "" echo "Next actions:" -echo "1) In EACH qube, get underlay IP: ip -4 -o addr show dev eth0" -echo "2) From this qube, connect to remote: sudo fipsctl connect ${REMOTE_NPUB} :2121 udp" -echo "3) From remote qube, connect back to this qube: sudo fipsctl connect :2121 udp" -echo "4) Re-run this script in both qubes." +echo "1) If name did not resolve, ensure /etc/resolv.conf has nameserver 127.0.0.1 and dnsmasq forwards /fips/ to 127.0.0.1#5354" +echo "2) In EACH qube, get underlay IP: ip -4 -o addr show dev eth0" +echo "3) From this qube, connect to remote: sudo fipsctl connect ${REMOTE_NPUB} :2121 udp" +echo "4) From remote qube, connect back to this qube: sudo fipsctl connect :2121 udp" +echo "5) If app is up but script still fails, verify remote listener is IPv6-bound and set REMOTE_HTTP_PORT if non-80" +echo "6) Re-run this script in both qubes." exit 1 diff --git a/update_and_deploy_fips.sh b/update_and_deploy_fips.sh index 0219e1c..d792ff1 100755 --- a/update_and_deploy_fips.sh +++ b/update_and_deploy_fips.sh @@ -383,10 +383,12 @@ persist_qubes_rc_local_network_overrides() { fi awk ' - BEGIN { skip = 0 } - /^# BEGIN FIPS_DEPLOY_NETWORK_OVERRIDES$/ { skip = 1; next } - /^# END FIPS_DEPLOY_NETWORK_OVERRIDES$/ { skip = 0; next } - skip == 0 { print } + BEGIN { skip_fips = 0; skip_legacy_ai_route = 0 } + /^# BEGIN FIPS_DEPLOY_NETWORK_OVERRIDES$/ { skip_fips = 1; next } + /^# END FIPS_DEPLOY_NETWORK_OVERRIDES$/ { skip_fips = 0; next } + /^# === AI FIPS ROUTE START ===$/ { skip_legacy_ai_route = 1; next } + /^# === AI FIPS ROUTE END ===$/ { skip_legacy_ai_route = 0; next } + skip_fips == 0 && skip_legacy_ai_route == 0 { print } ' "${tmp_file}" > "${tmp_file}.clean" mv "${tmp_file}.clean" "${tmp_file}" @@ -454,6 +456,60 @@ ensure_fips_ingress_allowed() { fi } +install_fips_route_cleanup_hook() { + local helper_path="/usr/local/libexec/fips-route-sanitize.sh" + local dropin_dir="/etc/systemd/system/fips.service.d" + local dropin_file="${dropin_dir}/10-route-sanitize.conf" + local tmp_file + + require_sudo + + echo "==> Installing boot-time route cleanup hook for fips.service" + sudo mkdir -p "$(dirname "${helper_path}")" "${dropin_dir}" + + tmp_file="$(mktemp)" + cat > "${tmp_file}" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +# Remove stale fd00::/8 routes that are not owned by fips0. +routes="$(ip -6 route show fd00::/8 2>/dev/null || true)" +[[ -n "${routes}" ]] || exit 0 + +while IFS= read -r line; do + [[ -n "${line}" ]] || continue + if [[ "${line}" == *" dev fips0"* ]]; then + continue + fi + + if [[ "${line}" =~ ^fd00::/8[[:space:]]+via[[:space:]]+([^[:space:]]+)[[:space:]]+dev[[:space:]]+([^[:space:]]+) ]]; then + gw="${BASH_REMATCH[1]}" + dev="${BASH_REMATCH[2]}" + ip -6 route del fd00::/8 via "${gw}" dev "${dev}" >/dev/null 2>&1 || true + elif [[ "${line}" =~ ^fd00::/8[[:space:]]+dev[[:space:]]+([^[:space:]]+) ]]; then + dev="${BASH_REMATCH[1]}" + if [[ "${dev}" != "fips0" ]]; then + ip -6 route del fd00::/8 dev "${dev}" >/dev/null 2>&1 || true + fi + else + ip -6 route del fd00::/8 >/dev/null 2>&1 || true + fi +done <<< "${routes}" +SH + sudo install -m 0755 "${tmp_file}" "${helper_path}" + rm -f "${tmp_file}" + + tmp_file="$(mktemp)" + cat > "${tmp_file}" < Cleaning potentially stale fd00::/8 route state" require_sudo sudo ip -6 route del fd00::/8 >/dev/null 2>&1 || true