From 8c349f524ec13b019a3983de51891973c4b473f6 Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 18 Mar 2026 15:26:43 +0000 Subject: [PATCH] Add Kubernetes sidecar deployment Adds a complete Kubernetes sidecar setup under testing/k8s/ that runs FIPS as a sidecar container, injecting the mesh TUN interface (fips0) into any co-located app container via shared pod network namespace. - Multi-stage Dockerfile builds fips and fipsctl from source (debian trixie / rust slim-trixie), producing a minimal runtime image with iproute2, iptables, dnsmasq, and the two binaries - entrypoint.sh generates fips.yaml from environment variables, rewrites /etc/resolv.conf to route .fips DNS through dnsmasq, applies optional iptables isolation, clamps TCP MSS, then starts dnsmasq and execs the daemon - pod.yaml annotated example Pod manifest with Secret, sysctl tuning, readiness/liveness probes, and resource limits - scripts/build.sh convenience wrapper for docker build - testing/k8s/.dockerignore keeps the build context small - README.md usage guide covering quick start, isolation modes, env vars, multi-peer JSON config, troubleshooting --- testing/k8s/.dockerignore | 11 ++ testing/k8s/Dockerfile | 63 +++++++ testing/k8s/README.md | 176 ++++++++++++++++++++ testing/k8s/entrypoint.sh | 309 +++++++++++++++++++++++++++++++++++ testing/k8s/pod.yaml | 207 +++++++++++++++++++++++ testing/k8s/scripts/build.sh | 37 +++++ 6 files changed, 803 insertions(+) create mode 100644 testing/k8s/.dockerignore create mode 100644 testing/k8s/Dockerfile create mode 100644 testing/k8s/README.md create mode 100755 testing/k8s/entrypoint.sh create mode 100644 testing/k8s/pod.yaml create mode 100755 testing/k8s/scripts/build.sh diff --git a/testing/k8s/.dockerignore b/testing/k8s/.dockerignore new file mode 100644 index 0000000..789afe4 --- /dev/null +++ b/testing/k8s/.dockerignore @@ -0,0 +1,11 @@ +target/ +.git/ +.github/ +testing/chaos/ +testing/static/ +testing/sidecar/ +testing/tor/ +fips +fipsctl +../sidecar/fips +../sidecar/fipsctl diff --git a/testing/k8s/Dockerfile b/testing/k8s/Dockerfile new file mode 100644 index 0000000..f5d9518 --- /dev/null +++ b/testing/k8s/Dockerfile @@ -0,0 +1,63 @@ +# ----------------------------------------------------------------------------- +# Stage 1 — build +# Compile fips and fipsctl inside a Rust toolchain image so no local Rust +# install is needed and the image is fully reproducible. +# ----------------------------------------------------------------------------- +FROM rust:slim-trixie AS builder + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Copy the full workspace source. .dockerignore filters target/ and other +# large paths so the context stays small. +COPY . . + +# Build only the two binaries needed at runtime; skip the TUI (fipstop) +# to avoid pulling in ratatui and crossterm. The daemon does not require +# any default features to run. +RUN cargo build --release --no-default-features \ + --bin fips \ + --bin fipsctl + +# ----------------------------------------------------------------------------- +# Stage 2 — runtime +# Minimal Debian image with only the packages required at runtime. +# ----------------------------------------------------------------------------- +FROM debian:trixie-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + iproute2 \ + iptables \ + dnsmasq \ + procps \ + curl && \ + rm -rf /var/lib/apt/lists/* + +# dnsmasq: forward .fips queries to the FIPS daemon DNS resolver; +# forward everything else to the upstream resolver passed in at runtime. +# The upstream placeholder is replaced by entrypoint.sh with the pod's +# original nameserver(s) from /etc/resolv.conf before dnsmasq starts. +RUN printf '%s\n' \ + 'port=53' \ + 'listen-address=127.0.0.1' \ + 'bind-interfaces' \ + 'server=/fips/127.0.0.1#5354' \ + 'no-resolv' \ + > /etc/dnsmasq.d/fips.conf + +COPY --from=builder /build/target/release/fips /usr/local/bin/fips +COPY --from=builder /build/target/release/fipsctl /usr/local/bin/fipsctl + +COPY testing/k8s/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD fipsctl show status + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/testing/k8s/README.md b/testing/k8s/README.md new file mode 100644 index 0000000..ed3b3c6 --- /dev/null +++ b/testing/k8s/README.md @@ -0,0 +1,176 @@ +# FIPS Kubernetes Sidecar + +Run FIPS as a sidecar container in a Kubernetes Pod, injecting the FIPS mesh +network interface into every other container in the Pod. All containers in +the Pod share the same network namespace, so once the sidecar creates `fips0` +it is immediately visible to the app container(s) without any further +configuration. + +## Quick Start + +```bash +# 1. Build the sidecar image +cd testing/k8s +./scripts/build.sh --tag fips-k8s-sidecar:latest + +# 2. Push to your registry (replace with your own) +docker tag fips-k8s-sidecar:latest registry.example.com/fips-k8s-sidecar:latest +docker push registry.example.com/fips-k8s-sidecar:latest + +# 3. Create the identity secret +kubectl create secret generic fips-identity \ + --from-literal=nsec=$(openssl rand -hex 32) + +# 4. Deploy +kubectl apply -f pod.yaml +``` + +## How It Works + +In Kubernetes, all containers in a Pod share the same network namespace. The +fips sidecar: + +1. Generates `/etc/fips/fips.yaml` from environment variables. +2. Rewrites `/etc/resolv.conf` to route `.fips` DNS through dnsmasq (which + forwards `.fips` names to the FIPS daemon's built-in DNS resolver and + everything else to the original cluster DNS). +3. Applies `iptables` isolation rules so that the pod's physical interface + (`eth0`) only carries FIPS transport traffic. +4. Starts dnsmasq and then `exec`s the FIPS daemon. + +The app container starts concurrently and immediately sees `lo`, `eth0`, and +`fips0`. DNS for `.fips` names resolves to `fd::/8` addresses via the +dnsmasq → FIPS daemon pipeline. + +```text +┌────────────────────────────────────────────────────┐ +│ Kubernetes Pod (shared network namespace) │ +│ │ +│ ┌─────────────────┐ ┌───────────────────────────┐│ +│ │ fips (sidecar) │ │ app container(s) ││ +│ │ │ │ ││ +│ │ fips daemon │ │ your workload ││ +│ │ fipsctl │ │ sees: lo, eth0, fips0 ││ +│ │ dnsmasq │ │ ││ +│ └─────────────────┘ └───────────────────────────┘│ +│ │ +│ Interfaces: │ +│ lo — loopback (unrestricted) │ +│ eth0 — pod CNI interface (iptables: FIPS only) │ +│ fips0 — FIPS TUN (fd00::/8, unrestricted) │ +└────────────────────────────────────────────────────┘ +``` + +## Network Isolation + +`FIPS_ISOLATE` controls whether the sidecar locks down `eth0`: + +| Value | Behaviour | +|---|---| +| `false` **(default)** | `fips0` is added alongside normal cluster networking. `eth0` continues to work — services, DNS, other pods, and the internet are all reachable as normal. Application traffic to FIPS mesh peers uses `fips0`. | +| `true` | All traffic on `eth0` is dropped except FIPS transport (UDP 2121). The app container can **only** communicate via `fips0`. Use this for deployments where the workload must never bypass the mesh. | + +> **Common gotcha**: if you deploy the sidecar and find that services or +> other pods are suddenly unreachable, check that `FIPS_ISOLATE` is not +> set to `true`. The mesh-only mode is intentionally strict and will break +> normal cluster connectivity. + +## Requirements + +| Requirement | Notes | +|---|---| +| `NET_ADMIN` capability | Required for TUN creation and iptables | +| `/dev/net/tun` | HostPath volume mount (see `pod.yaml`) | +| Linux kernel ≥ 4.9 | Standard on all current distributions | +| No gVisor / kata-containers | Requires real kernel TUN support | + +## Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `FIPS_NSEC` | *(required)* | Node secret key (hex or nsec1 bech32) | +| `FIPS_PEER_NPUB` | *(empty)* | Single peer npub | +| `FIPS_PEER_ADDR` | *(empty)* | Single peer transport address (`host:port`) | +| `FIPS_PEER_ALIAS` | `peer` | Single peer alias | +| `FIPS_PEER_TRANSPORT` | `udp` | Single peer transport type | +| `FIPS_PEERS_JSON` | *(empty)* | JSON array of peer objects (overrides single-peer vars). **Note**: the included parser handles simple host:port addresses only; IPv6 addresses with brackets may not parse correctly. | +| `FIPS_UDP_BIND` | `0.0.0.0:2121` | UDP transport bind address | +| `FIPS_UDP_PORT` | *(from `FIPS_UDP_BIND`)* | UDP port for iptables rules | +| `FIPS_TCP_BIND` | *(disabled)* | TCP transport bind address | +| `FIPS_TUN_NAME` | `fips0` | TUN interface name | +| `FIPS_TUN_MTU` | `1280` | TUN MTU | +| `FIPS_ISOLATE` | `false` | Apply iptables isolation (blocks all eth0 traffic except FIPS transport) | +| `FIPS_POD_IFACE` | `eth0` | Pod physical interface name | +| `FIPS_REWRITE_DNS` | `true` | Rewrite `/etc/resolv.conf` | +| `RUST_LOG` | `info` | FIPS log level | + +### Multiple Peers + +Use `FIPS_PEERS_JSON` to configure more than one peer: + +```yaml +- name: FIPS_PEERS_JSON + value: | + [ + {"npub":"npub1abc...","alias":"gw1","addr":"203.0.113.10:2121","transport":"udp"}, + {"npub":"npub1def...","alias":"gw2","addr":"198.51.100.5:2121","transport":"udp"} + ] +``` + +Each object supports the keys: `npub` (required), `addr` (required), +`alias`, `transport`, `priority`. + +Alternatively, mount a hand-crafted `fips.yaml` and set `FIPS_NSEC` to +anything (the mount takes precedence because the entrypoint writes to +`/etc/fips/fips.yaml` which is then overridden by the volume): + +```yaml +volumes: + - name: fips-config + configMap: + name: my-fips-config +containers: + - name: fips + volumeMounts: + - name: fips-config + mountPath: /etc/fips/fips.yaml + subPath: fips.yaml +``` + +### JSON Parsing Limitations + +`FIPS_PEERS_JSON` supports parsing with a simple awk-based parser. It handles common `host:port` addresses but does not support IPv6 addresses with brackets (e.g., `[::1]:2121`) or values containing embedded commas/colons. For complex configs, install `jq` in the container and modify the entrypoint, or use a mounted `fips.yaml`. + +## Files + +| File | Purpose | +|---|---| +| `Dockerfile` | Builds the sidecar image | +| `entrypoint.sh` | Generates config, rewrites DNS, applies iptables, starts fips | +| `pod.yaml` | Example Pod manifest (single app container + fips sidecar) | +| `scripts/build.sh` | Compiles FIPS and builds the Docker image | + +## Troubleshooting + +**`FIPS_NSEC is required`** — The secret was not injected. Verify the +`secretKeyRef` name and key match the secret you created with `kubectl +create secret`. + +**`fips0` not visible in app container** — Check that the fips sidecar +started successfully: `kubectl logs -c fips`. The sidecar must start +and run the daemon before `fips0` appears. Add a startup probe or +`postStart` lifecycle hook to your app container if you need to wait. + +**iptables errors** — The `NET_ADMIN` capability and `/dev/net/tun` volume +mount are both required. Verify they are present in the Pod spec. + +**`.fips` DNS not resolving** — Check that `FIPS_REWRITE_DNS=true` and that +dnsmasq started: `kubectl exec -c fips -- pgrep dnsmasq`. Verify the +FIPS DNS listener: `kubectl exec -c fips -- fipsctl show status`. + +**CNI uses a different interface name** — Set `FIPS_POD_IFACE` to match your +CNI's interface name (e.g. `ens3`, `net1` for Multus secondary interfaces). + +**gVisor / kata-containers** — These sandboxed runtimes intercept syscalls +and do not support `AF_PACKET` or `/dev/net/tun` in the same way as a +standard kernel. Use a standard RuntimeClass for FIPS sidecar pods. diff --git a/testing/k8s/entrypoint.sh b/testing/k8s/entrypoint.sh new file mode 100755 index 0000000..ad07165 --- /dev/null +++ b/testing/k8s/entrypoint.sh @@ -0,0 +1,309 @@ +#!/bin/bash +# FIPS Kubernetes sidecar entrypoint. +# +# Generates /etc/fips/fips.yaml from environment variables, rewrites the +# pod's resolv.conf to route .fips DNS through dnsmasq, applies iptables +# isolation rules so that the shared pod network namespace can only reach +# the outside world via the FIPS mesh, then launches the FIPS daemon. +# +# Required environment variables: +# FIPS_NSEC Node secret key (hex or nsec1 bech32) +# +# Optional environment variables (single peer shorthand): +# FIPS_PEER_NPUB Peer's npub +# FIPS_PEER_ADDR Peer's transport address (e.g. 203.0.113.10:2121) +# FIPS_PEER_ALIAS Human-readable peer name (default: peer) +# FIPS_PEER_TRANSPORT Transport type (default: udp) +# +# Multiple peers via JSON (overrides single-peer vars when set): +# FIPS_PEERS_JSON JSON array of peer objects, e.g.: +# '[{"npub":"npub1...","alias":"gw","addr":"1.2.3.4:2121","transport":"udp"}]' +# +# Transport / TUN tuning: +# FIPS_UDP_BIND UDP bind address (default: 0.0.0.0:2121) +# FIPS_UDP_PORT UDP transport port (derived from FIPS_UDP_BIND when unset) +# FIPS_TCP_BIND TCP bind address (default: disabled) +# FIPS_TUN_NAME TUN interface name (default: fips0) +# FIPS_TUN_MTU TUN interface MTU (default: 1280, IPv6 minimum) +# +# Network isolation: +# FIPS_ISOLATE Apply iptables isolation (default: false) +# Set to "true" for mesh-only deployments where the app +# must not communicate outside the FIPS mesh. +# Set to "false" to add fips0 alongside normal cluster +# networking without restricting eth0. +# FIPS_POD_IFACE Pod network interface (default: eth0) +# Adjust if your CNI uses a different name (e.g. ens3). +# +# DNS: +# FIPS_REWRITE_DNS Rewrite /etc/resolv.conf (default: true) +# Set to "false" if you manage DNS via another mechanism. +# +# Logging: +# RUST_LOG FIPS log level (default: info) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# IPv6 kernel configuration +# +# net.ipv6.conf.all.disable_ipv6, net.ipv6.conf.default.disable_ipv6, and +# net.ipv6.bindv6only are set via the pod securityContext.sysctls field in +# pod.yaml so that kubelet applies them before any container starts. +# Nothing to do here. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- + +FIPS_NSEC="${FIPS_NSEC:?FIPS_NSEC is required — provide your node secret key}" + +FIPS_UDP_BIND="${FIPS_UDP_BIND:-0.0.0.0:2121}" +FIPS_TCP_BIND="${FIPS_TCP_BIND:-}" +FIPS_TUN_NAME="${FIPS_TUN_NAME:-fips0}" +FIPS_TUN_MTU="${FIPS_TUN_MTU:-1280}" +FIPS_ISOLATE="${FIPS_ISOLATE:-false}" +FIPS_POD_IFACE="${FIPS_POD_IFACE:-eth0}" +FIPS_REWRITE_DNS="${FIPS_REWRITE_DNS:-true}" + +FIPS_PEER_TRANSPORT="${FIPS_PEER_TRANSPORT:-udp}" +FIPS_PEER_ALIAS="${FIPS_PEER_ALIAS:-peer}" + +# Derive the UDP port for iptables rules from FIPS_UDP_BIND (last colon-field). +FIPS_UDP_PORT="${FIPS_UDP_PORT:-${FIPS_UDP_BIND##*:}}" + +mkdir -p /etc/fips + +# --------------------------------------------------------------------------- +# Build peers YAML section +# --------------------------------------------------------------------------- + +build_peers_yaml() { + # FIPS_PEERS_JSON wins when set (supports multiple peers). + if [ -n "${FIPS_PEERS_JSON:-}" ]; then + # Parse the JSON array with awk — no jq required. + # Expected element keys: npub, alias, addr, transport, priority + # + # Note: This parser handles simple host:port addresses only. + # IPv6 addresses with brackets ([::1]:2121) or values containing + # commas/colons may not parse correctly. Use single-peer env vars + # for complex configs or install jq and replace this block. + echo "$FIPS_PEERS_JSON" | awk ' + BEGIN { RS="}"; FS="," } + { + npub=""; alias="peer"; addr=""; transport="udp"; priority=100 + for (i=1; i<=NF; i++) { + gsub(/[{}\[\] \t\n\r]/, "", $i) + if ($i ~ /"npub"/) { split($i,a,":"); npub=a[2]; gsub(/"/,"",npub) } + if ($i ~ /"alias"/) { split($i,a,":"); alias=a[2]; gsub(/"/,"",alias) } + if ($i ~ /"addr"/) { n=split($i,a,":"); addr=a[2]":"a[3]; gsub(/"/,"",addr) } + if ($i ~ /"transport"/) { split($i,a,":"); transport=a[2]; gsub(/"/,"",transport) } + if ($i ~ /"priority"/) { split($i,a,":"); priority=a[2]; gsub(/"/,"",priority) } + } + if (npub != "" && addr != "") { + printf " - npub: \"%s\"\n alias: \"%s\"\n addresses:\n - transport: %s\n addr: \"%s\"\n priority: %s\n connect_policy: auto_connect\n auto_reconnect: true\n", npub, alias, transport, addr, priority + } + }' + return + fi + + # Single-peer shorthand via individual env vars. + if [ -n "${FIPS_PEER_NPUB:-}" ] && [ -n "${FIPS_PEER_ADDR:-}" ]; then + cat < /etc/fips/fips.yaml <> /etc/dnsmasq.d/fips.conf + done + + # If no upstream was found fall back to Google (better than no resolution). + if [ "${#UPSTREAM_NS[@]}" -eq 0 ]; then + echo "[fips-sidecar] WARNING: no upstream nameserver found in /etc/resolv.conf; falling back to 8.8.8.8" + echo "server=8.8.8.8" >> /etc/dnsmasq.d/fips.conf + fi + + # Preserve the search/domain lines so pod DNS short-names still work. + SEARCH_LINE="" + while IFS= read -r line; do + if [[ "$line" =~ ^(search|domain)[[:space:]] ]]; then + SEARCH_LINE="$line" + break + fi + done < /etc/resolv.conf + + { + echo "nameserver 127.0.0.1" + [ -n "$SEARCH_LINE" ] && echo "$SEARCH_LINE" + } > /etc/resolv.conf + + echo "[fips-sidecar] DNS rewritten: upstream ${UPSTREAM_NS[*]:-8.8.8.8} → dnsmasq → fips/cluster" +fi + +# --------------------------------------------------------------------------- +# iptables isolation +# +# Goal: only FIPS transport traffic may leave/enter the pod via the physical +# interface. All other pod-to-external traffic is dropped. Traffic on the +# fips0 TUN and loopback is unrestricted. The app container sharing this +# network namespace therefore can only communicate over the FIPS mesh. +# +# Skip when FIPS_ISOLATE=false (e.g. already handled by a NetworkPolicy, +# or when the operator wants unrestricted egress alongside mesh traffic). +# --------------------------------------------------------------------------- + +if [ "$FIPS_ISOLATE" = "true" ]; then + IFACE="$FIPS_POD_IFACE" + UDP_PORT="$FIPS_UDP_PORT" + + # IPv4: allow loopback + FIPS UDP transport; drop everything else on + # the pod's physical interface. + iptables -A OUTPUT -o lo -j ACCEPT + iptables -A INPUT -i lo -j ACCEPT + + iptables -A OUTPUT -o "$IFACE" -p udp --dport "$UDP_PORT" -j ACCEPT + iptables -A OUTPUT -o "$IFACE" -p udp --sport "$UDP_PORT" -j ACCEPT + iptables -A INPUT -i "$IFACE" -p udp --dport "$UDP_PORT" -j ACCEPT + iptables -A INPUT -i "$IFACE" -p udp --sport "$UDP_PORT" -j ACCEPT + + # Allow TCP transport port when configured. + if [ -n "$FIPS_TCP_BIND" ]; then + TCP_PORT="${FIPS_TCP_BIND##*:}" + iptables -A OUTPUT -o "$IFACE" -p tcp --dport "$TCP_PORT" -j ACCEPT + iptables -A INPUT -i "$IFACE" -p tcp --sport "$TCP_PORT" -j ACCEPT + iptables -A OUTPUT -o "$IFACE" -p tcp --sport "$TCP_PORT" -j ACCEPT + iptables -A INPUT -i "$IFACE" -p tcp --dport "$TCP_PORT" -j ACCEPT + fi + + # Allow established/related so existing connections aren't torn down mid-flight. + iptables -A INPUT -i "$IFACE" -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + iptables -A OUTPUT -o "$IFACE" -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + + iptables -A OUTPUT -o "$IFACE" -j DROP + iptables -A INPUT -i "$IFACE" -j DROP + + # IPv6: allow loopback and fips TUN; block physical interface entirely. + ip6tables -A OUTPUT -o lo -j ACCEPT + ip6tables -A INPUT -i lo -j ACCEPT + ip6tables -A OUTPUT -o "$FIPS_TUN_NAME" -j ACCEPT + ip6tables -A INPUT -i "$FIPS_TUN_NAME" -j ACCEPT + ip6tables -A OUTPUT -o "$IFACE" -j DROP + ip6tables -A INPUT -i "$IFACE" -j DROP + + echo "[fips-sidecar] iptables isolation applied on ${IFACE} (UDP ${UDP_PORT})" +fi + +# --------------------------------------------------------------------------- +# TCP MSS clamping on fips0 +# +# The kernel derives MSS from the TUN MTU: 1280 - 60 = 1220. But FIPS adds +# FIPS_IPV6_OVERHEAD (77) bytes of encapsulation that the kernel doesn't know +# Clamp the MSS in TCP SYN packets traversing fips0 to the path MTU so that +# segments fit within the effective MTU after FIPS encapsulation. +# This is applied regardless of FIPS_ISOLATE mode since it's needed for +# correct TCP negotiation even when normal network access is allowed. +# --------------------------------------------------------------------------- + +ip6tables -t mangle -A FORWARD -o "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu +ip6tables -t mangle -A FORWARD -i "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu +ip6tables -t mangle -A OUTPUT -o "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu + +echo "[fips-sidecar] TCP MSS clamped to PMTU on ${FIPS_TUN_NAME}" + +# --------------------------------------------------------------------------- +# Start dnsmasq and launch FIPS daemon +# --------------------------------------------------------------------------- + +dnsmasq --conf-dir=/etc/dnsmasq.d +echo "[fips-sidecar] dnsmasq started" + +exec fips --config /etc/fips/fips.yaml diff --git a/testing/k8s/pod.yaml b/testing/k8s/pod.yaml new file mode 100644 index 0000000..5b688bb --- /dev/null +++ b/testing/k8s/pod.yaml @@ -0,0 +1,207 @@ +# FIPS Kubernetes sidecar — example Pod manifest +# +# WARNING: The Secret below contains a placeholder nsec. Replace it with your +# real key before applying. Do NOT commit real keys to version control. +# Generate a fresh key with: openssl rand -hex 32 +# +# Apply: +# kubectl apply -f pod.yaml +# +# Requirements on the node: +# - /dev/net/tun must exist (standard on all modern Linux kernels) +# - The container runtime must honour NET_ADMIN (true for containerd/CRI-O +# with default settings; may require a RuntimeClass tweak for gVisor) +# +# Kubernetes version: 1.28+ + +# ----------------------------------------------------------------------------- +# Secret — node identity +# Replace the nsec value with your real key (hex or nsec1 bech32). +# In production, manage this with a secrets operator (External Secrets, +# Sealed Secrets, Vault Agent, etc.) rather than a plaintext manifest. +# ----------------------------------------------------------------------------- +apiVersion: v1 +kind: Secret +metadata: + name: fips-identity +type: Opaque +stringData: + nsec: "replace-this-with-your-hex-or-nsec1-secret-key" + +--- + +apiVersion: v1 +kind: Pod +metadata: + name: fips-sidecar-example + labels: + app: fips-sidecar-example +spec: + # Containers in the same Pod share the same network namespace automatically. + # The fips sidecar creates fips0; the app container sees it immediately. + # + # Start order: initContainers run first, then all containers start + # concurrently. If your app needs the TUN interface to be ready before it + # starts, use a readiness probe on the sidecar and a dependency in the app + # (or use a startup init container that waits for fips0 to appear). + + # net.ipv6.* sysctls are namespaced per-pod in Linux. Declaring them here + # allows the fips sidecar entrypoint to set them at runtime via sysctl(8). + # bindv6only=0 means apps listening on 0.0.0.0 will also receive IPv6 + # connections arriving via fips0, without needing to be rewritten. + # These are "unsafe" sysctls and require the kubelet --allowed-unsafe-sysctls + # flag or a corresponding RuntimeClass / PodSecurityPolicy exception. + securityContext: + sysctls: + - name: net.ipv6.conf.all.disable_ipv6 + value: "0" + - name: net.ipv6.conf.default.disable_ipv6 + value: "0" + - name: net.ipv6.bindv6only + value: "0" + + volumes: + - name: tun-device + hostPath: + path: /dev/net/tun + type: CharDevice + # Optional: mount a pre-built fips.yaml instead of generating from env. + # - name: fips-config + # configMap: + # name: fips-config + + containers: + # ----------------------------------------------------------------------- + # FIPS sidecar — owns the network namespace setup + # ----------------------------------------------------------------------- + - name: fips + image: your-registry.example.com/fips-sidecar:latest + imagePullPolicy: Always + + securityContext: + capabilities: + add: + - NET_ADMIN # required: TUN creation and iptables + runAsNonRoot: false # fips needs root for iptables/TUN + runAsUser: 0 + + volumeMounts: + - name: tun-device + mountPath: /dev/net/tun + + env: + # --- Identity (required) --- + - name: FIPS_NSEC + valueFrom: + secretKeyRef: + name: fips-identity + key: nsec + + # --- Single peer shorthand --- + # Remove these and use FIPS_PEERS_JSON below for multiple peers. + - name: FIPS_PEER_NPUB + value: "" # e.g. npub1abc... + - name: FIPS_PEER_ADDR + value: "" # e.g. 203.0.113.10:2121 + - name: FIPS_PEER_ALIAS + value: "gateway" + - name: FIPS_PEER_TRANSPORT + value: "udp" + + # --- Multiple peers via JSON (overrides single-peer vars when non-empty) --- + # - name: FIPS_PEERS_JSON + # value: | + # [ + # {"npub":"npub1abc...","alias":"gw1","addr":"203.0.113.10:2121","transport":"udp"}, + # {"npub":"npub1def...","alias":"gw2","addr":"198.51.100.5:2121","transport":"udp"} + # ] + + # --- Transport tuning --- + - name: FIPS_UDP_BIND + value: "0.0.0.0:2121" + # Uncomment to enable TCP transport (useful on networks that block UDP): + # - name: FIPS_TCP_BIND + # value: "0.0.0.0:8443" + + # --- TUN interface --- + - name: FIPS_TUN_NAME + value: "fips0" + - name: FIPS_TUN_MTU + value: "1280" + + # --- Network isolation --- + # "false" — fips0 is added alongside normal cluster networking. eth0 + # continues to work as usual (services, DNS, other pods all reachable). + # "true" — eth0 is locked to FIPS transport only; app can only talk + # via fips0. Use for high-security mesh-only deployments. + - name: FIPS_ISOLATE + value: "false" + # Adjust if your CNI names the pod interface differently (e.g. ens3, net1). + - name: FIPS_POD_IFACE + value: "eth0" + + # --- DNS --- + # Set to "false" if you manage pod DNS externally. + - name: FIPS_REWRITE_DNS + value: "true" + + # --- Logging --- + - name: RUST_LOG + value: "info" + + ports: + - name: fips-udp + containerPort: 2121 + protocol: UDP + # - name: fips-tcp + # containerPort: 8443 + # protocol: TCP + + readinessProbe: + exec: + command: ["fipsctl", "show", "status"] + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + + livenessProbe: + exec: + command: ["fipsctl", "show", "status"] + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 3 + + resources: + requests: + cpu: "50m" + memory: "32Mi" + limits: + cpu: "500m" + memory: "128Mi" + + # ----------------------------------------------------------------------- + # App container — shares the network namespace created by the fips sidecar + # ----------------------------------------------------------------------- + - name: app + image: your-app:latest + imagePullPolicy: IfNotPresent + + # No special capabilities needed — app uses fips0 like a normal interface. + securityContext: + runAsNonRoot: true + + # The app sees: lo, eth0 (isolated by iptables), fips0 (mesh). + # All outbound traffic from the app flows through fips0. + # DNS resolution for .fips names works via dnsmasq on 127.0.0.1. + command: ["sleep", "infinity"] # replace with your actual workload command + + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "1000m" + memory: "512Mi" + + # Restart policy for the Pod. Use "Always" for long-running workloads. + restartPolicy: Always diff --git a/testing/k8s/scripts/build.sh b/testing/k8s/scripts/build.sh new file mode 100755 index 0000000..f9b8c81 --- /dev/null +++ b/testing/k8s/scripts/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Build the FIPS Kubernetes sidecar Docker image. +# The build happens entirely inside Docker (multi-arch friendly). +# Usage: ./scripts/build.sh [--tag TAG] [-- ] +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DOCKER_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$DOCKER_DIR/../.." && pwd)" + +IMAGE_TAG="fips-k8s-sidecar:latest" +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) IMAGE_TAG="$2"; shift 2 ;; + --) shift; break ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then + echo "Error: Cannot find Cargo.toml at $PROJECT_ROOT" >&2 + echo "Expected layout: /testing/k8s/scripts/build.sh" >&2 + exit 1 +fi + +echo "Building Docker image: $IMAGE_TAG" +docker build \ + -t "$IMAGE_TAG" \ + -f "$DOCKER_DIR/Dockerfile" \ + "$@" \ + "$PROJECT_ROOT" + +echo "" +echo "Done. Image: $IMAGE_TAG" +echo "" +echo "Push to your registry, then apply the example manifest:" +echo " kubectl apply -f testing/k8s/pod.yaml"