Improve Docker mesh setup: data-driven topologies, identity derivation, services

Config system overhaul:
- Add mesh-public topology (5 Docker nodes + external pub node at 217.77.8.91)
- Refactor generate-configs.sh to read topology YAML files directly,
  replacing hardcoded lookup functions, with multi-char node ID support
- Generate npubs.env with all node npubs; sourced by test scripts and
  injected into containers via docker-compose env_file directive
- Add .env with COMPOSE_PROFILES=mesh so docker compose defaults to
  mesh topology without requiring --profile

Deterministic mesh identity derivation:
- Add derive-keys.py: pure Python tool (no deps) deriving nsec/npub
  from sha256(mesh-name|node-id) via secp256k1 and BIP-173 bech32
- generate-configs.sh and build.sh accept optional mesh-name argument;
  Docker node identities are derived while external nodes keep
  hardcoded keys from topology YAML

Docker compose improvements:
- Add mesh-public profile service definitions
- build.sh now runs docker compose build automatically, providing a
  single command for the full binary+configs+images pipeline
- Ping test script supports mesh-public topology

Container services:
- Add HTTP server on port 8000 (IPv6-bound) serving static page,
  accessible over FIPS overlay via npub.fips hostnames
- Add rsync to container packages

Documentation:
- Comprehensive README update covering topology system, identity
  derivation, npubs.env, and container background services (SSH,
  iperf3, HTTP) with usage examples
This commit is contained in:
Johnathan Corgan
2026-02-18 13:20:03 +00:00
parent a1649ba4b7
commit a50473fe9f
10 changed files with 620 additions and 208 deletions
+15 -3
View File
@@ -1,11 +1,18 @@
#!/bin/bash
# Build the FIPS binary and copy it to the docker build context.
# Build the FIPS binary, generate configs, and build Docker images.
# Supports cross-compilation from macOS to Linux using cargo-zigbuild.
# Usage: ./build.sh [topology] [mesh-name]
# topology: mesh, mesh-public, chain, etc. (default: mesh)
# mesh-name: optional; derives unique node identities via sha256(mesh-name|node-id)
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
DOCKER_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# Topology to use (default: mesh)
TOPOLOGY="${1:-mesh}"
MESH_NAME="${2:-}"
# Find project root (directory containing Cargo.toml)
PROJECT_ROOT="$(cd "$DOCKER_DIR/../.." && pwd)"
if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then
@@ -14,6 +21,8 @@ if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then
exit 1
fi
echo "Using topology: $TOPOLOGY"
# Detect host OS
UNAME_S=$(uname -s)
CARGO_TARGET="x86_64-unknown-linux-musl"
@@ -54,6 +63,9 @@ fi
echo "Done. Binary at $DOCKER_DIR/fips"
echo ""
echo "Generating node configurations from templates..."
"$SCRIPT_DIR/generate-configs.sh" all
"$SCRIPT_DIR/generate-configs.sh" "$TOPOLOGY" $MESH_NAME
echo ""
echo "Next: cd $DOCKER_DIR && docker compose --profile mesh build"
echo "Building Docker images..."
docker compose --profile "$TOPOLOGY" build
echo ""
echo "Ready: docker compose --profile $TOPOLOGY up -d"
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Derive deterministic nostr nsec/npub from mesh-name and node-name.
Usage: derive-keys.py <mesh-name> <node-name>
Output: nsec=<hex>\nnpub=<bech32>
Derivation: nsec = sha256(mesh_name + "|" + node_name)
npub = bech32("npub", secp256k1_pubkey_x(nsec))
Pure Python, no external dependencies.
"""
import hashlib
import sys
# --- secp256k1 ---
P = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F
Gx = 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798
Gy = 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8
def _modinv(a, m):
return pow(a, m - 2, m)
def _point_add(p1, p2):
if p1 is None:
return p2
if p2 is None:
return p1
x1, y1 = p1
x2, y2 = p2
if x1 == x2 and y1 != y2:
return None
if x1 == x2:
lam = (3 * x1 * x1) * _modinv(2 * y1, P) % P
else:
lam = (y2 - y1) * _modinv(x2 - x1, P) % P
x3 = (lam * lam - x1 - x2) % P
y3 = (lam * (x1 - x3) - y1) % P
return (x3, y3)
def _scalar_mult(k, point):
result = None
addend = point
while k:
if k & 1:
result = _point_add(result, addend)
addend = _point_add(addend, addend)
k >>= 1
return result
# --- bech32 (BIP-173) ---
_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
def _bech32_polymod(values):
gen = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3]
chk = 1
for v in values:
b = chk >> 25
chk = (chk & 0x1FFFFFF) << 5 ^ v
for i in range(5):
chk ^= gen[i] if ((b >> i) & 1) else 0
return chk
def _bech32_encode(hrp, data_5bit):
hrp_expand = [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp]
polymod = _bech32_polymod(hrp_expand + data_5bit + [0] * 6) ^ 1
checksum = [(polymod >> 5 * (5 - i)) & 31 for i in range(6)]
return hrp + "1" + "".join(_CHARSET[d] for d in data_5bit + checksum)
def _convertbits(data, frombits, tobits):
acc, bits, ret = 0, 0, []
maxv = (1 << tobits) - 1
for value in data:
acc = (acc << frombits) | value
bits += frombits
while bits >= tobits:
bits -= tobits
ret.append((acc >> bits) & maxv)
if bits:
ret.append((acc << (tobits - bits)) & maxv)
return ret
# --- public API ---
def derive(mesh_name, node_name):
nsec_hex = hashlib.sha256(f"{mesh_name}|{node_name}".encode()).hexdigest()
k = int(nsec_hex, 16)
pub = _scalar_mult(k, (Gx, Gy))
x_hex = format(pub[0], "064x")
data_5bit = _convertbits(list(bytes.fromhex(x_hex)), 8, 5)
npub = _bech32_encode("npub", data_5bit)
return nsec_hex, npub
if __name__ == "__main__":
if len(sys.argv) != 3:
print(f"Usage: {sys.argv[0]} <mesh-name> <node-name>", file=sys.stderr)
sys.exit(1)
nsec, npub = derive(sys.argv[1], sys.argv[2])
print(f"nsec={nsec}")
print(f"npub={npub}")
@@ -1,7 +1,10 @@
#!/bin/bash
# Generate FIPS node configuration files from template and topology definition.
#
# Usage: ./generate-configs.sh [mesh|chain|all]
# Usage: ./generate-configs.sh <topology> [mesh-name]
# topology: mesh, mesh-public, chain, etc.
# mesh-name: optional; when given, docker node identities are derived
# deterministically via sha256(mesh-name|node-id)
set -e
@@ -9,79 +12,97 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_DIR="$SCRIPT_DIR/../configs"
GENERATED_DIR="$SCRIPT_DIR/../generated-configs"
TEMPLATE_FILE="$CONFIG_DIR/node.template.yaml"
DERIVE_KEYS="$SCRIPT_DIR/derive-keys.py"
# Node data lookup functions
get_nsec() {
case "$1" in
a) echo "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" ;;
b) echo "b102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fb0" ;;
c) echo "c102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fc0" ;;
d) echo "d102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fd0" ;;
e) echo "e102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fe0" ;;
esac
# Parse topology YAML to extract node attributes
# Usage: get_node_attr <topology_file> <node_id> <attr_name>
get_node_attr() {
local topology_file="$1"
local node_id="$2"
local attr="$3"
# Handle both docker_ip and external_ip as "address"
if [ "$attr" = "address" ]; then
local ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "docker_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
if [ -z "$ip" ]; then
ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "external_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
fi
echo "$ip"
else
grep -A 10 "^ $node_id:" "$topology_file" | grep "${attr}:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/'
fi
}
get_npub() {
case "$1" in
a) echo "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m" ;;
b) echo "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le" ;;
c) echo "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6" ;;
d) echo "npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl" ;;
e) echo "npub1wf8akf8lu2zdkjkmwhl75pqvven654mpv4sz2x2tprl5265mgrzq8nhak4" ;;
esac
# Check if a node is external (has external_ip instead of docker_ip)
is_external_node() {
local topology_file="$1"
local node_id="$2"
local docker_ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "docker_ip:" | head -1)
[ -z "$docker_ip" ]
}
get_docker_ip() {
case "$1" in
a) echo "172.20.0.10" ;;
b) echo "172.20.0.11" ;;
c) echo "172.20.0.12" ;;
d) echo "172.20.0.13" ;;
e) echo "172.20.0.14" ;;
esac
# Get peers list from topology
get_peers() {
local topology_file="$1"
local node_id="$2"
grep -A 10 "^ $node_id:" "$topology_file" | grep "peers:" | head -1 | \
sed 's/.*: *\[\(.*\)\].*/\1/' | \
sed 's/,/ /g' | \
tr -s ' ' | \
sed 's/^ *//;s/ *$//'
}
get_mesh_peers() {
case "$1" in
a) echo "d e" ;;
b) echo "c" ;;
c) echo "b d e" ;;
d) echo "a c e" ;;
e) echo "a c d" ;;
esac
# Get all node IDs from topology file
get_node_ids() {
local topology_file="$1"
grep "^ [a-z][a-z0-9_-]*:" "$topology_file" | sed 's/^ \([a-z][a-z0-9_-]*\):.*/\1/'
}
get_chain_peers() {
case "$1" in
a) echo "b" ;;
b) echo "a c" ;;
c) echo "b d" ;;
d) echo "c e" ;;
e) echo "d" ;;
esac
# Resolve nsec and npub for a node.
# If MESH_NAME is set and node is not external, derive from mesh-name.
# Otherwise use the value from the topology YAML.
# Output: two lines: nsec=<hex>\nnpub=<bech32>
resolve_keys() {
local topology_file="$1"
local node_id="$2"
if [ -n "$MESH_NAME" ] && ! is_external_node "$topology_file" "$node_id"; then
python3 "$DERIVE_KEYS" "$MESH_NAME" "$node_id"
else
local nsec
local npub
nsec=$(get_node_attr "$topology_file" "$node_id" "nsec")
npub=$(get_node_attr "$topology_file" "$node_id" "npub")
echo "nsec=$nsec"
echo "npub=$npub"
fi
}
generate_peer_block() {
local peer_id="$1"
local topology_file="$1"
local peer_id="$2"
local peer_npub="${RESOLVED_NPUB[$peer_id]}"
local peer_ip=$(get_node_attr "$topology_file" "$peer_id" "address")
cat <<EOF
- npub: "$(get_npub "$peer_id")"
- npub: "$peer_npub"
alias: "node-$peer_id"
addresses:
- transport: udp
addr: "$(get_docker_ip "$peer_id"):4000"
addr: "$peer_ip:4000"
connect_policy: auto_connect
EOF
}
generate_config() {
local node_id="$1"
local topology="$2"
local peers="$3"
local node_name=$(echo "$node_id" | tr '[:lower:]' '[:upper:]')
local template
template=$(cat "$TEMPLATE_FILE")
local topology_file="$2"
local output_file="$3"
local node_npub="${RESOLVED_NPUB[$node_id]}"
local node_nsec="${RESOLVED_NSEC[$node_id]}"
local peers=$(get_peers "$topology_file" "$node_id")
# Generate peers section
local peers_config=""
if [ -n "$peers" ]; then
@@ -89,67 +110,97 @@ generate_config() {
if [ -n "$peers_config" ]; then
peers_config="$peers_config"$'\n'
fi
peers_config="$peers_config$(generate_peer_block "$peer_id")"
peers_config="$peers_config$(generate_peer_block "$topology_file" "$peer_id")"
done
else
peers_config=" []"
fi
# Replace template variables
# Read and process template
local template=$(cat "$TEMPLATE_FILE")
local config="$template"
config="${config//\{\{NODE_NAME\}\}/$node_name}"
config="${config//\{\{TOPOLOGY\}\}/$topology}"
config="${config//\{\{NPUB\}\}/$(get_npub "$node_id")}"
config="${config//\{\{NSEC\}\}/$(get_nsec "$node_id")}"
config="${config//\{\{NODE_NAME\}\}/$(echo "$node_id" | tr '[:lower:]' '[:upper:]')}"
config="${config//\{\{TOPOLOGY\}\}/$(basename "$topology_file" .yaml)}"
config="${config//\{\{NPUB\}\}/$node_npub}"
config="${config//\{\{NSEC\}\}/$node_nsec}"
config="${config//\{\{PEERS\}\}/$peers_config}"
echo "$config"
echo "$config" > "$output_file"
}
# Associative arrays for resolved keys (populated in generate_topology)
declare -A RESOLVED_NSEC
declare -A RESOLVED_NPUB
generate_topology() {
local topology="$1"
local output_dir="$GENERATED_DIR/$topology"
echo "Generating $topology topology configs..."
local topology_name="$1"
local topology_file="$CONFIG_DIR/topologies/$topology_name.yaml"
local output_dir="$GENERATED_DIR/$topology_name"
if [ ! -f "$topology_file" ]; then
echo "Error: Topology file not found: $topology_file"
exit 1
fi
echo "Generating $topology_name topology configs..."
if [ -n "$MESH_NAME" ]; then
echo " Mesh name: $MESH_NAME (deriving docker node identities)"
fi
mkdir -p "$output_dir"
for node_id in a b c d e; do
local peers
if [ "$topology" = "mesh" ]; then
peers=$(get_mesh_peers "$node_id")
else
peers=$(get_chain_peers "$node_id")
# Phase 1: resolve keys for all nodes
for node_id in $(get_node_ids "$topology_file"); do
local keys=""
keys=$(resolve_keys "$topology_file" "$node_id")
RESOLVED_NSEC[$node_id]=$(echo "$keys" | grep "^nsec=" | cut -d= -f2)
RESOLVED_NPUB[$node_id]=$(echo "$keys" | grep "^npub=" | cut -d= -f2)
done
# Phase 2: generate config files for docker nodes
for node_id in $(get_node_ids "$topology_file"); do
# Skip external nodes (they don't need Docker config files)
if is_external_node "$topology_file" "$node_id"; then
echo " ⚠ Skipping $node_id (external node)"
continue
fi
local output_file="$output_dir/node-$node_id.yaml"
generate_config "$node_id" "$topology" "$peers" > "$output_file"
generate_config "$node_id" "$topology_file" "$output_file"
echo " ✓ Generated $output_file"
done
# Phase 3: write npubs.env
local env_file="$GENERATED_DIR/npubs.env"
echo "# Generated by generate-configs.sh (topology: $topology_name)" > "$env_file"
if [ -n "$MESH_NAME" ]; then
echo "# Mesh name: $MESH_NAME" >> "$env_file"
fi
for node_id in $(get_node_ids "$topology_file"); do
local var_name="NPUB_$(echo "$node_id" | tr '[:lower:]' '[:upper:]')"
echo "${var_name}=${RESOLVED_NPUB[$node_id]}" >> "$env_file"
done
echo " ✓ Generated $env_file"
}
main() {
local requested="${1:-all}"
case "$requested" in
mesh)
generate_topology "mesh"
;;
chain)
generate_topology "chain"
;;
all)
generate_topology "mesh"
generate_topology "chain"
;;
*)
echo "Error: Unknown topology '$requested'"
echo "Usage: $0 [mesh|chain|all]"
exit 1
;;
esac
local requested="${1:-mesh}"
# Support any topology file in the topologies directory
if [ -f "$CONFIG_DIR/topologies/$requested.yaml" ]; then
generate_topology "$requested"
else
echo "Error: Unknown topology '$requested'"
echo "Usage: $0 <topology> [mesh-name]"
echo ""
echo "Available topologies:"
ls -1 "$CONFIG_DIR/topologies/" | sed 's/\.yaml$//' | sed 's/^/ - /'
exit 1
fi
echo ""
echo "✓ All configurations generated successfully!"
}
main "$@"
MESH_NAME="${2:-}"
main "$@"
+9 -13
View File
@@ -23,19 +23,15 @@ PARALLEL=8
PASSED=0
FAILED=0
# Node identities
NPUB_A="npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
NPUB_B="npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le"
NPUB_C="npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6"
NPUB_D="npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl"
NPUB_E="npub1wf8akf8lu2zdkjkmwhl75pqvven654mpv4sz2x2tprl5265mgrzq8nhak4"
# FIPS IPv6 addresses (derived from npubs)
ADDR_A="fd69:e08d:65cc:3a6b:9c2c:2ac4:bd40:5e4b"
ADDR_B="fd8e:302c:287e:b48d:e5b7:6eee:e8a7:4f1e"
ADDR_C="fdac:a221:4069:5044:e0e5:7a5f:5a0f:4d43"
ADDR_D="fdb6:8411:a191:6d48:c5f7:e5f5:e5a5:2a50"
ADDR_E="fded:7dee:d386:a546:e5f7:e5f5:e5a5:2a50"
# Node identities (from generated env file)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
fi
# shellcheck source=../generated-configs/npubs.env
source "$ENV_FILE"
iperf_test() {
local server_node="$1"
+10 -7
View File
@@ -16,12 +16,15 @@ TIMEOUT=5
PASSED=0
FAILED=0
# Node identities
NPUB_A="npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
NPUB_B="npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le"
NPUB_C="npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6"
NPUB_D="npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl"
NPUB_E="npub1wf8akf8lu2zdkjkmwhl75pqvven654mpv4sz2x2tprl5265mgrzq8nhak4"
# Node identities (from generated env file)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
fi
# shellcheck source=../generated-configs/npubs.env
source "$ENV_FILE"
ping_test() {
local from="$1"
@@ -52,7 +55,7 @@ echo ""
echo "Waiting 3s for mesh convergence..."
sleep 3
if [ "$PROFILE" = "mesh" ]; then
if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
# Sparse mesh topology: A-B, B-C, C-D, D-E, E-A, A-D
# Test all 20 directed pairs (5 nodes × 4 targets each)
echo ""