v2.1.21 - Fixed caching_service_binary_path to ./caching_relay (relative to relay CWD), added caching_relay build step to make_and_restart_relay.sh, added default caching config values (admin npub, 4 bootstrap relays, kinds 0 and 10002)

api/index.html

@@ -622,17 +622,20 @@ WEB OF TRUST
 
             <div class="form-group">
                 <label for="caching-root-npubs">Root Npubs (one per line):</label>
-                <textarea id="caching-root-npubs" rows="4" placeholder="npub1..."></textarea>
+                <textarea id="caching-root-npubs" rows="4" placeholder="npub1...">npub13lm5wf8dvsdnc2894pkhch9uf8phvw9varrv8zf4sc885hhdmc8q6lx7ks</textarea>
             </div>
 
             <div class="form-group">
                 <label for="caching-bootstrap-relays">Bootstrap Relays (one per line):</label>
-                <textarea id="caching-bootstrap-relays" rows="4" placeholder="wss://relay.damus.io"></textarea>
+                <textarea id="caching-bootstrap-relays" rows="4" placeholder="wss://relay.damus.io">wss://relay.damus.io
+wss://nos.lol
+wss://relay.primal.net
+wss://laantungir.net/relay</textarea>
             </div>
 
             <div class="form-group">
                 <label for="caching-kinds">Kinds (comma-separated):</label>
-                <input type="text" id="caching-kinds" placeholder="1,3,6,10000,30023">
+                <input type="text" id="caching-kinds" placeholder="1,3,6,10000,30023" value="0,1,3,6,10000,10002,30023">
             </div>
 
             <div class="form-group">
@@ -688,16 +691,11 @@ WEB OF TRUST
 
             <div class="form-group">
                 <label for="caching-service-binary-path">Caching Service Binary Path:</label>
-                <input type="text" id="caching-service-binary-path" placeholder="/absolute/path/to/caching_relay" value="/home/user/lt/caching_relay/caching_relay">
+                <input type="text" id="caching-service-binary-path" placeholder="./caching_relay" value="./caching_relay">
             </div>
 
             <div class="form-group">
-                <label for="caching-service-config-path">Caching Service Config Path:</label>
-                <input type="text" id="caching-service-config-path" placeholder="/absolute/path/to/caching_relay_config.jsonc" value="/home/user/lt/caching_relay/caching_relay_config.jsonc">
-            </div>
-
-            <div class="form-group">
-                <label for="caching-service-pg-conn">Caching Service PG Connection (future):</label>
+                <label for="caching-service-pg-conn">Caching Service PG Connection:</label>
                 <input type="text" id="caching-service-pg-conn" placeholder="host=localhost port=5432 dbname=crelay user=crelay password=crelay" value="host=localhost port=5432 dbname=crelay user=crelay password=crelay">
             </div>
 

api/index.js

@@ -6841,7 +6841,6 @@ const CACHING_CONFIG_FIELDS = [
     { key: 'caching_inbox_active_poll_ms',     field: 'caching-inbox-active-poll',   type: 'integer' },
     { key: 'caching_inbox_idle_poll_ms',       field: 'caching-inbox-idle-poll',     type: 'integer' },
     { key: 'caching_service_binary_path',      field: 'caching-service-binary-path', type: 'string'  },
-    { key: 'caching_service_config_path',      field: 'caching-service-config-path', type: 'string'  },
     { key: 'caching_service_pg_conn',          field: 'caching-service-pg-conn',     type: 'string'  }
 ];
 
@@ -6863,9 +6862,26 @@ async function fetchCachingStatus() {
             throw new Error('Must be logged in to fetch caching status');
         }
 
-        // Reuse the existing config query infrastructure to refresh currentConfig
+        // Reuse the existing config query infrastructure to refresh currentConfig.
+        // fetchConfiguration() sends the config_query command but returns before
+        // the response arrives (the response updates currentConfig asynchronously
+        // via handleConfigQueryResponse -> displayConfiguration). We capture a
+        // marker so we can detect when currentConfig has been refreshed.
+        const prevConfigId = currentConfig ? (currentConfig.id || null) : null;
         await fetchConfiguration();
 
+        // Wait for currentConfig to be refreshed by the async config_query response.
+        // Poll for up to 5 seconds for a new currentConfig (or a changed id).
+        let waited = 0;
+        while (waited < 5000) {
+            const curId = currentConfig ? (currentConfig.id || null) : null;
+            if (currentConfig && curId !== prevConfigId) {
+                break;
+            }
+            await new Promise(resolve => setTimeout(resolve, 200));
+            waited += 200;
+        }
+
         // Populate form fields from currentConfig
         CACHING_CONFIG_FIELDS.forEach(field => {
             const value = getCachingConfigValue(field.key);
@@ -6956,6 +6972,16 @@ async function applyCachingConfig() {
                 value = String(value).trim();
             }
 
+            // Skip empty values for optional string/textarea-list fields.
+            // The backend (update_config_in_table) rejects empty strings to
+            // prevent accidental data loss, and config_update is atomic —
+            // one rejected field rolls back the entire batch. Fields like
+            // caching_root_npubs and caching_bootstrap_relays are optional
+            // and may legitimately be empty until configured.
+            if (value === '' && (field.type === 'textarea-list' || field.type === 'string')) {
+                return;
+            }
+
             let dataType = 'string';
             if (field.type === 'integer') {
                 dataType = 'integer';

caching/.gitignore

@@ -0,0 +1,31 @@
+# Database files (from c-relay running in this directory)
+*.db
+*.db-shm
+*.db-wal
+
+# Build artifacts
+*.o
+*.a
+
+# Binary (built into root, not version-controlled)
+caching_relay
+caching_relay_static_*
+
+# Logs
+*.log
+crelay.log
+
+# Test configs (not for version control)
+test_config*.jsonc
+
+# Note: caching_relay_config.jsonc IS version-controlled (it's the default
+# config + state store). Don't ignore it.
+
+# Docker build context temp (copied nostr_core_lib during static build)
+nostr_core_lib/
+
+# Editor
+*.swp
+*.swo
+*~
+.vscode/

caching/Dockerfile.alpine-musl

@@ -0,0 +1,125 @@
+# Alpine-based MUSL static binary builder for caching_relay
+# Produces a truly portable binary with zero runtime dependencies.
+# Adapted from c-relay's Dockerfile.alpine-musl.
+
+ARG DEBUG_BUILD=false
+
+FROM alpine:3.19 AS builder
+
+ARG DEBUG_BUILD=false
+
+# Install build dependencies
+RUN apk add --no-cache \
+    build-base \
+    musl-dev \
+    git \
+    cmake \
+    pkgconfig \
+    autoconf \
+    automake \
+    libtool \
+    openssl-dev \
+    openssl-libs-static \
+    zlib-dev \
+    zlib-static \
+    curl-dev \
+    curl-static \
+    sqlite-dev \
+    linux-headers \
+    wget \
+    bash
+
+WORKDIR /build
+
+# Build libsecp256k1 static
+RUN cd /tmp && \
+    git clone https://github.com/bitcoin-core/secp256k1.git && \
+    cd secp256k1 && \
+    ./autogen.sh && \
+    ./configure --enable-static --disable-shared --prefix=/usr \
+        CFLAGS="-fPIC" && \
+    make -j$(nproc) && \
+    make install && \
+    rm -rf /tmp/secp256k1
+
+# Build libwebsockets static with minimal features
+RUN cd /tmp && \
+    git clone --depth 1 --branch v4.3.3 https://github.com/warmcat/libwebsockets.git && \
+    cd libwebsockets && \
+    mkdir build && cd build && \
+    cmake .. \
+        -DLWS_WITH_STATIC=ON \
+        -DLWS_WITH_SHARED=OFF \
+        -DLWS_WITH_SSL=ON \
+        -DLWS_WITHOUT_TESTAPPS=ON \
+        -DLWS_WITHOUT_TEST_SERVER=ON \
+        -DLWS_WITHOUT_TEST_CLIENT=ON \
+        -DLWS_WITHOUT_TEST_PING=ON \
+        -DLWS_WITH_HTTP2=OFF \
+        -DLWS_WITH_LIBUV=OFF \
+        -DLWS_WITH_LIBEVENT=OFF \
+        -DLWS_IPV6=ON \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_C_FLAGS="-fPIC" && \
+    make -j$(nproc) && \
+    make install && \
+    rm -rf /tmp/libwebsockets
+
+# Copy nostr_core_lib source (sibling project)
+COPY nostr_core_lib /build/nostr_core_lib/
+
+# Build nostr_core_lib with the NIPs needed by the relay pool + signer:
+# 1 (basic), 6 (keys), 19 (npub bech32), 4 (legacy encryption, signer dep),
+# 42 (auth, relay pool dep), 44 (modern encryption, signer dep)
+RUN cd nostr_core_lib && \
+    chmod +x build.sh && \
+    sed -i 's/CFLAGS="-Wall -Wextra -std=c99 -fPIC -O2"/CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 -Wall -Wextra -std=c99 -fPIC -O2"/' build.sh && \
+    rm -f *.o *.a 2>/dev/null || true && \
+    ./build.sh --nips=1,4,6,19,42,44 && \
+    if [ -f libnostr_core_arm64.a ]; then \
+        cp libnostr_core_arm64.a libnostr_core.a; \
+    elif [ -f libnostr_core_x64.a ]; then \
+        cp libnostr_core_x64.a libnostr_core.a; \
+    else \
+        echo "ERROR: No supported nostr_core static library produced"; \
+        ls -la *.a 2>/dev/null || true; \
+        exit 1; \
+    fi
+
+# Copy caching_relay source LAST (only this layer rebuilds on source changes)
+COPY src/ /build/src/
+
+# Build caching_relay with full static linking.
+# No sqlite (the daemon doesn't use it), no c_utils (not needed).
+RUN if [ "$DEBUG_BUILD" = "true" ]; then \
+        CFLAGS="-g -O2 -DDEBUG -fno-omit-frame-pointer"; \
+        STRIP_CMD="echo 'Keeping debug symbols'"; \
+    else \
+        CFLAGS="-O2"; \
+        STRIP_CMD="strip /build/caching_relay_static"; \
+    fi && \
+    gcc -static $CFLAGS -Wall -Wextra -std=c99 \
+    -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 \
+    -I. -Isrc -Inostr_core_lib -Inostr_core_lib/nostr_core \
+    -Inostr_core_lib/cjson -Inostr_core_lib/nostr_websocket \
+    src/main.c src/debug.c src/jsonc_strip.c src/config.c src/state.c \
+    src/follow_graph.c src/relay_sink.c src/live_subscriber.c \
+    src/backfill.c src/relay_discovery.c \
+    -o /build/caching_relay_static \
+    nostr_core_lib/libnostr_core.a \
+    -lwebsockets -lssl -lcrypto -lsecp256k1 \
+    -lcurl -lz -lpthread -lm -ldl && \
+    eval "$STRIP_CMD"
+
+# Verify it's truly static
+RUN echo "=== Binary Information ===" && \
+    file /build/caching_relay_static && \
+    ls -lh /build/caching_relay_static && \
+    echo "=== Checking for dynamic dependencies ===" && \
+    (ldd /build/caching_relay_static 2>&1 || echo "Binary is static") && \
+    echo "=== Build complete ==="
+
+# Output stage - just the binary
+FROM scratch AS output
+COPY --from=builder /build/caching_relay_static /caching_relay_static

caching/Makefile

@@ -0,0 +1,67 @@
+# caching_relay Makefile - statically linked C99 binary, c-relay style
+
+# Use bash for reliable glob expansion in recipes (dash handles globs differently).
+SHELL := /bin/bash
+
+CC = gcc
+CFLAGS = -Wall -Wextra -std=c99 -g -O2
+
+# nostr_core_lib is a sibling of c-relay-pg (one dir up from caching/).
+NOSTR_CORE_DIR = ../nostr_core_lib
+
+INCLUDES = -I. -Isrc -I$(NOSTR_CORE_DIR) -I$(NOSTR_CORE_DIR)/nostr_core \
+           -I$(NOSTR_CORE_DIR)/cjson -I$(NOSTR_CORE_DIR)/nostr_websocket \
+           -I/usr/include/postgresql
+
+# -lsqlite3: nostr_core_lib's request_validator.x64.o references SQLite symbols;
+# we use --whole-archive so the linker pulls in all objects (needed for NIP-42
+# cross-references within the archive), which means SQLite must be satisfied.
+# No c_utils_lib: nostr_core_lib does not depend on it for the NIPs we use.
+# -lpq: PostgreSQL client library for the caching_event_inbox integration.
+LIBS = -lwebsockets -lssl -lcrypto -lsecp256k1 -lcurl -lz -ldl -lpthread -lm -lpq -lsqlite3
+
+MAIN_SRC = src/main.c src/debug.c src/jsonc_strip.c src/config.c src/state.c \
+           src/follow_graph.c src/relay_sink.c src/live_subscriber.c \
+           src/backfill.c src/relay_discovery.c \
+           src/pg_inbox.c src/pg_config.c
+
+# Architecture detection
+ARCH = $(shell uname -m)
+ifeq ($(ARCH),x86_64)
+    NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_x64.a
+else ifeq ($(ARCH),aarch64)
+    NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_arm64.a
+else ifeq ($(ARCH),arm64)
+    NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_arm64.a
+else
+    NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_x64.a
+endif
+
+# Binary goes in the c-relay-pg build/ directory so the launcher can find it.
+TARGET = ../build/caching_relay
+
+all: $(TARGET)
+
+# Build nostr_core_lib with the NIPs needed by the relay pool + signer:
+# 1 (basic), 6 (keys), 19 (npub bech32), 4 (legacy encryption, signer dep),
+# 42 (auth, relay pool dep), 44 (modern encryption, signer dep)
+$(NOSTR_CORE_LIB):
+	@echo "Building nostr_core_lib with required NIPs..."
+	cd $(NOSTR_CORE_DIR) && ./build.sh --nips=1,4,6,19,42,44
+
+$(TARGET): $(MAIN_SRC) $(NOSTR_CORE_LIB)
+	@echo "Compiling caching_relay for architecture: $(ARCH)"
+	@# Extract all objects from the static library and link them directly.
+	@# This avoids archive symbol resolution ordering issues (NIP-42 cross-refs).
+	@# Use a fixed temp dir and chain all commands with && so failures stop the build.
+	@rm -rf /tmp/cr_lib && mkdir -p /tmp/cr_lib && \
+	    ar x $(NOSTR_CORE_LIB) --output=/tmp/cr_lib && \
+	    echo "Extracted $$(ls /tmp/cr_lib/*.o | wc -l) objects" && \
+	    $(CC) $(CFLAGS) $(INCLUDES) $(MAIN_SRC) /tmp/cr_lib/*.o -o $(TARGET) $(LIBS) && \
+	    rm -rf /tmp/cr_lib && \
+	    echo "Build complete: $(TARGET)"
+
+clean:
+	rm -f $(TARGET)
+
+.PHONY: all clean

caching/README.md

@@ -0,0 +1,297 @@
+# caching_relay
+
+A C99 daemon that caches Nostr events from people you follow into a local relay,
+so your Nostr client can point at a single fast local relay instead of fanning
+out to dozens of upstream relays.
+
+## What it does
+
+1. Reads a `.jsonc` config file listing your **root npub(s)**, upstream relays,
+   a local relay URL, and the event kinds to cache.
+2. For each root npub, fetches its kind-3 contact list to discover **followed
+   pubkeys** (your follows + their follows).
+3. **Live-subscribes** to new events of the configured kinds from the union of
+   followed pubkeys.
+4. **Backfills** historical events using a progressive window-expansion strategy
+   (24h → 7d → 30d → 90d → 365d), round-robin per pubkey, throttled to be polite
+   to upstream relays.
+5. **Re-publishes** every fetched event to the local relay via a plain WebSocket
+   `EVENT` client connection (relay-agnostic - works with c-relay, c-relay-pg,
+   or any Nostr relay).
+6. Persists its backfill progress **in the config file itself** so a restart
+   resumes where it left off.
+
+## Build
+
+### Static binary (recommended, c-relay style)
+
+Produces a truly portable statically-linked MUSL binary with zero runtime
+dependencies:
+
+```bash
+./build_static.sh
+# or with debug symbols:
+./build_static.sh --debug
+# or cross-compile for arm64:
+./build_static.sh --arch arm64
+```
+
+Output: `caching_relay` (in the project root).
+
+Requires Docker. The build runs in an Alpine container that compiles
+libsecp256k1, libwebsockets, and nostr_core_lib from source, then statically
+links everything.
+
+### Local build (if you have the shared libs installed)
+
+```bash
+make
+```
+
+Output: `caching_relay` (in the project root).
+
+Requires: `libwebsockets`, `openssl`, `libsecp256k1`, `libcurl`, `zlib` shared
+libraries, and a pre-built `../nostr_core_lib/libnostr_core_x64.a`.
+
+## Usage
+
+```bash
+./caching_relay -d 3
+```
+
+The daemon looks for `caching_relay_config.jsonc` in the current directory by default.
+You can override with `-c`:
+
+```bash
+./caching_relay -c /path/to/my_config.jsonc -d 3
+```
+
+Options:
+- `-c, --config <file>` - Path to `.jsonc` config file (default: `./caching_relay_config.jsonc`)
+- `-d, --debug <level>` - Log level 1-4 (1=error, 2=warn, 3=info, 4=debug). Default 3.
+- `-h, --help` - Show help
+
+Signals:
+- `SIGINT` / `SIGTERM` - Graceful shutdown (saves state, closes connections)
+- `SIGHUP` - Reload config (preserves backfill state)
+
+## Config file
+
+The config file [`caching_relay_config.jsonc`](caching_relay_config.jsonc) lives in the
+project root. It is JSONC (JSON with `//` and `/* */` comments). The daemon
+rewrites it as plain JSON when saving state (comments are not preserved on
+rewrite).
+
+Key fields:
+
+| Field | Description |
+|-------|-------------|
+| `root_npubs` | npubs whose kind-3 follows list we crawl |
+| `upstream_relays` | bootstrap relays for initial discovery (kind-3, kind-10002). Outbox relays are discovered dynamically via NIP-65. |
+| `local_relay` | relay to publish cached events into (never queried) |
+| `kinds` | event kinds to cache for followed people (e.g. `[1, 3, 6, 10000, 30023]`) |
+| `admin_kinds` | kinds to follow specifically for root (admin) npubs. Use `["*"]` for all kinds. If omitted, admin uses same `kinds` as everyone else. |
+| `backfill.window_schedule_seconds` | progressive window sizes in seconds |
+| `backfill.events_per_tick` | max events per pubkey per backfill tick |
+| `backfill.tick_interval_seconds` | delay between pubkey backfills (throttle) |
+| `live.enabled` | enable live subscription |
+| `follow_graph_refresh_seconds` | how often to re-resolve the follow graph |
+| `state.*` | managed by the daemon - do not hand-edit |
+
+## Architecture
+
+See [`plans/plan.md`](plans/plan.md) for the full architecture document with
+Mermaid diagrams.
+
+The daemon uses a **NIP-65 outbox model**: instead of querying a fixed set of
+bootstrap relays for everything, it discovers which relays each followed pubkey
+actually posts to (via kind 10002 relay lists) and connects to those relays
+dynamically. The `upstream_relays` in the config serve only as **bootstrap
+relays** -- the initial set used to discover kind-3 and kind-10002 events before
+the outbox relay map is built.
+
+### Relay pool selection: minimum covering set
+
+After discovering each followed pubkey's outbox relays (from their kind 10002),
+the daemon computes the **minimum set of relays that covers all followed
+pubkeys**. This is the classic set cover problem, solved with a greedy
+approximation:
+
+1. Build a map: `{relay_url -> set of pubkeys that list it in their 10002}`
+2. Greedily pick the relay that covers the most uncovered pubkeys
+3. Repeat until all pubkeys are covered (or no more relays to pick)
+4. Always include the bootstrap relays in the final set (they may have events
+   from pubkeys that don't publish a kind 10002)
+
+This minimizes the number of WebSocket connections while ensuring every
+followed pubkey is reachable. The daemon logs the selected relay set and which
+pubkeys each relay covers.
+
+Pubkeys that have no kind 10002 (or whose 10002 lists no relays) are covered by
+the bootstrap relays as a fallback.
+
+```
+                    ┌─────────────────────────────────────────┐
+                    │              caching_relay               │
+                    │                                          │
+   upstream relays  │  upstream_pool          sink_pool        │  local relay
+   (damus, nos.lol) │  (query + subscribe)    (publish only)   │  (c-relay)
+        │           │          │                   │           │      │
+        │           │   follow_graph         relay_sink        │      │
+        └──────────►│   live_subscriber ──────►│              ├─────►│
+                    │   backfill ──────────────►│              │      │
+                    │          │                   │           │      │
+                    │       state + seen ring      │           │      │
+                    └──────────────────────────────────────────┘      │
+```
+
+Two `nostr_relay_pool_t` instances:
+- **upstream_pool** - holds bootstrap relays initially, then dynamically adds
+  outbox relays discovered from kind 10002. Used for `query_sync` (kind-3 fetch,
+  kind-10002 fetch, backfill) and the long-lived live subscription.
+- **sink_pool** - holds only the local relay; used exclusively for
+  `publish_async`. Never queried.
+
+## Startup Flow
+
+### First-time startup (empty local relay)
+
+The daemon has never run before. The local relay has no cached events. The
+daemon must bootstrap from the config's `upstream_relays` to discover the
+outbox relays for each followed pubkey.
+
+```
+ START
+   |
+   v
+ Load config (caching_relay_config.jsonc)
+   |  - root_npubs, upstream_relays (bootstrap), kinds, admin_kinds
+   |  - state.backfilled_until == 0  =>  first-time startup
+   |
+   v
+ Create upstream_pool with bootstrap relays only
+ Create sink_pool with local_relay
+   |
+   v
+ Phase 1: Resolve follow graph (from bootstrap relays)
+   |  - For each root npub: query_sync kind=3 from bootstrap relays
+   |  - Parse "p" tags => followed pubkey set
+   |  - Log: "follow: resolved N followed pubkeys"
+   |
+   v
+ Phase 2: Discover outbox relays (NIP-65, kind 10002) from bootstrap relays
+   |  - For each followed pubkey: query_sync kind=10002 from bootstrap relays
+   |  - Parse "r" tags => per-pubkey relay list
+   |  - Publish all kind-10002 events to local relay (cache them)
+   |  - Dynamically add discovered relays to upstream_pool
+   |  - Log: "relay_discovery: found N relays from M pubkeys"
+   |  - Log: "upstream_pool: now connected to N relays" + list them
+   |
+   v
+ Phase 3: Open live subscriptions on upstream_pool
+   |  - follows_sub: non-admin pubkeys + regular kinds
+   |  - admin_sub: admin npubs + admin_kinds (or all kinds)
+   |
+   v
+ Phase 4: Begin progressive backfill
+   |  - Window 0: 24h  -> query each pubkey from their outbox relays
+   |  - Window 1: 7d   -> ...
+   |  - Window 2: 30d  -> ...
+   |  - Publish all fetched events to local relay
+   |  - Save state to config file as each window completes
+   |
+   v
+ Steady-state: live sub + periodic follow-graph refresh + periodic
+   kind-10002 refresh + backfill re-cycle
+   |
+   v
+ SHUTDOWN (SIGINT/SIGTERM) -> save state -> clean exit
+```
+
+### Subsequent startup (local relay already has cached events)
+
+The daemon has run before. The local relay already has kind-3 and kind-10002
+events cached from the previous run. The daemon can read these from the local
+relay directly (fast, no network round-trip to bootstrap relays) and only falls
+back to bootstrap relays for pubkeys it cannot find locally.
+
+```
+ START
+   |
+   v
+ Load config (caching_relay_config.jsonc)
+   |  - state.backfilled_until > 0  =>  subsequent startup
+   |  - state.current_window_index, backfill_cursor preserved
+   |
+   v
+ Create upstream_pool with bootstrap relays
+ Create sink_pool with local_relay
+   |
+   v
+ Phase 1: Resolve follow graph (from LOCAL relay first, bootstrap fallback)
+   |  - Query local relay for kind=3 per root npub
+   |  - If found locally: use it (fast, no upstream query)
+   |  - If not found: fall back to bootstrap relays
+   |  - Parse "p" tags => followed pubkey set
+   |  - Log: "follow: resolved N followed pubkeys (L local, B bootstrap)"
+   |
+   v
+ Phase 2: Discover outbox relays (from LOCAL relay first, bootstrap fallback)
+   |  - Query local relay for kind=10002 per followed pubkey
+   |  - If found locally: use it (fast)
+   |  - If not found: fall back to bootstrap relays, cache result to local
+   |  - Parse "r" tags => per-pubkey relay list
+   |  - Dynamically add discovered relays to upstream_pool
+   |  - Log: "relay_discovery: found N relays (L local, B bootstrap)"
+   |  - Log: "upstream_pool: now connected to N relays" + list them
+   |
+   v
+ Phase 3: Open live subscriptions on upstream_pool
+   |  - (same as first-time)
+   |
+   v
+ Phase 4: Resume backfill from saved state
+   |  - Resume at state.current_window_index, state.backfill_cursor
+   |  - No re-pull of already-backfilled windows
+   |  - Continue progressive window expansion from where it left off
+   |
+   v
+ Steady-state (same as first-time)
+   |
+   v
+ SHUTDOWN -> save state -> clean exit
+```
+
+### Relay logging
+
+The daemon logs all relay activity so you can see exactly which relays are
+being used:
+
+- `upstream: added wss://relay.damus.io` -- each relay added to the pool
+- `relay_discovery: found 47 relays from 195 pubkeys` -- outbox discovery summary
+- `upstream_pool: 50 relays connected:` -- full relay list at startup
+- `relay_discovery: pubkey X -> wss://relay.example.com (local)` -- per-pubkey
+  relay source (local cache vs bootstrap)
+- `backfill: pubkey[3/195] abc123 -> wss://relay.example.com (12 events)` --
+  which relay served each backfill query
+
+## Dependencies
+
+- [nostr_core_lib](../nostr_core_lib) - built with NIPs 1, 4, 6, 19, 42, 44
+- libwebsockets, openssl, libsecp256k1, libcurl, zlib (all statically linked
+  in the Docker build)
+
+## Testing
+
+Start a local c-relay, then run the daemon:
+
+```bash
+# Start local relay (from c-relay project)
+cd ../c-relay/build && ./c_relay_static_x86_64 -p 8888 &
+
+# Run the caching daemon (uses ./caching_relay_config.jsonc by default)
+./caching_relay -d 3
+
+# Verify events are cached (from another terminal)
+nak req -k 1 -l 10 ws://127.0.0.1:8888
+```

caching/VERSION

@@ -0,0 +1 @@
+0.0.2

caching/build_static.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+# Build fully static MUSL binary for caching_relay using Alpine Docker.
+# Produces a truly portable binary with zero runtime dependencies.
+# Adapted from c-relay's build_static.sh.
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DOCKERFILE="$SCRIPT_DIR/Dockerfile.alpine-musl"
+
+# Parse command line arguments
+DEBUG_BUILD=false
+TARGET_ARCH=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --debug)
+            DEBUG_BUILD=true
+            shift
+            ;;
+        --arch)
+            if [[ -z "$2" ]]; then
+                echo "ERROR: --arch requires a value"
+                echo "Usage: $0 [--debug] [--arch <arm64|x86_64>]"
+                exit 1
+            fi
+            case "$2" in
+                arm64|x86_64)
+                    TARGET_ARCH="$2"
+                    ;;
+                *)
+                    echo "ERROR: Unsupported architecture '$2'"
+                    echo "Supported values: arm64, x86_64"
+                    exit 1
+                    ;;
+            esac
+            shift 2
+            ;;
+        *)
+            echo "ERROR: Unknown argument '$1'"
+            echo "Usage: $0 [--debug] [--arch <arm64|x86_64>]"
+            exit 1
+            ;;
+    esac
+done
+
+if [ "$DEBUG_BUILD" = true ]; then
+    echo "=========================================="
+    echo "caching_relay MUSL Static Binary Builder (DEBUG MODE)"
+    echo "=========================================="
+else
+    echo "=========================================="
+    echo "caching_relay MUSL Static Binary Builder (PRODUCTION MODE)"
+    echo "=========================================="
+fi
+echo "Project directory: $SCRIPT_DIR"
+echo ""
+
+if ! command -v docker &> /dev/null; then
+    echo "ERROR: Docker is not installed or not in PATH"
+    exit 1
+fi
+if ! docker info &> /dev/null; then
+    echo "ERROR: Docker daemon is not running or user not in docker group"
+    exit 1
+fi
+echo "Docker is available and running"
+echo ""
+
+# Detect host architecture
+ARCH=$(uname -m)
+case "$ARCH" in
+    x86_64)  ARCH="x86_64";;
+    aarch64) ARCH="arm64";;
+    arm64)   ARCH="arm64";;
+esac
+if [[ -n "$TARGET_ARCH" ]]; then
+    ARCH="$TARGET_ARCH"
+    echo "Using target architecture: $ARCH"
+else
+    echo "Detected host architecture: $ARCH"
+fi
+echo ""
+
+# nostr_core_lib is a sibling directory; copy it into the build context.
+# Docker cannot follow symlinks outside the context, so we do a real copy
+# (excluding .git and build artifacts) and clean up afterwards.
+NOSTR_CORE_DIR="$SCRIPT_DIR/../nostr_core_lib"
+if [ ! -d "$NOSTR_CORE_DIR" ]; then
+    echo "ERROR: nostr_core_lib not found at $NOSTR_CORE_DIR"
+    exit 1
+fi
+
+NEEDS_COPY=1
+if [ -d "$SCRIPT_DIR/nostr_core_lib" ] && [ ! -L "$SCRIPT_DIR/nostr_core_lib" ]; then
+    # Already a real directory (e.g. from a previous run); assume it's good.
+    NEEDS_COPY=0
+fi
+if [ "$NEEDS_COPY" = "1" ]; then
+    echo "Copying nostr_core_lib into build context..."
+    rm -rf "$SCRIPT_DIR/nostr_core_lib"
+    mkdir -p "$SCRIPT_DIR/nostr_core_lib"
+    rsync -a --exclude='.git' --exclude='*.a' --exclude='*.o' \
+          --exclude='.venv*' --exclude='backups' --exclude='verify_*' \
+          --exclude='rewrite_mirror' --exclude='websocket_debug' \
+          "$NOSTR_CORE_DIR/" "$SCRIPT_DIR/nostr_core_lib/"
+    COPIED_NOSTR_CORE=1
+fi
+
+# Build args
+BUILD_ARGS="--build-arg DEBUG_BUILD=$DEBUG_BUILD"
+if [ "$ARCH" = "arm64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "arm64" ]; then
+    BUILD_ARGS="$BUILD_ARGS --platform linux/arm64"
+fi
+
+IMAGE_TAG="caching_relay_builder:latest"
+
+echo "Building Docker image..."
+docker build $BUILD_ARGS \
+    -f "$DOCKERFILE" \
+    -t "$IMAGE_TAG" \
+    "$SCRIPT_DIR" 2>&1
+
+BUILD_RC=$?
+
+# Clean up the copied nostr_core_lib to keep the project dir tidy.
+if [ "${COPIED_NOSTR_CORE:-0}" = "1" ]; then
+    echo "Cleaning up copied nostr_core_lib from build context..."
+    rm -rf "$SCRIPT_DIR/nostr_core_lib"
+fi
+
+if [ $BUILD_RC -ne 0 ]; then
+    exit $BUILD_RC
+fi
+
+echo ""
+echo "Extracting binary from image..."
+# The output stage is FROM scratch with no CMD, so pass an empty command.
+CONTAINER_ID=$(docker create "$IMAGE_TAG" "")
+docker cp "$CONTAINER_ID:/caching_relay_static" "$SCRIPT_DIR/caching_relay"
+docker rm "$CONTAINER_ID" >/dev/null
+chmod +x "$SCRIPT_DIR/caching_relay"
+
+echo ""
+echo "=== Build complete ==="
+ls -lh "$SCRIPT_DIR/caching_relay"
+file "$SCRIPT_DIR/caching_relay"

caching/src/backfill.c

@@ -0,0 +1,432 @@
+/*
+ * caching_relay - progressive window-expansion backfill
+ *
+ * Uses until-based pagination so authors with more events than the page size
+ * are fully drained across multiple ticks instead of being skipped. In
+ * PostgreSQL mode, per-author/window progress is persisted to the
+ * caching_backfill_progress table so the service can resume after restart.
+ */
+#define _GNU_SOURCE
+#include "backfill.h"
+#include "follow_graph.h"
+#include "pg_inbox.h"
+#include "debug.h"
+
+#include "../nostr_core_lib/cjson/cJSON.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+
+/* ------------------------------------------------------------------ */
+/* Helpers                                                             */
+/* ------------------------------------------------------------------ */
+
+/* Returns 1 if the PostgreSQL inbox is initialized (PG mode active).
+ * Probes the config table; a non-NULL result means the connection is up. */
+static int pg_mode_active(void) {
+    char *v = pg_inbox_get_config_value("caching_root_npubs");
+    if (v) { free(v); return 1; }
+    char *v2 = pg_inbox_get_config_value("caching_kinds");
+    if (v2) { free(v2); return 1; }
+    return 0;
+}
+
+/* Find the oldest (minimum) created_at among an array of event JSON objects.
+ * Returns 1 and sets *out_oldest, or 0 if no events / no valid created_at. */
+static int find_oldest_created_at(cJSON **events, int count, long *out_oldest) {
+    long oldest = 0;
+    int found = 0;
+    for (int i = 0; i < count; i++) {
+        cJSON *ca = cJSON_GetObjectItem(events[i], "created_at");
+        if (!ca || !cJSON_IsNumber(ca)) continue;
+        long ts = (long)ca->valuedouble;
+        if (!found || ts < oldest) {
+            oldest = ts;
+            found = 1;
+        }
+    }
+    if (found && out_oldest) *out_oldest = oldest;
+    return found;
+}
+
+/* Count how many events in the array have created_at == ts. */
+static int count_at_timestamp(cJSON **events, int count, long ts) {
+    int n = 0;
+    for (int i = 0; i < count; i++) {
+        cJSON *ca = cJSON_GetObjectItem(events[i], "created_at");
+        if (ca && cJSON_IsNumber(ca) && (long)ca->valuedouble == ts) n++;
+    }
+    return n;
+}
+
+/* Build the kinds array for a given pubkey (admin vs regular).
+ * Returns NULL if no kinds filter should be applied (admin_all_kinds). */
+static cJSON *build_kinds(cr_config_t *cfg, const char *pk) {
+    int is_admin = cr_follow_is_root(cfg, pk);
+    if (is_admin && cfg->admin_all_kinds) {
+        return NULL;
+    }
+    cJSON *kinds = cJSON_CreateArray();
+    if (is_admin && cfg->admin_kind_count > 0) {
+        for (int i = 0; i < cfg->admin_kind_count; i++)
+            cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
+    } else {
+        for (int i = 0; i < cfg->kind_count; i++)
+            cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
+    }
+    return kinds;
+}
+
+/* Resolve the relay URL list to query for a given pubkey. Sets *out_urls and
+ * *out_n. Caller frees *out_urls (but not the strings, which alias internal
+ * storage). Returns 0 on success, -1 if no relays available. */
+static int resolve_relays(nostr_relay_pool_t *upstream,
+                          const cr_relay_map_t *relay_map,
+                          const char *pk,
+                          const char ***out_urls,
+                          int *out_n) {
+    *out_urls = NULL;
+    *out_n = 0;
+
+    if (relay_map) {
+        const cr_outbox_entry_t *oe = cr_relay_map_get_outbox(relay_map, pk);
+        if (oe && oe->relay_count > 0) {
+            const char **urls = malloc(oe->relay_count * sizeof(char *));
+            if (!urls) return -1;
+            for (int j = 0; j < oe->relay_count; j++) urls[j] = oe->relays[j];
+            *out_urls = urls;
+            *out_n = oe->relay_count;
+            return 0;
+        }
+    }
+
+    /* Fallback: all upstream relays. */
+    char **listed = NULL;
+    nostr_pool_relay_status_t *statuses = NULL;
+    int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+    if (n > 0) {
+        const char **urls = malloc(n * sizeof(char *));
+        if (!urls) { free(listed); free(statuses); return -1; }
+        for (int j = 0; j < n; j++) urls[j] = listed[j];
+        *out_urls = urls;
+        *out_n = n;
+    }
+    free(listed);
+    free(statuses);
+    return 0;
+}
+
+/* Issue a query_sync for one author with the given since/until/limit.
+ * Returns the events array (caller frees each event + the array) or NULL.
+ * Sets *out_count. */
+static cJSON **query_author(nostr_relay_pool_t *upstream,
+                            const cr_relay_map_t *relay_map,
+                            cr_config_t *cfg, const char *pk,
+                            long since, long until, int limit,
+                            int *out_count) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    cJSON_AddItemToArray(authors, cJSON_CreateString(pk));
+    cJSON_AddItemToObject(filter, "authors", authors);
+
+    cJSON *kinds = build_kinds(cfg, pk);
+    if (kinds) cJSON_AddItemToObject(filter, "kinds", kinds);
+
+    cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)since));
+    cJSON_AddItemToObject(filter, "until", cJSON_CreateNumber((double)until));
+    cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber((double)limit));
+
+    const char **urls = NULL;
+    int n = 0;
+    resolve_relays(upstream, relay_map, pk, &urls, &n);
+
+    cJSON **events = nostr_relay_pool_query_sync(upstream, urls, n, filter,
+                                                 out_count, 15000);
+    free(urls);
+    cJSON_Delete(filter);
+    return events;
+}
+
+/* ------------------------------------------------------------------ */
+/* Window advancement                                                  */
+/* ------------------------------------------------------------------ */
+
+static void advance_window(cr_backfill_t *bf, cr_config_t *cfg) {
+    /* Record that we've backfilled up to (now - current_window). */
+    cfg->state.backfilled_until = (long)time(NULL) - bf->current_window_s;
+    cfg->state.current_window_index = bf->window_index + 1;
+    cfg->state.backfill_cursor = 0;
+    if (!pg_mode_active()) {
+        cr_config_save_state(cfg);
+    }
+
+    DEBUG_INFO("backfill: window %d complete (%ld events). backfilled_until=%ld",
+                bf->window_index, bf->events_this_window, cfg->state.backfilled_until);
+
+    bf->window_index++;
+    if (bf->window_index >= cfg->backfill.window_count) {
+        DEBUG_INFO("backfill: all windows complete, entering steady-state");
+        bf->in_progress = 0;
+        return;
+    }
+    bf->current_window_s = cfg->backfill.window_schedule_seconds[bf->window_index];
+    bf->cursor = 0;
+    bf->until_cursor = 0;
+    bf->author_complete = 0;
+    bf->blocked = 0;
+    bf->blocked_until_ts = 0;
+    bf->events_this_window = 0;
+    bf->window_started = time(NULL);
+    DEBUG_INFO("backfill: advancing to window %d (%ld seconds)",
+                bf->window_index, bf->current_window_s);
+}
+
+/* Persist the current cursor state. In PG mode we rely on the per-author
+ * progress table; in legacy mode we write the .jsonc state. */
+static void persist_cursor(cr_backfill_t *bf, cr_config_t *cfg) {
+    cfg->state.backfill_cursor = bf->cursor;
+    if (!pg_mode_active()) {
+        cr_config_save_state(cfg);
+    }
+}
+
+/* Save progress for the current author to PG (no-op outside PG mode). */
+static void pg_save_current(cr_backfill_t *bf, cr_config_t *cfg,
+                            const char *pk, int complete) {
+    (void)cfg; /* window anchor is derived from bf + time(), cfg not needed */
+    if (!pg_mode_active() || !pk) return;
+    long anchor = (long)time(NULL) - bf->current_window_s;
+    pg_inbox_save_backfill_progress(pk, bf->window_index, anchor,
+                                    bf->until_cursor, complete);
+}
+
+/* ------------------------------------------------------------------ */
+/* Init                                                                */
+/* ------------------------------------------------------------------ */
+
+void cr_backfill_init(cr_backfill_t *bf, cr_config_t *cfg) {
+    memset(bf, 0, sizeof(*bf));
+    bf->window_index = cfg->state.current_window_index;
+    if (bf->window_index >= cfg->backfill.window_count) {
+        /* Already done; steady state. */
+        bf->in_progress = 0;
+        DEBUG_INFO("backfill: already complete (window index %d >= %d), steady-state",
+                    bf->window_index, cfg->backfill.window_count);
+        return;
+    }
+    bf->current_window_s = cfg->backfill.window_schedule_seconds[bf->window_index];
+    bf->cursor = cfg->state.backfill_cursor;
+    bf->until_cursor = 0;
+    bf->author_complete = 0;
+    bf->blocked = 0;
+    bf->blocked_until_ts = 0;
+    bf->in_progress = 1;
+    bf->window_started = time(NULL);
+
+    /* In PG mode, per-author restore is deferred to the first tick (the
+     * followed set is not resolved yet at init time). The tick loop skips
+     * already-complete authors via the progress table. */
+    DEBUG_INFO("backfill: starting at window %d (%ld seconds), cursor %d",
+                bf->window_index, bf->current_window_s, bf->cursor);
+}
+
+/* ------------------------------------------------------------------ */
+/* Tick                                                                */
+/* ------------------------------------------------------------------ */
+
+int cr_backfill_tick(cr_backfill_t *bf, cr_config_t *cfg,
+                     nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
+                     cr_sink_t *sink, const cr_relay_map_t *relay_map) {
+    if (!cfg->backfill.enabled) return -2;
+    if (!bf->in_progress) return -2;
+    if (followed->count == 0) return 0;
+
+    /* Throttle: only one query per tick_interval. */
+    time_t now = time(NULL);
+    if (bf->last_tick && (now - bf->last_tick) < cfg->backfill.tick_interval_seconds) {
+        return 0;
+    }
+
+    /* In PG mode, skip past authors already marked complete for this window
+     * (resume case). Restore the until_cursor for the first incomplete author. */
+    if (pg_mode_active()) {
+        while (bf->cursor < followed->count) {
+            const char *pk = followed->items[bf->cursor];
+            long uc = 0;
+            int complete = 0, found = 0;
+            if (pg_inbox_load_backfill_progress(pk, bf->window_index,
+                                                &uc, &complete, &found) == 0
+                && found && complete) {
+                DEBUG_TRACE("backfill: author[%d] %s already complete, skipping",
+                            bf->cursor, pk);
+                bf->cursor++;
+                cfg->state.backfill_cursor = bf->cursor;
+                continue;
+            }
+            if (found && uc > 0 && bf->until_cursor == 0) {
+                bf->until_cursor = uc;
+                DEBUG_TRACE("backfill: resumed author[%d] %s at until=%ld",
+                            bf->cursor, pk, bf->until_cursor);
+            }
+            break;
+        }
+        if (bf->cursor >= followed->count) {
+            advance_window(bf, cfg);
+            return 1;
+        }
+    }
+
+    const char *pk = followed->items[bf->cursor];
+
+    /* Determine the until cursor for this query. First query for an author:
+     * until = now. Subsequent queries: until = oldest event created_at - 1
+     * from the previous page (set at the end of the previous tick). */
+    long until = bf->until_cursor;
+    if (until == 0) {
+        until = (long)now;
+    }
+    long since = (long)now - bf->current_window_s;
+
+    int page_size = cfg->backfill.events_per_tick;
+    if (page_size < 1) page_size = 50;
+    int max_cap = CR_BACKFILL_MAX_PAGE_CAP;
+    if (max_cap < page_size) max_cap = page_size;
+
+    bf->last_tick = now;
+
+    DEBUG_TRACE("backfill: querying %s (window %ld, since %ld, until %ld, limit %d)",
+                pk, bf->current_window_s, since, until, page_size);
+
+    int ev_count = 0;
+    cJSON **events = query_author(upstream, relay_map, cfg, pk,
+                                  since, until, page_size, &ev_count);
+
+    /* Saturation handling: if we got exactly page_size events, the page may
+     * be truncated. Bump the limit up to max_cap and retry once. */
+    int limit_used = page_size;
+    if (events && ev_count == page_size && page_size < max_cap) {
+        for (int k = 0; k < ev_count; k++) cJSON_Delete(events[k]);
+        free(events);
+
+        DEBUG_TRACE("backfill: saturated page for %s, retrying with limit %d",
+                    pk, max_cap);
+        events = query_author(upstream, relay_map, cfg, pk,
+                              since, until, max_cap, &ev_count);
+        limit_used = max_cap;
+    }
+
+    /* If still saturated at the cap, we cannot safely paginate past the
+     * oldest timestamp (the relay may have more events at that exact ts).
+     * Mark the author blocked at that timestamp and move on; it will be
+     * retried in a later window sweep. */
+    if (events && ev_count == max_cap && max_cap > page_size) {
+        long oldest = 0;
+        find_oldest_created_at(events, ev_count, &oldest);
+        int at_oldest = count_at_timestamp(events, ev_count, oldest);
+
+        DEBUG_WARN("backfill: author %s saturated at limit %d (oldest ts %ld, "
+                    "%d events at that ts) - marking blocked, will retry next window",
+                    pk, max_cap, oldest, at_oldest);
+
+        /* Publish what we have. */
+        cr_sink_set_source_class(sink, CR_SINK_CLASS_BACKFILL);
+        for (int k = 0; k < ev_count; k++) {
+            cr_sink_publish(sink, events[k]);
+            cJSON_Delete(events[k]);
+            bf->events_this_window++;
+            bf->events_total++;
+        }
+        free(events);
+
+        bf->blocked = 1;
+        bf->blocked_until_ts = oldest;
+        bf->until_cursor = oldest; /* resume point for next window */
+        pg_save_current(bf, cfg, pk, 0); /* not complete - retry next window */
+
+        bf->cursor++;
+        persist_cursor(bf, cfg);
+        if (bf->cursor >= followed->count) {
+            advance_window(bf, cfg);
+        }
+        return 1;
+    }
+
+    /* Normal page processing. Capture the oldest timestamp BEFORE publishing
+     * so we can paginate if saturated. */
+    long oldest_ts = 0;
+    int have_oldest = 0;
+    if (events && ev_count > 0) {
+        have_oldest = find_oldest_created_at(events, ev_count, &oldest_ts);
+    }
+
+    if (events && ev_count > 0) {
+        cr_sink_set_source_class(sink, CR_SINK_CLASS_BACKFILL);
+        for (int k = 0; k < ev_count; k++) {
+            cr_sink_publish(sink, events[k]);
+            cJSON_Delete(events[k]);
+            bf->events_this_window++;
+            bf->events_total++;
+        }
+        free(events);
+    }
+
+    DEBUG_LOG("backfill: pubkey[%d/%d] %s -> %d events (until=%ld)",
+              bf->cursor, followed->count, pk, ev_count, until);
+
+    int saturated = (ev_count == limit_used) && (ev_count > 0);
+
+    if (ev_count == 0) {
+        /* No more events for this author in the window. */
+        bf->author_complete = 1;
+        bf->until_cursor = 0;
+        pg_save_current(bf, cfg, pk, 1);
+        bf->cursor++;
+        persist_cursor(bf, cfg);
+        if (bf->cursor >= followed->count) {
+            advance_window(bf, cfg);
+        }
+    } else if (!saturated) {
+        /* Last page for this author. */
+        bf->author_complete = 1;
+        bf->until_cursor = 0;
+        pg_save_current(bf, cfg, pk, 1);
+        bf->cursor++;
+        persist_cursor(bf, cfg);
+        if (bf->cursor >= followed->count) {
+            advance_window(bf, cfg);
+        }
+    } else {
+        /* Saturated: paginate. Set until = oldest - 1 and keep the same
+         * author for the next tick. The `until` parameter is inclusive, so
+         * subtracting 1 avoids re-fetching the oldest event. */
+        if (have_oldest) {
+            bf->until_cursor = oldest_ts - 1;
+            if (bf->until_cursor < since) {
+                /* Walked back past the window start; author is done. */
+                bf->author_complete = 1;
+                bf->until_cursor = 0;
+                pg_save_current(bf, cfg, pk, 1);
+                bf->cursor++;
+                persist_cursor(bf, cfg);
+                if (bf->cursor >= followed->count) {
+                    advance_window(bf, cfg);
+                }
+            } else {
+                pg_save_current(bf, cfg, pk, 0);
+            }
+        } else {
+            /* Could not determine oldest (no valid created_at); mark done. */
+            bf->author_complete = 1;
+            bf->until_cursor = 0;
+            pg_save_current(bf, cfg, pk, 1);
+            bf->cursor++;
+            persist_cursor(bf, cfg);
+            if (bf->cursor >= followed->count) {
+                advance_window(bf, cfg);
+            }
+        }
+    }
+
+    return 1;
+}

caching/src/backfill.h

@@ -0,0 +1,65 @@
+/*
+ * caching_relay - progressive window-expansion backfill.
+ *
+ * Round-robin through the followed-pubkey set, issuing query_sync per pubkey
+ * for the current window (since = now - window_seconds). When a full pass
+ * completes, advance the window and persist state to the config file.
+ *
+ * Pagination: each author is queried with an `until` cursor that walks
+ * backwards in time. If a query returns exactly `page_size` events (saturated),
+ * the limit is bumped up to a configurable cap and retried; if still saturated
+ * the author is marked "blocked" at that timestamp and retried in a later
+ * window sweep. Otherwise the until cursor advances to
+ * (oldest_event_created_at - 1) and the same author is queried again on the
+ * next tick until no more events are returned.
+ */
+#ifndef CACHING_RELAY_BACKFILL_H
+#define CACHING_RELAY_BACKFILL_H
+
+#include "config.h"
+#include "state.h"
+#include "relay_sink.h"
+#include "relay_discovery.h"
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+
+/* Maximum limit we will bump a saturated page to before marking an author
+ * blocked at a timestamp. Keeps a single query bounded. */
+#define CR_BACKFILL_MAX_PAGE_CAP 200
+
+typedef struct {
+    int      window_index;
+    long     current_window_s;
+    int      cursor;             /* current index into followed set */
+    long     until_cursor;       /* current until timestamp for the current author */
+    int      in_progress;        /* 1 if a window pass is in progress */
+    long     events_this_window; /* events pulled in current window */
+    long     events_total;       /* events pulled across all windows */
+    time_t   last_tick;          /* last time a pubkey was queried */
+    time_t   window_started;     /* when current window pass started */
+    int      author_complete;    /* 1 when current author is done, advance cursor */
+    int      blocked;            /* 1 when current author is blocked at a saturated timestamp */
+    long     blocked_until_ts;   /* timestamp where the author is blocked */
+} cr_backfill_t;
+
+/* Initialize backfill state from the persisted config state.
+ * In PostgreSQL mode (pg_inbox initialized), restores per-author progress
+ * for the current window from the caching_backfill_progress table. */
+void cr_backfill_init(cr_backfill_t *bf, cr_config_t *cfg);
+
+/* Perform one backfill tick. This queries ONE followed pubkey (the one at
+ * cfg->state.backfill_cursor) and publishes results to the sink. Uses
+ * until-based pagination so authors with more events than the page size are
+ * fully drained across multiple ticks. When a full round-robin pass completes,
+ * advances the window and saves state.
+ *
+ * Returns:
+ *   1  if a tick was performed (a pubkey was queried)
+ *   0  if throttled (tick_interval not elapsed) - caller should pump pools
+ *  -1  on hard error
+ *  -2  if backfill is complete (all windows done) - caller should go steady-state
+ */
+int cr_backfill_tick(cr_backfill_t *bf, cr_config_t *cfg,
+                     nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
+                     cr_sink_t *sink, const cr_relay_map_t *relay_map);
+
+#endif /* CACHING_RELAY_BACKFILL_H */

caching/src/config.c

@@ -0,0 +1,315 @@
+/*
+ * caching_relay - config parsing + persistent state (in the .jsonc file itself)
+ */
+#define _GNU_SOURCE
+#include "config.h"
+#include "jsonc_strip.h"
+#include "debug.h"
+
+#include "../nostr_core_lib/cjson/cJSON.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+static int read_file(const char *path, char **out, size_t *out_len) {
+    FILE *f = fopen(path, "rb");
+    if (!f) {
+        DEBUG_ERROR("cannot open config '%s': %s", path, strerror(errno));
+        return -1;
+    }
+    if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; }
+    long sz = ftell(f);
+    if (sz < 0) { fclose(f); return -1; }
+    rewind(f);
+    char *buf = malloc(sz + 1);
+    if (!buf) { fclose(f); return -1; }
+    size_t n = fread(buf, 1, sz, f);
+    fclose(f);
+    buf[n] = '\0';
+    *out = buf;
+    if (out_len) *out_len = n;
+    return 0;
+}
+
+static void copy_string_array(cJSON *arr, char *dst, int max, int *count_out,
+                              int elem_len) {
+    int count = 0;
+    if (arr && cJSON_IsArray(arr)) {
+        cJSON *item;
+        cJSON_ArrayForEach(item, arr) {
+            if (count >= max) break;
+            if (!cJSON_IsString(item)) continue;
+            const char *s = cJSON_GetStringValue(item);
+            if (!s) continue;
+            strncpy(dst + count * elem_len, s, elem_len - 1);
+            dst[count * elem_len + (elem_len - 1)] = '\0';
+            count++;
+        }
+    }
+    *count_out = count;
+}
+
+static void copy_int_array(cJSON *arr, int *dst, int max, int *count_out) {
+    int count = 0;
+    if (arr && cJSON_IsArray(arr)) {
+        cJSON *item;
+        cJSON_ArrayForEach(item, arr) {
+            if (count >= max) break;
+            if (cJSON_IsNumber(item)) {
+                dst[count++] = (int)cJSON_GetNumberValue(item);
+            }
+        }
+    }
+    *count_out = count;
+}
+
+static void copy_long_array(cJSON *arr, long *dst, int max, int *count_out) {
+    int count = 0;
+    if (arr && cJSON_IsArray(arr)) {
+        cJSON *item;
+        cJSON_ArrayForEach(item, arr) {
+            if (count >= max) break;
+            if (cJSON_IsNumber(item)) {
+                dst[count++] = (long)cJSON_GetNumberValue(item);
+            }
+        }
+    }
+    *count_out = count;
+}
+
+int cr_config_load(cr_config_t *cfg, const char *path) {
+    memset(cfg, 0, sizeof(*cfg));
+    strncpy(cfg->path, path, sizeof(cfg->path) - 1);
+
+    char *raw = NULL;
+    size_t raw_len = 0;
+    if (read_file(path, &raw, &raw_len) != 0) return -1;
+
+    char *stripped = jsonc_strip_comments(raw, raw_len);
+    free(raw);
+    if (!stripped) {
+        DEBUG_ERROR("failed to strip comments from config");
+        return -1;
+    }
+
+    cJSON *root = cJSON_Parse(stripped);
+    free(stripped);
+    if (!root) {
+        DEBUG_ERROR("failed to parse config JSON");
+        return -1;
+    }
+
+    /* root_npubs */
+    copy_string_array(cJSON_GetObjectItem(root, "root_npubs"),
+                      (char *)cfg->root_npubs, CR_MAX_ROOT_NPUBS,
+                      &cfg->root_npub_count, CR_NPUB_LEN);
+
+    /* upstream_relays */
+    copy_string_array(cJSON_GetObjectItem(root, "upstream_relays"),
+                      (char *)cfg->upstream_relays, CR_MAX_UPSTREAM,
+                      &cfg->upstream_count, CR_URL_LEN);
+
+    /* local_relay */
+    cJSON *local = cJSON_GetObjectItem(root, "local_relay");
+    if (local && cJSON_IsString(local)) {
+        strncpy(cfg->local_relay, cJSON_GetStringValue(local), CR_URL_LEN - 1);
+    }
+
+    /* kinds */
+    copy_int_array(cJSON_GetObjectItem(root, "kinds"),
+                   cfg->kinds, CR_MAX_KINDS, &cfg->kind_count);
+
+    /* admin_kinds - kinds to follow specifically for root (admin) npubs.
+     * Supports [*] to mean "all kinds" (no kind filter for admin). */
+    cJSON *ak = cJSON_GetObjectItem(root, "admin_kinds");
+    if (ak && cJSON_IsArray(ak)) {
+        cJSON *item;
+        cJSON_ArrayForEach(item, ak) {
+            if (cJSON_IsString(item)) {
+                const char *s = cJSON_GetStringValue(item);
+                if (s && strcmp(s, "*") == 0) {
+                    cfg->admin_all_kinds = 1;
+                    break;
+                }
+            }
+        }
+        if (!cfg->admin_all_kinds) {
+            copy_int_array(ak, cfg->admin_kinds, CR_MAX_KINDS, &cfg->admin_kind_count);
+        }
+    }
+
+    /* backfill */
+    cJSON *bf = cJSON_GetObjectItem(root, "backfill");
+    if (bf) {
+        cfg->backfill.enabled = cJSON_IsTrue(cJSON_GetObjectItem(bf, "enabled"));
+        copy_long_array(cJSON_GetObjectItem(bf, "window_schedule_seconds"),
+                        cfg->backfill.window_schedule_seconds, CR_MAX_WINDOWS,
+                        &cfg->backfill.window_count);
+        cJSON *ept = cJSON_GetObjectItem(bf, "events_per_tick");
+        if (ept) cfg->backfill.events_per_tick = (int)cJSON_GetNumberValue(ept);
+        cJSON *tis = cJSON_GetObjectItem(bf, "tick_interval_seconds");
+        if (tis) cfg->backfill.tick_interval_seconds = (int)cJSON_GetNumberValue(tis);
+        cJSON *wcs = cJSON_GetObjectItem(bf, "window_cooldown_seconds");
+        if (wcs) cfg->backfill.window_cooldown_seconds = (int)cJSON_GetNumberValue(wcs);
+    }
+
+    /* live */
+    cJSON *lv = cJSON_GetObjectItem(root, "live");
+    if (lv) {
+        cfg->live.enabled = cJSON_IsTrue(cJSON_GetObjectItem(lv, "enabled"));
+        cJSON *rsi = cJSON_GetObjectItem(lv, "resubscribe_interval_seconds");
+        if (rsi) cfg->live.resubscribe_interval_seconds = (int)cJSON_GetNumberValue(rsi);
+    }
+
+    /* follow_graph_refresh_seconds */
+    cJSON *fgr = cJSON_GetObjectItem(root, "follow_graph_refresh_seconds");
+    if (fgr) cfg->follow_graph_refresh_seconds = (int)cJSON_GetNumberValue(fgr);
+
+    /* state */
+    cJSON *st = cJSON_GetObjectItem(root, "state");
+    if (st) {
+        cJSON *bu = cJSON_GetObjectItem(st, "backfilled_until");
+        if (bu) cfg->state.backfilled_until = (long)cJSON_GetNumberValue(bu);
+        cJSON *cwi = cJSON_GetObjectItem(st, "current_window_index");
+        if (cwi) cfg->state.current_window_index = (int)cJSON_GetNumberValue(cwi);
+        cJSON *bc = cJSON_GetObjectItem(st, "backfill_cursor");
+        if (bc) cfg->state.backfill_cursor = (int)cJSON_GetNumberValue(bc);
+    }
+
+    cJSON_Delete(root);
+
+    /* Defaults if missing. */
+    if (cfg->backfill.window_count == 0) {
+        static const long def_windows[] = {86400, 604800, 2592000, 7776000, 31536000};
+        int n = (int)(sizeof(def_windows) / sizeof(def_windows[0]));
+        if (n > CR_MAX_WINDOWS) n = CR_MAX_WINDOWS;
+        for (int i = 0; i < n; i++) cfg->backfill.window_schedule_seconds[i] = def_windows[i];
+        cfg->backfill.window_count = n;
+    }
+    if (cfg->backfill.events_per_tick == 0) cfg->backfill.events_per_tick = 50;
+    if (cfg->backfill.tick_interval_seconds == 0) cfg->backfill.tick_interval_seconds = 5;
+    if (cfg->backfill.window_cooldown_seconds == 0) cfg->backfill.window_cooldown_seconds = 60;
+    if (cfg->live.resubscribe_interval_seconds == 0) cfg->live.resubscribe_interval_seconds = 300;
+    if (cfg->follow_graph_refresh_seconds == 0) cfg->follow_graph_refresh_seconds = 600;
+
+    /* Validation. */
+    if (cfg->root_npub_count == 0) {
+        DEBUG_ERROR("config: at least one root_npub required");
+        return -1;
+    }
+    if (cfg->upstream_count == 0) {
+        DEBUG_ERROR("config: at least one upstream_relay required");
+        return -1;
+    }
+    if (cfg->local_relay[0] == '\0') {
+        DEBUG_ERROR("config: local_relay required");
+        return -1;
+    }
+    if (cfg->kind_count == 0) {
+        DEBUG_ERROR("config: at least one kind required");
+        return -1;
+    }
+
+    DEBUG_INFO("config loaded: %d root npubs, %d upstream relays, %d kinds",
+                cfg->root_npub_count, cfg->upstream_count, cfg->kind_count);
+    return 0;
+}
+
+/* Build a fresh cJSON tree from the in-memory config and write it out.
+ * We re-emit plain JSON (comments are not preserved on rewrite). */
+int cr_config_save_state(cr_config_t *cfg) {
+    cJSON *root = cJSON_CreateObject();
+
+    cJSON *npubs = cJSON_CreateArray();
+    for (int i = 0; i < cfg->root_npub_count; i++)
+        cJSON_AddItemToArray(npubs, cJSON_CreateString(cfg->root_npubs[i]));
+    cJSON_AddItemToObject(root, "root_npubs", npubs);
+
+    cJSON *upstream = cJSON_CreateArray();
+    for (int i = 0; i < cfg->upstream_count; i++)
+        cJSON_AddItemToArray(upstream, cJSON_CreateString(cfg->upstream_relays[i]));
+    cJSON_AddItemToObject(root, "upstream_relays", upstream);
+
+    cJSON_AddItemToObject(root, "local_relay", cJSON_CreateString(cfg->local_relay));
+
+    cJSON *kinds = cJSON_CreateArray();
+    for (int i = 0; i < cfg->kind_count; i++)
+        cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
+    cJSON_AddItemToObject(root, "kinds", kinds);
+
+    /* admin_kinds */
+    cJSON *akinds = cJSON_CreateArray();
+    if (cfg->admin_all_kinds) {
+        cJSON_AddItemToArray(akinds, cJSON_CreateString("*"));
+    } else {
+        for (int i = 0; i < cfg->admin_kind_count; i++)
+            cJSON_AddItemToArray(akinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
+    }
+    cJSON_AddItemToObject(root, "admin_kinds", akinds);
+
+    cJSON *bf = cJSON_CreateObject();
+    cJSON_AddItemToObject(bf, "enabled", cJSON_CreateBool(cfg->backfill.enabled));
+    cJSON *ws = cJSON_CreateArray();
+    for (int i = 0; i < cfg->backfill.window_count; i++)
+        cJSON_AddItemToArray(ws, cJSON_CreateNumber(cfg->backfill.window_schedule_seconds[i]));
+    cJSON_AddItemToObject(bf, "window_schedule_seconds", ws);
+    cJSON_AddItemToObject(bf, "events_per_tick", cJSON_CreateNumber(cfg->backfill.events_per_tick));
+    cJSON_AddItemToObject(bf, "tick_interval_seconds", cJSON_CreateNumber(cfg->backfill.tick_interval_seconds));
+    cJSON_AddItemToObject(bf, "window_cooldown_seconds", cJSON_CreateNumber(cfg->backfill.window_cooldown_seconds));
+    cJSON_AddItemToObject(root, "backfill", bf);
+
+    cJSON *lv = cJSON_CreateObject();
+    cJSON_AddItemToObject(lv, "enabled", cJSON_CreateBool(cfg->live.enabled));
+    cJSON_AddItemToObject(lv, "resubscribe_interval_seconds", cJSON_CreateNumber(cfg->live.resubscribe_interval_seconds));
+    cJSON_AddItemToObject(root, "live", lv);
+
+    cJSON_AddItemToObject(root, "follow_graph_refresh_seconds",
+                          cJSON_CreateNumber(cfg->follow_graph_refresh_seconds));
+
+    cJSON *st = cJSON_CreateObject();
+    cJSON_AddItemToObject(st, "backfilled_until", cJSON_CreateNumber(cfg->state.backfilled_until));
+    cJSON_AddItemToObject(st, "current_window_index", cJSON_CreateNumber(cfg->state.current_window_index));
+    cJSON_AddItemToObject(st, "backfill_cursor", cJSON_CreateNumber(cfg->state.backfill_cursor));
+    cJSON_AddItemToObject(root, "state", st);
+
+    char *json = cJSON_Print(root);
+    cJSON_Delete(root);
+    if (!json) {
+        DEBUG_ERROR("config save: failed to serialize");
+        return -1;
+    }
+
+    /* Atomic write: temp file + rename. */
+    char tmp[1100];
+    snprintf(tmp, sizeof(tmp), "%s.tmp", cfg->path);
+    int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+    if (fd < 0) {
+        DEBUG_ERROR("config save: cannot open tmp '%s': %s", tmp, strerror(errno));
+        free(json);
+        return -1;
+    }
+    size_t jlen = strlen(json);
+    ssize_t w = write(fd, json, jlen);
+    close(fd);
+    free(json);
+    if (w < 0 || (size_t)w != jlen) {
+        DEBUG_ERROR("config save: short write");
+        return -1;
+    }
+    if (rename(tmp, cfg->path) != 0) {
+        DEBUG_ERROR("config save: rename failed: %s", strerror(errno));
+        return -1;
+    }
+    DEBUG_LOG("config state saved: backfilled_until=%ld window=%d cursor=%d",
+                 cfg->state.backfilled_until, cfg->state.current_window_index,
+                 cfg->state.backfill_cursor);
+    return 0;
+}
+
+void cr_config_free(cr_config_t *cfg) {
+    (void)cfg;
+}

caching/src/config.h

@@ -0,0 +1,81 @@
+/*
+ * caching_relay - config parsing + persistent state (in the .jsonc file itself)
+ */
+#ifndef CACHING_RELAY_CONFIG_H
+#define CACHING_RELAY_CONFIG_H
+
+#include <stddef.h>
+#include <time.h>
+
+/* Maximums to keep things statically sized and simple. */
+#define CR_MAX_ROOT_NPUBS   16
+#define CR_MAX_UPSTREAM     32
+#define CR_MAX_KINDS        32
+#define CR_MAX_WINDOWS      16
+#define CR_NPUB_LEN         64   /* npub1... bech32, generous */
+#define CR_URL_LEN          256
+#define CR_HEX_PUBKEY_LEN   65   /* 64 hex chars + NUL */
+
+typedef struct {
+    int    enabled;
+    long   window_schedule_seconds[CR_MAX_WINDOWS];
+    int    window_count;
+    int    events_per_tick;
+    int    tick_interval_seconds;
+    int    window_cooldown_seconds;
+} cr_backfill_config_t;
+
+typedef struct {
+    int enabled;
+    int resubscribe_interval_seconds;
+} cr_live_config_t;
+
+typedef struct {
+    long backfilled_until;       /* unix ts; 0 = nothing backfilled yet */
+    int  current_window_index;   /* index into window_schedule */
+    int  backfill_cursor;        /* round-robin cursor over followed pubkeys */
+} cr_state_t;
+
+typedef struct {
+    char  root_npubs[CR_MAX_ROOT_NPUBS][CR_NPUB_LEN];
+    int   root_npub_count;
+    /* Decoded hex pubkeys for root npubs (filled by follow_graph). */
+    char  root_hex[CR_MAX_ROOT_NPUBS][CR_HEX_PUBKEY_LEN];
+    int   root_hex_ready;
+
+    char  upstream_relays[CR_MAX_UPSTREAM][CR_URL_LEN];
+    int   upstream_count;
+
+    char  local_relay[CR_URL_LEN];
+
+    int   kinds[CR_MAX_KINDS];
+    int   kind_count;
+
+    /* Kinds to follow specifically for the root (admin) npubs.
+     * If admin_all_kinds is 1, grab everything (no kind filter). */
+    int   admin_kinds[CR_MAX_KINDS];
+    int   admin_kind_count;
+    int   admin_all_kinds;
+
+    cr_backfill_config_t backfill;
+    cr_live_config_t     live;
+    int                  follow_graph_refresh_seconds;
+
+    cr_state_t state;
+
+    /* Path the config was loaded from (for state write-back). */
+    char path[1024];
+} cr_config_t;
+
+/* Load + validate config from a .jsonc file. Returns 0 on success, -1 on error.
+ * On success cfg->path is set and cfg->state is populated from the file. */
+int cr_config_load(cr_config_t *cfg, const char *path);
+
+/* Persist the state sub-object back into the config file (temp + rename).
+ * Preserves all other config fields. Returns 0 on success, -1 on error. */
+int cr_config_save_state(cr_config_t *cfg);
+
+/* Free any heap resources held by cfg (currently none, but kept for future). */
+void cr_config_free(cr_config_t *cfg);
+
+#endif /* CACHING_RELAY_CONFIG_H */

caching/src/debug.c

@@ -0,0 +1,71 @@
+#include "debug.h"
+#include <stdarg.h>
+#include <string.h>
+
+/**
+ * @file debug.c
+ * @brief Debug and logging system implementation
+ *
+ * Provides a configurable logging system with timestamp formatting,
+ * level-based filtering, and optional file:line information.
+ */
+
+// Global debug level (default: no debug output)
+debug_level_t g_debug_level = DEBUG_LEVEL_NONE;
+
+/**
+ * @brief Initialize the debug system with a specific level
+ * @param level Debug level (0-5, clamped to valid range)
+ */
+void debug_init(int level) {
+    if (level < 0) level = 0;
+    if (level > 5) level = 5;
+    g_debug_level = (debug_level_t)level;
+}
+
+/**
+ * @brief Core logging function
+ * @param level Debug level for this message
+ * @param file Source file name (__FILE__)
+ * @param line Source line number (__LINE__)
+ * @param format printf-style format string
+ * @param ... Variable arguments for format string
+ */
+void debug_log(debug_level_t level, const char* file, int line, const char* format, ...) {
+    // Get timestamp
+    time_t now = time(NULL);
+    struct tm* tm_info = localtime(&now);
+    char timestamp[32];
+    strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", tm_info);
+
+    // Get level string
+    const char* level_str = "UNKNOWN";
+    switch (level) {
+        case DEBUG_LEVEL_ERROR: level_str = "ERROR"; break;
+        case DEBUG_LEVEL_WARN:  level_str = "WARN "; break;
+        case DEBUG_LEVEL_INFO:  level_str = "INFO "; break;
+        case DEBUG_LEVEL_DEBUG: level_str = "DEBUG"; break;
+        case DEBUG_LEVEL_TRACE: level_str = "TRACE"; break;
+        default: break;
+    }
+
+    // Print prefix with timestamp and level
+    printf("[%s] [%s] ", timestamp, level_str);
+
+    // Print source location when debug level is TRACE (5) or higher
+    if (file && g_debug_level >= DEBUG_LEVEL_TRACE) {
+        // Extract just the filename (not full path)
+        const char* filename = strrchr(file, '/');
+        filename = filename ? filename + 1 : file;
+        printf("[%s:%d] ", filename, line);
+    }
+
+    // Print message
+    va_list args;
+    va_start(args, format);
+    vprintf(format, args);
+    va_end(args);
+
+    printf("\n");
+    fflush(stdout);
+}

caching/src/debug.h

@@ -0,0 +1,65 @@
+#ifndef C_UTILS_DEBUG_H
+#define C_UTILS_DEBUG_H
+
+/**
+ * @file debug.h
+ * @brief Debug and logging system with configurable verbosity levels
+ *
+ * Provides a simple, efficient logging system with 5 levels:
+ * - ERROR: Critical errors
+ * - WARN: Warnings
+ * - INFO: Informational messages
+ * - DEBUG: Debug messages
+ * - TRACE: Detailed trace with file:line info
+ */
+
+#include <stdio.h>
+#include <time.h>
+
+// Debug levels
+typedef enum {
+    DEBUG_LEVEL_NONE = 0,  /**< No debug output */
+    DEBUG_LEVEL_ERROR = 1, /**< Critical errors only */
+    DEBUG_LEVEL_WARN = 2,  /**< Warnings and above */
+    DEBUG_LEVEL_INFO = 3,  /**< Informational messages and above */
+    DEBUG_LEVEL_DEBUG = 4, /**< Debug messages and above */
+    DEBUG_LEVEL_TRACE = 5  /**< Detailed trace with file:line info */
+} debug_level_t;
+
+// Global debug level (set at runtime via CLI)
+extern debug_level_t g_debug_level;
+
+/**
+ * @brief Initialize the debug system
+ * @param level Debug level (0-5, clamped to valid range)
+ */
+void debug_init(int level);
+
+/**
+ * @brief Core logging function
+ * @param level Debug level for this message
+ * @param file Source file name (__FILE__)
+ * @param line Source line number (__LINE__)
+ * @param format printf-style format string
+ * @param ... Variable arguments for format string
+ */
+void debug_log(debug_level_t level, const char* file, int line, const char* format, ...);
+
+// Convenience macros that check level before calling
+// Note: TRACE level (5) and above include file:line information for ALL messages
+#define DEBUG_ERROR(...) \
+    do { if (g_debug_level >= DEBUG_LEVEL_ERROR) debug_log(DEBUG_LEVEL_ERROR, __FILE__, __LINE__, __VA_ARGS__); } while(0)
+
+#define DEBUG_WARN(...) \
+    do { if (g_debug_level >= DEBUG_LEVEL_WARN) debug_log(DEBUG_LEVEL_WARN, __FILE__, __LINE__, __VA_ARGS__); } while(0)
+
+#define DEBUG_INFO(...) \
+    do { if (g_debug_level >= DEBUG_LEVEL_INFO) debug_log(DEBUG_LEVEL_INFO, __FILE__, __LINE__, __VA_ARGS__); } while(0)
+
+#define DEBUG_LOG(...) \
+    do { if (g_debug_level >= DEBUG_LEVEL_DEBUG) debug_log(DEBUG_LEVEL_DEBUG, __FILE__, __LINE__, __VA_ARGS__); } while(0)
+
+#define DEBUG_TRACE(...) \
+    do { if (g_debug_level >= DEBUG_LEVEL_TRACE) debug_log(DEBUG_LEVEL_TRACE, __FILE__, __LINE__, __VA_ARGS__); } while(0)
+
+#endif /* C_UTILS_DEBUG_H */

caching/src/follow_graph.c

@@ -0,0 +1,110 @@
+/*
+ * caching_relay - follow graph resolution
+ */
+#define _GNU_SOURCE
+#include "follow_graph.h"
+#include "debug.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+int cr_follow_is_root(const cr_config_t *cfg, const char *hex) {
+    if (!cfg->root_hex_ready) return 0;
+    for (int i = 0; i < cfg->root_npub_count; i++) {
+        if (strcmp(cfg->root_hex[i], hex) == 0) return 1;
+    }
+    return 0;
+}
+
+int cr_follow_decode_roots(cr_config_t *cfg) {
+    for (int i = 0; i < cfg->root_npub_count; i++) {
+        unsigned char pubkey[32];
+        if (nostr_decode_npub(cfg->root_npubs[i], pubkey) != NOSTR_SUCCESS) {
+            DEBUG_ERROR("follow: failed to decode npub '%s'", cfg->root_npubs[i]);
+            return -1;
+        }
+        nostr_bytes_to_hex(pubkey, 32, cfg->root_hex[i]);
+    }
+    cfg->root_hex_ready = 1;
+    DEBUG_INFO("follow: decoded %d root npubs", cfg->root_npub_count);
+    return 0;
+}
+
+/* Parse "p" tags from a kind-3 event and add pubkeys to the set. */
+static int parse_p_tags(cJSON *event, cr_pubkey_set_t *followed) {
+    cJSON *tags = cJSON_GetObjectItem(event, "tags");
+    if (!tags || !cJSON_IsArray(tags)) return 0;
+    int added = 0;
+    cJSON *tag;
+    cJSON_ArrayForEach(tag, tags) {
+        if (!cJSON_IsArray(tag)) continue;
+        cJSON *name = cJSON_GetArrayItem(tag, 0);
+        if (!name || !cJSON_IsString(name)) continue;
+        if (strcmp(cJSON_GetStringValue(name), "p") != 0) continue;
+        cJSON *pk = cJSON_GetArrayItem(tag, 1);
+        if (!pk || !cJSON_IsString(pk)) continue;
+        const char *hex = cJSON_GetStringValue(pk);
+        if (strlen(hex) != 64) continue;
+        if (cr_pubkey_set_add(followed, hex) == 1) added++;
+    }
+    return added;
+}
+
+int cr_follow_resolve(cr_config_t *cfg, nostr_relay_pool_t *upstream,
+                      cr_pubkey_set_t *followed) {
+    if (!cfg->root_hex_ready) {
+        if (cr_follow_decode_roots(cfg) != 0) return -1;
+    }
+
+    /* Seed the followed set with the root pubkeys themselves. */
+    for (int i = 0; i < cfg->root_npub_count; i++) {
+        cr_pubkey_set_add(followed, cfg->root_hex[i]);
+    }
+
+    int total_follows = 0;
+    for (int i = 0; i < cfg->root_npub_count; i++) {
+        /* Build filter: authors=[root], kinds=[3], limit=1 (most recent). */
+        cJSON *filter = cJSON_CreateObject();
+        cJSON *authors = cJSON_CreateArray();
+        cJSON_AddItemToArray(authors, cJSON_CreateString(cfg->root_hex[i]));
+        cJSON_AddItemToObject(filter, "authors", authors);
+        cJSON *kinds = cJSON_CreateArray();
+        cJSON_AddItemToArray(kinds, cJSON_CreateNumber(3));
+        cJSON_AddItemToObject(filter, "kinds", kinds);
+        cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(1));
+
+        const char **urls = NULL;
+        int n = 0;
+        char **listed = NULL;
+        nostr_pool_relay_status_t *statuses = NULL;
+        int listed_count = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+        if (listed_count > 0) {
+            urls = malloc(listed_count * sizeof(char *));
+            for (int j = 0; j < listed_count; j++) { urls[n++] = listed[j]; }
+        }
+        free(listed);
+        free(statuses);
+
+        int ev_count = 0;
+        cJSON **events = nostr_relay_pool_query_sync(upstream, urls, n,
+                                                     filter, &ev_count, 15000);
+        free(urls);
+        cJSON_Delete(filter);
+
+        if (events && ev_count > 0) {
+            int added = parse_p_tags(events[0], followed);
+            total_follows += added;
+            DEBUG_INFO("follow: root %d: kind-3 found, +%d follows (total set %d)",
+                        i, added, followed->count);
+            for (int k = 0; k < ev_count; k++) cJSON_Delete(events[k]);
+            free(events);
+        } else {
+            DEBUG_WARN("follow: root %d: no kind-3 found", i);
+            free(events);
+        }
+    }
+
+    DEBUG_INFO("follow: resolved %d total followed pubkeys (set size %d)",
+                total_follows, followed->count);
+    return 0;
+}

caching/src/follow_graph.h

@@ -0,0 +1,27 @@
+/*
+ * caching_relay - follow graph resolution.
+ *
+ * Decodes root npubs (NIP-19) to hex, queries their most recent kind-3
+ * contact list from the upstream pool, parses the "p" tags, and populates
+ * the in-memory followed-pubkey set (root pubkeys + their follows).
+ */
+#ifndef CACHING_RELAY_FOLLOW_GRAPH_H
+#define CACHING_RELAY_FOLLOW_GRAPH_H
+
+#include "config.h"
+#include "state.h"
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+
+/* Check if a hex pubkey is one of the root (admin) npubs. Returns 1 if yes. */
+int cr_follow_is_root(const cr_config_t *cfg, const char *hex);
+
+/* Decode all root_npubs in cfg into cfg->root_hex[]. Returns 0 on success. */
+int cr_follow_decode_roots(cr_config_t *cfg);
+
+/* Resolve the follow graph: for each root hex pubkey, query the most recent
+ * kind-3 from the upstream pool, parse "p" tags, and add root + follows to
+ * the followed set. Returns 0 on success, -1 on hard failure. */
+int cr_follow_resolve(cr_config_t *cfg, nostr_relay_pool_t *upstream,
+                      cr_pubkey_set_t *followed);
+
+#endif /* CACHING_RELAY_FOLLOW_GRAPH_H */

caching/src/jsonc_strip.c

@@ -0,0 +1,69 @@
+/*
+ * caching_relay - strip JSONC comments so cJSON can parse it.
+ */
+#define _GNU_SOURCE
+#include "jsonc_strip.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+char *jsonc_strip_comments(const char *input, size_t len) {
+    if (!input) return NULL;
+    if (len == 0) len = strlen(input);
+
+    /* Worst case: output is same size as input (no comments). +1 for NUL. */
+    char *out = malloc(len + 1);
+    if (!out) return NULL;
+
+    size_t i = 0, o = 0;
+    int in_string = 0;
+    int escape = 0;
+
+    while (i < len) {
+        char c = input[i];
+
+        if (in_string) {
+            out[o++] = c;
+            if (escape) {
+                escape = 0;
+            } else if (c == '\\') {
+                escape = 1;
+            } else if (c == '"') {
+                in_string = 0;
+            }
+            i++;
+            continue;
+        }
+
+        /* Not in a string. */
+        if (c == '"') {
+            in_string = 1;
+            out[o++] = c;
+            i++;
+            continue;
+        }
+
+        /* Line comment // */
+        if (c == '/' && i + 1 < len && input[i + 1] == '/') {
+            i += 2;
+            while (i < len && input[i] != '\n') i++;
+            continue;
+        }
+
+        /* Block comment / * ... * / */
+        if (c == '/' && i + 1 < len && input[i + 1] == '*') {
+            i += 2;
+            while (i < len && !(input[i] == '*' && i + 1 < len && input[i + 1] == '/')) i++;
+            if (i < len) i += 2; /* skip closing */
+            /* Preserve a space so tokens don't merge. */
+            out[o++] = ' ';
+            continue;
+        }
+
+        out[o++] = c;
+        i++;
+    }
+
+    out[o] = '\0';
+    return out;
+}

caching/src/jsonc_strip.h

@@ -0,0 +1,16 @@
+/*
+ * caching_relay - strip JSONC comments so cJSON can parse it.
+ *
+ * cJSON does not natively understand JSONC. This produces a heap buffer the
+ * caller must free() containing the comment-stripped JSON. String literals are
+ * respected so a double-slash inside a string is preserved.
+ */
+#ifndef CACHING_RELAY_JSONC_STRIP_H
+#define CACHING_RELAY_JSONC_STRIP_H
+
+#include <stddef.h>
+
+/* Returns malloc'd buffer (null-terminated) or NULL on alloc failure. */
+char *jsonc_strip_comments(const char *input, size_t len);
+
+#endif /* CACHING_RELAY_JSONC_STRIP_H */

caching/src/live_subscriber.c

@@ -0,0 +1,171 @@
+/*
+ * caching_relay - live subscription on the upstream pool
+ */
+#define _GNU_SOURCE
+#include "live_subscriber.h"
+#include "follow_graph.h"
+#include "debug.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+
+/* Context passed as user_data to the subscription callbacks. */
+typedef struct {
+    cr_sink_t *sink;
+    cr_live_t *live;
+} cr_live_ctx_t;
+
+static cr_live_ctx_t g_live_ctx;
+
+static void live_on_event(cJSON *event, const char *relay_url, void *user_data) {
+    cr_live_ctx_t *ctx = (cr_live_ctx_t *)user_data;
+    (void)relay_url;
+    ctx->live->events_received++;
+    cr_sink_publish(ctx->sink, event);
+}
+
+static void live_on_eose(cJSON **events, int event_count, void *user_data) {
+    (void)events; (void)event_count; (void)user_data;
+    DEBUG_LOG("live: EOSE (stored events flushed), staying open");
+}
+
+/* Build a filter for non-admin followed pubkeys with regular kinds. */
+static cJSON *build_follows_filter(cr_config_t *cfg, cr_pubkey_set_t *followed) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    for (int i = 0; i < followed->count; i++) {
+        /* Skip admin pubkeys - they get their own subscription. */
+        if (cr_follow_is_root(cfg, followed->items[i])) continue;
+        cJSON_AddItemToArray(authors, cJSON_CreateString(followed->items[i]));
+    }
+    cJSON_AddItemToObject(filter, "authors", authors);
+
+    cJSON *kinds = cJSON_CreateArray();
+    for (int i = 0; i < cfg->kind_count; i++)
+        cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
+    cJSON_AddItemToObject(filter, "kinds", kinds);
+
+    cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)time(NULL)));
+    return filter;
+}
+
+/* Build a filter for admin (root) pubkeys with admin_kinds (or all kinds). */
+static cJSON *build_admin_filter(cr_config_t *cfg) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    for (int i = 0; i < cfg->root_npub_count; i++)
+        cJSON_AddItemToArray(authors, cJSON_CreateString(cfg->root_hex[i]));
+    cJSON_AddItemToObject(filter, "authors", authors);
+
+    /* If admin_all_kinds, omit the kinds filter entirely (grab everything). */
+    if (!cfg->admin_all_kinds && cfg->admin_kind_count > 0) {
+        cJSON *kinds = cJSON_CreateArray();
+        for (int i = 0; i < cfg->admin_kind_count; i++)
+            cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
+        cJSON_AddItemToObject(filter, "kinds", kinds);
+    }
+
+    cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)time(NULL)));
+    return filter;
+}
+
+static int get_relay_urls(nostr_relay_pool_t *upstream, const char ***urls_out) {
+    char **listed = NULL;
+    nostr_pool_relay_status_t *statuses = NULL;
+    int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+    const char **urls = NULL;
+    if (n > 0) {
+        urls = malloc(n * sizeof(char *));
+        for (int j = 0; j < n; j++) urls[j] = listed[j];
+    }
+    free(listed);
+    free(statuses);
+    *urls_out = urls;
+    return n;
+}
+
+static nostr_pool_subscription_t *open_subscription(nostr_relay_pool_t *upstream,
+                                                     cJSON *filter) {
+    const char **urls = NULL;
+    int n = get_relay_urls(upstream, &urls);
+
+    nostr_pool_subscription_t *sub = nostr_relay_pool_subscribe(
+        upstream, urls, n, filter,
+        live_on_event, live_on_eose, &g_live_ctx,
+        0, 1, NOSTR_POOL_EOSE_FULL_SET, 0, 0);
+    free(urls);
+    return sub;
+}
+
+static int open_subs(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
+                     cr_pubkey_set_t *followed, cr_sink_t *sink) {
+    g_live_ctx.sink = sink;
+    g_live_ctx.live = live;
+
+    int admin_count = 0;
+    int follows_count = 0;
+    for (int i = 0; i < followed->count; i++) {
+        if (cr_follow_is_root(cfg, followed->items[i])) admin_count++;
+        else follows_count++;
+    }
+
+    /* Follows subscription (non-admin pubkeys, regular kinds). */
+    if (follows_count > 0) {
+        cJSON *filter = build_follows_filter(cfg, followed);
+        live->follows_sub = open_subscription(upstream, filter);
+        cJSON_Delete(filter);
+        if (!live->follows_sub) {
+            DEBUG_ERROR("live: follows subscribe failed");
+        } else {
+            DEBUG_INFO("live: follows sub - %d authors, %d kinds",
+                        follows_count, cfg->kind_count);
+        }
+    }
+
+    /* Admin subscription (root npubs, admin_kinds or all kinds). */
+    if (admin_count > 0 || cfg->root_npub_count > 0) {
+        cJSON *filter = build_admin_filter(cfg);
+        live->admin_sub = open_subscription(upstream, filter);
+        cJSON_Delete(filter);
+        if (!live->admin_sub) {
+            DEBUG_ERROR("live: admin subscribe failed");
+        } else {
+            if (cfg->admin_all_kinds) {
+                DEBUG_INFO("live: admin sub - %d authors, ALL kinds",
+                            cfg->root_npub_count);
+            } else {
+                DEBUG_INFO("live: admin sub - %d authors, %d kinds",
+                            cfg->root_npub_count, cfg->admin_kind_count);
+            }
+        }
+    }
+
+    live->last_resubscribe = time(NULL);
+    return 0;
+}
+
+int cr_live_open(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
+                 cr_pubkey_set_t *followed, cr_sink_t *sink) {
+    memset(live, 0, sizeof(*live));
+    return open_subs(live, cfg, upstream, followed, sink);
+}
+
+int cr_live_resubscribe(cr_live_t *live, cr_config_t *cfg,
+                        nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
+                        cr_sink_t *sink) {
+    cr_live_close(live);
+    DEBUG_INFO("live: resubscribing (events so far: %ld)", live->events_received);
+    return open_subs(live, cfg, upstream, followed, sink);
+}
+
+void cr_live_close(cr_live_t *live) {
+    if (live->follows_sub) {
+        nostr_pool_subscription_close(live->follows_sub);
+        live->follows_sub = NULL;
+    }
+    if (live->admin_sub) {
+        nostr_pool_subscription_close(live->admin_sub);
+        live->admin_sub = NULL;
+    }
+}

caching/src/live_subscriber.h

@@ -0,0 +1,37 @@
+/*
+ * caching_relay - live subscription on the upstream pool.
+ *
+ * Builds filters from the followed-pubkey set + configured kinds, opens
+ * long-lived subscriptions, and routes on_event to the relay sink.
+ *
+ * Two subscriptions are opened:
+ *   1. "follows" sub: non-admin followed pubkeys + regular kinds
+ *   2. "admin" sub: root (admin) npubs + admin_kinds (or all kinds if [*])
+ */
+#ifndef CACHING_RELAY_LIVE_SUBSCRIBER_H
+#define CACHING_RELAY_LIVE_SUBSCRIBER_H
+
+#include "config.h"
+#include "state.h"
+#include "relay_sink.h"
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+
+typedef struct {
+    nostr_pool_subscription_t *follows_sub;  /* non-admin pubkeys, regular kinds */
+    nostr_pool_subscription_t *admin_sub;    /* admin pubkeys, admin_kinds */
+    long                       events_received;
+    time_t                     last_resubscribe;
+} cr_live_t;
+
+/* Open the live subscription(s). Returns 0 on success. */
+int  cr_live_open(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
+                  cr_pubkey_set_t *followed, cr_sink_t *sink);
+
+/* Close + reopen the subscription(s). Returns 0 on success. */
+int  cr_live_resubscribe(cr_live_t *live, cr_config_t *cfg,
+                         nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
+                         cr_sink_t *sink);
+
+void cr_live_close(cr_live_t *live);
+
+#endif /* CACHING_RELAY_LIVE_SUBSCRIBER_H */

caching/src/main.c

@@ -0,0 +1,463 @@
+/*
+ * caching_relay - a daemon that caches Nostr events from followed people
+ * into a local relay.
+ *
+ * Architecture: see plans/plan.md. Two nostr_relay_pool_t instances:
+ *   - upstream_pool: query/subscribe to upstream relays
+ *   - sink_pool (in cr_sink_t): publish-only to the local relay
+ *
+ * Build: see Makefile. Statically linked C99 binary, c-relay style.
+ */
+#define _GNU_SOURCE
+#include "main.h"
+#include "debug.h"
+#include "config.h"
+#include "state.h"
+#include "follow_graph.h"
+#include "relay_sink.h"
+#include "live_subscriber.h"
+#include "backfill.h"
+#include "relay_discovery.h"
+#include "pg_inbox.h"
+#include "pg_config.h"
+
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+#include "../nostr_core_lib/nostr_core/nostr_log.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <unistd.h>
+#include <getopt.h>
+
+/* Forward nostr_core_lib's internal logging to our debug system. */
+static void nostr_log_forwarder(int level, const char *component,
+                                const char *message, void *user_data) {
+    (void)user_data;
+    /* Map nostr log levels to our debug levels (they're the same 1-5). */
+    if (level >= 5) {
+        DEBUG_TRACE("[nostr:%s] %s", component ? component : "?", message ? message : "");
+    } else if (level >= 4) {
+        DEBUG_LOG("[nostr:%s] %s", component ? component : "?", message ? message : "");
+    } else if (level >= 3) {
+        DEBUG_INFO("[nostr:%s] %s", component ? component : "?", message ? message : "");
+    } else if (level >= 2) {
+        DEBUG_WARN("[nostr:%s] %s", component ? component : "?", message ? message : "");
+    } else {
+        DEBUG_ERROR("[nostr:%s] %s", component ? component : "?", message ? message : "");
+    }
+}
+
+volatile sig_atomic_t g_shutdown = 0;
+static volatile sig_atomic_t g_reload  = 0;
+
+static void on_signal(int sig) {
+    if (sig == SIGINT || sig == SIGTERM) g_shutdown = 1;
+    else if (sig == SIGHUP) g_reload = 1;
+}
+
+static void usage(const char *prog) {
+    fprintf(stderr,
+        "caching_relay %s - cache Nostr events from followed people into a local relay\n"
+        "\n"
+        "Usage: %s [-c <config.jsonc>] [-p <pg-conn>] [options]\n"
+        "\n"
+        "Options:\n"
+        "  -c, --config <file>   Path to .jsonc config file (default: ./caching_relay_config.jsonc)\n"
+        "  -p, --pg-conn <str>   PostgreSQL connection string (libpq format). When provided,\n"
+        "                        config is read from the c-relay-pg config table and fetched\n"
+        "                        events are inserted into the caching_event_inbox table instead\n"
+        "                        of being published via WebSocket to a local relay.\n"
+        "  -d, --debug <level>   Log level 0-5 (0=none, 1=error, 2=warn, 3=info, 4=debug, 5=trace). Default 3.\n"
+        "  -r, --restart         Reset state to first-time startup (ignore local relay cache,\n"
+        "                        re-discover all kind-10002 from bootstrap relays, reset backfill)\n"
+        "  -h, --help            Show this help\n"
+        "\n"
+        "Without -p, the daemon uses the .jsonc config file and publishes to a local relay\n"
+        "via WebSocket (legacy mode). With -p, it uses PostgreSQL for config and inbox.\n"
+        "The config file is also the persistent state store in legacy mode; the daemon\n"
+        "rewrites it as backfill progresses. See plans/plan.md and caching_relay_config.jsonc.\n",
+        CR_VERSION, prog);
+}
+
+int main(int argc, char **argv) {
+    const char *config_path = NULL;
+    const char *pg_conn = NULL;
+    int log_level = DEBUG_LEVEL_INFO;
+    int restart = 0;
+
+    static struct option longopts[] = {
+        {"config",  required_argument, 0, 'c'},
+        {"pg-conn", required_argument, 0, 'p'},
+        {"debug",   required_argument, 0, 'd'},
+        {"restart", no_argument,       0, 'r'},
+        {"help",    no_argument,       0, 'h'},
+        {0, 0, 0, 0}
+    };
+    int opt;
+    while ((opt = getopt_long(argc, argv, "c:p:d:rh", longopts, NULL)) != -1) {
+        switch (opt) {
+            case 'c': config_path = optarg; break;
+            case 'p': pg_conn = optarg; break;
+            case 'd': log_level = atoi(optarg); break;
+            case 'r': restart = 1; break;
+            case 'h': usage(argv[0]); return 0;
+            default:  usage(argv[0]); return 1;
+        }
+    }
+    /* In PostgreSQL mode, the .jsonc config file is optional. */
+    if (!config_path && !pg_conn) {
+        /* Default: look for caching_relay.jsonc in the current directory. */
+        config_path = "caching_relay_config.jsonc";
+        if (access(config_path, F_OK) != 0) {
+            fprintf(stderr, "ERROR: no config specified and default '%s' not found\n\n", config_path);
+            usage(argv[0]);
+            return 1;
+        }
+    }
+    if (log_level < 0) log_level = 0;
+    if (log_level > 5) log_level = 5;
+
+    debug_init(log_level);
+
+    /* Enable nostr_core_lib internal logging and forward to our debug system.
+     * This shows the actual WebSocket messages (REQ filters, EVENT responses)
+     * at debug level 5 (trace). */
+    nostr_set_log_callback(nostr_log_forwarder, NULL);
+    nostr_set_log_level((nostr_log_level_t)log_level);
+
+    DEBUG_INFO("caching_relay %s starting (config=%s, pg-conn=%s, loglevel=%d%s)",
+                CR_VERSION,
+                config_path ? config_path : "(none)",
+                pg_conn ? "yes" : "no",
+                log_level, restart ? ", RESTART" : "");
+
+    /* Signals. */
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_handler = on_signal;
+    sigaction(SIGINT, &sa, NULL);
+    sigaction(SIGTERM, &sa, NULL);
+    sigaction(SIGHUP, &sa, NULL);
+    /* Ignore SIGPIPE - relay pool handles its own socket errors. */
+    signal(SIGPIPE, SIG_IGN);
+
+    /* PostgreSQL inbox mode: connect and load config from the config table. */
+    long config_generation = -1;
+    if (pg_conn) {
+        if (pg_inbox_init(pg_conn) != 0) {
+            DEBUG_ERROR("failed to connect to PostgreSQL");
+            return 1;
+        }
+        config_generation = pg_inbox_get_config_generation();
+        if (config_generation < 0) {
+            DEBUG_WARN("could not read caching_config_generation (defaulting to 0)");
+            config_generation = 0;
+        }
+        DEBUG_INFO("config generation: %ld", config_generation);
+    }
+
+    /* Load config. */
+    cr_config_t cfg;
+    if (pg_conn) {
+        if (pg_config_load(&cfg) != 0) {
+            DEBUG_ERROR("failed to load config from PostgreSQL");
+            pg_inbox_shutdown();
+            return 1;
+        }
+    } else {
+        if (cr_config_load(&cfg, config_path) != 0) return 1;
+    }
+
+    /* --restart: reset state to first-time startup. */
+    if (restart) {
+        DEBUG_INFO("RESTART: resetting state to first-time startup");
+        cfg.state.backfilled_until = 0;
+        cfg.state.current_window_index = 0;
+        cfg.state.backfill_cursor = 0;
+        if (pg_conn) {
+            /* Clear per-author backfill progress so the next run starts fresh. */
+            pg_inbox_reset_backfill_progress();
+        } else {
+            cr_config_save_state(&cfg);
+        }
+    }
+
+    /* Init crypto. */
+    if (nostr_crypto_init() != NOSTR_SUCCESS) {
+        DEBUG_ERROR("nostr_crypto_init failed");
+        return 1;
+    }
+
+    /* In-memory state. */
+    cr_seen_ring_t seen;
+    cr_seen_ring_init(&seen);
+    cr_pubkey_set_t followed;
+    cr_pubkey_set_init(&followed);
+
+    /* Upstream pool. */
+    nostr_relay_pool_t *upstream = nostr_relay_pool_create(nostr_pool_reconnect_config_default());
+    if (!upstream) {
+        DEBUG_ERROR("failed to create upstream pool");
+        return 1;
+    }
+    for (int i = 0; i < cfg.upstream_count; i++) {
+        if (nostr_relay_pool_add_relay(upstream, cfg.upstream_relays[i]) != NOSTR_SUCCESS) {
+            DEBUG_WARN("failed to add upstream relay %s (continuing)", cfg.upstream_relays[i]);
+        } else {
+            DEBUG_INFO("upstream: added %s", cfg.upstream_relays[i]);
+        }
+    }
+
+    /* Sink: WebSocket pool (legacy) or PostgreSQL inbox. */
+    cr_sink_t sink;
+    if (pg_conn) {
+        if (cr_sink_init_pg(&sink, &seen) != 0) {
+            DEBUG_ERROR("failed to init sink (pg)");
+            return 1;
+        }
+    } else {
+        if (cr_sink_init(&sink, cfg.local_relay, &seen) != 0) {
+            DEBUG_ERROR("failed to init sink");
+            return 1;
+        }
+    }
+
+    /* Give the sink pool a moment to connect to the local relay before we
+     * start publishing (relay discovery caches kind-10002 events to local).
+     * No-op in PostgreSQL inbox mode (cr_sink_pump does nothing). */
+    if (!pg_conn) {
+        DEBUG_INFO("waiting for sink connection to establish...");
+        for (int i = 0; i < 30 && !g_shutdown; i++) {
+            cr_sink_pump(&sink, 100);
+        }
+    }
+
+    /* Resolve follow graph. */
+    if (g_shutdown) goto shutdown;
+    if (cr_follow_resolve(&cfg, upstream, &followed) != 0) {
+        DEBUG_ERROR("follow graph resolution failed");
+        return 1;
+    }
+
+    if (g_shutdown) goto shutdown;
+
+    /* Discover outbox relays (NIP-65) and compute minimum covering set. */
+    int is_first_time = (cfg.state.backfilled_until == 0);
+    cr_relay_map_t relay_map;
+    if (cr_relay_discovery_run(&relay_map, &cfg, upstream, &sink, &followed,
+                                is_first_time) != 0) {
+        DEBUG_WARN("relay discovery failed, continuing with bootstrap relays only");
+        memset(&relay_map, 0, sizeof(relay_map));
+    }
+
+    /* Add discovered outbox relays to the upstream pool. */
+    for (int i = 0; i < relay_map.selected_count; i++) {
+        /* Check if already in the pool (bootstrap relays may already be there). */
+        char **listed = NULL;
+        nostr_pool_relay_status_t *statuses = NULL;
+        int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+        int already = 0;
+        for (int j = 0; j < n; j++) {
+            if (strcmp(listed[j], relay_map.selected_relays[i]) == 0) {
+                already = 1; break;
+            }
+        }
+        free(listed);
+        free(statuses);
+        if (!already) {
+            if (nostr_relay_pool_add_relay(upstream, relay_map.selected_relays[i]) == NOSTR_SUCCESS) {
+                DEBUG_INFO("upstream: added outbox relay %s", relay_map.selected_relays[i]);
+            }
+        }
+    }
+
+    /* Log final upstream pool. */
+    {
+        char **listed = NULL;
+        nostr_pool_relay_status_t *statuses = NULL;
+        int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+        DEBUG_INFO("upstream_pool: %d relays connected:", n);
+        for (int j = 0; j < n; j++) {
+            DEBUG_INFO("  %s", listed[j]);
+        }
+        free(listed);
+        free(statuses);
+    }
+
+    /* Open live subscription. */
+    cr_live_t live;
+    if (cfg.live.enabled) {
+        if (cr_live_open(&live, &cfg, upstream, &followed, &sink) != 0) {
+            DEBUG_WARN("live subscription failed to open (will retry on resubscribe)");
+        }
+    } else {
+        memset(&live, 0, sizeof(live));
+    }
+
+    /* Init backfill. */
+    cr_backfill_t bf;
+    cr_backfill_init(&bf, &cfg);
+
+    time_t last_follow_refresh = time(NULL);
+    time_t last_state_save = time(NULL);
+    time_t last_status_heartbeat = 0;
+
+    /* Initial status heartbeat in PostgreSQL mode. */
+    if (pg_conn) {
+        pg_inbox_update_status("starting", config_generation, (long)time(NULL),
+                               followed.count, relay_map.selected_count, 0,
+                               cfg.state.current_window_index,
+                               cfg.state.backfill_cursor, 0, 0, NULL, 0);
+    }
+
+    DEBUG_INFO("entering main loop");
+
+    while (!g_shutdown) {
+        if (g_reload) {
+            g_reload = 0;
+            DEBUG_INFO("SIGHUP: reloading config (state preserved)");
+            cr_config_t newcfg;
+            int reload_ok = 0;
+            if (pg_conn) {
+                if (pg_config_load(&newcfg) == 0) reload_ok = 1;
+            } else if (config_path) {
+                if (cr_config_load(&newcfg, config_path) == 0) reload_ok = 1;
+            }
+            if (reload_ok) {
+                /* Preserve runtime state across reload. */
+                newcfg.state = cfg.state;
+                cr_config_free(&cfg);
+                cfg = newcfg;
+                DEBUG_INFO("config reloaded");
+            } else {
+                DEBUG_ERROR("config reload failed, keeping old config");
+            }
+        }
+
+        /* PostgreSQL: check for config generation change and reload. */
+        if (pg_conn) {
+            int changed = pg_config_generation_changed(config_generation);
+            if (changed == 1) {
+                long new_gen = pg_inbox_get_config_generation();
+                DEBUG_INFO("config generation changed (%ld -> %ld), reloading",
+                           config_generation, new_gen);
+                cr_config_t newcfg;
+                if (pg_config_load(&newcfg) == 0) {
+                    newcfg.state = cfg.state;
+                    cr_config_free(&cfg);
+                    cfg = newcfg;
+                    config_generation = new_gen;
+                    DEBUG_INFO("config reloaded from PostgreSQL");
+                    /* Resubscribe live with new config. */
+                    if (cfg.live.enabled) {
+                        cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
+                    }
+                } else {
+                    DEBUG_ERROR("PostgreSQL config reload failed, keeping old config");
+                }
+            } else if (changed < 0) {
+                DEBUG_WARN("config generation check failed");
+            }
+        }
+
+        /* Pump upstream pool (drives live subscription callbacks). */
+        nostr_relay_pool_run(upstream, 100);
+
+        /* Pump sink pool (flush publish callbacks). */
+        cr_sink_pump(&sink, 50);
+
+        /* Backfill tick. */
+        int brc = cr_backfill_tick(&bf, &cfg, upstream, &followed, &sink, &relay_map);
+        (void)brc;
+
+        /* Periodic follow-graph refresh. */
+        time_t now = time(NULL);
+        if (cfg.follow_graph_refresh_seconds > 0 &&
+            (now - last_follow_refresh) >= cfg.follow_graph_refresh_seconds) {
+            DEBUG_INFO("refreshing follow graph");
+            cr_pubkey_set_t new_followed;
+            cr_pubkey_set_init(&new_followed);
+            if (cr_follow_resolve(&cfg, upstream, &new_followed) == 0) {
+                /* If the set changed, resubscribe live. */
+                int changed = (new_followed.count != followed.count);
+                if (!changed) {
+                    for (int i = 0; i < followed.count; i++) {
+                        if (!cr_pubkey_set_contains(&new_followed, followed.items[i])) {
+                            changed = 1; break;
+                        }
+                    }
+                }
+                cr_pubkey_set_free(&followed);
+                followed = new_followed;
+                if (changed && cfg.live.enabled) {
+                    cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
+                }
+            } else {
+                DEBUG_WARN("follow graph refresh failed");
+                cr_pubkey_set_free(&new_followed);
+            }
+            last_follow_refresh = now;
+        }
+
+        /* Periodic live resubscribe. */
+        if (cfg.live.enabled && cfg.live.resubscribe_interval_seconds > 0 &&
+            (now - live.last_resubscribe) >= cfg.live.resubscribe_interval_seconds) {
+            cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
+        }
+
+        /* Periodic state save (in case backfill didn't just save). */
+        if ((now - last_state_save) >= 60) {
+            if (!pg_conn) cr_config_save_state(&cfg);
+            last_state_save = now;
+        }
+
+        /* PostgreSQL: periodic status heartbeat. */
+        if (pg_conn && (now - last_status_heartbeat) >= 15) {
+            long events_fetched = live.events_received + bf.events_total;
+            long inbox_inserts  = sink.published_ok;
+            int connected = 0;
+            {
+                char **listed = NULL;
+                nostr_pool_relay_status_t *statuses = NULL;
+                int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
+                for (int j = 0; j < n; j++) {
+                    if (statuses[j] == NOSTR_POOL_RELAY_CONNECTED) connected++;
+                }
+                free(listed);
+                free(statuses);
+            }
+            pg_inbox_update_status("running", config_generation, (long)now,
+                                   followed.count, relay_map.selected_count,
+                                   connected, cfg.state.current_window_index,
+                                   cfg.state.backfill_cursor,
+                                   events_fetched, inbox_inserts, NULL, 0);
+            last_status_heartbeat = now;
+        }
+    }
+
+    /* Graceful shutdown. */
+ shutdown:
+     DEBUG_INFO("shutting down...");
+     cr_live_close(&live);
+     if (!pg_conn) cr_config_save_state(&cfg);
+     if (pg_conn) {
+         pg_inbox_update_status("stopped", config_generation, (long)time(NULL),
+                                followed.count, relay_map.selected_count, 0,
+                                cfg.state.current_window_index,
+                                cfg.state.backfill_cursor,
+                                live.events_received + bf.events_total,
+                                sink.published_ok, NULL, 0);
+     }
+     cr_sink_destroy(&sink);
+     nostr_relay_pool_destroy(upstream);
+     cr_pubkey_set_free(&followed);
+     cr_relay_map_free(&relay_map);
+     cr_config_free(&cfg);
+     if (pg_conn) pg_inbox_shutdown();
+     nostr_crypto_cleanup();
+     DEBUG_INFO("clean exit");
+     return 0;
+ }

caching/src/main.h

@@ -0,0 +1,16 @@
+/*
+ * caching_relay - main header
+ *
+ * Version information is auto-updated by increment_and_push.sh.
+ * Git tags are the source of truth for versioning.
+ */
+#ifndef CACHING_RELAY_MAIN_H
+#define CACHING_RELAY_MAIN_H
+
+// Version information (auto-updated by increment_and_push.sh)
+#define CR_VERSION_MAJOR 0
+#define CR_VERSION_MINOR 0
+#define CR_VERSION_PATCH 2
+#define CR_VERSION "v0.0.2"
+
+#endif /* CACHING_RELAY_MAIN_H */

caching/src/pg_config.c

@@ -0,0 +1,236 @@
+/*
+ * caching_relay - read caching configuration from the c-relay-pg PostgreSQL
+ * config table and populate the existing cr_config_t structure.
+ */
+#define _GNU_SOURCE
+#include "pg_config.h"
+#include "pg_inbox.h"
+#include "debug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/* ------------------------------------------------------------------ */
+/* Helpers                                                             */
+/* ------------------------------------------------------------------ */
+
+/* Split a comma-separated string into a string array (fixed-size rows).
+ * Trims leading/trailing whitespace from each token. */
+static void split_csv_str(const char *s, char *dst, int max, int elem_len,
+                          int *count_out) {
+    int count = 0;
+    if (!s) { *count_out = 0; return; }
+
+    /* Work on a mutable copy because strtok_r modifies its input. */
+    char *copy = strdup(s);
+    if (!copy) { *count_out = 0; return; }
+
+    char *saveptr = NULL;
+    char *tok = strtok_r(copy, ",", &saveptr);
+    while (tok && count < max) {
+        /* Trim leading spaces. */
+        while (*tok == ' ' || *tok == '\t') tok++;
+        /* Trim trailing spaces. */
+        char *end = tok + strlen(tok) - 1;
+        while (end > tok && (*end == ' ' || *end == '\t' || *end == '\r' || *end == '\n')) {
+            *end-- = '\0';
+        }
+        if (*tok != '\0') {
+            strncpy(dst + count * elem_len, tok, elem_len - 1);
+            dst[count * elem_len + (elem_len - 1)] = '\0';
+            count++;
+        }
+        tok = strtok_r(NULL, ",", &saveptr);
+    }
+    free(copy);
+    *count_out = count;
+}
+
+/* Split a comma-separated string of integers into an int array. */
+static void split_csv_int(const char *s, int *dst, int max, int *count_out) {
+    int count = 0;
+    if (!s) { *count_out = 0; return; }
+
+    char *copy = strdup(s);
+    if (!copy) { *count_out = 0; return; }
+
+    char *saveptr = NULL;
+    char *tok = strtok_r(copy, ",", &saveptr);
+    while (tok && count < max) {
+        while (*tok == ' ' || *tok == '\t') tok++;
+        if (*tok != '\0') {
+            dst[count++] = atoi(tok);
+        }
+        tok = strtok_r(NULL, ",", &saveptr);
+    }
+    free(copy);
+    *count_out = count;
+}
+
+/* Split a comma-separated string of longs into a long array. */
+static void split_csv_long(const char *s, long *dst, int max, int *count_out) {
+    int count = 0;
+    if (!s) { *count_out = 0; return; }
+
+    char *copy = strdup(s);
+    if (!copy) { *count_out = 0; return; }
+
+    char *saveptr = NULL;
+    char *tok = strtok_r(copy, ",", &saveptr);
+    while (tok && count < max) {
+        while (*tok == ' ' || *tok == '\t') tok++;
+        if (*tok != '\0') {
+            dst[count++] = strtol(tok, NULL, 10);
+        }
+        tok = strtok_r(NULL, ",", &saveptr);
+    }
+    free(copy);
+    *count_out = count;
+}
+
+/* Read a config key as a freshly-allocated string. Returns NULL if missing. */
+static char *get_key(const char *key) {
+    return pg_inbox_get_config_value(key);
+}
+
+/* Read a config key as a boolean ("true"/"1"/"yes" -> 1). */
+static int get_bool(const char *key, int default_val) {
+    char *v = get_key(key);
+    if (!v) return default_val;
+    int r = (strcmp(v, "true") == 0 || strcmp(v, "1") == 0 ||
+             strcmp(v, "yes") == 0 || strcmp(v, "on") == 0);
+    free(v);
+    return r;
+}
+
+/* Read a config key as a long. */
+static long get_long(const char *key, long default_val) {
+    char *v = get_key(key);
+    if (!v) return default_val;
+    long r = strtol(v, NULL, 10);
+    free(v);
+    return r;
+}
+
+/* Read a config key as an int. */
+static int get_int(const char *key, int default_val) {
+    return (int)get_long(key, (long)default_val);
+}
+
+/* ------------------------------------------------------------------ */
+/* Public API                                                          */
+/* ------------------------------------------------------------------ */
+
+int pg_config_load(cr_config_t *cfg) {
+    if (!cfg) return -1;
+    memset(cfg, 0, sizeof(*cfg));
+
+    /* root_npubs */
+    char *npubs = get_key("caching_root_npubs");
+    split_csv_str(npubs, (char *)cfg->root_npubs, CR_MAX_ROOT_NPUBS,
+                  CR_NPUB_LEN, &cfg->root_npub_count);
+    free(npubs);
+
+    /* bootstrap_relays -> upstream_relays */
+    char *relays = get_key("caching_bootstrap_relays");
+    split_csv_str(relays, (char *)cfg->upstream_relays, CR_MAX_UPSTREAM,
+                  CR_URL_LEN, &cfg->upstream_count);
+    free(relays);
+
+    /* local_relay - no longer required for PostgreSQL mode, but keep a
+     * placeholder for compatibility. The caller may override it. */
+    cfg->local_relay[0] = '\0';
+
+    /* kinds */
+    char *kinds = get_key("caching_kinds");
+    split_csv_int(kinds, cfg->kinds, CR_MAX_KINDS, &cfg->kind_count);
+    free(kinds);
+
+    /* admin_kinds - supports "*" to mean all kinds */
+    char *akinds = get_key("caching_admin_kinds");
+    if (akinds) {
+        /* Check for "*" anywhere in the list. */
+        if (strstr(akinds, "*") != NULL) {
+            cfg->admin_all_kinds = 1;
+        } else {
+            split_csv_int(akinds, cfg->admin_kinds, CR_MAX_KINDS,
+                          &cfg->admin_kind_count);
+        }
+        free(akinds);
+    }
+
+    /* live */
+    cfg->live.enabled = get_bool("caching_live_enabled", 1);
+    cfg->live.resubscribe_interval_seconds =
+        get_int("caching_live_resubscribe_seconds", 300);
+
+    /* backfill */
+    cfg->backfill.enabled = get_bool("caching_backfill_enabled", 1);
+    char *windows = get_key("caching_backfill_windows");
+    split_csv_long(windows, cfg->backfill.window_schedule_seconds,
+                   CR_MAX_WINDOWS, &cfg->backfill.window_count);
+    free(windows);
+    cfg->backfill.events_per_tick =
+        get_int("caching_backfill_page_size", 50);
+    /* tick interval is stored in ms in the config table; convert to seconds. */
+    int tick_ms = get_int("caching_backfill_tick_interval_ms", 5000);
+    cfg->backfill.tick_interval_seconds = (tick_ms + 999) / 1000;
+    if (cfg->backfill.tick_interval_seconds < 1)
+        cfg->backfill.tick_interval_seconds = 1;
+    cfg->backfill.window_cooldown_seconds =
+        get_int("caching_backfill_window_cooldown_seconds", 60);
+
+    /* follow graph refresh */
+    cfg->follow_graph_refresh_seconds =
+        get_int("caching_follow_graph_refresh_seconds", 600);
+
+    /* Defaults if missing (mirror cr_config_load). */
+    if (cfg->backfill.window_count == 0) {
+        static const long def_windows[] = {86400, 604800, 2592000, 7776000, 31536000};
+        int n = (int)(sizeof(def_windows) / sizeof(def_windows[0]));
+        if (n > CR_MAX_WINDOWS) n = CR_MAX_WINDOWS;
+        for (int i = 0; i < n; i++)
+            cfg->backfill.window_schedule_seconds[i] = def_windows[i];
+        cfg->backfill.window_count = n;
+    }
+    if (cfg->backfill.events_per_tick == 0)
+        cfg->backfill.events_per_tick = 50;
+    if (cfg->backfill.tick_interval_seconds == 0)
+        cfg->backfill.tick_interval_seconds = 5;
+    if (cfg->backfill.window_cooldown_seconds == 0)
+        cfg->backfill.window_cooldown_seconds = 60;
+    if (cfg->live.resubscribe_interval_seconds == 0)
+        cfg->live.resubscribe_interval_seconds = 300;
+    if (cfg->follow_graph_refresh_seconds == 0)
+        cfg->follow_graph_refresh_seconds = 600;
+
+    /* State: initialize to defaults for this phase. Phase 7 will read
+     * caching_backfill_progress / caching_service_state. */
+    cfg->state.backfilled_until = 0;
+    cfg->state.current_window_index = 0;
+    cfg->state.backfill_cursor = 0;
+
+    /* Validation. */
+    if (cfg->root_npub_count == 0) {
+        DEBUG_ERROR("pg_config: at least one caching_root_npubs required");
+        return -1;
+    }
+    if (cfg->upstream_count == 0) {
+        DEBUG_ERROR("pg_config: at least one caching_bootstrap_relays required");
+        return -1;
+    }
+    if (cfg->kind_count == 0) {
+        DEBUG_ERROR("pg_config: at least one caching_kinds required");
+        return -1;
+    }
+
+    DEBUG_INFO("pg_config loaded: %d root npubs, %d upstream relays, %d kinds",
+               cfg->root_npub_count, cfg->upstream_count, cfg->kind_count);
+    return 0;
+}
+
+int pg_config_generation_changed(long last_generation) {
+    long cur = pg_inbox_get_config_generation();
+    if (cur < 0) return -1;
+    return (cur != last_generation) ? 1 : 0;
+}

caching/src/pg_config.h

@@ -0,0 +1,19 @@
+/*
+ * caching_relay - read caching configuration from the c-relay-pg PostgreSQL
+ * config table and populate the existing cr_config_t structure.
+ */
+#ifndef CACHING_RELAY_PG_CONFIG_H
+#define CACHING_RELAY_PG_CONFIG_H
+
+#include "config.h" /* cr_config_t */
+
+/* Load caching configuration from the c-relay-pg PostgreSQL config table.
+ * Populates the cr_config_t fields from the config table.
+ * Returns 0 on success, -1 on error. */
+int pg_config_load(cr_config_t *cfg);
+
+/* Check if the config generation has changed since the last load.
+ * Returns 1 if changed, 0 if not, -1 on error. */
+int pg_config_generation_changed(long last_generation);
+
+#endif /* CACHING_RELAY_PG_CONFIG_H */

caching/src/pg_inbox.c

@@ -0,0 +1,488 @@
+/*
+ * caching_relay - PostgreSQL inbox connection and operations.
+ */
+#define _GNU_SOURCE
+#include "pg_inbox.h"
+#include "debug.h"
+
+#include <libpq-fe.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../nostr_core_lib/cjson/cJSON.h"
+
+static PGconn *g_pg = NULL;
+
+/* ------------------------------------------------------------------ */
+
+int pg_inbox_init(const char *connection_string) {
+    if (g_pg) {
+        PQfinish(g_pg);
+        g_pg = NULL;
+    }
+    g_pg = PQconnectdb(connection_string);
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: PQconnectdb returned NULL (out of memory)");
+        return -1;
+    }
+    if (PQstatus(g_pg) != CONNECTION_OK) {
+        DEBUG_ERROR("pg_inbox: connection failed: %s",
+                    PQerrorMessage(g_pg));
+        PQfinish(g_pg);
+        g_pg = NULL;
+        return -1;
+    }
+    DEBUG_INFO("pg_inbox: connected to PostgreSQL (%s)",
+             PQdb(g_pg) ? PQdb(g_pg) : "?");
+    return 0;
+}
+
+void pg_inbox_shutdown(void) {
+    if (g_pg) {
+        PQfinish(g_pg);
+        g_pg = NULL;
+        DEBUG_INFO("pg_inbox: connection closed");
+    }
+}
+
+/* ------------------------------------------------------------------ */
+/* Inbox insert                                                        */
+/* ------------------------------------------------------------------ */
+
+int pg_inbox_insert_event(const char *event_json, const char *source_relay,
+                          const char *source_class, int priority) {
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return -1;
+    }
+    if (!event_json) {
+        DEBUG_WARN("pg_inbox: insert called with NULL event_json");
+        return -1;
+    }
+
+    /* Extract the event id from the JSON for the event_id column.
+     * The schema CHECK requires event_id = event_json->>'id'. */
+    cJSON *ev = cJSON_Parse(event_json);
+    if (!ev) {
+        DEBUG_WARN("pg_inbox: event_json is not valid JSON, skipping");
+        return -1;
+    }
+    cJSON *id_node = cJSON_GetObjectItem(ev, "id");
+    const char *event_id = NULL;
+    if (id_node && cJSON_IsString(id_node)) {
+        event_id = cJSON_GetStringValue(id_node);
+    }
+    if (!event_id || strlen(event_id) != 64) {
+        DEBUG_WARN("pg_inbox: event missing valid 64-char id, skipping");
+        cJSON_Delete(ev);
+        return -1;
+    }
+
+    /* priority is an integer; pass as a separate param via a small buffer. */
+    char prio_buf[16];
+    snprintf(prio_buf, sizeof(prio_buf), "%d", priority);
+
+    const char *vals[5];
+    int lens[5];
+    int fmts[5];
+
+    /* $1: event_id (text) */
+    vals[0] = event_id;
+    lens[0] = (int)strlen(event_id);
+    fmts[0] = 0;
+
+    /* $2: event_json (cast to jsonb in SQL) */
+    vals[1] = event_json;
+    lens[1] = (int)strlen(event_json);
+    fmts[1] = 0;
+
+    /* $3: source_relay (text, may be NULL) */
+    vals[2] = source_relay;
+    lens[2] = source_relay ? (int)strlen(source_relay) : 0;
+    fmts[2] = 0;
+
+    /* $4: source_class (text) */
+    vals[3] = source_class ? source_class : "backfill";
+    lens[3] = (int)strlen(vals[3]);
+    fmts[3] = 0;
+
+    /* $5: priority (smallint, passed as text) */
+    vals[4] = prio_buf;
+    lens[4] = (int)strlen(prio_buf);
+    fmts[4] = 0;
+
+    const char *sql =
+        "INSERT INTO caching_event_inbox "
+        "(event_id, event_json, source_relay, source_class, priority) "
+        "VALUES ($1, $2::jsonb, $3, $4, $5) "
+        "ON CONFLICT (event_id) DO NOTHING";
+
+    PGresult *res = PQexecParams(g_pg, sql, 5, NULL, vals, lens, fmts, 0);
+    cJSON_Delete(ev);
+
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: insert returned NULL result for %s", event_id);
+        return -1;
+    }
+    ExecStatusType st = PQresultStatus(res);
+    if (st != PGRES_COMMAND_OK) {
+        DEBUG_WARN("pg_inbox: insert failed for %s: %s",
+                   event_id, PQresultErrorMessage(res));
+        PQclear(res);
+        return -1;
+    }
+    /* ON CONFLICT DO NOTHING returns COMMAND_OK either way; the row count
+     * is not directly available without RETURNING, so we treat both
+     * inserted and conflict-skipped as success (0). */
+    PQclear(res);
+    return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* Service state update                                                */
+/* ------------------------------------------------------------------ */
+
+int pg_inbox_update_status(const char *service_state,
+                           long config_generation,
+                           long heartbeat_at,
+                           int followed_author_count,
+                           int selected_relay_count,
+                           int connected_relay_count,
+                           int current_window_index,
+                           int backfill_cursor,
+                           long events_fetched,
+                           long inbox_inserts,
+                           const char *last_error,
+                           long last_error_at) {
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return -1;
+    }
+
+    char gen_buf[32], hb_buf[32], fac_buf[16], src_buf[16], crc_buf[16];
+    char cwi_buf[16], bc_buf[16], ef_buf[32], ii_buf[32], lea_buf[32];
+
+    snprintf(gen_buf, sizeof(gen_buf), "%ld", config_generation);
+    snprintf(hb_buf,  sizeof(hb_buf),  "%ld", heartbeat_at);
+    snprintf(fac_buf, sizeof(fac_buf), "%d",  followed_author_count);
+    snprintf(src_buf, sizeof(src_buf), "%d",  selected_relay_count);
+    snprintf(crc_buf, sizeof(crc_buf), "%d",  connected_relay_count);
+    snprintf(cwi_buf, sizeof(cwi_buf), "%d",  current_window_index);
+    snprintf(bc_buf,  sizeof(bc_buf),  "%d",  backfill_cursor);
+    snprintf(ef_buf,  sizeof(ef_buf),  "%ld", events_fetched);
+    snprintf(ii_buf,  sizeof(ii_buf),  "%ld", inbox_inserts);
+    if (last_error_at > 0) {
+        snprintf(lea_buf, sizeof(lea_buf), "%ld", last_error_at);
+    } else {
+        lea_buf[0] = '\0';
+    }
+
+    const char *vals[12];
+    int lens[12];
+    int fmts[12];
+    int n = 0;
+
+    /* $1 service_state (may be NULL) */
+    vals[n] = service_state; lens[n] = service_state ? (int)strlen(service_state) : 0; fmts[n] = 0; n++;
+    /* $2 config_generation */
+    vals[n] = gen_buf; lens[n] = (int)strlen(gen_buf); fmts[n] = 0; n++;
+    /* $3 heartbeat_at */
+    vals[n] = hb_buf; lens[n] = (int)strlen(hb_buf); fmts[n] = 0; n++;
+    /* $4 followed_author_count */
+    vals[n] = fac_buf; lens[n] = (int)strlen(fac_buf); fmts[n] = 0; n++;
+    /* $5 selected_relay_count */
+    vals[n] = src_buf; lens[n] = (int)strlen(src_buf); fmts[n] = 0; n++;
+    /* $6 connected_relay_count */
+    vals[n] = crc_buf; lens[n] = (int)strlen(crc_buf); fmts[n] = 0; n++;
+    /* $7 current_window_index */
+    vals[n] = cwi_buf; lens[n] = (int)strlen(cwi_buf); fmts[n] = 0; n++;
+    /* $8 backfill_cursor */
+    vals[n] = bc_buf; lens[n] = (int)strlen(bc_buf); fmts[n] = 0; n++;
+    /* $9 events_fetched */
+    vals[n] = ef_buf; lens[n] = (int)strlen(ef_buf); fmts[n] = 0; n++;
+    /* $10 inbox_inserts */
+    vals[n] = ii_buf; lens[n] = (int)strlen(ii_buf); fmts[n] = 0; n++;
+    /* $11 last_error (may be NULL) */
+    vals[n] = last_error; lens[n] = last_error ? (int)strlen(last_error) : 0; fmts[n] = 0; n++;
+    /* $12 last_error_at (empty string -> NULL) */
+    vals[n] = lea_buf; lens[n] = (int)strlen(lea_buf); fmts[n] = 0; n++;
+
+    const char *sql =
+        "INSERT INTO caching_service_state "
+        "(id, service_state, config_generation, heartbeat_at, "
+        " followed_author_count, selected_relay_count, connected_relay_count, "
+        " current_window_index, backfill_cursor, events_fetched, inbox_inserts, "
+        " last_error, last_error_at, updated_at) "
+        "VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, "
+        "        CASE WHEN $12 = '' THEN NULL ELSE $12::BIGINT END, "
+        "        EXTRACT(EPOCH FROM NOW())::BIGINT) "
+        "ON CONFLICT (id) DO UPDATE SET "
+        "  service_state        = COALESCE(EXCLUDED.service_state, caching_service_state.service_state), "
+        "  config_generation    = EXCLUDED.config_generation, "
+        "  heartbeat_at         = EXCLUDED.heartbeat_at, "
+        "  followed_author_count= EXCLUDED.followed_author_count, "
+        "  selected_relay_count = EXCLUDED.selected_relay_count, "
+        "  connected_relay_count= EXCLUDED.connected_relay_count, "
+        "  current_window_index = EXCLUDED.current_window_index, "
+        "  backfill_cursor      = EXCLUDED.backfill_cursor, "
+        "  events_fetched       = EXCLUDED.events_fetched, "
+        "  inbox_inserts        = EXCLUDED.inbox_inserts, "
+        "  last_error           = COALESCE(EXCLUDED.last_error, caching_service_state.last_error), "
+        "  last_error_at        = CASE WHEN EXCLUDED.last_error_at IS NULL "
+        "                              THEN caching_service_state.last_error_at "
+        "                              ELSE EXCLUDED.last_error_at END, "
+        "  updated_at           = EXCLUDED.updated_at";
+
+    PGresult *res = PQexecParams(g_pg, sql, n, NULL, vals, lens, fmts, 0);
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: update_status returned NULL result");
+        return -1;
+    }
+    ExecStatusType st = PQresultStatus(res);
+    if (st != PGRES_COMMAND_OK) {
+        DEBUG_ERROR("pg_inbox: update_status failed: %s",
+                    PQresultErrorMessage(res));
+        PQclear(res);
+        return -1;
+    }
+    PQclear(res);
+    return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* Config reads                                                        */
+/* ------------------------------------------------------------------ */
+
+char *pg_inbox_get_config_value(const char *key) {
+    if (!g_pg || !key) return NULL;
+
+    const char *vals[1] = { key };
+    int lens[1] = { (int)strlen(key) };
+    int fmts[1] = { 0 };
+
+    PGresult *res = PQexecParams(g_pg,
+        "SELECT value FROM config WHERE key = $1",
+        1, NULL, vals, lens, fmts, 0);
+    if (!res) return NULL;
+    if (PQresultStatus(res) != PGRES_TUPLES_OK) {
+        DEBUG_WARN("pg_inbox: config select failed for '%s': %s",
+                   key, PQresultErrorMessage(res));
+        PQclear(res);
+        return NULL;
+    }
+    if (PQntuples(res) < 1 || PQgetisnull(res, 0, 0)) {
+        PQclear(res);
+        return NULL;
+    }
+    char *v = strdup(PQgetvalue(res, 0, 0));
+    PQclear(res);
+    return v;
+}
+
+long pg_inbox_get_config_generation(void) {
+    char *v = pg_inbox_get_config_value("caching_config_generation");
+    if (!v) return -1;
+    long gen = strtol(v, NULL, 10);
+    free(v);
+    return gen;
+}
+
+/* ------------------------------------------------------------------ */
+/* Backfill progress persistence                                       */
+/* ------------------------------------------------------------------ */
+
+int pg_inbox_save_backfill_progress(const char *author_pubkey,
+                                    int window_index,
+                                    long window_anchor,
+                                    long until_cursor,
+                                    int complete) {
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return -1;
+    }
+    if (!author_pubkey) {
+        DEBUG_WARN("pg_inbox: save_backfill_progress called with NULL author");
+        return -1;
+    }
+
+    char wi_buf[16], wa_buf[32], uc_buf[32], cmp_buf[8];
+    snprintf(wi_buf,  sizeof(wi_buf),  "%d",  window_index);
+    snprintf(wa_buf,  sizeof(wa_buf),  "%ld", window_anchor);
+    snprintf(uc_buf,  sizeof(uc_buf),  "%ld", until_cursor);
+    snprintf(cmp_buf, sizeof(cmp_buf),  "%d",  complete ? 1 : 0);
+
+    const char *vals[5] = { author_pubkey, wi_buf, wa_buf, uc_buf, cmp_buf };
+    int lens[5] = {
+        (int)strlen(author_pubkey),
+        (int)strlen(wi_buf),
+        (int)strlen(wa_buf),
+        (int)strlen(uc_buf),
+        (int)strlen(cmp_buf),
+    };
+    int fmts[5] = { 0, 0, 0, 0, 0 };
+
+    const char *sql =
+        "INSERT INTO caching_backfill_progress "
+        "(author_pubkey, window_index, window_anchor, until_cursor, complete, updated_at) "
+        "VALUES ($1, $2, $3, $4, $5::BOOLEAN, EXTRACT(EPOCH FROM NOW())::BIGINT) "
+        "ON CONFLICT (author_pubkey, window_index) DO UPDATE SET "
+        "  until_cursor = EXCLUDED.until_cursor, "
+        "  complete     = EXCLUDED.complete, "
+        "  updated_at   = EXCLUDED.updated_at";
+
+    PGresult *res = PQexecParams(g_pg, sql, 5, NULL, vals, lens, fmts, 0);
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: save_backfill_progress returned NULL result");
+        return -1;
+    }
+    ExecStatusType st = PQresultStatus(res);
+    if (st != PGRES_COMMAND_OK) {
+        DEBUG_ERROR("pg_inbox: save_backfill_progress failed: %s",
+                    PQresultErrorMessage(res));
+        PQclear(res);
+        return -1;
+    }
+    PQclear(res);
+    return 0;
+}
+
+int pg_inbox_load_backfill_progress(const char *author_pubkey,
+                                    int window_index,
+                                    long *out_until_cursor,
+                                    int *out_complete,
+                                    int *out_found) {
+    if (out_found) *out_found = 0;
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return -1;
+    }
+    if (!author_pubkey) return -1;
+
+    char wi_buf[16];
+    snprintf(wi_buf, sizeof(wi_buf), "%d", window_index);
+
+    const char *vals[2] = { author_pubkey, wi_buf };
+    int lens[2] = { (int)strlen(author_pubkey), (int)strlen(wi_buf) };
+    int fmts[2] = { 0, 0 };
+
+    const char *sql =
+        "SELECT until_cursor, complete FROM caching_backfill_progress "
+        "WHERE author_pubkey = $1 AND window_index = $2";
+
+    PGresult *res = PQexecParams(g_pg, sql, 2, NULL, vals, lens, fmts, 0);
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: load_backfill_progress returned NULL result");
+        return -1;
+    }
+    if (PQresultStatus(res) != PGRES_TUPLES_OK) {
+        DEBUG_ERROR("pg_inbox: load_backfill_progress failed: %s",
+                    PQresultErrorMessage(res));
+        PQclear(res);
+        return -1;
+    }
+    if (PQntuples(res) < 1) {
+        /* Not found - not an error. */
+        PQclear(res);
+        return 0;
+    }
+
+    if (out_until_cursor) {
+        if (PQgetisnull(res, 0, 0)) {
+            *out_until_cursor = 0;
+        } else {
+            *out_until_cursor = strtol(PQgetvalue(res, 0, 0), NULL, 10);
+        }
+    }
+    if (out_complete) {
+        if (PQgetisnull(res, 0, 1)) {
+            *out_complete = 0;
+        } else {
+            const char *v = PQgetvalue(res, 0, 1);
+            /* boolean comes back as 't'/'f' or '1'/'0' depending on output mode. */
+            *out_complete = (v && (v[0] == 't' || v[0] == 'T' || v[0] == '1')) ? 1 : 0;
+        }
+    }
+    if (out_found) *out_found = 1;
+    PQclear(res);
+    return 0;
+}
+
+cJSON *pg_inbox_load_incomplete_progress(int window_index) {
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return NULL;
+    }
+
+    char wi_buf[16];
+    snprintf(wi_buf, sizeof(wi_buf), "%d", window_index);
+
+    const char *vals[1] = { wi_buf };
+    int lens[1] = { (int)strlen(wi_buf) };
+    int fmts[1] = { 0 };
+
+    const char *sql =
+        "SELECT author_pubkey, until_cursor, complete FROM caching_backfill_progress "
+        "WHERE window_index = $1 AND complete = FALSE";
+
+    PGresult *res = PQexecParams(g_pg, sql, 1, NULL, vals, lens, fmts, 0);
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: load_incomplete_progress returned NULL result");
+        return NULL;
+    }
+    if (PQresultStatus(res) != PGRES_TUPLES_OK) {
+        DEBUG_ERROR("pg_inbox: load_incomplete_progress failed: %s",
+                    PQresultErrorMessage(res));
+        PQclear(res);
+        return NULL;
+    }
+
+    int n = PQntuples(res);
+    if (n <= 0) {
+        PQclear(res);
+        return NULL;
+    }
+
+    cJSON *arr = cJSON_CreateArray();
+    if (!arr) {
+        PQclear(res);
+        return NULL;
+    }
+    for (int i = 0; i < n; i++) {
+        cJSON *obj = cJSON_CreateObject();
+        if (!obj) continue;
+        const char *pk = PQgetisnull(res, i, 0) ? "" : PQgetvalue(res, i, 0);
+        long uc = PQgetisnull(res, i, 1) ? 0 : strtol(PQgetvalue(res, i, 1), NULL, 10);
+        int cmp = 0;
+        if (!PQgetisnull(res, i, 1)) {
+            const char *v = PQgetvalue(res, i, 2);
+            cmp = (v && (v[0] == 't' || v[0] == 'T' || v[0] == '1')) ? 1 : 0;
+        }
+        cJSON_AddStringToObject(obj, "author_pubkey", pk);
+        cJSON_AddNumberToObject(obj, "until_cursor", (double)uc);
+        cJSON_AddBoolToObject(obj, "complete", cmp ? 1 : 0);
+        cJSON_AddItemToArray(arr, obj);
+    }
+    PQclear(res);
+    return arr;
+}
+
+int pg_inbox_reset_backfill_progress(void) {
+    if (!g_pg) {
+        DEBUG_ERROR("pg_inbox: not initialized");
+        return -1;
+    }
+    PGresult *res = PQexec(g_pg, "DELETE FROM caching_backfill_progress");
+    if (!res) {
+        DEBUG_ERROR("pg_inbox: reset_backfill_progress returned NULL result");
+        return -1;
+    }
+    ExecStatusType st = PQresultStatus(res);
+    if (st != PGRES_COMMAND_OK) {
+        DEBUG_ERROR("pg_inbox: reset_backfill_progress failed: %s",
+                    PQresultErrorMessage(res));
+        PQclear(res);
+        return -1;
+    }
+    PQclear(res);
+    return 0;
+}

caching/src/pg_inbox.h

@@ -0,0 +1,88 @@
+/*
+ * caching_relay - PostgreSQL inbox connection and operations.
+ *
+ * Manages the libpq connection to the c-relay-pg PostgreSQL database and
+ * provides functions to insert fetched events into the caching_event_inbox
+ * table, update the caching_service_state singleton, and read configuration
+ * from the shared config table.
+ */
+#ifndef CACHING_RELAY_PG_INBOX_H
+#define CACHING_RELAY_PG_INBOX_H
+
+#include <stddef.h>
+
+/* Forward declaration so the header does not pull in cJSON everywhere. */
+typedef struct cJSON cJSON;
+
+/* Initialize PostgreSQL connection using the given connection string.
+ * Returns 0 on success, -1 on error. */
+int pg_inbox_init(const char *connection_string);
+
+/* Close the PostgreSQL connection. */
+void pg_inbox_shutdown(void);
+
+/* Insert an event into the caching_event_inbox table.
+ * event_json is the raw Nostr event JSON string.
+ * source_relay may be NULL. source_class is "live", "discovery", or "backfill".
+ * priority is 0 (live/discovery) or 1 (backfill).
+ * Returns 0 on success (including conflict-skip), -1 on error. */
+int pg_inbox_insert_event(const char *event_json, const char *source_relay,
+                          const char *source_class, int priority);
+
+/* Update the caching_service_state singleton row.
+ * All string parameters may be NULL (left unchanged).
+ * Returns 0 on success, -1 on error. */
+int pg_inbox_update_status(const char *service_state,
+                           long config_generation,
+                           long heartbeat_at,
+                           int followed_author_count,
+                           int selected_relay_count,
+                           int connected_relay_count,
+                           int current_window_index,
+                           int backfill_cursor,
+                           long events_fetched,
+                           long inbox_inserts,
+                           const char *last_error,
+                           long last_error_at);
+
+/* Read a config value from the c-relay-pg config table.
+ * Returns a malloc'd string (caller must free) or NULL if not found/error. */
+char *pg_inbox_get_config_value(const char *key);
+
+/* Read the current config generation.
+ * Returns the generation number, or -1 on error. */
+long pg_inbox_get_config_generation(void);
+
+/* ------------------------------------------------------------------ */
+/* Backfill progress persistence                                       */
+/* ------------------------------------------------------------------ */
+
+/* Save backfill progress for a specific author+window (UPSERT).
+ * Returns 0 on success, -1 on error. */
+int pg_inbox_save_backfill_progress(const char *author_pubkey,
+                                    int window_index,
+                                    long window_anchor,
+                                    long until_cursor,
+                                    int complete);
+
+/* Load backfill progress for a specific author+window.
+ * Returns 0 on success (found), -1 on error.
+ * Sets *out_until_cursor and *out_complete. If not found, sets *out_found = 0
+ * and returns 0. */
+int pg_inbox_load_backfill_progress(const char *author_pubkey,
+                                    int window_index,
+                                    long *out_until_cursor,
+                                    int *out_complete,
+                                    int *out_found);
+
+/* Load all incomplete backfill progress rows for a given window.
+ * Returns a cJSON array of objects with "author_pubkey", "until_cursor" and
+ * "complete" fields. Caller must cJSON_Delete the result.
+ * Returns NULL on error or empty. */
+cJSON *pg_inbox_load_incomplete_progress(int window_index);
+
+/* Reset all backfill progress (delete all rows).
+ * Returns 0 on success, -1 on error. */
+int pg_inbox_reset_backfill_progress(void);
+
+#endif /* CACHING_RELAY_PG_INBOX_H */

caching/src/relay_discovery.c

@@ -0,0 +1,471 @@
+/*
+ * caching_relay - NIP-65 outbox relay discovery
+ */
+#define _GNU_SOURCE
+#include "relay_discovery.h"
+#include "debug.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <signal.h>
+
+/* Access the shutdown flag from main.c so we can abort discovery early. */
+extern volatile sig_atomic_t g_shutdown;
+
+/* Normalize a relay URL: strip trailing slash for consistency. */
+static void normalize_url(char *url, size_t maxlen) {
+    size_t len = strlen(url);
+    while (len > 0 && url[len - 1] == '/') {
+        url[--len] = '\0';
+    }
+    (void)maxlen;
+}
+
+/* ---- helpers ---- */
+
+static int get_pool_urls(nostr_relay_pool_t *pool, const char ***urls_out) {
+    char **listed = NULL;
+    nostr_pool_relay_status_t *statuses = NULL;
+    int n = nostr_relay_pool_list_relays(pool, &listed, &statuses);
+    const char **urls = NULL;
+    if (n > 0) {
+        urls = malloc(n * sizeof(char *));
+        for (int j = 0; j < n; j++) urls[j] = listed[j];
+    }
+    free(listed);
+    free(statuses);
+    *urls_out = urls;
+    return n;
+}
+
+/* Query the most recent kind 10002 for a single pubkey from a pool.
+ * Returns the event (caller must cJSON_Delete) or NULL. */
+static cJSON *query_kind10002(nostr_relay_pool_t *pool, const char *hex) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    cJSON_AddItemToArray(authors, cJSON_CreateString(hex));
+    cJSON_AddItemToObject(filter, "authors", authors);
+    cJSON *kinds = cJSON_CreateArray();
+    cJSON_AddItemToArray(kinds, cJSON_CreateNumber(10002));
+    cJSON_AddItemToObject(filter, "kinds", kinds);
+    cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(1));
+
+    const char **urls = NULL;
+    int n = get_pool_urls(pool, &urls);
+    int ev_count = 0;
+    cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 10000);
+    free(urls);
+    cJSON_Delete(filter);
+
+    cJSON *result = NULL;
+    if (events && ev_count > 0) {
+        result = events[0]; /* take first (most recent) */
+        for (int k = 1; k < ev_count; k++) cJSON_Delete(events[k]);
+    }
+    free(events);
+    return result;
+}
+
+/* Parse "r" tags from a kind 10002 event into an outbox entry.
+ * Only includes relays with "read" marker or no marker (assume both).
+ * Skips "write" only relays. */
+static int parse_r_tags(cJSON *event, cr_outbox_entry_t *entry) {
+    memset(entry, 0, sizeof(*entry));
+    cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
+    if (pubkey && cJSON_IsString(pubkey)) {
+        strncpy(entry->pubkey, cJSON_GetStringValue(pubkey), CR_HEX_LEN - 1);
+    }
+
+    cJSON *tags = cJSON_GetObjectItem(event, "tags");
+    if (!tags || !cJSON_IsArray(tags)) return 0;
+
+    int added = 0;
+    cJSON *tag;
+    cJSON_ArrayForEach(tag, tags) {
+        if (!cJSON_IsArray(tag)) continue;
+        cJSON *name = cJSON_GetArrayItem(tag, 0);
+        if (!name || !cJSON_IsString(name)) continue;
+        if (strcmp(cJSON_GetStringValue(name), "r") != 0) continue;
+        cJSON *url = cJSON_GetArrayItem(tag, 1);
+        if (!url || !cJSON_IsString(url)) continue;
+        const char *relay_url = cJSON_GetStringValue(url);
+        if (!relay_url || !relay_url[0]) continue;
+
+        /* Check marker (3rd element): "read", "write", or absent. */
+        cJSON *marker = cJSON_GetArrayItem(tag, 2);
+        if (marker && cJSON_IsString(marker)) {
+            const char *m = cJSON_GetStringValue(marker);
+            if (strcmp(m, "write") == 0) continue; /* skip write-only relays */
+        }
+        /* No marker or "read" => include. */
+
+        if (entry->relay_count >= CR_MAX_RELAYS_PER_PUBKEY) break;
+        strncpy(entry->relays[entry->relay_count], relay_url, CR_URL_LEN - 1);
+        entry->relays[entry->relay_count][CR_URL_LEN - 1] = '\0';
+        normalize_url(entry->relays[entry->relay_count], CR_URL_LEN);
+        entry->relay_count++;
+        added++;
+    }
+    return added;
+}
+
+/* ---- greedy set cover ---- */
+
+/* Build relay -> pubkey coverage map and compute minimum covering set. */
+static void compute_covering_set(cr_relay_map_t *map, cr_config_t *cfg,
+                                 const cr_pubkey_set_t *followed) {
+    /* Build a temporary relay -> pubkeys map. */
+    typedef struct {
+        char url[CR_URL_LEN];
+        char pubkeys[CR_MAX_PUBKEYS_PER_RELAY][CR_HEX_LEN];
+        int  pubkey_count;
+    } relay_cov_t;
+
+    relay_cov_t *relays = calloc(CR_MAX_DISCOVERED_RELAYS, sizeof(relay_cov_t));
+    int relay_count = 0;
+
+    /* Always include bootstrap relays in the coverage map. */
+    for (int i = 0; i < cfg->upstream_count && relay_count < CR_MAX_DISCOVERED_RELAYS; i++) {
+        strncpy(relays[relay_count].url, cfg->upstream_relays[i], CR_URL_LEN - 1);
+        relays[relay_count].url[CR_URL_LEN - 1] = '\0';
+        normalize_url(relays[relay_count].url, CR_URL_LEN);
+        relay_count++;
+    }
+
+    /* Add outbox relays and build coverage. */
+    for (int i = 0; i < map->outbox_count; i++) {
+        cr_outbox_entry_t *oe = &map->outboxes[i];
+        for (int r = 0; r < oe->relay_count; r++) {
+            /* Find or create relay entry. */
+            int found = -1;
+            for (int j = 0; j < relay_count; j++) {
+                if (strcmp(relays[j].url, oe->relays[r]) == 0) { found = j; break; }
+            }
+            if (found < 0) {
+                if (relay_count >= CR_MAX_DISCOVERED_RELAYS) break;
+                strncpy(relays[relay_count].url, oe->relays[r], CR_URL_LEN - 1);
+                found = relay_count++;
+            }
+            /* Add this pubkey to the relay's coverage. */
+            if (relays[found].pubkey_count < CR_MAX_PUBKEYS_PER_RELAY) {
+                strncpy(relays[found].pubkeys[relays[found].pubkey_count],
+                        oe->pubkey, CR_HEX_LEN - 1);
+                relays[found].pubkey_count++;
+            }
+        }
+    }
+
+    /* Greedy set cover. */
+    char *covered = calloc(followed->count, 1); /* 1 if pubkey is covered */
+    map->selected_count = 0;
+    map->coverage = calloc(map->outbox_count, sizeof(int));
+    for (int i = 0; i < map->outbox_count; i++) map->coverage[i] = -1;
+
+    int uncovered_count = followed->count;
+
+    /* Helper: find index of a pubkey in the followed set. */
+    /* (linear search, fine for our scale) */
+
+    while (uncovered_count > 0 && map->selected_count < CR_MAX_DISCOVERED_RELAYS) {
+        int best_relay = -1;
+        int best_count = 0;
+
+        for (int r = 0; r < relay_count; r++) {
+            /* Count how many uncovered pubkeys this relay covers. */
+            int count = 0;
+            for (int p = 0; p < relays[r].pubkey_count; p++) {
+                /* Find this pubkey in followed set. */
+                for (int i = 0; i < followed->count; i++) {
+                    if (strcmp(followed->items[i], relays[r].pubkeys[p]) == 0) {
+                        if (!covered[i]) count++;
+                        break;
+                    }
+                }
+            }
+            if (count > best_count) {
+                best_count = count;
+                best_relay = r;
+            }
+        }
+
+        if (best_relay < 0 || best_count == 0) break; /* no more coverage possible */
+
+        /* Select this relay. */
+        strncpy(map->selected_relays[map->selected_count],
+                relays[best_relay].url, CR_URL_LEN - 1);
+        map->selected_count++;
+
+        /* Mark covered pubkeys and record coverage. */
+        for (int p = 0; p < relays[best_relay].pubkey_count; p++) {
+            for (int i = 0; i < followed->count; i++) {
+                if (strcmp(followed->items[i], relays[best_relay].pubkeys[p]) == 0) {
+                    if (!covered[i]) {
+                        covered[i] = 1;
+                        uncovered_count--;
+                        /* Record which selected relay covers this outbox entry. */
+                        for (int oe_i = 0; oe_i < map->outbox_count; oe_i++) {
+                            if (strcmp(map->outboxes[oe_i].pubkey,
+                                       relays[best_relay].pubkeys[p]) == 0) {
+                                if (map->coverage[oe_i] < 0) {
+                                    map->coverage[oe_i] = map->selected_count - 1;
+                                }
+                            }
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+    }
+
+    /* Log results. */
+    DEBUG_INFO("relay_discovery: selected %d relays to cover %d/%d pubkeys",
+                map->selected_count, followed->count - uncovered_count, followed->count);
+    for (int i = 0; i < map->selected_count; i++) {
+        DEBUG_INFO("  relay[%d]: %s", i, map->selected_relays[i]);
+    }
+    if (uncovered_count > 0) {
+        DEBUG_WARN("relay_discovery: %d pubkeys have no outbox relays (covered by bootstrap)",
+                    uncovered_count);
+    }
+
+    free(covered);
+    free(relays);
+}
+
+/* ---- batched kind-10002 query ---- */
+
+/* Query kind 10002 for a batch of up to 100 pubkeys from a pool.
+ * Returns a cJSON array of events (caller must cJSON_Delete each + free).
+ * Sets *out_count to the number of events returned. */
+static cJSON **query_kind10002_batch(nostr_relay_pool_t *pool,
+                                     const char **pubkeys, int pubkey_count,
+                                     int *out_count) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    for (int i = 0; i < pubkey_count; i++)
+        cJSON_AddItemToArray(authors, cJSON_CreateString(pubkeys[i]));
+    cJSON_AddItemToObject(filter, "authors", authors);
+    cJSON *kinds = cJSON_CreateArray();
+    cJSON_AddItemToArray(kinds, cJSON_CreateNumber(10002));
+    cJSON_AddItemToObject(filter, "kinds", kinds);
+
+    const char **urls = NULL;
+    int n = get_pool_urls(pool, &urls);
+
+    /* TRACE: log the query being sent. */
+    if (g_debug_level >= DEBUG_LEVEL_TRACE) {
+        char *filter_str = cJSON_PrintUnformatted(filter);
+        DEBUG_TRACE("query_kind10002_batch: querying %d relays for %d pubkeys", n, pubkey_count);
+        if (filter_str) {
+            /* Only print first 200 chars of filter to avoid huge output */
+            if (strlen(filter_str) > 200) filter_str[200] = '\0';
+            DEBUG_TRACE("  filter: %s...", filter_str);
+            free(filter_str);
+        }
+    }
+
+    int ev_count = 0;
+    cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 30000);
+    free(urls);
+    cJSON_Delete(filter);
+
+    DEBUG_TRACE("query_kind10002_batch: returned %d events", ev_count);
+
+    *out_count = ev_count;
+    return events;
+}
+
+/* Process a batch of kind-10002 events: parse r-tags, populate outbox entries,
+ * publish to local relay. Returns count of events processed. */
+static int process_10002_batch(cJSON **events, int ev_count,
+                               cr_relay_map_t *map, cr_sink_t *sink,
+                               const char *source) {
+    int processed = 0;
+    for (int k = 0; k < ev_count; k++) {
+        cJSON *event = events[k];
+        if (!event) continue;
+
+        /* Get pubkey from event. */
+        cJSON *pk = cJSON_GetObjectItem(event, "pubkey");
+        if (!pk || !cJSON_IsString(pk)) { cJSON_Delete(event); continue; }
+        const char *hex = cJSON_GetStringValue(pk);
+
+        /* Find or create outbox entry for this pubkey. */
+        cr_outbox_entry_t *oe = NULL;
+        for (int i = 0; i < map->outbox_count; i++) {
+            if (strcmp(map->outboxes[i].pubkey, hex) == 0) { oe = &map->outboxes[i]; break; }
+        }
+        if (!oe) {
+            if (map->outbox_count >= /* followed->count passed in via map size */ 99999) {
+                cJSON_Delete(event); continue;
+            }
+            oe = &map->outboxes[map->outbox_count];
+            strncpy(oe->pubkey, hex, CR_HEX_LEN - 1);
+            map->outbox_count++;
+        }
+
+        int n = parse_r_tags(event, oe);
+        (void)n;
+
+        if (strcmp(source, "bootstrap") == 0) {
+            cr_sink_publish(sink, event);
+            /* Pump sink pool every 16 events to avoid pending publish overflow
+             * (NOSTR_POOL_MAX_PENDING_PUBLISHES = 32). */
+            if (processed > 0 && (processed % 16) == 0) {
+                cr_sink_pump(sink, 200);
+            }
+        }
+
+        processed++;
+        cJSON_Delete(event);
+    }
+    free(events);
+
+    /* Log sink stats after each batch. */
+    DEBUG_INFO("relay_discovery: batch processed %d events (sink: ok=%ld reject=%ld dup=%ld err=%ld)",
+               processed, sink->published_ok, sink->published_failed,
+               sink->published_dup, 0L);
+
+    return processed;
+}
+
+/* ---- main discovery function ---- */
+
+#define CR_10002_BATCH_SIZE 100
+
+int cr_relay_discovery_run(cr_relay_map_t *map, cr_config_t *cfg,
+                           nostr_relay_pool_t *upstream, cr_sink_t *sink,
+                           const cr_pubkey_set_t *followed, int is_first_time) {
+    memset(map, 0, sizeof(*map));
+    map->outboxes = calloc(followed->count, sizeof(cr_outbox_entry_t));
+    if (!map->outboxes) return -1;
+    map->outbox_count = 0;
+
+    DEBUG_INFO("relay_discovery: discovering outbox relays for %d pubkeys (%s)",
+                followed->count, is_first_time ? "first-time" : "local-first");
+
+    int found_local = 0, found_bootstrap = 0, found_none = 0;
+
+    /* Pre-populate outbox entries for all followed pubkeys (empty, for coverage
+     * tracking of pubkeys with no 10002). */
+    for (int i = 0; i < followed->count; i++) {
+        strncpy(map->outboxes[i].pubkey, followed->items[i], CR_HEX_LEN - 1);
+    }
+    map->outbox_count = followed->count;
+
+    /* Track which pubkeys we've found 10002 for. */
+    char *found = calloc(followed->count, 1);
+
+    /* Phase 1: If subsequent startup, batch-query local relay first. */
+    if (!is_first_time && sink && sink->pool) {
+        DEBUG_INFO("relay_discovery: querying local relay for kind-10002 (batched)...");
+
+        for (int start = 0; start < followed->count && !g_shutdown; start += CR_10002_BATCH_SIZE) {
+            int batch = followed->count - start;
+            if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
+
+            int ev_count = 0;
+            cJSON **events = query_kind10002_batch(sink->pool,
+                                                   (const char **)&followed->items[start],
+                                                   batch, &ev_count);
+            if (events && ev_count > 0) {
+                for (int k = 0; k < ev_count; k++) {
+                    cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
+                    if (pk && cJSON_IsString(pk)) {
+                        const char *hex = cJSON_GetStringValue(pk);
+                        for (int i = 0; i < followed->count; i++) {
+                            if (strcmp(followed->items[i], hex) == 0) {
+                                if (!found[i]) { found[i] = 1; found_local++; }
+                                break;
+                            }
+                        }
+                    }
+                }
+                process_10002_batch(events, ev_count, map, sink, "local");
+                /* Pump sink pool to flush published events. */
+                cr_sink_pump(sink, 500);
+            } else {
+                free(events);
+            }
+        }
+        DEBUG_INFO("relay_discovery: local relay provided %d kind-10002 events", found_local);
+    }
+
+    /* Phase 2: Batch-query bootstrap relays for pubkeys not found locally. */
+    int remaining = 0;
+    for (int i = 0; i < followed->count; i++) if (!found[i]) remaining++;
+
+    if (remaining > 0) {
+        DEBUG_INFO("relay_discovery: querying bootstrap relays for %d remaining pubkeys (batched)...",
+                    remaining);
+
+        /* Build list of remaining pubkeys. */
+        const char **remaining_pk = malloc(remaining * sizeof(char *));
+        int ridx = 0;
+        for (int i = 0; i < followed->count; i++) {
+            if (!found[i]) remaining_pk[ridx++] = followed->items[i];
+        }
+
+        for (int start = 0; start < remaining && !g_shutdown; start += CR_10002_BATCH_SIZE) {
+            int batch = remaining - start;
+            if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
+
+            int ev_count = 0;
+            cJSON **events = query_kind10002_batch(upstream,
+                                                   &remaining_pk[start],
+                                                   batch, &ev_count);
+            if (events && ev_count > 0) {
+                for (int k = 0; k < ev_count; k++) {
+                    cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
+                    if (pk && cJSON_IsString(pk)) {
+                        const char *hex = cJSON_GetStringValue(pk);
+                        for (int i = 0; i < followed->count; i++) {
+                            if (strcmp(followed->items[i], hex) == 0) {
+                                if (!found[i]) { found[i] = 1; found_bootstrap++; }
+                                break;
+                            }
+                        }
+                    }
+                }
+                process_10002_batch(events, ev_count, map, sink, "bootstrap");
+                /* Pump sink pool to flush published events. */
+                cr_sink_pump(sink, 500);
+            } else {
+                free(events);
+            }
+        }
+        free(remaining_pk);
+    }
+
+    found_none = followed->count - found_local - found_bootstrap;
+    DEBUG_INFO("relay_discovery: %d local, %d bootstrap, %d none",
+                found_local, found_bootstrap, found_none);
+
+    free(found);
+
+    /* Compute minimum covering set. */
+    compute_covering_set(map, cfg, followed);
+
+    return 0;
+}
+
+void cr_relay_map_free(cr_relay_map_t *map) {
+    if (map->outboxes) {
+        free(map->outboxes);
+        map->outboxes = NULL;
+    }
+    if (map->coverage) {
+        free(map->coverage);
+        map->coverage = NULL;
+    }
+    map->outbox_count = 0;
+    map->selected_count = 0;
+}
+
+const cr_outbox_entry_t *cr_relay_map_get_outbox(const cr_relay_map_t *map, const char *hex) {
+    for (int i = 0; i < map->outbox_count; i++) {
+        if (strcmp(map->outboxes[i].pubkey, hex) == 0) return &map->outboxes[i];
+    }
+    return NULL;
+}

caching/src/relay_discovery.h

@@ -0,0 +1,67 @@
+/*
+ * caching_relay - NIP-65 outbox relay discovery.
+ *
+ * For each followed pubkey, fetches their kind 10002 (relay list) and parses
+ * the "r" tags to build a per-pubkey outbox relay map. Then computes the
+ * minimum covering set of relays (greedy set cover) that covers all followed
+ * pubkeys.
+ *
+ * On first-time startup, queries bootstrap relays for kind 10002.
+ * On subsequent startups, queries the local relay first (fast), falls back
+ * to bootstrap relays for any pubkey not found locally.
+ */
+#ifndef CACHING_RELAY_RELAY_DISCOVERY_H
+#define CACHING_RELAY_RELAY_DISCOVERY_H
+
+#include "config.h"
+#include "state.h"
+#include "relay_sink.h"
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+
+#define CR_MAX_RELAYS_PER_PUBKEY  8
+#define CR_MAX_PUBKEYS_PER_RELAY  512
+#define CR_MAX_DISCOVERED_RELAYS  128
+
+/* Per-pubkey outbox relay list. */
+typedef struct {
+    char pubkey[CR_HEX_LEN];
+    char relays[CR_MAX_RELAYS_PER_PUBKEY][CR_URL_LEN];
+    int  relay_count;
+} cr_outbox_entry_t;
+
+/* Result of relay discovery. */
+typedef struct {
+    cr_outbox_entry_t *outboxes;      /* per-pubkey relay lists (heap) */
+    int                outbox_count;
+
+    /* The selected minimum covering set of relay URLs. */
+    char               selected_relays[CR_MAX_DISCOVERED_RELAYS][CR_URL_LEN];
+    int                selected_count;
+
+    /* Per-pubkey: which selected relay covers this pubkey (index into selected_relays).
+     * -1 means covered by bootstrap relays only. */
+    int               *coverage;      /* heap, outbox_count entries */
+} cr_relay_map_t;
+
+/* Discover outbox relays for all followed pubkeys.
+ *
+ *   cfg       - config (provides bootstrap relays, local_relay, root_hex)
+ *   upstream  - upstream pool (has bootstrap relays; discovered relays will be added)
+ *   sink      - sink pool (local relay, for publishing 10002 events + querying on subsequent startup)
+ *   followed  - the followed pubkey set
+ *   is_first_time - 1 if first startup (query bootstrap only), 0 if subsequent (local-first)
+ *
+ * Returns 0 on success, -1 on hard error. On success, map is populated.
+ * Caller must call cr_relay_map_free() when done.
+ */
+int  cr_relay_discovery_run(cr_relay_map_t *map, cr_config_t *cfg,
+                            nostr_relay_pool_t *upstream, cr_sink_t *sink,
+                            const cr_pubkey_set_t *followed, int is_first_time);
+
+/* Free heap resources in a relay map. */
+void cr_relay_map_free(cr_relay_map_t *map);
+
+/* Get the outbox relays for a specific pubkey. Returns NULL if not found. */
+const cr_outbox_entry_t *cr_relay_map_get_outbox(const cr_relay_map_t *map, const char *hex);
+
+#endif /* CACHING_RELAY_RELAY_DISCOVERY_H */

caching/src/relay_sink.c

@@ -0,0 +1,161 @@
+/*
+ * caching_relay - relay sink: publishes fetched events to the local relay.
+ *
+ * See relay_sink.h for the dual-mode (WebSocket / PostgreSQL inbox) design.
+ */
+#define _GNU_SOURCE
+#include "relay_sink.h"
+#include "pg_inbox.h"
+#include "debug.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+/* ------------------------------------------------------------------ */
+/* WebSocket mode callback                                             */
+/* ------------------------------------------------------------------ */
+
+static void sink_publish_cb(const char *relay_url, const char *event_id,
+                             int success, const char *message, void *user_data) {
+    cr_sink_t *sink = (cr_sink_t *)user_data;
+    if (success) {
+        sink->published_ok++;
+        DEBUG_TRACE("sink: OK %s from %s", event_id ? event_id : "?",
+                    relay_url ? relay_url : "?");
+    } else {
+        sink->published_failed++;
+        /* REJECT is important - show at INFO level so it's visible at level 3. */
+        DEBUG_INFO("sink: REJECT %s from %s: %s", event_id ? event_id : "?",
+                    relay_url ? relay_url : "?",
+                    message ? message : "(no message)");
+    }
+}
+
+/* ------------------------------------------------------------------ */
+/* Init                                                                */
+/* ------------------------------------------------------------------ */
+
+int cr_sink_init(cr_sink_t *sink, const char *local_relay_url, cr_seen_ring_t *seen) {
+    memset(sink, 0, sizeof(*sink));
+    sink->seen = seen;
+    sink->source_class = CR_SINK_CLASS_LIVE;
+    strncpy(sink->url, local_relay_url, sizeof(sink->url) - 1);
+
+    sink->pool = nostr_relay_pool_create(nostr_pool_reconnect_config_default());
+    if (!sink->pool) {
+        DEBUG_ERROR("sink: failed to create pool");
+        return -1;
+    }
+    if (nostr_relay_pool_add_relay(sink->pool, local_relay_url) != NOSTR_SUCCESS) {
+        DEBUG_ERROR("sink: failed to add relay %s", local_relay_url);
+        nostr_relay_pool_destroy(sink->pool);
+        sink->pool = NULL;
+        return -1;
+    }
+    DEBUG_INFO("sink: initialized (WebSocket) for %s", local_relay_url);
+    return 0;
+}
+
+int cr_sink_init_pg(cr_sink_t *sink, cr_seen_ring_t *seen) {
+    memset(sink, 0, sizeof(*sink));
+    sink->seen = seen;
+    sink->source_class = CR_SINK_CLASS_LIVE;
+    sink->pool = NULL; /* PostgreSQL inbox mode - no WebSocket pool. */
+    DEBUG_INFO("sink: initialized (PostgreSQL inbox)");
+    return 0;
+}
+
+void cr_sink_set_source_class(cr_sink_t *sink, const char *class_ptr) {
+    if (sink) sink->source_class = class_ptr ? class_ptr : CR_SINK_CLASS_LIVE;
+}
+
+void cr_sink_destroy(cr_sink_t *sink) {
+    if (sink && sink->pool) {
+        nostr_relay_pool_destroy(sink->pool);
+        sink->pool = NULL;
+    }
+}
+
+/* ------------------------------------------------------------------ */
+/* Publish                                                             */
+/* ------------------------------------------------------------------ */
+
+int cr_sink_publish_from(cr_sink_t *sink, cJSON *event, const char *source_relay) {
+    if (!sink || !event) return -1;
+
+    cJSON *id = cJSON_GetObjectItem(event, "id");
+    if (!id || !cJSON_IsString(id)) {
+        DEBUG_WARN("sink: event missing id, skipping");
+        return -1;
+    }
+    const char *eid = cJSON_GetStringValue(id);
+
+    if (cr_seen_ring_add(sink->seen, eid) == 0) {
+        sink->published_dup++;
+        return 0; /* already seen recently */
+    }
+
+    /* PostgreSQL inbox mode. */
+    if (!sink->pool) {
+        char *json = cJSON_PrintUnformatted(event);
+        if (!json) {
+            DEBUG_WARN("sink: failed to serialize event %s", eid);
+            sink->published_failed++;
+            return -1;
+        }
+        const char *klass = sink->source_class ? sink->source_class
+                                               : CR_SINK_CLASS_LIVE;
+        int priority = (strcmp(klass, CR_SINK_CLASS_BACKFILL) == 0) ? 1 : 0;
+
+        cJSON *kind = cJSON_GetObjectItem(event, "kind");
+        cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
+        int kind_num = (kind && cJSON_IsNumber(kind)) ? (int)cJSON_GetNumberValue(kind) : -1;
+        DEBUG_TRACE("sink: INBOX event id=%s kind=%d pubkey=%s class=%s",
+                    eid, kind_num,
+                    pubkey && cJSON_IsString(pubkey) ? cJSON_GetStringValue(pubkey) : "?",
+                    klass);
+
+        int rc = pg_inbox_insert_event(json, source_relay, klass, priority);
+        free(json);
+        if (rc != 0) {
+            sink->published_failed++;
+            return -1;
+        }
+        sink->published_ok++;
+        return 1;
+    }
+
+    /* WebSocket mode. */
+    const char *urls[1] = { sink->url };
+    cJSON *kind = cJSON_GetObjectItem(event, "kind");
+    cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
+    int kind_num = (kind && cJSON_IsNumber(kind)) ? (int)cJSON_GetNumberValue(kind) : -1;
+    DEBUG_TRACE("sink: SEND event id=%s kind=%d pubkey=%s to %s",
+                eid, kind_num,
+                pubkey && cJSON_IsString(pubkey) ? cJSON_GetStringValue(pubkey) : "?",
+                sink->url);
+
+    int rc = nostr_relay_pool_publish_async(sink->pool, urls, 1, event,
+                                            sink_publish_cb, sink);
+    if (rc < 0) {
+        DEBUG_WARN("sink: publish_async error rc=%d for %s", rc, eid);
+        return -1;
+    }
+    if (rc == 0) {
+        DEBUG_INFO("sink: publish SKIPPED (not connected) for %s", eid);
+        return 0;
+    }
+    DEBUG_TRACE("sink: publish_async rc=%d for %s", rc, eid);
+    return 1;
+}
+
+int cr_sink_publish(cr_sink_t *sink, cJSON *event) {
+    return cr_sink_publish_from(sink, event, NULL);
+}
+
+void cr_sink_pump(cr_sink_t *sink, int timeout_ms) {
+    if (sink && sink->pool) {
+        nostr_relay_pool_run(sink->pool, timeout_ms);
+    }
+    /* No-op in PostgreSQL inbox mode. */
+}

caching/src/relay_sink.h

@@ -0,0 +1,75 @@
+/*
+ * caching_relay - relay sink: publishes fetched events to the local relay.
+ *
+ * Two modes:
+ *   - WebSocket mode (legacy): a dedicated single-relay pool publishes via
+ *     nostr_relay_pool_publish_async(). Used when no PostgreSQL connection
+ *     string is provided.
+ *   - PostgreSQL inbox mode: events are inserted directly into the
+ *     caching_event_inbox table via pg_inbox_insert_event(). Used when the
+ *     daemon is started with --pg-conn.
+ *
+ * The public function signatures are identical in both modes so that callers
+ * (live_subscriber, backfill, relay_discovery) do not need to change.
+ */
+#ifndef CACHING_RELAY_RELAY_SINK_H
+#define CACHING_RELAY_RELAY_SINK_H
+
+#include "state.h"
+#include "../nostr_core_lib/nostr_core/nostr_core.h"
+
+/* Source class labels for the inbox priority field. */
+#define CR_SINK_CLASS_LIVE      "live"
+#define CR_SINK_CLASS_DISCOVERY "discovery"
+#define CR_SINK_CLASS_BACKFILL  "backfill"
+
+typedef struct {
+    /* WebSocket mode only (NULL in PostgreSQL inbox mode). */
+    nostr_relay_pool_t     *pool;
+    char                    url[256];
+
+    /* PostgreSQL inbox mode: source class used for priority assignment.
+     * Defaults to "live". Set to "backfill" by the backfill module.
+     * Ignored in WebSocket mode. */
+    const char             *source_class;
+
+    cr_seen_ring_t         *seen;   /* shared seen ring for dedup */
+    long                    published_ok;
+    long                    published_failed;
+    long                    published_dup;
+} cr_sink_t;
+
+/* Create the sink.
+ *
+ * In WebSocket mode (pg_mode = 0): creates a relay pool and adds
+ * local_relay_url.
+ * In PostgreSQL inbox mode (pg_mode != 0): no pool is created; the URL is
+ * ignored. Events are inserted via pg_inbox_insert_event() (the pg_inbox
+ * module must be initialized beforehand).
+ *
+ * The original two-argument signature is preserved via cr_sink_init() below
+ * for backward compatibility; it defaults to WebSocket mode. */
+int  cr_sink_init(cr_sink_t *sink, const char *local_relay_url, cr_seen_ring_t *seen);
+
+/* Initialize the sink in PostgreSQL inbox mode. */
+int  cr_sink_init_pg(cr_sink_t *sink, cr_seen_ring_t *seen);
+
+/* Set the source class for priority assignment in PostgreSQL inbox mode.
+ * class_ptr must point to a static string literal (e.g. CR_SINK_CLASS_LIVE). */
+void cr_sink_set_source_class(cr_sink_t *sink, const char *class_ptr);
+
+void cr_sink_destroy(cr_sink_t *sink);
+
+/* Publish a cJSON event. Dedups against the seen ring.
+ * Returns 1 if enqueued/inserted, 0 if dup (skipped), -1 on error. */
+int  cr_sink_publish(cr_sink_t *sink, cJSON *event);
+
+/* Variant that records the originating relay URL in PostgreSQL inbox mode.
+ * In WebSocket mode the source_relay is ignored. */
+int  cr_sink_publish_from(cr_sink_t *sink, cJSON *event, const char *source_relay);
+
+/* Pump the sink pool briefly to flush pending publish callbacks.
+ * No-op in PostgreSQL inbox mode. */
+void cr_sink_pump(cr_sink_t *sink, int timeout_ms);
+
+#endif /* CACHING_RELAY_RELAY_SINK_H */

caching/src/state.c

@@ -0,0 +1,76 @@
+/*
+ * caching_relay - in-memory runtime state
+ */
+#define _GNU_SOURCE
+#include "state.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/* ---- pubkey set ---- */
+
+void cr_pubkey_set_init(cr_pubkey_set_t *s) {
+    s->items = NULL;
+    s->count = 0;
+    s->capacity = 0;
+}
+
+void cr_pubkey_set_free(cr_pubkey_set_t *s) {
+    for (int i = 0; i < s->count; i++) free(s->items[i]);
+    free(s->items);
+    s->items = NULL;
+    s->count = 0;
+    s->capacity = 0;
+}
+
+void cr_pubkey_set_clear(cr_pubkey_set_t *s) {
+    cr_pubkey_set_free(s);
+    cr_pubkey_set_init(s);
+}
+
+int cr_pubkey_set_contains(const cr_pubkey_set_t *s, const char *hex) {
+    for (int i = 0; i < s->count; i++) {
+        if (strcmp(s->items[i], hex) == 0) return 1;
+    }
+    return 0;
+}
+
+int cr_pubkey_set_add(cr_pubkey_set_t *s, const char *hex) {
+    if (!hex || !hex[0]) return 0;
+    if (cr_pubkey_set_contains(s, hex)) return 0;
+    if (s->count >= s->capacity) {
+        int newcap = s->capacity ? s->capacity * 2 : 64;
+        char **ni = realloc(s->items, newcap * sizeof(char *));
+        if (!ni) return -1;
+        s->items = ni;
+        s->capacity = newcap;
+    }
+    s->items[s->count] = strdup(hex);
+    if (!s->items[s->count]) return -1;
+    s->count++;
+    return 1;
+}
+
+/* ---- seen-event ring buffer ---- */
+
+void cr_seen_ring_init(cr_seen_ring_t *r) {
+    memset(r, 0, sizeof(*r));
+}
+
+int cr_seen_ring_contains(const cr_seen_ring_t *r, const char *id) {
+    for (int i = 0; i < r->count; i++) {
+        int idx = (r->head - 1 - i + CR_SEEN_RING_SIZE) % CR_SEEN_RING_SIZE;
+        if (strcmp(r->ids[idx], id) == 0) return 1;
+    }
+    return 0;
+}
+
+int cr_seen_ring_add(cr_seen_ring_t *r, const char *id) {
+    if (!id || !id[0]) return 0;
+    if (cr_seen_ring_contains(r, id)) return 0;
+    strncpy(r->ids[r->head], id, CR_HEX_LEN - 1);
+    r->ids[r->head][CR_HEX_LEN - 1] = '\0';
+    r->head = (r->head + 1) % CR_SEEN_RING_SIZE;
+    if (r->count < CR_SEEN_RING_SIZE) r->count++;
+    return 1;
+}

caching/src/state.h

@@ -0,0 +1,41 @@
+/*
+ * caching_relay - in-memory runtime state:
+ *   - followed-pubkey set (hex pubkeys)
+ *   - seen-event-id ring buffer (to avoid redundant publish attempts)
+ */
+#ifndef CACHING_RELAY_STATE_H
+#define CACHING_RELAY_STATE_H
+
+#include <stddef.h>
+
+/* Hex pubkey is 64 chars + NUL. Event id is 64 hex chars + NUL. */
+#define CR_HEX_LEN 65
+
+/* A simple dynamic string set for followed pubkeys. */
+typedef struct {
+    char  **items;
+    int     count;
+    int     capacity;
+} cr_pubkey_set_t;
+
+void cr_pubkey_set_init(cr_pubkey_set_t *s);
+void cr_pubkey_set_free(cr_pubkey_set_t *s);
+int  cr_pubkey_set_add(cr_pubkey_set_t *s, const char *hex); /* 1 if added, 0 if dup */
+int  cr_pubkey_set_contains(const cr_pubkey_set_t *s, const char *hex);
+void cr_pubkey_set_clear(cr_pubkey_set_t *s);
+
+/* Fixed-size ring buffer of event ids for dedup. */
+#define CR_SEEN_RING_SIZE 4096
+
+typedef struct {
+    char  ids[CR_SEEN_RING_SIZE][CR_HEX_LEN];
+    int   head;
+    int   count;
+} cr_seen_ring_t;
+
+void cr_seen_ring_init(cr_seen_ring_t *r);
+/* Returns 1 if the id was newly inserted, 0 if it was already present. */
+int  cr_seen_ring_add(cr_seen_ring_t *r, const char *id);
+int  cr_seen_ring_contains(const cr_seen_ring_t *r, const char *id);
+
+#endif /* CACHING_RELAY_STATE_H */

make_and_restart_relay.sh

@@ -455,6 +455,19 @@ fi
 
 echo "Using static binary: $BINARY_PATH"
 
+# Build the caching_relay binary (from caching/ directory)
+# This is needed because the caching service launcher forks this binary.
+if [ -d "caching" ] && [ -f "caching/Makefile" ]; then
+    echo "Building caching_relay binary..."
+    (cd caching && make) > /dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        echo "WARNING: caching_relay build failed. The caching service will not be available."
+        echo "You can build it manually with: cd caching && make"
+    else
+        echo "✓ caching_relay binary built successfully"
+    fi
+fi
+
 echo "Build successful. Proceeding with relay restart..."
 
 # Kill existing relay if running - start aggressive immediately

plans/caching_code_consolidation_plan.md

@@ -0,0 +1,72 @@
+# Caching Code Consolidation Plan
+
+## Goal
+
+Move all caching_relay source code into a `caching/` directory inside c-relay-pg,
+and switch the launcher to use PostgreSQL mode (`-p <pg-conn>`) so all config
+comes from the c-relay-pg config table — no JSON config file needed.
+
+## Key Finding
+
+The caching_relay binary **already supports PG mode** via `-p <pg-conn>`. Its
+`pg_config.c` reads the same config table keys the caching page sets
+(`caching_root_npubs`, `caching_bootstrap_relays`, `caching_kinds`, etc.), and
+`pg_inbox.c` writes fetched events to the `caching_event_inbox` table. The PG
+schema tables already exist in `src/pg_schema.sql`.
+
+## Steps
+
+### Step 1: Create `caching/` directory and copy source files
+
+Copy all `src/*.c` and `src/*.h` from `/home/user/lt/caching_relay/src/` into
+`caching/src/` in this repo. Also copy the Makefile, build_static.sh,
+Dockerfile.alpine-musl, and VERSION.
+
+Files to copy:
+- src/main.c, src/main.h
+- src/config.c, src/config.h (legacy JSON config — kept for fallback)
+- src/pg_config.c, src/pg_config.h (PG config reader)
+- src/pg_inbox.c, src/pg_inbox.h (PG inbox writer)
+- src/backfill.c, src/backfill.h
+- src/follow_graph.c, src/follow_graph.h
+- src/live_subscriber.c, src/live_subscriber.h
+- src/relay_sink.c, src/relay_sink.h (legacy WebSocket sink — kept for fallback)
+- src/relay_discovery.c, src/relay_discovery.h
+- src/state.c, src/state.h
+- src/debug.c, src/debug.h
+- src/jsonc_strip.c, src/jsonc_strip.h
+- Makefile
+- VERSION
+
+### Step 2: Adapt the Makefile
+
+Update `caching/Makefile` to:
+- Point `NOSTR_CORE_DIR` at `../nostr_core_lib` (sibling in this repo)
+- Output binary to `caching/caching_relay` or `build/caching_relay`
+
+### Step 3: Update the launcher to use PG mode
+
+Update `src/caching_service_launcher.c`:
+- Change `caching_service_start_fork()` to pass `-p <pg_conn>` instead of
+  `-c <config_path>`
+- Update `caching_service_start()` to read `caching_service_pg_conn` from config
+  table instead of `caching_service_config_path`
+
+### Step 4: Remove config_path from defaults and UI
+
+- Remove `caching_service_config_path` from `src/default_config_event.h`
+- Remove the "Caching Service Config Path" field from `api/index.html`
+- Remove the field from `CACHING_CONFIG_FIELDS` in `api/index.js`
+- Update `caching_service_binary_path` default to point at the new build
+  location (`./caching/caching_relay` or `./build/caching_relay`)
+
+### Step 5: Build and test
+
+- Build the caching_relay binary from the new `caching/` directory
+- Build c-relay-pg with the updated launcher
+- Test: set caching config on the page, apply, start service, verify it runs
+  in PG mode reading config from the database
+
+### Step 6: Push
+
+Run `./increment_and_push.sh` with a descriptive commit message.

relay.pid

@@ -1 +1 @@
-194734
+549408

src/caching_service_launcher.c

@@ -21,12 +21,17 @@ static pid_t g_caching_child_pid = -1;
 // Fork/exec implementation
 // ---------------------------------------------------------------------------
 
-static int caching_service_start_fork(const char* binary_path, const char* config_path) {
+static int caching_service_start_fork(const char* binary_path, const char* pg_conn) {
     if (!binary_path || binary_path[0] == '\0') {
         DEBUG_ERROR("caching_service_start: caching_service_binary_path is not set");
         return -1;
     }
 
+    if (!pg_conn || pg_conn[0] == '\0') {
+        DEBUG_ERROR("caching_service_start: caching_service_pg_conn is not set");
+        return -1;
+    }
+
     // Check that the binary exists and is executable
     if (access(binary_path, X_OK) != 0) {
         DEBUG_ERROR("caching_service_start: binary '%s' not found or not executable: %s",
@@ -49,8 +54,7 @@ static int caching_service_start_fork(const char* binary_path, const char* confi
         g_caching_child_pid = -1;
     }
 
-    DEBUG_INFO("caching_service_start: forking '%s' with config '%s'",
-               binary_path, config_path ? config_path : "(default)");
+    DEBUG_INFO("caching_service_start: forking '%s' with -p (PG mode)", binary_path);
 
     pid_t pid = fork();
     if (pid < 0) {
@@ -82,18 +86,15 @@ static int caching_service_start_fork(const char* binary_path, const char* confi
         }
 
         // Build argv for the caching_relay binary.
-        // The caching_relay binary uses a JSON config file, not a --pg-conn flag.
-        // Interface: caching_relay [-c <config.jsonc>] [-d <level>] [-r]
-        // If a config path is provided, pass it via -c; otherwise let the binary
-        // use its default (./caching_relay_config.jsonc).
+        // The caching_relay binary supports PostgreSQL mode via -p <pg-conn>,
+        // which reads all config from the c-relay-pg config table and writes
+        // fetched events to the caching_event_inbox table. No JSON config file
+        // is needed in this mode.
         char* argv[5];
         int argc = 0;
         argv[argc++] = (char*)binary_path;
-
-        if (config_path && config_path[0] != '\0') {
-            argv[argc++] = (char*)"-c";
-            argv[argc++] = (char*)config_path;
-        }
+        argv[argc++] = (char*)"-p";
+        argv[argc++] = (char*)pg_conn;
         argv[argc] = NULL;
 
         execv(binary_path, argv);
@@ -200,12 +201,13 @@ int caching_service_start(void) {
     const char* launch_mode = get_config_value("caching_service_launch_mode");
     if (!launch_mode || strcmp(launch_mode, "fork") == 0) {
         const char* binary_path = get_config_value("caching_service_binary_path");
-        // The caching_relay binary uses a JSON config file (-c <path>), not a
-        // --pg-conn flag. Read the config path from the config table.
-        const char* config_path = get_config_value("caching_service_config_path");
-        int rc = caching_service_start_fork(binary_path, config_path);
+        // The caching_relay binary uses PostgreSQL mode (-p <pg-conn>) to read
+        // all config from the c-relay-pg config table and write events to the
+        // caching_event_inbox table. No JSON config file is needed.
+        const char* pg_conn = get_config_value("caching_service_pg_conn");
+        int rc = caching_service_start_fork(binary_path, pg_conn);
         if (binary_path) free((char*)binary_path);
-        if (config_path) free((char*)config_path);
+        if (pg_conn) free((char*)pg_conn);
         if (launch_mode) free((char*)launch_mode);
         return rc;
     }

src/default_config_event.h

@@ -140,9 +140,9 @@ static const struct {
     // External caching service configuration. All caching_ keys are dynamic (no restart required).
     {"caching_enabled", "false"},                              // Enable caching inbox consumer
     {"caching_config_generation", "0"},                        // Configuration generation counter for external service
-    {"caching_root_npubs", ""},                                // Comma-separated root npubs to follow
-    {"caching_bootstrap_relays", ""},                          // Comma-separated bootstrap relay URLs
-    {"caching_kinds", "1,3,6,10000,30023"},                    // Event kinds to cache for non-root follows
+    {"caching_root_npubs", "npub13lm5wf8dvsdnc2894pkhch9uf8phvw9varrv8zf4sc885hhdmc8q6lx7ks"}, // Comma-separated root npubs to follow (default: admin npub)
+    {"caching_bootstrap_relays", "wss://relay.damus.io,wss://nos.lol,wss://relay.primal.net,wss://laantungir.net/relay"}, // Comma-separated bootstrap relay URLs
+    {"caching_kinds", "0,1,3,6,10000,10002,30023"},            // Event kinds to cache for non-root follows
     {"caching_admin_kinds", "*"},                              // Event kinds for root npubs (* = all)
     {"caching_live_enabled", "true"},                          // Enable live subscriptions
     {"caching_live_resubscribe_seconds", "300"},               // Live subscription resubscribe interval
@@ -162,9 +162,8 @@ static const struct {
     {"caching_inbox_active_poll_ms", "200"},                   // Poll interval when inbox has events
     {"caching_inbox_idle_poll_ms", "5000"},                    // Poll interval when inbox is empty
     {"caching_max_event_json_bytes", "65536"},                  // Maximum event JSON size for inbox insert
-    {"caching_service_binary_path", "/home/user/lt/caching_relay/caching_relay"}, // Path to caching_relay binary (for fork/exec launch)
-    {"caching_service_config_path", "/home/user/lt/caching_relay/caching_relay_config.jsonc"}, // Path to caching_relay JSON config file
-    {"caching_service_pg_conn", "host=localhost port=5432 dbname=crelay user=crelay password=crelay"}, // PostgreSQL connection string for caching service (reserved for future PG-inbox integration)
+    {"caching_service_binary_path", "./caching_relay"}, // Path to caching_relay binary (relative to relay CWD which is build/)
+    {"caching_service_pg_conn", "host=localhost port=5432 dbname=crelay user=crelay password=crelay"}, // PostgreSQL connection string for caching service (PG-inbox mode)
     {"caching_service_launch_mode", "fork"}                     // Launch mode: "fork" or "systemd" (systemd not yet implemented)
 };
 

src/main.h

@@ -13,8 +13,8 @@
 // Using CRELAY_ prefix to avoid conflicts with nostr_core_lib VERSION macros
 #define CRELAY_VERSION_MAJOR 2
 #define CRELAY_VERSION_MINOR 1
-#define CRELAY_VERSION_PATCH 18
-#define CRELAY_VERSION "v2.1.18"
+#define CRELAY_VERSION_PATCH 21
+#define CRELAY_VERSION "v2.1.21"
 
 // Relay metadata (authoritative source for NIP-11 information)
 #define RELAY_NAME "C-Relay-PG"