v2.1.28 - Add kind-3 fallback for relay discovery (NIP-65 pre-standard relay lists in contact list r tags)

admin/api/caching.php

@@ -99,7 +99,7 @@ try {
                (SELECT count(*) FROM caching_backfill_relay_progress WHERE author_pubkey = fp.pubkey AND complete = false) AS relay_incomplete
         FROM caching_followed_pubkeys fp
         LEFT JOIN profiles p ON p.pubkey = fp.pubkey
-        ORDER BY fp.is_root DESC, fp.events_fetched DESC LIMIT 100
+        ORDER BY fp.is_root DESC, fp.events_fetched DESC LIMIT 1000
     ")->fetchAll();
     // Fetch per-relay progress for all followed pubkeys in one query
     $relayProgress = [];

admin/assets/app.js

@@ -447,8 +447,10 @@ async function loadCaching() {
             if (d.follows.length === 0) {
                 tbody.innerHTML = '<tr><td colspan="6" style="text-align:center;font-style:italic">No followed pubkeys</td></tr>';
             } else {
+                const activePk = d.active && d.active.pubkey ? d.active.pubkey : '';
                 tbody.innerHTML = d.follows.map((f, i) => {
                     const npub = f.npub || f.pubkey.substring(0, 20);
+                    const isActive = activePk && f.pubkey === activePk;
                     const relaySummary = f.relay_incomplete > 0
                         ? `<span class="status-working">${f.relay_count - f.relay_incomplete}/${f.relay_count} done</span>`
                         : `${f.relay_count ?? 0} relays`;
@@ -467,7 +469,9 @@ async function loadCaching() {
                             '</div>';
                     }
                     const refreshBtn = `<button type="button" class="refresh-user-btn" onclick="event.stopPropagation(); refreshCachingUser('${esc(f.pubkey)}', '${esc(f.name || '')}')">↻ Refresh this user</button>`;
-                    return `<tr class="follows-row" onclick="this.nextElementSibling.style.display = this.nextElementSibling.style.display === 'none' ? '' : 'none'"><td><bdi>${esc(f.name) || '<i>unknown</i>'}</bdi></td><td class="npub-link">${esc(npub)}…</td><td>${f.is_root ? '✓' : ''}</td><td>${f.total_events ?? 0}</td><td>${f.backfill_complete ? '✓' : '…'}</td><td>${relaySummary}</td></tr><tr class="follows-detail" style="display:none"><td colspan="6"><div class="follows-detail-content">${relayDetail || '<i>No relay progress data</i>'}<div class="follows-detail-actions">${refreshBtn}</div></div></td></tr>`;
+                    const rowClass = isActive ? 'follows-row follows-row-active' : 'follows-row';
+                    const activeIcon = isActive ? '⚡ ' : '';
+                    return `<tr class="${rowClass}" onclick="this.nextElementSibling.style.display = this.nextElementSibling.style.display === 'none' ? '' : 'none'"><td><bdi>${activeIcon}${esc(f.name) || '<i>unknown</i>'}</bdi></td><td class="npub-link">${esc(npub)}…</td><td>${f.is_root ? '✓' : ''}</td><td>${f.total_events ?? 0}</td><td>${f.backfill_complete ? '✓' : '…'}</td><td>${relaySummary}</td></tr><tr class="follows-detail" style="display:none"><td colspan="6"><div class="follows-detail-content">${relayDetail || '<i>No relay progress data</i>'}<div class="follows-detail-actions">${refreshBtn}</div></div></td></tr>`;
                 }).join('');
             }
         }

admin/assets/index.css

@@ -1602,6 +1602,12 @@ bdi {
 .follows-row:hover {
     background: var(--hover-bg, rgba(255,255,255,0.05));
 }
+.follows-row-active {
+    border-left: 3px solid red;
+}
+.follows-row-active:hover {
+    border-left: 3px solid red;
+}
 .follows-detail td {
     padding: 8px 12px;
     background: var(--bg-color);

caching/Dockerfile.alpine-musl

@@ -25,6 +25,7 @@ RUN apk add --no-cache \
     curl-dev \
     curl-static \
     sqlite-dev \
+    postgresql-dev \
     linux-headers \
     wget \
     bash
@@ -103,13 +104,15 @@ RUN if [ "$DEBUG_BUILD" = "true" ]; then \
     -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 \
     -I. -Isrc -Inostr_core_lib -Inostr_core_lib/nostr_core \
     -Inostr_core_lib/cjson -Inostr_core_lib/nostr_websocket \
+    -I/usr/include/postgresql \
     src/main.c src/debug.c src/jsonc_strip.c src/config.c src/state.c \
     src/follow_graph.c src/relay_sink.c src/live_subscriber.c \
     src/backfill.c src/relay_discovery.c \
+    src/pg_inbox.c src/pg_config.c src/forward_catchup.c \
     -o /build/caching_relay_static \
     nostr_core_lib/libnostr_core.a \
     -lwebsockets -lssl -lcrypto -lsecp256k1 \
-    -lcurl -lz -lpthread -lm -ldl && \
+    -lcurl -lz -lpthread -lm -ldl -lpq -lpgcommon -lpgport && \
     eval "$STRIP_CMD"
 
 # Verify it's truly static

caching/src/follow_graph.c

@@ -7,6 +7,7 @@
 
 #include <string.h>
 #include <stdlib.h>
+#include <ctype.h>
 
 int cr_follow_is_root(const cr_config_t *cfg, const char *hex) {
     if (!cfg->root_hex_ready) return 0;
@@ -16,17 +17,35 @@ int cr_follow_is_root(const cr_config_t *cfg, const char *hex) {
     return 0;
 }
 
+/* Check if a string is a 64-character hex pubkey (no npub prefix). */
+static int is_hex_pubkey(const char *s) {
+    if (!s || strlen(s) != 64) return 0;
+    for (int i = 0; i < 64; i++) {
+        if (!isxdigit((unsigned char)s[i])) return 0;
+    }
+    return 1;
+}
+
 int cr_follow_decode_roots(cr_config_t *cfg) {
     for (int i = 0; i < cfg->root_npub_count; i++) {
-        unsigned char pubkey[32];
-        if (nostr_decode_npub(cfg->root_npubs[i], pubkey) != NOSTR_SUCCESS) {
-            DEBUG_ERROR("follow: failed to decode npub '%s'", cfg->root_npubs[i]);
-            return -1;
+        const char *input = cfg->root_npubs[i];
+        if (is_hex_pubkey(input)) {
+            /* Already a 64-char hex pubkey — copy directly. */
+            strncpy(cfg->root_hex[i], input, 64);
+            cfg->root_hex[i][64] = '\0';
+        } else {
+            /* Assume npub (bech32) format — decode to hex. */
+            unsigned char pubkey[32];
+            if (nostr_decode_npub(input, pubkey) != NOSTR_SUCCESS) {
+                DEBUG_ERROR("follow: failed to decode root pubkey '%s' "
+                            "(not a valid npub or 64-char hex)", input);
+                return -1;
+            }
+            nostr_bytes_to_hex(pubkey, 32, cfg->root_hex[i]);
         }
-        nostr_bytes_to_hex(pubkey, 32, cfg->root_hex[i]);
     }
     cfg->root_hex_ready = 1;
-    DEBUG_INFO("follow: decoded %d root npubs", cfg->root_npub_count);
+    DEBUG_INFO("follow: decoded %d root pubkeys", cfg->root_npub_count);
     return 0;
 }
 

caching/src/forward_catchup.c

@@ -6,6 +6,7 @@
 #include "forward_catchup.h"
 #include "pg_inbox.h"
 #include "config.h"
+#include "follow_graph.h"
 #include "relay_sink.h"
 #include "debug.h"
 

caching/src/relay_discovery.c

@@ -239,8 +239,8 @@ static void compute_covering_set(cr_relay_map_t *map, cr_config_t *cfg,
  * Returns a cJSON array of events (caller must cJSON_Delete each + free).
  * Sets *out_count to the number of events returned. */
 static cJSON **query_kind10002_batch(nostr_relay_pool_t *pool,
-                                     const char **pubkeys, int pubkey_count,
-                                     int *out_count) {
+                                      const char **pubkeys, int pubkey_count,
+                                      int *out_count) {
     cJSON *filter = cJSON_CreateObject();
     cJSON *authors = cJSON_CreateArray();
     for (int i = 0; i < pubkey_count; i++)
@@ -276,6 +276,44 @@ static cJSON **query_kind10002_batch(nostr_relay_pool_t *pool,
     return events;
 }
 
+/* ---- batched kind-3 fallback query ---- */
+
+/* Query kind 3 (contact list) for a batch of pubkeys from a pool.
+ * Kind-3 events historically contain "r" tags with relay URLs (pre-NIP-65).
+ * We use this as a fallback for pubkeys that don't have kind-10002.
+ * Returns a cJSON array of events (caller must cJSON_Delete each + free).
+ * Sets *out_count to the number of events returned. */
+static cJSON **query_kind3_batch(nostr_relay_pool_t *pool,
+                                  const char **pubkeys, int pubkey_count,
+                                  int *out_count) {
+    cJSON *filter = cJSON_CreateObject();
+    cJSON *authors = cJSON_CreateArray();
+    for (int i = 0; i < pubkey_count; i++)
+        cJSON_AddItemToArray(authors, cJSON_CreateString(pubkeys[i]));
+    cJSON_AddItemToObject(filter, "authors", authors);
+    cJSON *kinds = cJSON_CreateArray();
+    cJSON_AddItemToArray(kinds, cJSON_CreateNumber(3));
+    cJSON_AddItemToObject(filter, "kinds", kinds);
+    /* limit=1 per author: we only need the most recent kind-3.
+     * The relay pool query_sync returns the most recent events. */
+    cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(pubkey_count));
+
+    const char **urls = NULL;
+    int n = get_pool_urls(pool, &urls);
+
+    DEBUG_TRACE("query_kind3_batch: querying %d relays for %d pubkeys", n, pubkey_count);
+
+    int ev_count = 0;
+    cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 30000);
+    free(urls);
+    cJSON_Delete(filter);
+
+    DEBUG_TRACE("query_kind3_batch: returned %d events", ev_count);
+
+    *out_count = ev_count;
+    return events;
+}
+
 /* Process a batch of kind-10002 events: parse r-tags, populate outbox entries,
  * publish to local relay. Returns count of events processed. */
 static int process_10002_batch(cJSON **events, int ev_count,
@@ -438,9 +476,61 @@ int cr_relay_discovery_run(cr_relay_map_t *map, cr_config_t *cfg,
         free(remaining_pk);
     }
 
+    /* Phase 3: Kind-3 fallback for pubkeys still without relay lists.
+     * Kind-3 (contact list) events historically contain "r" tags with relay
+     * URLs (pre-NIP-65). This catches older users who never published kind-10002. */
+    int remaining_k3 = 0;
+    for (int i = 0; i < followed->count; i++) if (!found[i]) remaining_k3++;
+
+    if (remaining_k3 > 0) {
+        DEBUG_INFO("relay_discovery: querying bootstrap relays for kind-3 fallback (%d pubkeys)...",
+                    remaining_k3);
+
+        const char **remaining_pk = malloc(remaining_k3 * sizeof(char *));
+        int ridx = 0;
+        for (int i = 0; i < followed->count; i++) {
+            if (!found[i]) remaining_pk[ridx++] = followed->items[i];
+        }
+
+        int found_k3 = 0;
+        for (int start = 0; start < remaining_k3 && !g_shutdown; start += CR_10002_BATCH_SIZE) {
+            int batch = remaining_k3 - start;
+            if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
+
+            int ev_count = 0;
+            cJSON **events = query_kind3_batch(upstream,
+                                                &remaining_pk[start],
+                                                batch, &ev_count);
+            if (events && ev_count > 0) {
+                for (int k = 0; k < ev_count; k++) {
+                    cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
+                    if (pk && cJSON_IsString(pk)) {
+                        const char *hex = cJSON_GetStringValue(pk);
+                        for (int i = 0; i < followed->count; i++) {
+                            if (strcmp(followed->items[i], hex) == 0) {
+                                if (!found[i]) { found[i] = 1; found_k3++; }
+                                break;
+                            }
+                        }
+                    }
+                }
+                /* process_10002_batch works for kind-3 too — same r-tag format */
+                process_10002_batch(events, ev_count, map, sink, "bootstrap");
+                cr_sink_pump(sink, 500);
+            } else {
+                free(events);
+            }
+        }
+        free(remaining_pk);
+        DEBUG_INFO("relay_discovery: kind-3 fallback found %d additional pubkeys with relays", found_k3);
+    }
+
     found_none = followed->count - found_local - found_bootstrap;
+    /* Recalculate found_none after kind-3 fallback */
+    int still_none = 0;
+    for (int i = 0; i < followed->count; i++) if (!found[i]) still_none++;
     DEBUG_INFO("relay_discovery: %d local, %d bootstrap, %d none",
-                found_local, found_bootstrap, found_none);
+                found_local, found_bootstrap, still_none);
 
     free(found);
 

deploy_lt.sh

@@ -47,6 +47,16 @@ else
     echo "WARNING: caching_relay binary not found locally ($LOCAL_CACHING_BINARY) — skipping caching binary upload"
 fi
 
+# Upload admin PHP files (tar to preserve directory structure, exclude
+# cache logs and the dev-only serve.sh / php_server.log).
+echo "==> Packaging admin files"
+tar czf /tmp/admin_deploy.tar.gz \
+    --exclude='admin/serve.sh' \
+    --exclude='admin/php_server.log' \
+    --exclude='admin/cache' \
+    -C . admin/
+scp /tmp/admin_deploy.tar.gz "$REMOTE_HOST:/tmp/admin_deploy.tar.gz"
+
 echo "==> Running remote install/configuration"
 ssh "$REMOTE_HOST" 'bash -s' <<'EOF'
 set -euo pipefail
@@ -102,6 +112,46 @@ else
   echo "  -> no caching_relay binary uploaded, skipping"
 fi
 
+echo "[remote] Installing admin PHP files"
+# Back up existing config.php and .htpasswd so we don't lose production
+# credentials (DB password, basic-auth) across deploys.
+if [ -f /opt/c-relay-pg/admin/lib/config.php ]; then
+  sudo cp /opt/c-relay-pg/admin/lib/config.php /tmp/admin_config_backup
+  echo "  -> backed up config.php"
+fi
+if [ -f /opt/c-relay-pg/admin/.htpasswd ]; then
+  sudo cp /opt/c-relay-pg/admin/.htpasswd /tmp/admin_htpasswd_backup
+  echo "  -> backed up .htpasswd"
+fi
+# Extract the new admin files (overwrites old admin at /opt/c-relay-pg/admin/).
+sudo mkdir -p /opt/c-relay-pg/admin
+sudo tar xzf /tmp/admin_deploy.tar.gz -C /opt/c-relay-pg/
+# Restore config.php and .htpasswd if they were backed up.
+if [ -f /tmp/admin_config_backup ]; then
+  sudo cp /tmp/admin_config_backup /opt/c-relay-pg/admin/lib/config.php
+  echo "  -> restored config.php"
+fi
+if [ -f /tmp/admin_htpasswd_backup ]; then
+  sudo cp /tmp/admin_htpasswd_backup /opt/c-relay-pg/admin/.htpasswd
+  echo "  -> restored .htpasswd"
+fi
+# Set ownership: www-data (nginx/php-fpm) for web-served files,
+# but keep .htpasswd readable by nginx.
+sudo chown -R www-data:www-data /opt/c-relay-pg/admin/
+sudo chmod 640 /opt/c-relay-pg/admin/.htpasswd 2>/dev/null || true
+# Ensure the cache directory exists (for chart text files).
+sudo mkdir -p /opt/c-relay-pg/admin/cache
+sudo chown www-data:www-data /opt/c-relay-pg/admin/cache
+echo "  -> admin files installed at /opt/c-relay-pg/admin/"
+
+# Also update admin2 if it exists (symlink or copy to keep it in sync).
+if [ -d /opt/c-relay-pg/admin2 ]; then
+  sudo rm -rf /opt/c-relay-pg/admin2
+  sudo cp -a /opt/c-relay-pg/admin /opt/c-relay-pg/admin2
+  sudo chown -R www-data:www-data /opt/c-relay-pg/admin2/
+  echo "  -> admin2 synced at /opt/c-relay-pg/admin2/"
+fi
+
 echo "[remote] Installing systemd unit"
 sudo mv /tmp/c-relay-pg.service /etc/systemd/system/c-relay-pg.service
 sudo chown root:root /etc/systemd/system/c-relay-pg.service
@@ -130,6 +180,24 @@ sudo systemctl restart "$SERVICE_NAME"
 echo "[remote] Health checks"
 sudo systemctl --no-pager --full status "$SERVICE_NAME" | sed -n '1,25p'
 sudo -u "$RELAY_USER" psql -d crelay -c "SELECT current_user, current_database();"
+
+# Verify admin page is accessible (HTTP 200 or 401 = auth required, both are OK).
+ADMIN_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1/admin/" 2>/dev/null || echo "000")
+if [ "$ADMIN_CODE" = "200" ] || [ "$ADMIN_CODE" = "401" ]; then
+  echo "  -> admin page accessible (HTTP $ADMIN_CODE)"
+else
+  echo "  -> WARNING: admin page returned HTTP $ADMIN_CODE (expected 200 or 401)"
+fi
+
+# Clean up temp files.
+rm -f /tmp/admin_deploy.tar.gz
+sudo rm -f /tmp/admin_htpasswd_backup /tmp/admin_config_backup
 EOF
 
+# Clean up local temp tarball.
+rm -f /tmp/admin_deploy.tar.gz
+
 echo "Deployment complete: $REMOTE_HOST"
+echo ""
+echo "Admin interface: https://laantungir.net/admin/"
+echo "Caching is NOT auto-started. Enable it via the admin UI when ready."

relay.pid

@@ -1 +1 @@
-2276497
+2356597

src/main.c

@@ -2815,6 +2815,15 @@ int main(int argc, char* argv[]) {
         }
         DEBUG_INFO("Synchronized relay_version to %s", CRELAY_VERSION);
 
+        // Insert any config keys that were added in newer versions but are
+        // missing from this existing database (e.g. caching_* keys added after
+        // the relay was first deployed). This only inserts MISSING keys —
+        // existing values are preserved.
+        int added = populate_default_config_values();
+        if (added > 0) {
+            DEBUG_INFO("Inserted %d missing default config keys", added);
+        }
+
         // Apply CLI overrides atomically (now that database is initialized)
         if (apply_cli_overrides_atomic(&cli_options) != 0) {
             DEBUG_ERROR("Failed to apply CLI overrides for existing relay");

src/main.h

@@ -13,8 +13,8 @@
 // Using CRELAY_ prefix to avoid conflicts with nostr_core_lib VERSION macros
 #define CRELAY_VERSION_MAJOR 2
 #define CRELAY_VERSION_MINOR 1
-#define CRELAY_VERSION_PATCH 24
-#define CRELAY_VERSION "v2.1.24"
+#define CRELAY_VERSION_PATCH 28
+#define CRELAY_VERSION "v2.1.28"
 
 // Relay metadata (authoritative source for NIP-11 information)
 #define RELAY_NAME "C-Relay-PG"