Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Laan Tungir
|
81c5339063 | ||
|
Laan Tungir
|
5ec5ed9bd4 | ||
|
Laan Tungir
|
bd5150cd09 | ||
|
Laan Tungir
|
2e1d731de4 | ||
|
Laan Tungir
|
75d983b2df | ||
|
Laan Tungir
|
a766f08738 | ||
|
Laan Tungir
|
23c37abe3a | ||
|
Laan Tungir
|
5b3ccfd1f4 | ||
|
Laan Tungir
|
38a027e626 | ||
|
Laan Tungir
|
a042aca524 | ||
|
Laan Tungir
|
20902e1c5d | ||
|
Laan Tungir
|
7233818da9 | ||
|
Laan Tungir
|
86cbf7ff16 | ||
|
Laan Tungir
|
3035e71ca5 | ||
|
Laan Tungir
|
179b160cf3 | ||
|
Laan Tungir
|
aedd22436e | ||
|
Laan Tungir
|
8fdc362a8b | ||
|
Laan Tungir
|
67cf3973c5 | ||
|
Laan Tungir
|
c02ae2be5b | ||
|
Laan Tungir
|
fd3efdd01f |
+1
-1
@@ -11,4 +11,4 @@ copy_executable_local.sh
|
||||
nostr_login_lite/
|
||||
style_guide/
|
||||
nostr-tools
|
||||
.test_keys
|
||||
.test_keysadmin/cache/
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
[submodule "nostr_core_lib"]
|
||||
path = nostr_core_lib
|
||||
url = https://git.laantungir.net/laantungir/nostr_core_lib.git
|
||||
[submodule "c_utils_lib"]
|
||||
path = c_utils_lib
|
||||
url = ssh://git@git.laantungir.net:2222/laantungir/c_utils_lib.git
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
ADMIN_PUBKEY='6a04ab98d9e4774ad806e302dddeb63bea16b5cb5f223ee77478e861bb583eb3'
|
||||
SERVER_PRIVKEY='1111111111111111111111111111111111111111111111111111111111111111'
|
||||
@@ -0,0 +1,2 @@
|
||||
ADMIN_PUBKEY='8ff74724ed641b3c28e5a86d7c5cbc49c37638ace8c6c38935860e7a5eedde0e'
|
||||
SERVER_PRIVKEY='1111111111111111111111111111111111111111111111111111111111111111'
|
||||
@@ -91,12 +91,13 @@ RUN cd c_utils_lib && \
|
||||
|
||||
# Build nostr_core_lib with required NIPs (cached unless nostr_core_lib changes)
|
||||
# Disable fortification in build.sh to prevent __*_chk symbol issues
|
||||
# NIPs: 001(Basic), 006(Keys), 013(PoW), 017(DMs), 019(Bech32), 044(Encryption), 059(Gift Wrap - required by NIP-17)
|
||||
# NIPs: 001(Basic), 004(Encryption), 006(Keys), 013(PoW), 017(DMs), 019(Bech32), 044(Encryption), 059(Gift Wrap - required by NIP-17)
|
||||
# NIP-04 is required by nostr_signer.c (NIP-46 bunker signer uses NIP-04 encryption)
|
||||
RUN cd nostr_core_lib && \
|
||||
chmod +x build.sh && \
|
||||
sed -i 's/CFLAGS="-Wall -Wextra -std=c99 -fPIC -O2"/CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 -Wall -Wextra -std=c99 -fPIC -O2"/' build.sh && \
|
||||
rm -f *.o *.a 2>/dev/null || true && \
|
||||
./build.sh --nips=1,6,13,17,19,44,59
|
||||
./build.sh --nips=1,4,6,13,17,19,44,59
|
||||
|
||||
# Copy c-relay-pg source files LAST (only this layer rebuilds on source changes)
|
||||
COPY src/ /build/src/
|
||||
@@ -130,6 +131,7 @@ RUN if [ "$DEBUG_BUILD" = "true" ]; then \
|
||||
src/main.c src/config.c src/dm_admin.c src/request_validator.c \
|
||||
src/nip009.c src/nip011.c src/nip013.c src/nip040.c src/nip042.c \
|
||||
src/websockets.c src/subscriptions.c src/api.c src/embedded_web_content.c src/ip_ban.c \
|
||||
src/caching_inbox_poller.c src/caching_service_launcher.c \
|
||||
src/db_ops.c src/db_ops_sqlite.c src/db_ops_postgres.c src/thread_pool.c \
|
||||
-o /build/c_relay_pg_static \
|
||||
c_utils_lib/libc_utils.a \
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
CC = gcc
|
||||
CFLAGS = -Wall -Wextra -std=c99 -g -O2
|
||||
INCLUDES = -I. -Ic_utils_lib/src -Inostr_core_lib -Inostr_core_lib/nostr_core -Inostr_core_lib/cjson -Inostr_core_lib/nostr_websocket
|
||||
INCLUDES = -I. -Ic_utils_lib/src -Inostr_core_lib -Inostr_core_lib/nostr_core -Inostr_core_lib/cjson -Inostr_core_lib/nostr_websocket -I/usr/include/postgresql
|
||||
LIBS = -lsqlite3 -lwebsockets -lz -ldl -lpthread -lm -L/usr/local/lib -lsecp256k1 -lssl -lcrypto -L/usr/local/lib -lcurl -Lc_utils_lib -lc_utils
|
||||
|
||||
DB_BACKEND ?= sqlite
|
||||
@@ -11,7 +11,7 @@ DB_BACKEND ?= sqlite
|
||||
BUILD_DIR = build
|
||||
|
||||
# Source files
|
||||
MAIN_SRC = src/main.c src/config.c src/dm_admin.c src/request_validator.c src/nip009.c src/nip011.c src/nip013.c src/nip040.c src/nip042.c src/websockets.c src/subscriptions.c src/api.c src/embedded_web_content.c src/ip_ban.c src/thread_pool.c
|
||||
MAIN_SRC = src/main.c src/config.c src/dm_admin.c src/request_validator.c src/nip009.c src/nip011.c src/nip013.c src/nip040.c src/nip042.c src/websockets.c src/subscriptions.c src/api.c src/embedded_web_content.c src/ip_ban.c src/thread_pool.c src/caching_inbox_poller.c src/caching_service_launcher.c
|
||||
DB_OPS_SRC = src/db_ops.c
|
||||
|
||||
ifeq ($(DB_BACKEND),postgres)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
/** admin2/api/auth.php — Auth rules + WoT status. */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$input = json_decode(file_get_contents('php://input'), true);
|
||||
$action = $input['action'] ?? '';
|
||||
try {
|
||||
if ($action === 'add') {
|
||||
$pdo->prepare("INSERT INTO auth_rules (rule_type, pattern_type, pattern_value) VALUES (?, 'pubkey', ?)")->execute([$input['rule_type'], $input['pattern_value']]);
|
||||
json_response(['message' => 'Rule added']);
|
||||
} elseif ($action === 'remove') {
|
||||
$pdo->prepare("DELETE FROM auth_rules WHERE id = ?")->execute([intval($input['id'])]);
|
||||
json_response(['message' => 'Rule removed']);
|
||||
} elseif ($action === 'set_wot_level') {
|
||||
$pdo->prepare("UPDATE config SET value = ? WHERE key = 'wot_level'")->execute([strval($input['level'])]);
|
||||
json_response(['message' => 'WoT level set']);
|
||||
} elseif ($action === 'sync_wot') {
|
||||
json_response(['message' => 'WoT sync not available via PHP (requires relay)']);
|
||||
}
|
||||
} catch (PDOException $e) { json_response(['error' => $e->getMessage()]); }
|
||||
}
|
||||
|
||||
// GET: WoT status
|
||||
if (isset($_GET['action']) && $_GET['action'] === 'wot') {
|
||||
$level = $pdo->query("SELECT value FROM config WHERE key = 'wot_level'")->fetchColumn() ?: '0';
|
||||
$wl_count = 0;
|
||||
try { $wl_count = intval($pdo->query("SELECT count(*) FROM auth_rules WHERE rule_type = 'whitelist'")->fetchColumn()); } catch (PDOException $e) {}
|
||||
$descriptions = ['0' => 'Level 0: Open relay — anyone can read and write', '1' => 'Level 1: Write only — WoT users can write', '2' => 'Level 2: Full — WoT users only'];
|
||||
json_response(['kind3_status' => 'N/A (PHP admin)', 'whitelist_count' => $wl_count, 'level_description' => $descriptions[$level] ?? $descriptions['0']]);
|
||||
}
|
||||
|
||||
// GET: auth rules
|
||||
$rules = [];
|
||||
try { $rules = $pdo->query("SELECT id, rule_type, pattern_type, pattern_value FROM auth_rules ORDER BY id")->fetchAll(); } catch (PDOException $e) {}
|
||||
json_response(['rules' => $rules]);
|
||||
@@ -0,0 +1,123 @@
|
||||
<?php
|
||||
/** admin2/api/caching.php — Caching service status + follows + reset actions. */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
// --- POST actions: reset backfill (all or per-user) ---
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$input = json_decode(file_get_contents('php://input'), true) ?: [];
|
||||
$action = $input['action'] ?? '';
|
||||
$pubkey = $input['pubkey'] ?? '';
|
||||
|
||||
if ($action === 'reset_all') {
|
||||
// Reset backfill progress for ALL followed authors, then bump config
|
||||
// generation so the running caching service hot-reloads and re-drains.
|
||||
try {
|
||||
$pdo->beginTransaction();
|
||||
$pdo->exec("UPDATE caching_followed_pubkeys
|
||||
SET until_cursor = 0, backfill_complete = FALSE,
|
||||
events_fetched = 0,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW())::BIGINT");
|
||||
// Also reset per-relay progress rows
|
||||
$pdo->exec("UPDATE caching_backfill_relay_progress
|
||||
SET complete = FALSE, events_fetched = 0,
|
||||
until_cursor = 0, last_status = 'reset',
|
||||
consecutive_errors = 0,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW())::BIGINT");
|
||||
// Bump config generation to trigger hot-reload
|
||||
$pdo->exec("UPDATE config SET value = (COALESCE(value::int, 0) + 1)::text
|
||||
WHERE key = 'caching_config_generation'");
|
||||
$pdo->commit();
|
||||
json_response(['ok' => true, 'message' => 'All backfill progress reset. Caching service will re-drain.']);
|
||||
} catch (PDOException $e) {
|
||||
$pdo->rollBack();
|
||||
json_response(['ok' => false, 'error' => $e->getMessage()], 500);
|
||||
}
|
||||
} elseif ($action === 'reset_user' && $pubkey) {
|
||||
// Reset backfill progress for a single followed author.
|
||||
try {
|
||||
$pdo->beginTransaction();
|
||||
$stmt = $pdo->prepare("UPDATE caching_followed_pubkeys
|
||||
SET until_cursor = 0, backfill_complete = FALSE,
|
||||
events_fetched = 0,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
WHERE pubkey = ?");
|
||||
$stmt->execute([$pubkey]);
|
||||
// Also reset per-relay progress for this author
|
||||
$stmt2 = $pdo->prepare("UPDATE caching_backfill_relay_progress
|
||||
SET complete = FALSE, events_fetched = 0,
|
||||
until_cursor = 0, last_status = 'reset',
|
||||
consecutive_errors = 0,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
WHERE author_pubkey = ?");
|
||||
$stmt2->execute([$pubkey]);
|
||||
// Bump config generation to trigger hot-reload
|
||||
$pdo->exec("UPDATE config SET value = (COALESCE(value::int, 0) + 1)::text
|
||||
WHERE key = 'caching_config_generation'");
|
||||
$pdo->commit();
|
||||
json_response(['ok' => true, 'message' => "Backfill reset for pubkey. Caching service will re-fetch."]);
|
||||
} catch (PDOException $e) {
|
||||
$pdo->rollBack();
|
||||
json_response(['ok' => false, 'error' => $e->getMessage()], 500);
|
||||
}
|
||||
} else {
|
||||
json_response(['ok' => false, 'error' => 'Unknown action'], 400);
|
||||
}
|
||||
exit;
|
||||
}
|
||||
|
||||
// Caching config values (for toggle controls)
|
||||
$config = [];
|
||||
try {
|
||||
$cfg_rows = $pdo->query("SELECT key, value FROM config WHERE key IN ('caching_enabled','caching_inbox_enabled','caching_live_enabled','caching_backfill_enabled')")->fetchAll();
|
||||
foreach ($cfg_rows as $r) { $config[$r['key']] = $r['value']; }
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Service state
|
||||
$state = [];
|
||||
try { $state = $pdo->query("SELECT * FROM caching_service_state WHERE id = 1")->fetch() ?: []; } catch (PDOException $e) {}
|
||||
|
||||
// Active target
|
||||
$active = ['pubkey' => '', 'relay' => ''];
|
||||
try { $active = $pdo->query("SELECT active_pubkey AS pubkey, active_relay AS relay FROM caching_backfill_active WHERE id = 1")->fetch() ?: $active; } catch (PDOException $e) {}
|
||||
|
||||
// Inbox
|
||||
$inbox = ['pending' => 0, 'live' => 0, 'backfill' => 0];
|
||||
try {
|
||||
$inbox = $pdo->query("SELECT COUNT(*) AS pending, COUNT(*) FILTER (WHERE source_class='live') AS live, COUNT(*) FILTER (WHERE source_class='backfill') AS backfill FROM caching_event_inbox")->fetch() ?: $inbox;
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Follows (paginated, with names from profiles cache + per-relay progress)
|
||||
$follows = [];
|
||||
try {
|
||||
$follows = $pdo->query("
|
||||
SELECT fp.pubkey, fp.is_root, fp.backfill_complete, fp.events_fetched,
|
||||
fp.last_event_at,
|
||||
p.name, p.display_name,
|
||||
(SELECT count(*) FROM events WHERE pubkey = fp.pubkey) AS total_events,
|
||||
(SELECT count(*) FROM caching_backfill_relay_progress WHERE author_pubkey = fp.pubkey) AS relay_count,
|
||||
(SELECT count(*) FROM caching_backfill_relay_progress WHERE author_pubkey = fp.pubkey AND complete = false) AS relay_incomplete
|
||||
FROM caching_followed_pubkeys fp
|
||||
LEFT JOIN profiles p ON p.pubkey = fp.pubkey
|
||||
ORDER BY fp.is_root DESC, fp.events_fetched DESC LIMIT 1000
|
||||
")->fetchAll();
|
||||
// Fetch per-relay progress for all followed pubkeys in one query
|
||||
$relayProgress = [];
|
||||
try {
|
||||
$rpRows = $pdo->query("
|
||||
SELECT author_pubkey, relay_url, complete, events_fetched, until_cursor, last_status, updated_at
|
||||
FROM caching_backfill_relay_progress
|
||||
ORDER BY author_pubkey, events_fetched DESC
|
||||
")->fetchAll();
|
||||
foreach ($rpRows as $rp) {
|
||||
$relayProgress[$rp['author_pubkey']][] = $rp;
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
foreach ($follows as &$f) {
|
||||
$f['name'] = profile_display_name($f);
|
||||
$f['npub'] = function_exists('hex_to_npub') ? hex_to_npub($f['pubkey']) : substr($f['pubkey'], 0, 20);
|
||||
$f['relays'] = $relayProgress[$f['pubkey']] ?? [];
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
json_response(['state' => $state, 'active' => $active, 'inbox' => $inbox, 'follows' => $follows, 'config' => $config]);
|
||||
@@ -0,0 +1,120 @@
|
||||
<?php
|
||||
/**
|
||||
* admin2/api/chart.php — Standalone ASCII chart endpoint.
|
||||
*
|
||||
* Returns a plain-text ASCII X-bar chart of event counts over time.
|
||||
* Works in both the browser (injected into a <div>) and the terminal:
|
||||
*
|
||||
* curl http://localhost:8088/api/chart.php?range=hour
|
||||
* curl http://localhost:8088/api/chart.php?range=day
|
||||
* curl http://localhost:8088/api/chart.php?range=month
|
||||
* curl http://localhost:8088/api/chart.php?range=year
|
||||
*
|
||||
* Caching: the hour chart is never cached (live). Day/month/year are
|
||||
* cached to file with TTLs to avoid expensive queries on every request.
|
||||
*/
|
||||
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
require_once __DIR__ . '/../lib/ascii_chart.php';
|
||||
|
||||
// --- Range configuration ---
|
||||
$ranges = [
|
||||
'hour' => ['span' => 3600, 'bin' => 45, 'bins' => 80, 'ttl' => 0, 'title' => 'New Events — Last Hour'],
|
||||
'day' => ['span' => 86400, 'bin' => 1080, 'bins' => 80, 'ttl' => 3600, 'title' => 'New Events — Last 24 Hours'],
|
||||
'month' => ['span' => 2592000, 'bin' => 32400, 'bins' => 80, 'ttl' => 86400, 'title' => 'New Events — Last 30 Days'],
|
||||
'year' => ['span' => 31536000, 'bin' => 394200, 'bins' => 80, 'ttl' => 2592000, 'title' => 'New Events — Last Year'],
|
||||
];
|
||||
|
||||
$range = $_GET['range'] ?? 'hour';
|
||||
if (!isset($ranges[$range])) {
|
||||
http_response_code(400);
|
||||
header('Content-Type: text/plain; charset=utf-8');
|
||||
echo "Invalid range. Use: hour, day, month, or year.\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
$cfg = $ranges[$range];
|
||||
|
||||
// --- Set content type ---
|
||||
header('Content-Type: text/plain; charset=utf-8');
|
||||
|
||||
// --- Check cache ---
|
||||
if ($cfg['ttl'] > 0) {
|
||||
$cached = get_cached_chart($range, $cfg['ttl']);
|
||||
if ($cached !== null) {
|
||||
echo $cached;
|
||||
exit;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Run binning query ---
|
||||
$pdo = db();
|
||||
$now = time();
|
||||
$epoch = $now - $cfg['span'];
|
||||
$bin_size = $cfg['bin'];
|
||||
$num_bins = $cfg['bins'];
|
||||
|
||||
try {
|
||||
// Calculate the bin offset so the rightmost bin aligns with "now"
|
||||
// bin index 0 = oldest, bin index (num_bins-1) = newest
|
||||
$base_bin = (int)floor($epoch / $bin_size);
|
||||
|
||||
$sql = "SELECT FLOOR(created_at / {$bin_size})::BIGINT - {$base_bin} AS bin, COUNT(*) AS cnt
|
||||
FROM events
|
||||
WHERE created_at >= {$epoch}
|
||||
GROUP BY bin
|
||||
ORDER BY bin";
|
||||
$rows = $pdo->query($sql)->fetchAll();
|
||||
} catch (PDOException $e) {
|
||||
echo "Chart query failed: " . $e->getMessage() . "\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
// --- Build bin array (zero-filled, fixed length) ---
|
||||
// Index 0 = oldest, index N-1 = newest (natural time order from SQL)
|
||||
$bins = build_bin_array($rows, $num_bins);
|
||||
|
||||
// Reverse so newest bin is at index 0 (leftmost) and oldest at right.
|
||||
// This matches the original text_graph.js display: new data enters
|
||||
// from the left and proceeds rightward as it ages.
|
||||
$bins = array_reverse($bins);
|
||||
|
||||
// --- Render ASCII chart ---
|
||||
$ascii = render_ascii_chart($bins, [
|
||||
'title' => $cfg['title'],
|
||||
'max_height' => 11,
|
||||
'bin_duration' => $bin_size,
|
||||
'label_interval' => max(1, (int)floor($num_bins / 15)), // ~15 labels across
|
||||
]);
|
||||
|
||||
// --- Write to cache (if TTL > 0) ---
|
||||
if ($cfg['ttl'] > 0) {
|
||||
set_cached_chart($range, $ascii);
|
||||
}
|
||||
|
||||
echo $ascii;
|
||||
|
||||
|
||||
// ================================
|
||||
// CACHE HELPERS
|
||||
// ================================
|
||||
|
||||
function get_cache_dir(): string {
|
||||
return __DIR__ . '/../cache';
|
||||
}
|
||||
|
||||
function get_cached_chart(string $range, int $ttl): ?string {
|
||||
$file = get_cache_dir() . "/chart_{$range}.txt";
|
||||
if (!file_exists($file)) return null;
|
||||
if (filemtime($file) < (time() - $ttl)) return null;
|
||||
$content = @file_get_contents($file);
|
||||
return ($content !== false) ? $content : null;
|
||||
}
|
||||
|
||||
function set_cached_chart(string $range, string $ascii): void {
|
||||
$dir = get_cache_dir();
|
||||
if (!is_dir($dir)) {
|
||||
@mkdir($dir, 0775, true);
|
||||
}
|
||||
@file_put_contents($dir . "/chart_{$range}.txt", $ascii);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
/** admin2/api/config.php — Config table read/edit (all keys). */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$input = json_decode(file_get_contents('php://input'), true);
|
||||
$key = $input['key'] ?? '';
|
||||
$value = $input['value'] ?? '';
|
||||
if (!$key) json_response(['error' => 'Missing key']);
|
||||
try {
|
||||
$pdo->prepare("UPDATE config SET value = ? WHERE key = ?")->execute([$value, $key]);
|
||||
json_response(['message' => "Updated $key"]);
|
||||
} catch (PDOException $e) {
|
||||
json_response(['error' => $e->getMessage()]);
|
||||
}
|
||||
}
|
||||
|
||||
$rows = $pdo->query("SELECT key, value FROM config ORDER BY key")->fetchAll();
|
||||
json_response(['config' => $rows]);
|
||||
@@ -0,0 +1,16 @@
|
||||
<?php
|
||||
/** admin2/api/dm.php — Direct messages (kind 4/14/15). */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
$limit = min(200, intval($_GET['limit'] ?? 50));
|
||||
$rows = $pdo->prepare("SELECT id, pubkey, kind, created_at, content FROM events WHERE kind IN (4, 14, 15) ORDER BY created_at DESC LIMIT $limit");
|
||||
$rows->execute();
|
||||
$messages = $rows->fetchAll();
|
||||
|
||||
foreach ($messages as &$m) {
|
||||
$m['created_at'] = date('Y-m-d H:i:s', intval($m['created_at']));
|
||||
$m['content'] = substr($m['content'] ?? '', 0, 300);
|
||||
}
|
||||
|
||||
json_response(['messages' => $messages]);
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
/** admin2/api/events.php — Recent relay events (paginated). */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
$limit = min(200, intval($_GET['limit'] ?? 50));
|
||||
$offset = intval($_GET['offset'] ?? 0);
|
||||
$kind = $_GET['kind'] ?? '';
|
||||
|
||||
$where = [];
|
||||
$params = [];
|
||||
if ($kind !== '') { $where[] = 'kind = ?'; $params[] = intval($kind); }
|
||||
$where_sql = $where ? 'WHERE ' . implode(' AND ', $where) : '';
|
||||
|
||||
$rows = $pdo->prepare("SELECT e.id, e.pubkey, e.kind, e.created_at, e.content,
|
||||
p.name, p.display_name
|
||||
FROM events e
|
||||
LEFT JOIN profiles p ON p.pubkey = e.pubkey
|
||||
$where_sql
|
||||
ORDER BY e.created_at DESC LIMIT $limit OFFSET $offset");
|
||||
$rows->execute($params);
|
||||
$events = $rows->fetchAll();
|
||||
|
||||
foreach ($events as &$e) {
|
||||
$e['created_at'] = date('Y-m-d H:i:s', intval($e['created_at']));
|
||||
$e['content'] = substr($e['content'] ?? '', 0, 200);
|
||||
// Resolve display name from profiles cache; fall back to truncated pubkey.
|
||||
$best = profile_display_name($e);
|
||||
$e['display_name'] = $best !== '' ? $best : substr($e['pubkey'] ?? '', 0, 16) . '…';
|
||||
}
|
||||
|
||||
json_response(['events' => $events]);
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
/** admin2/api/ipbans.php — IP ban management. */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$input = json_decode(file_get_contents('php://input'), true);
|
||||
$action = $input['action'] ?? '';
|
||||
try {
|
||||
if ($action === 'add') {
|
||||
$until = time() + intval($input['duration'] ?? 86400);
|
||||
$pdo->prepare("INSERT INTO ip_bans (ip, banned_until, ban_count) VALUES (?, ?, 1) ON CONFLICT (ip) DO UPDATE SET banned_until = ?, ban_count = ip_bans.ban_count + 1")->execute([$input['ip'], $until, $until]);
|
||||
json_response(['message' => 'IP banned']);
|
||||
} elseif ($action === 'remove') {
|
||||
$pdo->prepare("DELETE FROM ip_bans WHERE ip = ?")->execute([$input['ip']]);
|
||||
json_response(['message' => 'Ban removed']);
|
||||
}
|
||||
} catch (PDOException $e) { json_response(['error' => $e->getMessage()]); }
|
||||
}
|
||||
|
||||
$now = time();
|
||||
$total = 0; $active = 0; $issued = 0;
|
||||
$bans = [];
|
||||
try {
|
||||
$total = intval($pdo->query("SELECT count(*) FROM ip_bans")->fetchColumn());
|
||||
$active = intval($pdo->query("SELECT count(*) FROM ip_bans WHERE banned_until > $now")->fetchColumn());
|
||||
$issued = intval($pdo->query("SELECT COALESCE(sum(ban_count),0) FROM ip_bans")->fetchColumn());
|
||||
$bans = $pdo->query("SELECT ip, banned_until, ban_count, failures, authed_successfully, connection_attempts FROM ip_bans ORDER BY banned_until DESC LIMIT 100")->fetchAll();
|
||||
foreach ($bans as &$b) {
|
||||
$b['status'] = intval($b['banned_until']) > $now ? 'banned' : 'expired';
|
||||
$b['banned_until'] = date('Y-m-d H:i:s', intval($b['banned_until']));
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
json_response(['total' => $total, 'active' => $active, 'issued' => $issued, 'bans' => $bans]);
|
||||
@@ -0,0 +1,38 @@
|
||||
<?php
|
||||
/**
|
||||
* admin/api/profile.php — Single-profile lookup from the profiles cache.
|
||||
* Returns cached kind-0 metadata for a given pubkey.
|
||||
*
|
||||
* Usage: profile.php?pubkey=<64-char-hex>
|
||||
* Returns: { "pubkey": "...", "name": "...", "display_name": "...",
|
||||
* "best_name": "...", "picture": "...", "nip05": "...",
|
||||
* "about": "...", "website": "...", "lud16": "..." }
|
||||
* or { "error": "not found" } with HTTP 404 if no profile is cached.
|
||||
*/
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
|
||||
$pubkey = trim($_GET['pubkey'] ?? '');
|
||||
if (!preg_match('/^[0-9a-fA-F]{64}$/', $pubkey)) {
|
||||
http_response_code(400);
|
||||
json_response(['error' => 'Invalid pubkey']);
|
||||
}
|
||||
|
||||
$pdo = db();
|
||||
try {
|
||||
$stmt = $pdo->prepare(
|
||||
"SELECT pubkey, name, display_name, about, picture, banner,
|
||||
nip05, website, lud16, lud06
|
||||
FROM profiles WHERE pubkey = ?"
|
||||
);
|
||||
$stmt->execute([$pubkey]);
|
||||
$row = $stmt->fetch();
|
||||
if (!$row) {
|
||||
http_response_code(404);
|
||||
json_response(['error' => 'not found']);
|
||||
}
|
||||
$row['best_name'] = profile_display_name($row);
|
||||
json_response($row);
|
||||
} catch (PDOException $e) {
|
||||
http_response_code(500);
|
||||
json_response(['error' => 'Database error']);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<?php
|
||||
/** admin2/api/query.php — Direct SQL query (SELECT only, admin only). */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
json_response(['error' => 'POST only']);
|
||||
}
|
||||
|
||||
$input = json_decode(file_get_contents('php://input'), true);
|
||||
$sql = trim($input['sql'] ?? '');
|
||||
|
||||
if (!$sql) json_response(['error' => 'No query provided']);
|
||||
if (!preg_match('/^\s*SELECT/i', $sql)) json_response(['error' => 'Only SELECT queries are allowed']);
|
||||
|
||||
$start = microtime(true);
|
||||
try {
|
||||
$stmt = $pdo->query($sql);
|
||||
$rows = $stmt->fetchAll();
|
||||
$time_ms = round((microtime(true) - $start) * 1000, 1);
|
||||
json_response(['rows' => $rows, 'row_count' => count($rows), 'time_ms' => $time_ms]);
|
||||
} catch (PDOException $e) {
|
||||
json_response(['error' => $e->getMessage()]);
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
<?php
|
||||
/**
|
||||
* admin2/api/stats.php — Statistics page data.
|
||||
* Returns DB size, event counts, kind distribution, top pubkeys,
|
||||
* time-based stats, and event-rate delta (for the chart).
|
||||
*/
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
|
||||
$pdo = db();
|
||||
|
||||
// Database size
|
||||
$db_size = '-';
|
||||
try {
|
||||
$bytes = $pdo->query("SELECT pg_database_size(current_database())")->fetchColumn();
|
||||
$db_size = format_bytes(intval($bytes));
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Total events
|
||||
$total_events = intval($pdo->query("SELECT COUNT(*) FROM events")->fetchColumn());
|
||||
|
||||
// Event rate: events with first_seen in last 10 seconds
|
||||
$events_delta = 0;
|
||||
try {
|
||||
$events_delta = intval($pdo->query("SELECT COUNT(*) FROM events WHERE first_seen >= EXTRACT(EPOCH FROM NOW())::BIGINT - 10")->fetchColumn());
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Process info (from pg_stat_activity)
|
||||
$process_id = '-';
|
||||
$ws_connections = '-';
|
||||
try {
|
||||
$pid = $pdo->query("SELECT pg_backend_pid()")->fetchColumn();
|
||||
$process_id = strval($pid);
|
||||
$ws_connections = intval($pdo->query("SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND pid != pg_backend_pid()")->fetchColumn());
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Active subscriptions
|
||||
$active_subscriptions = 0;
|
||||
try {
|
||||
$active_subscriptions = intval($pdo->query("SELECT count(*) FROM subscriptions WHERE active = true")->fetchColumn());
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Memory/CPU (from /proc on Linux)
|
||||
$memory_usage = '-';
|
||||
$cpu_usage = '-';
|
||||
$cpu_core = '-';
|
||||
if (is_readable('/proc/meminfo')) {
|
||||
$mem = parse_ini_file('/proc/meminfo');
|
||||
$total = intval($mem['MemTotal'] ?? 0) * 1024;
|
||||
$avail = intval($mem['MemAvailable'] ?? 0) * 1024;
|
||||
if ($total > 0) $memory_usage = format_bytes($total - $avail) . ' / ' . format_bytes($total);
|
||||
}
|
||||
if (is_readable('/proc/cpuinfo')) {
|
||||
$cores = intval(shell_exec('nproc 2>/dev/null') ?: 1);
|
||||
$cpu_core = strval($cores) . ' cores';
|
||||
}
|
||||
$cpu_usage = @file_get_contents('/proc/loadavg');
|
||||
if ($cpu_usage !== false) $cpu_usage = trim(explode(' ', $cpu_usage)[0] ?? '-');
|
||||
|
||||
// Oldest / newest event
|
||||
$oldest_event = '-';
|
||||
$newest_event = '-';
|
||||
try {
|
||||
$oldest = $pdo->query("SELECT to_timestamp(MIN(created_at)) FROM events")->fetchColumn();
|
||||
$newest = $pdo->query("SELECT to_timestamp(MAX(created_at)) FROM events")->fetchColumn();
|
||||
if ($oldest) $oldest_event = substr($oldest, 0, 19);
|
||||
if ($newest) $newest_event = substr($newest, 0, 19);
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Time-based stats
|
||||
$now = time();
|
||||
$events_24h = 0; $events_7d = 0; $events_30d = 0;
|
||||
try {
|
||||
$events_24h = intval($pdo->query("SELECT COUNT(*) FROM events WHERE created_at >= $now - 86400")->fetchColumn());
|
||||
$events_7d = intval($pdo->query("SELECT COUNT(*) FROM events WHERE created_at >= $now - 604800")->fetchColumn());
|
||||
$events_30d = intval($pdo->query("SELECT COUNT(*) FROM events WHERE created_at >= $now - 2592000")->fetchColumn());
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Kind distribution
|
||||
$kinds = [];
|
||||
try {
|
||||
$rows = $pdo->query("SELECT kind, COUNT(*) AS cnt FROM events GROUP BY kind ORDER BY cnt DESC LIMIT 20")->fetchAll();
|
||||
foreach ($rows as $r) {
|
||||
$kinds[] = ['kind' => intval($r['kind']), 'count' => intval($r['cnt']), 'pct' => $total_events > 0 ? round(intval($r['cnt']) / $total_events * 100, 1) : 0];
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Top pubkeys (with names from profiles cache)
|
||||
$top_pubkeys = [];
|
||||
try {
|
||||
$rows = $pdo->query("
|
||||
SELECT pubkey, COUNT(*) AS cnt
|
||||
FROM events
|
||||
GROUP BY pubkey
|
||||
ORDER BY cnt DESC LIMIT 20
|
||||
")->fetchAll();
|
||||
// Batch-resolve profile names from the profiles cache table.
|
||||
$pubkeys = array_column($rows, 'pubkey');
|
||||
$pmap = profile_map($pubkeys);
|
||||
foreach ($rows as $r) {
|
||||
$pk = $r['pubkey'];
|
||||
$prof = $pmap[$pk] ?? null;
|
||||
$top_pubkeys[] = [
|
||||
'pubkey' => $pk,
|
||||
'name' => $prof ? $prof['best_name'] : '',
|
||||
'count' => intval($r['cnt']),
|
||||
'pct' => $total_events > 0 ? round(intval($r['cnt']) / $total_events * 100, 1) : 0,
|
||||
];
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Name-field usage stats (from profiles cache)
|
||||
$name_field_usage = ['both' => 0, 'name_only' => 0, 'display_only' => 0, 'neither' => 0, 'both_differ' => 0, 'total' => 0];
|
||||
try {
|
||||
$row = $pdo->query("
|
||||
SELECT count(*) FILTER (WHERE name <> '' AND display_name <> '') AS both,
|
||||
count(*) FILTER (WHERE name <> '' AND display_name = '') AS name_only,
|
||||
count(*) FILTER (WHERE name = '' AND display_name <> '') AS display_only,
|
||||
count(*) FILTER (WHERE name = '' AND display_name = '') AS neither,
|
||||
count(*) FILTER (WHERE name <> '' AND display_name <> '' AND name <> display_name) AS both_differ,
|
||||
count(*) AS total
|
||||
FROM profiles
|
||||
")->fetch();
|
||||
if ($row) {
|
||||
$name_field_usage = array_map('intval', $row);
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
json_response([
|
||||
'db_size' => $db_size,
|
||||
'total_events' => $total_events,
|
||||
'events_delta' => $events_delta,
|
||||
'process_id' => $process_id,
|
||||
'ws_connections' => $ws_connections,
|
||||
'active_subscriptions' => $active_subscriptions,
|
||||
'memory_usage' => $memory_usage,
|
||||
'cpu_usage' => $cpu_usage,
|
||||
'cpu_core' => $cpu_core,
|
||||
'oldest_event' => $oldest_event,
|
||||
'newest_event' => $newest_event,
|
||||
'events_24h' => $events_24h,
|
||||
'events_7d' => $events_7d,
|
||||
'events_30d' => $events_30d,
|
||||
'kinds' => $kinds,
|
||||
'top_pubkeys' => $top_pubkeys,
|
||||
'name_field_usage' => $name_field_usage,
|
||||
]);
|
||||
|
||||
/** Format bytes as human-readable. */
|
||||
function format_bytes(int $bytes): string {
|
||||
if ($bytes < 1024) return $bytes . ' B';
|
||||
if ($bytes < 1048576) return round($bytes / 1024, 1) . ' KB';
|
||||
if ($bytes < 1073741824) return round($bytes / 1048576, 1) . ' MB';
|
||||
return round($bytes / 1073741824, 2) . ' GB';
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
/** admin2/api/subscriptions.php — Active subscription details. */
|
||||
require_once __DIR__ . '/../lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
$subs = [];
|
||||
try {
|
||||
$subs = $pdo->query("SELECT id, sub_id, pubkey, filters, created_at, events_sent FROM subscriptions ORDER BY created_at DESC LIMIT 100")->fetchAll();
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
json_response(['subscriptions' => $subs]);
|
||||
@@ -0,0 +1,956 @@
|
||||
/*
|
||||
* admin2 — Full PHP admin for C-Relay-PG.
|
||||
* Replaces the 7457-line api/index.js with a lightweight AJAX-based
|
||||
* controller that fetches data from PHP API endpoints (api/*.php)
|
||||
* instead of Nostr WebSocket admin commands.
|
||||
*/
|
||||
|
||||
const REFRESH_MS = 10000;
|
||||
let currentPage = 'statistics';
|
||||
let statsInterval = null;
|
||||
|
||||
// HTML-escape helper for safe interpolation into innerHTML.
|
||||
// Prevents stored XSS from user-controlled data (profile names, event
|
||||
// content, config values, etc.) being parsed as HTML by the browser.
|
||||
const esc = (s) => {
|
||||
const str = String(s ?? '');
|
||||
return str.replace(/[&<>"']/g, (ch) => {
|
||||
return '&#' + ch.charCodeAt(0) + ';';
|
||||
});
|
||||
};
|
||||
|
||||
// Auth state
|
||||
let nlLite = null;
|
||||
let userPubkey = null;
|
||||
let isLoggedIn = false;
|
||||
let relayAnimationTimer = null;
|
||||
|
||||
// Server-rendered ASCII chart state
|
||||
let currentChartRange = 'hour';
|
||||
let chartLoadTimer = null;
|
||||
|
||||
// ================================
|
||||
// NAVIGATION
|
||||
// ================================
|
||||
|
||||
function toggleNav() {
|
||||
document.getElementById('side-nav').classList.toggle('open');
|
||||
document.getElementById('side-nav-overlay').classList.toggle('show');
|
||||
}
|
||||
|
||||
document.getElementById('side-nav-overlay')?.addEventListener('click', toggleNav);
|
||||
|
||||
document.querySelectorAll('.nav-item').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
const page = btn.getAttribute('data-page');
|
||||
switchPage(page);
|
||||
document.getElementById('side-nav').classList.remove('open');
|
||||
document.getElementById('side-nav-overlay').classList.remove('show');
|
||||
});
|
||||
});
|
||||
|
||||
function switchPage(pageName) {
|
||||
currentPage = pageName;
|
||||
document.querySelectorAll('.nav-item').forEach(item => {
|
||||
item.classList.remove('active');
|
||||
if (item.getAttribute('data-page') === pageName) item.classList.add('active');
|
||||
});
|
||||
|
||||
const sections = [
|
||||
'databaseStatisticsSection', 'subscriptionDetailsSection', 'div_config',
|
||||
'authRulesSection', 'wotSection', 'ipBansSection', 'relayEventsSection',
|
||||
'cachingSection', 'nip17DMSection', 'sqlQuerySection'
|
||||
];
|
||||
sections.forEach(id => { const el = document.getElementById(id); if (el) el.style.display = 'none'; });
|
||||
|
||||
const pageMap = {
|
||||
'statistics': 'databaseStatisticsSection',
|
||||
'subscriptions': 'subscriptionDetailsSection',
|
||||
'configuration': 'div_config',
|
||||
'ip-bans': 'ipBansSection',
|
||||
'relay-events': 'relayEventsSection',
|
||||
'caching': 'cachingSection',
|
||||
'dm': 'nip17DMSection',
|
||||
'database': 'sqlQuerySection'
|
||||
};
|
||||
|
||||
if (pageName === 'authorization') {
|
||||
document.getElementById('authRulesSection').style.display = 'block';
|
||||
document.getElementById('wotSection').style.display = 'block';
|
||||
loadAuthRules();
|
||||
loadWotStatus();
|
||||
} else {
|
||||
const target = pageMap[pageName];
|
||||
if (target) document.getElementById(target).style.display = 'block';
|
||||
}
|
||||
|
||||
// Load data for the page
|
||||
const loaders = {
|
||||
'statistics': loadStats,
|
||||
'subscriptions': loadSubscriptions,
|
||||
'configuration': loadConfig,
|
||||
'ip-bans': loadIpBans,
|
||||
'relay-events': loadEvents,
|
||||
'caching': loadCaching,
|
||||
'dm': loadDMs,
|
||||
};
|
||||
if (loaders[pageName]) loaders[pageName]();
|
||||
|
||||
// Start/stop auto-refresh for statistics
|
||||
if (pageName === 'statistics') {
|
||||
if (!statsInterval) {
|
||||
loadStats();
|
||||
statsInterval = setInterval(loadStats, REFRESH_MS);
|
||||
console.log('[admin2] auto-refresh started, interval:', REFRESH_MS, 'ms');
|
||||
}
|
||||
} else {
|
||||
if (statsInterval) { clearInterval(statsInterval); statsInterval = null; }
|
||||
}
|
||||
}
|
||||
|
||||
// ================================
|
||||
// STATISTICS
|
||||
// ================================
|
||||
|
||||
async function loadStats() {
|
||||
console.log('[admin2] loading stats at', new Date().toLocaleTimeString());
|
||||
// Fire the RELAY letter animation on every refresh — visual indicator the page is updating
|
||||
startRelayAnimation();
|
||||
try {
|
||||
const res = await fetch('api/stats.php');
|
||||
if (!res.ok) { console.warn('[admin2] stats.php returned', res.status); return; }
|
||||
const d = await res.json();
|
||||
console.log('[admin2] stats loaded — events:', d.total_events, 'rate:', d.events_delta, '/10s');
|
||||
|
||||
const set = (id, val) => { const el = document.getElementById(id); if (el) el.textContent = val; };
|
||||
set('db-size', d.db_size || '-');
|
||||
set('total-events', (d.total_events ?? 0).toLocaleString());
|
||||
set('process-id', d.process_id || '-');
|
||||
set('websocket-connections', d.ws_connections ?? '-');
|
||||
set('active-subscriptions', d.active_subscriptions ?? '-');
|
||||
set('memory-usage', d.memory_usage || '-');
|
||||
set('cpu-usage', d.cpu_usage || '-');
|
||||
set('cpu-core', d.cpu_core || '-');
|
||||
set('oldest-event', d.oldest_event || '-');
|
||||
set('newest-event', d.newest_event || '-');
|
||||
set('events-24h', (d.events_24h ?? 0).toLocaleString());
|
||||
set('events-7d', (d.events_7d ?? 0).toLocaleString());
|
||||
set('events-30d', (d.events_30d ?? 0).toLocaleString());
|
||||
|
||||
// Kind distribution
|
||||
if (d.kinds) {
|
||||
const tbody = document.getElementById('stats-kinds-table-body');
|
||||
tbody.innerHTML = d.kinds.map(k =>
|
||||
`<tr><td>${k.kind}</td><td>${k.count.toLocaleString()}</td><td>${k.pct}%</td></tr>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
// Top pubkeys
|
||||
if (d.top_pubkeys) {
|
||||
const tbody = document.getElementById('stats-pubkeys-table-body');
|
||||
tbody.innerHTML = d.top_pubkeys.map((p, i) =>
|
||||
`<tr><td>${i+1}</td><td><bdi>${esc(p.name) || '<i>unknown</i>'}</bdi></td><td class="pubkey">${esc(p.pubkey.substring(0,16))}…</td><td>${p.count.toLocaleString()}</td><td>${p.pct}%</td></tr>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
// Name-field usage stats (from profiles cache)
|
||||
if (d.name_field_usage && d.name_field_usage.total > 0) {
|
||||
const u = d.name_field_usage;
|
||||
const el = document.getElementById('name-field-usage');
|
||||
if (el) {
|
||||
el.innerHTML = `
|
||||
<div class="name-usage-stats">
|
||||
<span class="name-usage-label">Profile name fields (${u.total} profiles):</span>
|
||||
<span class="name-usage-item">Both: ${u.both}</span>
|
||||
<span class="name-usage-item">name only: ${u.name_only}</span>
|
||||
<span class="name-usage-item">display_name only: ${u.display_only}</span>
|
||||
<span class="name-usage-item name-usage-differ">Differ: ${u.both_differ}</span>
|
||||
</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh the chart on every stats poll (server handles caching for non-hour ranges)
|
||||
loadChart(currentChartRange);
|
||||
} catch (e) { console.error('[admin2] stats error:', e); }
|
||||
}
|
||||
|
||||
// ================================
|
||||
// EVENT RATE CHART (server-rendered ASCII via chart.php)
|
||||
// ================================
|
||||
|
||||
// Fetch the ASCII chart from the server and inject it into the div.
|
||||
// The chart is rendered server-side as plain text — works in both
|
||||
// browser and terminal (curl http://localhost:8088/api/chart.php?range=hour)
|
||||
async function loadChart(range) {
|
||||
const el = document.getElementById('event-rate-chart');
|
||||
if (!el) return;
|
||||
try {
|
||||
const res = await fetch('api/chart.php?range=' + encodeURIComponent(range));
|
||||
if (!res.ok) { el.textContent = 'Chart load failed (HTTP ' + res.status + ')'; return; }
|
||||
const text = await res.text();
|
||||
el.textContent = text;
|
||||
// Newest data is at the left (index 0) — scroll to start so it's visible
|
||||
el.scrollLeft = 0;
|
||||
} catch (e) {
|
||||
console.error('[admin2] chart load error:', e);
|
||||
el.textContent = 'Chart load error: ' + e.message;
|
||||
}
|
||||
}
|
||||
|
||||
// Chart range tab click handlers
|
||||
document.querySelectorAll('.chart-tab').forEach(tab => {
|
||||
tab.addEventListener('click', () => {
|
||||
const range = tab.getAttribute('data-range');
|
||||
if (!range || range === currentChartRange) return;
|
||||
currentChartRange = range;
|
||||
document.querySelectorAll('.chart-tab').forEach(t => t.classList.remove('active'));
|
||||
tab.classList.add('active');
|
||||
loadChart(range);
|
||||
});
|
||||
});
|
||||
|
||||
// ================================
|
||||
// SUBSCRIPTIONS
|
||||
// ================================
|
||||
|
||||
async function loadSubscriptions() {
|
||||
try {
|
||||
const res = await fetch('api/subscriptions.php');
|
||||
const d = await res.json();
|
||||
const tbody = document.getElementById('subscription-details-table-body');
|
||||
if (!d.subscriptions || d.subscriptions.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="5" style="text-align:center;font-style:italic">No subscriptions active</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = d.subscriptions.map(s =>
|
||||
`<tr><td>${esc(s.sub_id) || '-'}</td><td class="pubkey">${esc((s.pubkey||'-').substring(0,16))}</td><td>${esc(s.filters) || '-'}</td><td>${esc(s.created) || '-'}</td><td>${s.events_sent ?? 0}</td></tr>`
|
||||
).join('');
|
||||
} catch (e) { console.error('[admin2] subscriptions error:', e); }
|
||||
}
|
||||
|
||||
// ================================
|
||||
// CONFIGURATION
|
||||
// ================================
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
const res = await fetch('api/config.php');
|
||||
const d = await res.json();
|
||||
const tbody = document.getElementById('config-table-body');
|
||||
if (!d.config || d.config.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="3" style="text-align:center;font-style:italic">No config entries</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = d.config.map(c =>
|
||||
`<tr><td>${esc(c.key)}</td><td>${esc(c.value)}</td><td><button onclick="editConfig('${esc(c.key)}')">EDIT</button></td></tr>`
|
||||
).join('');
|
||||
} catch (e) { console.error('[admin2] config error:', e); }
|
||||
}
|
||||
|
||||
function editConfig(key) {
|
||||
const newVal = prompt('Enter new value for ' + key + ':');
|
||||
if (newVal === null) return;
|
||||
fetch('api/config.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({key, value: newVal})
|
||||
}).then(r => r.json()).then(d => {
|
||||
alert(d.message || 'Saved');
|
||||
loadConfig();
|
||||
}).catch(e => alert('Error: ' + e));
|
||||
}
|
||||
|
||||
// ================================
|
||||
// AUTH RULES
|
||||
// ================================
|
||||
|
||||
async function loadAuthRules() {
|
||||
try {
|
||||
const res = await fetch('api/auth.php');
|
||||
const d = await res.json();
|
||||
const tbody = document.getElementById('authRulesTableBody');
|
||||
if (!d.rules || d.rules.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="5" style="text-align:center;font-style:italic">No auth rules</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = d.rules.map(r =>
|
||||
`<tr><td>${esc(r.rule_type)}</td><td>${esc(r.pattern_type)}</td><td>${esc(r.pattern_value)}</td><td>${esc(r.status) || 'active'}</td><td><button onclick="removeAuthRule(${r.id})">REMOVE</button></td></tr>`
|
||||
).join('');
|
||||
} catch (e) { console.error('[admin2] auth error:', e); }
|
||||
}
|
||||
|
||||
async function addAuthRule(type) {
|
||||
const pk = document.getElementById('authRulePubkey').value.trim();
|
||||
if (!pk) { alert('Enter a pubkey first'); return; }
|
||||
const res = await fetch('api/auth.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'add', rule_type: type, pattern_value: pk})
|
||||
});
|
||||
const d = await res.json();
|
||||
alert(d.message || 'Done');
|
||||
loadAuthRules();
|
||||
}
|
||||
|
||||
async function removeAuthRule(id) {
|
||||
const res = await fetch('api/auth.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'remove', id})
|
||||
});
|
||||
const d = await res.json();
|
||||
alert(d.message || 'Done');
|
||||
loadAuthRules();
|
||||
}
|
||||
|
||||
// ================================
|
||||
// WEB OF TRUST
|
||||
// ================================
|
||||
|
||||
async function loadWotStatus() {
|
||||
try {
|
||||
const res = await fetch('api/auth.php?action=wot');
|
||||
const d = await res.json();
|
||||
const ind = document.getElementById('wotKind3Indicator');
|
||||
if (ind) ind.textContent = d.kind3_status || 'Unknown';
|
||||
const cnt = document.getElementById('wotWhitelistCount');
|
||||
if (cnt) cnt.textContent = d.whitelist_count ?? '—';
|
||||
const desc = document.getElementById('wotLevelDescription');
|
||||
if (desc) desc.textContent = d.level_description || '';
|
||||
} catch (e) { console.error('[admin2] wot error:', e); }
|
||||
}
|
||||
|
||||
async function setWotLevel(level) {
|
||||
const res = await fetch('api/auth.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'set_wot_level', level})
|
||||
});
|
||||
const d = await res.json();
|
||||
alert(d.message || 'Done');
|
||||
loadWotStatus();
|
||||
}
|
||||
|
||||
async function syncWot() {
|
||||
const res = await fetch('api/auth.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'sync_wot'})
|
||||
});
|
||||
const d = await res.json();
|
||||
alert(d.message || 'Done');
|
||||
loadWotStatus();
|
||||
}
|
||||
|
||||
// ================================
|
||||
// IP BANS
|
||||
// ================================
|
||||
|
||||
async function loadIpBans() {
|
||||
try {
|
||||
const res = await fetch('api/ipbans.php');
|
||||
const d = await res.json();
|
||||
document.getElementById('ip-bans-total').textContent = d.total ?? '-';
|
||||
document.getElementById('ip-bans-active').textContent = d.active ?? '-';
|
||||
document.getElementById('ip-bans-issued').textContent = d.issued ?? '-';
|
||||
const tbody = document.getElementById('ip-bans-tbody');
|
||||
if (!d.bans || d.bans.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="5" style="text-align:center">No IP bans</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = d.bans.map(b =>
|
||||
`<tr><td>${esc(b.ip)}</td><td>${esc(b.status)}</td><td>${esc(b.banned_until)}</td><td>${b.failures ?? 0}</td><td><button onclick="removeBan('${esc(b.ip)}')">REMOVE</button></td></tr>`
|
||||
).join('');
|
||||
} catch (e) { console.error('[admin2] ipbans error:', e); }
|
||||
}
|
||||
|
||||
async function addBan() {
|
||||
const ip = document.getElementById('ban-ip-input').value.trim();
|
||||
const duration = document.getElementById('ban-duration-select').value;
|
||||
if (!ip) { alert('Enter an IP address'); return; }
|
||||
const res = await fetch('api/ipbans.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'add', ip, duration: parseInt(duration)})
|
||||
});
|
||||
const d = await res.json();
|
||||
document.getElementById('add-ban-status').textContent = d.message || 'Done';
|
||||
loadIpBans();
|
||||
}
|
||||
|
||||
async function removeBan(ip) {
|
||||
const res = await fetch('api/ipbans.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'remove', ip})
|
||||
});
|
||||
const d = await res.json();
|
||||
alert(d.message || 'Done');
|
||||
loadIpBans();
|
||||
}
|
||||
|
||||
// ================================
|
||||
// RELAY EVENTS
|
||||
// ================================
|
||||
|
||||
async function loadEvents() {
|
||||
try {
|
||||
const res = await fetch('api/events.php?limit=50');
|
||||
const d = await res.json();
|
||||
const tbody = document.getElementById('live-relay-events-table-body');
|
||||
if (!d.events || d.events.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="5" style="text-align:center">No events</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = d.events.map(e =>
|
||||
`<tr><td>${esc(e.created_at)}</td><td>${e.kind}</td><td><bdi>${esc(e.display_name || (e.pubkey||'').substring(0,16) + '…')}</bdi></td><td class="pubkey">${esc((e.id||'').substring(0,16))}…</td><td>${esc((e.content||'').substring(0,80))}</td></tr>`
|
||||
).join('');
|
||||
} catch (err) { console.error('[admin2] events error:', err); }
|
||||
}
|
||||
|
||||
// ================================
|
||||
// CACHING
|
||||
// ================================
|
||||
|
||||
async function loadCaching() {
|
||||
try {
|
||||
const res = await fetch('api/caching.php');
|
||||
const d = await res.json();
|
||||
// Config toggle state
|
||||
if (d.config) {
|
||||
const enabled = d.config.caching_enabled === 'true';
|
||||
const inboxEnabled = d.config.caching_inbox_enabled === 'true';
|
||||
const enLabel = document.getElementById('caching-enabled-label');
|
||||
const enBtn = document.getElementById('caching-toggle-btn');
|
||||
const inLabel = document.getElementById('caching-inbox-enabled-label');
|
||||
const inBtn = document.getElementById('caching-inbox-toggle-btn');
|
||||
if (enLabel) enLabel.textContent = 'Caching: ' + (enabled ? 'ON' : 'OFF');
|
||||
if (enBtn) enBtn.textContent = enabled ? 'Turn OFF' : 'Turn ON';
|
||||
if (inLabel) inLabel.textContent = 'Inbox: ' + (inboxEnabled ? 'ON' : 'OFF');
|
||||
if (inBtn) inBtn.textContent = inboxEnabled ? 'Turn OFF' : 'Turn ON';
|
||||
}
|
||||
// Service status
|
||||
const ssEl = document.getElementById('caching-service-status');
|
||||
if (ssEl && d.state) {
|
||||
const s = d.state;
|
||||
const hb = s.heartbeat_at ? new Date(s.heartbeat_at * 1000).toLocaleTimeString() : '—';
|
||||
ssEl.innerHTML = `<p>State: <strong>${esc(s.service_state)}</strong> | Follows: ${s.followed_author_count ?? 0} | Connected relays: ${s.connected_relay_count ?? 0}/${s.selected_relay_count ?? 0} | Backfill: ${s.backfill_authors_complete ?? 0}/${s.backfill_authors_total ?? 0} | Events fetched: ${s.events_fetched ?? 0} | Inbox inserts: ${s.inbox_inserts ?? 0} | Heartbeat: ${hb}</p>`;
|
||||
}
|
||||
// Inbox status
|
||||
const isEl = document.getElementById('caching-inbox-status');
|
||||
if (isEl && d.inbox) {
|
||||
isEl.innerHTML = `<p>Pending: ${d.inbox.pending} | Live: ${d.inbox.live} | Backfill: ${d.inbox.backfill}</p>`;
|
||||
}
|
||||
// Follows table
|
||||
const tbody = document.getElementById('caching-follows-table-body');
|
||||
if (tbody && d.follows) {
|
||||
if (d.follows.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="6" style="text-align:center;font-style:italic">No followed pubkeys</td></tr>';
|
||||
} else {
|
||||
const activePk = d.active && d.active.pubkey ? d.active.pubkey : '';
|
||||
tbody.innerHTML = d.follows.map((f, i) => {
|
||||
const npub = f.npub || f.pubkey.substring(0, 20);
|
||||
const isActive = activePk && f.pubkey === activePk;
|
||||
const relaySummary = f.relay_incomplete > 0
|
||||
? `<span class="status-working">${f.relay_count - f.relay_incomplete}/${f.relay_count} done</span>`
|
||||
: `${f.relay_count ?? 0} relays`;
|
||||
// Per-relay detail row (hidden by default, toggle via click)
|
||||
let relayDetail = '';
|
||||
if (f.relays && f.relays.length > 0) {
|
||||
relayDetail = '<div class="relay-progress-list">' +
|
||||
f.relays.map(r => {
|
||||
const statusIcon = r.complete ? '✓' : '⏳';
|
||||
const statusBadge = r.complete
|
||||
? `<span class="relay-status relay-done">${esc(r.last_status || 'eose')}</span>`
|
||||
: `<span class="relay-status relay-pending">${esc(r.last_status || 'pending')}</span>`;
|
||||
const relayHost = r.relay_url.replace(/^wss?:\/\//, '').replace(/\/relay$/, '');
|
||||
return `<div class="relay-progress-row"><span class="relay-icon">${statusIcon}</span><span class="relay-url" title="${esc(r.relay_url)}">${esc(relayHost)}</span><span class="relay-events">${r.events_fetched} evts</span>${statusBadge}</div>`;
|
||||
}).join('') +
|
||||
'</div>';
|
||||
}
|
||||
const refreshBtn = `<button type="button" class="refresh-user-btn" onclick="event.stopPropagation(); refreshCachingUser('${esc(f.pubkey)}', '${esc(f.name || '')}')">↻ Refresh this user</button>`;
|
||||
const rowClass = isActive ? 'follows-row follows-row-active' : 'follows-row';
|
||||
const activeIcon = isActive ? '⚡ ' : '';
|
||||
return `<tr class="${rowClass}" onclick="this.nextElementSibling.style.display = this.nextElementSibling.style.display === 'none' ? '' : 'none'"><td><bdi>${activeIcon}${esc(f.name) || '<i>unknown</i>'}</bdi></td><td class="npub-link">${esc(npub)}…</td><td>${f.is_root ? '✓' : ''}</td><td>${f.total_events ?? 0}</td><td>${f.backfill_complete ? '✓' : '…'}</td><td>${relaySummary}</td></tr><tr class="follows-detail" style="display:none"><td colspan="6"><div class="follows-detail-content">${relayDetail || '<i>No relay progress data</i>'}<div class="follows-detail-actions">${refreshBtn}</div></div></td></tr>`;
|
||||
}).join('');
|
||||
}
|
||||
}
|
||||
// Active target
|
||||
const fsEl = document.getElementById('caching-follows-status');
|
||||
if (fsEl) {
|
||||
if (d.active && d.active.pubkey && d.active.relay) {
|
||||
fsEl.innerHTML = `<span class="status-working">⚡ Backfilling: ${d.active.pubkey.substring(0,16)}… @ ${esc(d.active.relay)}</span>`;
|
||||
} else if (d.state && d.state.service_state === 'running') {
|
||||
const incomplete = d.state.backfill_authors_complete < d.state.backfill_authors_total;
|
||||
fsEl.innerHTML = incomplete
|
||||
? `<span class="status-working">⚡ Backfill in progress…</span>`
|
||||
: `<span class="status-complete">✓ Backfill complete — listening for live events</span>`;
|
||||
} else {
|
||||
fsEl.innerHTML = `<span class="status-complete">✓ Caching complete</span>`;
|
||||
}
|
||||
}
|
||||
} catch (e) { console.error('[admin2] caching error:', e); }
|
||||
}
|
||||
|
||||
// Toggle caching_enabled config via the config API.
|
||||
async function toggleCachingEnabled() {
|
||||
try {
|
||||
const res = await fetch('api/caching.php');
|
||||
const d = await res.json();
|
||||
const current = d.config?.caching_enabled === 'true';
|
||||
const newVal = current ? 'false' : 'true';
|
||||
await fetch('api/config.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({key: 'caching_enabled', value: newVal})
|
||||
});
|
||||
loadCaching();
|
||||
} catch (e) { console.error('[admin2] toggle caching error:', e); }
|
||||
}
|
||||
|
||||
// Toggle caching_inbox_enabled config via the config API.
|
||||
async function toggleCachingInboxEnabled() {
|
||||
try {
|
||||
const res = await fetch('api/caching.php');
|
||||
const d = await res.json();
|
||||
const current = d.config?.caching_inbox_enabled === 'true';
|
||||
const newVal = current ? 'false' : 'true';
|
||||
await fetch('api/config.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({key: 'caching_inbox_enabled', value: newVal})
|
||||
});
|
||||
loadCaching();
|
||||
} catch (e) { console.error('[admin2] toggle inbox error:', e); }
|
||||
}
|
||||
|
||||
// Re-run all caching: resets backfill progress for all followed authors and
|
||||
// bumps caching_config_generation so the running service hot-reloads and
|
||||
// re-drains from the beginning.
|
||||
async function rerunAllCaching() {
|
||||
if (!confirm('Reset backfill progress for ALL followed authors? The caching service will re-download everything from scratch.')) return;
|
||||
const btn = document.getElementById('caching-rerun-all-btn');
|
||||
if (btn) { btn.disabled = true; btn.textContent = 'Resetting…'; }
|
||||
try {
|
||||
const res = await fetch('api/caching.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'reset_all'})
|
||||
});
|
||||
const d = await res.json();
|
||||
if (d.ok) {
|
||||
if (btn) { btn.textContent = '✓ Reset — re-draining'; }
|
||||
setTimeout(() => { if (btn) { btn.disabled = false; btn.textContent = 'Re-run All Caching'; } loadCaching(); }, 2000);
|
||||
} else {
|
||||
alert('Reset failed: ' + (d.error || 'unknown error'));
|
||||
if (btn) { btn.disabled = false; btn.textContent = 'Re-run All Caching'; }
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[admin2] rerunAllCaching error:', e);
|
||||
alert('Reset failed: ' + e.message);
|
||||
if (btn) { btn.disabled = false; btn.textContent = 'Re-run All Caching'; }
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh a single user: resets backfill progress for one followed author
|
||||
// and bumps caching_config_generation so the service re-fetches that author.
|
||||
async function refreshCachingUser(pubkey, name) {
|
||||
if (!confirm('Reset backfill progress for ' + (name || pubkey.substring(0, 16) + '…') + '? The caching service will re-download this user\'s events from scratch.')) return;
|
||||
try {
|
||||
const res = await fetch('api/caching.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({action: 'reset_user', pubkey: pubkey})
|
||||
});
|
||||
const d = await res.json();
|
||||
if (d.ok) {
|
||||
loadCaching();
|
||||
} else {
|
||||
alert('Refresh failed: ' + (d.error || 'unknown error'));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[admin2] refreshCachingUser error:', e);
|
||||
alert('Refresh failed: ' + e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// ================================
|
||||
// DMs
|
||||
// ================================
|
||||
|
||||
async function loadDMs() {
|
||||
try {
|
||||
const res = await fetch('api/dm.php?limit=50');
|
||||
const d = await res.json();
|
||||
const el = document.getElementById('dm-inbox');
|
||||
if (!d.messages || d.messages.length === 0) {
|
||||
el.innerHTML = '<div class="log-entry">No messages found.</div>';
|
||||
return;
|
||||
}
|
||||
el.innerHTML = d.messages.map(m =>
|
||||
`<div class="log-entry"><strong>kind ${m.kind}</strong> from <span class="pubkey">${esc((m.pubkey||'').substring(0,16))}…</span> at ${esc(m.created_at)}:<br>${esc((m.content||'').substring(0,200))}</div>`
|
||||
).join('');
|
||||
} catch (e) { console.error('[admin2] dm error:', e); }
|
||||
}
|
||||
|
||||
// ================================
|
||||
// SQL QUERY
|
||||
// ================================
|
||||
|
||||
async function executeQuery() {
|
||||
const sql = document.getElementById('sql-input').value.trim();
|
||||
if (!sql) { alert('Enter a SQL query'); return; }
|
||||
if (!sql.toUpperCase().startsWith('SELECT')) { alert('Only SELECT queries are allowed'); return; }
|
||||
try {
|
||||
const res = await fetch('api/query.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({sql})
|
||||
});
|
||||
const d = await res.json();
|
||||
const info = document.getElementById('query-info');
|
||||
const tableDiv = document.getElementById('query-table');
|
||||
if (d.error) {
|
||||
info.textContent = 'Error: ' + d.error;
|
||||
tableDiv.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
info.textContent = `${d.row_count} rows in ${d.time_ms}ms`;
|
||||
if (d.rows && d.rows.length > 0) {
|
||||
const cols = Object.keys(d.rows[0]);
|
||||
let html = '<table class="config-table"><thead><tr>';
|
||||
cols.forEach(c => html += `<th>${c}</th>`);
|
||||
html += '</tr></thead><tbody>';
|
||||
d.rows.forEach(r => {
|
||||
html += '<tr>';
|
||||
cols.forEach(c => html += `<td>${r[c] ?? ''}</td>`);
|
||||
html += '</tr>';
|
||||
});
|
||||
html += '</tbody></table>';
|
||||
tableDiv.innerHTML = html;
|
||||
} else {
|
||||
tableDiv.innerHTML = '<p>No rows returned.</p>';
|
||||
}
|
||||
} catch (e) { console.error('[admin2] query error:', e); }
|
||||
}
|
||||
|
||||
// ================================
|
||||
// DARK MODE
|
||||
// ================================
|
||||
|
||||
document.getElementById('nav-dark-mode-btn')?.addEventListener('click', () => {
|
||||
document.body.classList.toggle('dark-mode');
|
||||
localStorage.setItem('admin2-dark-mode', document.body.classList.contains('dark-mode'));
|
||||
const btn = document.getElementById('nav-dark-mode-btn');
|
||||
if (btn) btn.textContent = document.body.classList.contains('dark-mode') ? 'LIGHT MODE' : 'DARK MODE';
|
||||
});
|
||||
|
||||
if (localStorage.getItem('admin2-dark-mode') === 'true') {
|
||||
document.body.classList.add('dark-mode');
|
||||
const btn = document.getElementById('nav-dark-mode-btn');
|
||||
if (btn) btn.textContent = 'LIGHT MODE';
|
||||
}
|
||||
|
||||
// ================================
|
||||
// RELAY LETTER ANIMATION
|
||||
// ================================
|
||||
|
||||
// Animate the RELAY letters with an underline sweep. Fires on every stats
|
||||
// refresh so the user gets a visual cue that the page is updating.
|
||||
function startRelayAnimation() {
|
||||
const letters = document.querySelectorAll('.relay-letter');
|
||||
if (letters.length === 0) return;
|
||||
|
||||
// Cancel any in-flight animation so rapid refreshes don't overlap
|
||||
if (relayAnimationTimer) { clearTimeout(relayAnimationTimer); relayAnimationTimer = null; }
|
||||
|
||||
let currentIndex = 0;
|
||||
letters.forEach(l => l.classList.remove('underlined'));
|
||||
|
||||
function animateLetter() {
|
||||
letters.forEach(letter => letter.classList.remove('underlined'));
|
||||
if (letters[currentIndex]) {
|
||||
letters[currentIndex].classList.add('underlined');
|
||||
}
|
||||
currentIndex++;
|
||||
if (currentIndex > letters.length) {
|
||||
// Sweep complete — clear underlines and pause before next refresh
|
||||
letters.forEach(letter => letter.classList.remove('underlined'));
|
||||
relayAnimationTimer = null;
|
||||
return;
|
||||
}
|
||||
relayAnimationTimer = setTimeout(animateLetter, 100);
|
||||
}
|
||||
animateLetter();
|
||||
}
|
||||
|
||||
// ================================
|
||||
// RELAY PUBKEY COPY-TO-CLIPBOARD
|
||||
// ================================
|
||||
|
||||
document.getElementById('relay-pubkey-container')?.addEventListener('click', async () => {
|
||||
const el = document.getElementById('relay-pubkey');
|
||||
if (!el || !el.textContent.trim()) return;
|
||||
try {
|
||||
await navigator.clipboard.writeText(el.textContent.replace(/\s+/g, ''));
|
||||
const container = document.getElementById('relay-pubkey-container');
|
||||
container.classList.add('copied');
|
||||
setTimeout(() => container.classList.remove('copied'), 500);
|
||||
} catch (e) { console.warn('[admin2] clipboard copy failed:', e); }
|
||||
});
|
||||
|
||||
// ================================
|
||||
// AUTH — nostr_login_lite modal
|
||||
// ================================
|
||||
|
||||
const loginModal = document.getElementById('login-modal');
|
||||
const loginModalContainer = document.getElementById('login-modal-container');
|
||||
const profileArea = document.getElementById('profile-area');
|
||||
const headerUserName = document.getElementById('header-user-name');
|
||||
const headerUserImage = document.getElementById('header-user-image');
|
||||
const logoutDropdown = document.getElementById('logout-dropdown');
|
||||
|
||||
function showLoginModal() {
|
||||
if (loginModal && loginModalContainer) {
|
||||
if (window.NOSTR_LOGIN_LITE && typeof window.NOSTR_LOGIN_LITE.embed === 'function') {
|
||||
// Clear previous embed before re-embedding
|
||||
loginModalContainer.innerHTML = '';
|
||||
window.NOSTR_LOGIN_LITE.embed('#login-modal-container', { seamless: true });
|
||||
}
|
||||
loginModal.style.display = 'flex';
|
||||
}
|
||||
}
|
||||
|
||||
function hideLoginModal() {
|
||||
if (loginModal) loginModal.style.display = 'none';
|
||||
}
|
||||
|
||||
function showProfileInHeader() {
|
||||
if (profileArea) profileArea.style.display = 'flex';
|
||||
}
|
||||
|
||||
function hideProfileFromHeader() {
|
||||
if (profileArea) profileArea.style.display = 'none';
|
||||
}
|
||||
|
||||
// Toggle logout dropdown when clicking the profile area
|
||||
profileArea?.addEventListener('click', (e) => {
|
||||
// Only toggle if the click wasn't on the logout button itself
|
||||
if (e.target.closest('.logout-btn')) return;
|
||||
if (logoutDropdown) {
|
||||
logoutDropdown.style.display = (logoutDropdown.style.display === 'none' || !logoutDropdown.style.display) ? 'block' : 'none';
|
||||
}
|
||||
});
|
||||
|
||||
// Hide logout dropdown when clicking elsewhere
|
||||
document.addEventListener('click', (e) => {
|
||||
if (logoutDropdown && logoutDropdown.style.display === 'block' && !e.target.closest('#profile-area')) {
|
||||
logoutDropdown.style.display = 'none';
|
||||
}
|
||||
});
|
||||
|
||||
// Update header profile display from logged-in user's pubkey.
|
||||
// Sets a placeholder from the npub immediately, then fetches the kind 0
|
||||
// profile event from public relays to populate the name + picture.
|
||||
function updateProfileDisplay(pubkey) {
|
||||
if (!pubkey) return;
|
||||
let npub = '';
|
||||
try {
|
||||
if (pubkey.length === 64 && /^[0-9a-fA-F]+$/.test(pubkey)) {
|
||||
npub = window.NostrTools.nip19.npubEncode(pubkey);
|
||||
}
|
||||
} catch (err) { console.warn('[admin2] npub encode failed:', err); }
|
||||
|
||||
// Placeholder until the profile fetch resolves
|
||||
if (headerUserName) {
|
||||
headerUserName.textContent = npub ? npub.substring(0, 16) + '…' : pubkey.substring(0, 16) + '…';
|
||||
}
|
||||
if (headerUserImage) headerUserImage.style.display = 'none';
|
||||
|
||||
// Fetch kind 0 profile from public relays (same approach as original api page)
|
||||
loadUserProfile(pubkey, npub);
|
||||
}
|
||||
|
||||
// Apply profile data to the header name + profile picture.
|
||||
// Uses best_name from the server (resolved per profile_name_preference).
|
||||
function applyProfileToHeader(name, picture) {
|
||||
if (headerUserName) headerUserName.textContent = name || 'Anonymous User';
|
||||
if (headerUserImage && picture && typeof picture === 'string' &&
|
||||
(picture.startsWith('http://') || picture.startsWith('https://'))) {
|
||||
headerUserImage.src = picture;
|
||||
headerUserImage.style.display = 'block';
|
||||
headerUserImage.onerror = function() { this.style.display = 'none'; };
|
||||
} else if (headerUserImage) {
|
||||
headerUserImage.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the user's profile. Tries the local profiles cache first
|
||||
// (admin/api/profile.php), then falls back to public relays if the
|
||||
// relay has no cached kind-0 for this pubkey.
|
||||
async function loadUserProfile(pubkey, npub) {
|
||||
if (!pubkey) return;
|
||||
|
||||
// Try local profiles cache first.
|
||||
try {
|
||||
const res = await fetch('api/profile.php?pubkey=' + encodeURIComponent(pubkey));
|
||||
if (res.ok) {
|
||||
const profile = await res.json();
|
||||
if (profile && profile.best_name !== undefined) {
|
||||
applyProfileToHeader(profile.best_name, profile.picture);
|
||||
console.log('[admin2] profile loaded from local cache for', profile.best_name);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Local endpoint not available — fall through to public relays.
|
||||
}
|
||||
|
||||
// Fall back to public relays.
|
||||
if (!window.NostrTools || !window.NostrTools.SimplePool) {
|
||||
if (headerUserName) headerUserName.textContent = npub ? npub.substring(0, 16) + '…' : 'Anonymous User';
|
||||
return;
|
||||
}
|
||||
const relays = [
|
||||
'wss://relay.damus.io',
|
||||
'wss://relay.nostr.band',
|
||||
'wss://nos.lol',
|
||||
'wss://relay.primal.net',
|
||||
'wss://relay.snort.social'
|
||||
];
|
||||
try {
|
||||
const pool = new window.NostrTools.SimplePool();
|
||||
const timeoutPromise = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Profile query timeout')), 5000)
|
||||
);
|
||||
const queryPromise = pool.querySync(relays, {
|
||||
kinds: [0],
|
||||
authors: [pubkey],
|
||||
limit: 1
|
||||
});
|
||||
const events = await Promise.race([queryPromise, timeoutPromise]);
|
||||
try { await pool.close(relays); } catch (e) {}
|
||||
|
||||
if (events && events.length > 0) {
|
||||
const profile = JSON.parse(events[0].content);
|
||||
// Use best_name resolution: display_name first, then name.
|
||||
const name = profile.display_name || profile.name || profile.displayName || 'Anonymous User';
|
||||
const picture = profile.picture || profile.image || null;
|
||||
applyProfileToHeader(name, picture);
|
||||
console.log('[admin2] profile loaded from public relays for', name);
|
||||
} else {
|
||||
if (headerUserName) headerUserName.textContent = 'Anonymous User';
|
||||
console.log('[admin2] no profile event found for', pubkey);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[admin2] profile load failed:', err.message);
|
||||
if (headerUserName) headerUserName.textContent = npub ? npub.substring(0, 16) + '…' : 'Error loading profile';
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize nostr_login_lite and show modal if not already authenticated
|
||||
async function initializeAuth() {
|
||||
if (!window.NOSTR_LOGIN_LITE) {
|
||||
console.warn('[admin2] NOSTR_LOGIN_LITE not loaded — skipping auth modal');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await window.NOSTR_LOGIN_LITE.init({
|
||||
theme: 'default',
|
||||
methods: {
|
||||
extension: true,
|
||||
local: true,
|
||||
seedphrase: true,
|
||||
readonly: true,
|
||||
connect: true,
|
||||
remote: true,
|
||||
otp: false
|
||||
},
|
||||
floatingTab: { enabled: false }
|
||||
});
|
||||
nlLite = window.NOSTR_LOGIN_LITE;
|
||||
console.log('[admin2] nostr_login_lite initialized');
|
||||
|
||||
// Check for existing auth state
|
||||
let alreadyLoggedIn = false;
|
||||
try {
|
||||
const stored = localStorage.getItem('nostr_login_lite_auth');
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored);
|
||||
if (parsed && parsed.pubkey) {
|
||||
userPubkey = parsed.pubkey;
|
||||
isLoggedIn = true;
|
||||
alreadyLoggedIn = true;
|
||||
showProfileInHeader();
|
||||
updateProfileDisplay(userPubkey);
|
||||
hideLoginModal();
|
||||
console.log('[admin2] existing auth restored for', userPubkey);
|
||||
}
|
||||
}
|
||||
} catch (e) { /* no stored auth */ }
|
||||
|
||||
if (!alreadyLoggedIn) {
|
||||
console.log('[admin2] no existing auth — showing login modal');
|
||||
showLoginModal();
|
||||
}
|
||||
|
||||
// Listen for auth events
|
||||
window.addEventListener('nlMethodSelected', (event) => {
|
||||
const { pubkey, method, error } = event.detail || {};
|
||||
if (method && pubkey) {
|
||||
userPubkey = pubkey;
|
||||
isLoggedIn = true;
|
||||
console.log('[admin2] login success via', method, pubkey);
|
||||
hideLoginModal();
|
||||
showProfileInHeader();
|
||||
updateProfileDisplay(pubkey);
|
||||
} else if (error) {
|
||||
console.warn('[admin2] auth error:', error);
|
||||
}
|
||||
});
|
||||
|
||||
window.addEventListener('nlLogout', () => {
|
||||
console.log('[admin2] logout event received');
|
||||
userPubkey = null;
|
||||
isLoggedIn = false;
|
||||
hideProfileFromHeader();
|
||||
if (logoutDropdown) logoutDropdown.style.display = 'none';
|
||||
showLoginModal();
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('[admin2] nostr_login_lite init failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
// Logout function — clears auth state and re-shows login modal
|
||||
async function logout() {
|
||||
console.log('[admin2] logging out...');
|
||||
try {
|
||||
if (nlLite && typeof nlLite.logout === 'function') {
|
||||
await nlLite.logout();
|
||||
}
|
||||
} catch (e) { console.warn('[admin2] nlLite.logout error:', e); }
|
||||
userPubkey = null;
|
||||
isLoggedIn = false;
|
||||
hideProfileFromHeader();
|
||||
if (logoutDropdown) logoutDropdown.style.display = 'none';
|
||||
showLoginModal();
|
||||
console.log('[admin2] logged out');
|
||||
}
|
||||
|
||||
// ================================
|
||||
// INIT
|
||||
// ================================
|
||||
|
||||
switchPage('statistics');
|
||||
|
||||
// Start the RELAY animation immediately on page load
|
||||
startRelayAnimation();
|
||||
|
||||
// Initialize auth + load initial chart on DOM ready
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
setTimeout(initializeAuth, 100);
|
||||
// Load the default (hour) chart immediately
|
||||
loadChart(currentChartRange);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
New Events — Last 24 Hours
|
||||
|
||||
11 | X
|
||||
10 | XX
|
||||
9 | XX
|
||||
8 | XX
|
||||
7 | XX X X
|
||||
6 | XX X X
|
||||
5 | XX X X
|
||||
4 | X XX X XX
|
||||
3 | X XXX X XX
|
||||
2 | X X X XXX X XX
|
||||
1 | X XX X X X X X X X X X X X X XX X X XXXX X X X X X XX
|
||||
+--------------------------------------------------------------------------------
|
||||
0s 1h 3h 4h 6h 7h 9h 10h 12h 13h 15h 16h 18h 19h 21h 22h
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
New Events — Last 30 Days
|
||||
|
||||
81 | X
|
||||
73 | XX X
|
||||
65 | XX X
|
||||
57 | XX X X X
|
||||
49 | X X XX XX X X X
|
||||
41 | X X X XX X X X XXX XXX XXX X X X X
|
||||
33 | X X X XXXX XX X X X XX XXX XXXXXXXX XX X X X X X XX
|
||||
25 |X XX XX XX XXXX XX XXX X XX XXXXXXXXXXXXXXXXXXXX XX XXX X XX XX XXXXX
|
||||
17 |X XXXXX XX XXXXXXXXXXXXXXX X XX XXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXX XX XXXXXXXX
|
||||
9 |XXX XXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXX
|
||||
1 |XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
+--------------------------------------------------------------------------------
|
||||
0s 1d 3d 5d 7d 9d 11d 13d 15d 16d 18d 20d 22d 24d 26d 28d
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
New Events — Last Year
|
||||
|
||||
531 | X XX
|
||||
478 | X X X XXX X
|
||||
425 | X X XX X XX XXX X
|
||||
372 | XXX XX X XX XXX X
|
||||
319 |XXXXXXXXXXX X X X XXXXXXX X
|
||||
266 |XXXXXXXXXXXX X XX XXX XXXXXXXXX X
|
||||
213 |XXXXXXXXXXXXXX XXX XXXXXXXXXXXXX XX X
|
||||
160 |XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX X X XXXX X
|
||||
107 |XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XX XX XX X X X XX XXX X XXXXXXX X
|
||||
54 |XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXX
|
||||
1 |XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
+--------------------------------------------------------------------------------
|
||||
0s 22d 45d 68d 91d 114d 136d 159d 182d 205d 228d 250d 273d 296d 319d 342d
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
<?php
|
||||
/**
|
||||
* admin2 — Full PHP admin for C-Relay-PG.
|
||||
* Recreates the original api/ admin interface with a PHP/PDO backend
|
||||
* instead of Nostr WebSocket admin commands.
|
||||
*/
|
||||
require_once __DIR__ . '/lib/helpers.php';
|
||||
$pdo = db();
|
||||
|
||||
// Fetch relay info for the header (from config table)
|
||||
$relay_name = '';
|
||||
$relay_desc = '';
|
||||
$relay_pubkey = '';
|
||||
$relay_version = '';
|
||||
try {
|
||||
$relay_name = $pdo->query("SELECT value FROM config WHERE key = 'relay_name'")->fetchColumn() ?: 'C-Relay-PG';
|
||||
$relay_desc = $pdo->query("SELECT value FROM config WHERE key = 'relay_description'")->fetchColumn() ?: '';
|
||||
$relay_pubkey = $pdo->query("SELECT value FROM config WHERE key = 'relay_pubkey'")->fetchColumn() ?: '';
|
||||
$relay_version = $pdo->query("SELECT value FROM config WHERE key = 'relay_version'")->fetchColumn() ?: '';
|
||||
} catch (PDOException $e) {}
|
||||
|
||||
// Convert relay pubkey (hex) to npub and format into 3 lines of 3 groups of 7 chars
|
||||
$relay_npub = $relay_pubkey;
|
||||
if ($relay_pubkey && preg_match('/^[0-9a-fA-F]{64}$/', $relay_pubkey)) {
|
||||
$relay_npub = hex_to_npub($relay_pubkey);
|
||||
}
|
||||
// Format npub (63 chars) into 3 lines of "xxxxxxx xxxxxxx xxxxxxx"
|
||||
$formatted_npub = $relay_npub;
|
||||
if (strlen($relay_npub) === 63) {
|
||||
$line1 = substr($relay_npub, 0, 7) . ' ' . substr($relay_npub, 7, 7) . ' ' . substr($relay_npub, 14, 7);
|
||||
$line2 = substr($relay_npub, 21, 7) . ' ' . substr($relay_npub, 28, 7) . ' ' . substr($relay_npub, 35, 7);
|
||||
$line3 = substr($relay_npub, 42, 7) . ' ' . substr($relay_npub, 49, 7) . ' ' . substr($relay_npub, 56, 7);
|
||||
$formatted_npub = $line1 . "\n" . $line2 . "\n" . $line3;
|
||||
}
|
||||
$display_name = $relay_version ? ($relay_name . ' ' . $relay_version) : $relay_name;
|
||||
?>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>C-Relay-PG Admin</title>
|
||||
<link rel="stylesheet" href="assets/index.css">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<!-- Side Navigation Menu -->
|
||||
<nav class="side-nav" id="side-nav">
|
||||
<ul class="nav-menu">
|
||||
<li><button class="nav-item" data-page="statistics">Statistics</button></li>
|
||||
<li><button class="nav-item" data-page="subscriptions">Subscriptions</button></li>
|
||||
<li><button class="nav-item" data-page="configuration">Configuration</button></li>
|
||||
<li><button class="nav-item" data-page="authorization">Authorization</button></li>
|
||||
<li><button class="nav-item" data-page="ip-bans">IP BAN</button></li>
|
||||
<li><button class="nav-item" data-page="relay-events">Relay Events</button></li>
|
||||
<li><button class="nav-item" data-page="caching">Caching</button></li>
|
||||
<li><button class="nav-item" data-page="dm">DM</button></li>
|
||||
<li><button class="nav-item" data-page="database">Database Query</button></li>
|
||||
</ul>
|
||||
<div class="nav-footer">
|
||||
<button class="nav-footer-btn" id="nav-dark-mode-btn">DARK MODE</button>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<!-- Side Navigation Overlay -->
|
||||
<div class="side-nav-overlay" id="side-nav-overlay"></div>
|
||||
|
||||
<!-- Header with title and profile display -->
|
||||
<div class="section">
|
||||
<div class="header-content">
|
||||
<div class="header-title clickable" id="header-title" onclick="toggleNav()">
|
||||
<span class="relay-letter" data-letter="R">R</span>
|
||||
<span class="relay-letter" data-letter="E">E</span>
|
||||
<span class="relay-letter" data-letter="L">L</span>
|
||||
<span class="relay-letter" data-letter="A">A</span>
|
||||
<span class="relay-letter" data-letter="Y">Y</span>
|
||||
</div>
|
||||
<div class="relay-info">
|
||||
<div id="relay-name" class="relay-name"><?= e($display_name) ?></div>
|
||||
<div id="relay-description" class="relay-description"><?= e($relay_desc) ?></div>
|
||||
<div id="relay-pubkey-container" class="relay-pubkey-container" title="Click to copy npub">
|
||||
<div id="relay-pubkey" class="relay-pubkey"><?= e($formatted_npub) ?></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="profile-area" id="profile-area" style="display: none;">
|
||||
<div class="admin-label">admin</div>
|
||||
<div class="profile-container">
|
||||
<img id="header-user-image" class="header-user-image" alt="Profile" style="display: none;">
|
||||
<span id="header-user-name" class="header-user-name">Loading...</span>
|
||||
</div>
|
||||
<!-- Logout dropdown (under admin picture) -->
|
||||
<div class="logout-dropdown" id="logout-dropdown" style="display: none;">
|
||||
<button type="button" class="logout-btn" onclick="logout()">LOGOUT</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Login Modal Overlay (nostr_login_lite) -->
|
||||
<div id="login-modal" class="login-modal-overlay" style="display: none;">
|
||||
<div class="login-modal-content">
|
||||
<div id="login-modal-container"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Access Denied Overlay (shown when non-admin user logs in) -->
|
||||
<div id="access-denied-overlay" class="access-denied-overlay" style="display: none;">
|
||||
<div class="access-denied-content">
|
||||
<div class="access-denied-icon">⛔</div>
|
||||
<h2>ACCESS DENIED</h2>
|
||||
<p class="access-denied-message">This interface is restricted to the relay administrator.</p>
|
||||
<p class="access-denied-submessage">The logged-in account does not have admin privileges.</p>
|
||||
<button type="button" class="access-denied-logout-btn" onclick="logout()">LOGOUT</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- DATABASE STATISTICS Section -->
|
||||
<div class="section flex-section" id="databaseStatisticsSection">
|
||||
<div class="section-header">DATABASE STATISTICS</div>
|
||||
|
||||
<!-- Event Rate Graph — range selector tabs + ASCII chart (server-rendered) -->
|
||||
<div class="chart-range-tabs">
|
||||
<button class="chart-tab active" data-range="hour">1H</button>
|
||||
<button class="chart-tab" data-range="day">1D</button>
|
||||
<button class="chart-tab" data-range="month">1M</button>
|
||||
<button class="chart-tab" data-range="year">1Y</button>
|
||||
</div>
|
||||
<div id="event-rate-chart" class="event-rate-chart-container">Loading chart...</div>
|
||||
|
||||
<!-- Database Overview Table -->
|
||||
<div class="input-group">
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="stats-overview-table">
|
||||
<thead><tr><th>Metric</th><th>Value</th></tr></thead>
|
||||
<tbody id="stats-overview-table-body">
|
||||
<tr><td>Database Size</td><td id="db-size">-</td></tr>
|
||||
<tr><td>Total Events</td><td id="total-events">-</td></tr>
|
||||
<tr><td>Process ID</td><td id="process-id">-</td></tr>
|
||||
<tr><td>WebSocket Connections</td><td id="websocket-connections">-</td></tr>
|
||||
<tr><td>Active Subscriptions</td><td id="active-subscriptions">-</td></tr>
|
||||
<tr><td>Memory Usage</td><td id="memory-usage">-</td></tr>
|
||||
<tr><td>CPU Usage</td><td id="cpu-usage">-</td></tr>
|
||||
<tr><td>CPU Core</td><td id="cpu-core">-</td></tr>
|
||||
<tr><td>Oldest Event</td><td id="oldest-event">-</td></tr>
|
||||
<tr><td>Newest Event</td><td id="newest-event">-</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Event Kind Distribution Table -->
|
||||
<div class="input-group">
|
||||
<label>Event Kind Distribution:</label>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="stats-kinds-table">
|
||||
<thead><tr><th>Event Kind</th><th>Count</th><th>Percentage</th></tr></thead>
|
||||
<tbody id="stats-kinds-table-body">
|
||||
<tr><td colspan="3" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Time-based Statistics Table -->
|
||||
<div class="input-group">
|
||||
<label>Time-based Statistics:</label>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="stats-time-table">
|
||||
<thead><tr><th>Period</th><th>Events</th></tr></thead>
|
||||
<tbody id="stats-time-table-body">
|
||||
<tr><td>Last 24 Hours</td><td id="events-24h">-</td></tr>
|
||||
<tr><td>Last 7 Days</td><td id="events-7d">-</td></tr>
|
||||
<tr><td>Last 30 Days</td><td id="events-30d">-</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Top Pubkeys Table -->
|
||||
<div class="input-group">
|
||||
<label>Top Pubkeys by Event Count:</label>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="stats-pubkeys-table">
|
||||
<thead><tr><th>Rank</th><th>Name</th><th>Pubkey</th><th>Event Count</th><th>Percentage</th></tr></thead>
|
||||
<tbody id="stats-pubkeys-table-body">
|
||||
<tr><td colspan="5" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Name-field usage stats (from profiles cache) -->
|
||||
<div class="input-group" id="name-field-usage" style="display:none;"></div>
|
||||
</div>
|
||||
|
||||
<!-- SUBSCRIPTION DETAILS Section -->
|
||||
<div class="section flex-section" id="subscriptionDetailsSection" style="display: none;">
|
||||
<div class="section-header">ACTIVE SUBSCRIPTION DETAILS</div>
|
||||
<div class="input-group">
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="subscription-details-table">
|
||||
<thead><tr><th>Sub ID</th><th>Pubkey</th><th>Filters</th><th>Created</th><th>Events Sent</th></tr></thead>
|
||||
<tbody id="subscription-details-table-body">
|
||||
<tr><td colspan="5" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- CONFIGURATION Section -->
|
||||
<div id="div_config" class="section flex-section" style="display: none;">
|
||||
<div class="section-header">RELAY CONFIGURATION</div>
|
||||
<div id="config-display">
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="config-table">
|
||||
<thead><tr><th>Parameter</th><th>Value</th><th>Actions</th></tr></thead>
|
||||
<tbody id="config-table-body">
|
||||
<tr><td colspan="3" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" id="fetch-config-btn" onclick="loadConfig()">REFRESH</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- AUTH RULES Section -->
|
||||
<div class="section flex-section" id="authRulesSection" style="display: none;">
|
||||
<div class="section-header">AUTH RULES MANAGEMENT</div>
|
||||
<div id="authRulesTableContainer">
|
||||
<table class="config-table" id="authRulesTable">
|
||||
<thead><tr><th>Rule Type</th><th>Pattern Type</th><th>Pattern Value</th><th>Status</th><th>Actions</th></tr></thead>
|
||||
<tbody id="authRulesTableBody">
|
||||
<tr><td colspan="5" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="authRuleInputSections" style="display: block;">
|
||||
<div class="input-group">
|
||||
<label for="authRulePubkey">Public Key (npub or hex):</label>
|
||||
<input type="text" id="authRulePubkey" placeholder="npub1... or 64-character hex pubkey">
|
||||
</div>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="addAuthRule('whitelist')">ADD TO WHITELIST</button>
|
||||
<button type="button" onclick="addAuthRule('blacklist')">ADD TO BLACKLIST</button>
|
||||
<button type="button" onclick="loadAuthRules()">REFRESH</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- WEB OF TRUST Section -->
|
||||
<div class="section flex-section" id="wotSection" style="display: none;">
|
||||
<div class="section-header">WEB OF TRUST</div>
|
||||
<div id="wotKind3Status" class="wot-status-row">
|
||||
<span>Admin Contact List (kind 3):</span>
|
||||
<span id="wotKind3Indicator" class="wot-indicator wot-unknown">Loading...</span>
|
||||
</div>
|
||||
<div class="wot-level-selector">
|
||||
<label>WoT Level:</label>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" class="wot-level-btn" onclick="setWotLevel(0)">OFF</button>
|
||||
<button type="button" class="wot-level-btn" onclick="setWotLevel(1)">WRITE ONLY</button>
|
||||
<button type="button" class="wot-level-btn" onclick="setWotLevel(2)">FULL</button>
|
||||
</div>
|
||||
<div class="wot-level-description" id="wotLevelDescription">Loading...</div>
|
||||
</div>
|
||||
<div class="wot-stats-row">
|
||||
<span>Whitelisted Pubkeys:</span>
|
||||
<span id="wotWhitelistCount">—</span>
|
||||
</div>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="syncWot()">SYNC FROM KIND 3</button>
|
||||
<button type="button" onclick="loadWotStatus()">REFRESH STATUS</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- NIP-17 DIRECT MESSAGES Section -->
|
||||
<div class="section" id="nip17DMSection" style="display: none;">
|
||||
<div class="section-header"><h2>NIP-17 DIRECT MESSAGES</h2></div>
|
||||
<div class="input-group">
|
||||
<label>Received Messages (kind 4/14/15):</label>
|
||||
<div id="dm-inbox" class="log-panel" style="height: 300px; overflow-y: auto;">
|
||||
<div class="log-entry">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- IP BANS Section -->
|
||||
<div class="section" id="ipBansSection" style="display: none;">
|
||||
<div class="section-header">IP BAN MANAGEMENT</div>
|
||||
<div class="input-group">
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="ip-bans-stats-table">
|
||||
<thead><tr><th>Total IPs Tracked</th><th>Currently Banned</th><th>Total Bans Issued</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td id="ip-bans-total">-</td><td id="ip-bans-active">-</td><td id="ip-bans-issued">-</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<h3>Manually Ban IP Address</h3>
|
||||
<div class="form-group">
|
||||
<label for="ban-ip-input">IP Address:</label>
|
||||
<input type="text" id="ban-ip-input" placeholder="192.168.1.100">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="ban-duration-select">Ban Duration:</label>
|
||||
<select id="ban-duration-select">
|
||||
<option value="3600">1 Hour</option>
|
||||
<option value="86400" selected>24 Hours</option>
|
||||
<option value="604800">7 Days</option>
|
||||
<option value="2592000">30 Days</option>
|
||||
<option value="31536000">1 Year</option>
|
||||
<option value="999999999">Permanent</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="addBan()">BAN IP</button>
|
||||
<button type="button" onclick="loadIpBans()">REFRESH</button>
|
||||
</div>
|
||||
<div id="add-ban-status" class="status-message"></div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<label>Banned IP Addresses:</label>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="ip-bans-table">
|
||||
<thead><tr><th>IP Address</th><th>Status</th><th>Banned Until</th><th>Failures</th><th>Actions</th></tr></thead>
|
||||
<tbody id="ip-bans-tbody">
|
||||
<tr><td colspan="5" style="text-align: center;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- RELAY EVENTS Section -->
|
||||
<div class="section" id="relayEventsSection" style="display: none;">
|
||||
<div class="section-header">RELAY EVENTS MANAGEMENT</div>
|
||||
<div class="input-group">
|
||||
<h3>Recent Events</h3>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="loadEvents()">REFRESH</button>
|
||||
</div>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="live-relay-events-table">
|
||||
<thead><tr><th>Time</th><th>Kind</th><th>Name</th><th>ID</th><th>Content Preview</th></tr></thead>
|
||||
<tbody id="live-relay-events-table-body">
|
||||
<tr><td colspan="5" style="text-align: center;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- CACHING Section -->
|
||||
<div class="section" id="cachingSection" style="display: none;">
|
||||
<div class="section-header">CACHING</div>
|
||||
<div class="input-group">
|
||||
<h3>Caching Controls</h3>
|
||||
<div id="caching-controls" class="inline-buttons">
|
||||
<span id="caching-enabled-label">Caching: ?</span>
|
||||
<button type="button" id="caching-toggle-btn" onclick="toggleCachingEnabled()">Toggle Caching</button>
|
||||
<span id="caching-inbox-enabled-label">Inbox: ?</span>
|
||||
<button type="button" id="caching-inbox-toggle-btn" onclick="toggleCachingInboxEnabled()">Toggle Inbox</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<h3>Caching Service Status</h3>
|
||||
<div id="caching-service-status" class="status-display">
|
||||
<p>Loading...</p>
|
||||
</div>
|
||||
<div class="inline-buttons" style="margin-top:8px">
|
||||
<button type="button" id="caching-rerun-all-btn" onclick="rerunAllCaching()">Re-run All Caching</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<h3>Relay Inbox Status</h3>
|
||||
<div id="caching-inbox-status" class="status-display">
|
||||
<p>Loading...</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<h3>Followed Pubkeys <span style="font-size:11px;font-weight:normal;color:var(--text-muted,#888)">(click a row for per-relay details)</span></h3>
|
||||
<div id="caching-follows-status" class="status-message"></div>
|
||||
<div class="config-table-container">
|
||||
<table class="config-table" id="caching-follows-table">
|
||||
<thead><tr><th>Name</th><th>npub</th><th>Root?</th><th>Events in DB</th><th>Backfill</th><th>Relays</th></tr></thead>
|
||||
<tbody id="caching-follows-table-body">
|
||||
<tr><td colspan="6" style="text-align: center; font-style: italic;">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="loadCaching()">REFRESH</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- SQL QUERY Section -->
|
||||
<div class="section" id="sqlQuerySection" style="display: none;">
|
||||
<div class="section-header"><h2>SQL QUERY CONSOLE</h2></div>
|
||||
<div class="input-group">
|
||||
<label for="sql-input">SQL Query (SELECT only):</label>
|
||||
<textarea id="sql-input" rows="5" placeholder="SELECT * FROM events LIMIT 10"></textarea>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<div class="inline-buttons">
|
||||
<button type="button" onclick="executeQuery()">EXECUTE QUERY</button>
|
||||
<button type="button" onclick="document.getElementById('sql-input').value=''">CLEAR</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<label>Query Results:</label>
|
||||
<div id="query-info" class="info-box"></div>
|
||||
<div id="query-table" class="config-table-container"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Nostr libraries for nostr_login_lite auth modal -->
|
||||
<script src="assets/nostr.bundle.js"></script>
|
||||
<script src="assets/nostr-lite.js"></script>
|
||||
<script src="assets/app.js"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,147 @@
|
||||
<?php
|
||||
/**
|
||||
* admin2/lib/ascii_chart.php — Server-side ASCII X-bar chart renderer.
|
||||
*
|
||||
* Produces a monospaced ASCII bar chart (using 'X' characters) from an
|
||||
* array of bin counts. Mirrors the layout of the original text_graph.js
|
||||
* ASCIIBarChart: Y-axis count labels, X-bar columns, X-axis with
|
||||
* elapsed-time labels.
|
||||
*
|
||||
* The output is a plain text string suitable for both browser <div>
|
||||
* injection (white-space: pre) and terminal display via curl.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Render an ASCII bar chart from an array of bin counts.
|
||||
*
|
||||
* @param array $bins Array of integer counts (one per time bin, left=oldest)
|
||||
* @param array $options {
|
||||
* @var string $title Chart title (centered at top)
|
||||
* @var int $max_height Chart height in rows (default 11)
|
||||
* @var string $x_axis_label Label below the X-axis (default '')
|
||||
* @var int $bin_duration Seconds per bin — for X-axis elapsed labels
|
||||
* @var int $label_interval Label every N bins (default 5)
|
||||
* }
|
||||
* @return string The ASCII chart as a multi-line string.
|
||||
*/
|
||||
function render_ascii_chart(array $bins, array $options = []): string {
|
||||
$title = $options['title'] ?? 'New Events';
|
||||
$max_height = $options['max_height'] ?? 11;
|
||||
$x_axis_label = $options['x_axis_label'] ?? '';
|
||||
$bin_duration = $options['bin_duration'] ?? 10;
|
||||
$label_interval = $options['label_interval'] ?? 5;
|
||||
|
||||
$num_bins = count($bins);
|
||||
if ($num_bins === 0) {
|
||||
return "No data available.\n";
|
||||
}
|
||||
|
||||
$max_count = max($bins);
|
||||
if ($max_count < 1) $max_count = 1; // Avoid division by zero for all-empty bins
|
||||
|
||||
// Scaling: each X represents scale_factor counts
|
||||
$scale_factor = max(1, (int)ceil($max_count / $max_height));
|
||||
$scaled_max = (int)ceil($max_count / $scale_factor) * $scale_factor;
|
||||
|
||||
$output = '';
|
||||
|
||||
// --- Title (centered) ---
|
||||
$chart_width = 4 + $num_bins; // 4 = Y-axis number width (3) + separator (1)
|
||||
if ($title !== '') {
|
||||
$title_padding = (int)floor(($chart_width - strlen($title)) / 2);
|
||||
if ($title_padding < 0) $title_padding = 0;
|
||||
$output .= str_repeat(' ', $title_padding) . $title . "\n\n";
|
||||
}
|
||||
|
||||
// --- Bar rows (top to bottom) ---
|
||||
for ($row = $max_height; $row > 0; $row--) {
|
||||
$row_count = ($row - 1) * $scale_factor + 1;
|
||||
$line = str_pad((string)$row_count, 3, ' ', STR_PAD_LEFT) . ' |';
|
||||
|
||||
for ($i = 0; $i < $num_bins; $i++) {
|
||||
$count = $bins[$i];
|
||||
$scaled_height = ($count > 0) ? (int)ceil($count / $scale_factor) : 0;
|
||||
$line .= ($scaled_height >= $row) ? 'X' : ' ';
|
||||
}
|
||||
$output .= $line . "\n";
|
||||
}
|
||||
|
||||
// --- X-axis line ---
|
||||
$output .= ' +' . str_repeat('-', $num_bins) . "\n";
|
||||
|
||||
// --- X-axis labels (elapsed time every label_interval bins) ---
|
||||
$label_line = ' ';
|
||||
$labels = [];
|
||||
for ($i = 0; $i < $num_bins; $i++) {
|
||||
if ($i % $label_interval === 0) {
|
||||
$elapsed_sec = $i * $bin_duration;
|
||||
$labels[] = format_elapsed_time($elapsed_sec);
|
||||
}
|
||||
}
|
||||
// Build label line with spacing
|
||||
for ($i = 0; $i < count($labels); $i++) {
|
||||
$label_line .= $labels[$i];
|
||||
if ($i < count($labels) - 1) {
|
||||
$spacing = $label_interval - strlen($labels[$i]);
|
||||
if ($spacing < 1) $spacing = 1;
|
||||
$label_line .= str_repeat(' ', $spacing);
|
||||
}
|
||||
}
|
||||
// Pad to match X-axis dash line length
|
||||
$min_label_len = 4 + $num_bins;
|
||||
if (strlen($label_line) < $min_label_len) {
|
||||
$label_line .= str_repeat(' ', $min_label_len - strlen($label_line));
|
||||
}
|
||||
$output .= $label_line . "\n";
|
||||
|
||||
// --- X-axis label (if provided) ---
|
||||
if ($x_axis_label !== '') {
|
||||
$label_pad = (int)floor(($num_bins - strlen($x_axis_label)) / 2);
|
||||
if ($label_pad < 0) $label_pad = 0;
|
||||
$output .= "\n" . ' ' . str_repeat(' ', $label_pad) . $x_axis_label . "\n";
|
||||
}
|
||||
|
||||
return $output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an elapsed time in seconds as a compact label.
|
||||
* < 60s → "Ns"
|
||||
* < 1h → "Nm"
|
||||
* < 1d → "Nh"
|
||||
* else → "Nd"
|
||||
*/
|
||||
function format_elapsed_time(int $seconds): string {
|
||||
if ($seconds < 60) {
|
||||
return $seconds . 's';
|
||||
} elseif ($seconds < 3600) {
|
||||
return (int)floor($seconds / 60) . 'm';
|
||||
} elseif ($seconds < 86400) {
|
||||
return (int)floor($seconds / 3600) . 'h';
|
||||
} else {
|
||||
return (int)floor($seconds / 86400) . 'd';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a fixed-length bin array from SQL query rows.
|
||||
*
|
||||
* The query returns only non-empty bins. This function creates a
|
||||
* zero-filled array of $num_bins length and overlays the counts
|
||||
* at the correct positions, so empty time slots show as blank
|
||||
* columns — the chart always advances in time.
|
||||
*
|
||||
* @param array $rows Query rows with 'bin' (int) and 'cnt' (int)
|
||||
* @param int $num_bins Total number of bins (fixed length)
|
||||
* @return array Zero-filled array of counts
|
||||
*/
|
||||
function build_bin_array(array $rows, int $num_bins): array {
|
||||
$bins = array_fill(0, $num_bins, 0);
|
||||
foreach ($rows as $r) {
|
||||
$idx = (int)$r['bin'];
|
||||
if ($idx >= 0 && $idx < $num_bins) {
|
||||
$bins[$idx] = (int)$r['cnt'];
|
||||
}
|
||||
}
|
||||
return $bins;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
<?php
|
||||
/**
|
||||
* C-Relay-PG Admin — Database connection configuration.
|
||||
*
|
||||
* This file is NOT web-accessible (denied by nginx location block).
|
||||
* It contains the PostgreSQL connection string used by all admin pages.
|
||||
*
|
||||
* Edit these values to match your relay's PostgreSQL credentials.
|
||||
*/
|
||||
|
||||
// PostgreSQL connection parameters.
|
||||
// These should match the --db-* flags passed to the c-relay-pg binary
|
||||
// or the connection string passed to caching_relay -p "...".
|
||||
return [
|
||||
'db_host' => 'localhost',
|
||||
'db_port' => '5432',
|
||||
'db_name' => 'crelay',
|
||||
'db_user' => 'crelay',
|
||||
'db_password' => 'crelay',
|
||||
|
||||
// Page sizes for paginated tables.
|
||||
'follows_per_page' => 50,
|
||||
'relays_per_page' => 50,
|
||||
|
||||
// Auto-refresh interval for dashboard polling (milliseconds).
|
||||
'refresh_ms' => 10000,
|
||||
];
|
||||
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
/**
|
||||
* C-Relay-PG Admin — PDO database connection helper.
|
||||
*
|
||||
* Provides a singleton PDO connection to the relay's PostgreSQL database.
|
||||
* All admin pages use db() to get the connection.
|
||||
*/
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
function db(): PDO {
|
||||
static $pdo = null;
|
||||
if ($pdo === null) {
|
||||
$cfg = require __DIR__ . '/config.php';
|
||||
// Build DSN — supports both TCP (host=localhost) and Unix socket
|
||||
// (host=/var/run/postgresql). Omit password if empty (peer auth).
|
||||
$dsn = sprintf(
|
||||
'pgsql:host=%s;port=%d;dbname=%s;user=%s',
|
||||
$cfg['db_host'],
|
||||
$cfg['db_port'],
|
||||
$cfg['db_name'],
|
||||
$cfg['db_user']
|
||||
);
|
||||
if (!empty($cfg['db_password'])) {
|
||||
$dsn .= ';password=' . $cfg['db_password'];
|
||||
}
|
||||
try {
|
||||
$pdo = new PDO($dsn, null, null, [
|
||||
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
||||
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
||||
PDO::ATTR_EMULATE_PREPARES => false,
|
||||
]);
|
||||
} catch (PDOException $e) {
|
||||
http_response_code(500);
|
||||
die('Database connection failed: ' . htmlspecialchars($e->getMessage()));
|
||||
}
|
||||
}
|
||||
return $pdo;
|
||||
}
|
||||
|
||||
/** Get a config value from the admin config file. */
|
||||
function cfg(string $key, $default = null) {
|
||||
static $cfg = null;
|
||||
if ($cfg === null) {
|
||||
$cfg = require __DIR__ . '/config.php';
|
||||
}
|
||||
return $cfg[$key] ?? $default;
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
<?php
|
||||
/**
|
||||
* C-Relay-PG Admin — Shared helper functions.
|
||||
*/
|
||||
|
||||
require_once __DIR__ . '/db.php';
|
||||
|
||||
/** HTML-escape a string. */
|
||||
function e(?string $s): string {
|
||||
return htmlspecialchars($s ?? '', ENT_QUOTES, 'UTF-8');
|
||||
}
|
||||
|
||||
/** Format a Unix timestamp as a human-readable relative time. */
|
||||
function time_ago(int $ts): string {
|
||||
if ($ts <= 0) return 'never';
|
||||
$diff = time() - $ts;
|
||||
if ($diff < 60) return $diff . 's ago';
|
||||
if ($diff < 3600) return floor($diff / 60) . 'm ago';
|
||||
if ($diff < 86400) return floor($diff / 3600) . 'h ago';
|
||||
return floor($diff / 86400) . 'd ago';
|
||||
}
|
||||
|
||||
/** Format a Unix timestamp as a date/time string. */
|
||||
function fmt_date(int $ts): string {
|
||||
if ($ts <= 0) return '-';
|
||||
return date('Y-m-d H:i:s', $ts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a 64-char hex pubkey to npub (bech32).
|
||||
* Uses the standard bech32 encoding (NIP-19).
|
||||
*/
|
||||
function hex_to_npub(string $hex): string {
|
||||
if (!preg_match('/^[0-9a-fA-F]{64}$/', $hex)) return $hex;
|
||||
$hex = strtolower($hex);
|
||||
|
||||
// Bech32 constants
|
||||
$charset = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
|
||||
$data = [];
|
||||
for ($i = 0; $i < strlen($hex); $i += 2) {
|
||||
$data[] = intval(substr($hex, $i, 2), 16);
|
||||
}
|
||||
// Convert 8-bit groups to 5-bit groups
|
||||
$conv = [];
|
||||
$buffer = 0;
|
||||
$bits = 0;
|
||||
foreach ($data as $byte) {
|
||||
$buffer = ($buffer << 8) | $byte;
|
||||
$bits += 8;
|
||||
while ($bits >= 5) {
|
||||
$conv[] = ($buffer >> ($bits - 5)) & 31;
|
||||
$bits -= 5;
|
||||
}
|
||||
}
|
||||
if ($bits > 0) {
|
||||
$conv[] = ($buffer << (5 - $bits)) & 31;
|
||||
}
|
||||
|
||||
// Compute checksum — HRP values are positions of each char in the charset.
|
||||
$values = array_merge(
|
||||
array_map(function($c) {
|
||||
return strpos('qpzry9x8gf2tvdw0s3jn54khce6mua7l', $c);
|
||||
}, str_split('npub')),
|
||||
$conv
|
||||
);
|
||||
$gen = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3];
|
||||
$chk = 1;
|
||||
foreach ($values as $v) {
|
||||
$top = $chk >> 25;
|
||||
$chk = (($chk & 0x1ffffff) << 5) ^ $v;
|
||||
for ($i = 0; $i < 5; $i++) {
|
||||
if (($top >> $i) & 1) $chk ^= $gen[$i];
|
||||
}
|
||||
}
|
||||
$chk ^= 1;
|
||||
$checksum = [];
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
$checksum[] = ($chk >> (5 * (5 - $i))) & 31;
|
||||
}
|
||||
|
||||
$result = 'npub1';
|
||||
foreach (array_merge($conv, $checksum) as $v) {
|
||||
$result .= $charset[$v];
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/** Truncate a string to $len chars with ellipsis. */
|
||||
function trunc(string $s, int $len = 16): string {
|
||||
if (strlen($s) <= $len) return $s;
|
||||
return substr($s, 0, $len) . '…';
|
||||
}
|
||||
|
||||
/** Build pagination links HTML. */
|
||||
function pagination(int $page, int $per_page, int $total, string $base_url): string {
|
||||
$total_pages = max(1, ceil($total / $per_page));
|
||||
if ($total_pages <= 1) return '';
|
||||
|
||||
$html = '<div class="pagination">';
|
||||
if ($page > 1) {
|
||||
$html .= '<a href="' . e($base_url) . '?page=' . ($page - 1) . '">← Prev</a>';
|
||||
}
|
||||
$html .= '<span>Page ' . $page . ' of ' . $total_pages . ' (' . $total . ' total)</span>';
|
||||
if ($page < $total_pages) {
|
||||
$html .= '<a href="' . e($base_url) . '?page=' . ($page + 1) . '">Next →</a>';
|
||||
}
|
||||
$html .= '</div>';
|
||||
return $html;
|
||||
}
|
||||
|
||||
/** Get the current page number from the query string. */
|
||||
function current_page(): int {
|
||||
$p = intval($_GET['page'] ?? 1);
|
||||
return max(1, $p);
|
||||
}
|
||||
|
||||
/** JSON response helper for AJAX endpoints. */
|
||||
function json_response($data): void {
|
||||
header('Content-Type: application/json');
|
||||
echo json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
exit;
|
||||
}
|
||||
|
||||
/** Get a query parameter with a default. */
|
||||
function query_param(string $key, $default = null) {
|
||||
return $_GET[$key] ?? $default;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch-resolve profiles from the profiles cache table.
|
||||
* Returns [pubkey_hex => ['name'=>..., 'display_name'=>...,
|
||||
* 'best_name'=>..., 'picture'=>..., 'nip05'=>...]].
|
||||
* Pubkeys with no cached profile are absent from the result.
|
||||
*/
|
||||
function profile_map(array $pubkeys): array {
|
||||
if (empty($pubkeys)) return [];
|
||||
$pdo = db();
|
||||
$map = [];
|
||||
try {
|
||||
// Build parameterized IN clause: WHERE pubkey = ANY(?)
|
||||
// PDO PostgreSQL supports passing an array as a string literal.
|
||||
$placeholders = implode(',', array_fill(0, count($pubkeys), '?'));
|
||||
$stmt = $pdo->prepare(
|
||||
"SELECT pubkey, name, display_name, picture, nip05 FROM profiles "
|
||||
. "WHERE pubkey IN ($placeholders)"
|
||||
);
|
||||
$stmt->execute(array_values($pubkeys));
|
||||
$rows = $stmt->fetchAll();
|
||||
foreach ($rows as $r) {
|
||||
$map[$r['pubkey']] = [
|
||||
'name' => $r['name'] ?? '',
|
||||
'display_name' => $r['display_name'] ?? '',
|
||||
'best_name' => profile_display_name($r),
|
||||
'picture' => $r['picture'] ?? '',
|
||||
'nip05' => $r['nip05'] ?? '',
|
||||
];
|
||||
}
|
||||
} catch (PDOException $e) {}
|
||||
return $map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the profile_name_preference config key to resolve the display name.
|
||||
* Never returns null. Returns '' if neither field is set.
|
||||
*/
|
||||
function profile_display_name(array $profile): string {
|
||||
$name = $profile['name'] ?? '';
|
||||
$display_name = $profile['display_name'] ?? '';
|
||||
static $pref = null;
|
||||
if ($pref === null) {
|
||||
try {
|
||||
$pdo = db();
|
||||
$pref = $pdo->query("SELECT value FROM config WHERE key = 'profile_name_preference'")->fetchColumn() ?: 'display_name';
|
||||
} catch (PDOException $e) {
|
||||
$pref = 'display_name';
|
||||
}
|
||||
}
|
||||
if ($pref === 'name') {
|
||||
return $name !== '' ? $name : $display_name;
|
||||
}
|
||||
// Default: prefer display_name, fall back to name.
|
||||
return $display_name !== '' ? $display_name : $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the shared HTML header with side navigation.
|
||||
* Pass the active page name to highlight the current nav item.
|
||||
*/
|
||||
function admin_header(string $active = '', string $title = 'C-Relay-PG Admin'): void {
|
||||
$pages = [
|
||||
'dashboard' => ['index.php', 'Dashboard'],
|
||||
'follows' => ['follows.php', 'Follows'],
|
||||
'relays' => ['relays.php', 'Relays'],
|
||||
'config' => ['config-edit.php', 'Config'],
|
||||
'inbox' => ['inbox.php', 'Inbox'],
|
||||
];
|
||||
?>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title><?= e($title) ?></title>
|
||||
<link rel="stylesheet" href="assets/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<!-- Side Navigation -->
|
||||
<nav class="side-nav" id="side-nav">
|
||||
<ul class="nav-menu">
|
||||
<?php foreach ($pages as $key => [$url, $label]): ?>
|
||||
<li><a class="nav-item<?= $active === $key ? ' active' : '' ?>" href="<?= e($url) ?>"><?= e($label) ?></a></li>
|
||||
<?php endforeach; ?>
|
||||
</ul>
|
||||
</nav>
|
||||
<div class="side-nav-overlay" id="side-nav-overlay" onclick="closeNav()"></div>
|
||||
|
||||
<!-- Header -->
|
||||
<div class="main-header">
|
||||
<div class="header-content">
|
||||
<button class="menu-btn" onclick="openNav()">☰</button>
|
||||
<div class="header-title" onclick="location.href='index.php'">
|
||||
<span class="relay-letter">R</span><span class="relay-letter">E</span><span class="relay-letter">L</span><span class="relay-letter">A</span><span class="relay-letter">Y</span>
|
||||
</div>
|
||||
<div style="width:40px"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function openNav() {
|
||||
document.getElementById('side-nav').classList.add('open');
|
||||
document.getElementById('side-nav-overlay').classList.add('show');
|
||||
}
|
||||
function closeNav() {
|
||||
document.getElementById('side-nav').classList.remove('open');
|
||||
document.getElementById('side-nav-overlay').classList.remove('show');
|
||||
}
|
||||
</script>
|
||||
<?php
|
||||
}
|
||||
|
||||
/** Render the shared HTML footer. */
|
||||
function admin_footer(): void {
|
||||
?>
|
||||
</body>
|
||||
</html>
|
||||
<?php
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
# Launch the admin2 PHP built-in server, fully detached from the calling shell.
|
||||
# Usage: ./admin2/serve.sh [port] (default port 8088)
|
||||
PORT="${1:-8088}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# Kill any existing instance on this port.
|
||||
pkill -f "php -S 127.0.0.1:$PORT" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
# Start fully detached in its own session so SIGTERM to the launcher
|
||||
# does not propagate to the PHP server.
|
||||
setsid php -S 127.0.0.1:$PORT -t "$SCRIPT_DIR" > "$SCRIPT_DIR/php_server.log" 2>&1 < /dev/null &
|
||||
PHP_PID=$!
|
||||
disown $PHP_PID 2>/dev/null
|
||||
|
||||
# Give it a moment to bind, then report.
|
||||
sleep 1
|
||||
if kill -0 "$PHP_PID" 2>/dev/null; then
|
||||
echo "admin PHP server started on http://127.0.0.1:$PORT (PID $PHP_PID)"
|
||||
echo "Serving from: $SCRIPT_DIR"
|
||||
echo "Log: $SCRIPT_DIR/php_server.log"
|
||||
else
|
||||
echo "ERROR: PHP server failed to start. Log:"
|
||||
cat "$SCRIPT_DIR/php_server.log" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
+161
-1
@@ -18,6 +18,7 @@
|
||||
<li><button class="nav-item" data-page="authorization">Authorization</button></li>
|
||||
<li><button class="nav-item" data-page="ip-bans">IP BANS</button></li>
|
||||
<li><button class="nav-item" data-page="relay-events">Relay Events</button></li>
|
||||
<li><button class="nav-item" data-page="caching">Caching</button></li>
|
||||
<li><button class="nav-item" data-page="dm">DM</button></li>
|
||||
<li><button class="nav-item" data-page="database">Database Query</button></li>
|
||||
</ul>
|
||||
@@ -203,6 +204,7 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Rank</th>
|
||||
<th>Name</th>
|
||||
<th>Pubkey</th>
|
||||
<th>Event Count</th>
|
||||
<th>Percentage</th>
|
||||
@@ -210,7 +212,7 @@
|
||||
</thead>
|
||||
<tbody id="stats-pubkeys-table-body">
|
||||
<tr>
|
||||
<td colspan="4" style="text-align: center; font-style: italic;">No data loaded</td>
|
||||
<td colspan="5" style="text-align: center; font-style: italic;">No data loaded</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -579,6 +581,164 @@ WEB OF TRUST
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- CACHING Section -->
|
||||
<div class="section" id="cachingSection" style="display: none;">
|
||||
<div class="section-header">CACHING</div>
|
||||
|
||||
<!-- Caching Service Status (read-only) -->
|
||||
<div class="input-group">
|
||||
<h3>Caching Service Status</h3>
|
||||
<div id="caching-service-status" class="status-display">
|
||||
<p>Service status will appear here after connecting.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Relay Inbox Status (read-only) -->
|
||||
<div class="input-group">
|
||||
<h3>Relay Inbox Status</h3>
|
||||
<div id="caching-inbox-status" class="status-display">
|
||||
<p>Inbox status will appear here after connecting.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Caching Configuration (editable) -->
|
||||
<div class="input-group">
|
||||
<h3>Caching Configuration</h3>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-enabled">Enable Caching:</label>
|
||||
<select id="caching-enabled">
|
||||
<option value="false">Disabled</option>
|
||||
<option value="true">Enabled</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-inbox-enabled">Enable Inbox Consumer:</label>
|
||||
<select id="caching-inbox-enabled">
|
||||
<option value="false">Disabled</option>
|
||||
<option value="true">Enabled</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-root-npubs">Root Npubs (one per line):</label>
|
||||
<textarea id="caching-root-npubs" rows="4" placeholder="npub1...">npub13lm5wf8dvsdnc2894pkhch9uf8phvw9varrv8zf4sc885hhdmc8q6lx7ks</textarea>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-bootstrap-relays">Bootstrap Relays (one per line):</label>
|
||||
<textarea id="caching-bootstrap-relays" rows="4" placeholder="wss://relay.damus.io">wss://relay.damus.io
|
||||
wss://nos.lol
|
||||
wss://relay.primal.net
|
||||
wss://laantungir.net/relay</textarea>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-kinds">Kinds (comma-separated):</label>
|
||||
<input type="text" id="caching-kinds" placeholder="1,3,6,10000,30023" value="0,1,3,6,10000,10002,30023">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-admin-kinds">Admin Kinds (comma-separated, * for all):</label>
|
||||
<input type="text" id="caching-admin-kinds" placeholder="*">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-live-enabled">Live Subscriptions:</label>
|
||||
<select id="caching-live-enabled">
|
||||
<option value="true">Enabled</option>
|
||||
<option value="false">Disabled</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-backfill-enabled">Backfill:</label>
|
||||
<select id="caching-backfill-enabled">
|
||||
<option value="true">Enabled</option>
|
||||
<option value="false">Disabled</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-backfill-windows">Backfill Windows (seconds, comma-separated):</label>
|
||||
<input type="text" id="caching-backfill-windows" placeholder="86400,604800,2592000,7776000,31536000">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-backfill-page-size">Backfill Page Size:</label>
|
||||
<input type="number" id="caching-backfill-page-size" placeholder="50">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-backfill-tick-interval">Backfill Tick Interval (ms):</label>
|
||||
<input type="number" id="caching-backfill-tick-interval" placeholder="5000">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-inbox-batch-size">Inbox Batch Size:</label>
|
||||
<input type="number" id="caching-inbox-batch-size" placeholder="150">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-inbox-active-poll">Inbox Active Poll (ms):</label>
|
||||
<input type="number" id="caching-inbox-active-poll" placeholder="200">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-inbox-idle-poll">Inbox Idle Poll (ms):</label>
|
||||
<input type="number" id="caching-inbox-idle-poll" placeholder="5000">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-service-binary-path">Caching Service Binary Path:</label>
|
||||
<input type="text" id="caching-service-binary-path" placeholder="./caching_relay" value="./caching_relay">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="caching-service-pg-conn">Caching Service PG Connection:</label>
|
||||
<input type="text" id="caching-service-pg-conn" placeholder="host=localhost port=5432 dbname=crelay user=crelay password=crelay" value="host=localhost port=5432 dbname=crelay user=crelay password=crelay">
|
||||
</div>
|
||||
|
||||
<div class="inline-buttons">
|
||||
<button type="button" id="caching-apply-btn">APPLY CONFIGURATION</button>
|
||||
<button type="button" id="caching-reset-progress-btn">RESET BACKFILL PROGRESS</button>
|
||||
</div>
|
||||
<div id="caching-config-status" class="status-message"></div>
|
||||
</div>
|
||||
|
||||
<div class="input-group">
|
||||
<h3>Caching Service Control</h3>
|
||||
<p>Start or stop the external caching service process. Set <code>caching_service_binary_path</code> and <code>caching_service_pg_conn</code> above and click APPLY CONFIGURATION before starting.</p>
|
||||
<div class="inline-buttons">
|
||||
<button type="button" id="caching-start-service-btn">START CACHING SERVICE</button>
|
||||
<button type="button" id="caching-stop-service-btn">STOP CACHING SERVICE</button>
|
||||
</div>
|
||||
<div id="caching-service-control-status" class="status-message"></div>
|
||||
</div>
|
||||
|
||||
<div class="input-group">
|
||||
<h3>Followed Pubkeys</h3>
|
||||
<p>Pubkeys being cached with their profile names, event counts, outbox relays, and backfill status.</p>
|
||||
<div id="caching-follows-status" class="status-message"></div>
|
||||
<table id="caching-follows-table" style="width: 100%; border-collapse: collapse; font-size: 13px;">
|
||||
<thead>
|
||||
<tr style="border-bottom: 2px solid var(--border-color, #444); text-align: left;">
|
||||
<th style="padding: 6px 8px;">Name</th>
|
||||
<th style="padding: 6px 8px;">npub</th>
|
||||
<th style="padding: 6px 8px;">Root?</th>
|
||||
<th style="padding: 6px 8px;">Events in DB</th>
|
||||
<th style="padding: 6px 8px;">Backfill</th>
|
||||
<th style="padding: 6px 8px; width: 40%;">Relays (status, events fetched)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="caching-follows-table-body">
|
||||
<tr><td colspan="6" style="text-align: center; font-style: italic; padding: 12px;">Loading follows status...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- SQL QUERY Section -->
|
||||
<div class="section" id="sqlQuerySection" style="display: none;">
|
||||
<div class="section-header">
|
||||
|
||||
+681
-3
@@ -19,6 +19,8 @@ let nlLite = null;
|
||||
let userPubkey = null;
|
||||
let isLoggedIn = false;
|
||||
let currentConfig = null;
|
||||
// Caching page periodic refresh interval handle
|
||||
let cachingRefreshInterval = null;
|
||||
// Global subscription state
|
||||
let relayPool = null;
|
||||
let subscriptionId = null;
|
||||
@@ -2076,6 +2078,25 @@ function handleSystemCommandResponse(responseData) {
|
||||
handleWotStatusResponse(responseData);
|
||||
}
|
||||
|
||||
// Handle caching status response
|
||||
if (responseData.command === 'caching_status') {
|
||||
handleCachingStatusResponse(responseData);
|
||||
}
|
||||
|
||||
// Handle caching reset progress response
|
||||
if (responseData.command === 'caching_reset_progress') {
|
||||
if (responseData.status === 'success') {
|
||||
log('Backfill progress reset successfully', 'INFO');
|
||||
} else {
|
||||
log(`Failed to reset backfill progress: ${responseData.message || 'Unknown error'}`, 'ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Handle caching follows status response
|
||||
if (responseData.command === 'caching_follows_status') {
|
||||
handleCachingFollowsResponse(responseData);
|
||||
}
|
||||
|
||||
if (typeof logTestEvent === 'function') {
|
||||
logTestEvent('RECV', `System command response: ${responseData.command} - ${responseData.status}`, 'SYSTEM_CMD');
|
||||
}
|
||||
@@ -4833,7 +4854,7 @@ function populateStatsPubkeys(data) {
|
||||
|
||||
if (data.top_pubkeys.length === 0) {
|
||||
const row = document.createElement('tr');
|
||||
row.innerHTML = '<td colspan="4" style="text-align: center; font-style: italic;">No pubkey data</td>';
|
||||
row.innerHTML = '<td colspan="5" style="text-align: center; font-style: italic;">No pubkey data</td>';
|
||||
tableBody.appendChild(row);
|
||||
return;
|
||||
}
|
||||
@@ -4852,8 +4873,12 @@ function populateStatsPubkeys(data) {
|
||||
} catch (error) {
|
||||
console.log('Failed to encode pubkey to npub:', error.message);
|
||||
}
|
||||
// Name from kind-0 metadata (enriched by backend)
|
||||
const name = pubkey.name || '';
|
||||
const nameDisplay = name ? escapeHtml(name) : '<span style="color: var(--text-muted, #888); font-style: italic;">unknown</span>';
|
||||
row.innerHTML = `
|
||||
<td>${index + 1}</td>
|
||||
<td>${nameDisplay}</td>
|
||||
<td style="font-family: 'Courier New', monospace; font-size: 12px; word-break: break-all;">${npubLink}</td>
|
||||
<td>${pubkey.event_count}</td>
|
||||
<td>${pubkey.percentage}%</td>
|
||||
@@ -4871,7 +4896,7 @@ function populateStatsPubkeysFromMonitoring(pubkeysData, totalEvents) {
|
||||
|
||||
if (pubkeysData.length === 0) {
|
||||
const row = document.createElement('tr');
|
||||
row.innerHTML = '<td colspan="4" style="text-align: center; font-style: italic;">No pubkey data</td>';
|
||||
row.innerHTML = '<td colspan="5" style="text-align: center; font-style: italic;">No pubkey data</td>';
|
||||
tableBody.appendChild(row);
|
||||
return;
|
||||
}
|
||||
@@ -4894,8 +4919,13 @@ function populateStatsPubkeysFromMonitoring(pubkeysData, totalEvents) {
|
||||
// Calculate percentage using totalEvents parameter
|
||||
const percentage = totalEvents > 0 ? ((pubkey.event_count / totalEvents) * 100).toFixed(1) : '0.0';
|
||||
|
||||
// Name from kind-0 metadata (enriched by backend)
|
||||
const name = pubkey.name || '';
|
||||
const nameDisplay = name ? escapeHtml(name) : '<span style="color: var(--text-muted, #888); font-style: italic;">unknown</span>';
|
||||
|
||||
row.innerHTML = `
|
||||
<td>${index + 1}</td>
|
||||
<td>${nameDisplay}</td>
|
||||
<td style="font-family: 'Courier New', monospace; font-size: 12px; word-break: break-all;">${npubLink}</td>
|
||||
<td>${pubkey.event_count}</td>
|
||||
<td>${percentage}%</td>
|
||||
@@ -5368,6 +5398,7 @@ function switchPage(pageName) {
|
||||
'wotSection',
|
||||
'ipBansSection',
|
||||
'relayEventsSection',
|
||||
'cachingSection',
|
||||
'nip17DMSection',
|
||||
'sqlQuerySection'
|
||||
];
|
||||
@@ -5386,6 +5417,7 @@ function switchPage(pageName) {
|
||||
'configuration': 'div_config',
|
||||
'ip-bans': 'ipBansSection',
|
||||
'relay-events': 'relayEventsSection',
|
||||
'caching': 'cachingSection',
|
||||
'dm': 'nip17DMSection',
|
||||
'database': 'sqlQuerySection'
|
||||
};
|
||||
@@ -5423,6 +5455,36 @@ function switchPage(pageName) {
|
||||
});
|
||||
}
|
||||
|
||||
// Special handling for caching page - fetch status and start periodic refresh
|
||||
if (pageName === 'caching') {
|
||||
fetchCachingStatus().catch(error => {
|
||||
console.log('Failed to refresh caching status on page switch: ' + error.message);
|
||||
});
|
||||
fetchCachingFollowsStatus().catch(error => {
|
||||
console.log('Failed to refresh caching follows status on page switch: ' + error.message);
|
||||
});
|
||||
// Start periodic refresh every 10 seconds while the caching page is visible
|
||||
if (cachingRefreshInterval) {
|
||||
clearInterval(cachingRefreshInterval);
|
||||
}
|
||||
cachingRefreshInterval = setInterval(() => {
|
||||
if (currentPage === 'caching') {
|
||||
fetchCachingStatus().catch(error => {
|
||||
console.log('Periodic caching status refresh failed: ' + error.message);
|
||||
});
|
||||
fetchCachingFollowsStatus().catch(error => {
|
||||
console.log('Periodic caching follows refresh failed: ' + error.message);
|
||||
});
|
||||
}
|
||||
}, 10000);
|
||||
} else {
|
||||
// Clear periodic refresh when leaving the caching page
|
||||
if (cachingRefreshInterval) {
|
||||
clearInterval(cachingRefreshInterval);
|
||||
cachingRefreshInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Close side navigation
|
||||
closeSideNav();
|
||||
|
||||
@@ -6776,4 +6838,620 @@ function initIpBansEventListeners() {
|
||||
}
|
||||
|
||||
// Initialize when DOM is ready
|
||||
document.addEventListener('DOMContentLoaded', initIpBansEventListeners);
|
||||
document.addEventListener('DOMContentLoaded', initIpBansEventListeners);
|
||||
|
||||
// ================================
|
||||
// CACHING PAGE FUNCTIONS
|
||||
// ================================
|
||||
|
||||
// Mapping of caching config keys to form field ids and metadata
|
||||
const CACHING_CONFIG_FIELDS = [
|
||||
{ key: 'caching_enabled', field: 'caching-enabled', type: 'select' },
|
||||
{ key: 'caching_inbox_enabled', field: 'caching-inbox-enabled', type: 'select' },
|
||||
{ key: 'caching_root_npubs', field: 'caching-root-npubs', type: 'textarea-list' },
|
||||
{ key: 'caching_bootstrap_relays', field: 'caching-bootstrap-relays', type: 'textarea-list' },
|
||||
{ key: 'caching_kinds', field: 'caching-kinds', type: 'string' },
|
||||
{ key: 'caching_admin_kinds', field: 'caching-admin-kinds', type: 'string' },
|
||||
{ key: 'caching_live_enabled', field: 'caching-live-enabled', type: 'select' },
|
||||
{ key: 'caching_backfill_enabled', field: 'caching-backfill-enabled', type: 'select' },
|
||||
{ key: 'caching_backfill_windows', field: 'caching-backfill-windows', type: 'string' },
|
||||
{ key: 'caching_backfill_page_size', field: 'caching-backfill-page-size', type: 'integer' },
|
||||
{ key: 'caching_backfill_tick_interval_ms',field: 'caching-backfill-tick-interval', type: 'integer' },
|
||||
{ key: 'caching_inbox_batch_size', field: 'caching-inbox-batch-size', type: 'integer' },
|
||||
{ key: 'caching_inbox_active_poll_ms', field: 'caching-inbox-active-poll', type: 'integer' },
|
||||
{ key: 'caching_inbox_idle_poll_ms', field: 'caching-inbox-idle-poll', type: 'integer' },
|
||||
{ key: 'caching_service_binary_path', field: 'caching-service-binary-path', type: 'string' },
|
||||
{ key: 'caching_service_pg_conn', field: 'caching-service-pg-conn', type: 'string' }
|
||||
];
|
||||
|
||||
// Helper: read a config value from currentConfig (which stores values in tags as [key, value])
|
||||
function getCachingConfigValue(key) {
|
||||
if (!currentConfig || !currentConfig.tags) return null;
|
||||
for (const tag of currentConfig.tags) {
|
||||
if (Array.isArray(tag) && tag.length >= 2 && tag[0] === key) {
|
||||
return tag[1];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Fetch caching status: populates form fields from config and queries service/inbox status
|
||||
async function fetchCachingStatus() {
|
||||
try {
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
throw new Error('Must be logged in to fetch caching status');
|
||||
}
|
||||
|
||||
// Reuse the existing config query infrastructure to refresh currentConfig.
|
||||
// fetchConfiguration() sends the config_query command but returns before
|
||||
// the response arrives (the response updates currentConfig asynchronously
|
||||
// via handleConfigQueryResponse -> displayConfiguration). We capture a
|
||||
// marker so we can detect when currentConfig has been refreshed.
|
||||
const prevConfigId = currentConfig ? (currentConfig.id || null) : null;
|
||||
await fetchConfiguration();
|
||||
|
||||
// Wait for currentConfig to be refreshed by the async config_query response.
|
||||
// Poll for up to 5 seconds for a new currentConfig (or a changed id).
|
||||
let waited = 0;
|
||||
while (waited < 5000) {
|
||||
const curId = currentConfig ? (currentConfig.id || null) : null;
|
||||
if (currentConfig && curId !== prevConfigId) {
|
||||
break;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
waited += 200;
|
||||
}
|
||||
|
||||
// Populate form fields from currentConfig.
|
||||
// IMPORTANT: skip fields the user is actively editing so the periodic
|
||||
// auto-refresh (every 10s via cachingRefreshInterval) does not clobber
|
||||
// in-progress edits before "Apply configuration" is pressed. A field is
|
||||
// considered "being edited" if it currently has focus OR it has been
|
||||
// modified by the user since the last populate (dirty). Dirty state is
|
||||
// tracked by the input/change listeners installed in
|
||||
// initCachingEventListeners() via markCachingFieldDirty().
|
||||
CACHING_CONFIG_FIELDS.forEach(field => {
|
||||
const value = getCachingConfigValue(field.key);
|
||||
const el = document.getElementById(field.field);
|
||||
if (!el) return;
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Don't stomp on a field the user is currently focused on.
|
||||
if (document.activeElement === el) {
|
||||
return;
|
||||
}
|
||||
// Don't stomp on a field the user has edited but not yet applied.
|
||||
if (el.dataset.cachingDirty === '1') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (field.type === 'textarea-list') {
|
||||
// Config stores comma-separated; display as newline-separated
|
||||
el.value = String(value).split(',').map(s => s.trim()).filter(s => s.length > 0).join('\n');
|
||||
} else {
|
||||
el.value = String(value);
|
||||
}
|
||||
});
|
||||
|
||||
// Attempt to fetch service/inbox status via the caching_status system command.
|
||||
// Show a neutral "loading" placeholder; handleCachingStatusResponse() will
|
||||
// overwrite it when the response arrives. If the command fails (e.g. admin
|
||||
// auth rejection), the catch block surfaces the real error.
|
||||
const serviceStatusEl = document.getElementById('caching-service-status');
|
||||
const inboxStatusEl = document.getElementById('caching-inbox-status');
|
||||
|
||||
if (serviceStatusEl) {
|
||||
serviceStatusEl.innerHTML = '<p>Loading caching service status...</p>';
|
||||
}
|
||||
if (inboxStatusEl) {
|
||||
inboxStatusEl.innerHTML = '<p>Loading relay inbox status...</p>';
|
||||
}
|
||||
|
||||
// Best-effort: send caching_status system command. Response handling is in
|
||||
// handleSystemCommandResponse() -> handleCachingStatusResponse() below.
|
||||
try {
|
||||
await sendAdminCommand(['system_command', 'caching_status']);
|
||||
} catch (e) {
|
||||
console.log('caching_status command failed: ' + e.message);
|
||||
const errMsg = escapeHtml(e.message || 'Unknown error');
|
||||
// Detect admin authorization failures and surface a clear message,
|
||||
// since the relay rejects the kind 23456 event before the command
|
||||
// handler ever runs.
|
||||
const isAuthError = /unauthorized admin event attempt|invalid admin pubkey|not admin/i.test(e.message || '');
|
||||
const authNote = isAuthError
|
||||
? ' <em>(Your browser pubkey is not registered as an admin on this relay. Load the admin private key in your Nostr extension.)</em>'
|
||||
: '';
|
||||
if (serviceStatusEl) {
|
||||
serviceStatusEl.innerHTML = `<p style="color:red">Failed to load service status: ${errMsg}${authNote}</p>`;
|
||||
}
|
||||
if (inboxStatusEl) {
|
||||
inboxStatusEl.innerHTML = `<p style="color:red">Failed to load inbox status: ${errMsg}${authNote}</p>`;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch caching status:', error);
|
||||
const statusEl = document.getElementById('caching-config-status');
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = `<span style="color:red">Failed to load caching status: ${escapeHtml(error.message)}</span>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply caching configuration: collect form values and send config_update
|
||||
async function applyCachingConfig() {
|
||||
const statusEl = document.getElementById('caching-config-status');
|
||||
try {
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
throw new Error('Must be logged in to apply caching configuration');
|
||||
}
|
||||
|
||||
const configObjects = [];
|
||||
|
||||
CACHING_CONFIG_FIELDS.forEach(field => {
|
||||
const el = document.getElementById(field.field);
|
||||
if (!el) return;
|
||||
|
||||
let value = el.value;
|
||||
|
||||
if (field.type === 'textarea-list') {
|
||||
// Convert newlines back to comma-separated for storage
|
||||
value = value.split('\n').map(s => s.trim()).filter(s => s.length > 0).join(',');
|
||||
} else if (field.type === 'integer') {
|
||||
// Trim whitespace; leave empty for the skip check below.
|
||||
// Do NOT coerce empty to '0' — the backend validates numeric
|
||||
// ranges (e.g. caching_inbox_batch_size 1-500) and rejects 0,
|
||||
// and config_update is atomic, so a single rejected field
|
||||
// rolls back the entire batch (including unrelated boolean
|
||||
// toggles like caching_enabled). Empty here means the form
|
||||
// field was not populated, so we skip it and preserve the
|
||||
// existing stored value.
|
||||
value = String(value).trim();
|
||||
} else {
|
||||
value = String(value).trim();
|
||||
}
|
||||
|
||||
// Skip empty values for optional string/textarea-list/integer fields.
|
||||
// The backend (update_config_in_table) rejects empty strings to
|
||||
// prevent accidental data loss, and config_update is atomic —
|
||||
// one rejected field rolls back the entire batch. Fields like
|
||||
// caching_root_npubs and caching_bootstrap_relays are optional
|
||||
// and may legitimately be empty until configured. Integer fields
|
||||
// are also skipped when empty so their existing DB value is
|
||||
// preserved instead of being overwritten with an out-of-range 0.
|
||||
if (value === '' && (field.type === 'textarea-list' || field.type === 'string' || field.type === 'integer')) {
|
||||
return;
|
||||
}
|
||||
|
||||
let dataType = 'string';
|
||||
if (field.type === 'integer') {
|
||||
dataType = 'integer';
|
||||
} else if (field.type === 'select') {
|
||||
dataType = 'boolean';
|
||||
}
|
||||
|
||||
configObjects.push({
|
||||
key: field.key,
|
||||
value: value,
|
||||
data_type: dataType,
|
||||
category: 'caching'
|
||||
});
|
||||
});
|
||||
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = '<span style="color:var(--accent-color,#ff0000)">Applying caching configuration...</span>';
|
||||
}
|
||||
|
||||
// Send all caching config keys in a single config_update command
|
||||
await sendConfigUpdateCommand(configObjects);
|
||||
|
||||
// Increment caching_config_generation so the caching service picks up changes
|
||||
const currentGen = getCachingConfigValue('caching_config_generation');
|
||||
let nextGen = 1;
|
||||
if (currentGen !== null && !isNaN(parseInt(currentGen, 10))) {
|
||||
nextGen = parseInt(currentGen, 10) + 1;
|
||||
}
|
||||
await sendConfigUpdateCommand([{
|
||||
key: 'caching_config_generation',
|
||||
value: String(nextGen),
|
||||
data_type: 'integer',
|
||||
category: 'caching'
|
||||
}]);
|
||||
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = '<span style="color:green">✓ Caching configuration applied successfully.</span>';
|
||||
}
|
||||
log('Caching configuration applied successfully', 'INFO');
|
||||
|
||||
// Clear dirty flags so the upcoming refresh can repopulate fields from
|
||||
// the freshly-applied DB values without skipping user-edited fields.
|
||||
clearCachingFieldsDirty();
|
||||
|
||||
// Refresh status to reflect applied values
|
||||
setTimeout(() => fetchCachingStatus().catch(() => {}), 1500);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to apply caching configuration:', error);
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = `<span style="color:red">Failed to apply configuration: ${escapeHtml(error.message)}</span>`;
|
||||
}
|
||||
log('Failed to apply caching configuration: ' + error.message, 'ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Reset backfill progress via system_command
|
||||
async function resetBackfillProgress() {
|
||||
const statusEl = document.getElementById('caching-config-status');
|
||||
try {
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
throw new Error('Must be logged in to reset backfill progress');
|
||||
}
|
||||
|
||||
const confirmed = confirm('Reset all backfill progress? This will cause the caching service to re-fetch events for all backfill windows.');
|
||||
if (!confirmed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = '<span style="color:var(--accent-color,#ff0000)">Resetting backfill progress...</span>';
|
||||
}
|
||||
|
||||
await sendAdminCommand(['system_command', 'caching_reset_progress']);
|
||||
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = '<span style="color:green">✓ Backfill progress reset command sent.</span>';
|
||||
}
|
||||
log('Backfill progress reset command sent', 'INFO');
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to reset backfill progress:', error);
|
||||
if (statusEl) {
|
||||
statusEl.innerHTML = `<span style="color:red">Failed to reset backfill progress: ${escapeHtml(error.message)}</span>`;
|
||||
}
|
||||
log('Failed to reset backfill progress: ' + error.message, 'ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Handle caching_status system command responses (service + inbox status)
|
||||
function handleCachingStatusResponse(responseData) {
|
||||
const serviceStatusEl = document.getElementById('caching-service-status');
|
||||
const inboxStatusEl = document.getElementById('caching-inbox-status');
|
||||
|
||||
if (responseData.status !== 'success') {
|
||||
const msg = responseData.message || 'Unknown error';
|
||||
if (serviceStatusEl) {
|
||||
serviceStatusEl.innerHTML = `<p style="color:red">Service status error: ${escapeHtml(msg)}</p>`;
|
||||
}
|
||||
if (inboxStatusEl) {
|
||||
inboxStatusEl.innerHTML = `<p style="color:red">Inbox status error: ${escapeHtml(msg)}</p>`;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const data = responseData.data || responseData;
|
||||
|
||||
// Helper: pick the first defined value from a list of candidate keys/paths.
|
||||
function pick(...vals) {
|
||||
for (const v of vals) {
|
||||
if (v !== undefined && v !== null) return v;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Service status block (external caching service process)
|
||||
if (serviceStatusEl) {
|
||||
const service = data.service || data.caching_service || {};
|
||||
// Backward-compat: fall back to top-level flat fields if nested object absent.
|
||||
const enabled = pick(service.enabled, data.caching_enabled, data.enabled);
|
||||
const running = pick(service.running, data.running);
|
||||
const connectedRelays = pick(service.connected_relays, data.connected_relays);
|
||||
const eventsCached = pick(service.events_cached, data.events_cached);
|
||||
// Per-author until-cursor drain backfill progress.
|
||||
const bfComplete = pick(service.backfill_authors_complete, data.backfill_authors_complete);
|
||||
const bfTotal = pick(service.backfill_authors_total, data.backfill_authors_total);
|
||||
|
||||
let html = '<ul style="list-style:none;padding:0;margin:0;">';
|
||||
if (enabled !== null) html += `<li><strong>Enabled:</strong> ${escapeHtml(String(enabled))}</li>`;
|
||||
if (running !== null) html += `<li><strong>Running:</strong> ${escapeHtml(String(running))}</li>`;
|
||||
if (connectedRelays !== null) html += `<li><strong>Connected Relays:</strong> ${escapeHtml(String(connectedRelays))}</li>`;
|
||||
if (eventsCached !== null) html += `<li><strong>Events Cached:</strong> ${escapeHtml(String(eventsCached))}</li>`;
|
||||
if (bfComplete !== null && bfTotal !== null) {
|
||||
html += `<li><strong>Backfill:</strong> ${escapeHtml(String(bfComplete))}/${escapeHtml(String(bfTotal))} authors complete</li>`;
|
||||
}
|
||||
html += '</ul>';
|
||||
serviceStatusEl.innerHTML = html;
|
||||
}
|
||||
|
||||
// Inbox status block (relay-owned inbox poller)
|
||||
if (inboxStatusEl) {
|
||||
const inbox = data.inbox || data.caching_inbox || {};
|
||||
// Backward-compat: derive from flat top-level inbox_* fields if nested object absent.
|
||||
const inboxEnabled = pick(inbox.enabled, data.caching_inbox_enabled);
|
||||
const inboxRunning = pick(inbox.running);
|
||||
const queueDepth = pick(inbox.queue_depth,
|
||||
(data.inbox_pending_live !== undefined && data.inbox_pending_backfill !== undefined)
|
||||
? (Number(data.inbox_pending_live) + Number(data.inbox_pending_backfill))
|
||||
: undefined);
|
||||
const lastPoll = pick(inbox.last_poll);
|
||||
const totalDequeued = pick(inbox.total_dequeued, data.inbox_total_dequeued);
|
||||
const totalAccepted = pick(inbox.total_accepted, data.inbox_total_accepted);
|
||||
const totalRejected = pick(inbox.total_rejected, data.inbox_total_rejected);
|
||||
const totalDuplicates = pick(inbox.total_duplicates, data.inbox_total_duplicates);
|
||||
const lastBatchSize = pick(inbox.last_batch_size, data.inbox_last_batch_size);
|
||||
const pendingLive = pick(inbox.pending_live, data.inbox_pending_live);
|
||||
const pendingBackfill = pick(inbox.pending_backfill, data.inbox_pending_backfill);
|
||||
const oldestAge = pick(inbox.oldest_age_seconds, data.inbox_oldest_age_seconds);
|
||||
|
||||
let html = '<ul style="list-style:none;padding:0;margin:0;">';
|
||||
if (inboxEnabled !== null) html += `<li><strong>Inbox Enabled:</strong> ${escapeHtml(String(inboxEnabled))}</li>`;
|
||||
if (inboxRunning !== null) html += `<li><strong>Inbox Running:</strong> ${escapeHtml(String(inboxRunning))}</li>`;
|
||||
if (queueDepth !== null) html += `<li><strong>Queue Depth:</strong> ${escapeHtml(String(queueDepth))}</li>`;
|
||||
if (lastPoll !== null) html += `<li><strong>Last Poll:</strong> ${escapeHtml(String(lastPoll))}</li>`;
|
||||
if (totalDequeued !== null) html += `<li><strong>Total Dequeued:</strong> ${escapeHtml(String(totalDequeued))}</li>`;
|
||||
if (totalAccepted !== null) html += `<li><strong>Total Accepted:</strong> ${escapeHtml(String(totalAccepted))}</li>`;
|
||||
if (totalRejected !== null) html += `<li><strong>Total Rejected:</strong> ${escapeHtml(String(totalRejected))}</li>`;
|
||||
if (totalDuplicates !== null) html += `<li><strong>Total Duplicates:</strong> ${escapeHtml(String(totalDuplicates))}</li>`;
|
||||
if (lastBatchSize !== null) html += `<li><strong>Last Batch Size:</strong> ${escapeHtml(String(lastBatchSize))}</li>`;
|
||||
if (pendingLive !== null) html += `<li><strong>Pending (Live):</strong> ${escapeHtml(String(pendingLive))}</li>`;
|
||||
if (pendingBackfill !== null) html += `<li><strong>Pending (Backfill):</strong> ${escapeHtml(String(pendingBackfill))}</li>`;
|
||||
if (oldestAge !== null) html += `<li><strong>Oldest Pending Age:</strong> ${escapeHtml(String(oldestAge))}s</li>`;
|
||||
html += '</ul>';
|
||||
inboxStatusEl.innerHTML = html;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch caching follows status (per-follow detail with usernames, event counts, relays)
|
||||
async function fetchCachingFollowsStatus() {
|
||||
try {
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
throw new Error('Must be logged in to fetch caching follows status');
|
||||
}
|
||||
await sendAdminCommand(['system_command', 'caching_follows_status']);
|
||||
} catch (e) {
|
||||
console.log('caching_follows_status command failed: ' + e.message);
|
||||
const tbody = document.getElementById('caching-follows-table-body');
|
||||
if (tbody) {
|
||||
tbody.innerHTML = `<tr><td colspan="7" style="text-align: center; color: red; padding: 8px;">Failed to load: ${escapeHtml(e.message || 'Unknown error')}</td></tr>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle caching_follows_status response and render the follows table
|
||||
function handleCachingFollowsResponse(responseData) {
|
||||
const tbody = document.getElementById('caching-follows-table-body');
|
||||
const statusEl = document.getElementById('caching-follows-status');
|
||||
if (!tbody) return;
|
||||
|
||||
if (responseData.status !== 'success') {
|
||||
tbody.innerHTML = `<tr><td colspan="7" style="text-align: center; color: red; padding: 8px;">Error: ${escapeHtml(responseData.message || 'Unknown error')}</td></tr>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const follows = responseData.follows || [];
|
||||
const totalFollows = responseData.total_follows || follows.length;
|
||||
const totalEvents = responseData.total_events_in_db || 0;
|
||||
|
||||
// Active backfill target (written by the caching_relay process to PG).
|
||||
const activePk = responseData.active_backfill_pubkey || '';
|
||||
const activeRelay = responseData.active_backfill_relay || '';
|
||||
|
||||
if (statusEl) {
|
||||
let html = `<span style="color: var(--text-muted, #888);">${totalFollows} follows, ${totalEvents} total events in DB</span>`;
|
||||
if (activePk && activeRelay) {
|
||||
html += `<br><span style="color: var(--accent-color, #ff0);">⚡ Working on: ${escapeHtml(activeRelay)}</span>`;
|
||||
} else if (activePk) {
|
||||
html += `<br><span style="color: var(--accent-color, #ff0);">⚡ Working on: ${escapeHtml(activePk.substring(0,12))}...</span>`;
|
||||
} else {
|
||||
// No active backfill target — check if all follows are complete.
|
||||
const allComplete = follows.length > 0 && follows.every(f => f.backfill_complete);
|
||||
if (allComplete) {
|
||||
html += `<br><span style="color: green;">✓ Caching complete</span>`;
|
||||
}
|
||||
}
|
||||
statusEl.innerHTML = html;
|
||||
}
|
||||
|
||||
if (follows.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="6" style="text-align: center; font-style: italic; padding: 12px;">No followed pubkeys found.</td></tr>';
|
||||
return;
|
||||
}
|
||||
|
||||
// Sort by total_events_in_db descending (most active first)
|
||||
follows.sort((a, b) => (b.total_events_in_db || 0) - (a.total_events_in_db || 0));
|
||||
|
||||
tbody.innerHTML = '';
|
||||
follows.forEach((follow, index) => {
|
||||
const row = document.createElement('tr');
|
||||
row.style.borderBottom = '1px solid var(--border-color, #333)';
|
||||
|
||||
// Highlight the row if this is the currently-active backfill target.
|
||||
const isActiveAuthor = activePk && follow.pubkey === activePk;
|
||||
if (isActiveAuthor) {
|
||||
row.style.borderLeft = '4px solid red';
|
||||
row.style.borderRight = '4px solid red';
|
||||
row.style.borderTop = '2px solid red';
|
||||
row.style.borderBottom = '2px solid red';
|
||||
}
|
||||
|
||||
// Name column
|
||||
const name = follow.name || '';
|
||||
const nameDisplay = name ? escapeHtml(name) : '<span style="color: var(--text-muted, #888); font-style: italic;">unknown</span>';
|
||||
|
||||
// npub column (convert hex to npub)
|
||||
let npubLink = follow.pubkey ? follow.pubkey.substring(0, 12) + '...' : '-';
|
||||
try {
|
||||
if (follow.pubkey && follow.pubkey.length === 64 && /^[0-9a-fA-F]+$/.test(follow.pubkey)) {
|
||||
const npub = window.NostrTools.nip19.npubEncode(follow.pubkey);
|
||||
npubLink = `<a href="https://njump.me/${npub}" target="_blank" class="npub-link" style="font-family: 'Courier New', monospace; font-size: 11px;">${npub.substring(0, 20)}...</a>`;
|
||||
}
|
||||
} catch (e) { /* keep short hex */ }
|
||||
|
||||
// Root badge
|
||||
const rootBadge = follow.is_root
|
||||
? '<span style="color: var(--accent-color, #ff0); font-weight: bold;">★ Root</span>'
|
||||
: '<span style="color: var(--text-muted, #888);">—</span>';
|
||||
|
||||
// Events in DB
|
||||
const totalInDb = follow.total_events_in_db || 0;
|
||||
|
||||
// Backfill status
|
||||
const relayProgress = follow.relay_progress || [];
|
||||
const doneRelays = relayProgress.filter(r => r.complete).length;
|
||||
const totalRelays = relayProgress.length;
|
||||
const bfStatus = follow.backfill_complete
|
||||
? `<span style="color: green;">✓ Complete</span>`
|
||||
: `<span style="color: orange;">${doneRelays}/${totalRelays} relays</span>`;
|
||||
|
||||
// Outbox relays with per-relay event counts from relay_progress.
|
||||
// Normalize URLs by stripping trailing slashes for dedup.
|
||||
const normUrl = u => u ? u.replace(/\/+$/, '') : u;
|
||||
const relayProgressMap = {};
|
||||
(follow.relay_progress || []).forEach(rp => {
|
||||
relayProgressMap[normUrl(rp.relay_url)] = rp;
|
||||
});
|
||||
const outboxRelays = (follow.outbox_relays || []).map(normUrl);
|
||||
// Merge outbox relays with relay_progress relays, deduplicating by
|
||||
// normalized URL. Relays in relay_progress that aren't in outbox_relays
|
||||
// are bootstrap relays that were queried but aren't outbox relays.
|
||||
const allRelays = [...new Set([...outboxRelays, ...Object.keys(relayProgressMap)])];
|
||||
const outboxDisplay = allRelays.length > 0
|
||||
? allRelays.map(r => {
|
||||
const short = escapeHtml(r.replace('wss://', '').replace('https://', '').replace('ws://', ''));
|
||||
const rp = relayProgressMap[r];
|
||||
// Highlight if this is the relay currently being queried.
|
||||
const isActiveRelay = isActiveAuthor && activeRelay && r === normUrl(activeRelay);
|
||||
const activePrefix = isActiveRelay ? '⚡ ' : '';
|
||||
if (rp) {
|
||||
const evCount = rp.events_fetched || 0;
|
||||
const done = rp.complete ? '✓' : '⏳';
|
||||
const status = rp.last_status || '';
|
||||
const statusDisplay = status ? ` <span style="color: var(--text-muted, #888); font-size: 10px;">[${escapeHtml(status)}]</span>` : '';
|
||||
const color = isActiveRelay
|
||||
? 'color: red; font-weight: bold;'
|
||||
: (evCount > 0 ? 'color: green;' : 'color: var(--text-muted, #888);');
|
||||
return `<span style="${color}">${activePrefix}${done} ${short} (${evCount})${statusDisplay}</span>`;
|
||||
}
|
||||
const color = isActiveRelay
|
||||
? 'color: red; font-weight: bold;'
|
||||
: 'color: var(--text-muted, #888);';
|
||||
return `<span style="${color}">${activePrefix}${short}</span>`;
|
||||
}).join('<br>')
|
||||
: '<span style="color: var(--text-muted, #888); font-style: italic;">none</span>';
|
||||
|
||||
row.innerHTML = `
|
||||
<td style="padding: 6px 8px;">${nameDisplay}</td>
|
||||
<td style="padding: 6px 8px;">${npubLink}</td>
|
||||
<td style="padding: 6px 8px; text-align: center;">${rootBadge}</td>
|
||||
<td style="padding: 6px 8px; text-align: right;">${totalInDb}</td>
|
||||
<td style="padding: 6px 8px;">${bfStatus}</td>
|
||||
<td style="padding: 6px 8px; font-size: 11px;">${outboxDisplay}</td>
|
||||
`;
|
||||
tbody.appendChild(row);
|
||||
});
|
||||
}
|
||||
|
||||
// Start the external caching service
|
||||
async function startCachingService() {
|
||||
const statusDiv = document.getElementById('caching-service-control-status');
|
||||
if (!statusDiv) return;
|
||||
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
statusDiv.innerHTML = '<span style="color:red">Must be logged in to start caching service</span>';
|
||||
return;
|
||||
}
|
||||
|
||||
statusDiv.innerHTML = '<span style="color:blue">Starting caching service...</span>';
|
||||
|
||||
try {
|
||||
const command_array = ["system_command", "caching_start_service"];
|
||||
await sendAdminCommand(command_array);
|
||||
statusDiv.innerHTML = '<span style="color:green">✓ Caching service start initiated</span>';
|
||||
log('Caching service start initiated', 'INFO');
|
||||
} catch (error) {
|
||||
statusDiv.innerHTML = `<span style="color:red">✗ Failed to start caching service: ${error.message}</span>`;
|
||||
log(`Failed to start caching service: ${error.message}`, 'ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the external caching service
|
||||
async function stopCachingService() {
|
||||
const statusDiv = document.getElementById('caching-service-control-status');
|
||||
if (!statusDiv) return;
|
||||
|
||||
if (!isLoggedIn || !userPubkey) {
|
||||
statusDiv.innerHTML = '<span style="color:red">Must be logged in to stop caching service</span>';
|
||||
return;
|
||||
}
|
||||
|
||||
if (!confirm('Are you sure you want to stop the caching service?')) {
|
||||
return;
|
||||
}
|
||||
|
||||
statusDiv.innerHTML = '<span style="color:blue">Stopping caching service...</span>';
|
||||
|
||||
try {
|
||||
const command_array = ["system_command", "caching_stop_service"];
|
||||
await sendAdminCommand(command_array);
|
||||
statusDiv.innerHTML = '<span style="color:green">✓ Caching service stop initiated</span>';
|
||||
log('Caching service stop initiated', 'INFO');
|
||||
} catch (error) {
|
||||
statusDiv.innerHTML = `<span style="color:red">✗ Failed to stop caching service: ${error.message}</span>`;
|
||||
log(`Failed to stop caching service: ${error.message}`, 'ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Mark a caching config field as dirty (user-modified) so the periodic
|
||||
// fetchCachingStatus() refresh does not overwrite in-progress edits.
|
||||
function markCachingFieldDirty() {
|
||||
this.dataset.cachingDirty = '1';
|
||||
}
|
||||
|
||||
// Clear the dirty flag on all caching config fields. Called after a successful
|
||||
// apply so subsequent refreshes can repopulate fields from the DB again.
|
||||
function clearCachingFieldsDirty() {
|
||||
CACHING_CONFIG_FIELDS.forEach(field => {
|
||||
const el = document.getElementById(field.field);
|
||||
if (el) delete el.dataset.cachingDirty;
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize Caching page event listeners
|
||||
function initCachingEventListeners() {
|
||||
const applyBtn = document.getElementById('caching-apply-btn');
|
||||
const resetBtn = document.getElementById('caching-reset-progress-btn');
|
||||
const startServiceBtn = document.getElementById('caching-start-service-btn');
|
||||
const stopServiceBtn = document.getElementById('caching-stop-service-btn');
|
||||
|
||||
if (applyBtn) {
|
||||
applyBtn.addEventListener('click', applyCachingConfig);
|
||||
}
|
||||
if (resetBtn) {
|
||||
resetBtn.addEventListener('click', resetBackfillProgress);
|
||||
}
|
||||
if (startServiceBtn) {
|
||||
startServiceBtn.addEventListener('click', startCachingService);
|
||||
}
|
||||
if (stopServiceBtn) {
|
||||
stopServiceBtn.addEventListener('click', stopCachingService);
|
||||
}
|
||||
|
||||
// Track user edits to caching config fields so the 10s auto-refresh
|
||||
// (fetchCachingStatus -> populate form) does not clobber in-progress
|
||||
// edits before "Apply configuration" is pressed. Both 'input' (fires
|
||||
// on every keystroke) and 'change' (fires on blur/commit for selects
|
||||
// and number inputs) are tracked.
|
||||
CACHING_CONFIG_FIELDS.forEach(field => {
|
||||
const el = document.getElementById(field.field);
|
||||
if (!el) return;
|
||||
el.addEventListener('input', markCachingFieldDirty);
|
||||
el.addEventListener('change', markCachingFieldDirty);
|
||||
});
|
||||
|
||||
console.log('Caching page event listeners initialized');
|
||||
}
|
||||
|
||||
// Initialize caching event listeners when DOM is ready
|
||||
document.addEventListener('DOMContentLoaded', initCachingEventListeners);
|
||||
@@ -0,0 +1,813 @@
|
||||
<!DOCTYPE html>
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<html lang="en" dir="ltr">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>TEMPLATE</title>
|
||||
|
||||
<link rel="stylesheet" href="./css/client.css" />
|
||||
|
||||
<!-- Initialize theme BEFORE any components load -->
|
||||
<script>
|
||||
(function () {
|
||||
const savedTheme = localStorage.getItem('theme');
|
||||
if (savedTheme === 'dark') {
|
||||
document.documentElement.classList.add('dark-mode');
|
||||
if (document.body) {
|
||||
document.body.classList.add('dark-mode');
|
||||
}
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
<link rel="shortcut icon" type="image/x-icon" href="./favicon/favicon-dots2.ico" />
|
||||
|
||||
<!-- SVG.js library (required by HamburgerMorphing) -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/@svgdotjs/svg.js@3.0/dist/svg.min.js"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<!-- ================================================================
|
||||
HAMBURGER BUTTON (Fixed, separate from header)
|
||||
================================================================
|
||||
The hamburger button is a fixed element outside the header
|
||||
to ensure it stays visible above the sidenav (z-index: 10 > 3).
|
||||
================================================================ -->
|
||||
<div id="divSvgHam" class="divHeaderButtons">
|
||||
<!-- HamburgerMorphing will be injected here -->
|
||||
</div>
|
||||
|
||||
<!-- ================================================================
|
||||
HEADER
|
||||
================================================================
|
||||
Standard header with title (center).
|
||||
================================================================ -->
|
||||
<div id="divHeader">
|
||||
<div id="divHeaderFlexLeft">
|
||||
<!-- Hamburger is now separate fixed element -->
|
||||
</div>
|
||||
|
||||
<div id="divHeaderFlexCenter">
|
||||
<div class="divHeaderText"></div>
|
||||
</div>
|
||||
|
||||
<div id="divHeaderFlexRight">
|
||||
<!-- No button in header right - logout is in sidenav footer -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ================================================================
|
||||
BODY
|
||||
================================================================
|
||||
Main content area. Add your page-specific content here.
|
||||
================================================================ -->
|
||||
<div id="divBody">
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<!-- ================================================================
|
||||
FOOTER
|
||||
================================================================
|
||||
Three-section footer layout:
|
||||
- Left: Relay status animations (HamburgerMorphing instances)
|
||||
- Center: General status information
|
||||
- Right: Additional information
|
||||
================================================================ -->
|
||||
<div id="divFooter">
|
||||
<div id="divFooterLeft" class="divFooterBox"></div>
|
||||
<div id="divFooterCenter" class="divFooterBox"></div>
|
||||
<div id="divFooterRight" class="divFooterBox"></div>
|
||||
<div id="divFooterBalance" class="divFooterBox">0 sats</div>
|
||||
</div>
|
||||
|
||||
<!-- ================================================================
|
||||
SIDENAV
|
||||
================================================================
|
||||
Slide-out navigation panel. Opens from left when hamburger clicked.
|
||||
Uses flexbox layout to pin version bar to bottom.
|
||||
Includes a version bar footer with theme toggle and logout buttons.
|
||||
================================================================ -->
|
||||
<div id="divSideNav">
|
||||
<div id="divSideNavHeader">
|
||||
<!-- No close button - use main hamburger to close -->
|
||||
</div>
|
||||
|
||||
<div id="divSideNavBody">
|
||||
<div id="divFiles"></div>
|
||||
</div>
|
||||
|
||||
<div id="divAiSection" class="sidenavSection">
|
||||
<div id="divAiSectionTitle" class="sidenavSectionTitle">AI</div>
|
||||
<div id="divAiList" class="sidenavSectionList">
|
||||
<div id="divAiProvidersList">No saved providers yet.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="divRelaySection">
|
||||
<div id="divRelaySectionTitle">
|
||||
リレー
|
||||
</div>
|
||||
<div id="divRelayList">
|
||||
Loading relays...
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="divBlossomSection">
|
||||
|
||||
<div id="divBlossomSectionTitle">ブロッサム</div>
|
||||
|
||||
<div id="divBlossomList">Loading blossom servers...</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div id="divVersionBar">
|
||||
<span id="versionDisplay">v0.0.1</span>
|
||||
<div id="divVersionBarButtons">
|
||||
<button id="themeToggleButton" title="Toggle Dark/Light Mode">
|
||||
<div id="themeToggleHamburgerContainer"></div>
|
||||
</button>
|
||||
<button id="logoutButton" title="Logout">
|
||||
<div id="logoutHamburgerContainer"></div>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ================================================================
|
||||
REQUIRED SCRIPTS
|
||||
================================================================
|
||||
These scripts must be loaded in this order:
|
||||
1. nostr.bundle.js - Nostr tools library
|
||||
2. nostr-lite.js - Authentication modal (nostr-login-lite)
|
||||
================================================================ -->
|
||||
<script src="./nostr.bundle.js"></script>
|
||||
<script src="./nostr-lite.js"></script>
|
||||
|
||||
<script type="module">
|
||||
/* ================================================================
|
||||
IMPORTS
|
||||
================================================================
|
||||
Import shared NDK functionality from init-ndk.mjs:
|
||||
- initNDKPage() - Initialize authentication and worker
|
||||
- getPubkey() - Get current user's pubkey
|
||||
- subscribe() - Create NDK subscriptions
|
||||
- publishEvent() - Publish events via NDK
|
||||
- disconnect() - Disconnect from worker
|
||||
- getRelayData() - Get relay connection data
|
||||
- getRelayStats() - Get relay activity statistics
|
||||
|
||||
Import HamburgerMorphing for animated icons
|
||||
================================================================ */
|
||||
import {
|
||||
initNDKPage,
|
||||
getPubkey, injectHeaderAvatar,
|
||||
subscribe,
|
||||
publishEvent,
|
||||
disconnect,
|
||||
getVersion,
|
||||
updateVersionDisplay,
|
||||
getUserSettings,
|
||||
patchUserSettings,
|
||||
onUserSettings
|
||||
} from './js/init-ndk.mjs';
|
||||
import { HamburgerMorphing } from "./hamburger_morphing/hamburger.mjs";
|
||||
import { initFooterRelayStatus, updateFooterRelayStatus, initSidenavRelaySection, updateSidenavRelaySection, setRelayActivityState } from './js/relay-ui.mjs';
|
||||
|
||||
import { initBlossomSection, updateBlossomSection } from './js/blossom-ui.mjs';
|
||||
|
||||
import { initAiSectionWithLocalConfig } from './js/ai-ui.mjs';
|
||||
// Version will be loaded asynchronously
|
||||
const versionInfo = await getVersion();
|
||||
const VERSION = versionInfo.VERSION;
|
||||
console.log(`[template.html ${VERSION}] Loading...`);
|
||||
|
||||
/* ================================================================
|
||||
GLOBAL VARIABLES
|
||||
================================================================
|
||||
Track state for hamburger menu, relay status, and theme.
|
||||
================================================================ */
|
||||
let updateIntervalId = null;
|
||||
let currentPubkey = null;
|
||||
|
||||
/*
|
||||
AUTH STATE MODEL (Template reference)
|
||||
------------------------------------------------------------------
|
||||
This template now demonstrates three auth modes for standalone pages:
|
||||
|
||||
- required (default):
|
||||
Behaves like existing pages: login is required immediately.
|
||||
|
||||
- optional:
|
||||
Page can render public/read-only data without login, but can still
|
||||
prompt login later for user actions (publish, settings, etc).
|
||||
|
||||
- none:
|
||||
Never auto-login on load (pure public page).
|
||||
|
||||
URL behavior in this template:
|
||||
- If ?auth=required|optional|none is present, it wins.
|
||||
- Otherwise, if URL includes ?npub=... or ?pubkey=..., mode defaults
|
||||
to optional because pages with explicit profile targets are commonly
|
||||
public-readable.
|
||||
- Otherwise, mode defaults to required.
|
||||
*/
|
||||
let isAuthenticated = false;
|
||||
let authMode = 'required';
|
||||
let authedPageInitialized = false;
|
||||
let relayActivityListenersBound = false;
|
||||
|
||||
// Hamburger menu
|
||||
let hamburgerInstance = null;
|
||||
let isNavOpen = false;
|
||||
|
||||
// Version bar buttons
|
||||
let logoutHamburger = null;
|
||||
let themeToggleHamburger = null;
|
||||
let isDarkMode = false;
|
||||
|
||||
// App-wide user settings (NIP-78 kind 30078, d:user-settings)
|
||||
let pageSettings = {};
|
||||
let unsubscribeUserSettings = null;
|
||||
|
||||
/* ================================================================
|
||||
DOM VARIABLES
|
||||
================================================================
|
||||
Cache DOM element references for better performance.
|
||||
================================================================ */
|
||||
const divBody = document.getElementById("divBody");
|
||||
const divSideNav = document.getElementById("divSideNav");
|
||||
const divSideNavBody = document.getElementById("divSideNavBody");
|
||||
const divFooterCenter = document.getElementById("divFooterCenter");
|
||||
const divFooterRight = document.getElementById("divFooterRight");
|
||||
|
||||
/* ================================================================
|
||||
HAMBURGER MENU
|
||||
================================================================
|
||||
Initialize and control the animated hamburger menu.
|
||||
================================================================ */
|
||||
function initHamburgerMenu() {
|
||||
hamburgerInstance = new HamburgerMorphing('#divSvgHam', {
|
||||
foreground: 'var(--primary-color)',
|
||||
background: 'var(--secondary-color)',
|
||||
hover: 'var(--accent-color)'
|
||||
});
|
||||
hamburgerInstance.animateTo('burger');
|
||||
}
|
||||
|
||||
/* ================================================================
|
||||
SIDENAV FUNCTIONS
|
||||
================================================================
|
||||
Open/close sidenav with hamburger morphing animation.
|
||||
================================================================ */
|
||||
function openNav() {
|
||||
divSideNav.style.zIndex = 3;
|
||||
divSideNav.style.width = "clamp(400px, 50vw, 600px)";
|
||||
isNavOpen = true;
|
||||
if (hamburgerInstance) {
|
||||
hamburgerInstance.animateTo('arrow_left');
|
||||
}
|
||||
|
||||
|
||||
// Initialize version bar buttons when sidenav opens (lazy load)
|
||||
if (!logoutHamburger) {
|
||||
logoutHamburger = new HamburgerMorphing('#logoutHamburgerContainer', {
|
||||
size: 24,
|
||||
foreground: 'var(--primary-color)',
|
||||
background: 'var(--secondary-color)',
|
||||
hover: 'var(--accent-color)'
|
||||
});
|
||||
logoutHamburger.animateTo('x');
|
||||
}
|
||||
|
||||
if (!themeToggleHamburger) {
|
||||
themeToggleHamburger = new HamburgerMorphing('#themeToggleHamburgerContainer', {
|
||||
size: 24,
|
||||
foreground: 'var(--primary-color)',
|
||||
background: 'var(--secondary-color)',
|
||||
hover: 'var(--accent-color)'
|
||||
});
|
||||
|
||||
// Determine current theme
|
||||
const savedTheme = localStorage.getItem('theme');
|
||||
isDarkMode = savedTheme === 'dark' || document.body.classList.contains('dark-mode');
|
||||
const initialShape = isDarkMode ? 'moon' : 'circle';
|
||||
themeToggleHamburger.animateTo(initialShape);
|
||||
}
|
||||
}
|
||||
|
||||
function closeNav() {
|
||||
divSideNav.style.width = "0vw";
|
||||
divSideNav.style.zIndex = -1;
|
||||
isNavOpen = false;
|
||||
if (hamburgerInstance) {
|
||||
hamburgerInstance.animateTo('burger');
|
||||
}
|
||||
}
|
||||
|
||||
function toggleNav() {
|
||||
if (isNavOpen) {
|
||||
closeNav();
|
||||
} else {
|
||||
openNav();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ================================================================
|
||||
AUTH MODE HELPERS
|
||||
================================================================
|
||||
These functions are meant as reusable guidance for future pages.
|
||||
================================================================ */
|
||||
function hasTargetPubkeyInUrl() {
|
||||
const params = new URLSearchParams(window.location.search || '');
|
||||
const npub = String(params.get('npub') || '').trim();
|
||||
const pubkey = String(params.get('pubkey') || '').trim();
|
||||
return Boolean(npub || pubkey);
|
||||
}
|
||||
|
||||
function resolveAuthModeFromUrl() {
|
||||
const params = new URLSearchParams(window.location.search || '');
|
||||
const explicitAuth = String(params.get('auth') || '').trim().toLowerCase();
|
||||
if (explicitAuth === 'required' || explicitAuth === 'optional' || explicitAuth === 'none') {
|
||||
return explicitAuth;
|
||||
}
|
||||
|
||||
// Convention: explicit target profiles are public-readable by default.
|
||||
if (hasTargetPubkeyInUrl()) {
|
||||
return 'optional';
|
||||
}
|
||||
|
||||
return 'required';
|
||||
}
|
||||
|
||||
function isAuthRequiredError(error) {
|
||||
const message = String(error?.message || error || '').toLowerCase();
|
||||
return message.includes('authentication required');
|
||||
}
|
||||
|
||||
async function initializeAuthentication(mode) {
|
||||
// required: existing behavior, throw if auth fails.
|
||||
if (mode === 'required') {
|
||||
await initNDKPage();
|
||||
currentPubkey = await getPubkey();
|
||||
isAuthenticated = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// none: public page, no login attempt on load.
|
||||
if (mode === 'none') {
|
||||
isAuthenticated = false;
|
||||
currentPubkey = null;
|
||||
return;
|
||||
}
|
||||
|
||||
// optional: try silent/normal init; if auth required, continue public.
|
||||
try {
|
||||
await initNDKPage();
|
||||
currentPubkey = await getPubkey();
|
||||
isAuthenticated = true;
|
||||
} catch (error) {
|
||||
if (isAuthRequiredError(error)) {
|
||||
console.log('[template.html] Optional auth mode: continuing unauthenticated');
|
||||
isAuthenticated = false;
|
||||
currentPubkey = null;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function initializeAuthenticatedPageFeatures() {
|
||||
if (!isAuthenticated || authedPageInitialized) return;
|
||||
|
||||
await injectHeaderAvatar(currentPubkey);
|
||||
console.log('[template.html] Authenticated as:', currentPubkey);
|
||||
|
||||
// Hydrate app-wide user settings for this page
|
||||
try {
|
||||
pageSettings = await getUserSettings();
|
||||
} catch (error) {
|
||||
console.warn('[template.html] getUserSettings failed:', error);
|
||||
pageSettings = {};
|
||||
}
|
||||
|
||||
// Subscribe to live user settings updates (cross-tab + publish echoes)
|
||||
if (!unsubscribeUserSettings) {
|
||||
unsubscribeUserSettings = onUserSettings((settings) => {
|
||||
pageSettings = settings || {};
|
||||
// TODO: Re-render page-specific UI from pageSettings here.
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize relay-dependent UI only once authenticated.
|
||||
initFooterRelayStatus();
|
||||
initSidenavRelaySection();
|
||||
await initBlossomSection();
|
||||
initAiSectionWithLocalConfig();
|
||||
await UpdateFooter();
|
||||
|
||||
if (!updateIntervalId) {
|
||||
updateIntervalId = setInterval(UpdateFooter, 1000);
|
||||
}
|
||||
|
||||
// Relay activity listeners only matter after worker init/auth.
|
||||
if (!relayActivityListenersBound) {
|
||||
window.addEventListener('ndkRelayActivity', (event) => {
|
||||
const { relayUrl, activity, stats } = event.detail;
|
||||
console.log(`[template.html] Relay activity: ${relayUrl} - ${activity}`, stats);
|
||||
setRelayActivityState(relayUrl, activity);
|
||||
});
|
||||
|
||||
window.addEventListener('message', (event) => {
|
||||
if (event.data && event.data.type === 'relayActivity') {
|
||||
const { relayUrl, activity } = event.data;
|
||||
console.log(`[template.html] Relay activity: ${relayUrl} - ${activity}`);
|
||||
setRelayActivityState(relayUrl, activity);
|
||||
}
|
||||
});
|
||||
|
||||
relayActivityListenersBound = true;
|
||||
}
|
||||
|
||||
authedPageInitialized = true;
|
||||
}
|
||||
|
||||
async function promptLoginIfNeeded() {
|
||||
if (isAuthenticated) return true;
|
||||
|
||||
await initNDKPage();
|
||||
currentPubkey = await getPubkey();
|
||||
isAuthenticated = true;
|
||||
await initializeAuthenticatedPageFeatures();
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ================================================================
|
||||
UPDATE FOOTER
|
||||
================================================================
|
||||
Update footer sections with relay status, pubkey, and other info.
|
||||
Called periodically by update loop.
|
||||
================================================================ */
|
||||
const UpdateFooter = async () => {
|
||||
|
||||
try {
|
||||
// Update relay status visuals in footer and sidenav
|
||||
await updateFooterRelayStatus();
|
||||
await updateSidenavRelaySection();
|
||||
|
||||
await updateBlossomSection();
|
||||
// Clear center and right sections
|
||||
divFooterCenter.innerHTML = '';
|
||||
divFooterRight.innerHTML = '';
|
||||
} catch (error) {
|
||||
console.error('[template.html] Error updating footer:', error);
|
||||
}
|
||||
};
|
||||
|
||||
/* ================================================================
|
||||
LOGOUT
|
||||
================================================================
|
||||
Complete logout process:
|
||||
1. Stop update loop
|
||||
2. Disconnect from NDK worker
|
||||
3. Logout from nostr-login-lite
|
||||
4. Clear all storage (localStorage, sessionStorage, IndexedDB)
|
||||
5. Reload page
|
||||
================================================================ */
|
||||
const Logout = async () => {
|
||||
console.log("[template.html] Starting logout process...");
|
||||
|
||||
// Stop the update loop
|
||||
if (updateIntervalId) {
|
||||
clearInterval(updateIntervalId);
|
||||
updateIntervalId = null;
|
||||
}
|
||||
|
||||
// Disconnect from worker
|
||||
disconnect();
|
||||
|
||||
// Logout from nostr-login-lite
|
||||
if (window.NOSTR_LOGIN_LITE && window.NOSTR_LOGIN_LITE.logout) {
|
||||
await window.NOSTR_LOGIN_LITE.logout();
|
||||
}
|
||||
|
||||
// Clear all storage
|
||||
localStorage.clear();
|
||||
sessionStorage.clear();
|
||||
|
||||
// Clear IndexedDB
|
||||
if (window.indexedDB) {
|
||||
const databases = await window.indexedDB.databases();
|
||||
for (const db of databases) {
|
||||
if (db.name) {
|
||||
window.indexedDB.deleteDatabase(db.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("[template.html] Logged out, reloading page");
|
||||
location.reload(true);
|
||||
};
|
||||
|
||||
/* ================================================================
|
||||
EVENT LISTENERS
|
||||
================================================================
|
||||
Wire up UI interactions.
|
||||
Main hamburger button click handler is set up in main() after initialization.
|
||||
================================================================ */
|
||||
|
||||
/* ================================================================
|
||||
SUBSCRIPTION EXAMPLE
|
||||
================================================================
|
||||
Example of how to subscribe to Nostr events:
|
||||
|
||||
const sub = subscribe(
|
||||
{ kinds: [1], authors: [pubkey], limit: 10 },
|
||||
{ closeOnEose: false, cacheUsage: 'CACHE_FIRST' }
|
||||
);
|
||||
|
||||
Cache usage options:
|
||||
- 'CACHE_FIRST' - Check cache first, then relays
|
||||
- 'ONLY_RELAY' - Only query relays
|
||||
- 'ONLY_CACHE' - Only query cache
|
||||
- 'PARALLEL' - Query cache and relays simultaneously
|
||||
|
||||
Listen for events via window events:
|
||||
window.addEventListener('ndkEvent', (event) => {
|
||||
const evt = event.detail;
|
||||
console.log('Received event:', evt);
|
||||
});
|
||||
================================================================ */
|
||||
|
||||
/* ================================================================
|
||||
PUBLISH EXAMPLE
|
||||
================================================================
|
||||
Example of how to publish a Nostr event:
|
||||
|
||||
const event = {
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
kind: 1,
|
||||
tags: [],
|
||||
content: "Hello, Nostr!"
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await publishEvent(event);
|
||||
console.log("✅ Published to:", result.relayResults.successful);
|
||||
console.log("❌ Failed:", result.relayResults.failed);
|
||||
console.log("Total relays:", result.totalRelays);
|
||||
} catch (error) {
|
||||
console.error("Publish error:", error);
|
||||
}
|
||||
|
||||
Note: Events are automatically signed by the NDK worker using
|
||||
the message-based signer (which calls window.nostr.signEvent).
|
||||
================================================================ */
|
||||
|
||||
/* ================================================================
|
||||
USER SETTINGS EXAMPLE (NIP-78)
|
||||
================================================================
|
||||
Read/subscribe/write helper pattern for all pages:
|
||||
|
||||
// Read latest merged settings (cache + relay hydrated by worker)
|
||||
const settings = await getUserSettings();
|
||||
|
||||
// Subscribe to cross-tab updates
|
||||
const unsubscribe = onUserSettings((nextSettings) => {
|
||||
// Re-render page from nextSettings
|
||||
});
|
||||
|
||||
// Patch only your feature namespace
|
||||
await patchUserSettings({
|
||||
myFeature: {
|
||||
someFlag: true
|
||||
}
|
||||
});
|
||||
|
||||
// On page teardown (if applicable)
|
||||
// unsubscribe();
|
||||
================================================================ */
|
||||
|
||||
/* ================================================================
|
||||
INITIALIZATION
|
||||
================================================================
|
||||
Main initialization sequence:
|
||||
1. Initialize hamburger menu
|
||||
2. Set up hamburger click handler
|
||||
3. Resolve auth mode from URL/query policy
|
||||
4. Initialize authentication based on mode
|
||||
5. Initialize authenticated-only features (if signed in)
|
||||
6. Set up version bar button listeners
|
||||
7. Restore sidenav state
|
||||
8. Update version display
|
||||
|
||||
Notes:
|
||||
- required mode = existing behavior (prompt login on load)
|
||||
- optional mode = allow public load, login later on demand
|
||||
- none mode = no auto-login on load
|
||||
================================================================ */
|
||||
(async function main() {
|
||||
console.log("[template.html] Starting initialization...");
|
||||
|
||||
try {
|
||||
// Initialize hamburger menu first
|
||||
initHamburgerMenu();
|
||||
|
||||
// Add click handler to hamburger
|
||||
const divSvgHam = document.getElementById('divSvgHam');
|
||||
if (divSvgHam) {
|
||||
divSvgHam.addEventListener('click', toggleNav);
|
||||
}
|
||||
|
||||
// Initialize version bar buttons
|
||||
const themeToggleButton = document.getElementById('themeToggleButton');
|
||||
const logoutButton = document.getElementById('logoutButton');
|
||||
|
||||
if (themeToggleButton) {
|
||||
themeToggleButton.addEventListener('click', () => {
|
||||
isDarkMode = !isDarkMode;
|
||||
if (isDarkMode) {
|
||||
localStorage.setItem('theme', 'dark');
|
||||
} else {
|
||||
localStorage.setItem('theme', 'light');
|
||||
}
|
||||
// Save sidenav state before reload
|
||||
localStorage.setItem('sidenavWasOpen', isNavOpen ? 'true' : 'false');
|
||||
window.location.reload();
|
||||
});
|
||||
}
|
||||
|
||||
if (logoutButton) {
|
||||
logoutButton.addEventListener('click', async () => {
|
||||
try {
|
||||
// In optional/none modes this doubles as a "Sign in" entry point.
|
||||
if (!isAuthenticated) {
|
||||
await promptLoginIfNeeded();
|
||||
return;
|
||||
}
|
||||
await Logout();
|
||||
} catch (error) {
|
||||
console.error('Logout/login action failed:', error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Resolve and initialize page auth policy.
|
||||
authMode = resolveAuthModeFromUrl();
|
||||
console.log('[template.html] Resolved auth mode:', authMode);
|
||||
await initializeAuthentication(authMode);
|
||||
|
||||
// Initialize authenticated features only when signed in.
|
||||
await initializeAuthenticatedPageFeatures();
|
||||
|
||||
/* ============================================================
|
||||
EXAMPLE: Subscribe to events
|
||||
============================================================
|
||||
Uncomment to subscribe to user's notes:
|
||||
|
||||
const notesSub = subscribe(
|
||||
{ kinds: [1], authors: [currentPubkey], limit: 10 },
|
||||
{ closeOnEose: false, cacheUsage: 'CACHE_FIRST' }
|
||||
);
|
||||
console.log("[template.html] Subscribed to kind 1");
|
||||
============================================================ */
|
||||
|
||||
/* ============================================================
|
||||
EXAMPLE: Listen for events from worker
|
||||
============================================================
|
||||
Uncomment to handle incoming events:
|
||||
|
||||
window.addEventListener('ndkEvent', (event) => {
|
||||
const evt = event.detail;
|
||||
console.log("[template.html] Received event:", evt.kind, evt.pubkey);
|
||||
|
||||
if (evt.pubkey === currentPubkey) {
|
||||
if (evt.kind === 1) {
|
||||
// Handle note event
|
||||
console.log("Note:", evt.content);
|
||||
}
|
||||
}
|
||||
});
|
||||
============================================================ */
|
||||
|
||||
/* ============================================================
|
||||
EXAMPLE: Listen for cached profile
|
||||
============================================================
|
||||
Uncomment to handle cached profile data:
|
||||
|
||||
window.addEventListener('ndkProfile', (event) => {
|
||||
console.log("[template.html] Cached profile:", event.detail);
|
||||
// event.detail contains profile object (name, about, etc.)
|
||||
});
|
||||
============================================================ */
|
||||
|
||||
// Restore sidenav state if it was open before theme toggle
|
||||
const sidenavWasOpen = localStorage.getItem('sidenavWasOpen');
|
||||
if (sidenavWasOpen === 'true') {
|
||||
localStorage.removeItem('sidenavWasOpen');
|
||||
openNav();
|
||||
}
|
||||
|
||||
// Optional UX note for public mode pages.
|
||||
if (!isAuthenticated && (authMode === 'optional' || authMode === 'none')) {
|
||||
divFooterCenter.textContent = 'Public mode';
|
||||
divFooterRight.textContent = 'Sign in from side menu for private features';
|
||||
}
|
||||
|
||||
// Update version display
|
||||
await updateVersionDisplay();
|
||||
|
||||
console.log('[template.html] Initialization complete');
|
||||
} catch (error) {
|
||||
console.error('[template.html] Initialization failed:', error);
|
||||
divBody.innerHTML = `<div style="text-align: center; padding: 50px;">
|
||||
<div style="font-size: 24px; margin-bottom: 20px; color: red;">❌ Authentication Error</div>
|
||||
<div style="font-size: 16px; color: #666;">${error.message}</div>
|
||||
<div style="margin-top: 20px;">
|
||||
<button onclick="location.reload()" style="padding: 10px 20px; font-size: 16px;">Retry</button>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
})();
|
||||
|
||||
/* ================================================================
|
||||
WORKER MESSAGE TYPES
|
||||
================================================================
|
||||
The NDK worker can send these message types:
|
||||
|
||||
1. 'response' - Response to init/subscribe/publish requests
|
||||
- data.profile - User profile (from init)
|
||||
- data.relays - User relays (from init)
|
||||
- data.success - Publish success status
|
||||
- data.relayResults - Relay publish results
|
||||
|
||||
2. 'event' - Nostr event from subscription
|
||||
- Dispatched as 'ndkEvent' window event
|
||||
- event.detail contains the Nostr event
|
||||
|
||||
3. 'eose' - End of stored events for subscription
|
||||
- Dispatched as 'ndkEose' window event
|
||||
- event.detail.subId contains subscription ID
|
||||
|
||||
4. 'signRequest' - Request to sign event/encrypt/decrypt
|
||||
- Handled automatically by init-ndk.mjs
|
||||
- Calls window.nostr methods and sends response
|
||||
|
||||
5. 'error' - Error from worker
|
||||
- Logged to console automatically
|
||||
|
||||
6. 'relayActivity' - Relay read/write activity notification
|
||||
- Dispatched as 'ndkRelayActivity' window event
|
||||
- Used to animate relay status icons in footer
|
||||
================================================================ */
|
||||
|
||||
/* ================================================================
|
||||
DISTRIBUTED ARCHITECTURE NOTES
|
||||
================================================================
|
||||
Each page is independently accessible and self-contained:
|
||||
|
||||
1. Authentication persists via nostr-login-lite localStorage
|
||||
- Login once on any page
|
||||
- All other pages automatically authenticated
|
||||
|
||||
2. NDK SharedWorker is shared across all tabs/pages
|
||||
- Single NDK instance manages all connections
|
||||
- Subscriptions from all pages handled by one worker
|
||||
- Events broadcast to all connected pages
|
||||
|
||||
3. Dexie cache is shared across all pages
|
||||
- IndexedDB persists across sessions
|
||||
- Cache-first queries are fast
|
||||
- Reduces relay load
|
||||
|
||||
4. User settings are centralized and shared
|
||||
- Worker hydrates kind 30078 (`d:user-settings`) on init
|
||||
- Pages read via getUserSettings()
|
||||
- Pages patch via patchUserSettings({ featureNamespace: ... })
|
||||
- Pages subscribe via onUserSettings() for live updates
|
||||
|
||||
5. Each page can be distributed independently
|
||||
- Copy template.html and customize
|
||||
- No dependencies on other pages
|
||||
- Works standalone or as part of suite
|
||||
|
||||
6. Message-based signer bridges worker and page
|
||||
- Worker's NDK uses MessageBasedSigner
|
||||
- Signer sends sign requests to page
|
||||
- Page calls window.nostr.signEvent()
|
||||
- Response sent back to worker
|
||||
- NDK completes signing and publishing
|
||||
|
||||
7. Relay status visualization
|
||||
- Footer left section shows connected relays
|
||||
- Each relay has animated icon (HamburgerMorphing)
|
||||
- Icons morph based on activity (read/write)
|
||||
- Temporary animations show real-time activity
|
||||
================================================================ */
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,31 @@
|
||||
# Database files (from c-relay running in this directory)
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
|
||||
# Build artifacts
|
||||
*.o
|
||||
*.a
|
||||
|
||||
# Binary (built into root, not version-controlled)
|
||||
caching_relay
|
||||
caching_relay_static_*
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
crelay.log
|
||||
|
||||
# Test configs (not for version control)
|
||||
test_config*.jsonc
|
||||
|
||||
# Note: caching_relay_config.jsonc IS version-controlled (it's the default
|
||||
# config + state store). Don't ignore it.
|
||||
|
||||
# Docker build context temp (copied nostr_core_lib during static build)
|
||||
nostr_core_lib/
|
||||
|
||||
# Editor
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.vscode/
|
||||
@@ -0,0 +1,128 @@
|
||||
# Alpine-based MUSL static binary builder for caching_relay
|
||||
# Produces a truly portable binary with zero runtime dependencies.
|
||||
# Adapted from c-relay's Dockerfile.alpine-musl.
|
||||
|
||||
ARG DEBUG_BUILD=false
|
||||
|
||||
FROM alpine:3.19 AS builder
|
||||
|
||||
ARG DEBUG_BUILD=false
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache \
|
||||
build-base \
|
||||
musl-dev \
|
||||
git \
|
||||
cmake \
|
||||
pkgconfig \
|
||||
autoconf \
|
||||
automake \
|
||||
libtool \
|
||||
openssl-dev \
|
||||
openssl-libs-static \
|
||||
zlib-dev \
|
||||
zlib-static \
|
||||
curl-dev \
|
||||
curl-static \
|
||||
sqlite-dev \
|
||||
postgresql-dev \
|
||||
linux-headers \
|
||||
wget \
|
||||
bash
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Build libsecp256k1 static
|
||||
RUN cd /tmp && \
|
||||
git clone https://github.com/bitcoin-core/secp256k1.git && \
|
||||
cd secp256k1 && \
|
||||
./autogen.sh && \
|
||||
./configure --enable-static --disable-shared --prefix=/usr \
|
||||
CFLAGS="-fPIC" && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
rm -rf /tmp/secp256k1
|
||||
|
||||
# Build libwebsockets static with minimal features
|
||||
RUN cd /tmp && \
|
||||
git clone --depth 1 --branch v4.3.3 https://github.com/warmcat/libwebsockets.git && \
|
||||
cd libwebsockets && \
|
||||
mkdir build && cd build && \
|
||||
cmake .. \
|
||||
-DLWS_WITH_STATIC=ON \
|
||||
-DLWS_WITH_SHARED=OFF \
|
||||
-DLWS_WITH_SSL=ON \
|
||||
-DLWS_WITHOUT_TESTAPPS=ON \
|
||||
-DLWS_WITHOUT_TEST_SERVER=ON \
|
||||
-DLWS_WITHOUT_TEST_CLIENT=ON \
|
||||
-DLWS_WITHOUT_TEST_PING=ON \
|
||||
-DLWS_WITH_HTTP2=OFF \
|
||||
-DLWS_WITH_LIBUV=OFF \
|
||||
-DLWS_WITH_LIBEVENT=OFF \
|
||||
-DLWS_IPV6=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DCMAKE_C_FLAGS="-fPIC" && \
|
||||
make -j$(nproc) && \
|
||||
make install && \
|
||||
rm -rf /tmp/libwebsockets
|
||||
|
||||
# Copy nostr_core_lib source (sibling project)
|
||||
COPY nostr_core_lib /build/nostr_core_lib/
|
||||
|
||||
# Build nostr_core_lib with the NIPs needed by the relay pool + signer:
|
||||
# 1 (basic), 6 (keys), 19 (npub bech32), 4 (legacy encryption, signer dep),
|
||||
# 42 (auth, relay pool dep), 44 (modern encryption, signer dep)
|
||||
RUN cd nostr_core_lib && \
|
||||
chmod +x build.sh && \
|
||||
sed -i 's/CFLAGS="-Wall -Wextra -std=c99 -fPIC -O2"/CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 -Wall -Wextra -std=c99 -fPIC -O2"/' build.sh && \
|
||||
rm -f *.o *.a 2>/dev/null || true && \
|
||||
./build.sh --nips=1,4,6,19,42,44 && \
|
||||
if [ -f libnostr_core_arm64.a ]; then \
|
||||
cp libnostr_core_arm64.a libnostr_core.a; \
|
||||
elif [ -f libnostr_core_x64.a ]; then \
|
||||
cp libnostr_core_x64.a libnostr_core.a; \
|
||||
else \
|
||||
echo "ERROR: No supported nostr_core static library produced"; \
|
||||
ls -la *.a 2>/dev/null || true; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Copy caching_relay source LAST (only this layer rebuilds on source changes)
|
||||
COPY src/ /build/src/
|
||||
|
||||
# Build caching_relay with full static linking.
|
||||
# No sqlite (the daemon doesn't use it), no c_utils (not needed).
|
||||
RUN if [ "$DEBUG_BUILD" = "true" ]; then \
|
||||
CFLAGS="-g -O2 -DDEBUG -fno-omit-frame-pointer"; \
|
||||
STRIP_CMD="echo 'Keeping debug symbols'"; \
|
||||
else \
|
||||
CFLAGS="-O2"; \
|
||||
STRIP_CMD="strip /build/caching_relay_static"; \
|
||||
fi && \
|
||||
gcc -static $CFLAGS -Wall -Wextra -std=c99 \
|
||||
-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 \
|
||||
-I. -Isrc -Inostr_core_lib -Inostr_core_lib/nostr_core \
|
||||
-Inostr_core_lib/cjson -Inostr_core_lib/nostr_websocket \
|
||||
-I/usr/include/postgresql \
|
||||
src/main.c src/debug.c src/jsonc_strip.c src/config.c src/state.c \
|
||||
src/follow_graph.c src/relay_sink.c src/live_subscriber.c \
|
||||
src/backfill.c src/relay_discovery.c \
|
||||
src/pg_inbox.c src/pg_config.c src/forward_catchup.c \
|
||||
-o /build/caching_relay_static \
|
||||
nostr_core_lib/libnostr_core.a \
|
||||
-lwebsockets -lssl -lcrypto -lsecp256k1 \
|
||||
-lcurl -lz -lpthread -lm -ldl -lpq -lpgcommon -lpgport && \
|
||||
eval "$STRIP_CMD"
|
||||
|
||||
# Verify it's truly static
|
||||
RUN echo "=== Binary Information ===" && \
|
||||
file /build/caching_relay_static && \
|
||||
ls -lh /build/caching_relay_static && \
|
||||
echo "=== Checking for dynamic dependencies ===" && \
|
||||
(ldd /build/caching_relay_static 2>&1 || echo "Binary is static") && \
|
||||
echo "=== Build complete ==="
|
||||
|
||||
# Output stage - just the binary
|
||||
FROM scratch AS output
|
||||
COPY --from=builder /build/caching_relay_static /caching_relay_static
|
||||
@@ -0,0 +1,67 @@
|
||||
# caching_relay Makefile - statically linked C99 binary, c-relay style
|
||||
|
||||
# Use bash for reliable glob expansion in recipes (dash handles globs differently).
|
||||
SHELL := /bin/bash
|
||||
|
||||
CC = gcc
|
||||
CFLAGS = -Wall -Wextra -std=c99 -g -O2
|
||||
|
||||
# nostr_core_lib is inside the c-relay-pg project root (one dir up from caching/).
|
||||
NOSTR_CORE_DIR = ../nostr_core_lib
|
||||
|
||||
INCLUDES = -I. -Isrc -I$(NOSTR_CORE_DIR) -I$(NOSTR_CORE_DIR)/nostr_core \
|
||||
-I$(NOSTR_CORE_DIR)/cjson -I$(NOSTR_CORE_DIR)/nostr_websocket \
|
||||
-I/usr/include/postgresql
|
||||
|
||||
# -lsqlite3: nostr_core_lib's request_validator.x64.o references SQLite symbols;
|
||||
# we use --whole-archive so the linker pulls in all objects (needed for NIP-42
|
||||
# cross-references within the archive), which means SQLite must be satisfied.
|
||||
# No c_utils_lib: nostr_core_lib does not depend on it for the NIPs we use.
|
||||
# -lpq: PostgreSQL client library for the caching_event_inbox integration.
|
||||
LIBS = -lwebsockets -lssl -lcrypto -lsecp256k1 -lcurl -lz -ldl -lpthread -lm -lpq -lsqlite3
|
||||
|
||||
MAIN_SRC = src/main.c src/debug.c src/jsonc_strip.c src/config.c src/state.c \
|
||||
src/follow_graph.c src/relay_sink.c src/live_subscriber.c \
|
||||
src/backfill.c src/forward_catchup.c src/relay_discovery.c \
|
||||
src/pg_inbox.c src/pg_config.c
|
||||
|
||||
# Architecture detection
|
||||
ARCH = $(shell uname -m)
|
||||
ifeq ($(ARCH),x86_64)
|
||||
NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_x64.a
|
||||
else ifeq ($(ARCH),aarch64)
|
||||
NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_arm64.a
|
||||
else ifeq ($(ARCH),arm64)
|
||||
NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_arm64.a
|
||||
else
|
||||
NOSTR_CORE_LIB = $(NOSTR_CORE_DIR)/libnostr_core_x64.a
|
||||
endif
|
||||
|
||||
# Binary goes in the c-relay-pg build/ directory so the launcher can find it.
|
||||
TARGET = ../build/caching_relay
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
# Build nostr_core_lib with the NIPs needed by the relay pool + signer:
|
||||
# 1 (basic), 6 (keys), 19 (npub bech32), 4 (legacy encryption, signer dep),
|
||||
# 42 (auth, relay pool dep), 44 (modern encryption, signer dep)
|
||||
$(NOSTR_CORE_LIB):
|
||||
@echo "Building nostr_core_lib with required NIPs..."
|
||||
cd $(NOSTR_CORE_DIR) && ./build.sh --nips=1,4,6,19,42,44
|
||||
|
||||
$(TARGET): $(MAIN_SRC) $(NOSTR_CORE_LIB)
|
||||
@echo "Compiling caching_relay for architecture: $(ARCH)"
|
||||
@# Extract all objects from the static library and link them directly.
|
||||
@# This avoids archive symbol resolution ordering issues (NIP-42 cross-refs).
|
||||
@# Use a fixed temp dir and chain all commands with && so failures stop the build.
|
||||
@rm -rf /tmp/cr_lib && mkdir -p /tmp/cr_lib && \
|
||||
ar x $(NOSTR_CORE_LIB) --output=/tmp/cr_lib && \
|
||||
echo "Extracted $$(ls /tmp/cr_lib/*.o | wc -l) objects" && \
|
||||
$(CC) $(CFLAGS) $(INCLUDES) $(MAIN_SRC) /tmp/cr_lib/*.o -o $(TARGET) $(LIBS) && \
|
||||
rm -rf /tmp/cr_lib && \
|
||||
echo "Build complete: $(TARGET)"
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET)
|
||||
|
||||
.PHONY: all clean
|
||||
@@ -0,0 +1,297 @@
|
||||
# caching_relay
|
||||
|
||||
A C99 daemon that caches Nostr events from people you follow into a local relay,
|
||||
so your Nostr client can point at a single fast local relay instead of fanning
|
||||
out to dozens of upstream relays.
|
||||
|
||||
## What it does
|
||||
|
||||
1. Reads a `.jsonc` config file listing your **root npub(s)**, upstream relays,
|
||||
a local relay URL, and the event kinds to cache.
|
||||
2. For each root npub, fetches its kind-3 contact list to discover **followed
|
||||
pubkeys** (your follows + their follows).
|
||||
3. **Live-subscribes** to new events of the configured kinds from the union of
|
||||
followed pubkeys.
|
||||
4. **Backfills** historical events using a progressive window-expansion strategy
|
||||
(24h → 7d → 30d → 90d → 365d), round-robin per pubkey, throttled to be polite
|
||||
to upstream relays.
|
||||
5. **Re-publishes** every fetched event to the local relay via a plain WebSocket
|
||||
`EVENT` client connection (relay-agnostic - works with c-relay, c-relay-pg,
|
||||
or any Nostr relay).
|
||||
6. Persists its backfill progress **in the config file itself** so a restart
|
||||
resumes where it left off.
|
||||
|
||||
## Build
|
||||
|
||||
### Static binary (recommended, c-relay style)
|
||||
|
||||
Produces a truly portable statically-linked MUSL binary with zero runtime
|
||||
dependencies:
|
||||
|
||||
```bash
|
||||
./build_static.sh
|
||||
# or with debug symbols:
|
||||
./build_static.sh --debug
|
||||
# or cross-compile for arm64:
|
||||
./build_static.sh --arch arm64
|
||||
```
|
||||
|
||||
Output: `caching_relay` (in the project root).
|
||||
|
||||
Requires Docker. The build runs in an Alpine container that compiles
|
||||
libsecp256k1, libwebsockets, and nostr_core_lib from source, then statically
|
||||
links everything.
|
||||
|
||||
### Local build (if you have the shared libs installed)
|
||||
|
||||
```bash
|
||||
make
|
||||
```
|
||||
|
||||
Output: `caching_relay` (in the project root).
|
||||
|
||||
Requires: `libwebsockets`, `openssl`, `libsecp256k1`, `libcurl`, `zlib` shared
|
||||
libraries, and a pre-built `../nostr_core_lib/libnostr_core_x64.a`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
./caching_relay -d 3
|
||||
```
|
||||
|
||||
The daemon looks for `caching_relay_config.jsonc` in the current directory by default.
|
||||
You can override with `-c`:
|
||||
|
||||
```bash
|
||||
./caching_relay -c /path/to/my_config.jsonc -d 3
|
||||
```
|
||||
|
||||
Options:
|
||||
- `-c, --config <file>` - Path to `.jsonc` config file (default: `./caching_relay_config.jsonc`)
|
||||
- `-d, --debug <level>` - Log level 1-4 (1=error, 2=warn, 3=info, 4=debug). Default 3.
|
||||
- `-h, --help` - Show help
|
||||
|
||||
Signals:
|
||||
- `SIGINT` / `SIGTERM` - Graceful shutdown (saves state, closes connections)
|
||||
- `SIGHUP` - Reload config (preserves backfill state)
|
||||
|
||||
## Config file
|
||||
|
||||
The config file [`caching_relay_config.jsonc`](caching_relay_config.jsonc) lives in the
|
||||
project root. It is JSONC (JSON with `//` and `/* */` comments). The daemon
|
||||
rewrites it as plain JSON when saving state (comments are not preserved on
|
||||
rewrite).
|
||||
|
||||
Key fields:
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `root_npubs` | npubs whose kind-3 follows list we crawl |
|
||||
| `upstream_relays` | bootstrap relays for initial discovery (kind-3, kind-10002). Outbox relays are discovered dynamically via NIP-65. |
|
||||
| `local_relay` | relay to publish cached events into (never queried) |
|
||||
| `kinds` | event kinds to cache for followed people (e.g. `[1, 3, 6, 10000, 30023]`) |
|
||||
| `admin_kinds` | kinds to follow specifically for root (admin) npubs. Use `["*"]` for all kinds. If omitted, admin uses same `kinds` as everyone else. |
|
||||
| `backfill.window_schedule_seconds` | progressive window sizes in seconds |
|
||||
| `backfill.events_per_tick` | max events per pubkey per backfill tick |
|
||||
| `backfill.tick_interval_seconds` | delay between pubkey backfills (throttle) |
|
||||
| `live.enabled` | enable live subscription |
|
||||
| `follow_graph_refresh_seconds` | how often to re-resolve the follow graph |
|
||||
| `state.*` | managed by the daemon - do not hand-edit |
|
||||
|
||||
## Architecture
|
||||
|
||||
See [`plans/plan.md`](plans/plan.md) for the full architecture document with
|
||||
Mermaid diagrams.
|
||||
|
||||
The daemon uses a **NIP-65 outbox model**: instead of querying a fixed set of
|
||||
bootstrap relays for everything, it discovers which relays each followed pubkey
|
||||
actually posts to (via kind 10002 relay lists) and connects to those relays
|
||||
dynamically. The `upstream_relays` in the config serve only as **bootstrap
|
||||
relays** -- the initial set used to discover kind-3 and kind-10002 events before
|
||||
the outbox relay map is built.
|
||||
|
||||
### Relay pool selection: minimum covering set
|
||||
|
||||
After discovering each followed pubkey's outbox relays (from their kind 10002),
|
||||
the daemon computes the **minimum set of relays that covers all followed
|
||||
pubkeys**. This is the classic set cover problem, solved with a greedy
|
||||
approximation:
|
||||
|
||||
1. Build a map: `{relay_url -> set of pubkeys that list it in their 10002}`
|
||||
2. Greedily pick the relay that covers the most uncovered pubkeys
|
||||
3. Repeat until all pubkeys are covered (or no more relays to pick)
|
||||
4. Always include the bootstrap relays in the final set (they may have events
|
||||
from pubkeys that don't publish a kind 10002)
|
||||
|
||||
This minimizes the number of WebSocket connections while ensuring every
|
||||
followed pubkey is reachable. The daemon logs the selected relay set and which
|
||||
pubkeys each relay covers.
|
||||
|
||||
Pubkeys that have no kind 10002 (or whose 10002 lists no relays) are covered by
|
||||
the bootstrap relays as a fallback.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ caching_relay │
|
||||
│ │
|
||||
upstream relays │ upstream_pool sink_pool │ local relay
|
||||
(damus, nos.lol) │ (query + subscribe) (publish only) │ (c-relay)
|
||||
│ │ │ │ │ │
|
||||
│ │ follow_graph relay_sink │ │
|
||||
└──────────►│ live_subscriber ──────►│ ├─────►│
|
||||
│ backfill ──────────────►│ │ │
|
||||
│ │ │ │ │
|
||||
│ state + seen ring │ │ │
|
||||
└──────────────────────────────────────────┘ │
|
||||
```
|
||||
|
||||
Two `nostr_relay_pool_t` instances:
|
||||
- **upstream_pool** - holds bootstrap relays initially, then dynamically adds
|
||||
outbox relays discovered from kind 10002. Used for `query_sync` (kind-3 fetch,
|
||||
kind-10002 fetch, backfill) and the long-lived live subscription.
|
||||
- **sink_pool** - holds only the local relay; used exclusively for
|
||||
`publish_async`. Never queried.
|
||||
|
||||
## Startup Flow
|
||||
|
||||
### First-time startup (empty local relay)
|
||||
|
||||
The daemon has never run before. The local relay has no cached events. The
|
||||
daemon must bootstrap from the config's `upstream_relays` to discover the
|
||||
outbox relays for each followed pubkey.
|
||||
|
||||
```
|
||||
START
|
||||
|
|
||||
v
|
||||
Load config (caching_relay_config.jsonc)
|
||||
| - root_npubs, upstream_relays (bootstrap), kinds, admin_kinds
|
||||
| - state.backfilled_until == 0 => first-time startup
|
||||
|
|
||||
v
|
||||
Create upstream_pool with bootstrap relays only
|
||||
Create sink_pool with local_relay
|
||||
|
|
||||
v
|
||||
Phase 1: Resolve follow graph (from bootstrap relays)
|
||||
| - For each root npub: query_sync kind=3 from bootstrap relays
|
||||
| - Parse "p" tags => followed pubkey set
|
||||
| - Log: "follow: resolved N followed pubkeys"
|
||||
|
|
||||
v
|
||||
Phase 2: Discover outbox relays (NIP-65, kind 10002) from bootstrap relays
|
||||
| - For each followed pubkey: query_sync kind=10002 from bootstrap relays
|
||||
| - Parse "r" tags => per-pubkey relay list
|
||||
| - Publish all kind-10002 events to local relay (cache them)
|
||||
| - Dynamically add discovered relays to upstream_pool
|
||||
| - Log: "relay_discovery: found N relays from M pubkeys"
|
||||
| - Log: "upstream_pool: now connected to N relays" + list them
|
||||
|
|
||||
v
|
||||
Phase 3: Open live subscriptions on upstream_pool
|
||||
| - follows_sub: non-admin pubkeys + regular kinds
|
||||
| - admin_sub: admin npubs + admin_kinds (or all kinds)
|
||||
|
|
||||
v
|
||||
Phase 4: Begin progressive backfill
|
||||
| - Window 0: 24h -> query each pubkey from their outbox relays
|
||||
| - Window 1: 7d -> ...
|
||||
| - Window 2: 30d -> ...
|
||||
| - Publish all fetched events to local relay
|
||||
| - Save state to config file as each window completes
|
||||
|
|
||||
v
|
||||
Steady-state: live sub + periodic follow-graph refresh + periodic
|
||||
kind-10002 refresh + backfill re-cycle
|
||||
|
|
||||
v
|
||||
SHUTDOWN (SIGINT/SIGTERM) -> save state -> clean exit
|
||||
```
|
||||
|
||||
### Subsequent startup (local relay already has cached events)
|
||||
|
||||
The daemon has run before. The local relay already has kind-3 and kind-10002
|
||||
events cached from the previous run. The daemon can read these from the local
|
||||
relay directly (fast, no network round-trip to bootstrap relays) and only falls
|
||||
back to bootstrap relays for pubkeys it cannot find locally.
|
||||
|
||||
```
|
||||
START
|
||||
|
|
||||
v
|
||||
Load config (caching_relay_config.jsonc)
|
||||
| - state.backfilled_until > 0 => subsequent startup
|
||||
| - state.current_window_index, backfill_cursor preserved
|
||||
|
|
||||
v
|
||||
Create upstream_pool with bootstrap relays
|
||||
Create sink_pool with local_relay
|
||||
|
|
||||
v
|
||||
Phase 1: Resolve follow graph (from LOCAL relay first, bootstrap fallback)
|
||||
| - Query local relay for kind=3 per root npub
|
||||
| - If found locally: use it (fast, no upstream query)
|
||||
| - If not found: fall back to bootstrap relays
|
||||
| - Parse "p" tags => followed pubkey set
|
||||
| - Log: "follow: resolved N followed pubkeys (L local, B bootstrap)"
|
||||
|
|
||||
v
|
||||
Phase 2: Discover outbox relays (from LOCAL relay first, bootstrap fallback)
|
||||
| - Query local relay for kind=10002 per followed pubkey
|
||||
| - If found locally: use it (fast)
|
||||
| - If not found: fall back to bootstrap relays, cache result to local
|
||||
| - Parse "r" tags => per-pubkey relay list
|
||||
| - Dynamically add discovered relays to upstream_pool
|
||||
| - Log: "relay_discovery: found N relays (L local, B bootstrap)"
|
||||
| - Log: "upstream_pool: now connected to N relays" + list them
|
||||
|
|
||||
v
|
||||
Phase 3: Open live subscriptions on upstream_pool
|
||||
| - (same as first-time)
|
||||
|
|
||||
v
|
||||
Phase 4: Resume backfill from saved state
|
||||
| - Resume at state.current_window_index, state.backfill_cursor
|
||||
| - No re-pull of already-backfilled windows
|
||||
| - Continue progressive window expansion from where it left off
|
||||
|
|
||||
v
|
||||
Steady-state (same as first-time)
|
||||
|
|
||||
v
|
||||
SHUTDOWN -> save state -> clean exit
|
||||
```
|
||||
|
||||
### Relay logging
|
||||
|
||||
The daemon logs all relay activity so you can see exactly which relays are
|
||||
being used:
|
||||
|
||||
- `upstream: added wss://relay.damus.io` -- each relay added to the pool
|
||||
- `relay_discovery: found 47 relays from 195 pubkeys` -- outbox discovery summary
|
||||
- `upstream_pool: 50 relays connected:` -- full relay list at startup
|
||||
- `relay_discovery: pubkey X -> wss://relay.example.com (local)` -- per-pubkey
|
||||
relay source (local cache vs bootstrap)
|
||||
- `backfill: pubkey[3/195] abc123 -> wss://relay.example.com (12 events)` --
|
||||
which relay served each backfill query
|
||||
|
||||
## Dependencies
|
||||
|
||||
- [nostr_core_lib](../nostr_core_lib) - built with NIPs 1, 4, 6, 19, 42, 44
|
||||
- libwebsockets, openssl, libsecp256k1, libcurl, zlib (all statically linked
|
||||
in the Docker build)
|
||||
|
||||
## Testing
|
||||
|
||||
Start a local c-relay, then run the daemon:
|
||||
|
||||
```bash
|
||||
# Start local relay (from c-relay project)
|
||||
cd ../c-relay/build && ./c_relay_static_x86_64 -p 8888 &
|
||||
|
||||
# Run the caching daemon (uses ./caching_relay_config.jsonc by default)
|
||||
./caching_relay -d 3
|
||||
|
||||
# Verify events are cached (from another terminal)
|
||||
nak req -k 1 -l 10 ws://127.0.0.1:8888
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
0.0.2
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Build fully static MUSL binary for caching_relay using Alpine Docker.
|
||||
# Produces a truly portable binary with zero runtime dependencies.
|
||||
# Adapted from c-relay's build_static.sh.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DOCKERFILE="$SCRIPT_DIR/Dockerfile.alpine-musl"
|
||||
|
||||
# Parse command line arguments
|
||||
DEBUG_BUILD=false
|
||||
TARGET_ARCH=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--debug)
|
||||
DEBUG_BUILD=true
|
||||
shift
|
||||
;;
|
||||
--arch)
|
||||
if [[ -z "$2" ]]; then
|
||||
echo "ERROR: --arch requires a value"
|
||||
echo "Usage: $0 [--debug] [--arch <arm64|x86_64>]"
|
||||
exit 1
|
||||
fi
|
||||
case "$2" in
|
||||
arm64|x86_64)
|
||||
TARGET_ARCH="$2"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unsupported architecture '$2'"
|
||||
echo "Supported values: arm64, x86_64"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown argument '$1'"
|
||||
echo "Usage: $0 [--debug] [--arch <arm64|x86_64>]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$DEBUG_BUILD" = true ]; then
|
||||
echo "=========================================="
|
||||
echo "caching_relay MUSL Static Binary Builder (DEBUG MODE)"
|
||||
echo "=========================================="
|
||||
else
|
||||
echo "=========================================="
|
||||
echo "caching_relay MUSL Static Binary Builder (PRODUCTION MODE)"
|
||||
echo "=========================================="
|
||||
fi
|
||||
echo "Project directory: $SCRIPT_DIR"
|
||||
echo ""
|
||||
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo "ERROR: Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
if ! docker info &> /dev/null; then
|
||||
echo "ERROR: Docker daemon is not running or user not in docker group"
|
||||
exit 1
|
||||
fi
|
||||
echo "Docker is available and running"
|
||||
echo ""
|
||||
|
||||
# Detect host architecture
|
||||
ARCH=$(uname -m)
|
||||
case "$ARCH" in
|
||||
x86_64) ARCH="x86_64";;
|
||||
aarch64) ARCH="arm64";;
|
||||
arm64) ARCH="arm64";;
|
||||
esac
|
||||
if [[ -n "$TARGET_ARCH" ]]; then
|
||||
ARCH="$TARGET_ARCH"
|
||||
echo "Using target architecture: $ARCH"
|
||||
else
|
||||
echo "Detected host architecture: $ARCH"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# nostr_core_lib is inside the c-relay-pg project root (one dir up from caching/).
|
||||
# Copy it into the Docker build context (excluding .git and build artifacts).
|
||||
NOSTR_CORE_DIR="$SCRIPT_DIR/../nostr_core_lib"
|
||||
if [ ! -d "$NOSTR_CORE_DIR" ]; then
|
||||
echo "ERROR: nostr_core_lib not found at $NOSTR_CORE_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEEDS_COPY=1
|
||||
if [ -d "$SCRIPT_DIR/nostr_core_lib" ] && [ ! -L "$SCRIPT_DIR/nostr_core_lib" ]; then
|
||||
# Already a real directory (e.g. from a previous run); assume it's good.
|
||||
NEEDS_COPY=0
|
||||
fi
|
||||
if [ "$NEEDS_COPY" = "1" ]; then
|
||||
echo "Copying nostr_core_lib into build context..."
|
||||
rm -rf "$SCRIPT_DIR/nostr_core_lib"
|
||||
mkdir -p "$SCRIPT_DIR/nostr_core_lib"
|
||||
rsync -a --exclude='.git' --exclude='*.a' --exclude='*.o' \
|
||||
--exclude='.venv*' --exclude='backups' --exclude='verify_*' \
|
||||
--exclude='rewrite_mirror' --exclude='websocket_debug' \
|
||||
"$NOSTR_CORE_DIR/" "$SCRIPT_DIR/nostr_core_lib/"
|
||||
COPIED_NOSTR_CORE=1
|
||||
fi
|
||||
|
||||
# Build args
|
||||
BUILD_ARGS="--build-arg DEBUG_BUILD=$DEBUG_BUILD"
|
||||
if [ "$ARCH" = "arm64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "arm64" ]; then
|
||||
BUILD_ARGS="$BUILD_ARGS --platform linux/arm64"
|
||||
fi
|
||||
|
||||
IMAGE_TAG="caching_relay_builder:latest"
|
||||
|
||||
echo "Building Docker image..."
|
||||
docker build $BUILD_ARGS \
|
||||
-f "$DOCKERFILE" \
|
||||
-t "$IMAGE_TAG" \
|
||||
"$SCRIPT_DIR" 2>&1
|
||||
|
||||
BUILD_RC=$?
|
||||
|
||||
# Clean up the copied nostr_core_lib to keep the project dir tidy.
|
||||
if [ "${COPIED_NOSTR_CORE:-0}" = "1" ]; then
|
||||
echo "Cleaning up copied nostr_core_lib from build context..."
|
||||
rm -rf "$SCRIPT_DIR/nostr_core_lib"
|
||||
fi
|
||||
|
||||
if [ $BUILD_RC -ne 0 ]; then
|
||||
exit $BUILD_RC
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Extracting binary from image..."
|
||||
# The output stage is FROM scratch with no CMD, so pass an empty command.
|
||||
CONTAINER_ID=$(docker create "$IMAGE_TAG" "")
|
||||
docker cp "$CONTAINER_ID:/caching_relay_static" "$SCRIPT_DIR/caching_relay"
|
||||
docker rm "$CONTAINER_ID" >/dev/null
|
||||
chmod +x "$SCRIPT_DIR/caching_relay"
|
||||
|
||||
echo ""
|
||||
echo "=== Build complete ==="
|
||||
ls -lh "$SCRIPT_DIR/caching_relay"
|
||||
file "$SCRIPT_DIR/caching_relay"
|
||||
@@ -0,0 +1,456 @@
|
||||
/*
|
||||
* caching_relay - relay-by-relay per-author until-cursor drain backfill.
|
||||
*
|
||||
* Each (author, relay) pair has its own persistent until_cursor stored in
|
||||
* the caching_backfill_relay_progress table. Each backfill tick picks the
|
||||
* next author that still has at least one incomplete relay (round-robin),
|
||||
* then queries each incomplete relay one at a time with
|
||||
* since=0 & until=cursor & limit=page_size
|
||||
* publishes the page, and advances that relay's cursor to
|
||||
* oldest_event_created_at - 1. A relay is marked complete when a page
|
||||
* returns fewer than page_size events (the relay is drained for this
|
||||
* author). An author is marked complete in caching_followed_pubkeys when
|
||||
* all their relay progress rows are complete.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "backfill.h"
|
||||
#include "follow_graph.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include "../nostr_core_lib/cjson/cJSON.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Global access for status reporting */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Singleton pointer set in cr_backfill_init, used by the accessor
|
||||
* functions below so the main loop / status reporter can see what
|
||||
* the backfill is currently working on. */
|
||||
static cr_backfill_t *g_bf = NULL;
|
||||
|
||||
const char *cr_backfill_active_pubkey(void) {
|
||||
return g_bf ? g_bf->active_pubkey : "";
|
||||
}
|
||||
const char *cr_backfill_active_relay(void) {
|
||||
return g_bf ? g_bf->active_relay : "";
|
||||
}
|
||||
int cr_backfill_active_got_eose(void) {
|
||||
return g_bf ? g_bf->active_relay_got_eose : 0;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* EOSE callback for synchronous_query_relays_with_progress */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Context passed to the query callback. Records whether the relay sent
|
||||
* EOSE (we got everything) or timed out (partial result). */
|
||||
typedef struct {
|
||||
int got_eose; /* 1 if EOSE received, 0 if timed out/errored */
|
||||
int events_count; /* events reported by callback */
|
||||
char last_status[256]; /* last status string from callback (may include message) */
|
||||
} backfill_query_ctx_t;
|
||||
|
||||
static void backfill_query_callback(const char *relay_url, const char *status,
|
||||
const char *event_id, int events_received,
|
||||
int total_relays, int completed_relays,
|
||||
void *user_data) {
|
||||
(void)relay_url; (void)total_relays; (void)completed_relays;
|
||||
backfill_query_ctx_t *ctx = (backfill_query_ctx_t *)user_data;
|
||||
if (!ctx || !status) return;
|
||||
|
||||
/* Ignore the synthetic "all_complete" status from core_relays.c — it
|
||||
* is sent AFTER the real EOSE/timeout and would overwrite the real
|
||||
* status with a misleading "all_complete" string, and clobber
|
||||
* events_count with total_unique_events. The real per-relay status
|
||||
* (eose/timeout/error) is what we need for completion decisions. */
|
||||
if (strcmp(status, "all_complete") == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* For NOTICE/CLOSED/error, event_id carries the descriptive message
|
||||
* content (relay's own text for NOTICE/CLOSED, or our transport-level
|
||||
* diagnosis for error). Include it in the status for display.
|
||||
* Priority: relay's own response (NOTICE/CLOSED) > our error > bare
|
||||
* status string. */
|
||||
if ((strcmp(status, "NOTICE") == 0 || strcmp(status, "CLOSED") == 0) &&
|
||||
event_id && event_id[0] != '\0') {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status), "%s: %s", status, event_id);
|
||||
} else if (strcmp(status, "error") == 0) {
|
||||
if (event_id && event_id[0] != '\0') {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status), "error: %s", event_id);
|
||||
} else {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status), "error");
|
||||
}
|
||||
} else {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status), "%s", status);
|
||||
}
|
||||
|
||||
if (strcmp(status, "eose") == 0) {
|
||||
ctx->got_eose = 1;
|
||||
} else if (strcmp(status, "timeout") == 0) {
|
||||
ctx->got_eose = 0;
|
||||
} else if (strcmp(status, "error") == 0) {
|
||||
ctx->got_eose = 0;
|
||||
}
|
||||
/* NOTICE/CLOSED don't change got_eose — the relay might still send
|
||||
* events or EOSE after a NOTICE. */
|
||||
ctx->events_count = events_received;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Returns 1 if the PostgreSQL inbox is initialized (PG mode active).
|
||||
* Probes the config table; a non-NULL result means the connection is up. */
|
||||
static int pg_mode_active(void) {
|
||||
char *v = pg_inbox_get_config_value("caching_root_npubs");
|
||||
if (v) { free(v); return 1; }
|
||||
char *v2 = pg_inbox_get_config_value("caching_kinds");
|
||||
if (v2) { free(v2); return 1; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Find the oldest (minimum) created_at among an array of event JSON objects.
|
||||
* Returns 1 and sets *out_oldest, or 0 if no events / no valid created_at. */
|
||||
static int find_oldest_created_at(cJSON **events, int count, long *out_oldest) {
|
||||
long oldest = 0;
|
||||
int found = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
cJSON *ca = cJSON_GetObjectItem(events[i], "created_at");
|
||||
if (!ca || !cJSON_IsNumber(ca)) continue;
|
||||
long ts = (long)ca->valuedouble;
|
||||
if (!found || ts < oldest) {
|
||||
oldest = ts;
|
||||
found = 1;
|
||||
}
|
||||
}
|
||||
if (found && out_oldest) *out_oldest = oldest;
|
||||
return found;
|
||||
}
|
||||
|
||||
/* Find the newest (maximum) created_at among an array of event JSON objects.
|
||||
* Returns 1 and sets *out_newest, or 0 if no events / no valid created_at. */
|
||||
static int find_newest_created_at(cJSON **events, int count, long *out_newest) {
|
||||
long newest = 0;
|
||||
int found = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
cJSON *ca = cJSON_GetObjectItem(events[i], "created_at");
|
||||
if (!ca || !cJSON_IsNumber(ca)) continue;
|
||||
long ts = (long)ca->valuedouble;
|
||||
if (!found || ts > newest) {
|
||||
newest = ts;
|
||||
found = 1;
|
||||
}
|
||||
}
|
||||
if (found && out_newest) *out_newest = newest;
|
||||
return found;
|
||||
}
|
||||
|
||||
/* Build the kinds array for a given pubkey (admin vs regular).
|
||||
* Returns NULL if no kinds filter should be applied (admin_all_kinds). */
|
||||
static cJSON *build_kinds(cr_config_t *cfg, const char *pk) {
|
||||
int is_admin = cr_follow_is_root(cfg, pk);
|
||||
if (is_admin && cfg->admin_all_kinds) {
|
||||
return NULL;
|
||||
}
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
if (is_admin && cfg->admin_kind_count > 0) {
|
||||
for (int i = 0; i < cfg->admin_kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
|
||||
} else {
|
||||
for (int i = 0; i < cfg->kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
|
||||
}
|
||||
return kinds;
|
||||
}
|
||||
|
||||
/* Resolve the relay URL list to query for a given pubkey. Sets *out_urls and
|
||||
* *out_n. Caller frees *out_urls (but not the strings, which alias internal
|
||||
* storage). Returns 0 on success, -1 if no relays available. */
|
||||
static int resolve_relays(nostr_relay_pool_t *upstream,
|
||||
const cr_relay_map_t *relay_map,
|
||||
const char *pk,
|
||||
const char ***out_urls,
|
||||
int *out_n) {
|
||||
*out_urls = NULL;
|
||||
*out_n = 0;
|
||||
|
||||
/* Collect outbox relays for this author (from kind-10002). */
|
||||
int outbox_count = 0;
|
||||
const cr_outbox_entry_t *oe = NULL;
|
||||
if (relay_map) {
|
||||
oe = cr_relay_map_get_outbox(relay_map, pk);
|
||||
if (oe) outbox_count = oe->relay_count;
|
||||
}
|
||||
|
||||
/* Also collect all upstream (bootstrap) relays. We query BOTH the
|
||||
* author's outbox relays AND the bootstrap relays to maximize coverage.
|
||||
* Different relays may have different subsets of the author's events. */
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int upstream_count = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
|
||||
/* Merge outbox + upstream, deduplicating by URL. */
|
||||
int max_total = outbox_count + upstream_count;
|
||||
if (max_total <= 0) {
|
||||
free(listed);
|
||||
free(statuses);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char **urls = malloc(max_total * sizeof(char *));
|
||||
if (!urls) { free(listed); free(statuses); return -1; }
|
||||
|
||||
int n = 0;
|
||||
|
||||
/* Add outbox relays first. */
|
||||
if (oe) {
|
||||
for (int j = 0; j < oe->relay_count && n < max_total; j++) {
|
||||
urls[n++] = oe->relays[j];
|
||||
}
|
||||
}
|
||||
|
||||
/* Add upstream relays, skipping duplicates. */
|
||||
for (int j = 0; j < upstream_count && n < max_total; j++) {
|
||||
int dup = 0;
|
||||
for (int k = 0; k < n; k++) {
|
||||
if (strcmp(urls[k], listed[j]) == 0) { dup = 1; break; }
|
||||
}
|
||||
if (!dup) {
|
||||
urls[n++] = listed[j];
|
||||
}
|
||||
}
|
||||
|
||||
free(listed);
|
||||
free(statuses);
|
||||
|
||||
*out_urls = urls;
|
||||
*out_n = n;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Init */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
void cr_backfill_init(cr_backfill_t *bf, cr_config_t *cfg) {
|
||||
(void)cfg;
|
||||
memset(bf, 0, sizeof(*bf));
|
||||
g_bf = bf; /* Set singleton for status accessors */
|
||||
|
||||
/* In PG mode, check whether any incomplete authors remain in the DB.
|
||||
* If so, backfill is in progress; otherwise we're in steady state.
|
||||
* Outside PG mode there is no per-author table, so we just start. */
|
||||
if (pg_mode_active()) {
|
||||
int complete = 0, total = 0;
|
||||
if (pg_inbox_count_backfill_progress(&complete, &total) == 0) {
|
||||
bf->in_progress = (total > 0 && complete < total) ? 1 : 0;
|
||||
DEBUG_INFO("backfill: init complete=%d/%d -> in_progress=%d",
|
||||
complete, total, bf->in_progress);
|
||||
return;
|
||||
}
|
||||
/* Count query failed - assume in progress so we keep trying. */
|
||||
bf->in_progress = 1;
|
||||
DEBUG_WARN("backfill: init could not read progress counts, "
|
||||
"defaulting to in_progress=1");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Legacy (non-PG) mode: no per-author table; just start. */
|
||||
bf->in_progress = 1;
|
||||
DEBUG_INFO("backfill: init (legacy mode) in_progress=1");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Tick */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
int cr_backfill_tick(cr_backfill_t *bf, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed,
|
||||
cr_sink_t *sink, const cr_relay_map_t *relay_map) {
|
||||
(void)followed; /* followed set is tracked in the DB now */
|
||||
(void)relay_map; /* relays are tracked per-author in the DB */
|
||||
|
||||
if (!cfg->backfill.enabled) return -2;
|
||||
if (!bf->in_progress) return -2;
|
||||
|
||||
/* Throttle: one query per tick_interval. */
|
||||
time_t now = time(NULL);
|
||||
if (bf->last_tick && (now - bf->last_tick) < cfg->backfill.tick_interval_seconds)
|
||||
return 0;
|
||||
|
||||
/* Pick the next author that has at least one incomplete relay progress
|
||||
* row (round-robin). */
|
||||
char pk[CR_HEX_LEN];
|
||||
if (pg_inbox_pick_next_author_with_incomplete_relays(pk, sizeof(pk),
|
||||
&bf->author_round_cursor) != 0) {
|
||||
/* No incomplete authors - steady state. */
|
||||
bf->in_progress = 0;
|
||||
DEBUG_INFO("backfill: no authors with incomplete relays, steady-state");
|
||||
return -2;
|
||||
}
|
||||
|
||||
int page_size = cfg->backfill.events_per_tick;
|
||||
if (page_size < 1) page_size = 500;
|
||||
|
||||
bf->last_tick = now;
|
||||
|
||||
/* Get the incomplete relays for this author. */
|
||||
cJSON *relays = pg_inbox_get_incomplete_relays(pk);
|
||||
if (!relays) {
|
||||
DEBUG_WARN("backfill: no incomplete relays returned for %s", pk);
|
||||
/* Author has no relay rows - mark complete so we don't loop. */
|
||||
pg_inbox_check_and_mark_author_complete(pk);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int relay_count = cJSON_GetArraySize(relays);
|
||||
DEBUG_TRACE("backfill: author %s has %d incomplete relay(s)", pk, relay_count);
|
||||
|
||||
cr_sink_set_source_class(sink, CR_SINK_CLASS_BACKFILL);
|
||||
|
||||
/* Query each incomplete relay one at a time. */
|
||||
for (int r = 0; r < relay_count; r++) {
|
||||
cJSON *entry = cJSON_GetArrayItem(relays, r);
|
||||
if (!entry) continue;
|
||||
cJSON *url_node = cJSON_GetObjectItem(entry, "relay_url");
|
||||
cJSON *cur_node = cJSON_GetObjectItem(entry, "until_cursor");
|
||||
if (!url_node || !cJSON_IsString(url_node)) continue;
|
||||
const char *relay_url = cJSON_GetStringValue(url_node);
|
||||
long until_cursor = (cur_node && cJSON_IsNumber(cur_node))
|
||||
? (long)cur_node->valuedouble : 0;
|
||||
if (until_cursor == 0) until_cursor = (long)now;
|
||||
|
||||
/* Build the filter for this single relay. */
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(pk));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
cJSON *kinds = build_kinds(cfg, pk);
|
||||
if (kinds) cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber(0.0));
|
||||
cJSON_AddItemToObject(filter, "until", cJSON_CreateNumber((double)until_cursor));
|
||||
cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber((double)page_size));
|
||||
|
||||
DEBUG_TRACE("backfill: %s @ %s (until=%ld, limit=%d)",
|
||||
pk, relay_url, until_cursor, page_size);
|
||||
|
||||
/* Record the active target for UI status display (both
|
||||
* in-memory for local accessors and in PG for the relay). */
|
||||
strncpy(bf->active_pubkey, pk, sizeof(bf->active_pubkey) - 1);
|
||||
bf->active_pubkey[sizeof(bf->active_pubkey) - 1] = '\0';
|
||||
strncpy(bf->active_relay, relay_url, sizeof(bf->active_relay) - 1);
|
||||
bf->active_relay[sizeof(bf->active_relay) - 1] = '\0';
|
||||
bf->active_relay_got_eose = 0;
|
||||
pg_inbox_set_active_target(pk, relay_url);
|
||||
|
||||
/* Query this single relay using synchronous_query_relays_with_progress
|
||||
* which gives us an EOSE/timeout callback — the relay's own signal
|
||||
* of whether it sent everything or we timed out. */
|
||||
backfill_query_ctx_t qctx = {0, 0};
|
||||
const char *one_url = relay_url;
|
||||
int ev_count = 0;
|
||||
cJSON **events = synchronous_query_relays_with_progress(
|
||||
&one_url, 1, filter, RELAY_QUERY_ALL_RESULTS,
|
||||
&ev_count, 30, /* timeout in seconds */
|
||||
backfill_query_callback, &qctx,
|
||||
0, NULL /* nip42 disabled, no private key */);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
bf->active_relay_got_eose = qctx.got_eose;
|
||||
|
||||
if (!events || ev_count == 0) {
|
||||
/* No events from this relay - it is drained for this author.
|
||||
* (ev_count == 0 with EOSE means truly empty; without EOSE it
|
||||
* means a connection failure — keep incomplete to retry.
|
||||
* The pg_inbox_update_relay_progress SQL will auto-mark
|
||||
* complete once consecutive_errors reaches 3.) */
|
||||
int mark_complete = qctx.got_eose ? 1 : 0;
|
||||
pg_inbox_update_relay_progress(pk, relay_url, until_cursor,
|
||||
mark_complete, 0, qctx.last_status);
|
||||
free(events);
|
||||
if (!qctx.got_eose) {
|
||||
/* Error or timeout with no events — surface the
|
||||
* descriptive status at WARN so operators can see why. */
|
||||
DEBUG_WARN("backfill: %s @ %s failed: %s (will auto-complete after 3 consecutive errors)",
|
||||
pk, relay_url, qctx.last_status);
|
||||
} else {
|
||||
DEBUG_LOG("backfill: %s @ %s complete (no events, eose)",
|
||||
pk, relay_url);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Find oldest and newest timestamps, then publish every event. */
|
||||
long oldest_ts = 0;
|
||||
long newest_ts = 0;
|
||||
find_oldest_created_at(events, ev_count, &oldest_ts);
|
||||
find_newest_created_at(events, ev_count, &newest_ts);
|
||||
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cr_sink_publish(sink, events[k]);
|
||||
cJSON_Delete(events[k]);
|
||||
}
|
||||
free(events);
|
||||
bf->events_total += ev_count;
|
||||
|
||||
/* Update last_event_at for forward catch-up gap tracking. */
|
||||
if (newest_ts > 0) {
|
||||
pg_inbox_update_last_event_at(pk, newest_ts);
|
||||
}
|
||||
|
||||
DEBUG_LOG("backfill: %s @ %s -> %d events (until=%ld, oldest=%ld, eose=%d)",
|
||||
pk, relay_url, ev_count, until_cursor, oldest_ts, qctx.got_eose);
|
||||
|
||||
/* Completion decision based on EOSE status:
|
||||
* - ev_count >= page_size: saturated, advance cursor, keep incomplete
|
||||
* - ev_count < page_size AND got EOSE: relay is truly drained
|
||||
* - ev_count < page_size AND no EOSE (timeout): partial result,
|
||||
* advance cursor but keep incomplete to retry next tick
|
||||
* - Guard: if cursor didn't advance (oldest_ts >= until_cursor),
|
||||
* mark complete to avoid infinite loop on identical timestamps. */
|
||||
long next_cursor = (oldest_ts > 0) ? oldest_ts - 1 : 0;
|
||||
int cursor_advanced = (oldest_ts > 0 && oldest_ts < until_cursor);
|
||||
|
||||
if (ev_count >= page_size) {
|
||||
pg_inbox_update_relay_progress(pk, relay_url, next_cursor, 0, ev_count, qctx.last_status);
|
||||
} else if (qctx.got_eose) {
|
||||
/* Relay sent EOSE with < page_size events — truly drained. */
|
||||
pg_inbox_update_relay_progress(pk, relay_url, next_cursor, 1, ev_count, qctx.last_status);
|
||||
} else {
|
||||
/* Timed out before EOSE — partial result. Keep incomplete
|
||||
* to retry, unless the cursor didn't advance (stall guard).
|
||||
* Timeouts DO count toward the consecutive_errors limit
|
||||
* (a relay that consistently times out is effectively dead
|
||||
* for backfill purposes), so the SQL will auto-mark
|
||||
* complete after 3 consecutive timeouts/errors. */
|
||||
int mark_complete = cursor_advanced ? 0 : 1;
|
||||
pg_inbox_update_relay_progress(pk, relay_url, next_cursor,
|
||||
mark_complete, ev_count, qctx.last_status);
|
||||
DEBUG_WARN("backfill: %s @ %s partial: %s (%d events, will auto-complete after 3 consecutive timeouts/errors)",
|
||||
pk, relay_url, qctx.last_status, ev_count);
|
||||
}
|
||||
}
|
||||
|
||||
/* Clear active target — done with this author. */
|
||||
bf->active_relay[0] = '\0';
|
||||
pg_inbox_set_active_target(pk, "");
|
||||
|
||||
cJSON_Delete(relays);
|
||||
|
||||
/* After querying all incomplete relays for this author, check whether
|
||||
* the author is now fully complete. */
|
||||
int crc = pg_inbox_check_and_mark_author_complete(pk);
|
||||
if (crc == 1) {
|
||||
DEBUG_LOG("backfill: author %s fully complete (all relays drained)", pk);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* caching_relay - per-author until-cursor drain backfill.
|
||||
*
|
||||
* Each followed author has a single persistent cursor (starting at `now`)
|
||||
* stored in the caching_followed_pubkeys table. Each backfill tick picks
|
||||
* the next incomplete author (round-robin), queries
|
||||
* since=0 & until=cursor & limit=page_size
|
||||
* publishes the page, and sets cursor = oldest_event_created_at - 1.
|
||||
* Repeat until a page returns fewer than page_size events (beginning of
|
||||
* the author's history reached), then mark the author complete.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_BACKFILL_H
|
||||
#define CACHING_RELAY_BACKFILL_H
|
||||
|
||||
#include "config.h"
|
||||
#include "state.h"
|
||||
#include "relay_sink.h"
|
||||
#include "relay_discovery.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
typedef struct {
|
||||
int in_progress; /* 1 if any incomplete authors remain */
|
||||
long events_total; /* events pulled across all ticks (diagnostic) */
|
||||
time_t last_tick; /* last time an author was queried (throttle) */
|
||||
int author_round_cursor;/* round-robin cursor for author selection */
|
||||
/* Current backfill target (for UI status display). */
|
||||
char active_pubkey[65]; /* pubkey currently being backfilled (or "") */
|
||||
char active_relay[256]; /* relay URL currently being queried (or "") */
|
||||
int active_relay_got_eose; /* 1 if last query got EOSE, 0 if timed out */
|
||||
} cr_backfill_t;
|
||||
|
||||
/* Global access to the current backfill target for status reporting.
|
||||
* Returns pointers to static storage in the singleton cr_backfill_t.
|
||||
* Safe to call from the main loop; not thread-safe but backfill is
|
||||
* single-threaded. */
|
||||
const char *cr_backfill_active_pubkey(void);
|
||||
const char *cr_backfill_active_relay(void);
|
||||
int cr_backfill_active_got_eose(void);
|
||||
|
||||
/* Initialize backfill state. Checks the caching_followed_pubkeys table for
|
||||
* any incomplete authors; sets in_progress accordingly. */
|
||||
void cr_backfill_init(cr_backfill_t *bf, cr_config_t *cfg);
|
||||
|
||||
/* Perform one backfill tick. Picks the next incomplete author from the DB
|
||||
* (round-robin), queries one page of history, publishes it to the sink, and
|
||||
* updates the per-author cursor in the DB.
|
||||
*
|
||||
* Returns:
|
||||
* 1 if a tick was performed (an author was queried)
|
||||
* 0 if throttled (tick_interval not elapsed) - caller should pump pools
|
||||
* -2 if backfill is complete (no incomplete authors) - caller goes steady-state
|
||||
*/
|
||||
int cr_backfill_tick(cr_backfill_t *bf, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed,
|
||||
cr_sink_t *sink, const cr_relay_map_t *relay_map);
|
||||
|
||||
#endif /* CACHING_RELAY_BACKFILL_H */
|
||||
@@ -0,0 +1,290 @@
|
||||
/*
|
||||
* caching_relay - config parsing + persistent state (in the .jsonc file itself)
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "config.h"
|
||||
#include "jsonc_strip.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include "../nostr_core_lib/cjson/cJSON.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
static int read_file(const char *path, char **out, size_t *out_len) {
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) {
|
||||
DEBUG_ERROR("cannot open config '%s': %s", path, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; }
|
||||
long sz = ftell(f);
|
||||
if (sz < 0) { fclose(f); return -1; }
|
||||
rewind(f);
|
||||
char *buf = malloc(sz + 1);
|
||||
if (!buf) { fclose(f); return -1; }
|
||||
size_t n = fread(buf, 1, sz, f);
|
||||
fclose(f);
|
||||
buf[n] = '\0';
|
||||
*out = buf;
|
||||
if (out_len) *out_len = n;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void copy_string_array(cJSON *arr, char *dst, int max, int *count_out,
|
||||
int elem_len) {
|
||||
int count = 0;
|
||||
if (arr && cJSON_IsArray(arr)) {
|
||||
cJSON *item;
|
||||
cJSON_ArrayForEach(item, arr) {
|
||||
if (count >= max) break;
|
||||
if (!cJSON_IsString(item)) continue;
|
||||
const char *s = cJSON_GetStringValue(item);
|
||||
if (!s) continue;
|
||||
strncpy(dst + count * elem_len, s, elem_len - 1);
|
||||
dst[count * elem_len + (elem_len - 1)] = '\0';
|
||||
count++;
|
||||
}
|
||||
}
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
static void copy_int_array(cJSON *arr, int *dst, int max, int *count_out) {
|
||||
int count = 0;
|
||||
if (arr && cJSON_IsArray(arr)) {
|
||||
cJSON *item;
|
||||
cJSON_ArrayForEach(item, arr) {
|
||||
if (count >= max) break;
|
||||
if (cJSON_IsNumber(item)) {
|
||||
dst[count++] = (int)cJSON_GetNumberValue(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
static void copy_long_array(cJSON *arr, long *dst, int max, int *count_out) {
|
||||
int count = 0;
|
||||
if (arr && cJSON_IsArray(arr)) {
|
||||
cJSON *item;
|
||||
cJSON_ArrayForEach(item, arr) {
|
||||
if (count >= max) break;
|
||||
if (cJSON_IsNumber(item)) {
|
||||
dst[count++] = (long)cJSON_GetNumberValue(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
int cr_config_load(cr_config_t *cfg, const char *path) {
|
||||
memset(cfg, 0, sizeof(*cfg));
|
||||
strncpy(cfg->path, path, sizeof(cfg->path) - 1);
|
||||
|
||||
char *raw = NULL;
|
||||
size_t raw_len = 0;
|
||||
if (read_file(path, &raw, &raw_len) != 0) return -1;
|
||||
|
||||
char *stripped = jsonc_strip_comments(raw, raw_len);
|
||||
free(raw);
|
||||
if (!stripped) {
|
||||
DEBUG_ERROR("failed to strip comments from config");
|
||||
return -1;
|
||||
}
|
||||
|
||||
cJSON *root = cJSON_Parse(stripped);
|
||||
free(stripped);
|
||||
if (!root) {
|
||||
DEBUG_ERROR("failed to parse config JSON");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* root_npubs */
|
||||
copy_string_array(cJSON_GetObjectItem(root, "root_npubs"),
|
||||
(char *)cfg->root_npubs, CR_MAX_ROOT_NPUBS,
|
||||
&cfg->root_npub_count, CR_NPUB_LEN);
|
||||
|
||||
/* upstream_relays */
|
||||
copy_string_array(cJSON_GetObjectItem(root, "upstream_relays"),
|
||||
(char *)cfg->upstream_relays, CR_MAX_UPSTREAM,
|
||||
&cfg->upstream_count, CR_URL_LEN);
|
||||
|
||||
/* local_relay */
|
||||
cJSON *local = cJSON_GetObjectItem(root, "local_relay");
|
||||
if (local && cJSON_IsString(local)) {
|
||||
strncpy(cfg->local_relay, cJSON_GetStringValue(local), CR_URL_LEN - 1);
|
||||
}
|
||||
|
||||
/* kinds */
|
||||
copy_int_array(cJSON_GetObjectItem(root, "kinds"),
|
||||
cfg->kinds, CR_MAX_KINDS, &cfg->kind_count);
|
||||
|
||||
/* admin_kinds - kinds to follow specifically for root (admin) npubs.
|
||||
* Supports [*] to mean "all kinds" (no kind filter for admin). */
|
||||
cJSON *ak = cJSON_GetObjectItem(root, "admin_kinds");
|
||||
if (ak && cJSON_IsArray(ak)) {
|
||||
cJSON *item;
|
||||
cJSON_ArrayForEach(item, ak) {
|
||||
if (cJSON_IsString(item)) {
|
||||
const char *s = cJSON_GetStringValue(item);
|
||||
if (s && strcmp(s, "*") == 0) {
|
||||
cfg->admin_all_kinds = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!cfg->admin_all_kinds) {
|
||||
copy_int_array(ak, cfg->admin_kinds, CR_MAX_KINDS, &cfg->admin_kind_count);
|
||||
}
|
||||
}
|
||||
|
||||
/* backfill */
|
||||
cJSON *bf = cJSON_GetObjectItem(root, "backfill");
|
||||
if (bf) {
|
||||
cfg->backfill.enabled = cJSON_IsTrue(cJSON_GetObjectItem(bf, "enabled"));
|
||||
cJSON *ept = cJSON_GetObjectItem(bf, "events_per_tick");
|
||||
if (ept) cfg->backfill.events_per_tick = (int)cJSON_GetNumberValue(ept);
|
||||
cJSON *tis = cJSON_GetObjectItem(bf, "tick_interval_seconds");
|
||||
if (tis) cfg->backfill.tick_interval_seconds = (int)cJSON_GetNumberValue(tis);
|
||||
}
|
||||
|
||||
/* live */
|
||||
cJSON *lv = cJSON_GetObjectItem(root, "live");
|
||||
if (lv) {
|
||||
cfg->live.enabled = cJSON_IsTrue(cJSON_GetObjectItem(lv, "enabled"));
|
||||
cJSON *rsi = cJSON_GetObjectItem(lv, "resubscribe_interval_seconds");
|
||||
if (rsi) cfg->live.resubscribe_interval_seconds = (int)cJSON_GetNumberValue(rsi);
|
||||
}
|
||||
|
||||
/* follow_graph_refresh_seconds */
|
||||
cJSON *fgr = cJSON_GetObjectItem(root, "follow_graph_refresh_seconds");
|
||||
if (fgr) cfg->follow_graph_refresh_seconds = (int)cJSON_GetNumberValue(fgr);
|
||||
|
||||
/* state */
|
||||
cJSON *st = cJSON_GetObjectItem(root, "state");
|
||||
if (st) {
|
||||
cJSON *bu = cJSON_GetObjectItem(st, "backfilled_until");
|
||||
if (bu) cfg->state.backfilled_until = (long)cJSON_GetNumberValue(bu);
|
||||
}
|
||||
|
||||
cJSON_Delete(root);
|
||||
|
||||
/* Defaults if missing. */
|
||||
if (cfg->backfill.events_per_tick == 0) cfg->backfill.events_per_tick = 500;
|
||||
if (cfg->backfill.tick_interval_seconds == 0) cfg->backfill.tick_interval_seconds = 5;
|
||||
if (cfg->live.resubscribe_interval_seconds == 0) cfg->live.resubscribe_interval_seconds = 300;
|
||||
if (cfg->follow_graph_refresh_seconds == 0) cfg->follow_graph_refresh_seconds = 600;
|
||||
|
||||
/* Validation. */
|
||||
if (cfg->root_npub_count == 0) {
|
||||
DEBUG_ERROR("config: at least one root_npub required");
|
||||
return -1;
|
||||
}
|
||||
if (cfg->upstream_count == 0) {
|
||||
DEBUG_ERROR("config: at least one upstream_relay required");
|
||||
return -1;
|
||||
}
|
||||
if (cfg->local_relay[0] == '\0') {
|
||||
DEBUG_ERROR("config: local_relay required");
|
||||
return -1;
|
||||
}
|
||||
if (cfg->kind_count == 0) {
|
||||
DEBUG_ERROR("config: at least one kind required");
|
||||
return -1;
|
||||
}
|
||||
|
||||
DEBUG_INFO("config loaded: %d root npubs, %d upstream relays, %d kinds",
|
||||
cfg->root_npub_count, cfg->upstream_count, cfg->kind_count);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Build a fresh cJSON tree from the in-memory config and write it out.
|
||||
* We re-emit plain JSON (comments are not preserved on rewrite). */
|
||||
int cr_config_save_state(cr_config_t *cfg) {
|
||||
cJSON *root = cJSON_CreateObject();
|
||||
|
||||
cJSON *npubs = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->root_npub_count; i++)
|
||||
cJSON_AddItemToArray(npubs, cJSON_CreateString(cfg->root_npubs[i]));
|
||||
cJSON_AddItemToObject(root, "root_npubs", npubs);
|
||||
|
||||
cJSON *upstream = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->upstream_count; i++)
|
||||
cJSON_AddItemToArray(upstream, cJSON_CreateString(cfg->upstream_relays[i]));
|
||||
cJSON_AddItemToObject(root, "upstream_relays", upstream);
|
||||
|
||||
cJSON_AddItemToObject(root, "local_relay", cJSON_CreateString(cfg->local_relay));
|
||||
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
|
||||
cJSON_AddItemToObject(root, "kinds", kinds);
|
||||
|
||||
/* admin_kinds */
|
||||
cJSON *akinds = cJSON_CreateArray();
|
||||
if (cfg->admin_all_kinds) {
|
||||
cJSON_AddItemToArray(akinds, cJSON_CreateString("*"));
|
||||
} else {
|
||||
for (int i = 0; i < cfg->admin_kind_count; i++)
|
||||
cJSON_AddItemToArray(akinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
|
||||
}
|
||||
cJSON_AddItemToObject(root, "admin_kinds", akinds);
|
||||
|
||||
cJSON *bf = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(bf, "enabled", cJSON_CreateBool(cfg->backfill.enabled));
|
||||
cJSON_AddItemToObject(bf, "events_per_tick", cJSON_CreateNumber(cfg->backfill.events_per_tick));
|
||||
cJSON_AddItemToObject(bf, "tick_interval_seconds", cJSON_CreateNumber(cfg->backfill.tick_interval_seconds));
|
||||
cJSON_AddItemToObject(root, "backfill", bf);
|
||||
|
||||
cJSON *lv = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(lv, "enabled", cJSON_CreateBool(cfg->live.enabled));
|
||||
cJSON_AddItemToObject(lv, "resubscribe_interval_seconds", cJSON_CreateNumber(cfg->live.resubscribe_interval_seconds));
|
||||
cJSON_AddItemToObject(root, "live", lv);
|
||||
|
||||
cJSON_AddItemToObject(root, "follow_graph_refresh_seconds",
|
||||
cJSON_CreateNumber(cfg->follow_graph_refresh_seconds));
|
||||
|
||||
cJSON *st = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(st, "backfilled_until", cJSON_CreateNumber(cfg->state.backfilled_until));
|
||||
cJSON_AddItemToObject(root, "state", st);
|
||||
|
||||
char *json = cJSON_Print(root);
|
||||
cJSON_Delete(root);
|
||||
if (!json) {
|
||||
DEBUG_ERROR("config save: failed to serialize");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Atomic write: temp file + rename. */
|
||||
char tmp[1100];
|
||||
snprintf(tmp, sizeof(tmp), "%s.tmp", cfg->path);
|
||||
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||
if (fd < 0) {
|
||||
DEBUG_ERROR("config save: cannot open tmp '%s': %s", tmp, strerror(errno));
|
||||
free(json);
|
||||
return -1;
|
||||
}
|
||||
size_t jlen = strlen(json);
|
||||
ssize_t w = write(fd, json, jlen);
|
||||
close(fd);
|
||||
free(json);
|
||||
if (w < 0 || (size_t)w != jlen) {
|
||||
DEBUG_ERROR("config save: short write");
|
||||
return -1;
|
||||
}
|
||||
if (rename(tmp, cfg->path) != 0) {
|
||||
DEBUG_ERROR("config save: rename failed: %s", strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
DEBUG_LOG("config state saved: backfilled_until=%ld",
|
||||
cfg->state.backfilled_until);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cr_config_free(cr_config_t *cfg) {
|
||||
(void)cfg;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* caching_relay - config parsing + persistent state (in the .jsonc file itself)
|
||||
*/
|
||||
#ifndef CACHING_RELAY_CONFIG_H
|
||||
#define CACHING_RELAY_CONFIG_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Maximums to keep things statically sized and simple. */
|
||||
#define CR_MAX_ROOT_NPUBS 16
|
||||
#define CR_MAX_UPSTREAM 32
|
||||
#define CR_MAX_KINDS 32
|
||||
#define CR_NPUB_LEN 64 /* npub1... bech32, generous */
|
||||
#define CR_URL_LEN 256
|
||||
#define CR_HEX_PUBKEY_LEN 65 /* 64 hex chars + NUL */
|
||||
|
||||
typedef struct {
|
||||
int enabled;
|
||||
int events_per_tick; /* per-query page size (default 500) */
|
||||
int tick_interval_seconds;
|
||||
} cr_backfill_config_t;
|
||||
|
||||
typedef struct {
|
||||
int enabled;
|
||||
int resubscribe_interval_seconds;
|
||||
} cr_live_config_t;
|
||||
|
||||
/* Persistent state. The per-author until-cursor drain model stores all
|
||||
* backfill progress in the caching_followed_pubkeys table, so the only
|
||||
* state kept here is the legacy-mode first-time flag (used to drive relay
|
||||
* discovery caching behavior). */
|
||||
typedef struct {
|
||||
long backfilled_until; /* legacy: 0 = first-time startup */
|
||||
} cr_state_t;
|
||||
|
||||
typedef struct {
|
||||
char root_npubs[CR_MAX_ROOT_NPUBS][CR_NPUB_LEN];
|
||||
int root_npub_count;
|
||||
/* Decoded hex pubkeys for root npubs (filled by follow_graph). */
|
||||
char root_hex[CR_MAX_ROOT_NPUBS][CR_HEX_PUBKEY_LEN];
|
||||
int root_hex_ready;
|
||||
|
||||
char upstream_relays[CR_MAX_UPSTREAM][CR_URL_LEN];
|
||||
int upstream_count;
|
||||
|
||||
char local_relay[CR_URL_LEN];
|
||||
|
||||
int kinds[CR_MAX_KINDS];
|
||||
int kind_count;
|
||||
|
||||
/* Kinds to follow specifically for the root (admin) npubs.
|
||||
* If admin_all_kinds is 1, grab everything (no kind filter). */
|
||||
int admin_kinds[CR_MAX_KINDS];
|
||||
int admin_kind_count;
|
||||
int admin_all_kinds;
|
||||
|
||||
cr_backfill_config_t backfill;
|
||||
cr_live_config_t live;
|
||||
int follow_graph_refresh_seconds;
|
||||
|
||||
cr_state_t state;
|
||||
|
||||
/* Path the config was loaded from (for state write-back). */
|
||||
char path[1024];
|
||||
} cr_config_t;
|
||||
|
||||
/* Load + validate config from a .jsonc file. Returns 0 on success, -1 on error.
|
||||
* On success cfg->path is set and cfg->state is populated from the file. */
|
||||
int cr_config_load(cr_config_t *cfg, const char *path);
|
||||
|
||||
/* Persist the state sub-object back into the config file (temp + rename).
|
||||
* Preserves all other config fields. Returns 0 on success, -1 on error. */
|
||||
int cr_config_save_state(cr_config_t *cfg);
|
||||
|
||||
/* Free any heap resources held by cfg (currently none, but kept for future). */
|
||||
void cr_config_free(cr_config_t *cfg);
|
||||
|
||||
#endif /* CACHING_RELAY_CONFIG_H */
|
||||
@@ -0,0 +1,71 @@
|
||||
#include "debug.h"
|
||||
#include <stdarg.h>
|
||||
#include <string.h>
|
||||
|
||||
/**
|
||||
* @file debug.c
|
||||
* @brief Debug and logging system implementation
|
||||
*
|
||||
* Provides a configurable logging system with timestamp formatting,
|
||||
* level-based filtering, and optional file:line information.
|
||||
*/
|
||||
|
||||
// Global debug level (default: no debug output)
|
||||
debug_level_t g_debug_level = DEBUG_LEVEL_NONE;
|
||||
|
||||
/**
|
||||
* @brief Initialize the debug system with a specific level
|
||||
* @param level Debug level (0-5, clamped to valid range)
|
||||
*/
|
||||
void debug_init(int level) {
|
||||
if (level < 0) level = 0;
|
||||
if (level > 5) level = 5;
|
||||
g_debug_level = (debug_level_t)level;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Core logging function
|
||||
* @param level Debug level for this message
|
||||
* @param file Source file name (__FILE__)
|
||||
* @param line Source line number (__LINE__)
|
||||
* @param format printf-style format string
|
||||
* @param ... Variable arguments for format string
|
||||
*/
|
||||
void debug_log(debug_level_t level, const char* file, int line, const char* format, ...) {
|
||||
// Get timestamp
|
||||
time_t now = time(NULL);
|
||||
struct tm* tm_info = localtime(&now);
|
||||
char timestamp[32];
|
||||
strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", tm_info);
|
||||
|
||||
// Get level string
|
||||
const char* level_str = "UNKNOWN";
|
||||
switch (level) {
|
||||
case DEBUG_LEVEL_ERROR: level_str = "ERROR"; break;
|
||||
case DEBUG_LEVEL_WARN: level_str = "WARN "; break;
|
||||
case DEBUG_LEVEL_INFO: level_str = "INFO "; break;
|
||||
case DEBUG_LEVEL_DEBUG: level_str = "DEBUG"; break;
|
||||
case DEBUG_LEVEL_TRACE: level_str = "TRACE"; break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
// Print prefix with timestamp and level
|
||||
printf("[%s] [%s] ", timestamp, level_str);
|
||||
|
||||
// Print source location when debug level is TRACE (5) or higher
|
||||
if (file && g_debug_level >= DEBUG_LEVEL_TRACE) {
|
||||
// Extract just the filename (not full path)
|
||||
const char* filename = strrchr(file, '/');
|
||||
filename = filename ? filename + 1 : file;
|
||||
printf("[%s:%d] ", filename, line);
|
||||
}
|
||||
|
||||
// Print message
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
vprintf(format, args);
|
||||
va_end(args);
|
||||
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#ifndef C_UTILS_DEBUG_H
|
||||
#define C_UTILS_DEBUG_H
|
||||
|
||||
/**
|
||||
* @file debug.h
|
||||
* @brief Debug and logging system with configurable verbosity levels
|
||||
*
|
||||
* Provides a simple, efficient logging system with 5 levels:
|
||||
* - ERROR: Critical errors
|
||||
* - WARN: Warnings
|
||||
* - INFO: Informational messages
|
||||
* - DEBUG: Debug messages
|
||||
* - TRACE: Detailed trace with file:line info
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <time.h>
|
||||
|
||||
// Debug levels
|
||||
typedef enum {
|
||||
DEBUG_LEVEL_NONE = 0, /**< No debug output */
|
||||
DEBUG_LEVEL_ERROR = 1, /**< Critical errors only */
|
||||
DEBUG_LEVEL_WARN = 2, /**< Warnings and above */
|
||||
DEBUG_LEVEL_INFO = 3, /**< Informational messages and above */
|
||||
DEBUG_LEVEL_DEBUG = 4, /**< Debug messages and above */
|
||||
DEBUG_LEVEL_TRACE = 5 /**< Detailed trace with file:line info */
|
||||
} debug_level_t;
|
||||
|
||||
// Global debug level (set at runtime via CLI)
|
||||
extern debug_level_t g_debug_level;
|
||||
|
||||
/**
|
||||
* @brief Initialize the debug system
|
||||
* @param level Debug level (0-5, clamped to valid range)
|
||||
*/
|
||||
void debug_init(int level);
|
||||
|
||||
/**
|
||||
* @brief Core logging function
|
||||
* @param level Debug level for this message
|
||||
* @param file Source file name (__FILE__)
|
||||
* @param line Source line number (__LINE__)
|
||||
* @param format printf-style format string
|
||||
* @param ... Variable arguments for format string
|
||||
*/
|
||||
void debug_log(debug_level_t level, const char* file, int line, const char* format, ...);
|
||||
|
||||
// Convenience macros that check level before calling
|
||||
// Note: TRACE level (5) and above include file:line information for ALL messages
|
||||
#define DEBUG_ERROR(...) \
|
||||
do { if (g_debug_level >= DEBUG_LEVEL_ERROR) debug_log(DEBUG_LEVEL_ERROR, __FILE__, __LINE__, __VA_ARGS__); } while(0)
|
||||
|
||||
#define DEBUG_WARN(...) \
|
||||
do { if (g_debug_level >= DEBUG_LEVEL_WARN) debug_log(DEBUG_LEVEL_WARN, __FILE__, __LINE__, __VA_ARGS__); } while(0)
|
||||
|
||||
#define DEBUG_INFO(...) \
|
||||
do { if (g_debug_level >= DEBUG_LEVEL_INFO) debug_log(DEBUG_LEVEL_INFO, __FILE__, __LINE__, __VA_ARGS__); } while(0)
|
||||
|
||||
#define DEBUG_LOG(...) \
|
||||
do { if (g_debug_level >= DEBUG_LEVEL_DEBUG) debug_log(DEBUG_LEVEL_DEBUG, __FILE__, __LINE__, __VA_ARGS__); } while(0)
|
||||
|
||||
#define DEBUG_TRACE(...) \
|
||||
do { if (g_debug_level >= DEBUG_LEVEL_TRACE) debug_log(DEBUG_LEVEL_TRACE, __FILE__, __LINE__, __VA_ARGS__); } while(0)
|
||||
|
||||
#endif /* C_UTILS_DEBUG_H */
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* caching_relay - follow graph resolution
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "follow_graph.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <ctype.h>
|
||||
|
||||
int cr_follow_is_root(const cr_config_t *cfg, const char *hex) {
|
||||
if (!cfg->root_hex_ready) return 0;
|
||||
for (int i = 0; i < cfg->root_npub_count; i++) {
|
||||
if (strcmp(cfg->root_hex[i], hex) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Check if a string is a 64-character hex pubkey (no npub prefix). */
|
||||
static int is_hex_pubkey(const char *s) {
|
||||
if (!s || strlen(s) != 64) return 0;
|
||||
for (int i = 0; i < 64; i++) {
|
||||
if (!isxdigit((unsigned char)s[i])) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int cr_follow_decode_roots(cr_config_t *cfg) {
|
||||
for (int i = 0; i < cfg->root_npub_count; i++) {
|
||||
const char *input = cfg->root_npubs[i];
|
||||
if (is_hex_pubkey(input)) {
|
||||
/* Already a 64-char hex pubkey — copy directly. */
|
||||
strncpy(cfg->root_hex[i], input, 64);
|
||||
cfg->root_hex[i][64] = '\0';
|
||||
} else {
|
||||
/* Assume npub (bech32) format — decode to hex. */
|
||||
unsigned char pubkey[32];
|
||||
if (nostr_decode_npub(input, pubkey) != NOSTR_SUCCESS) {
|
||||
DEBUG_ERROR("follow: failed to decode root pubkey '%s' "
|
||||
"(not a valid npub or 64-char hex)", input);
|
||||
return -1;
|
||||
}
|
||||
nostr_bytes_to_hex(pubkey, 32, cfg->root_hex[i]);
|
||||
}
|
||||
}
|
||||
cfg->root_hex_ready = 1;
|
||||
DEBUG_INFO("follow: decoded %d root pubkeys", cfg->root_npub_count);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Parse "p" tags from a kind-3 event and add pubkeys to the set. */
|
||||
static int parse_p_tags(cJSON *event, cr_pubkey_set_t *followed) {
|
||||
cJSON *tags = cJSON_GetObjectItem(event, "tags");
|
||||
if (!tags || !cJSON_IsArray(tags)) return 0;
|
||||
int added = 0;
|
||||
cJSON *tag;
|
||||
cJSON_ArrayForEach(tag, tags) {
|
||||
if (!cJSON_IsArray(tag)) continue;
|
||||
cJSON *name = cJSON_GetArrayItem(tag, 0);
|
||||
if (!name || !cJSON_IsString(name)) continue;
|
||||
if (strcmp(cJSON_GetStringValue(name), "p") != 0) continue;
|
||||
cJSON *pk = cJSON_GetArrayItem(tag, 1);
|
||||
if (!pk || !cJSON_IsString(pk)) continue;
|
||||
const char *hex = cJSON_GetStringValue(pk);
|
||||
if (strlen(hex) != 64) continue;
|
||||
if (cr_pubkey_set_add(followed, hex) == 1) added++;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
int cr_follow_resolve(cr_config_t *cfg, nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed) {
|
||||
if (!cfg->root_hex_ready) {
|
||||
if (cr_follow_decode_roots(cfg) != 0) return -1;
|
||||
}
|
||||
|
||||
/* Seed the followed set with the root pubkeys themselves. */
|
||||
for (int i = 0; i < cfg->root_npub_count; i++) {
|
||||
cr_pubkey_set_add(followed, cfg->root_hex[i]);
|
||||
}
|
||||
|
||||
int total_follows = 0;
|
||||
for (int i = 0; i < cfg->root_npub_count; i++) {
|
||||
/* Build filter: authors=[root], kinds=[3], limit=1 (most recent). */
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(cfg->root_hex[i]));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(3));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(1));
|
||||
|
||||
const char **urls = NULL;
|
||||
int n = 0;
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int listed_count = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
if (listed_count > 0) {
|
||||
urls = malloc(listed_count * sizeof(char *));
|
||||
for (int j = 0; j < listed_count; j++) { urls[n++] = listed[j]; }
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = nostr_relay_pool_query_sync(upstream, urls, n,
|
||||
filter, &ev_count, 15000);
|
||||
free(urls);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
if (events && ev_count > 0) {
|
||||
int added = parse_p_tags(events[0], followed);
|
||||
total_follows += added;
|
||||
DEBUG_INFO("follow: root %d: kind-3 found, +%d follows (total set %d)",
|
||||
i, added, followed->count);
|
||||
for (int k = 0; k < ev_count; k++) cJSON_Delete(events[k]);
|
||||
free(events);
|
||||
} else {
|
||||
DEBUG_WARN("follow: root %d: no kind-3 found", i);
|
||||
free(events);
|
||||
}
|
||||
}
|
||||
|
||||
DEBUG_INFO("follow: resolved %d total followed pubkeys (set size %d)",
|
||||
total_follows, followed->count);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* caching_relay - follow graph resolution.
|
||||
*
|
||||
* Decodes root npubs (NIP-19) to hex, queries their most recent kind-3
|
||||
* contact list from the upstream pool, parses the "p" tags, and populates
|
||||
* the in-memory followed-pubkey set (root pubkeys + their follows).
|
||||
*/
|
||||
#ifndef CACHING_RELAY_FOLLOW_GRAPH_H
|
||||
#define CACHING_RELAY_FOLLOW_GRAPH_H
|
||||
|
||||
#include "config.h"
|
||||
#include "state.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
/* Check if a hex pubkey is one of the root (admin) npubs. Returns 1 if yes. */
|
||||
int cr_follow_is_root(const cr_config_t *cfg, const char *hex);
|
||||
|
||||
/* Decode all root_npubs in cfg into cfg->root_hex[]. Returns 0 on success. */
|
||||
int cr_follow_decode_roots(cr_config_t *cfg);
|
||||
|
||||
/* Resolve the follow graph: for each root hex pubkey, query the most recent
|
||||
* kind-3 from the upstream pool, parse "p" tags, and add root + follows to
|
||||
* the followed set. Returns 0 on success, -1 on hard failure. */
|
||||
int cr_follow_resolve(cr_config_t *cfg, nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed);
|
||||
|
||||
#endif /* CACHING_RELAY_FOLLOW_GRAPH_H */
|
||||
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* caching_relay - forward catch-up implementation.
|
||||
*
|
||||
* See forward_catchup.h for design.
|
||||
*/
|
||||
#include "forward_catchup.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "config.h"
|
||||
#include "follow_graph.h"
|
||||
#include "relay_sink.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <cjson/cJSON.h>
|
||||
#include <time.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* Find the newest (maximum) created_at among an array of event JSON objects.
|
||||
* Returns 1 and sets *out_newest, or 0 if no events / no valid created_at. */
|
||||
static int find_newest_created_at(cJSON **events, int count, long *out_newest) {
|
||||
long newest = 0;
|
||||
int found = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
cJSON *ca = cJSON_GetObjectItem(events[i], "created_at");
|
||||
if (!ca || !cJSON_IsNumber(ca)) continue;
|
||||
long ts = (long)ca->valuedouble;
|
||||
if (!found || ts > newest) {
|
||||
newest = ts;
|
||||
found = 1;
|
||||
}
|
||||
}
|
||||
if (found && out_newest) *out_newest = newest;
|
||||
return found;
|
||||
}
|
||||
|
||||
/* Build the kinds array for a given pubkey (admin vs regular).
|
||||
* Returns NULL if no kinds filter should be applied (admin_all_kinds). */
|
||||
static cJSON *build_kinds(cr_config_t *cfg, const char *pk) {
|
||||
int is_admin = cr_follow_is_root(cfg, pk);
|
||||
if (is_admin && cfg->admin_all_kinds) {
|
||||
return NULL;
|
||||
}
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
if (is_admin && cfg->admin_kind_count > 0) {
|
||||
for (int i = 0; i < cfg->admin_kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
|
||||
} else {
|
||||
for (int i = 0; i < cfg->kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
|
||||
}
|
||||
return kinds;
|
||||
}
|
||||
|
||||
/* Get relay URLs for an author from the relay progress table.
|
||||
* Returns a cJSON array of relay_url strings. Caller must cJSON_Delete().
|
||||
* Returns NULL on error or if no relays found. */
|
||||
static cJSON *get_author_relays(const char *pk) {
|
||||
/* Use pg_inbox_get_incomplete_relays for incomplete authors.
|
||||
* For completed authors, we need all relays — query directly. */
|
||||
cJSON *incomplete = pg_inbox_get_incomplete_relays(pk);
|
||||
if (incomplete && cJSON_GetArraySize(incomplete) > 0) {
|
||||
/* Extract just the relay_url strings. */
|
||||
cJSON *urls = cJSON_CreateArray();
|
||||
int n = cJSON_GetArraySize(incomplete);
|
||||
for (int i = 0; i < n; i++) {
|
||||
cJSON *entry = cJSON_GetArrayItem(incomplete, i);
|
||||
cJSON *url = cJSON_GetObjectItemCaseSensitive(entry, "relay_url");
|
||||
if (url && cJSON_IsString(url) && url->valuestring[0]) {
|
||||
cJSON_AddItemToArray(urls, cJSON_CreateString(url->valuestring));
|
||||
}
|
||||
}
|
||||
cJSON_Delete(incomplete);
|
||||
if (cJSON_GetArraySize(urls) > 0) return urls;
|
||||
cJSON_Delete(urls);
|
||||
}
|
||||
if (incomplete) cJSON_Delete(incomplete);
|
||||
|
||||
/* For completed authors, fall back to the upstream pool's connected
|
||||
* relays. The caller will use the pool directly. */
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int cr_forward_catchup(cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_sink_t *sink) {
|
||||
if (!cfg || !upstream || !sink) return -1;
|
||||
|
||||
cJSON *authors = pg_inbox_get_authors_for_catchup();
|
||||
if (!authors) {
|
||||
DEBUG_INFO("forward_catchup: no authors with last_event_at > 0");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int author_count = cJSON_GetArraySize(authors);
|
||||
if (author_count == 0) {
|
||||
cJSON_Delete(authors);
|
||||
DEBUG_INFO("forward_catchup: no authors need catch-up");
|
||||
return 0;
|
||||
}
|
||||
|
||||
DEBUG_INFO("forward_catchup: checking %d authors for missed events", author_count);
|
||||
|
||||
long now = (long)time(NULL);
|
||||
int page_size = cfg->backfill.events_per_tick;
|
||||
if (page_size < 1) page_size = 500;
|
||||
|
||||
cr_sink_set_source_class(sink, CR_SINK_CLASS_BACKFILL);
|
||||
|
||||
int total_events_published = 0;
|
||||
int authors_caught_up = 0;
|
||||
|
||||
for (int i = 0; i < author_count; i++) {
|
||||
cJSON *entry = cJSON_GetArrayItem(authors, i);
|
||||
if (!entry) continue;
|
||||
|
||||
cJSON *pk_node = cJSON_GetObjectItemCaseSensitive(entry, "pubkey");
|
||||
cJSON *lea_node = cJSON_GetObjectItemCaseSensitive(entry, "last_event_at");
|
||||
if (!pk_node || !cJSON_IsString(pk_node) || !lea_node || !cJSON_IsNumber(lea_node))
|
||||
continue;
|
||||
|
||||
const char *pk = pk_node->valuestring;
|
||||
long last_event_at = (long)lea_node->valuedouble;
|
||||
|
||||
if (last_event_at <= 0 || last_event_at >= now) continue;
|
||||
|
||||
/* Build filter: since = last_event_at + 1, until = now. */
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors_arr = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(authors_arr, cJSON_CreateString(pk));
|
||||
cJSON_AddItemToObject(filter, "authors", authors_arr);
|
||||
cJSON *kinds = build_kinds(cfg, pk);
|
||||
if (kinds) cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)(last_event_at + 1)));
|
||||
cJSON_AddItemToObject(filter, "until", cJSON_CreateNumber((double)now));
|
||||
cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber((double)page_size));
|
||||
|
||||
/* Try to get author-specific relays first; fall back to upstream pool. */
|
||||
cJSON *author_relays = get_author_relays(pk);
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = NULL;
|
||||
|
||||
if (author_relays && cJSON_GetArraySize(author_relays) > 0) {
|
||||
/* Query using the author's outbox relays. */
|
||||
int n_relays = cJSON_GetArraySize(author_relays);
|
||||
const char **urls = malloc(sizeof(char *) * n_relays);
|
||||
for (int r = 0; r < n_relays; r++) {
|
||||
cJSON *u = cJSON_GetArrayItem(author_relays, r);
|
||||
urls[r] = cJSON_IsString(u) ? u->valuestring : "";
|
||||
}
|
||||
events = synchronous_query_relays_with_progress(
|
||||
urls, n_relays, filter, RELAY_QUERY_ALL_RESULTS,
|
||||
&ev_count, 30, NULL, NULL, 0, NULL);
|
||||
free(urls);
|
||||
} else {
|
||||
/* Fall back: query all connected upstream relays.
|
||||
* Use the pool's relay list. We pass NULL for urls to let
|
||||
* the pool use all connected relays. */
|
||||
events = synchronous_query_relays_with_progress(
|
||||
NULL, 0, filter, RELAY_QUERY_ALL_RESULTS,
|
||||
&ev_count, 30, NULL, NULL, 0, NULL);
|
||||
}
|
||||
cJSON_Delete(author_relays);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
if (events && ev_count > 0) {
|
||||
/* Publish all events. */
|
||||
long newest = 0;
|
||||
find_newest_created_at(events, ev_count, &newest);
|
||||
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cr_sink_publish(sink, events[k]);
|
||||
cJSON_Delete(events[k]);
|
||||
}
|
||||
free(events);
|
||||
total_events_published += ev_count;
|
||||
authors_caught_up++;
|
||||
|
||||
/* Update last_event_at to the newest event we saw. */
|
||||
if (newest > 0) {
|
||||
pg_inbox_update_last_event_at(pk, newest);
|
||||
}
|
||||
|
||||
DEBUG_LOG("forward_catchup: %s -> %d events (since=%ld, until=%ld)",
|
||||
pk, ev_count, last_event_at + 1, now);
|
||||
} else {
|
||||
/* No events in the gap — update last_event_at to now so we
|
||||
* don't re-query the same empty window next time. */
|
||||
pg_inbox_update_last_event_at(pk, now);
|
||||
if (events) free(events);
|
||||
}
|
||||
}
|
||||
|
||||
cJSON_Delete(authors);
|
||||
|
||||
DEBUG_INFO("forward_catchup: %d authors checked, %d had events, %d total events published",
|
||||
author_count, authors_caught_up, total_events_published);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* caching_relay - forward catch-up for gap bridging.
|
||||
*
|
||||
* When the caching service starts (or caching is re-enabled after being
|
||||
* off), events posted by followed authors during the downtime are missed
|
||||
* by the backward-drain backfill (which walks toward the beginning of
|
||||
* time) and by the live subscriber (which starts at `since = now`).
|
||||
*
|
||||
* The forward catch-up queries `since = last_event_at + 1, until = now`
|
||||
* for each followed author to bridge that gap. `last_event_at` is the
|
||||
* `created_at` of the most recent event we've ever seen for the author,
|
||||
* maintained by the backfill and live subscriber.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_FORWARD_CATCHUP_H
|
||||
#define CACHING_RELAY_FORWARD_CATCHUP_H
|
||||
|
||||
#include "config.h"
|
||||
#include "relay_sink.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
/* Run forward catch-up for all followed authors with last_event_at > 0.
|
||||
* For each author, queries events from last_event_at + 1 to now on one
|
||||
* outbox relay and publishes them to the sink.
|
||||
*
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int cr_forward_catchup(cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_sink_t *sink);
|
||||
|
||||
#endif /* CACHING_RELAY_FORWARD_CATCHUP_H */
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* caching_relay - strip JSONC comments so cJSON can parse it.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "jsonc_strip.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
char *jsonc_strip_comments(const char *input, size_t len) {
|
||||
if (!input) return NULL;
|
||||
if (len == 0) len = strlen(input);
|
||||
|
||||
/* Worst case: output is same size as input (no comments). +1 for NUL. */
|
||||
char *out = malloc(len + 1);
|
||||
if (!out) return NULL;
|
||||
|
||||
size_t i = 0, o = 0;
|
||||
int in_string = 0;
|
||||
int escape = 0;
|
||||
|
||||
while (i < len) {
|
||||
char c = input[i];
|
||||
|
||||
if (in_string) {
|
||||
out[o++] = c;
|
||||
if (escape) {
|
||||
escape = 0;
|
||||
} else if (c == '\\') {
|
||||
escape = 1;
|
||||
} else if (c == '"') {
|
||||
in_string = 0;
|
||||
}
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Not in a string. */
|
||||
if (c == '"') {
|
||||
in_string = 1;
|
||||
out[o++] = c;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Line comment // */
|
||||
if (c == '/' && i + 1 < len && input[i + 1] == '/') {
|
||||
i += 2;
|
||||
while (i < len && input[i] != '\n') i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Block comment / * ... * / */
|
||||
if (c == '/' && i + 1 < len && input[i + 1] == '*') {
|
||||
i += 2;
|
||||
while (i < len && !(input[i] == '*' && i + 1 < len && input[i + 1] == '/')) i++;
|
||||
if (i < len) i += 2; /* skip closing */
|
||||
/* Preserve a space so tokens don't merge. */
|
||||
out[o++] = ' ';
|
||||
continue;
|
||||
}
|
||||
|
||||
out[o++] = c;
|
||||
i++;
|
||||
}
|
||||
|
||||
out[o] = '\0';
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* caching_relay - strip JSONC comments so cJSON can parse it.
|
||||
*
|
||||
* cJSON does not natively understand JSONC. This produces a heap buffer the
|
||||
* caller must free() containing the comment-stripped JSON. String literals are
|
||||
* respected so a double-slash inside a string is preserved.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_JSONC_STRIP_H
|
||||
#define CACHING_RELAY_JSONC_STRIP_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Returns malloc'd buffer (null-terminated) or NULL on alloc failure. */
|
||||
char *jsonc_strip_comments(const char *input, size_t len);
|
||||
|
||||
#endif /* CACHING_RELAY_JSONC_STRIP_H */
|
||||
@@ -0,0 +1,199 @@
|
||||
/*
|
||||
* caching_relay - live subscription on the upstream pool
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "live_subscriber.h"
|
||||
#include "follow_graph.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Context passed as user_data to the subscription callbacks. */
|
||||
typedef struct {
|
||||
cr_sink_t *sink;
|
||||
cr_live_t *live;
|
||||
cr_config_t *cfg;
|
||||
} cr_live_ctx_t;
|
||||
|
||||
static cr_live_ctx_t g_live_ctx;
|
||||
|
||||
static void live_on_event(cJSON *event, const char *relay_url, void *user_data) {
|
||||
cr_live_ctx_t *ctx = (cr_live_ctx_t *)user_data;
|
||||
(void)relay_url;
|
||||
ctx->live->events_received++;
|
||||
cr_sink_publish(ctx->sink, event);
|
||||
|
||||
/* Update last_event_at for forward catch-up gap tracking. */
|
||||
cJSON *ev_pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
cJSON *ev_created = cJSON_GetObjectItem(event, "created_at");
|
||||
if (ev_pubkey && cJSON_IsString(ev_pubkey) &&
|
||||
ev_created && cJSON_IsNumber(ev_created)) {
|
||||
pg_inbox_update_last_event_at(ev_pubkey->valuestring,
|
||||
(long)ev_created->valuedouble);
|
||||
}
|
||||
|
||||
/* Detect admin kind-3 (contact list) changes from a root pubkey and
|
||||
* signal the main loop to refresh the follow graph immediately. */
|
||||
if (ctx->cfg) {
|
||||
cJSON *kind = cJSON_GetObjectItem(event, "kind");
|
||||
cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
if (kind && cJSON_IsNumber(kind) && kind->valuedouble == 3.0 &&
|
||||
pubkey && cJSON_IsString(pubkey)) {
|
||||
if (cr_follow_is_root(ctx->cfg, cJSON_GetStringValue(pubkey))) {
|
||||
ctx->live->follow_graph_changed = 1;
|
||||
DEBUG_INFO("live: root kind-3 detected from %s, "
|
||||
"signaling follow graph refresh",
|
||||
cJSON_GetStringValue(pubkey));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void live_on_eose(cJSON **events, int event_count, void *user_data) {
|
||||
(void)events; (void)event_count; (void)user_data;
|
||||
DEBUG_LOG("live: EOSE (stored events flushed), staying open");
|
||||
}
|
||||
|
||||
/* Build a filter for non-admin followed pubkeys with regular kinds. */
|
||||
static cJSON *build_follows_filter(cr_config_t *cfg, cr_pubkey_set_t *followed) {
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
/* Skip admin pubkeys - they get their own subscription. */
|
||||
if (cr_follow_is_root(cfg, followed->items[i])) continue;
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(followed->items[i]));
|
||||
}
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->kinds[i]));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
|
||||
cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)time(NULL)));
|
||||
return filter;
|
||||
}
|
||||
|
||||
/* Build a filter for admin (root) pubkeys with admin_kinds (or all kinds). */
|
||||
static cJSON *build_admin_filter(cr_config_t *cfg) {
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->root_npub_count; i++)
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(cfg->root_hex[i]));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
|
||||
/* If admin_all_kinds, omit the kinds filter entirely (grab everything). */
|
||||
if (!cfg->admin_all_kinds && cfg->admin_kind_count > 0) {
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
for (int i = 0; i < cfg->admin_kind_count; i++)
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(cfg->admin_kinds[i]));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
}
|
||||
|
||||
cJSON_AddItemToObject(filter, "since", cJSON_CreateNumber((double)time(NULL)));
|
||||
return filter;
|
||||
}
|
||||
|
||||
static int get_relay_urls(nostr_relay_pool_t *upstream, const char ***urls_out) {
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
const char **urls = NULL;
|
||||
if (n > 0) {
|
||||
urls = malloc(n * sizeof(char *));
|
||||
for (int j = 0; j < n; j++) urls[j] = listed[j];
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
*urls_out = urls;
|
||||
return n;
|
||||
}
|
||||
|
||||
static nostr_pool_subscription_t *open_subscription(nostr_relay_pool_t *upstream,
|
||||
cJSON *filter) {
|
||||
const char **urls = NULL;
|
||||
int n = get_relay_urls(upstream, &urls);
|
||||
|
||||
nostr_pool_subscription_t *sub = nostr_relay_pool_subscribe(
|
||||
upstream, urls, n, filter,
|
||||
live_on_event, live_on_eose, &g_live_ctx,
|
||||
0, 1, NOSTR_POOL_EOSE_FULL_SET, 0, 0);
|
||||
free(urls);
|
||||
return sub;
|
||||
}
|
||||
|
||||
static int open_subs(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed, cr_sink_t *sink) {
|
||||
g_live_ctx.sink = sink;
|
||||
g_live_ctx.live = live;
|
||||
g_live_ctx.cfg = cfg;
|
||||
|
||||
int admin_count = 0;
|
||||
int follows_count = 0;
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (cr_follow_is_root(cfg, followed->items[i])) admin_count++;
|
||||
else follows_count++;
|
||||
}
|
||||
|
||||
/* Follows subscription (non-admin pubkeys, regular kinds). */
|
||||
if (follows_count > 0) {
|
||||
cJSON *filter = build_follows_filter(cfg, followed);
|
||||
live->follows_sub = open_subscription(upstream, filter);
|
||||
cJSON_Delete(filter);
|
||||
if (!live->follows_sub) {
|
||||
DEBUG_ERROR("live: follows subscribe failed");
|
||||
} else {
|
||||
DEBUG_INFO("live: follows sub - %d authors, %d kinds",
|
||||
follows_count, cfg->kind_count);
|
||||
}
|
||||
}
|
||||
|
||||
/* Admin subscription (root npubs, admin_kinds or all kinds). */
|
||||
if (admin_count > 0 || cfg->root_npub_count > 0) {
|
||||
cJSON *filter = build_admin_filter(cfg);
|
||||
live->admin_sub = open_subscription(upstream, filter);
|
||||
cJSON_Delete(filter);
|
||||
if (!live->admin_sub) {
|
||||
DEBUG_ERROR("live: admin subscribe failed");
|
||||
} else {
|
||||
if (cfg->admin_all_kinds) {
|
||||
DEBUG_INFO("live: admin sub - %d authors, ALL kinds",
|
||||
cfg->root_npub_count);
|
||||
} else {
|
||||
DEBUG_INFO("live: admin sub - %d authors, %d kinds",
|
||||
cfg->root_npub_count, cfg->admin_kind_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
live->last_resubscribe = time(NULL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cr_live_open(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed, cr_sink_t *sink) {
|
||||
memset(live, 0, sizeof(*live));
|
||||
return open_subs(live, cfg, upstream, followed, sink);
|
||||
}
|
||||
|
||||
int cr_live_resubscribe(cr_live_t *live, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
|
||||
cr_sink_t *sink) {
|
||||
cr_live_close(live);
|
||||
DEBUG_INFO("live: resubscribing (events so far: %ld)", live->events_received);
|
||||
return open_subs(live, cfg, upstream, followed, sink);
|
||||
}
|
||||
|
||||
void cr_live_close(cr_live_t *live) {
|
||||
if (live->follows_sub) {
|
||||
nostr_pool_subscription_close(live->follows_sub);
|
||||
live->follows_sub = NULL;
|
||||
}
|
||||
if (live->admin_sub) {
|
||||
nostr_pool_subscription_close(live->admin_sub);
|
||||
live->admin_sub = NULL;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* caching_relay - live subscription on the upstream pool.
|
||||
*
|
||||
* Builds filters from the followed-pubkey set + configured kinds, opens
|
||||
* long-lived subscriptions, and routes on_event to the relay sink.
|
||||
*
|
||||
* Two subscriptions are opened:
|
||||
* 1. "follows" sub: non-admin followed pubkeys + regular kinds
|
||||
* 2. "admin" sub: root (admin) npubs + admin_kinds (or all kinds if [*])
|
||||
*/
|
||||
#ifndef CACHING_RELAY_LIVE_SUBSCRIBER_H
|
||||
#define CACHING_RELAY_LIVE_SUBSCRIBER_H
|
||||
|
||||
#include "config.h"
|
||||
#include "state.h"
|
||||
#include "relay_sink.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
typedef struct {
|
||||
nostr_pool_subscription_t *follows_sub; /* non-admin pubkeys, regular kinds */
|
||||
nostr_pool_subscription_t *admin_sub; /* admin pubkeys, admin_kinds */
|
||||
long events_received;
|
||||
time_t last_resubscribe;
|
||||
int follow_graph_changed; /* set when a root npub publishes a kind-3 */
|
||||
} cr_live_t;
|
||||
|
||||
/* Open the live subscription(s). Returns 0 on success. */
|
||||
int cr_live_open(cr_live_t *live, cr_config_t *cfg, nostr_relay_pool_t *upstream,
|
||||
cr_pubkey_set_t *followed, cr_sink_t *sink);
|
||||
|
||||
/* Close + reopen the subscription(s). Returns 0 on success. */
|
||||
int cr_live_resubscribe(cr_live_t *live, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream, cr_pubkey_set_t *followed,
|
||||
cr_sink_t *sink);
|
||||
|
||||
void cr_live_close(cr_live_t *live);
|
||||
|
||||
#endif /* CACHING_RELAY_LIVE_SUBSCRIBER_H */
|
||||
@@ -0,0 +1,639 @@
|
||||
/*
|
||||
* caching_relay - a daemon that caches Nostr events from followed people
|
||||
* into a local relay.
|
||||
*
|
||||
* Architecture: see plans/plan.md. Two nostr_relay_pool_t instances:
|
||||
* - upstream_pool: query/subscribe to upstream relays
|
||||
* - sink_pool (in cr_sink_t): publish-only to the local relay
|
||||
*
|
||||
* Build: see Makefile. Statically linked C99 binary, c-relay style.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "main.h"
|
||||
#include "debug.h"
|
||||
#include "config.h"
|
||||
#include "state.h"
|
||||
#include "follow_graph.h"
|
||||
#include "relay_sink.h"
|
||||
#include "live_subscriber.h"
|
||||
#include "backfill.h"
|
||||
#include "forward_catchup.h"
|
||||
#include "relay_discovery.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "pg_config.h"
|
||||
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_log.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <getopt.h>
|
||||
|
||||
/* Forward nostr_core_lib's internal logging to our debug system. */
|
||||
static void nostr_log_forwarder(int level, const char *component,
|
||||
const char *message, void *user_data) {
|
||||
(void)user_data;
|
||||
/* Map nostr log levels to our debug levels (they're the same 1-5). */
|
||||
if (level >= 5) {
|
||||
DEBUG_TRACE("[nostr:%s] %s", component ? component : "?", message ? message : "");
|
||||
} else if (level >= 4) {
|
||||
DEBUG_LOG("[nostr:%s] %s", component ? component : "?", message ? message : "");
|
||||
} else if (level >= 3) {
|
||||
DEBUG_INFO("[nostr:%s] %s", component ? component : "?", message ? message : "");
|
||||
} else if (level >= 2) {
|
||||
DEBUG_WARN("[nostr:%s] %s", component ? component : "?", message ? message : "");
|
||||
} else {
|
||||
DEBUG_ERROR("[nostr:%s] %s", component ? component : "?", message ? message : "");
|
||||
}
|
||||
}
|
||||
|
||||
/* Initialize per-relay backfill progress rows for every followed pubkey.
|
||||
* For each author, merges its discovered outbox relays (from relay_map)
|
||||
* with the bootstrap (upstream) relays from cfg, then inserts one
|
||||
* caching_backfill_relay_progress row per relay (ON CONFLICT DO NOTHING).
|
||||
* Safe to call repeatedly - existing rows are left untouched. */
|
||||
static void init_relay_progress_for_all(const cr_config_t *cfg,
|
||||
const cr_relay_map_t *relay_map,
|
||||
const cr_pubkey_set_t *followed) {
|
||||
if (!followed || followed->count <= 0) return;
|
||||
|
||||
/* Bootstrap relay URLs from cfg->upstream_relays. */
|
||||
const char **bootstrap = NULL;
|
||||
int bootstrap_n = cfg->upstream_count;
|
||||
if (bootstrap_n > 0) {
|
||||
bootstrap = malloc(bootstrap_n * sizeof(char *));
|
||||
if (!bootstrap) return;
|
||||
for (int i = 0; i < bootstrap_n; i++)
|
||||
bootstrap[i] = cfg->upstream_relays[i];
|
||||
}
|
||||
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
const char *pk = followed->items[i];
|
||||
if (!pk || !*pk) continue;
|
||||
|
||||
/* Collect this author's outbox relays. */
|
||||
const cr_outbox_entry_t *oe = NULL;
|
||||
int outbox_n = 0;
|
||||
if (relay_map) {
|
||||
oe = cr_relay_map_get_outbox(relay_map, pk);
|
||||
if (oe) outbox_n = oe->relay_count;
|
||||
}
|
||||
|
||||
int total = outbox_n + bootstrap_n;
|
||||
if (total <= 0) continue;
|
||||
|
||||
const char **urls = malloc(total * sizeof(char *));
|
||||
if (!urls) continue;
|
||||
int n = 0;
|
||||
if (oe) {
|
||||
for (int j = 0; j < oe->relay_count && n < total; j++)
|
||||
urls[n++] = oe->relays[j];
|
||||
}
|
||||
for (int j = 0; j < bootstrap_n && n < total; j++) {
|
||||
int dup = 0;
|
||||
for (int k = 0; k < n; k++) {
|
||||
if (strcmp(urls[k], bootstrap[j]) == 0) { dup = 1; break; }
|
||||
}
|
||||
if (!dup) urls[n++] = bootstrap[j];
|
||||
}
|
||||
|
||||
if (n > 0) {
|
||||
pg_inbox_init_relay_progress_for_author(pk, urls, n);
|
||||
}
|
||||
free(urls);
|
||||
}
|
||||
|
||||
free(bootstrap);
|
||||
}
|
||||
|
||||
volatile sig_atomic_t g_shutdown = 0;
|
||||
static volatile sig_atomic_t g_reload = 0;
|
||||
|
||||
static void on_signal(int sig) {
|
||||
if (sig == SIGINT || sig == SIGTERM) g_shutdown = 1;
|
||||
else if (sig == SIGHUP) g_reload = 1;
|
||||
}
|
||||
|
||||
static void usage(const char *prog) {
|
||||
fprintf(stderr,
|
||||
"caching_relay %s - cache Nostr events from followed people into a local relay\n"
|
||||
"\n"
|
||||
"Usage: %s [-c <config.jsonc>] [-p <pg-conn>] [options]\n"
|
||||
"\n"
|
||||
"Options:\n"
|
||||
" -c, --config <file> Path to .jsonc config file (default: ./caching_relay_config.jsonc)\n"
|
||||
" -p, --pg-conn <str> PostgreSQL connection string (libpq format). When provided,\n"
|
||||
" config is read from the c-relay-pg config table and fetched\n"
|
||||
" events are inserted into the caching_event_inbox table instead\n"
|
||||
" of being published via WebSocket to a local relay.\n"
|
||||
" -d, --debug <level> Log level 0-5 (0=none, 1=error, 2=warn, 3=info, 4=debug, 5=trace). Default 3.\n"
|
||||
" -r, --restart Reset state to first-time startup (ignore local relay cache,\n"
|
||||
" re-discover all kind-10002 from bootstrap relays, reset backfill)\n"
|
||||
" -h, --help Show this help\n"
|
||||
"\n"
|
||||
"Without -p, the daemon uses the .jsonc config file and publishes to a local relay\n"
|
||||
"via WebSocket (legacy mode). With -p, it uses PostgreSQL for config and inbox.\n"
|
||||
"The config file is also the persistent state store in legacy mode; the daemon\n"
|
||||
"rewrites it as backfill progresses. See plans/plan.md and caching_relay_config.jsonc.\n",
|
||||
CR_VERSION, prog);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
const char *config_path = NULL;
|
||||
const char *pg_conn = NULL;
|
||||
int log_level = DEBUG_LEVEL_INFO;
|
||||
int restart = 0;
|
||||
|
||||
static struct option longopts[] = {
|
||||
{"config", required_argument, 0, 'c'},
|
||||
{"pg-conn", required_argument, 0, 'p'},
|
||||
{"debug", required_argument, 0, 'd'},
|
||||
{"restart", no_argument, 0, 'r'},
|
||||
{"help", no_argument, 0, 'h'},
|
||||
{0, 0, 0, 0}
|
||||
};
|
||||
int opt;
|
||||
while ((opt = getopt_long(argc, argv, "c:p:d:rh", longopts, NULL)) != -1) {
|
||||
switch (opt) {
|
||||
case 'c': config_path = optarg; break;
|
||||
case 'p': pg_conn = optarg; break;
|
||||
case 'd': log_level = atoi(optarg); break;
|
||||
case 'r': restart = 1; break;
|
||||
case 'h': usage(argv[0]); return 0;
|
||||
default: usage(argv[0]); return 1;
|
||||
}
|
||||
}
|
||||
/* In PostgreSQL mode, the .jsonc config file is optional. */
|
||||
if (!config_path && !pg_conn) {
|
||||
/* Default: look for caching_relay.jsonc in the current directory. */
|
||||
config_path = "caching_relay_config.jsonc";
|
||||
if (access(config_path, F_OK) != 0) {
|
||||
fprintf(stderr, "ERROR: no config specified and default '%s' not found\n\n", config_path);
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (log_level < 0) log_level = 0;
|
||||
if (log_level > 5) log_level = 5;
|
||||
|
||||
debug_init(log_level);
|
||||
|
||||
/* Enable nostr_core_lib internal logging and forward to our debug system.
|
||||
* This shows the actual WebSocket messages (REQ filters, EVENT responses)
|
||||
* at debug level 5 (trace). */
|
||||
nostr_set_log_callback(nostr_log_forwarder, NULL);
|
||||
nostr_set_log_level((nostr_log_level_t)log_level);
|
||||
|
||||
DEBUG_INFO("caching_relay %s starting (config=%s, pg-conn=%s, loglevel=%d%s)",
|
||||
CR_VERSION,
|
||||
config_path ? config_path : "(none)",
|
||||
pg_conn ? "yes" : "no",
|
||||
log_level, restart ? ", RESTART" : "");
|
||||
|
||||
/* Signals. */
|
||||
struct sigaction sa;
|
||||
memset(&sa, 0, sizeof(sa));
|
||||
sa.sa_handler = on_signal;
|
||||
sigaction(SIGINT, &sa, NULL);
|
||||
sigaction(SIGTERM, &sa, NULL);
|
||||
sigaction(SIGHUP, &sa, NULL);
|
||||
/* Ignore SIGPIPE - relay pool handles its own socket errors. */
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
|
||||
/* PostgreSQL inbox mode: connect and load config from the config table. */
|
||||
long config_generation = -1;
|
||||
if (pg_conn) {
|
||||
if (pg_inbox_init(pg_conn) != 0) {
|
||||
DEBUG_ERROR("failed to connect to PostgreSQL");
|
||||
return 1;
|
||||
}
|
||||
config_generation = pg_inbox_get_config_generation();
|
||||
if (config_generation < 0) {
|
||||
DEBUG_WARN("could not read caching_config_generation (defaulting to 0)");
|
||||
config_generation = 0;
|
||||
}
|
||||
DEBUG_INFO("config generation: %ld", config_generation);
|
||||
}
|
||||
|
||||
/* Load config. */
|
||||
cr_config_t cfg;
|
||||
if (pg_conn) {
|
||||
if (pg_config_load(&cfg) != 0) {
|
||||
DEBUG_ERROR("failed to load config from PostgreSQL");
|
||||
pg_inbox_shutdown();
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
if (cr_config_load(&cfg, config_path) != 0) return 1;
|
||||
}
|
||||
|
||||
/* --restart: reset state to first-time startup. */
|
||||
if (restart) {
|
||||
DEBUG_INFO("RESTART: resetting state to first-time startup");
|
||||
cfg.state.backfilled_until = 0;
|
||||
if (pg_conn) {
|
||||
/* Reset per-author backfill progress (caching_followed_pubkeys)
|
||||
* so the next run starts fresh. The followed set itself is
|
||||
* preserved; only cursor/completion state is reset. */
|
||||
pg_inbox_reset_backfill_progress();
|
||||
} else {
|
||||
cr_config_save_state(&cfg);
|
||||
}
|
||||
}
|
||||
|
||||
/* Init crypto. */
|
||||
if (nostr_crypto_init() != NOSTR_SUCCESS) {
|
||||
DEBUG_ERROR("nostr_crypto_init failed");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* In-memory state. */
|
||||
cr_seen_ring_t seen;
|
||||
cr_seen_ring_init(&seen);
|
||||
cr_pubkey_set_t followed;
|
||||
cr_pubkey_set_init(&followed);
|
||||
|
||||
/* Upstream pool. */
|
||||
nostr_relay_pool_t *upstream = nostr_relay_pool_create(nostr_pool_reconnect_config_default());
|
||||
if (!upstream) {
|
||||
DEBUG_ERROR("failed to create upstream pool");
|
||||
return 1;
|
||||
}
|
||||
for (int i = 0; i < cfg.upstream_count; i++) {
|
||||
if (nostr_relay_pool_add_relay(upstream, cfg.upstream_relays[i]) != NOSTR_SUCCESS) {
|
||||
DEBUG_WARN("failed to add upstream relay %s (continuing)", cfg.upstream_relays[i]);
|
||||
} else {
|
||||
DEBUG_INFO("upstream: added %s", cfg.upstream_relays[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/* Sink: WebSocket pool (legacy) or PostgreSQL inbox. */
|
||||
cr_sink_t sink;
|
||||
if (pg_conn) {
|
||||
if (cr_sink_init_pg(&sink, &seen) != 0) {
|
||||
DEBUG_ERROR("failed to init sink (pg)");
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
if (cr_sink_init(&sink, cfg.local_relay, &seen) != 0) {
|
||||
DEBUG_ERROR("failed to init sink");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Give the sink pool a moment to connect to the local relay before we
|
||||
* start publishing (relay discovery caches kind-10002 events to local).
|
||||
* No-op in PostgreSQL inbox mode (cr_sink_pump does nothing). */
|
||||
if (!pg_conn) {
|
||||
DEBUG_INFO("waiting for sink connection to establish...");
|
||||
for (int i = 0; i < 30 && !g_shutdown; i++) {
|
||||
cr_sink_pump(&sink, 100);
|
||||
}
|
||||
}
|
||||
|
||||
/* Resolve follow graph. */
|
||||
if (g_shutdown) goto shutdown;
|
||||
if (cr_follow_resolve(&cfg, upstream, &followed) != 0) {
|
||||
DEBUG_ERROR("follow graph resolution failed");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Sync the followed set to the DB so the backfill tick can iterate
|
||||
* over it (per-author until-cursor drain). Also run the one-time
|
||||
* migration from the old caching_backfill_progress table. */
|
||||
if (pg_conn) {
|
||||
pg_inbox_migrate_backfill_progress();
|
||||
const char **root_pks = malloc(cfg.root_npub_count * sizeof(char *));
|
||||
for (int i = 0; i < cfg.root_npub_count; i++)
|
||||
root_pks[i] = cfg.root_hex[i];
|
||||
pg_inbox_sync_followed_pubkeys((const char **)followed.items,
|
||||
followed.count, root_pks,
|
||||
cfg.root_npub_count);
|
||||
free(root_pks);
|
||||
}
|
||||
|
||||
if (g_shutdown) goto shutdown;
|
||||
|
||||
/* Discover outbox relays (NIP-65) and compute minimum covering set. */
|
||||
int is_first_time = (cfg.state.backfilled_until == 0);
|
||||
cr_relay_map_t relay_map;
|
||||
if (cr_relay_discovery_run(&relay_map, &cfg, upstream, &sink, &followed,
|
||||
is_first_time) != 0) {
|
||||
DEBUG_WARN("relay discovery failed, continuing with bootstrap relays only");
|
||||
memset(&relay_map, 0, sizeof(relay_map));
|
||||
}
|
||||
|
||||
/* Add discovered outbox relays to the upstream pool. */
|
||||
for (int i = 0; i < relay_map.selected_count; i++) {
|
||||
/* Check if already in the pool (bootstrap relays may already be there). */
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
int already = 0;
|
||||
for (int j = 0; j < n; j++) {
|
||||
if (strcmp(listed[j], relay_map.selected_relays[i]) == 0) {
|
||||
already = 1; break;
|
||||
}
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
if (!already) {
|
||||
if (nostr_relay_pool_add_relay(upstream, relay_map.selected_relays[i]) == NOSTR_SUCCESS) {
|
||||
DEBUG_INFO("upstream: added outbox relay %s", relay_map.selected_relays[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Log final upstream pool. */
|
||||
{
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
DEBUG_INFO("upstream_pool: %d relays connected:", n);
|
||||
for (int j = 0; j < n; j++) {
|
||||
DEBUG_INFO(" %s", listed[j]);
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
}
|
||||
|
||||
/* Create per-relay backfill progress rows for every followed pubkey
|
||||
* (outbox relays + bootstrap relays). Existing rows are left untouched
|
||||
* so cursor/completion state is preserved across restarts. */
|
||||
if (pg_conn) {
|
||||
init_relay_progress_for_all(&cfg, &relay_map, &followed);
|
||||
}
|
||||
|
||||
/* Forward catch-up: bridge the gap for events posted while caching
|
||||
* was off. Runs once at startup (not on --restart, which does a full
|
||||
* re-drain from now). For each followed author with last_event_at > 0,
|
||||
* queries since = last_event_at + 1, until = now. */
|
||||
if (pg_conn && !restart && cfg.backfill.enabled) {
|
||||
DEBUG_INFO("forward catch-up: bridging gap for followed authors");
|
||||
cr_forward_catchup(&cfg, upstream, &sink);
|
||||
}
|
||||
|
||||
/* Open live subscription. */
|
||||
cr_live_t live;
|
||||
if (cfg.live.enabled) {
|
||||
if (cr_live_open(&live, &cfg, upstream, &followed, &sink) != 0) {
|
||||
DEBUG_WARN("live subscription failed to open (will retry on resubscribe)");
|
||||
}
|
||||
} else {
|
||||
memset(&live, 0, sizeof(live));
|
||||
}
|
||||
|
||||
/* Init backfill. */
|
||||
cr_backfill_t bf;
|
||||
cr_backfill_init(&bf, &cfg);
|
||||
|
||||
time_t last_follow_refresh = time(NULL);
|
||||
time_t last_state_save = time(NULL);
|
||||
time_t last_status_heartbeat = 0;
|
||||
|
||||
/* Initial status heartbeat in PostgreSQL mode. */
|
||||
if (pg_conn) {
|
||||
int bf_complete = 0, bf_total = 0;
|
||||
pg_inbox_count_backfill_progress(&bf_complete, &bf_total);
|
||||
pg_inbox_update_status("starting", config_generation, (long)time(NULL),
|
||||
followed.count, relay_map.selected_count, 0,
|
||||
bf_complete, bf_total, 0, 0, NULL, 0);
|
||||
}
|
||||
|
||||
DEBUG_INFO("entering main loop");
|
||||
|
||||
while (!g_shutdown) {
|
||||
if (g_reload) {
|
||||
g_reload = 0;
|
||||
DEBUG_INFO("SIGHUP: reloading config (state preserved)");
|
||||
cr_config_t newcfg;
|
||||
int reload_ok = 0;
|
||||
if (pg_conn) {
|
||||
if (pg_config_load(&newcfg) == 0 &&
|
||||
cr_follow_decode_roots(&newcfg) == 0) reload_ok = 1;
|
||||
} else if (config_path) {
|
||||
if (cr_config_load(&newcfg, config_path) == 0) reload_ok = 1;
|
||||
}
|
||||
if (reload_ok) {
|
||||
/* Preserve runtime state across reload. */
|
||||
newcfg.state = cfg.state;
|
||||
cr_config_free(&cfg);
|
||||
cfg = newcfg;
|
||||
DEBUG_INFO("config reloaded");
|
||||
/* Force an immediate follow-graph refresh so any new
|
||||
* root npubs are picked up right away. */
|
||||
last_follow_refresh = 0;
|
||||
} else {
|
||||
DEBUG_ERROR("config reload failed, keeping old config");
|
||||
}
|
||||
}
|
||||
|
||||
/* PostgreSQL: check for config generation change and reload. */
|
||||
if (pg_conn) {
|
||||
int changed = pg_config_generation_changed(config_generation);
|
||||
if (changed == 1) {
|
||||
long new_gen = pg_inbox_get_config_generation();
|
||||
DEBUG_INFO("config generation changed (%ld -> %ld), reloading",
|
||||
config_generation, new_gen);
|
||||
cr_config_t newcfg;
|
||||
if (pg_config_load(&newcfg) == 0) {
|
||||
/* Decode root npubs to hex — pg_config_load fills
|
||||
* root_npubs[] but NOT root_hex[] / root_hex_ready.
|
||||
* Without this, cr_follow_is_root() returns 0 for
|
||||
* everything and the new root's follows are never
|
||||
* resolved. */
|
||||
if (cr_follow_decode_roots(&newcfg) != 0) {
|
||||
DEBUG_ERROR("config reload: failed to decode root npubs, keeping old config");
|
||||
cr_config_free(&newcfg);
|
||||
} else {
|
||||
newcfg.state = cfg.state;
|
||||
cr_config_free(&cfg);
|
||||
cfg = newcfg;
|
||||
config_generation = new_gen;
|
||||
DEBUG_INFO("config reloaded from PostgreSQL");
|
||||
/* Force an immediate follow-graph refresh so the
|
||||
* new root npub's follows are picked up right
|
||||
* away (instead of waiting up to
|
||||
* follow_graph_refresh_seconds). */
|
||||
last_follow_refresh = 0;
|
||||
/* Resubscribe live with new config. */
|
||||
if (cfg.live.enabled) {
|
||||
cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DEBUG_ERROR("PostgreSQL config reload failed, keeping old config");
|
||||
}
|
||||
} else if (changed < 0) {
|
||||
DEBUG_WARN("config generation check failed");
|
||||
}
|
||||
}
|
||||
|
||||
/* Pump upstream pool (drives live subscription callbacks). */
|
||||
nostr_relay_pool_run(upstream, 100);
|
||||
|
||||
/* Pump sink pool (flush publish callbacks). */
|
||||
cr_sink_pump(&sink, 50);
|
||||
|
||||
/* Backfill tick. */
|
||||
int brc = cr_backfill_tick(&bf, &cfg, upstream, &followed, &sink, &relay_map);
|
||||
(void)brc;
|
||||
|
||||
/* Immediate follow-graph refresh when a root npub publishes a new
|
||||
* kind-3 contact list (detected by the live subscriber). */
|
||||
if (cfg.live.enabled && live.follow_graph_changed) {
|
||||
live.follow_graph_changed = 0;
|
||||
DEBUG_INFO("live: admin kind-3 detected, refreshing follow graph immediately");
|
||||
cr_pubkey_set_t new_followed;
|
||||
cr_pubkey_set_init(&new_followed);
|
||||
if (cr_follow_resolve(&cfg, upstream, &new_followed) == 0) {
|
||||
int changed = (new_followed.count != followed.count);
|
||||
if (!changed) {
|
||||
for (int i = 0; i < followed.count; i++) {
|
||||
if (!cr_pubkey_set_contains(&new_followed, followed.items[i])) {
|
||||
changed = 1; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
cr_pubkey_set_free(&followed);
|
||||
followed = new_followed;
|
||||
if (pg_conn) {
|
||||
const char **root_pks = malloc(cfg.root_npub_count * sizeof(char *));
|
||||
for (int i = 0; i < cfg.root_npub_count; i++)
|
||||
root_pks[i] = cfg.root_hex[i];
|
||||
pg_inbox_sync_followed_pubkeys((const char **)followed.items,
|
||||
followed.count, root_pks,
|
||||
cfg.root_npub_count);
|
||||
free(root_pks);
|
||||
/* Create relay progress rows for newly added follows.
|
||||
* Existing rows are preserved (ON CONFLICT DO NOTHING). */
|
||||
init_relay_progress_for_all(&cfg, &relay_map, &followed);
|
||||
}
|
||||
if (changed && cfg.live.enabled) {
|
||||
cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
|
||||
}
|
||||
/* Reset the backfill in_progress flag in case we had entered
|
||||
* steady-state; new follows need to be drained. */
|
||||
bf.in_progress = 1;
|
||||
last_follow_refresh = time(NULL);
|
||||
} else {
|
||||
DEBUG_WARN("immediate follow graph refresh failed");
|
||||
cr_pubkey_set_free(&new_followed);
|
||||
}
|
||||
}
|
||||
|
||||
/* Periodic follow-graph refresh. */
|
||||
time_t now = time(NULL);
|
||||
if (cfg.follow_graph_refresh_seconds > 0 &&
|
||||
(now - last_follow_refresh) >= cfg.follow_graph_refresh_seconds) {
|
||||
DEBUG_INFO("refreshing follow graph");
|
||||
cr_pubkey_set_t new_followed;
|
||||
cr_pubkey_set_init(&new_followed);
|
||||
if (cr_follow_resolve(&cfg, upstream, &new_followed) == 0) {
|
||||
/* If the set changed, resubscribe live. */
|
||||
int changed = (new_followed.count != followed.count);
|
||||
if (!changed) {
|
||||
for (int i = 0; i < followed.count; i++) {
|
||||
if (!cr_pubkey_set_contains(&new_followed, followed.items[i])) {
|
||||
changed = 1; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
cr_pubkey_set_free(&followed);
|
||||
followed = new_followed;
|
||||
/* Sync the refreshed followed set to the DB so newly added
|
||||
* follows get backfilled and dropped follows stop. Existing
|
||||
* cursor/completion state is preserved (upsert only touches
|
||||
* is_root and last_seen). */
|
||||
if (pg_conn) {
|
||||
const char **root_pks = malloc(cfg.root_npub_count * sizeof(char *));
|
||||
for (int i = 0; i < cfg.root_npub_count; i++)
|
||||
root_pks[i] = cfg.root_hex[i];
|
||||
pg_inbox_sync_followed_pubkeys((const char **)followed.items,
|
||||
followed.count, root_pks,
|
||||
cfg.root_npub_count);
|
||||
free(root_pks);
|
||||
/* Create relay progress rows for newly added follows.
|
||||
* Existing rows are preserved (ON CONFLICT DO NOTHING). */
|
||||
init_relay_progress_for_all(&cfg, &relay_map, &followed);
|
||||
}
|
||||
if (changed && cfg.live.enabled) {
|
||||
cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
|
||||
}
|
||||
/* Reset the backfill in_progress flag in case we had entered
|
||||
* steady-state; new follows need to be drained. The backfill
|
||||
* tick will quickly re-enter steady-state if there are no
|
||||
* incomplete authors, so this is cheap. */
|
||||
bf.in_progress = 1;
|
||||
} else {
|
||||
DEBUG_WARN("follow graph refresh failed");
|
||||
cr_pubkey_set_free(&new_followed);
|
||||
}
|
||||
last_follow_refresh = now;
|
||||
}
|
||||
|
||||
/* Periodic live resubscribe. */
|
||||
if (cfg.live.enabled && cfg.live.resubscribe_interval_seconds > 0 &&
|
||||
(now - live.last_resubscribe) >= cfg.live.resubscribe_interval_seconds) {
|
||||
cr_live_resubscribe(&live, &cfg, upstream, &followed, &sink);
|
||||
}
|
||||
|
||||
/* Periodic state save (in case backfill didn't just save). */
|
||||
if ((now - last_state_save) >= 60) {
|
||||
if (!pg_conn) cr_config_save_state(&cfg);
|
||||
last_state_save = now;
|
||||
}
|
||||
|
||||
/* PostgreSQL: periodic status heartbeat. */
|
||||
if (pg_conn && (now - last_status_heartbeat) >= 15) {
|
||||
long events_fetched = live.events_received + bf.events_total;
|
||||
long inbox_inserts = sink.published_ok;
|
||||
int connected = 0;
|
||||
{
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int n = nostr_relay_pool_list_relays(upstream, &listed, &statuses);
|
||||
for (int j = 0; j < n; j++) {
|
||||
if (statuses[j] == NOSTR_POOL_RELAY_CONNECTED) connected++;
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
}
|
||||
int bf_complete = 0, bf_total = 0;
|
||||
pg_inbox_count_backfill_progress(&bf_complete, &bf_total);
|
||||
pg_inbox_update_status("running", config_generation, (long)now,
|
||||
followed.count, relay_map.selected_count,
|
||||
connected, bf_complete, bf_total,
|
||||
events_fetched, inbox_inserts, NULL, 0);
|
||||
last_status_heartbeat = now;
|
||||
}
|
||||
}
|
||||
|
||||
/* Graceful shutdown. */
|
||||
shutdown:
|
||||
DEBUG_INFO("shutting down...");
|
||||
cr_live_close(&live);
|
||||
if (!pg_conn) cr_config_save_state(&cfg);
|
||||
if (pg_conn) {
|
||||
int bf_complete = 0, bf_total = 0;
|
||||
pg_inbox_count_backfill_progress(&bf_complete, &bf_total);
|
||||
pg_inbox_update_status("stopped", config_generation, (long)time(NULL),
|
||||
followed.count, relay_map.selected_count, 0,
|
||||
bf_complete, bf_total,
|
||||
live.events_received + bf.events_total,
|
||||
sink.published_ok, NULL, 0);
|
||||
}
|
||||
cr_sink_destroy(&sink);
|
||||
nostr_relay_pool_destroy(upstream);
|
||||
cr_pubkey_set_free(&followed);
|
||||
cr_relay_map_free(&relay_map);
|
||||
cr_config_free(&cfg);
|
||||
if (pg_conn) pg_inbox_shutdown();
|
||||
nostr_crypto_cleanup();
|
||||
DEBUG_INFO("clean exit");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* caching_relay - main header
|
||||
*
|
||||
* Version information is auto-updated by increment_and_push.sh.
|
||||
* Git tags are the source of truth for versioning.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_MAIN_H
|
||||
#define CACHING_RELAY_MAIN_H
|
||||
|
||||
// Version information (auto-updated by increment_and_push.sh)
|
||||
#define CR_VERSION_MAJOR 0
|
||||
#define CR_VERSION_MINOR 0
|
||||
#define CR_VERSION_PATCH 2
|
||||
#define CR_VERSION "v0.0.2"
|
||||
|
||||
#endif /* CACHING_RELAY_MAIN_H */
|
||||
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* caching_relay - read caching configuration from the c-relay-pg PostgreSQL
|
||||
* config table and populate the existing cr_config_t structure.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "pg_config.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Split a comma-separated string into a string array (fixed-size rows).
|
||||
* Trims leading/trailing whitespace from each token. */
|
||||
static void split_csv_str(const char *s, char *dst, int max, int elem_len,
|
||||
int *count_out) {
|
||||
int count = 0;
|
||||
if (!s) { *count_out = 0; return; }
|
||||
|
||||
/* Work on a mutable copy because strtok_r modifies its input. */
|
||||
char *copy = strdup(s);
|
||||
if (!copy) { *count_out = 0; return; }
|
||||
|
||||
char *saveptr = NULL;
|
||||
char *tok = strtok_r(copy, ",", &saveptr);
|
||||
while (tok && count < max) {
|
||||
/* Trim leading spaces. */
|
||||
while (*tok == ' ' || *tok == '\t') tok++;
|
||||
/* Trim trailing spaces. */
|
||||
char *end = tok + strlen(tok) - 1;
|
||||
while (end > tok && (*end == ' ' || *end == '\t' || *end == '\r' || *end == '\n')) {
|
||||
*end-- = '\0';
|
||||
}
|
||||
if (*tok != '\0') {
|
||||
strncpy(dst + count * elem_len, tok, elem_len - 1);
|
||||
dst[count * elem_len + (elem_len - 1)] = '\0';
|
||||
count++;
|
||||
}
|
||||
tok = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
free(copy);
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
/* Split a comma-separated string of integers into an int array. */
|
||||
static void split_csv_int(const char *s, int *dst, int max, int *count_out) {
|
||||
int count = 0;
|
||||
if (!s) { *count_out = 0; return; }
|
||||
|
||||
char *copy = strdup(s);
|
||||
if (!copy) { *count_out = 0; return; }
|
||||
|
||||
char *saveptr = NULL;
|
||||
char *tok = strtok_r(copy, ",", &saveptr);
|
||||
while (tok && count < max) {
|
||||
while (*tok == ' ' || *tok == '\t') tok++;
|
||||
if (*tok != '\0') {
|
||||
dst[count++] = atoi(tok);
|
||||
}
|
||||
tok = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
free(copy);
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
/* Split a comma-separated string of longs into a long array. */
|
||||
static void split_csv_long(const char *s, long *dst, int max, int *count_out) {
|
||||
int count = 0;
|
||||
if (!s) { *count_out = 0; return; }
|
||||
|
||||
char *copy = strdup(s);
|
||||
if (!copy) { *count_out = 0; return; }
|
||||
|
||||
char *saveptr = NULL;
|
||||
char *tok = strtok_r(copy, ",", &saveptr);
|
||||
while (tok && count < max) {
|
||||
while (*tok == ' ' || *tok == '\t') tok++;
|
||||
if (*tok != '\0') {
|
||||
dst[count++] = strtol(tok, NULL, 10);
|
||||
}
|
||||
tok = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
free(copy);
|
||||
*count_out = count;
|
||||
}
|
||||
|
||||
/* Read a config key as a freshly-allocated string. Returns NULL if missing. */
|
||||
static char *get_key(const char *key) {
|
||||
return pg_inbox_get_config_value(key);
|
||||
}
|
||||
|
||||
/* Read a config key as a boolean ("true"/"1"/"yes" -> 1). */
|
||||
static int get_bool(const char *key, int default_val) {
|
||||
char *v = get_key(key);
|
||||
if (!v) return default_val;
|
||||
int r = (strcmp(v, "true") == 0 || strcmp(v, "1") == 0 ||
|
||||
strcmp(v, "yes") == 0 || strcmp(v, "on") == 0);
|
||||
free(v);
|
||||
return r;
|
||||
}
|
||||
|
||||
/* Read a config key as a long. */
|
||||
static long get_long(const char *key, long default_val) {
|
||||
char *v = get_key(key);
|
||||
if (!v) return default_val;
|
||||
long r = strtol(v, NULL, 10);
|
||||
free(v);
|
||||
return r;
|
||||
}
|
||||
|
||||
/* Read a config key as an int. */
|
||||
static int get_int(const char *key, int default_val) {
|
||||
return (int)get_long(key, (long)default_val);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Public API */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
int pg_config_load(cr_config_t *cfg) {
|
||||
if (!cfg) return -1;
|
||||
memset(cfg, 0, sizeof(*cfg));
|
||||
|
||||
/* root_npubs */
|
||||
char *npubs = get_key("caching_root_npubs");
|
||||
split_csv_str(npubs, (char *)cfg->root_npubs, CR_MAX_ROOT_NPUBS,
|
||||
CR_NPUB_LEN, &cfg->root_npub_count);
|
||||
free(npubs);
|
||||
|
||||
/* bootstrap_relays -> upstream_relays */
|
||||
char *relays = get_key("caching_bootstrap_relays");
|
||||
split_csv_str(relays, (char *)cfg->upstream_relays, CR_MAX_UPSTREAM,
|
||||
CR_URL_LEN, &cfg->upstream_count);
|
||||
free(relays);
|
||||
|
||||
/* local_relay - no longer required for PostgreSQL mode, but keep a
|
||||
* placeholder for compatibility. The caller may override it. */
|
||||
cfg->local_relay[0] = '\0';
|
||||
|
||||
/* kinds */
|
||||
char *kinds = get_key("caching_kinds");
|
||||
split_csv_int(kinds, cfg->kinds, CR_MAX_KINDS, &cfg->kind_count);
|
||||
free(kinds);
|
||||
|
||||
/* admin_kinds - supports "*" to mean all kinds */
|
||||
char *akinds = get_key("caching_admin_kinds");
|
||||
if (akinds) {
|
||||
/* Check for "*" anywhere in the list. */
|
||||
if (strstr(akinds, "*") != NULL) {
|
||||
cfg->admin_all_kinds = 1;
|
||||
} else {
|
||||
split_csv_int(akinds, cfg->admin_kinds, CR_MAX_KINDS,
|
||||
&cfg->admin_kind_count);
|
||||
}
|
||||
free(akinds);
|
||||
}
|
||||
|
||||
/* live */
|
||||
cfg->live.enabled = get_bool("caching_live_enabled", 1);
|
||||
cfg->live.resubscribe_interval_seconds =
|
||||
get_int("caching_live_resubscribe_seconds", 300);
|
||||
|
||||
/* backfill */
|
||||
cfg->backfill.enabled = get_bool("caching_backfill_enabled", 1);
|
||||
cfg->backfill.events_per_tick =
|
||||
get_int("caching_backfill_page_size", 500);
|
||||
/* tick interval is stored in ms in the config table; convert to seconds. */
|
||||
int tick_ms = get_int("caching_backfill_tick_interval_ms", 5000);
|
||||
cfg->backfill.tick_interval_seconds = (tick_ms + 999) / 1000;
|
||||
if (cfg->backfill.tick_interval_seconds < 1)
|
||||
cfg->backfill.tick_interval_seconds = 1;
|
||||
|
||||
/* follow graph refresh */
|
||||
cfg->follow_graph_refresh_seconds =
|
||||
get_int("caching_follow_graph_refresh_seconds", 600);
|
||||
|
||||
/* Defaults if missing. */
|
||||
if (cfg->backfill.events_per_tick == 0)
|
||||
cfg->backfill.events_per_tick = 500;
|
||||
if (cfg->backfill.tick_interval_seconds == 0)
|
||||
cfg->backfill.tick_interval_seconds = 5;
|
||||
if (cfg->live.resubscribe_interval_seconds == 0)
|
||||
cfg->live.resubscribe_interval_seconds = 300;
|
||||
if (cfg->follow_graph_refresh_seconds == 0)
|
||||
cfg->follow_graph_refresh_seconds = 600;
|
||||
|
||||
/* State: per-author backfill progress lives in caching_followed_pubkeys.
|
||||
* backfilled_until is only used as a first-time flag in legacy mode. */
|
||||
cfg->state.backfilled_until = 0;
|
||||
|
||||
/* Validation. */
|
||||
if (cfg->root_npub_count == 0) {
|
||||
DEBUG_ERROR("pg_config: at least one caching_root_npubs required");
|
||||
return -1;
|
||||
}
|
||||
if (cfg->upstream_count == 0) {
|
||||
DEBUG_ERROR("pg_config: at least one caching_bootstrap_relays required");
|
||||
return -1;
|
||||
}
|
||||
if (cfg->kind_count == 0) {
|
||||
DEBUG_ERROR("pg_config: at least one caching_kinds required");
|
||||
return -1;
|
||||
}
|
||||
|
||||
DEBUG_INFO("pg_config loaded: %d root npubs, %d upstream relays, %d kinds",
|
||||
cfg->root_npub_count, cfg->upstream_count, cfg->kind_count);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int pg_config_generation_changed(long last_generation) {
|
||||
long cur = pg_inbox_get_config_generation();
|
||||
if (cur < 0) return -1;
|
||||
return (cur != last_generation) ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* caching_relay - read caching configuration from the c-relay-pg PostgreSQL
|
||||
* config table and populate the existing cr_config_t structure.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_PG_CONFIG_H
|
||||
#define CACHING_RELAY_PG_CONFIG_H
|
||||
|
||||
#include "config.h" /* cr_config_t */
|
||||
|
||||
/* Load caching configuration from the c-relay-pg PostgreSQL config table.
|
||||
* Populates the cr_config_t fields from the config table.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_config_load(cr_config_t *cfg);
|
||||
|
||||
/* Check if the config generation has changed since the last load.
|
||||
* Returns 1 if changed, 0 if not, -1 on error. */
|
||||
int pg_config_generation_changed(long last_generation);
|
||||
|
||||
#endif /* CACHING_RELAY_PG_CONFIG_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
* caching_relay - PostgreSQL inbox connection and operations.
|
||||
*
|
||||
* Manages the libpq connection to the c-relay-pg PostgreSQL database and
|
||||
* provides functions to insert fetched events into the caching_event_inbox
|
||||
* table, update the caching_service_state singleton, and read configuration
|
||||
* from the shared config table.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_PG_INBOX_H
|
||||
#define CACHING_RELAY_PG_INBOX_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Forward declaration so the header does not pull in cJSON everywhere. */
|
||||
typedef struct cJSON cJSON;
|
||||
|
||||
/* Initialize PostgreSQL connection using the given connection string.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_init(const char *connection_string);
|
||||
|
||||
/* Close the PostgreSQL connection. */
|
||||
void pg_inbox_shutdown(void);
|
||||
|
||||
/* Insert an event into the caching_event_inbox table.
|
||||
* event_json is the raw Nostr event JSON string.
|
||||
* source_relay may be NULL. source_class is "live", "discovery", or "backfill".
|
||||
* priority is 0 (live/discovery) or 1 (backfill).
|
||||
* Returns 0 on success (including conflict-skip), -1 on error. */
|
||||
int pg_inbox_insert_event(const char *event_json, const char *source_relay,
|
||||
const char *source_class, int priority);
|
||||
|
||||
/* Update the caching_service_state singleton row.
|
||||
* All string parameters may be NULL (left unchanged).
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_update_status(const char *service_state,
|
||||
long config_generation,
|
||||
long heartbeat_at,
|
||||
int followed_author_count,
|
||||
int selected_relay_count,
|
||||
int connected_relay_count,
|
||||
int backfill_authors_complete,
|
||||
int backfill_authors_total,
|
||||
long events_fetched,
|
||||
long inbox_inserts,
|
||||
const char *last_error,
|
||||
long last_error_at);
|
||||
|
||||
/* Read a config value from the c-relay-pg config table.
|
||||
* Returns a malloc'd string (caller must free) or NULL if not found/error. */
|
||||
char *pg_inbox_get_config_value(const char *key);
|
||||
|
||||
/* Read the current config generation.
|
||||
* Returns the generation number, or -1 on error. */
|
||||
long pg_inbox_get_config_generation(void);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-author until-cursor drain backfill model */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Reset all backfill progress: set until_cursor=0 and backfill_complete=FALSE
|
||||
* for all rows in caching_followed_pubkeys (the followed set itself is
|
||||
* preserved). Used by the "Reset Backfill" API button and --restart.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_reset_backfill_progress(void);
|
||||
|
||||
/* Upsert all followed pubkeys into caching_followed_pubkeys.
|
||||
* For each pubkey: INSERT if new (until_cursor=0, backfill_complete=FALSE),
|
||||
* or UPDATE is_root and last_seen if existing (until_cursor and
|
||||
* backfill_complete are NOT touched). root_pubkeys marks which entries are
|
||||
* root follows. Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_sync_followed_pubkeys(const char **pubkeys, int count,
|
||||
const char **root_pubkeys, int root_count);
|
||||
|
||||
/* Pick the next incomplete author for backfill, round-robin.
|
||||
* On success fills out_pk, out_until_cursor and out_is_root, advances
|
||||
* *round_cursor, and returns 0. Returns -1 if no incomplete authors or
|
||||
* on error. */
|
||||
int pg_inbox_pick_next_backfill_author(char *out_pk, int pk_len,
|
||||
long *out_until_cursor,
|
||||
int *out_is_root,
|
||||
int *round_cursor);
|
||||
|
||||
/* Update the cursor and completion state for an author.
|
||||
* events_this_page is added to the running events_fetched counter.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_update_backfill_progress(const char *pk, long until_cursor,
|
||||
int complete, int events_this_page);
|
||||
|
||||
/* One-time migration from the old caching_backfill_progress table to the
|
||||
* new caching_followed_pubkeys table. Only runs if the new table is empty
|
||||
* and the old table has rows. Returns 0 on success (including no-op),
|
||||
* -1 on error. */
|
||||
int pg_inbox_migrate_backfill_progress(void);
|
||||
|
||||
/* Count complete/total authors for status reporting.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_count_backfill_progress(int *out_complete, int *out_total);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-relay backfill progress (relay-by-relay drain model) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Initialize relay progress rows for an author. Creates one row per relay
|
||||
* URL with until_cursor=0, complete=FALSE. Skips rows that already exist
|
||||
* (ON CONFLICT DO NOTHING). Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_init_relay_progress_for_author(const char *pk,
|
||||
const char **relay_urls,
|
||||
int relay_count);
|
||||
|
||||
/* Pick the next author that has at least one incomplete relay progress row.
|
||||
* Round-robin via *round_cursor. On success fills out_pk and returns 0.
|
||||
* Returns -1 if no incomplete authors or on error. */
|
||||
int pg_inbox_pick_next_author_with_incomplete_relays(char *out_pk, int pk_len,
|
||||
int *round_cursor);
|
||||
|
||||
/* Get incomplete relays for an author. Returns a cJSON array of
|
||||
* {relay_url, until_cursor} objects (caller frees with cJSON_Delete).
|
||||
* Returns NULL on error or if no incomplete relays. */
|
||||
cJSON *pg_inbox_get_incomplete_relays(const char *pk);
|
||||
|
||||
/* Update relay progress: advance cursor and/or mark complete.
|
||||
* events_this_page is added to the running events_fetched counter.
|
||||
* last_status is a short string like "eose", "timeout", "error", or NULL
|
||||
* to leave unchanged. Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_update_relay_progress(const char *pk, const char *relay_url,
|
||||
long until_cursor, int complete,
|
||||
int events_this_page,
|
||||
const char *last_status);
|
||||
|
||||
/* Check if all relays for an author are complete. If so, mark the author
|
||||
* as backfill_complete=TRUE in caching_followed_pubkeys.
|
||||
* Returns 1 if marked complete, 0 if still incomplete, -1 on error. */
|
||||
int pg_inbox_check_and_mark_author_complete(const char *pk);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Active backfill target (for UI status display) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Set the currently-active backfill target (pubkey + relay being queried).
|
||||
* Either parameter may be NULL/empty to clear. Creates the
|
||||
* caching_backfill_active table if needed (single-row singleton).
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_set_active_target(const char *pubkey, const char *relay_url);
|
||||
|
||||
/* Read the active backfill target. Fills out_pubkey/out_relay with
|
||||
* the current values (or empty strings if none). Buffers must be
|
||||
* at least 65 and 256 bytes respectively.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_get_active_target(char *out_pubkey, int pk_len,
|
||||
char *out_relay, int relay_len);
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Forward catch-up support (last_event_at tracking) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* Returns a cJSON array of {pubkey, last_event_at} objects for all
|
||||
* followed authors where last_event_at > 0. Caller must cJSON_Delete().
|
||||
* Returns NULL on error. */
|
||||
cJSON* pg_inbox_get_authors_for_catchup(void);
|
||||
|
||||
/* Update last_event_at for a pubkey to the GREATEST of current and
|
||||
* the given value. Returns 0 on success, -1 on error. */
|
||||
int pg_inbox_update_last_event_at(const char *pk, long event_created_at);
|
||||
|
||||
#endif /* CACHING_RELAY_PG_INBOX_H */
|
||||
@@ -0,0 +1,561 @@
|
||||
/*
|
||||
* caching_relay - NIP-65 outbox relay discovery
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "relay_discovery.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <signal.h>
|
||||
|
||||
/* Access the shutdown flag from main.c so we can abort discovery early. */
|
||||
extern volatile sig_atomic_t g_shutdown;
|
||||
|
||||
/* Normalize a relay URL: strip trailing slash for consistency. */
|
||||
static void normalize_url(char *url, size_t maxlen) {
|
||||
size_t len = strlen(url);
|
||||
while (len > 0 && url[len - 1] == '/') {
|
||||
url[--len] = '\0';
|
||||
}
|
||||
(void)maxlen;
|
||||
}
|
||||
|
||||
/* ---- helpers ---- */
|
||||
|
||||
static int get_pool_urls(nostr_relay_pool_t *pool, const char ***urls_out) {
|
||||
char **listed = NULL;
|
||||
nostr_pool_relay_status_t *statuses = NULL;
|
||||
int n = nostr_relay_pool_list_relays(pool, &listed, &statuses);
|
||||
const char **urls = NULL;
|
||||
if (n > 0) {
|
||||
urls = malloc(n * sizeof(char *));
|
||||
for (int j = 0; j < n; j++) urls[j] = listed[j];
|
||||
}
|
||||
free(listed);
|
||||
free(statuses);
|
||||
*urls_out = urls;
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Query the most recent kind 10002 for a single pubkey from a pool.
|
||||
* Returns the event (caller must cJSON_Delete) or NULL. */
|
||||
static cJSON *query_kind10002(nostr_relay_pool_t *pool, const char *hex) {
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(hex));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(10002));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(1));
|
||||
|
||||
const char **urls = NULL;
|
||||
int n = get_pool_urls(pool, &urls);
|
||||
int ev_count = 0;
|
||||
cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 10000);
|
||||
free(urls);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
cJSON *result = NULL;
|
||||
if (events && ev_count > 0) {
|
||||
result = events[0]; /* take first (most recent) */
|
||||
for (int k = 1; k < ev_count; k++) cJSON_Delete(events[k]);
|
||||
}
|
||||
free(events);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Parse "r" tags from a kind 10002 event into an outbox entry.
|
||||
* Only includes relays with "read" marker or no marker (assume both).
|
||||
* Skips "write" only relays. */
|
||||
static int parse_r_tags(cJSON *event, cr_outbox_entry_t *entry) {
|
||||
memset(entry, 0, sizeof(*entry));
|
||||
cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
if (pubkey && cJSON_IsString(pubkey)) {
|
||||
strncpy(entry->pubkey, cJSON_GetStringValue(pubkey), CR_HEX_LEN - 1);
|
||||
}
|
||||
|
||||
cJSON *tags = cJSON_GetObjectItem(event, "tags");
|
||||
if (!tags || !cJSON_IsArray(tags)) return 0;
|
||||
|
||||
int added = 0;
|
||||
cJSON *tag;
|
||||
cJSON_ArrayForEach(tag, tags) {
|
||||
if (!cJSON_IsArray(tag)) continue;
|
||||
cJSON *name = cJSON_GetArrayItem(tag, 0);
|
||||
if (!name || !cJSON_IsString(name)) continue;
|
||||
if (strcmp(cJSON_GetStringValue(name), "r") != 0) continue;
|
||||
cJSON *url = cJSON_GetArrayItem(tag, 1);
|
||||
if (!url || !cJSON_IsString(url)) continue;
|
||||
const char *relay_url = cJSON_GetStringValue(url);
|
||||
if (!relay_url || !relay_url[0]) continue;
|
||||
|
||||
/* Check marker (3rd element): "read", "write", or absent. */
|
||||
cJSON *marker = cJSON_GetArrayItem(tag, 2);
|
||||
if (marker && cJSON_IsString(marker)) {
|
||||
const char *m = cJSON_GetStringValue(marker);
|
||||
if (strcmp(m, "write") == 0) continue; /* skip write-only relays */
|
||||
}
|
||||
/* No marker or "read" => include. */
|
||||
|
||||
if (entry->relay_count >= CR_MAX_RELAYS_PER_PUBKEY) break;
|
||||
strncpy(entry->relays[entry->relay_count], relay_url, CR_URL_LEN - 1);
|
||||
entry->relays[entry->relay_count][CR_URL_LEN - 1] = '\0';
|
||||
normalize_url(entry->relays[entry->relay_count], CR_URL_LEN);
|
||||
entry->relay_count++;
|
||||
added++;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
/* ---- greedy set cover ---- */
|
||||
|
||||
/* Build relay -> pubkey coverage map and compute minimum covering set. */
|
||||
static void compute_covering_set(cr_relay_map_t *map, cr_config_t *cfg,
|
||||
const cr_pubkey_set_t *followed) {
|
||||
/* Build a temporary relay -> pubkeys map. */
|
||||
typedef struct {
|
||||
char url[CR_URL_LEN];
|
||||
char pubkeys[CR_MAX_PUBKEYS_PER_RELAY][CR_HEX_LEN];
|
||||
int pubkey_count;
|
||||
} relay_cov_t;
|
||||
|
||||
relay_cov_t *relays = calloc(CR_MAX_DISCOVERED_RELAYS, sizeof(relay_cov_t));
|
||||
int relay_count = 0;
|
||||
|
||||
/* Always include bootstrap relays in the coverage map. */
|
||||
for (int i = 0; i < cfg->upstream_count && relay_count < CR_MAX_DISCOVERED_RELAYS; i++) {
|
||||
strncpy(relays[relay_count].url, cfg->upstream_relays[i], CR_URL_LEN - 1);
|
||||
relays[relay_count].url[CR_URL_LEN - 1] = '\0';
|
||||
normalize_url(relays[relay_count].url, CR_URL_LEN);
|
||||
relay_count++;
|
||||
}
|
||||
|
||||
/* Add outbox relays and build coverage. */
|
||||
for (int i = 0; i < map->outbox_count; i++) {
|
||||
cr_outbox_entry_t *oe = &map->outboxes[i];
|
||||
for (int r = 0; r < oe->relay_count; r++) {
|
||||
/* Find or create relay entry. */
|
||||
int found = -1;
|
||||
for (int j = 0; j < relay_count; j++) {
|
||||
if (strcmp(relays[j].url, oe->relays[r]) == 0) { found = j; break; }
|
||||
}
|
||||
if (found < 0) {
|
||||
if (relay_count >= CR_MAX_DISCOVERED_RELAYS) break;
|
||||
strncpy(relays[relay_count].url, oe->relays[r], CR_URL_LEN - 1);
|
||||
found = relay_count++;
|
||||
}
|
||||
/* Add this pubkey to the relay's coverage. */
|
||||
if (relays[found].pubkey_count < CR_MAX_PUBKEYS_PER_RELAY) {
|
||||
strncpy(relays[found].pubkeys[relays[found].pubkey_count],
|
||||
oe->pubkey, CR_HEX_LEN - 1);
|
||||
relays[found].pubkey_count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Greedy set cover. */
|
||||
char *covered = calloc(followed->count, 1); /* 1 if pubkey is covered */
|
||||
map->selected_count = 0;
|
||||
map->coverage = calloc(map->outbox_count, sizeof(int));
|
||||
for (int i = 0; i < map->outbox_count; i++) map->coverage[i] = -1;
|
||||
|
||||
int uncovered_count = followed->count;
|
||||
|
||||
/* Helper: find index of a pubkey in the followed set. */
|
||||
/* (linear search, fine for our scale) */
|
||||
|
||||
while (uncovered_count > 0 && map->selected_count < CR_MAX_DISCOVERED_RELAYS) {
|
||||
int best_relay = -1;
|
||||
int best_count = 0;
|
||||
|
||||
for (int r = 0; r < relay_count; r++) {
|
||||
/* Count how many uncovered pubkeys this relay covers. */
|
||||
int count = 0;
|
||||
for (int p = 0; p < relays[r].pubkey_count; p++) {
|
||||
/* Find this pubkey in followed set. */
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (strcmp(followed->items[i], relays[r].pubkeys[p]) == 0) {
|
||||
if (!covered[i]) count++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (count > best_count) {
|
||||
best_count = count;
|
||||
best_relay = r;
|
||||
}
|
||||
}
|
||||
|
||||
if (best_relay < 0 || best_count == 0) break; /* no more coverage possible */
|
||||
|
||||
/* Select this relay. */
|
||||
strncpy(map->selected_relays[map->selected_count],
|
||||
relays[best_relay].url, CR_URL_LEN - 1);
|
||||
map->selected_count++;
|
||||
|
||||
/* Mark covered pubkeys and record coverage. */
|
||||
for (int p = 0; p < relays[best_relay].pubkey_count; p++) {
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (strcmp(followed->items[i], relays[best_relay].pubkeys[p]) == 0) {
|
||||
if (!covered[i]) {
|
||||
covered[i] = 1;
|
||||
uncovered_count--;
|
||||
/* Record which selected relay covers this outbox entry. */
|
||||
for (int oe_i = 0; oe_i < map->outbox_count; oe_i++) {
|
||||
if (strcmp(map->outboxes[oe_i].pubkey,
|
||||
relays[best_relay].pubkeys[p]) == 0) {
|
||||
if (map->coverage[oe_i] < 0) {
|
||||
map->coverage[oe_i] = map->selected_count - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Log results. */
|
||||
DEBUG_INFO("relay_discovery: selected %d relays to cover %d/%d pubkeys",
|
||||
map->selected_count, followed->count - uncovered_count, followed->count);
|
||||
for (int i = 0; i < map->selected_count; i++) {
|
||||
DEBUG_INFO(" relay[%d]: %s", i, map->selected_relays[i]);
|
||||
}
|
||||
if (uncovered_count > 0) {
|
||||
DEBUG_WARN("relay_discovery: %d pubkeys have no outbox relays (covered by bootstrap)",
|
||||
uncovered_count);
|
||||
}
|
||||
|
||||
free(covered);
|
||||
free(relays);
|
||||
}
|
||||
|
||||
/* ---- batched kind-10002 query ---- */
|
||||
|
||||
/* Query kind 10002 for a batch of up to 100 pubkeys from a pool.
|
||||
* Returns a cJSON array of events (caller must cJSON_Delete each + free).
|
||||
* Sets *out_count to the number of events returned. */
|
||||
static cJSON **query_kind10002_batch(nostr_relay_pool_t *pool,
|
||||
const char **pubkeys, int pubkey_count,
|
||||
int *out_count) {
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
for (int i = 0; i < pubkey_count; i++)
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(pubkeys[i]));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(10002));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
|
||||
const char **urls = NULL;
|
||||
int n = get_pool_urls(pool, &urls);
|
||||
|
||||
/* TRACE: log the query being sent. */
|
||||
if (g_debug_level >= DEBUG_LEVEL_TRACE) {
|
||||
char *filter_str = cJSON_PrintUnformatted(filter);
|
||||
DEBUG_TRACE("query_kind10002_batch: querying %d relays for %d pubkeys", n, pubkey_count);
|
||||
if (filter_str) {
|
||||
/* Only print first 200 chars of filter to avoid huge output */
|
||||
if (strlen(filter_str) > 200) filter_str[200] = '\0';
|
||||
DEBUG_TRACE(" filter: %s...", filter_str);
|
||||
free(filter_str);
|
||||
}
|
||||
}
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 30000);
|
||||
free(urls);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
DEBUG_TRACE("query_kind10002_batch: returned %d events", ev_count);
|
||||
|
||||
*out_count = ev_count;
|
||||
return events;
|
||||
}
|
||||
|
||||
/* ---- batched kind-3 fallback query ---- */
|
||||
|
||||
/* Query kind 3 (contact list) for a batch of pubkeys from a pool.
|
||||
* Kind-3 events historically contain "r" tags with relay URLs (pre-NIP-65).
|
||||
* We use this as a fallback for pubkeys that don't have kind-10002.
|
||||
* Returns a cJSON array of events (caller must cJSON_Delete each + free).
|
||||
* Sets *out_count to the number of events returned. */
|
||||
static cJSON **query_kind3_batch(nostr_relay_pool_t *pool,
|
||||
const char **pubkeys, int pubkey_count,
|
||||
int *out_count) {
|
||||
cJSON *filter = cJSON_CreateObject();
|
||||
cJSON *authors = cJSON_CreateArray();
|
||||
for (int i = 0; i < pubkey_count; i++)
|
||||
cJSON_AddItemToArray(authors, cJSON_CreateString(pubkeys[i]));
|
||||
cJSON_AddItemToObject(filter, "authors", authors);
|
||||
cJSON *kinds = cJSON_CreateArray();
|
||||
cJSON_AddItemToArray(kinds, cJSON_CreateNumber(3));
|
||||
cJSON_AddItemToObject(filter, "kinds", kinds);
|
||||
/* limit=1 per author: we only need the most recent kind-3.
|
||||
* The relay pool query_sync returns the most recent events. */
|
||||
cJSON_AddItemToObject(filter, "limit", cJSON_CreateNumber(pubkey_count));
|
||||
|
||||
const char **urls = NULL;
|
||||
int n = get_pool_urls(pool, &urls);
|
||||
|
||||
DEBUG_TRACE("query_kind3_batch: querying %d relays for %d pubkeys", n, pubkey_count);
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = nostr_relay_pool_query_sync(pool, urls, n, filter, &ev_count, 30000);
|
||||
free(urls);
|
||||
cJSON_Delete(filter);
|
||||
|
||||
DEBUG_TRACE("query_kind3_batch: returned %d events", ev_count);
|
||||
|
||||
*out_count = ev_count;
|
||||
return events;
|
||||
}
|
||||
|
||||
/* Process a batch of kind-10002 events: parse r-tags, populate outbox entries,
|
||||
* publish to local relay. Returns count of events processed. */
|
||||
static int process_10002_batch(cJSON **events, int ev_count,
|
||||
cr_relay_map_t *map, cr_sink_t *sink,
|
||||
const char *source) {
|
||||
int processed = 0;
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cJSON *event = events[k];
|
||||
if (!event) continue;
|
||||
|
||||
/* Get pubkey from event. */
|
||||
cJSON *pk = cJSON_GetObjectItem(event, "pubkey");
|
||||
if (!pk || !cJSON_IsString(pk)) { cJSON_Delete(event); continue; }
|
||||
const char *hex = cJSON_GetStringValue(pk);
|
||||
|
||||
/* Find or create outbox entry for this pubkey. */
|
||||
cr_outbox_entry_t *oe = NULL;
|
||||
for (int i = 0; i < map->outbox_count; i++) {
|
||||
if (strcmp(map->outboxes[i].pubkey, hex) == 0) { oe = &map->outboxes[i]; break; }
|
||||
}
|
||||
if (!oe) {
|
||||
if (map->outbox_count >= /* followed->count passed in via map size */ 99999) {
|
||||
cJSON_Delete(event); continue;
|
||||
}
|
||||
oe = &map->outboxes[map->outbox_count];
|
||||
strncpy(oe->pubkey, hex, CR_HEX_LEN - 1);
|
||||
map->outbox_count++;
|
||||
}
|
||||
|
||||
int n = parse_r_tags(event, oe);
|
||||
(void)n;
|
||||
|
||||
if (strcmp(source, "bootstrap") == 0) {
|
||||
cr_sink_publish(sink, event);
|
||||
/* Pump sink pool every 16 events to avoid pending publish overflow
|
||||
* (NOSTR_POOL_MAX_PENDING_PUBLISHES = 32). */
|
||||
if (processed > 0 && (processed % 16) == 0) {
|
||||
cr_sink_pump(sink, 200);
|
||||
}
|
||||
}
|
||||
|
||||
processed++;
|
||||
cJSON_Delete(event);
|
||||
}
|
||||
free(events);
|
||||
|
||||
/* Log sink stats after each batch. */
|
||||
DEBUG_INFO("relay_discovery: batch processed %d events (sink: ok=%ld reject=%ld dup=%ld err=%ld)",
|
||||
processed, sink->published_ok, sink->published_failed,
|
||||
sink->published_dup, 0L);
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/* ---- main discovery function ---- */
|
||||
|
||||
#define CR_10002_BATCH_SIZE 100
|
||||
|
||||
int cr_relay_discovery_run(cr_relay_map_t *map, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream, cr_sink_t *sink,
|
||||
const cr_pubkey_set_t *followed, int is_first_time) {
|
||||
memset(map, 0, sizeof(*map));
|
||||
map->outboxes = calloc(followed->count, sizeof(cr_outbox_entry_t));
|
||||
if (!map->outboxes) return -1;
|
||||
map->outbox_count = 0;
|
||||
|
||||
DEBUG_INFO("relay_discovery: discovering outbox relays for %d pubkeys (%s)",
|
||||
followed->count, is_first_time ? "first-time" : "local-first");
|
||||
|
||||
int found_local = 0, found_bootstrap = 0, found_none = 0;
|
||||
|
||||
/* Pre-populate outbox entries for all followed pubkeys (empty, for coverage
|
||||
* tracking of pubkeys with no 10002). */
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
strncpy(map->outboxes[i].pubkey, followed->items[i], CR_HEX_LEN - 1);
|
||||
}
|
||||
map->outbox_count = followed->count;
|
||||
|
||||
/* Track which pubkeys we've found 10002 for. */
|
||||
char *found = calloc(followed->count, 1);
|
||||
|
||||
/* Phase 1: If subsequent startup, batch-query local relay first. */
|
||||
if (!is_first_time && sink && sink->pool) {
|
||||
DEBUG_INFO("relay_discovery: querying local relay for kind-10002 (batched)...");
|
||||
|
||||
for (int start = 0; start < followed->count && !g_shutdown; start += CR_10002_BATCH_SIZE) {
|
||||
int batch = followed->count - start;
|
||||
if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = query_kind10002_batch(sink->pool,
|
||||
(const char **)&followed->items[start],
|
||||
batch, &ev_count);
|
||||
if (events && ev_count > 0) {
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
|
||||
if (pk && cJSON_IsString(pk)) {
|
||||
const char *hex = cJSON_GetStringValue(pk);
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (strcmp(followed->items[i], hex) == 0) {
|
||||
if (!found[i]) { found[i] = 1; found_local++; }
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
process_10002_batch(events, ev_count, map, sink, "local");
|
||||
/* Pump sink pool to flush published events. */
|
||||
cr_sink_pump(sink, 500);
|
||||
} else {
|
||||
free(events);
|
||||
}
|
||||
}
|
||||
DEBUG_INFO("relay_discovery: local relay provided %d kind-10002 events", found_local);
|
||||
}
|
||||
|
||||
/* Phase 2: Batch-query bootstrap relays for pubkeys not found locally. */
|
||||
int remaining = 0;
|
||||
for (int i = 0; i < followed->count; i++) if (!found[i]) remaining++;
|
||||
|
||||
if (remaining > 0) {
|
||||
DEBUG_INFO("relay_discovery: querying bootstrap relays for %d remaining pubkeys (batched)...",
|
||||
remaining);
|
||||
|
||||
/* Build list of remaining pubkeys. */
|
||||
const char **remaining_pk = malloc(remaining * sizeof(char *));
|
||||
int ridx = 0;
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (!found[i]) remaining_pk[ridx++] = followed->items[i];
|
||||
}
|
||||
|
||||
for (int start = 0; start < remaining && !g_shutdown; start += CR_10002_BATCH_SIZE) {
|
||||
int batch = remaining - start;
|
||||
if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = query_kind10002_batch(upstream,
|
||||
&remaining_pk[start],
|
||||
batch, &ev_count);
|
||||
if (events && ev_count > 0) {
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
|
||||
if (pk && cJSON_IsString(pk)) {
|
||||
const char *hex = cJSON_GetStringValue(pk);
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (strcmp(followed->items[i], hex) == 0) {
|
||||
if (!found[i]) { found[i] = 1; found_bootstrap++; }
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
process_10002_batch(events, ev_count, map, sink, "bootstrap");
|
||||
/* Pump sink pool to flush published events. */
|
||||
cr_sink_pump(sink, 500);
|
||||
} else {
|
||||
free(events);
|
||||
}
|
||||
}
|
||||
free(remaining_pk);
|
||||
}
|
||||
|
||||
/* Phase 3: Kind-3 fallback for pubkeys still without relay lists.
|
||||
* Kind-3 (contact list) events historically contain "r" tags with relay
|
||||
* URLs (pre-NIP-65). This catches older users who never published kind-10002. */
|
||||
int remaining_k3 = 0;
|
||||
for (int i = 0; i < followed->count; i++) if (!found[i]) remaining_k3++;
|
||||
|
||||
if (remaining_k3 > 0) {
|
||||
DEBUG_INFO("relay_discovery: querying bootstrap relays for kind-3 fallback (%d pubkeys)...",
|
||||
remaining_k3);
|
||||
|
||||
const char **remaining_pk = malloc(remaining_k3 * sizeof(char *));
|
||||
int ridx = 0;
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (!found[i]) remaining_pk[ridx++] = followed->items[i];
|
||||
}
|
||||
|
||||
int found_k3 = 0;
|
||||
for (int start = 0; start < remaining_k3 && !g_shutdown; start += CR_10002_BATCH_SIZE) {
|
||||
int batch = remaining_k3 - start;
|
||||
if (batch > CR_10002_BATCH_SIZE) batch = CR_10002_BATCH_SIZE;
|
||||
|
||||
int ev_count = 0;
|
||||
cJSON **events = query_kind3_batch(upstream,
|
||||
&remaining_pk[start],
|
||||
batch, &ev_count);
|
||||
if (events && ev_count > 0) {
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cJSON *pk = cJSON_GetObjectItem(events[k], "pubkey");
|
||||
if (pk && cJSON_IsString(pk)) {
|
||||
const char *hex = cJSON_GetStringValue(pk);
|
||||
for (int i = 0; i < followed->count; i++) {
|
||||
if (strcmp(followed->items[i], hex) == 0) {
|
||||
if (!found[i]) { found[i] = 1; found_k3++; }
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/* process_10002_batch works for kind-3 too — same r-tag format */
|
||||
process_10002_batch(events, ev_count, map, sink, "bootstrap");
|
||||
cr_sink_pump(sink, 500);
|
||||
} else {
|
||||
free(events);
|
||||
}
|
||||
}
|
||||
free(remaining_pk);
|
||||
DEBUG_INFO("relay_discovery: kind-3 fallback found %d additional pubkeys with relays", found_k3);
|
||||
}
|
||||
|
||||
found_none = followed->count - found_local - found_bootstrap;
|
||||
/* Recalculate found_none after kind-3 fallback */
|
||||
int still_none = 0;
|
||||
for (int i = 0; i < followed->count; i++) if (!found[i]) still_none++;
|
||||
DEBUG_INFO("relay_discovery: %d local, %d bootstrap, %d none",
|
||||
found_local, found_bootstrap, still_none);
|
||||
|
||||
free(found);
|
||||
|
||||
/* Compute minimum covering set. */
|
||||
compute_covering_set(map, cfg, followed);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cr_relay_map_free(cr_relay_map_t *map) {
|
||||
if (map->outboxes) {
|
||||
free(map->outboxes);
|
||||
map->outboxes = NULL;
|
||||
}
|
||||
if (map->coverage) {
|
||||
free(map->coverage);
|
||||
map->coverage = NULL;
|
||||
}
|
||||
map->outbox_count = 0;
|
||||
map->selected_count = 0;
|
||||
}
|
||||
|
||||
const cr_outbox_entry_t *cr_relay_map_get_outbox(const cr_relay_map_t *map, const char *hex) {
|
||||
for (int i = 0; i < map->outbox_count; i++) {
|
||||
if (strcmp(map->outboxes[i].pubkey, hex) == 0) return &map->outboxes[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* caching_relay - NIP-65 outbox relay discovery.
|
||||
*
|
||||
* For each followed pubkey, fetches their kind 10002 (relay list) and parses
|
||||
* the "r" tags to build a per-pubkey outbox relay map. Then computes the
|
||||
* minimum covering set of relays (greedy set cover) that covers all followed
|
||||
* pubkeys.
|
||||
*
|
||||
* On first-time startup, queries bootstrap relays for kind 10002.
|
||||
* On subsequent startups, queries the local relay first (fast), falls back
|
||||
* to bootstrap relays for any pubkey not found locally.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_RELAY_DISCOVERY_H
|
||||
#define CACHING_RELAY_RELAY_DISCOVERY_H
|
||||
|
||||
#include "config.h"
|
||||
#include "state.h"
|
||||
#include "relay_sink.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
#define CR_MAX_RELAYS_PER_PUBKEY 8
|
||||
#define CR_MAX_PUBKEYS_PER_RELAY 512
|
||||
#define CR_MAX_DISCOVERED_RELAYS 128
|
||||
|
||||
/* Per-pubkey outbox relay list. */
|
||||
typedef struct {
|
||||
char pubkey[CR_HEX_LEN];
|
||||
char relays[CR_MAX_RELAYS_PER_PUBKEY][CR_URL_LEN];
|
||||
int relay_count;
|
||||
} cr_outbox_entry_t;
|
||||
|
||||
/* Result of relay discovery. */
|
||||
typedef struct {
|
||||
cr_outbox_entry_t *outboxes; /* per-pubkey relay lists (heap) */
|
||||
int outbox_count;
|
||||
|
||||
/* The selected minimum covering set of relay URLs. */
|
||||
char selected_relays[CR_MAX_DISCOVERED_RELAYS][CR_URL_LEN];
|
||||
int selected_count;
|
||||
|
||||
/* Per-pubkey: which selected relay covers this pubkey (index into selected_relays).
|
||||
* -1 means covered by bootstrap relays only. */
|
||||
int *coverage; /* heap, outbox_count entries */
|
||||
} cr_relay_map_t;
|
||||
|
||||
/* Discover outbox relays for all followed pubkeys.
|
||||
*
|
||||
* cfg - config (provides bootstrap relays, local_relay, root_hex)
|
||||
* upstream - upstream pool (has bootstrap relays; discovered relays will be added)
|
||||
* sink - sink pool (local relay, for publishing 10002 events + querying on subsequent startup)
|
||||
* followed - the followed pubkey set
|
||||
* is_first_time - 1 if first startup (query bootstrap only), 0 if subsequent (local-first)
|
||||
*
|
||||
* Returns 0 on success, -1 on hard error. On success, map is populated.
|
||||
* Caller must call cr_relay_map_free() when done.
|
||||
*/
|
||||
int cr_relay_discovery_run(cr_relay_map_t *map, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream, cr_sink_t *sink,
|
||||
const cr_pubkey_set_t *followed, int is_first_time);
|
||||
|
||||
/* Free heap resources in a relay map. */
|
||||
void cr_relay_map_free(cr_relay_map_t *map);
|
||||
|
||||
/* Get the outbox relays for a specific pubkey. Returns NULL if not found. */
|
||||
const cr_outbox_entry_t *cr_relay_map_get_outbox(const cr_relay_map_t *map, const char *hex);
|
||||
|
||||
#endif /* CACHING_RELAY_RELAY_DISCOVERY_H */
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* caching_relay - relay sink: publishes fetched events to the local relay.
|
||||
*
|
||||
* See relay_sink.h for the dual-mode (WebSocket / PostgreSQL inbox) design.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "relay_sink.h"
|
||||
#include "pg_inbox.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* WebSocket mode callback */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void sink_publish_cb(const char *relay_url, const char *event_id,
|
||||
int success, const char *message, void *user_data) {
|
||||
cr_sink_t *sink = (cr_sink_t *)user_data;
|
||||
if (success) {
|
||||
sink->published_ok++;
|
||||
DEBUG_TRACE("sink: OK %s from %s", event_id ? event_id : "?",
|
||||
relay_url ? relay_url : "?");
|
||||
} else {
|
||||
sink->published_failed++;
|
||||
/* REJECT is important - show at INFO level so it's visible at level 3. */
|
||||
DEBUG_INFO("sink: REJECT %s from %s: %s", event_id ? event_id : "?",
|
||||
relay_url ? relay_url : "?",
|
||||
message ? message : "(no message)");
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Init */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
int cr_sink_init(cr_sink_t *sink, const char *local_relay_url, cr_seen_ring_t *seen) {
|
||||
memset(sink, 0, sizeof(*sink));
|
||||
sink->seen = seen;
|
||||
sink->source_class = CR_SINK_CLASS_LIVE;
|
||||
strncpy(sink->url, local_relay_url, sizeof(sink->url) - 1);
|
||||
|
||||
sink->pool = nostr_relay_pool_create(nostr_pool_reconnect_config_default());
|
||||
if (!sink->pool) {
|
||||
DEBUG_ERROR("sink: failed to create pool");
|
||||
return -1;
|
||||
}
|
||||
if (nostr_relay_pool_add_relay(sink->pool, local_relay_url) != NOSTR_SUCCESS) {
|
||||
DEBUG_ERROR("sink: failed to add relay %s", local_relay_url);
|
||||
nostr_relay_pool_destroy(sink->pool);
|
||||
sink->pool = NULL;
|
||||
return -1;
|
||||
}
|
||||
DEBUG_INFO("sink: initialized (WebSocket) for %s", local_relay_url);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cr_sink_init_pg(cr_sink_t *sink, cr_seen_ring_t *seen) {
|
||||
memset(sink, 0, sizeof(*sink));
|
||||
sink->seen = seen;
|
||||
sink->source_class = CR_SINK_CLASS_LIVE;
|
||||
sink->pool = NULL; /* PostgreSQL inbox mode - no WebSocket pool. */
|
||||
DEBUG_INFO("sink: initialized (PostgreSQL inbox)");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cr_sink_set_source_class(cr_sink_t *sink, const char *class_ptr) {
|
||||
if (sink) sink->source_class = class_ptr ? class_ptr : CR_SINK_CLASS_LIVE;
|
||||
}
|
||||
|
||||
void cr_sink_destroy(cr_sink_t *sink) {
|
||||
if (sink && sink->pool) {
|
||||
nostr_relay_pool_destroy(sink->pool);
|
||||
sink->pool = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Publish */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
int cr_sink_publish_from(cr_sink_t *sink, cJSON *event, const char *source_relay) {
|
||||
if (!sink || !event) return -1;
|
||||
|
||||
cJSON *id = cJSON_GetObjectItem(event, "id");
|
||||
if (!id || !cJSON_IsString(id)) {
|
||||
DEBUG_WARN("sink: event missing id, skipping");
|
||||
return -1;
|
||||
}
|
||||
const char *eid = cJSON_GetStringValue(id);
|
||||
|
||||
if (cr_seen_ring_add(sink->seen, eid) == 0) {
|
||||
sink->published_dup++;
|
||||
return 0; /* already seen recently */
|
||||
}
|
||||
|
||||
/* PostgreSQL inbox mode. */
|
||||
if (!sink->pool) {
|
||||
char *json = cJSON_PrintUnformatted(event);
|
||||
if (!json) {
|
||||
DEBUG_WARN("sink: failed to serialize event %s", eid);
|
||||
sink->published_failed++;
|
||||
return -1;
|
||||
}
|
||||
const char *klass = sink->source_class ? sink->source_class
|
||||
: CR_SINK_CLASS_LIVE;
|
||||
int priority = (strcmp(klass, CR_SINK_CLASS_BACKFILL) == 0) ? 1 : 0;
|
||||
|
||||
cJSON *kind = cJSON_GetObjectItem(event, "kind");
|
||||
cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
int kind_num = (kind && cJSON_IsNumber(kind)) ? (int)cJSON_GetNumberValue(kind) : -1;
|
||||
DEBUG_TRACE("sink: INBOX event id=%s kind=%d pubkey=%s class=%s",
|
||||
eid, kind_num,
|
||||
pubkey && cJSON_IsString(pubkey) ? cJSON_GetStringValue(pubkey) : "?",
|
||||
klass);
|
||||
|
||||
int rc = pg_inbox_insert_event(json, source_relay, klass, priority);
|
||||
free(json);
|
||||
if (rc != 0) {
|
||||
sink->published_failed++;
|
||||
return -1;
|
||||
}
|
||||
sink->published_ok++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* WebSocket mode. */
|
||||
const char *urls[1] = { sink->url };
|
||||
cJSON *kind = cJSON_GetObjectItem(event, "kind");
|
||||
cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
int kind_num = (kind && cJSON_IsNumber(kind)) ? (int)cJSON_GetNumberValue(kind) : -1;
|
||||
DEBUG_TRACE("sink: SEND event id=%s kind=%d pubkey=%s to %s",
|
||||
eid, kind_num,
|
||||
pubkey && cJSON_IsString(pubkey) ? cJSON_GetStringValue(pubkey) : "?",
|
||||
sink->url);
|
||||
|
||||
int rc = nostr_relay_pool_publish_async(sink->pool, urls, 1, event,
|
||||
sink_publish_cb, sink);
|
||||
if (rc < 0) {
|
||||
DEBUG_WARN("sink: publish_async error rc=%d for %s", rc, eid);
|
||||
return -1;
|
||||
}
|
||||
if (rc == 0) {
|
||||
DEBUG_INFO("sink: publish SKIPPED (not connected) for %s", eid);
|
||||
return 0;
|
||||
}
|
||||
DEBUG_TRACE("sink: publish_async rc=%d for %s", rc, eid);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int cr_sink_publish(cr_sink_t *sink, cJSON *event) {
|
||||
return cr_sink_publish_from(sink, event, NULL);
|
||||
}
|
||||
|
||||
void cr_sink_pump(cr_sink_t *sink, int timeout_ms) {
|
||||
if (sink && sink->pool) {
|
||||
nostr_relay_pool_run(sink->pool, timeout_ms);
|
||||
}
|
||||
/* No-op in PostgreSQL inbox mode. */
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* caching_relay - relay sink: publishes fetched events to the local relay.
|
||||
*
|
||||
* Two modes:
|
||||
* - WebSocket mode (legacy): a dedicated single-relay pool publishes via
|
||||
* nostr_relay_pool_publish_async(). Used when no PostgreSQL connection
|
||||
* string is provided.
|
||||
* - PostgreSQL inbox mode: events are inserted directly into the
|
||||
* caching_event_inbox table via pg_inbox_insert_event(). Used when the
|
||||
* daemon is started with --pg-conn.
|
||||
*
|
||||
* The public function signatures are identical in both modes so that callers
|
||||
* (live_subscriber, backfill, relay_discovery) do not need to change.
|
||||
*/
|
||||
#ifndef CACHING_RELAY_RELAY_SINK_H
|
||||
#define CACHING_RELAY_RELAY_SINK_H
|
||||
|
||||
#include "state.h"
|
||||
#include "../nostr_core_lib/nostr_core/nostr_core.h"
|
||||
|
||||
/* Source class labels for the inbox priority field. */
|
||||
#define CR_SINK_CLASS_LIVE "live"
|
||||
#define CR_SINK_CLASS_DISCOVERY "discovery"
|
||||
#define CR_SINK_CLASS_BACKFILL "backfill"
|
||||
|
||||
typedef struct {
|
||||
/* WebSocket mode only (NULL in PostgreSQL inbox mode). */
|
||||
nostr_relay_pool_t *pool;
|
||||
char url[256];
|
||||
|
||||
/* PostgreSQL inbox mode: source class used for priority assignment.
|
||||
* Defaults to "live". Set to "backfill" by the backfill module.
|
||||
* Ignored in WebSocket mode. */
|
||||
const char *source_class;
|
||||
|
||||
cr_seen_ring_t *seen; /* shared seen ring for dedup */
|
||||
long published_ok;
|
||||
long published_failed;
|
||||
long published_dup;
|
||||
} cr_sink_t;
|
||||
|
||||
/* Create the sink.
|
||||
*
|
||||
* In WebSocket mode (pg_mode = 0): creates a relay pool and adds
|
||||
* local_relay_url.
|
||||
* In PostgreSQL inbox mode (pg_mode != 0): no pool is created; the URL is
|
||||
* ignored. Events are inserted via pg_inbox_insert_event() (the pg_inbox
|
||||
* module must be initialized beforehand).
|
||||
*
|
||||
* The original two-argument signature is preserved via cr_sink_init() below
|
||||
* for backward compatibility; it defaults to WebSocket mode. */
|
||||
int cr_sink_init(cr_sink_t *sink, const char *local_relay_url, cr_seen_ring_t *seen);
|
||||
|
||||
/* Initialize the sink in PostgreSQL inbox mode. */
|
||||
int cr_sink_init_pg(cr_sink_t *sink, cr_seen_ring_t *seen);
|
||||
|
||||
/* Set the source class for priority assignment in PostgreSQL inbox mode.
|
||||
* class_ptr must point to a static string literal (e.g. CR_SINK_CLASS_LIVE). */
|
||||
void cr_sink_set_source_class(cr_sink_t *sink, const char *class_ptr);
|
||||
|
||||
void cr_sink_destroy(cr_sink_t *sink);
|
||||
|
||||
/* Publish a cJSON event. Dedups against the seen ring.
|
||||
* Returns 1 if enqueued/inserted, 0 if dup (skipped), -1 on error. */
|
||||
int cr_sink_publish(cr_sink_t *sink, cJSON *event);
|
||||
|
||||
/* Variant that records the originating relay URL in PostgreSQL inbox mode.
|
||||
* In WebSocket mode the source_relay is ignored. */
|
||||
int cr_sink_publish_from(cr_sink_t *sink, cJSON *event, const char *source_relay);
|
||||
|
||||
/* Pump the sink pool briefly to flush pending publish callbacks.
|
||||
* No-op in PostgreSQL inbox mode. */
|
||||
void cr_sink_pump(cr_sink_t *sink, int timeout_ms);
|
||||
|
||||
#endif /* CACHING_RELAY_RELAY_SINK_H */
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* caching_relay - in-memory runtime state
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include "state.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ---- pubkey set ---- */
|
||||
|
||||
void cr_pubkey_set_init(cr_pubkey_set_t *s) {
|
||||
s->items = NULL;
|
||||
s->count = 0;
|
||||
s->capacity = 0;
|
||||
}
|
||||
|
||||
void cr_pubkey_set_free(cr_pubkey_set_t *s) {
|
||||
for (int i = 0; i < s->count; i++) free(s->items[i]);
|
||||
free(s->items);
|
||||
s->items = NULL;
|
||||
s->count = 0;
|
||||
s->capacity = 0;
|
||||
}
|
||||
|
||||
void cr_pubkey_set_clear(cr_pubkey_set_t *s) {
|
||||
cr_pubkey_set_free(s);
|
||||
cr_pubkey_set_init(s);
|
||||
}
|
||||
|
||||
int cr_pubkey_set_contains(const cr_pubkey_set_t *s, const char *hex) {
|
||||
for (int i = 0; i < s->count; i++) {
|
||||
if (strcmp(s->items[i], hex) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cr_pubkey_set_add(cr_pubkey_set_t *s, const char *hex) {
|
||||
if (!hex || !hex[0]) return 0;
|
||||
if (cr_pubkey_set_contains(s, hex)) return 0;
|
||||
if (s->count >= s->capacity) {
|
||||
int newcap = s->capacity ? s->capacity * 2 : 64;
|
||||
char **ni = realloc(s->items, newcap * sizeof(char *));
|
||||
if (!ni) return -1;
|
||||
s->items = ni;
|
||||
s->capacity = newcap;
|
||||
}
|
||||
s->items[s->count] = strdup(hex);
|
||||
if (!s->items[s->count]) return -1;
|
||||
s->count++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ---- seen-event ring buffer ---- */
|
||||
|
||||
void cr_seen_ring_init(cr_seen_ring_t *r) {
|
||||
memset(r, 0, sizeof(*r));
|
||||
}
|
||||
|
||||
int cr_seen_ring_contains(const cr_seen_ring_t *r, const char *id) {
|
||||
for (int i = 0; i < r->count; i++) {
|
||||
int idx = (r->head - 1 - i + CR_SEEN_RING_SIZE) % CR_SEEN_RING_SIZE;
|
||||
if (strcmp(r->ids[idx], id) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cr_seen_ring_add(cr_seen_ring_t *r, const char *id) {
|
||||
if (!id || !id[0]) return 0;
|
||||
if (cr_seen_ring_contains(r, id)) return 0;
|
||||
strncpy(r->ids[r->head], id, CR_HEX_LEN - 1);
|
||||
r->ids[r->head][CR_HEX_LEN - 1] = '\0';
|
||||
r->head = (r->head + 1) % CR_SEEN_RING_SIZE;
|
||||
if (r->count < CR_SEEN_RING_SIZE) r->count++;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* caching_relay - in-memory runtime state:
|
||||
* - followed-pubkey set (hex pubkeys)
|
||||
* - seen-event-id ring buffer (to avoid redundant publish attempts)
|
||||
*/
|
||||
#ifndef CACHING_RELAY_STATE_H
|
||||
#define CACHING_RELAY_STATE_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Hex pubkey is 64 chars + NUL. Event id is 64 hex chars + NUL. */
|
||||
#define CR_HEX_LEN 65
|
||||
|
||||
/* A simple dynamic string set for followed pubkeys. */
|
||||
typedef struct {
|
||||
char **items;
|
||||
int count;
|
||||
int capacity;
|
||||
} cr_pubkey_set_t;
|
||||
|
||||
void cr_pubkey_set_init(cr_pubkey_set_t *s);
|
||||
void cr_pubkey_set_free(cr_pubkey_set_t *s);
|
||||
int cr_pubkey_set_add(cr_pubkey_set_t *s, const char *hex); /* 1 if added, 0 if dup */
|
||||
int cr_pubkey_set_contains(const cr_pubkey_set_t *s, const char *hex);
|
||||
void cr_pubkey_set_clear(cr_pubkey_set_t *s);
|
||||
|
||||
/* Fixed-size ring buffer of event ids for dedup. */
|
||||
#define CR_SEEN_RING_SIZE 4096
|
||||
|
||||
typedef struct {
|
||||
char ids[CR_SEEN_RING_SIZE][CR_HEX_LEN];
|
||||
int head;
|
||||
int count;
|
||||
} cr_seen_ring_t;
|
||||
|
||||
void cr_seen_ring_init(cr_seen_ring_t *r);
|
||||
/* Returns 1 if the id was newly inserted, 0 if it was already present. */
|
||||
int cr_seen_ring_add(cr_seen_ring_t *r, const char *id);
|
||||
int cr_seen_ring_contains(const cr_seen_ring_t *r, const char *id);
|
||||
|
||||
#endif /* CACHING_RELAY_STATE_H */
|
||||
-9970
File diff suppressed because it is too large
Load Diff
+105
@@ -8,8 +8,10 @@ set -euo pipefail
|
||||
# Configuration
|
||||
REMOTE_HOST="ubuntu@laantungir.net"
|
||||
LOCAL_BINARY="build/c_relay_pg_static_x86_64"
|
||||
LOCAL_CACHING_BINARY="build/caching_relay"
|
||||
REMOTE_BINARY_DIR="/usr/local/bin/c_relay_pg"
|
||||
REMOTE_BINARY_PATH="/usr/local/bin/c_relay_pg/c_relay_pg"
|
||||
REMOTE_CACHING_BINARY_PATH="/opt/c-relay-pg/caching_relay"
|
||||
SERVICE_NAME="c-relay-pg"
|
||||
|
||||
LOCAL_SERVICE_FILE="systemd/c-relay.service"
|
||||
@@ -35,6 +37,25 @@ echo "==> Uploading artifacts to $REMOTE_HOST"
|
||||
scp "$LOCAL_BINARY" "$REMOTE_HOST:/tmp/c_relay_pg.tmp"
|
||||
scp "$LOCAL_SERVICE_FILE" "$REMOTE_HOST:/tmp/c-relay-pg.service"
|
||||
scp "$LOCAL_PG_SETUP_SCRIPT" "$REMOTE_HOST:/tmp/setup_postgres_18.sh"
|
||||
# Upload the caching_relay binary if it was built locally. The caching
|
||||
# service is NOT started by this deploy (the systemd unit does not pass
|
||||
# --start-caching), but the binary is placed in the relay's WorkingDirectory
|
||||
# so it is ready for when caching is enabled via event-based config.
|
||||
if [ -f "$LOCAL_CACHING_BINARY" ]; then
|
||||
scp "$LOCAL_CACHING_BINARY" "$REMOTE_HOST:/tmp/caching_relay.tmp"
|
||||
else
|
||||
echo "WARNING: caching_relay binary not found locally ($LOCAL_CACHING_BINARY) — skipping caching binary upload"
|
||||
fi
|
||||
|
||||
# Upload admin PHP files (tar to preserve directory structure, exclude
|
||||
# cache logs and the dev-only serve.sh / php_server.log).
|
||||
echo "==> Packaging admin files"
|
||||
tar czf /tmp/admin_deploy.tar.gz \
|
||||
--exclude='admin/serve.sh' \
|
||||
--exclude='admin/php_server.log' \
|
||||
--exclude='admin/cache' \
|
||||
-C . admin/
|
||||
scp /tmp/admin_deploy.tar.gz "$REMOTE_HOST:/tmp/admin_deploy.tar.gz"
|
||||
|
||||
echo "==> Running remote install/configuration"
|
||||
ssh "$REMOTE_HOST" 'bash -s' <<'EOF'
|
||||
@@ -44,6 +65,7 @@ SERVICE_NAME="c-relay-pg"
|
||||
RELAY_USER="c-relay-pg"
|
||||
REMOTE_BINARY_DIR="/usr/local/bin/c_relay_pg"
|
||||
REMOTE_BINARY_PATH="/usr/local/bin/c_relay_pg/c_relay_pg"
|
||||
REMOTE_CACHING_BINARY_PATH="/opt/c-relay-pg/caching_relay"
|
||||
|
||||
echo "[remote] Ensuring service user exists"
|
||||
if ! id "$RELAY_USER" >/dev/null 2>&1; then
|
||||
@@ -80,11 +102,76 @@ sudo mv /tmp/c_relay_pg.tmp "$REMOTE_BINARY_PATH"
|
||||
sudo chown "$RELAY_USER:$RELAY_USER" "$REMOTE_BINARY_PATH"
|
||||
sudo chmod +x "$REMOTE_BINARY_PATH"
|
||||
|
||||
echo "[remote] Installing caching_relay binary (if uploaded)"
|
||||
if [ -f /tmp/caching_relay.tmp ]; then
|
||||
sudo mv /tmp/caching_relay.tmp "$REMOTE_CACHING_BINARY_PATH"
|
||||
sudo chown "$RELAY_USER:$RELAY_USER" "$REMOTE_CACHING_BINARY_PATH"
|
||||
sudo chmod +x "$REMOTE_CACHING_BINARY_PATH"
|
||||
echo " -> caching_relay installed at $REMOTE_CACHING_BINARY_PATH (not started; enable via config when ready)"
|
||||
else
|
||||
echo " -> no caching_relay binary uploaded, skipping"
|
||||
fi
|
||||
|
||||
echo "[remote] Installing admin PHP files"
|
||||
# Back up existing config.php and .htpasswd so we don't lose production
|
||||
# credentials (DB password, basic-auth) across deploys.
|
||||
if [ -f /opt/c-relay-pg/admin/lib/config.php ]; then
|
||||
sudo cp /opt/c-relay-pg/admin/lib/config.php /tmp/admin_config_backup
|
||||
echo " -> backed up config.php"
|
||||
fi
|
||||
if [ -f /opt/c-relay-pg/admin/.htpasswd ]; then
|
||||
sudo cp /opt/c-relay-pg/admin/.htpasswd /tmp/admin_htpasswd_backup
|
||||
echo " -> backed up .htpasswd"
|
||||
fi
|
||||
# Extract the new admin files (overwrites old admin at /opt/c-relay-pg/admin/).
|
||||
sudo mkdir -p /opt/c-relay-pg/admin
|
||||
sudo tar xzf /tmp/admin_deploy.tar.gz -C /opt/c-relay-pg/
|
||||
# Restore config.php and .htpasswd if they were backed up.
|
||||
if [ -f /tmp/admin_config_backup ]; then
|
||||
sudo cp /tmp/admin_config_backup /opt/c-relay-pg/admin/lib/config.php
|
||||
echo " -> restored config.php"
|
||||
fi
|
||||
if [ -f /tmp/admin_htpasswd_backup ]; then
|
||||
sudo cp /tmp/admin_htpasswd_backup /opt/c-relay-pg/admin/.htpasswd
|
||||
echo " -> restored .htpasswd"
|
||||
fi
|
||||
# Set ownership: www-data (nginx/php-fpm) for web-served files,
|
||||
# but keep .htpasswd readable by nginx.
|
||||
sudo chown -R www-data:www-data /opt/c-relay-pg/admin/
|
||||
sudo chmod 640 /opt/c-relay-pg/admin/.htpasswd 2>/dev/null || true
|
||||
# Ensure the cache directory exists (for chart text files).
|
||||
sudo mkdir -p /opt/c-relay-pg/admin/cache
|
||||
sudo chown www-data:www-data /opt/c-relay-pg/admin/cache
|
||||
echo " -> admin files installed at /opt/c-relay-pg/admin/"
|
||||
|
||||
# Also update admin2 if it exists (symlink or copy to keep it in sync).
|
||||
if [ -d /opt/c-relay-pg/admin2 ]; then
|
||||
sudo rm -rf /opt/c-relay-pg/admin2
|
||||
sudo cp -a /opt/c-relay-pg/admin /opt/c-relay-pg/admin2
|
||||
sudo chown -R www-data:www-data /opt/c-relay-pg/admin2/
|
||||
echo " -> admin2 synced at /opt/c-relay-pg/admin2/"
|
||||
fi
|
||||
|
||||
echo "[remote] Installing systemd unit"
|
||||
sudo mv /tmp/c-relay-pg.service /etc/systemd/system/c-relay-pg.service
|
||||
sudo chown root:root /etc/systemd/system/c-relay-pg.service
|
||||
sudo chmod 644 /etc/systemd/system/c-relay-pg.service
|
||||
|
||||
echo "[remote] Killing stale caching_relay processes (safety)"
|
||||
# The caching service is forked by the relay with setsid(), so it detaches
|
||||
# from the relay's process group and survives when the relay is killed.
|
||||
# Without this, every restart orphans the previous caching_relay child and
|
||||
# a new relay forks another, leading to many stale processes with dead PG
|
||||
# connections spamming errors. Kill them here so the new relay starts clean.
|
||||
CACHING_PIDS=$(pgrep -f "caching_relay" || echo "")
|
||||
if [ -n "$CACHING_PIDS" ]; then
|
||||
echo " -> killing stale caching_relay PIDs: $CACHING_PIDS"
|
||||
kill -9 $CACHING_PIDS 2>/dev/null || true
|
||||
sleep 1
|
||||
else
|
||||
echo " -> no stale caching_relay processes found"
|
||||
fi
|
||||
|
||||
echo "[remote] Reloading and restarting service"
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable "$SERVICE_NAME"
|
||||
@@ -93,6 +180,24 @@ sudo systemctl restart "$SERVICE_NAME"
|
||||
echo "[remote] Health checks"
|
||||
sudo systemctl --no-pager --full status "$SERVICE_NAME" | sed -n '1,25p'
|
||||
sudo -u "$RELAY_USER" psql -d crelay -c "SELECT current_user, current_database();"
|
||||
|
||||
# Verify admin page is accessible (HTTP 200 or 401 = auth required, both are OK).
|
||||
ADMIN_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1/admin/" 2>/dev/null || echo "000")
|
||||
if [ "$ADMIN_CODE" = "200" ] || [ "$ADMIN_CODE" = "401" ]; then
|
||||
echo " -> admin page accessible (HTTP $ADMIN_CODE)"
|
||||
else
|
||||
echo " -> WARNING: admin page returned HTTP $ADMIN_CODE (expected 200 or 401)"
|
||||
fi
|
||||
|
||||
# Clean up temp files.
|
||||
rm -f /tmp/admin_deploy.tar.gz
|
||||
sudo rm -f /tmp/admin_htpasswd_backup /tmp/admin_config_backup
|
||||
EOF
|
||||
|
||||
# Clean up local temp tarball.
|
||||
rm -f /tmp/admin_deploy.tar.gz
|
||||
|
||||
echo "Deployment complete: $REMOTE_HOST"
|
||||
echo ""
|
||||
echo "Admin interface: https://laantungir.net/admin/"
|
||||
echo "Caching is NOT auto-started. Enable it via the admin UI when ready."
|
||||
|
||||
@@ -155,6 +155,34 @@ http {
|
||||
log_not_found off;
|
||||
}
|
||||
|
||||
# PHP admin page for caching service (direct PostgreSQL access).
|
||||
# Bypasses the NIP-44 64KB encryption limit of the Nostr admin API.
|
||||
# Requires: php-fpm + php-pgsql installed and configured.
|
||||
location /admin/ {
|
||||
alias /opt/c-relay-pg/admin/;
|
||||
index index.php;
|
||||
|
||||
# HTTP Basic Auth
|
||||
auth_basic "Relay Admin";
|
||||
auth_basic_user_file /opt/c-relay-pg/admin/.htpasswd;
|
||||
|
||||
# Pass .php files to PHP-FPM
|
||||
location ~ \.php$ {
|
||||
# Adjust socket path for your distro:
|
||||
# Debian/Ubuntu: unix:/run/php/php8.2-fpm.sock
|
||||
# RHEL/Fedora: unix:/run/php-fpm/www.sock
|
||||
fastcgi_pass unix:/run/php/php8.2-fpm.sock;
|
||||
fastcgi_index index.php;
|
||||
include fastcgi_params;
|
||||
fastcgi_param SCRIPT_FILENAME $request_filename;
|
||||
}
|
||||
|
||||
# Deny access to the lib/ directory (contains DB credentials)
|
||||
location ^~ /admin/lib/ {
|
||||
deny all;
|
||||
}
|
||||
}
|
||||
|
||||
# Optional: Metrics endpoint (if implemented)
|
||||
location /metrics {
|
||||
proxy_pass http://c_relay_pg_backend/metrics;
|
||||
|
||||
@@ -13,6 +13,8 @@ ADMIN_KEY=""
|
||||
RELAY_KEY=""
|
||||
PORT_OVERRIDE=""
|
||||
DEBUG_LEVEL="5"
|
||||
START_CACHING=false
|
||||
RESET_BACKFILL=false
|
||||
DB_BACKEND="postgres"
|
||||
DB_CONNSTRING=""
|
||||
DB_HOST=""
|
||||
@@ -75,6 +77,14 @@ while [[ $# -gt 0 ]]; do
|
||||
PRESERVE_DATABASE=true
|
||||
shift
|
||||
;;
|
||||
--start-caching)
|
||||
START_CACHING=true
|
||||
shift
|
||||
;;
|
||||
--reset-backfill)
|
||||
RESET_BACKFILL=true
|
||||
shift
|
||||
;;
|
||||
--test-keys|-t)
|
||||
USE_TEST_KEYS=true
|
||||
# Read keys from .test_keys file
|
||||
@@ -359,6 +369,8 @@ if [ "$HELP" = true ]; then
|
||||
echo " --db-name <name> PostgreSQL database name"
|
||||
echo " --db-user <user> PostgreSQL database user"
|
||||
echo " --db-password <pass> PostgreSQL database password"
|
||||
echo " --start-caching Auto-start the caching service on startup"
|
||||
echo " --reset-backfill Clear caching backfill progress (use with --start-caching)"
|
||||
echo " --help, -h Show this help message"
|
||||
echo ""
|
||||
echo "Event-Based Configuration:"
|
||||
@@ -455,6 +467,19 @@ fi
|
||||
|
||||
echo "Using static binary: $BINARY_PATH"
|
||||
|
||||
# Build the caching_relay binary (from caching/ directory)
|
||||
# This is needed because the caching service launcher forks this binary.
|
||||
if [ -d "caching" ] && [ -f "caching/Makefile" ]; then
|
||||
echo "Building caching_relay binary..."
|
||||
(cd caching && make) > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "WARNING: caching_relay build failed. The caching service will not be available."
|
||||
echo "You can build it manually with: cd caching && make"
|
||||
else
|
||||
echo "✓ caching_relay binary built successfully"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Build successful. Proceeding with relay restart..."
|
||||
|
||||
# Kill existing relay if running - start aggressive immediately
|
||||
@@ -469,6 +494,21 @@ else
|
||||
echo "No existing relay processes found"
|
||||
fi
|
||||
|
||||
# Kill any stale caching_relay processes from previous runs.
|
||||
# The caching service is forked by the relay with setsid(), so it detaches
|
||||
# from the relay's process group and survives when the relay is killed -9.
|
||||
# Without this, every restart orphans the previous caching_relay child and
|
||||
# a new relay forks another, leading to many stale processes with dead PG
|
||||
# connections spamming errors. Kill them here so the new relay starts fresh.
|
||||
CACHING_PIDS=$(pgrep -f "caching_relay" || echo "")
|
||||
if [ -n "$CACHING_PIDS" ]; then
|
||||
echo "Killing stale caching_relay processes: $CACHING_PIDS"
|
||||
kill -9 $CACHING_PIDS 2>/dev/null || true
|
||||
sleep 1
|
||||
else
|
||||
echo "No existing caching_relay processes found"
|
||||
fi
|
||||
|
||||
# Ensure port 8888 is completely free with retry loop
|
||||
echo "Ensuring port 8888 is available..."
|
||||
for attempt in {1..15}; do
|
||||
@@ -499,14 +539,21 @@ for attempt in {1..15}; do
|
||||
fi
|
||||
done
|
||||
|
||||
# Final safety check - ensure no relay processes remain
|
||||
# Final safety check - ensure no relay or caching processes remain
|
||||
FINAL_PIDS=$(pgrep -f "c_relay_pg_" || echo "")
|
||||
if [ -n "$FINAL_PIDS" ]; then
|
||||
echo "Final cleanup: killing processes $FINAL_PIDS"
|
||||
echo "Final cleanup: killing relay processes $FINAL_PIDS"
|
||||
kill -9 $FINAL_PIDS 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
FINAL_CACHING_PIDS=$(pgrep -f "caching_relay" || echo "")
|
||||
if [ -n "$FINAL_CACHING_PIDS" ]; then
|
||||
echo "Final cleanup: killing stale caching_relay processes $FINAL_CACHING_PIDS"
|
||||
kill -9 $FINAL_CACHING_PIDS 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
# Clean up PID file
|
||||
rm -f relay.pid
|
||||
|
||||
@@ -566,6 +613,16 @@ if [ -n "$DB_PASSWORD" ]; then
|
||||
RELAY_ARGS="$RELAY_ARGS --db-password '$DB_PASSWORD'"
|
||||
fi
|
||||
|
||||
if [ "$START_CACHING" = true ]; then
|
||||
RELAY_ARGS="$RELAY_ARGS --start-caching"
|
||||
echo "Auto-starting caching service on startup"
|
||||
fi
|
||||
|
||||
if [ "$RESET_BACKFILL" = true ]; then
|
||||
RELAY_ARGS="$RELAY_ARGS --reset-backfill"
|
||||
echo "Resetting caching backfill progress before start"
|
||||
fi
|
||||
|
||||
# Change to build directory before starting relay so database files are created there
|
||||
cd build
|
||||
# Start relay in background and capture its PID
|
||||
@@ -647,8 +704,10 @@ if ps -p "$RELAY_PID" >/dev/null 2>&1; then
|
||||
echo "Configuration: Event-based (kind 33334 Nostr events)"
|
||||
echo "Database: Automatically created with relay pubkey naming"
|
||||
echo "To kill relay: pkill -f 'c_relay_pg_'"
|
||||
echo "To kill caching service: pkill -f 'caching_relay'"
|
||||
echo "To check status: ps aux | grep c_relay_pg_"
|
||||
echo "To view logs: tail -f relay.log"
|
||||
echo "To view caching logs: tail -f build/caching_relay.log"
|
||||
echo "Binary: $BINARY_PATH (zero configuration needed)"
|
||||
echo "Ready for Nostr client connections!"
|
||||
else
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
curl -H "Accept: application/nostr+json" http://localhost:8888/
|
||||
curl -H "Accept: application/nostr+json" https://relay.laantungir.net
|
||||
|
||||
Submodule nostr_core_lib deleted from 98cfd81b01
@@ -0,0 +1,206 @@
|
||||
# admin2: Full PHP Admin for C-Relay-PG
|
||||
|
||||
## Goal
|
||||
|
||||
Recreate the entire c-relay-pg admin interface as a PHP application in a new
|
||||
`admin2/` directory, starting from the existing `api/` HTML/CSS/JS files and
|
||||
replacing the Nostr admin command backend with direct PostgreSQL queries via
|
||||
PHP PDO. This covers ALL admin sections, not just caching.
|
||||
|
||||
## Architecture
|
||||
|
||||
Same as `admin/` (PHP + nginx + PDO + 10s polling), but with the full
|
||||
section set from the original `api/index.html`:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Browser -->|HTTPS /admin2/| Nginx
|
||||
Nginx -->|*.php| PHP_FPM
|
||||
Nginx -->|/relay/| Relay[C-Relay-PG]
|
||||
PHP_FPM -->|PDO| PostgreSQL
|
||||
Relay -->|libpq| PostgreSQL
|
||||
```
|
||||
|
||||
## Sections (9 pages, matching original nav)
|
||||
|
||||
| # | Nav Label | Section ID | Data Source | PHP API Endpoint |
|
||||
|---|-----------|------------|-------------|------------------|
|
||||
| 1 | Statistics | databaseStatisticsSection | `pg_database_size()`, `events` count, `pg_stat_activity`, process stats | `api/stats.php` |
|
||||
| 2 | Subscriptions | subscriptionDetailsSection | `subscriptions` + `subscription_metrics` tables | `api/subscriptions.php` |
|
||||
| 3 | Configuration | div_config | `config` table (read/edit all keys) | `api/config.php` |
|
||||
| 4 | Authorization | authRulesSection + wotSection | `auth_rules` table | `api/auth.php` |
|
||||
| 5 | IP BAN | ipBansSection | `ip_bans` table | `api/ipbans.php` |
|
||||
| 6 | Relay Events | relayEventsSection | `events` table (recent events, kind filter) | `api/events.php` |
|
||||
| 7 | Caching | cachingSection | `caching_*` tables (already built in `admin/`) | `api/caching.php` |
|
||||
| 8 | DM | nip17DMSection | `events` table (kind 4/14/15 DMs) | `api/dm.php` |
|
||||
| 9 | Database Query | sqlQuerySection | Direct SQL execution (admin only) | `api/query.php` |
|
||||
|
||||
## Graph on Statistics Page
|
||||
|
||||
The original has an event-rate chart. For the PHP version:
|
||||
- **X-axis:** one tick = 10 seconds (the refresh interval)
|
||||
- **Y-axis:** events per interval
|
||||
- **Data source:** `api/stats.php` returns `events_delta` (count of new events since last poll)
|
||||
- **JS:** maintains a rolling array of the last 60 data points (10 minutes of history), appends a new point on each 10s refresh, renders as a simple canvas/SVG line chart
|
||||
- **No external chart library** — use a lightweight inline canvas renderer to keep it dependency-free
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
admin2/
|
||||
├── index.php ← Single-page app (all 9 sections in one page, show/hide via JS)
|
||||
├── assets/
|
||||
│ ├── index.css ← Copied from api/index.css (unchanged)
|
||||
│ └── app.js ← Adapted from api/index.js (replace Nostr commands with fetch() to PHP APIs)
|
||||
├── lib/
|
||||
│ ├── config.php ← DB connection config (same as admin/)
|
||||
│ ├── db.php ← PDO helper (same as admin/)
|
||||
│ └── helpers.php ← Shared helpers (same as admin/)
|
||||
├── api/
|
||||
│ ├── stats.php ← Statistics: DB size, event counts, process info, event-rate delta
|
||||
│ ├── subscriptions.php ← Subscription details + metrics
|
||||
│ ├── config.php ← Config table read/edit (all keys, not just caching)
|
||||
│ ├── auth.php ← Auth rules + WoT status
|
||||
│ ├── ipbans.php ← IP ban list (read/add/remove)
|
||||
│ ├── events.php ← Recent relay events (paginated, kind filter)
|
||||
│ ├── caching.php ← Caching status (reuses admin/api/status.php logic)
|
||||
│ ├── dm.php ← DM list (kind 4/14/15 events)
|
||||
│ └── query.php ← Direct SQL query execution (admin only, read-only by default)
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Step 1 — Scaffold admin2/ directory
|
||||
- Create `admin2/` with `lib/` (copy from `admin/`), `assets/`, `api/`
|
||||
- Copy `api/index.css` → `admin2/assets/index.css` (unchanged)
|
||||
- Copy `api/index.html` → `admin2/index.php` (will be adapted)
|
||||
|
||||
### Step 2 — Adapt index.php from api/index.html
|
||||
- Remove Nostr login/auth (NIP-07 extension, kind-23456 admin commands)
|
||||
- Remove WebSocket connection logic (replaced by PHP AJAX polling)
|
||||
- Keep all 9 section HTML structures exactly as-is
|
||||
- Keep the side-nav, header, section show/hide JS
|
||||
- Replace `sendAdminCommand()` calls with `fetch('api/*.php')` calls
|
||||
- Add 10s auto-refresh via `setInterval` for statistics page
|
||||
- Add event-rate chart canvas renderer
|
||||
|
||||
### Step 3 — Adapt app.js from api/index.js
|
||||
- Keep: `switchPage()`, nav handling, section show/hide, config table rendering
|
||||
- Remove: WebSocket connection, NIP-07 login, NIP-44 encryption, kind-23456 event sending
|
||||
- Replace: each `sendAdminCommand(['system_command', '...'])` with `fetch('api/....php')`
|
||||
- Add: `refreshStats()` function that polls `api/stats.php` every 10s
|
||||
- Add: event-rate chart rendering on canvas (rolling 60-point window)
|
||||
|
||||
### Step 4 — Build PHP API endpoints (9 files)
|
||||
Each endpoint returns JSON, queried directly from PostgreSQL via PDO:
|
||||
|
||||
**api/stats.php** — Statistics page data:
|
||||
```sql
|
||||
SELECT pg_database_size(current_database()) AS db_size;
|
||||
SELECT COUNT(*) FROM events;
|
||||
SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'active';
|
||||
-- Event rate: count events with first_seen in last 10s
|
||||
SELECT COUNT(*) FROM events WHERE first_seen >= EXTRACT(EPOCH FROM NOW())::BIGINT - 10;
|
||||
-- Kind distribution
|
||||
SELECT kind, COUNT(*) FROM events GROUP BY kind ORDER BY count DESC;
|
||||
```
|
||||
|
||||
**api/config.php** — Configuration (all config keys, not just caching):
|
||||
```sql
|
||||
SELECT key, value FROM config ORDER BY key;
|
||||
-- POST: UPDATE config SET value = ? WHERE key = ?
|
||||
```
|
||||
|
||||
**api/subscriptions.php** — Subscription details:
|
||||
```sql
|
||||
SELECT * FROM subscriptions ORDER BY created_at DESC LIMIT 100;
|
||||
SELECT * FROM subscription_metrics ORDER BY id DESC LIMIT 100;
|
||||
```
|
||||
|
||||
**api/auth.php** — Auth rules:
|
||||
```sql
|
||||
SELECT * FROM auth_rules ORDER BY id;
|
||||
```
|
||||
|
||||
**api/ipbans.php** — IP bans:
|
||||
```sql
|
||||
SELECT * FROM ip_bans ORDER BY banned_at DESC;
|
||||
-- POST: INSERT/DELETE
|
||||
```
|
||||
|
||||
**api/events.php** — Recent relay events (paginated):
|
||||
```sql
|
||||
SELECT id, pubkey, kind, created_at, content, tags
|
||||
FROM events ORDER BY created_at DESC LIMIT 50 OFFSET ?;
|
||||
-- Filter by kind, pubkey
|
||||
```
|
||||
|
||||
**api/caching.php** — Caching status (reuse from admin/):
|
||||
```sql
|
||||
SELECT * FROM caching_service_state;
|
||||
SELECT * FROM caching_followed_pubkeys LIMIT 50 OFFSET ?;
|
||||
SELECT * FROM caching_backfill_relay_progress LIMIT 50 OFFSET ?;
|
||||
```
|
||||
|
||||
**api/dm.php** — Direct messages:
|
||||
```sql
|
||||
SELECT id, pubkey, kind, created_at, content
|
||||
FROM events WHERE kind IN (4, 14, 15) ORDER BY created_at DESC LIMIT 50;
|
||||
```
|
||||
|
||||
**api/query.php** — SQL query (admin only):
|
||||
```sql
|
||||
-- Execute arbitrary SELECT, return JSON
|
||||
-- Read-only by default; write mode behind a flag
|
||||
```
|
||||
|
||||
### Step 5 — Event-rate chart
|
||||
- Canvas element in the statistics section
|
||||
- JS maintains `eventRateHistory = []` (max 60 points)
|
||||
- On each 10s refresh, push new event count, shift if > 60
|
||||
- Render as line chart on canvas (no external library)
|
||||
- X-axis labels: time (mm:ss), one tick per 10s
|
||||
- Y-axis labels: event count per interval
|
||||
|
||||
### Step 6 — nginx config
|
||||
Add `/admin2/` location block (same pattern as `/admin/`):
|
||||
```nginx
|
||||
location ^~ /admin2/ {
|
||||
alias /opt/c-relay-pg/admin2/;
|
||||
index index.php;
|
||||
auth_basic "Relay Admin";
|
||||
auth_basic_user_file /opt/c-relay-pg/admin/.htpasswd;
|
||||
location ~ \.php$ {
|
||||
fastcgi_pass unix:/run/php/php8.3-fpm.sock;
|
||||
include fastcgi_params;
|
||||
fastcgi_param SCRIPT_FILENAME $request_filename;
|
||||
}
|
||||
location ^~ /admin2/lib/ { deny all; }
|
||||
}
|
||||
```
|
||||
|
||||
### Step 7 — Deploy and test
|
||||
- Copy `admin2/` to server
|
||||
- Add nginx location block, reload
|
||||
- Test all 9 sections
|
||||
- Verify 10s auto-refresh on statistics page
|
||||
- Verify event-rate chart updates
|
||||
|
||||
## Key Differences from Original api/
|
||||
|
||||
| Aspect | Original (api/) | New (admin2/) |
|
||||
|--------|-----------------|---------------|
|
||||
| Auth | NIP-07 Nostr extension login | HTTP Basic Auth (nginx) |
|
||||
| Data fetch | WebSocket + kind-23456 encrypted admin commands | PHP AJAX `fetch()` to `api/*.php` |
|
||||
| Encryption | NIP-44 (64KB limit) | None (direct DB, no limit) |
|
||||
| Real-time | WebSocket push | 10s polling |
|
||||
| Backend | C relay binary (embedded JS) | PHP-FPM + PostgreSQL PDO |
|
||||
| Sections | 9 (same) | 9 (same HTML, different data source) |
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- NIP-07 Nostr login (use HTTP Basic Auth)
|
||||
- WebSocket push (use 10s polling)
|
||||
- Write operations that require relay-level logic (event publishing, NIP-42 auth) — those stay on the Nostr admin API
|
||||
- The original `api/` page remains unchanged and functional
|
||||
@@ -0,0 +1,178 @@
|
||||
# API Upgrade Implementation Plan — Phases 1, 2, and 5
|
||||
|
||||
Goal: finish the remaining work from `plans/api_upgrade_plan.md`:
|
||||
- **Phase 1 (design fix):** convert the `api-worker` thread from job-queue-driven to **timer + subscriber-check** driven.
|
||||
- **Phase 2:** remove the per-event and per-subscription monitoring triggers so monitoring no longer runs on the event-processing path.
|
||||
- **Phase 5:** add **PostgreSQL `LISTEN`/`NOTIFY`** so the worker wakes on real data changes instead of pure polling.
|
||||
|
||||
Target test relay: **port 7777** (local).
|
||||
|
||||
---
|
||||
|
||||
## Current State (verified in code)
|
||||
|
||||
- `api_worker_main()` ([`src/api.c:172`](src/api.c:172)) blocks on `api_worker_pop_job_blocking()` and reacts to three job types: `API_WORK_JOB_EVENT_STORED`, `API_WORK_JOB_SUBSCRIPTION_CHANGE`, `API_WORK_JOB_STATUS_POST`.
|
||||
- Monitoring hooks `monitoring_on_event_stored()` ([`src/api.c:752`](src/api.c:752)) and `monitoring_on_subscription_change()` ([`src/api.c:759`](src/api.c:759)) are called from [`src/main.c:1169`](src/main.c:1169) and [`src/subscriptions.c:483`](src/subscriptions.c:483), [`src/subscriptions.c:536`](src/subscriptions.c:536). They enqueue jobs; sync fallback runs if the worker is down.
|
||||
- Throttling lives in `monitoring_on_event_stored_sync()` / `monitoring_on_subscription_change_sync()` ([`src/api.c:711`](src/api.c:711), [`src/api.c:731`](src/api.c:731)).
|
||||
- Round-robin selection in `generate_round_robin_monitoring_event()` ([`src/api.c:571`](src/api.c:571)); per-type build/sign/broadcast in `generate_monitoring_event_for_type()` ([`src/api.c:621`](src/api.c:621)).
|
||||
- `has_subscriptions_for_kind(int)` ([`src/subscriptions.c:980`](src/subscriptions.c:980)) checks only subscriptions with an explicit `kinds` filter — it does **not** consider the `no_kind_filter_subs` list, so it alone is insufficient for a "is anyone listening to kind 24567" check.
|
||||
- Worker DB connection is a `PGconn*` opened via `db_open_worker_connection()` ([`src/db_ops_postgres.c:226`](src/db_ops_postgres.c:226)). Schema is embedded in [`src/pg_schema.h`](src/pg_schema.h) (version `"4"`) and mirrored in [`src/pg_schema.sql`](src/pg_schema.sql).
|
||||
- No `LISTEN`/`NOTIFY`/`pg_notify`/`PQnotifies` usage exists anywhere in `src/`.
|
||||
|
||||
---
|
||||
|
||||
## Design
|
||||
|
||||
### Phase 1 + 2 + 5 combined worker loop
|
||||
|
||||
The three phases are coupled, so they are implemented together in one rewrite of `api_worker_main()` and its helpers.
|
||||
|
||||
### CRITICAL BUG FIX (why the web API isn't getting updates today)
|
||||
|
||||
`postgres_db_open_worker_connection()` ([`src/db_ops_postgres.c:226`](src/db_ops_postgres.c:226)) calls
|
||||
`postgres_db_set_thread_connection(conn)` at line 243 — but `g_thread_pg_conn` is declared `__thread`
|
||||
([`src/db_ops_postgres.c:45`](src/db_ops_postgres.c:45)), so the binding is set in the **caller's thread**
|
||||
(lws-main, which calls `start_api_worker`), NOT in the api-worker thread. When `api_worker_main` runs and
|
||||
the monitoring query functions call `postgres_db_active_connection()`, `g_thread_pg_conn` is NULL in that
|
||||
thread, so they fall back to the shared global `g_pg_conn` — concurrent access with the main thread. This
|
||||
causes monitoring queries to fail or interleave, which is why the dashboard stops receiving kind 24567
|
||||
events.
|
||||
|
||||
**Fix:** inside `api_worker_main()`, after opening `worker_db`, call
|
||||
`db_set_thread_connection(worker_db)` so the `__thread` pointer is bound in the api-worker thread itself.
|
||||
Call `db_clear_thread_connection()` before closing the connection on shutdown. This must be done in the
|
||||
new rewrite regardless of the loop design.
|
||||
|
||||
```
|
||||
api-worker thread:
|
||||
1. Open own PG connection.
|
||||
2. db_set_thread_connection(worker_db) // bind __thread pointer in THIS thread
|
||||
3. If PG backend: LISTEN event_stored on this connection.
|
||||
4. Loop while running:
|
||||
a. Determine if anyone is listening to kind 24567
|
||||
(has_subscriptions_for_kind(24567) OR no_kind_filter_subs non-empty).
|
||||
b. If NO subscribers:
|
||||
- condvar_timedwait(throttle_sec) // wakeable for shutdown / STATUS_POST job
|
||||
- on wake: process any pending STATUS_POST job, then continue.
|
||||
// Zero DB work, zero notify polling.
|
||||
c. If subscribers exist:
|
||||
- wait_for_notify_or_timeout(throttle_sec):
|
||||
select() on { PQsocket, self-pipe read fd } with timeout = throttle_sec.
|
||||
- If PQsocket readable: PQconsumeInput + drain PQnotifies.
|
||||
- If self-pipe readable: drain (shutdown or STATUS_POST signal).
|
||||
- If timeout: fall through.
|
||||
- generate_round_robin_monitoring_event() // one d-tag per tick
|
||||
- process any pending STATUS_POST job.
|
||||
4. On shutdown: UNLISTEN, close PG connection.
|
||||
```
|
||||
|
||||
Key properties:
|
||||
- **Zero overhead when no subscribers** — condvar sleep, no `select`, no `LISTEN` polling.
|
||||
- **Reactive when subscribers exist** — wakes within `throttle_sec` of an `event_stored` NOTIFY, but never more than once per `throttle_sec` (rate-limited by the select timeout + the round-robin cadence already in `generate_round_robin_monitoring_event`).
|
||||
- **STATUS_POST preserved** — still enqueued from [`src/websockets.c:3470`](src/websockets.c:3470); the self-pipe wakes the worker's `select` so status posts are not delayed by a pending notify wait.
|
||||
- **SQLite fallback** — `db_worker_poll_notify()` returns "not supported" on SQLite, so the worker falls back to pure timer behavior (condvar timed wait) — identical to the no-PG path.
|
||||
|
||||
### Self-pipe for job/shutdown signalling
|
||||
|
||||
A self-pipe (or `eventfd` on Linux) is added to `api.c`:
|
||||
- `api_worker_enqueue_job(STATUS_POST)` writes one byte to the pipe → wakes `select`.
|
||||
- `stop_api_worker()` sets `g_api_worker_running = 0`, writes a byte, and broadcasts the condvar (covers both the subscriber and no-subscriber wait paths).
|
||||
|
||||
### Subscriber-check helper
|
||||
|
||||
Add `int has_any_subscription_for_kind(int event_kind)` in [`src/subscriptions.c`](src/subscriptions.c) that returns true if `has_subscriptions_for_kind(event_kind)` OR the `no_kind_filter_subs` list is non-empty. Export it in [`src/subscriptions.h`](src/subscriptions.h). The worker calls `has_any_subscription_for_kind(24567)`.
|
||||
|
||||
### DB abstraction for LISTEN/NOTIFY
|
||||
|
||||
Add to [`src/db_ops.h`](src/db_ops.h) / [`src/db_ops_postgres.c`](src/db_ops_postgres.c) / [`src/db_ops_sqlite.c`](src/db_ops_sqlite.c):
|
||||
- `int db_worker_listen(void* conn, const char* channel);` — issues `LISTEN <channel>`.
|
||||
- `int db_worker_poll_notify(void* conn, int timeout_ms);` — returns `1` if a notification was consumed, `0` on timeout, `-1` on error/unsupported. Uses `PQsocket()`, `select()`, `PQconsumeInput()`, `PQnotifies()`.
|
||||
- SQLite stubs return `-1` (unsupported) so the worker falls back to timer mode.
|
||||
|
||||
### Schema change (Phase 5)
|
||||
|
||||
Add to both [`src/pg_schema.sql`](src/pg_schema.sql) and [`src/pg_schema.h`](src/pg_schema.h):
|
||||
```sql
|
||||
CREATE OR REPLACE FUNCTION notify_event_stored() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('event_stored', json_build_object(
|
||||
'kind', NEW.kind,
|
||||
'pubkey', substring(NEW.pubkey, 1, 8)
|
||||
)::text);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_notify_event_stored ON events;
|
||||
CREATE TRIGGER trg_notify_event_stored
|
||||
AFTER INSERT ON events
|
||||
FOR EACH ROW EXECUTE FUNCTION notify_event_stored();
|
||||
```
|
||||
Bump `EMBEDDED_PG_SCHEMA_VERSION` from `"4"` to `"5"` in [`src/pg_schema.h`](src/pg_schema.h) and the matching `schema_info` version write in [`src/pg_schema.sql`](src/pg_schema.sql). The trigger is idempotent (`DROP TRIGGER IF EXISTS` + `CREATE`), so existing databases upgrade automatically on next startup via `postgres_db_apply_schema()` ([`src/db_ops_postgres.c:134`](src/db_ops_postgres.c:134)).
|
||||
|
||||
### Phase 2 removals
|
||||
|
||||
- Delete the `monitoring_on_event_stored()` call at [`src/main.c:1169`](src/main.c:1169) (and its forward decl at [`src/main.c:459`](src/main.c:459)).
|
||||
- Delete the `monitoring_on_subscription_change()` calls at [`src/subscriptions.c:483`](src/subscriptions.c:483) and [`src/subscriptions.c:536`](src/subscriptions.c:536) (and its forward decl at [`src/subscriptions.c:58`](src/subscriptions.c:58)).
|
||||
- Remove from [`src/api.h`](src/api.h): `monitoring_on_event_stored`, `monitoring_on_subscription_change`.
|
||||
- Remove from [`src/api.c`](src/api.c): `API_WORK_JOB_EVENT_STORED`, `API_WORK_JOB_SUBSCRIPTION_CHANGE`, `monitoring_on_event_stored`, `monitoring_on_subscription_change`, `monitoring_on_event_stored_sync`, `monitoring_on_subscription_change_sync`. Keep `API_WORK_JOB_STATUS_POST`, `generate_round_robin_monitoring_event`, `generate_monitoring_event_for_type`, `generate_event_driven_monitoring`/`generate_subscription_driven_monitoring` (still used by `generate_monitoring_event` legacy path and tests).
|
||||
- The `generate_event_driven_monitoring` / `generate_subscription_driven_monitoring` wrappers become unused by the worker but remain as public helpers for any external/test callers; the worker calls `generate_round_robin_monitoring_event()` directly.
|
||||
|
||||
### Thread layout (unchanged from plan target)
|
||||
|
||||
```
|
||||
c_relay_pg process
|
||||
├── lws-main — WebSocket event loop (no monitoring DB work)
|
||||
├── db-read-1..N — Async REQ/COUNT queries
|
||||
├── event-worker — Async EVENT ingestion (no monitoring hook)
|
||||
├── db-write-1..N — Async sub logging, misc writes
|
||||
└── api-worker — Timer + LISTEN/NOTIFY driven monitoring (own PG conn)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Mermaid: api-worker state machine
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> OpenConn
|
||||
OpenConn --> BindThread: db_set_thread_connection
|
||||
BindThread --> Listen: PG backend
|
||||
BindThread --> Ready: SQLite/no-PG
|
||||
Listen --> Ready
|
||||
Ready --> CheckSubs
|
||||
CheckSubs --> NoSubSleep: no kind-24567 subs
|
||||
CheckSubs --> WaitNotify: subs present
|
||||
NoSubSleep --> CheckSubs: throttle_sec or wake
|
||||
WaitNotify --> Generate: NOTIFY or throttle_sec
|
||||
Generate --> CheckSubs: after round-robin tick
|
||||
WaitNotify --> NoSubSleep: subs dropped + wake
|
||||
Ready --> [*]: stop_api_worker
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Todo List
|
||||
|
||||
1. Add `has_any_subscription_for_kind()` to `subscriptions.c`/`subscriptions.h` (checks kind index + no-kind-filter list).
|
||||
2. Add `db_worker_listen()` and `db_worker_poll_notify()` to `db_ops.h`, `db_ops_postgres.c` (libpq), and `db_ops_sqlite.c` (stub).
|
||||
3. Add `notify_event_stored()` function + `trg_notify_event_stored` trigger to `pg_schema.sql` and `pg_schema.h`; bump schema version to 5.
|
||||
4. Rewrite `api_worker_main()` in `api.c`: bind the worker PG connection to the thread via `db_set_thread_connection()` (fixes the dashboard-not-updating bug), then run the timer + LISTEN/NOTIFY + subscriber-check loop; add self-pipe for STATUS_POST/shutdown wakeups; keep `start_api_worker`/`stop_api_worker`/`api_worker_process_completions`/`api_worker_enqueue_status_post` signatures.
|
||||
5. Remove `API_WORK_JOB_EVENT_STORED`/`API_WORK_JOB_SUBSCRIPTION_CHANGE` job types and the `monitoring_on_event_stored`/`monitoring_on_subscription_change` functions + sync wrappers from `api.c` and `api.h`.
|
||||
6. Remove the `monitoring_on_event_stored()` call from `main.c` (event storage path).
|
||||
7. Remove the `monitoring_on_subscription_change()` calls from `subscriptions.c` (subscription create/close paths).
|
||||
8. Build with `./make_and_restart_relay.sh` on port 7777; fix any compile errors.
|
||||
9. Test: subscribe to kind 24567 on ws://localhost:7777 → confirm monitoring events arrive on throttle cadence (this validates the thread-connection fix).
|
||||
10. Test: publish a kind-1 event → confirm a NOTIFY-driven monitoring event fires within throttle_sec.
|
||||
11. Test: with no kind-24567 subscribers → confirm no monitoring events generated (zero overhead) and event storage latency unaffected.
|
||||
12. Run `tests/quick_error_tests.sh` and `tests/subscribe_all.sh` against port 7777 to confirm no regressions.
|
||||
|
||||
---
|
||||
|
||||
## Risks & Mitigations
|
||||
|
||||
- **Self-pipe + condvar mix complexity.** Mitigation: the no-subscriber path uses only the condvar; the subscriber path uses `select` on the self-pipe + PQ socket. The condvar is only used to wake the no-subscriber sleep, so the two wait mechanisms never overlap.
|
||||
- **`LISTEN` on a worker connection that also runs monitoring queries.** `LISTEN` is connection-local and persists; running `SELECT`s on the same connection does not cancel it. `PQconsumeInput` must be called before `PQnotifies`. Mitigation: encapsulate all of this in `db_worker_poll_notify()`.
|
||||
- **Schema trigger on every insert adds per-event overhead.** `pg_notify` is cheap (in-memory queue) and the payload is tiny. The existing `sync_event_tags_from_events` AFTER INSERT trigger already does far more work per row, so this is negligible by comparison.
|
||||
- **SQLite has no LISTEN/NOTIFY.** Mitigation: `db_worker_poll_notify` returns `-1` on SQLite and the worker falls back to pure timer mode — functionally identical to the no-notify path.
|
||||
- **Removing the per-event monitoring hook changes dashboard freshness behavior.** Mitigation: with subscribers present, NOTIFY now drives near-real-time updates (better than before); with no subscribers, nothing is generated (matches the plan's "zero overhead" goal). Dashboard already handles event-driven updates via its kind-24567 subscription.
|
||||
@@ -0,0 +1,368 @@
|
||||
# Backfill Redesign: Per-Author Until-Cursor Drain (Option B)
|
||||
|
||||
## Problem
|
||||
|
||||
The current window-expansion backfill (`caching/src/backfill.c`) has four
|
||||
compounding structural defects that make cache coverage unacceptably poor
|
||||
(~10-28% of upstream availability even for a 7-pubkey follow set):
|
||||
|
||||
1. **Overlapping windows re-fetch the same events.** Window N+1's time range
|
||||
fully contains window N's range. The only dedup is a 4096-entry in-memory
|
||||
ring + inbox `ON CONFLICT DO NOTHING`, which overflows for high-volume
|
||||
authors, wasting query budget on duplicates.
|
||||
2. **Broken saturation detection.** When a relay caps at 500 events, the code
|
||||
retries with `limit=5000`, still gets 500, compares `500 == 5000` (false),
|
||||
and incorrectly concludes the page was not saturated — marking the author
|
||||
complete without paginating.
|
||||
3. **`until`-cursor walk-back never triggers** because it's gated on the
|
||||
broken saturation check.
|
||||
4. **Window 4 (full history) was skipped for 5/7 pubkeys** and the loop won't
|
||||
retry once `current_window_index` advances past the last window.
|
||||
|
||||
## Solution
|
||||
|
||||
Replace the window-expansion model with a **per-author `until`-cursor drain**:
|
||||
each followed author has a single persistent cursor (starting at `now`). Each
|
||||
backfill tick queries `since=0&until=cursor&limit=500` for one author,
|
||||
publishes the page, and sets `cursor = oldest_event_created_at - 1`. Repeat
|
||||
until a page returns fewer than the relay's cap (beginning of history reached).
|
||||
The live subscriber stays as-is (it's already correct).
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[cr_follow_resolve] --> B[caching_followed_pubkeys table]
|
||||
B --> C[Backfill tick: pick next incomplete author]
|
||||
C --> D[Query since=0 and until=cursor and limit=500]
|
||||
D --> E{ev_count == relay_cap?}
|
||||
E -- yes --> F[Publish page, set cursor = oldest - 1]
|
||||
F --> D
|
||||
E -- no < relay_cap --> G[Publish page, mark backfill_complete = true]
|
||||
G --> H{More incomplete authors?}
|
||||
H -- yes --> C
|
||||
H -- no --> I[Steady state: live sub only]
|
||||
J[Live subscriber] --> K[caching_event_inbox]
|
||||
K --> L[c-relay-pg inbox poller]
|
||||
L --> M[events table]
|
||||
```
|
||||
|
||||
## Changes
|
||||
|
||||
### 1. New table: `caching_followed_pubkeys`
|
||||
|
||||
Replaces both the ephemeral in-memory `cr_pubkey_set_t` (for persistence) and
|
||||
the window-coupled `caching_backfill_progress` (for cursor state).
|
||||
|
||||
**Schema** (add to `src/pg_schema.sql`):
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS caching_followed_pubkeys (
|
||||
pubkey TEXT PRIMARY KEY,
|
||||
is_root BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
until_cursor BIGINT NOT NULL DEFAULT 0,
|
||||
backfill_complete BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
events_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,
|
||||
last_seen BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,
|
||||
updated_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_caching_followed_incomplete
|
||||
ON caching_followed_pubkeys(backfill_complete, last_seen)
|
||||
WHERE backfill_complete = FALSE;
|
||||
```
|
||||
|
||||
- `until_cursor`: the `until` timestamp for the next backfill query. `0` means
|
||||
"not started yet" (treat as `now` on first query). After each page, set to
|
||||
`oldest_event_created_at - 1`.
|
||||
- `backfill_complete`: set true when a page returns fewer than the relay cap
|
||||
(we've reached the beginning of the author's history).
|
||||
- `is_root`: true for root/admin npubs (used to apply `admin_kinds`).
|
||||
- `events_fetched`: cumulative count for diagnostics.
|
||||
- `last_seen`: updated each follow-graph refresh (prune stale follows).
|
||||
|
||||
**Migration**: On startup, if `caching_followed_pubkeys` is empty but
|
||||
`caching_backfill_progress` has rows, migrate: for each distinct
|
||||
`author_pubkey` in the old table, insert a row with `until_cursor = 0`,
|
||||
`backfill_complete = FALSE` (re-backfill from scratch — the old data was
|
||||
incomplete anyway). Drop `caching_backfill_progress` after migration.
|
||||
|
||||
### 2. Follow graph resolver: persist to `caching_followed_pubkeys`
|
||||
|
||||
**File**: `caching/src/follow_graph.c` + `caching/src/pg_inbox.c`
|
||||
|
||||
After `cr_follow_resolve()` builds the in-memory set, sync it to the table:
|
||||
|
||||
```c
|
||||
/* For each pubkey in the followed set: */
|
||||
/* INSERT INTO caching_followed_pubkeys (pubkey, is_root, last_seen)
|
||||
* VALUES ($1, $2, now)
|
||||
* ON CONFLICT (pubkey) DO UPDATE SET
|
||||
* is_root = EXCLUDED.is_root,
|
||||
* last_seen = EXCLUDED.last_seen;
|
||||
*
|
||||
* Mark follows not seen this refresh as stale (for pruning / diagnostics).
|
||||
* Do NOT touch until_cursor or backfill_complete on existing rows. */
|
||||
```
|
||||
|
||||
New function: `pg_inbox_sync_followed_pubkeys(cr_pubkey_set_t *followed,
|
||||
cr_config_t *cfg)` — upserts all followed pubkeys, updates `last_seen`, sets
|
||||
`is_root` based on `cr_follow_is_root()`.
|
||||
|
||||
New function: `pg_inbox_prune_stale_follows(long threshold_seconds)` —
|
||||
optionally deletes follows not seen in `threshold_seconds` (e.g. 24h). This is
|
||||
off by default; stale follows just stop getting backfilled.
|
||||
|
||||
The in-memory `cr_pubkey_set_t followed` stays as the runtime working set
|
||||
(used by the live subscriber). The table is the durable source of truth for
|
||||
backfill iteration.
|
||||
|
||||
### 3. Rewrite `cr_backfill_tick` — per-author until-cursor drain
|
||||
|
||||
**File**: `caching/src/backfill.c`
|
||||
|
||||
Replace the entire window-expansion logic. New `cr_backfill_tick`:
|
||||
|
||||
```c
|
||||
int cr_backfill_tick(cr_backfill_t *bf, cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_sink_t *sink, const cr_relay_map_t *relay_map) {
|
||||
if (!cfg->backfill.enabled) return -2;
|
||||
|
||||
/* Throttle: one query per tick_interval. */
|
||||
time_t now = time(NULL);
|
||||
if (bf->last_tick && (now - bf->last_tick) < cfg->backfill.tick_interval_seconds)
|
||||
return 0;
|
||||
bf->last_tick = now;
|
||||
|
||||
/* Pick the next incomplete author from the DB (round-robin via
|
||||
* last_seen ordering, or a cursor we persist in bf). */
|
||||
char pk[CR_HEX_LEN];
|
||||
long until_cursor = 0;
|
||||
int is_root = 0;
|
||||
if (pg_inbox_pick_next_backfill_author(pk, &until_cursor, &is_root,
|
||||
&bf->author_round_cursor) != 0) {
|
||||
/* No incomplete authors — steady state. */
|
||||
bf->in_progress = 0;
|
||||
return -2;
|
||||
}
|
||||
|
||||
/* First-time: cursor 0 means start from now. */
|
||||
if (until_cursor == 0) until_cursor = (long)now;
|
||||
|
||||
int page_size = 500; /* matches typical relay cap */
|
||||
int ev_count = 0;
|
||||
cJSON **events = query_author(upstream, relay_map, cfg, pk,
|
||||
0 /* since=0 full history */,
|
||||
until_cursor, page_size, &ev_count);
|
||||
|
||||
if (!events || ev_count == 0) {
|
||||
/* No events at all (or no more) — author is complete. */
|
||||
pg_inbox_update_backfill_progress(pk, 0, 1 /* complete */, 0);
|
||||
free(events);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find oldest timestamp BEFORE publishing. */
|
||||
long oldest_ts = 0;
|
||||
find_oldest_created_at(events, ev_count, &oldest_ts);
|
||||
|
||||
/* Publish all events. */
|
||||
cr_sink_set_source_class(sink, CR_SINK_CLASS_BACKFILL);
|
||||
for (int k = 0; k < ev_count; k++) {
|
||||
cr_sink_publish(sink, events[k]);
|
||||
cJSON_Delete(events[k]);
|
||||
}
|
||||
free(events);
|
||||
bf->events_total += ev_count;
|
||||
|
||||
/* Saturation check: if we got exactly page_size, the page may be
|
||||
* truncated. Set cursor = oldest - 1 and keep going next tick.
|
||||
* If we got fewer than page_size, we've reached the beginning. */
|
||||
if (ev_count >= page_size) {
|
||||
/* Saturated — more history likely exists. Advance cursor. */
|
||||
long next_cursor = (oldest_ts > 0) ? oldest_ts - 1 : 0;
|
||||
pg_inbox_update_backfill_progress(pk, next_cursor, 0 /* not complete */,
|
||||
ev_count);
|
||||
} else {
|
||||
/* Not saturated — beginning of history reached. */
|
||||
pg_inbox_update_backfill_progress(pk, until_cursor, 1 /* complete */,
|
||||
ev_count);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
```
|
||||
|
||||
Key differences from the old code:
|
||||
- **No windows.** Single `since=0` query per author, always full history.
|
||||
- **Saturation check compares against `page_size` (500)**, not `max_cap`
|
||||
(5000). This fixes Defect 2 — if the relay returns 500, we paginate.
|
||||
- **Cursor persisted per-author** in `caching_followed_pubkeys.until_cursor`,
|
||||
not in a window-coupled table. This fixes Defect 4 — no window to skip.
|
||||
- **No overlapping queries.** Each event is fetched exactly once as the cursor
|
||||
walks backwards. This fixes Defect 1.
|
||||
- **`pg_inbox_pick_next_backfill_author`** selects the next
|
||||
`backfill_complete = FALSE` author, round-robin, so all authors get fair
|
||||
backfill time.
|
||||
|
||||
### 4. New PG inbox functions
|
||||
|
||||
**File**: `caching/src/pg_inbox.c` + `caching/src/pg_inbox.h`
|
||||
|
||||
```c
|
||||
/* Sync the followed set to the DB (upsert + update last_seen).
|
||||
* Does NOT touch until_cursor or backfill_complete on existing rows. */
|
||||
int pg_inbox_sync_followed_pubkeys(const cr_pubkey_set_t *followed,
|
||||
const cr_config_t *cfg);
|
||||
|
||||
/* Pick the next incomplete author for backfill (round-robin).
|
||||
* Sets out_pk, out_until_cursor, out_is_root.
|
||||
* Returns 0 on success, -1 if no incomplete authors remain. */
|
||||
int pg_inbox_pick_next_backfill_author(char *out_pk,
|
||||
long *out_until_cursor,
|
||||
int *out_is_root,
|
||||
int *round_cursor);
|
||||
|
||||
/* Update backfill progress for an author. */
|
||||
int pg_inbox_update_backfill_progress(const char *pk,
|
||||
long until_cursor,
|
||||
int complete,
|
||||
int events_this_page);
|
||||
|
||||
/* Migrate old caching_backfill_progress rows to caching_followed_pubkeys.
|
||||
* Called once on startup if the new table is empty and the old one has rows. */
|
||||
int pg_inbox_migrate_backfill_progress(void);
|
||||
|
||||
/* Reset all backfill progress (set until_cursor=0, backfill_complete=FALSE
|
||||
* for all followed pubkeys). Used by the "Reset Backfill" API button. */
|
||||
int pg_inbox_reset_backfill_progress(void);
|
||||
```
|
||||
|
||||
### 5. Simplify `cr_backfill_t` and `cr_config_t` state
|
||||
|
||||
**File**: `caching/src/backfill.h`, `caching/src/config.h`
|
||||
|
||||
Remove from `cr_backfill_t`:
|
||||
- `window_index`, `current_window_s`, `events_this_window`, `window_started`
|
||||
- `author_complete`, `blocked`, `blocked_until_ts`
|
||||
|
||||
Keep:
|
||||
- `cursor` (now: round-robin index into followed set, for fair scheduling)
|
||||
- `until_cursor` (transient, per-tick — not persisted here, read from DB)
|
||||
- `in_progress` (1 if any incomplete authors remain)
|
||||
- `events_total` (cumulative diagnostic)
|
||||
- `last_tick` (throttle)
|
||||
|
||||
Remove from `cr_config_t` state:
|
||||
- `state.backfilled_until` (no longer meaningful)
|
||||
- `state.current_window_index` (no windows)
|
||||
|
||||
Remove from `cr_config_t` backfill config:
|
||||
- `backfill.window_schedule_seconds[]`, `backfill.window_count`
|
||||
- `backfill.window_cooldown_seconds`
|
||||
- `backfill.events_per_tick` → rename to `backfill.page_size` (default 500)
|
||||
|
||||
Keep:
|
||||
- `backfill.enabled`
|
||||
- `backfill.page_size` (was `events_per_tick`)
|
||||
- `backfill.tick_interval_seconds`
|
||||
|
||||
### 6. Update `caching_service_state` table
|
||||
|
||||
**File**: `src/pg_schema.sql`
|
||||
|
||||
The `current_window_index` and `backfill_cursor` columns are no longer
|
||||
meaningful. Replace with:
|
||||
|
||||
```sql
|
||||
ALTER TABLE caching_service_state
|
||||
DROP COLUMN IF EXISTS current_window_index,
|
||||
DROP COLUMN IF EXISTS backfill_cursor,
|
||||
ADD COLUMN IF NOT EXISTS backfill_authors_complete INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS backfill_authors_total INTEGER NOT NULL DEFAULT 0;
|
||||
```
|
||||
|
||||
Update `pg_inbox_update_status()` to report
|
||||
`backfill_authors_complete / backfill_authors_total` instead of
|
||||
`current_window_index / backfill_cursor`. The API status panel shows this as a
|
||||
progress bar (e.g. "Backfill: 3/7 authors complete").
|
||||
|
||||
### 7. Update main loop
|
||||
|
||||
**File**: `caching/src/main.c`
|
||||
|
||||
- Remove `advance_window` calls (no windows).
|
||||
- After `cr_follow_resolve()`, call `pg_inbox_sync_followed_pubkeys()`.
|
||||
- On startup, call `pg_inbox_migrate_backfill_progress()` if needed.
|
||||
- `cr_backfill_init` just sets `in_progress = 1` (check if any incomplete
|
||||
authors exist in the DB).
|
||||
- The follow-graph refresh at [`main.c:378`](caching/src/main.c:378) now also
|
||||
calls `pg_inbox_sync_followed_pubkeys()` so new follows get backfilled and
|
||||
unfollowed pubkeys stop getting backfilled.
|
||||
|
||||
### 8. Update API status display
|
||||
|
||||
**File**: `api/index.js`, `src/config.c` (caching_status handler)
|
||||
|
||||
- Show "Backfill: X/Y authors complete" instead of "Window N, cursor M".
|
||||
- The "Reset Backfill Progress" button now calls
|
||||
`pg_inbox_reset_backfill_progress()` which sets
|
||||
`until_cursor=0, backfill_complete=FALSE` for all rows in
|
||||
`caching_followed_pubkeys`.
|
||||
|
||||
### 9. Config changes
|
||||
|
||||
**File**: `src/pg_schema.sql` (default config inserts), `caching/src/pg_config.c`
|
||||
|
||||
Remove config keys:
|
||||
- `caching_backfill_windows` (no window schedule)
|
||||
- `caching_backfill_window_cooldown_seconds` (no windows)
|
||||
|
||||
Add config key:
|
||||
- `caching_backfill_page_size` already exists (value 500) — repurpose as the
|
||||
per-query limit (was already used this way, just rename in docs).
|
||||
|
||||
Keep:
|
||||
- `caching_backfill_enabled`
|
||||
- `caching_backfill_tick_interval_ms` (throttle between queries)
|
||||
- `caching_backfill_page_size` (per-query limit, default 500)
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Schema**: Add `caching_followed_pubkeys` table + alter
|
||||
`caching_service_state` in `src/pg_schema.sql`.
|
||||
2. **PG inbox functions**: Add `pg_inbox_sync_followed_pubkeys`,
|
||||
`pg_inbox_pick_next_backfill_author`, `pg_inbox_update_backfill_progress`,
|
||||
`pg_inbox_migrate_backfill_progress`, `pg_inbox_reset_backfill_progress`
|
||||
in `caching/src/pg_inbox.c` + `.h`.
|
||||
3. **Follow graph sync**: Call `pg_inbox_sync_followed_pubkeys()` after
|
||||
`cr_follow_resolve()` in `caching/src/main.c`.
|
||||
4. **Rewrite backfill**: Replace `cr_backfill_tick` in
|
||||
`caching/src/backfill.c`. Simplify `cr_backfill_t` in `backfill.h`.
|
||||
5. **Config cleanup**: Remove window config from `caching/src/pg_config.c`
|
||||
and `caching/src/config.c`. Update defaults in `src/pg_schema.sql`.
|
||||
6. **State reporting**: Update `pg_inbox_update_status()` in
|
||||
`caching/src/pg_inbox.c` to report authors complete/total.
|
||||
7. **API status**: Update `api/index.js` and `src/config.c` status handler
|
||||
to show backfill author progress.
|
||||
8. **Migration**: Implement `pg_inbox_migrate_backfill_progress()` and call
|
||||
on startup.
|
||||
9. **Test**: Rebuild, restart with `--reset-backfill`, verify 100% coverage
|
||||
for all 7 followed pubkeys via `nak req` comparison against upstream.
|
||||
|
||||
## Risk and Mitigation
|
||||
|
||||
- **Relays that don't honor `until`**: If a relay ignores the `until` filter,
|
||||
the cursor won't advance and we'll re-fetch the same page. Mitigation:
|
||||
detect when `oldest_ts` doesn't decrease between consecutive pages for the
|
||||
same author and mark the author complete (or skip to a different relay).
|
||||
- **High-volume authors take many ticks**: At 500 events/tick and 5s
|
||||
tick_interval, a 10,000-event author takes ~100s. This is acceptable — the
|
||||
live subscriber keeps you current, and the backfill makes steady progress.
|
||||
If faster backfill is needed later, Option C (parallel workers) can be added
|
||||
on top of this design.
|
||||
- **Follow graph churn**: If follows change frequently, the sync upsert is
|
||||
O(N) per refresh (every 10 min). Fine for personal-scale (hundreds of
|
||||
follows). For thousands, batch the upsert.
|
||||
@@ -0,0 +1,307 @@
|
||||
# Backfill v2: Relay-by-Relay Per-Author Until-Cursor Drain
|
||||
|
||||
## Overview
|
||||
|
||||
Refine the per-author until-cursor backfill to query each relay **one at a
|
||||
time** with **per-relay cursor tracking**, and pick up new follows
|
||||
**immediately** when the admin's kind-3 contact list changes.
|
||||
|
||||
## Problem with the current implementation
|
||||
|
||||
The current backfill (`caching/src/backfill.c`) calls `nostr_relay_pool_query_sync`
|
||||
with **all relay URLs at once**. This function:
|
||||
|
||||
1. Sends the same REQ to all connected relays
|
||||
2. Collects and deduplicates events from all relays into one merged set
|
||||
3. Waits until ALL relays send EOSE, or the timeout (now 30s) expires
|
||||
4. Returns the merged set — but **does not tell the caller whether it exited
|
||||
via all-EOSE or timeout**
|
||||
|
||||
If one relay is slow (e.g. nos.lol streaming 500+ events), the timeout can
|
||||
expire before that relay sends EOSE. The result is a partial event set, but
|
||||
the backfill code can't distinguish "partial due to timeout" from "complete
|
||||
because the author only has 225 events." This causes premature completion
|
||||
marking.
|
||||
|
||||
Additionally, the current design has a single `until_cursor` per author in
|
||||
`caching_followed_pubkeys`, which doesn't track per-relay progress. And new
|
||||
follows are only discovered on the 10-minute follow-graph refresh cycle.
|
||||
|
||||
## Solution
|
||||
|
||||
### 1. Per-relay cursor tracking
|
||||
|
||||
Add a new table to track the backfill cursor for each author on each relay
|
||||
independently:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS caching_backfill_relay_progress (
|
||||
author_pubkey TEXT NOT NULL,
|
||||
relay_url TEXT NOT NULL,
|
||||
until_cursor BIGINT NOT NULL DEFAULT 0,
|
||||
complete BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
events_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,
|
||||
PRIMARY KEY (author_pubkey, relay_url)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_caching_relay_progress_incomplete
|
||||
ON caching_backfill_relay_progress(author_pubkey)
|
||||
WHERE complete = FALSE;
|
||||
```
|
||||
|
||||
- `until_cursor`: the `until` timestamp for the next query to this specific
|
||||
relay for this author. `0` means "not started" (treat as `now`).
|
||||
- `complete`: set true when this relay returns < page_size events for the
|
||||
current cursor range (this relay is drained for this author).
|
||||
- `events_fetched`: cumulative count for diagnostics.
|
||||
|
||||
The existing `caching_followed_pubkeys.backfill_complete` becomes a derived
|
||||
value: an author is complete when all their relay progress rows are complete
|
||||
(or when they have no relay progress rows at all — no relays to query).
|
||||
|
||||
### 2. Relay-by-relay querying
|
||||
|
||||
Instead of one `query_sync` call with N relays, make N `query_sync` calls
|
||||
with 1 relay each. Each call waits for that single relay's EOSE (or timeout).
|
||||
No merge/dedup needed — publish every event directly to the inbox; the
|
||||
database handles deduplication (`ON CONFLICT DO NOTHING`).
|
||||
|
||||
### 3. Immediate new-follow detection
|
||||
|
||||
When the admin publishes a new kind-3 contact list, the live subscriber's
|
||||
`on_event` callback receives it in real-time. We detect this and trigger an
|
||||
immediate follow-graph refresh, which:
|
||||
- Resolves the new follow set
|
||||
- Syncs new pubkeys to `caching_followed_pubkeys`
|
||||
- Creates relay progress rows for new follows (cursor=0, complete=false)
|
||||
- The backfill loop picks them up on the next tick
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Per backfill tick:
|
||||
|
||||
```
|
||||
1. Pick next author that has at least one incomplete relay progress row
|
||||
(round-robin from caching_followed_pubkeys WHERE backfill_complete=FALSE)
|
||||
2. Get incomplete relays for this author:
|
||||
SELECT relay_url, until_cursor FROM caching_backfill_relay_progress
|
||||
WHERE author_pubkey = $1 AND complete = FALSE
|
||||
3. For each incomplete relay (one at a time):
|
||||
a. cursor = until_cursor (0 = now)
|
||||
b. query_sync(pool, &relay_url, 1, filter, &ev_count, 30000)
|
||||
filter = {authors: [pk], kinds: [configured], since: 0,
|
||||
until: cursor, limit: 500}
|
||||
c. Publish every event directly to inbox (cr_sink_publish)
|
||||
d. Find oldest_ts in this batch
|
||||
e. If ev_count == 0:
|
||||
- Mark this relay complete for this author
|
||||
(UPDATE caching_backfill_relay_progress SET complete=TRUE)
|
||||
f. If ev_count >= page_size (saturated):
|
||||
- Advance this relay's cursor: cursor = oldest_ts - 1
|
||||
(UPDATE caching_backfill_relay_progress SET until_cursor = oldest-1)
|
||||
g. If ev_count < page_size but > 0:
|
||||
- This relay is drained for this cursor range
|
||||
- Advance cursor to oldest_ts - 1 AND mark complete
|
||||
(the next query at this lower cursor would return 0 anyway,
|
||||
but we mark complete to avoid an extra round-trip)
|
||||
4. After querying all incomplete relays for this author:
|
||||
Check if ALL relays for this author are now complete:
|
||||
SELECT COUNT(*) WHERE author_pubkey=$1 AND complete=FALSE
|
||||
If 0: mark author complete in caching_followed_pubkeys
|
||||
```
|
||||
|
||||
### Why per-relay cursors are better
|
||||
|
||||
- **Relay A has 500+ events, relay B has 50**: Relay A gets paginated across
|
||||
multiple ticks (cursor walks back 500 at a time). Relay B is drained in one
|
||||
query and marked complete. The author stays incomplete until relay A is
|
||||
also drained. With a single shared cursor, we'd advance the cursor based on
|
||||
the oldest across both relays, potentially skipping events on relay A.
|
||||
|
||||
- **Resume after restart**: each relay's cursor is persisted independently.
|
||||
On restart, we resume each relay exactly where it left off.
|
||||
|
||||
- **Relay goes offline**: if a relay is unreachable, its progress row stays
|
||||
incomplete. The author stays incomplete. When the relay comes back, we
|
||||
resume querying it. No events are lost.
|
||||
|
||||
### Completion criteria
|
||||
|
||||
An author is marked `backfill_complete = TRUE` in `caching_followed_pubkeys`
|
||||
when ALL their relay progress rows in `caching_backfill_relay_progress` are
|
||||
marked `complete = TRUE`. This means every known relay has been queried down
|
||||
to the beginning of the author's history.
|
||||
|
||||
### New follow detection
|
||||
|
||||
In `caching/src/live_subscriber.c`, the `live_on_event` callback currently
|
||||
just publishes events. We add a check:
|
||||
|
||||
```c
|
||||
static void live_on_event(cJSON *event, const char *relay_url, void *user_data) {
|
||||
cr_live_ctx_t *ctx = (cr_live_ctx_t *)user_data;
|
||||
ctx->live->events_received++;
|
||||
cr_sink_publish(ctx->sink, event);
|
||||
|
||||
/* Detect admin kind-3 (contact list) changes → trigger follow refresh */
|
||||
cJSON *kind = cJSON_GetObjectItem(event, "kind");
|
||||
cJSON *pubkey = cJSON_GetObjectItem(event, "pubkey");
|
||||
if (kind && cJSON_IsNumber(kind) && kind->valuedouble == 3 &&
|
||||
pubkey && cJSON_IsString(pubkey)) {
|
||||
if (cr_follow_is_root(ctx->cfg, cJSON_GetStringValue(pubkey))) {
|
||||
ctx->live->follow_graph_changed = 1; /* signal to main loop */
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The main loop checks this flag each iteration:
|
||||
|
||||
```c
|
||||
if (live.follow_graph_changed) {
|
||||
live.follow_graph_changed = 0;
|
||||
/* Immediate follow-graph refresh */
|
||||
cr_pubkey_set_t new_followed;
|
||||
cr_pubkey_set_init(&new_followed);
|
||||
if (cr_follow_resolve(&cfg, upstream, &new_followed) == 0) {
|
||||
/* Sync to DB — new follows get relay progress rows automatically */
|
||||
pg_inbox_sync_followed_pubkeys(...);
|
||||
/* Create relay progress rows for new follows */
|
||||
pg_inbox_init_relay_progress_for_new_follows(&new_followed, &relay_map);
|
||||
/* Resubscribe live with new follow set */
|
||||
cr_live_resubscribe(&live, &cfg, upstream, &new_followed, &sink);
|
||||
cr_pubkey_set_free(&followed);
|
||||
followed = new_followed;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Relay progress row creation
|
||||
|
||||
When a new follow is discovered, we need to create relay progress rows for
|
||||
it. The relays to query are determined by the relay discovery phase (NIP-65
|
||||
outbox relays + bootstrap relays). New function:
|
||||
|
||||
```c
|
||||
int pg_inbox_init_relay_progress_for_author(const char *pk,
|
||||
const char **relay_urls,
|
||||
int relay_count);
|
||||
```
|
||||
|
||||
For each relay URL, insert a row with `until_cursor=0, complete=FALSE` (if
|
||||
not already present). This is called:
|
||||
- After follow-graph resolution for new follows
|
||||
- After relay discovery for newly discovered outbox relays
|
||||
|
||||
## Database schema changes
|
||||
|
||||
### New table: `caching_backfill_relay_progress`
|
||||
|
||||
(see SQL above)
|
||||
|
||||
### `caching_followed_pubkeys` — unchanged
|
||||
|
||||
The `until_cursor` and `backfill_complete` columns remain. `until_cursor`
|
||||
becomes less important (per-relay cursors are the source of truth), but we
|
||||
keep it for backward compatibility and as a quick "has this author started"
|
||||
flag. `backfill_complete` is updated when all relay progress rows are
|
||||
complete.
|
||||
|
||||
## New PG inbox functions
|
||||
|
||||
### `caching/src/pg_inbox.c` + `.h`
|
||||
|
||||
```c
|
||||
/* Initialize relay progress rows for an author. Creates one row per relay
|
||||
* with until_cursor=0, complete=FALSE. Skips rows that already exist. */
|
||||
int pg_inbox_init_relay_progress_for_author(const char *pk,
|
||||
const char **relay_urls,
|
||||
int relay_count);
|
||||
|
||||
/* Pick the next author that has at least one incomplete relay progress row.
|
||||
* Round-robin via round_cursor. Returns 0 on success, -1 if none. */
|
||||
int pg_inbox_pick_next_author_with_incomplete_relays(char *out_pk, int pk_len,
|
||||
int *round_cursor);
|
||||
|
||||
/* Get incomplete relays for an author. Returns a JSON array of
|
||||
* {relay_url, until_cursor} objects. Caller frees. */
|
||||
cJSON *pg_inbox_get_incomplete_relays(const char *pk);
|
||||
|
||||
/* Update relay progress: advance cursor and/or mark complete. */
|
||||
int pg_inbox_update_relay_progress(const char *pk, const char *relay_url,
|
||||
long until_cursor, int complete,
|
||||
int events_this_page);
|
||||
|
||||
/* Check if all relays for an author are complete. Returns 1 if all complete,
|
||||
* 0 if some incomplete, -1 on error. */
|
||||
int pg_inbox_check_author_relays_complete(const char *pk);
|
||||
|
||||
/* Mark author complete in caching_followed_pubkeys if all relay progress
|
||||
* rows are complete. Called after each relay update. */
|
||||
int pg_inbox_check_and_mark_author_complete(const char *pk);
|
||||
```
|
||||
|
||||
## Changes
|
||||
|
||||
### 1. `src/pg_schema.sql` + `src/pg_schema.h`
|
||||
|
||||
Add the `caching_backfill_relay_progress` table and index.
|
||||
|
||||
### 2. `caching/src/pg_inbox.c` + `.h`
|
||||
|
||||
Add the new relay progress functions listed above.
|
||||
|
||||
### 3. `caching/src/backfill.c` — rewrite `cr_backfill_tick`
|
||||
|
||||
Replace the single `query_author` call with a relay-by-relay loop using
|
||||
per-relay cursors from the database.
|
||||
|
||||
### 4. `caching/src/live_subscriber.c` — detect admin kind-3 changes
|
||||
|
||||
Add `follow_graph_changed` flag to `cr_live_t`. Set it in `live_on_event`
|
||||
when a root npub publishes a kind-3.
|
||||
|
||||
### 5. `caching/src/live_subscriber.h` — add flag to struct
|
||||
|
||||
Add `int follow_graph_changed;` to `cr_live_t`.
|
||||
|
||||
### 6. `caching/src/main.c` — handle follow_graph_changed signal
|
||||
|
||||
Check `live.follow_graph_changed` each main loop iteration. If set, trigger
|
||||
immediate follow-graph refresh + relay progress initialization for new
|
||||
follows.
|
||||
|
||||
### 7. `caching/src/main.c` — create relay progress rows after relay discovery
|
||||
|
||||
After `cr_relay_discovery_run()`, call `pg_inbox_init_relay_progress_for_author`
|
||||
for each followed pubkey with its discovered outbox relays + bootstrap relays.
|
||||
|
||||
## Implementation order
|
||||
|
||||
1. Schema: add `caching_backfill_relay_progress` table
|
||||
2. PG inbox functions: relay progress CRUD
|
||||
3. Backfill rewrite: relay-by-relay loop with per-relay cursors
|
||||
4. Live subscriber: kind-3 detection + `follow_graph_changed` flag
|
||||
5. Main loop: handle `follow_graph_changed`, create relay progress rows
|
||||
6. Build and test
|
||||
|
||||
## Testing
|
||||
|
||||
1. Rebuild: `cd caching && make`
|
||||
2. Reset backfill: restart relay with `--reset-backfill`
|
||||
3. Monitor relay progress:
|
||||
```sql
|
||||
SELECT author_pubkey, relay_url, until_cursor, complete, events_fetched
|
||||
FROM caching_backfill_relay_progress
|
||||
ORDER BY author_pubkey, relay_url;
|
||||
```
|
||||
4. Monitor author completion:
|
||||
```sql
|
||||
SELECT pubkey, backfill_complete
|
||||
FROM caching_followed_pubkeys ORDER BY pubkey;
|
||||
```
|
||||
5. Compare coverage: `nak req -k 1 -a <pk> ws://localhost:7777` vs upstream
|
||||
6. Test new-follow detection: follow a new pubkey from the admin account,
|
||||
verify it appears in `caching_followed_pubkeys` and gets backfilled within
|
||||
seconds (not 10 minutes)
|
||||
@@ -0,0 +1,240 @@
|
||||
# C-Relay-PG Admin Page Migration Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Migrate the relay admin dashboard from an embedded HTTP-served page (`api/index.html` + `api/index.js`) to a full client-ndk page (`c-relay-pg.html`) built on the client-ndk template. The page will live in the client-ndk project (`~/lt/client-ndk/www/`) alongside other pages and communicate with the relay purely through Nostr protocol subscriptions.
|
||||
|
||||
## Why This Migration
|
||||
|
||||
The current `/api` page is served directly by the relay's `lws-main` thread, which causes:
|
||||
- **Slow page delivery** — HTTP file serving competes with WebSocket event processing on a single thread
|
||||
- **Incomplete page loads** — 10-second `timeout_secs` kills connections when the main thread is busy
|
||||
- **6+ sequential HTTP round-trips** — each asset requires a two-phase header/body delivery cycle
|
||||
- **Unnecessary complexity** — embedded file generation, `lws_set_wsi_user()` overwriting, session data type dispatch
|
||||
|
||||
The admin page already communicates entirely via Nostr protocol:
|
||||
- **Kind 23456** → Admin commands (encrypted, sent by admin)
|
||||
- **Kind 23457** → Admin responses (encrypted, sent by relay)
|
||||
- **Kind 24567** → Monitoring/stats events (ephemeral, broadcast by relay)
|
||||
- **NIP-11** → Relay info (fetched via HTTP, independent of file serving)
|
||||
- **NIP-17** → DM admin commands (gift-wrapped)
|
||||
|
||||
Moving to client-ndk means the relay serves zero HTML — it only speaks Nostr protocol.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph client-ndk project
|
||||
A[c-relay-pg.html] --> B[init-ndk.mjs]
|
||||
B --> C[NDK SharedWorker]
|
||||
end
|
||||
|
||||
subgraph c-relay-pg
|
||||
D[WebSocket Server]
|
||||
E[api-worker thread]
|
||||
end
|
||||
|
||||
C -->|kind 23456 admin cmds| D
|
||||
D -->|kind 23457 responses| C
|
||||
E -->|kind 24567 monitoring| D
|
||||
D -->|kind 24567 events| C
|
||||
C -->|kind 1059 NIP-17 DMs| D
|
||||
```
|
||||
|
||||
## Current Admin Page Sections
|
||||
|
||||
| Section | Data Source | Communication |
|
||||
|---------|-----------|---------------|
|
||||
| Statistics | kind 24567 monitoring events with d-tags: event_kinds, time_stats, top_pubkeys, cpu_metrics | Subscribe to kind 24567 |
|
||||
| Subscriptions | kind 24567 monitoring events with d-tag: subscription_details | Subscribe to kind 24567 |
|
||||
| Configuration | kind 23457 admin responses | Send kind 23456 config_query, receive kind 23457 |
|
||||
| Authorization | kind 23457 admin responses | Send kind 23456 auth rule commands, receive kind 23457 |
|
||||
| IP Bans | kind 23457 admin responses | Send kind 23456 ip_ban_* commands, receive kind 23457 |
|
||||
| Relay Events | Live subscription to all events + kind 0/10050/10002 management | Standard Nostr subscriptions |
|
||||
| DM | NIP-17 gift-wrapped DMs kind 1059 | NIP-17 encrypt/wrap/publish |
|
||||
| SQL Query | kind 23457 admin responses | Send kind 23456 sql_query, receive kind 23457 |
|
||||
|
||||
## Design Decisions
|
||||
|
||||
1. **Full client-ndk page** — Uses `client.css`, `init-ndk.mjs`, `HamburgerMorphing`, `relay-ui.mjs`, `blossom-ui.mjs`, `ai-ui.mjs` — all standard template infrastructure stays intact
|
||||
2. **Sidenav preserved** — All existing sidenav sections (Relay, Blossom, AI) remain. Admin page navigation links are added to `divSideNavBody` as clickable items
|
||||
3. **Page-specific CSS** — Follows client-ndk convention: page styles in inline `<style>` block, using only CSS variables from `client.css`
|
||||
4. **Hardcoded relay URL** — `wss://relay.laantungir.net` for now; future enhancement to support relay selection
|
||||
5. **Auth mode: required** — Admin page requires authentication
|
||||
|
||||
## Migration Steps
|
||||
|
||||
### Step 1: Create `c-relay-pg.html` from Template
|
||||
|
||||
Start from `template.html` in client-ndk. Customize:
|
||||
|
||||
- **Title**: `C-Relay-PG`
|
||||
- **Header text**: `C-Relay-PG Admin`
|
||||
- **Auth mode**: `required` (default)
|
||||
- **`divBody` content**: All 8 admin sections, each in its own container div, all `display:none` except Statistics (default)
|
||||
- **`divSideNavBody`**: Add navigation links for each section:
|
||||
```html
|
||||
<div id="divAdminNav">
|
||||
<div class="adminNavItem active" data-section="statistics">Statistics</div>
|
||||
<div class="adminNavItem" data-section="subscriptions">Subscriptions</div>
|
||||
<div class="adminNavItem" data-section="configuration">Configuration</div>
|
||||
<div class="adminNavItem" data-section="authorization">Authorization</div>
|
||||
<div class="adminNavItem" data-section="ip-bans">IP Bans</div>
|
||||
<div class="adminNavItem" data-section="relay-events">Relay Events</div>
|
||||
<div class="adminNavItem" data-section="dm">DM</div>
|
||||
<div class="adminNavItem" data-section="database">Database Query</div>
|
||||
</div>
|
||||
```
|
||||
|
||||
### Step 2: Port Admin Communication Layer
|
||||
|
||||
Replace `SimplePool` + `nostr-lite.js` with NDK equivalents:
|
||||
|
||||
| Current index.js | New c-relay-pg.html |
|
||||
|-----------------|---------------------|
|
||||
| `new SimplePool()` | `subscribe()` from init-ndk.mjs |
|
||||
| `relayPool.subscribeMany()` | `subscribe(filter, opts)` |
|
||||
| `relayPool.publish()` | `publishEvent(event)` |
|
||||
| `window.nostr.signEvent()` | Handled by NDK MessageBasedSigner |
|
||||
| `window.NostrTools.nip42` | Handled by NDK NIP-42 support |
|
||||
| `window.NostrTools.nip44` | `window.NostrTools.nip44` from nostr.bundle.js — same as before |
|
||||
| `nlLite` nostr-login-lite | `initNDKPage()` + `getPubkey()` |
|
||||
|
||||
Key function — `sendAdminCommand(commandArray)`:
|
||||
```javascript
|
||||
async function sendAdminCommand(commandArray) {
|
||||
const conversationKey = window.NostrTools.nip44.v2.utils.getConversationKey(
|
||||
adminPrivateKey, relayPubkey
|
||||
);
|
||||
const encrypted = window.NostrTools.nip44.v2.encrypt(
|
||||
JSON.stringify(commandArray), conversationKey
|
||||
);
|
||||
|
||||
const event = {
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
kind: 23456,
|
||||
tags: [['p', relayPubkey]],
|
||||
content: encrypted
|
||||
};
|
||||
|
||||
return await publishEvent(event);
|
||||
}
|
||||
```
|
||||
|
||||
Note: NIP-44 encryption requires the admin's private key. The current page uses `window.nostr.nip44.encrypt()` which delegates to the browser extension. NDK's `publishEvent` handles signing, but encryption must be done before publishing. We'll use `window.nostr.nip44.encrypt()` if available (NIP-07 extension), same as the current page does.
|
||||
|
||||
### Step 3: Port Subscription Setup
|
||||
|
||||
Using NDK's `subscribe()` and `window.addEventListener('ndkEvent', ...)`:
|
||||
|
||||
```javascript
|
||||
// Subscribe to admin responses + monitoring events from relay
|
||||
const adminSub = subscribe(
|
||||
{ kinds: [23457, 24567], authors: [RELAY_PUBKEY] },
|
||||
{ closeOnEose: false, cacheUsage: 'ONLY_RELAY' }
|
||||
);
|
||||
|
||||
// Live event feed
|
||||
const liveSub = subscribe(
|
||||
{ limit: 50 },
|
||||
{ closeOnEose: false, cacheUsage: 'ONLY_RELAY' }
|
||||
);
|
||||
|
||||
// Route events to handlers
|
||||
window.addEventListener('ndkEvent', (e) => {
|
||||
const evt = e.detail;
|
||||
if (evt.kind === 23457) processAdminResponse(evt);
|
||||
else if (evt.kind === 24567) processMonitoringEvent(evt);
|
||||
else addEventToLiveFeed(evt);
|
||||
});
|
||||
```
|
||||
|
||||
### Step 4: Port NIP-44 Encryption
|
||||
|
||||
Admin commands require NIP-44 encryption. The template already loads `nostr.bundle.js` which includes `window.NostrTools.nip44`. Additionally, `window.nostr.nip44.encrypt()` is available from NIP-07 extensions. Use the same approach as the current page.
|
||||
|
||||
### Step 5: Port UI Sections into divBody
|
||||
|
||||
Each section's HTML goes into `divBody` as a container div. Page-specific styles go in the inline `<style>` block following client-ndk conventions:
|
||||
- Use CSS variables only (no hardcoded colors)
|
||||
- Tables use `var(--primary-color)`, `var(--muted-color)`, `var(--accent-color)`
|
||||
- Buttons follow the `var(--button-*)` pattern
|
||||
- Font is always `var(--font-family)`
|
||||
|
||||
Sections to port:
|
||||
1. **Statistics** — DB overview table, event kinds table, time stats table, top pubkeys table, event rate chart area
|
||||
2. **Subscriptions** — Active subscription details table
|
||||
3. **Configuration** — Config key/value table with inline edit, refresh button
|
||||
4. **Authorization** — Auth rules table, whitelist/blacklist inputs, WoT level selector
|
||||
5. **IP Bans** — Ban stats, manual ban form, IP whitelist, ban list table with filters
|
||||
6. **Relay Events** — Live event feed table, kind 0/10050/10002 management forms
|
||||
7. **DM** — NIP-17 message textarea + send button, inbox display
|
||||
8. **SQL Query** — Query dropdown, SQL textarea, execute button, results table
|
||||
|
||||
### Step 6: Page-Specific CSS
|
||||
|
||||
Following client-ndk rules from `client.css`:
|
||||
- Override `#divBody` layout: `flex-direction: column; overflow-y: auto; padding: 20px;`
|
||||
- Admin section containers: hidden by default, shown when nav item clicked
|
||||
- Table styles: use `var(--primary-color)` for headers, `var(--muted-color)` for borders
|
||||
- Button styles: border + transparent background, hover with `var(--accent-color)`
|
||||
- All in inline `<style media="screen">` block in `<head>`
|
||||
|
||||
### Step 7: Relay URL — Hardcoded for Now
|
||||
|
||||
```javascript
|
||||
const RELAY_WS_URL = 'wss://relay.laantungir.net';
|
||||
const RELAY_HTTP_URL = 'https://relay.laantungir.net';
|
||||
```
|
||||
|
||||
NIP-11 fetch to get relay pubkey:
|
||||
```javascript
|
||||
const nip11 = await fetch(RELAY_HTTP_URL, {
|
||||
headers: { 'Accept': 'application/nostr+json' }
|
||||
}).then(r => r.json());
|
||||
const RELAY_PUBKEY = nip11.pubkey;
|
||||
```
|
||||
|
||||
### Step 8: Admin Verification
|
||||
|
||||
Same flow as current page:
|
||||
1. Get relay pubkey from NIP-11
|
||||
2. Send kind 23456 `system_status` command
|
||||
3. If relay responds with kind 23457, user is verified as admin
|
||||
4. Show admin sections; otherwise show access denied
|
||||
|
||||
### Step 9 (Optional, Later): Remove Embedded HTTP Serving from Relay
|
||||
|
||||
After migration is complete and tested:
|
||||
- Remove `handle_embedded_file_request()` from `src/api.c`
|
||||
- Remove `handle_embedded_file_writeable()` from `src/api.c`
|
||||
- Remove embedded file dispatch from `LWS_CALLBACK_HTTP` in `src/websockets.c`
|
||||
- Remove `embed_web_files.sh` and `src/embedded_web_content.c`
|
||||
- Keep NIP-11 serving (lightweight and required by protocol)
|
||||
|
||||
## File Changes Summary
|
||||
|
||||
### New File (in client-ndk project: ~/lt/client-ndk/www/)
|
||||
- `c-relay-pg.html` — Complete admin page, self-contained with inline `<style>` and `<script type="module">`
|
||||
|
||||
### Files Unchanged
|
||||
- All relay C code for kind 23456/23457/24567 processing — stays exactly the same
|
||||
- `src/dm_admin.c` — NIP-17 DM handling stays the same
|
||||
- `src/nip011.c` — NIP-11 stays the same
|
||||
- All client-ndk shared files (client.css, init-ndk.mjs, relay-ui.mjs, etc.) — no modifications needed
|
||||
|
||||
### Files to Modify Later (optional relay cleanup)
|
||||
- `src/api.c` — Remove embedded file serving functions
|
||||
- `src/websockets.c` — Remove embedded file dispatch from HTTP callback
|
||||
- `Makefile` — Remove embedded web content compilation step
|
||||
|
||||
## Risks and Mitigations
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|-----------|
|
||||
| NIP-44 encryption via window.nostr.nip44 | Same mechanism as current page — no change in encryption approach |
|
||||
| NDK SharedWorker relay connection | Use ONLY_RELAY cache usage to ensure direct relay communication |
|
||||
| Large page size with all sections inline | Sections are display:none by default — no performance impact |
|
||||
| Event rate chart from text_graph.js | Port chart logic inline or skip initially |
|
||||
| Relay pubkey needed before subscriptions | Fetch NIP-11 first in initialization sequence |
|
||||
@@ -0,0 +1,72 @@
|
||||
# Caching Code Consolidation Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Move all caching_relay source code into a `caching/` directory inside c-relay-pg,
|
||||
and switch the launcher to use PostgreSQL mode (`-p <pg-conn>`) so all config
|
||||
comes from the c-relay-pg config table — no JSON config file needed.
|
||||
|
||||
## Key Finding
|
||||
|
||||
The caching_relay binary **already supports PG mode** via `-p <pg-conn>`. Its
|
||||
`pg_config.c` reads the same config table keys the caching page sets
|
||||
(`caching_root_npubs`, `caching_bootstrap_relays`, `caching_kinds`, etc.), and
|
||||
`pg_inbox.c` writes fetched events to the `caching_event_inbox` table. The PG
|
||||
schema tables already exist in `src/pg_schema.sql`.
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 1: Create `caching/` directory and copy source files
|
||||
|
||||
Copy all `src/*.c` and `src/*.h` from `/home/user/lt/caching_relay/src/` into
|
||||
`caching/src/` in this repo. Also copy the Makefile, build_static.sh,
|
||||
Dockerfile.alpine-musl, and VERSION.
|
||||
|
||||
Files to copy:
|
||||
- src/main.c, src/main.h
|
||||
- src/config.c, src/config.h (legacy JSON config — kept for fallback)
|
||||
- src/pg_config.c, src/pg_config.h (PG config reader)
|
||||
- src/pg_inbox.c, src/pg_inbox.h (PG inbox writer)
|
||||
- src/backfill.c, src/backfill.h
|
||||
- src/follow_graph.c, src/follow_graph.h
|
||||
- src/live_subscriber.c, src/live_subscriber.h
|
||||
- src/relay_sink.c, src/relay_sink.h (legacy WebSocket sink — kept for fallback)
|
||||
- src/relay_discovery.c, src/relay_discovery.h
|
||||
- src/state.c, src/state.h
|
||||
- src/debug.c, src/debug.h
|
||||
- src/jsonc_strip.c, src/jsonc_strip.h
|
||||
- Makefile
|
||||
- VERSION
|
||||
|
||||
### Step 2: Adapt the Makefile
|
||||
|
||||
Update `caching/Makefile` to:
|
||||
- Point `NOSTR_CORE_DIR` at `../nostr_core_lib` (sibling in this repo)
|
||||
- Output binary to `caching/caching_relay` or `build/caching_relay`
|
||||
|
||||
### Step 3: Update the launcher to use PG mode
|
||||
|
||||
Update `src/caching_service_launcher.c`:
|
||||
- Change `caching_service_start_fork()` to pass `-p <pg_conn>` instead of
|
||||
`-c <config_path>`
|
||||
- Update `caching_service_start()` to read `caching_service_pg_conn` from config
|
||||
table instead of `caching_service_config_path`
|
||||
|
||||
### Step 4: Remove config_path from defaults and UI
|
||||
|
||||
- Remove `caching_service_config_path` from `src/default_config_event.h`
|
||||
- Remove the "Caching Service Config Path" field from `api/index.html`
|
||||
- Remove the field from `CACHING_CONFIG_FIELDS` in `api/index.js`
|
||||
- Update `caching_service_binary_path` default to point at the new build
|
||||
location (`./caching/caching_relay` or `./build/caching_relay`)
|
||||
|
||||
### Step 5: Build and test
|
||||
|
||||
- Build the caching_relay binary from the new `caching/` directory
|
||||
- Build c-relay-pg with the updated launcher
|
||||
- Test: set caching config on the page, apply, start service, verify it runs
|
||||
in PG mode reading config from the database
|
||||
|
||||
### Step 6: Push
|
||||
|
||||
Run `./increment_and_push.sh` with a descriptive commit message.
|
||||
@@ -0,0 +1,338 @@
|
||||
# Caching Service: Descriptive Errors + Retry Limit
|
||||
|
||||
## Problem Summary
|
||||
|
||||
Two issues with the caching service backfill:
|
||||
|
||||
1. **Errors recorded as just "Error"** — When a relay fails, the
|
||||
`caching_backfill_relay_progress.last_status` column stores the literal
|
||||
string `"error"` with no detail. Investigation with `nak` showed the
|
||||
relays currently flagged as `error` actually fail for **three different
|
||||
reasons** that we conflate:
|
||||
- TCP/connect timeout (e.g. `wss://relay.orly.dev`,
|
||||
`wss://relay.snort.social` — "connection took too long")
|
||||
- Query timeout (relay connected, REQ sent, but no EOSE within 30s —
|
||||
e.g. `wss://nostr.wine`, `wss://nostr.land` which actually work fine
|
||||
via `nak` but our 30s timeout is too aggressive)
|
||||
- Connection refused / TLS handshake failure
|
||||
|
||||
Root cause: [`nostr_core_lib/nostr_core/core_relays.c:140-147`](../nostr_core_lib/nostr_core/core_relays.c:140)
|
||||
invokes the progress callback with `status="error"`, `message=NULL`
|
||||
for **all** failure paths (connect failure, REQ send failure). The
|
||||
underlying [`nostr_ws_connect()`](../nostr_core_lib/nostr_websocket/nostr_websocket_openssl.c:146)
|
||||
returns `NULL` with no error string surfaced. The caching callback in
|
||||
[`caching/src/backfill.c:88`](../caching/src/backfill.c:88) just stores
|
||||
`"error"` verbatim.
|
||||
|
||||
2. **No retry limit — infinite retry loop** —
|
||||
[`caching/src/backfill.c:342-354`](../caching/src/backfill.c:342) keeps
|
||||
`complete=false` when a relay errors with no events, so the next tick
|
||||
[`pg_inbox_pick_next_author_with_incomplete_relays`](../caching/src/pg_inbox.c)
|
||||
picks the same author+relay again. The schema
|
||||
`caching_backfill_relay_progress` has no `attempt_count` /
|
||||
`consecutive_errors` / `last_attempt_at` column, so a permanently-dead
|
||||
relay (e.g. `relay.orly.dev`) is retried **every tick forever**. The DB
|
||||
confirms this — error rows have `updated_at` bumped repeatedly within
|
||||
the same minute.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Surface **descriptive error messages** from the relay (or our end) all
|
||||
the way to the `last_status` column so an operator can see *why* a
|
||||
relay is failing.
|
||||
2. Stop retrying a (author, relay) pair after **3 consecutive errors**,
|
||||
marking it `complete=true` so backfill moves on. Log a `WARN`.
|
||||
3. The existing **"Reset Backfill Progress"** button (UI +
|
||||
`caching_reset_progress` admin command) must also clear the new error
|
||||
counters so a manual re-run starts fresh. (Duplicates are deduped by
|
||||
the inbox, so re-running is safe — this matches the user's mental
|
||||
model: "once we cache everyone's back events we never have to do it
|
||||
again, but if a relay was down we can re-run".)
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A[backfill_tick] --> B[nostr_ws_connect]
|
||||
B -->|fail| C[core_relays callback]
|
||||
C -->|status=error, msg=descriptive| D[backfill_query_callback]
|
||||
D --> E[pg_inbox_update_relay_progress]
|
||||
E --> F{consecutive_errors >= 3?}
|
||||
F -->|yes| G[mark complete=true, WARN log]
|
||||
F -->|no| H[keep incomplete, retry next tick]
|
||||
I[Reset Backfill button] --> J[caching_reset_progress]
|
||||
J --> K[clear consecutive_errors=0, complete=false, until_cursor=0]
|
||||
```
|
||||
|
||||
## Error Message Priority Order
|
||||
|
||||
The `last_status` column should reflect the **most descriptive** failure
|
||||
reason available, in this priority order:
|
||||
|
||||
1. **Relay's own response text** (highest priority) — when a relay sends
|
||||
a `NOTICE` or `CLOSED` message with content like
|
||||
`"auth-required: subscribers only"`, `"private relay"`,
|
||||
`"rate-limited: please slow down"`, etc. This **already works today**:
|
||||
[`core_relays.c:326-342`](../nostr_core_lib/nostr_core/core_relays.c:326)
|
||||
captures the message and
|
||||
[`backfill.c:75-82`](../caching/src/backfill.c:75) formats it as
|
||||
`"NOTICE: <msg>"` / `"CLOSED: <msg>"` into `last_status`. No change
|
||||
needed for this path.
|
||||
|
||||
**DB verification (2026-07-28):** Querying the live DB shows only
|
||||
`eose` (49 rows) and `error` (6 rows) — **zero NOTICE/CLOSED rows**.
|
||||
So the relay-response path is wired but not currently firing for the
|
||||
followed set. The 6 `error` rows are all connection/timeout failures
|
||||
(confirmed by `nak`: `relay.orly.dev` and `relay.snort.social`
|
||||
genuinely time out at TCP; the others work via `nak` but hit our
|
||||
30s query timeout). This means **the descriptive-transport-error
|
||||
work in Steps 1-3 is what will actually populate `last_status` for
|
||||
the errors we see today**. The NOTICE/CLOSED path remains as a
|
||||
ready fallback for when a relay does respond with a restriction
|
||||
message.
|
||||
2. **Our connection/transport error** (this plan, Steps 1-3) — when the
|
||||
relay never responds at all (TCP timeout, TLS handshake failure,
|
||||
connection refused), surface a descriptive string like
|
||||
`"error: connect failed: Connection refused"` instead of just
|
||||
`"error"`.
|
||||
3. **Generic fallback** — `"error"` / `"timeout"` only if no more
|
||||
specific information is available.
|
||||
|
||||
So after this change, an operator querying the DB will see:
|
||||
- `NOTICE: auth-required: please authenticate` (relay spoke, we listened)
|
||||
- `CLOSED: rate-limited: too many requests` (relay spoke, we listened)
|
||||
- `error: connect failed: Connection timed out` (relay never responded)
|
||||
- `error: tls handshake failed` (relay never responded)
|
||||
- `timeout` (relay connected but didn't send EOSE in 30s)
|
||||
|
||||
The relay's own words are always preferred over our transport-level
|
||||
diagnosis.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Step 1 — Surface descriptive errors from `nostr_ws_connect`
|
||||
|
||||
**File:** [`nostr_core_lib/nostr_websocket/nostr_websocket_openssl.c`](../nostr_core_lib/nostr_websocket/nostr_websocket_openssl.c)
|
||||
**File:** [`nostr_core_lib/nostr_websocket/nostr_websocket_tls.h`](../nostr_core_lib/nostr_websocket/nostr_websocket_tls.h)
|
||||
|
||||
Add a new variant that fills an error buffer:
|
||||
|
||||
```c
|
||||
/* Connect and on failure write a short descriptive error into err_buf
|
||||
* (at most err_buf_len-1 bytes, always null-terminated). err_buf may be
|
||||
* NULL. Returns client handle or NULL. */
|
||||
nostr_ws_client_t* nostr_ws_connect_with_error(const char* url,
|
||||
char* err_buf,
|
||||
size_t err_buf_len);
|
||||
```
|
||||
|
||||
Implementation reuses the existing `nostr_ws_connect` body but, on each
|
||||
early-return failure path, writes a descriptive string:
|
||||
- URL parse failure → `"invalid url"`
|
||||
- TCP connect failure → `"connect failed: <errno string>"`
|
||||
- TLS handshake failure → `"tls handshake failed"`
|
||||
- WebSocket handshake failure → `"ws handshake failed"`
|
||||
- Memory allocation failure → `"out of memory"`
|
||||
|
||||
Keep `nostr_ws_connect()` as a thin wrapper that calls the new function
|
||||
with `err_buf=NULL` (no behaviour change for existing callers).
|
||||
|
||||
### Step 2 — Pass descriptive error through `core_relays.c`
|
||||
|
||||
**File:** [`nostr_core_lib/nostr_core/core_relays.c`](../nostr_core_lib/nostr_core/core_relays.c)
|
||||
|
||||
In `synchronous_query_relays_with_progress` (lines 125-148), replace:
|
||||
|
||||
```c
|
||||
relays[i].client = nostr_ws_connect(relays[i].url);
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```c
|
||||
char connect_err[128] = {0};
|
||||
relays[i].client = nostr_ws_connect_with_error(relays[i].url,
|
||||
connect_err, sizeof(connect_err));
|
||||
```
|
||||
|
||||
Then on the error paths, pass the descriptive string as the `message`
|
||||
(4th) argument to the callback instead of `NULL`:
|
||||
|
||||
```c
|
||||
if (callback) {
|
||||
callback(relays[i].url, "error",
|
||||
connect_err[0] ? connect_err : "connect failed",
|
||||
0, relay_count, 0, user_data);
|
||||
}
|
||||
```
|
||||
|
||||
Also distinguish the REQ-send-failure path (line 138-142) with message
|
||||
`"req send failed"`.
|
||||
|
||||
For the **timeout** path (line 184-189), the callback already sends
|
||||
`status="timeout"` with `message=NULL` — leave that as-is (the caching
|
||||
callback already records `"timeout"` distinctly, which is the right
|
||||
semantic).
|
||||
|
||||
### Step 3 — Capture descriptive error in caching backfill callback
|
||||
|
||||
**File:** [`caching/src/backfill.c`](../caching/src/backfill.c)
|
||||
|
||||
Update `backfill_query_callback` (lines 58-94) so that when
|
||||
`status == "error"`, the `event_id` parameter (which now carries the
|
||||
descriptive message from Step 2) is included in `last_status`, the same
|
||||
way NOTICE/CLOSED already are:
|
||||
|
||||
```c
|
||||
if (strcmp(status, "error") == 0) {
|
||||
ctx->got_eose = 0;
|
||||
if (event_id && event_id[0] != '\0') {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status),
|
||||
"error: %s", event_id);
|
||||
} else {
|
||||
snprintf(ctx->last_status, sizeof(ctx->last_status), "error");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This makes `last_status` store e.g. `"error: connect failed: Connection refused"`
|
||||
or `"error: tls handshake failed"` instead of just `"error"`.
|
||||
|
||||
### Step 4 — Add error counter column to the schema
|
||||
|
||||
**File:** [`caching/src/pg_inbox.c`](../caching/src/pg_inbox.c)
|
||||
|
||||
In `pg_inbox_update_relay_progress` (around line 866, where
|
||||
`last_status` is already ensured via `ALTER TABLE ... ADD COLUMN IF NOT
|
||||
EXISTS`), add the same idempotent ALTER for the new column:
|
||||
|
||||
```c
|
||||
"ALTER TABLE caching_backfill_relay_progress "
|
||||
"ADD COLUMN IF NOT EXISTS consecutive_errors INTEGER NOT NULL DEFAULT 0"
|
||||
```
|
||||
|
||||
Extend the UPDATE statement to bump `consecutive_errors` on error and
|
||||
reset it to 0 on success (eose/timeout-with-events). The cleanest
|
||||
approach: add a new parameter `int is_error` to
|
||||
`pg_inbox_update_relay_progress` (or infer from `last_status` starting
|
||||
with `"error"` — simpler, no signature change). We'll infer from
|
||||
`last_status`:
|
||||
|
||||
```sql
|
||||
UPDATE caching_backfill_relay_progress
|
||||
SET until_cursor = $3,
|
||||
complete = $4 OR (consecutive_errors >= 3),
|
||||
events_fetched = events_fetched + $5,
|
||||
last_status = CASE WHEN $6 = '' THEN last_status ELSE $6 END,
|
||||
consecutive_errors = CASE
|
||||
WHEN $6 LIKE 'error%' THEN consecutive_errors + 1
|
||||
ELSE 0
|
||||
END,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
WHERE author_pubkey = $1 AND relay_url = $2
|
||||
```
|
||||
|
||||
The `complete = $4 OR (consecutive_errors >= 3)` clause auto-marks
|
||||
complete once the threshold is hit, **without** requiring the caller to
|
||||
know the count. This keeps the backfill.c logic simple.
|
||||
|
||||
> Note: `consecutive_errors >= 3` in the SET refers to the **pre-update**
|
||||
> value (the value before this tick's increment). To mark complete on
|
||||
> the *same* tick that the 3rd error is recorded, we want
|
||||
> `consecutive_errors + 1 >= 3` when `$6 LIKE 'error%'`. So the clause
|
||||
> becomes:
|
||||
> ```sql
|
||||
> complete = $4 OR ($6 LIKE 'error%' AND consecutive_errors + 1 >= 3)
|
||||
> ```
|
||||
|
||||
### Step 5 — Log a WARN when a relay is auto-completed by error limit
|
||||
|
||||
**File:** [`caching/src/backfill.c`](../caching/src/backfill.c)
|
||||
|
||||
After each `pg_inbox_update_relay_progress` call in `cr_backfill_tick`,
|
||||
check the returned status. The simplest approach: after the loop over
|
||||
incomplete relays, re-query `pg_inbox_get_incomplete_relays` is not
|
||||
needed — instead, in the per-relay branch where we currently log
|
||||
`"complete"` vs `"incomplete (will retry)"` (lines 350-353 and 381-391),
|
||||
detect the auto-complete case:
|
||||
|
||||
```c
|
||||
int will_auto_complete = (strncmp(qctx.last_status, "error", 5) == 0);
|
||||
/* The SQL will mark complete if consecutive_errors+1 >= 3. We can't
|
||||
* see the count from here without an extra query, so log at WARN
|
||||
* whenever an error path leaves the row incomplete — and at WARN
|
||||
* when the row would be auto-completed. Simplest: just log WARN
|
||||
* for every error result with the descriptive status. */
|
||||
if (will_auto_complete) {
|
||||
DEBUG_WARN("backfill: %s @ %s error: %s (will auto-complete after 3 consecutive)",
|
||||
pk, relay_url, qctx.last_status);
|
||||
}
|
||||
```
|
||||
|
||||
Optionally, add a `pg_inbox_get_relay_error_count(pk, relay_url)` helper
|
||||
to log the exact count. This is a nice-to-have, not required for the
|
||||
fix.
|
||||
|
||||
### Step 6 — Reset error counters in "Reset Backfill"
|
||||
|
||||
**File:** [`caching/src/pg_inbox.c`](../caching/src/pg_inbox.c) — `pg_inbox_reset_backfill_progress`
|
||||
|
||||
The existing UPDATE on `caching_followed_pubkeys` is fine, but we also
|
||||
need to reset the per-relay table. The relay-side admin command in
|
||||
[`src/config.c:4411`](../src/config.c:4411) already does
|
||||
`DELETE FROM caching_backfill_relay_progress` — which clears the new
|
||||
column too (since the rows are gone). So **no change needed** here:
|
||||
when the user clicks "Reset Backfill Progress", the relay progress
|
||||
rows are deleted, and `cr_backfill_tick` will re-create them via
|
||||
`pg_inbox_init_relay_progress_for_author` with
|
||||
`consecutive_errors=0` (column default).
|
||||
|
||||
Verify this by re-reading
|
||||
[`caching/src/pg_inbox.c`](../caching/src/pg_inbox.c)
|
||||
`pg_inbox_init_relay_progress_for_author` — the INSERT must not
|
||||
explicitly set `consecutive_errors` so the DEFAULT 0 applies. (If it
|
||||
uses an explicit column list, add `consecutive_errors` omission
|
||||
check.)
|
||||
|
||||
### Step 7 — Build and verify
|
||||
|
||||
1. Build nostr_core_lib: `cd nostr_core_lib && make`
|
||||
2. Build caching: `cd caching && make`
|
||||
3. Build relay: `./make_and_restart_relay.sh --start-caching --reset-backfill`
|
||||
4. Watch `build/caching_relay.log` for `WARN` lines mentioning
|
||||
`auto-complete after 3 consecutive`.
|
||||
5. Query the DB:
|
||||
```sql
|
||||
SELECT author_pubkey, relay_url, complete, consecutive_errors, last_status
|
||||
FROM caching_backfill_relay_progress
|
||||
WHERE last_status LIKE 'error%'
|
||||
ORDER BY consecutive_errors DESC;
|
||||
```
|
||||
Expect `last_status` values like `error: connect failed: ...` and
|
||||
rows with `consecutive_errors >= 3` marked `complete=true`.
|
||||
6. Manually verify a known-dead relay (`wss://relay.orly.dev`) hits the
|
||||
limit and stops being retried (no more `updated_at` bumps on
|
||||
subsequent ticks).
|
||||
7. Click "Reset Backfill Progress" in the UI → confirm
|
||||
`caching_backfill_relay_progress` is emptied and backfill restarts
|
||||
from scratch.
|
||||
|
||||
## Files Touched
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `nostr_core_lib/nostr_websocket/nostr_websocket_tls.h` | Add `nostr_ws_connect_with_error()` decl |
|
||||
| `nostr_core_lib/nostr_websocket/nostr_websocket_openssl.c` | Implement `nostr_ws_connect_with_error()`, make `nostr_ws_connect()` a wrapper |
|
||||
| `nostr_core_lib/nostr_core/core_relays.c` | Use new connect variant, pass descriptive msg in error callback |
|
||||
| `caching/src/backfill.c` | Capture descriptive msg in `last_status` for error status; WARN log on error |
|
||||
| `caching/src/pg_inbox.c` | Add `consecutive_errors` column (idempotent ALTER); extend UPDATE to auto-complete at 3 |
|
||||
|
||||
## Out of Scope (deliberately)
|
||||
|
||||
- No exponential backoff — the user explicitly chose "3 errors then
|
||||
stop". Backoff can be added later if needed.
|
||||
- No change to the 30s query timeout — separate concern; could be
|
||||
raised in a follow-up if `nostr.wine`/`nostr.land` keep timing out
|
||||
despite actually being reachable.
|
||||
- No new UI button — the existing "Reset Backfill Progress" button
|
||||
already covers the "re-run the whole cache" use case.
|
||||
@@ -0,0 +1,275 @@
|
||||
# Caching Follows Status UI with Username Resolution
|
||||
|
||||
## Goal
|
||||
|
||||
Show a list of all followed/cached pubkeys with their **usernames**, **event counts by kind**, **outbox relays**, and **backfill status** in the relay admin web UI. Also add username resolution to the existing stats top-pubkeys table.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
A[Admin Web UI] -->|system_command: caching_follows_status| B[relay config.c]
|
||||
B -->|SQL query| C[(PostgreSQL)]
|
||||
C -->|caching_followed_pubkeys| D[follows + backfill status]
|
||||
C -->|events kind=0| E[metadata: name, picture, about]
|
||||
C -->|events kind=10002| F[outbox relay URLs]
|
||||
C -->|events GROUP BY kind| G[event counts per kind]
|
||||
B -->|JSON response| A
|
||||
```
|
||||
|
||||
The relay already has DB access and an admin command pipeline. We add one new admin command that joins the caching tables with the events table to produce a rich per-follow status response.
|
||||
|
||||
## Data Sources
|
||||
|
||||
All data is already in PostgreSQL — no new tables needed:
|
||||
|
||||
| Data | Source Table | Query |
|
||||
|---|---|---|
|
||||
| Followed pubkeys + backfill status | `caching_followed_pubkeys` | `SELECT pubkey, is_root, backfill_complete, events_fetched, first_seen, last_seen` |
|
||||
| Per-relay backfill progress | `caching_backfill_relay_progress` | `SELECT relay_url, until_cursor, complete, events_fetched WHERE author_pubkey = $1` |
|
||||
| Username / profile metadata | `events` (kind=0) | `SELECT content FROM events WHERE pubkey = $1 AND kind = 0 ORDER BY created_at DESC LIMIT 1` — content is JSON with `name`, `picture`, `about`, `nip05` |
|
||||
| Outbox relays | `events` (kind=10002) | `SELECT content, tags FROM events WHERE pubkey = $1 AND kind = 10002 ORDER BY created_at DESC LIMIT 1` — tags contain `["r", "wss://..."]` entries |
|
||||
| Event counts by kind | `events` | `SELECT kind, count(*) FROM events WHERE pubkey = $1 GROUP BY kind ORDER BY kind` |
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
### Step 1: Username/Profile Resolution Helper
|
||||
|
||||
**File:** `src/db_ops_postgres.c` (and `src/db_ops.h` for the declaration)
|
||||
|
||||
Add a reusable function that, given a pubkey, returns the most recent kind-0 metadata as a JSON object with `name`, `picture`, `about`, `nip05`, `display_name` fields parsed from the event content.
|
||||
|
||||
```c
|
||||
// Returns a cJSON object: {"name":"...", "picture":"...", "about":"...", "nip05":"...", "display_name":"..."}
|
||||
// Returns NULL if no kind-0 event exists for this pubkey.
|
||||
cJSON* db_get_profile_metadata(const char* pubkey);
|
||||
```
|
||||
|
||||
**SQL:**
|
||||
```sql
|
||||
SELECT content FROM events
|
||||
WHERE pubkey = $1 AND kind = 0
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
```
|
||||
|
||||
The `content` column is a JSON string (NIP-01 metadata). Parse it with cJSON and extract the fields. This function will be used by both the caching follows command and the stats top-pubkeys enrichment.
|
||||
|
||||
**Also add to SQLite backend** (`src/db_ops_sqlite.c`) for parity, though the primary target is PostgreSQL.
|
||||
|
||||
### Step 2: Outbox Relay Resolution Helper
|
||||
|
||||
**File:** `src/db_ops_postgres.c` (and `src/db_ops.h`)
|
||||
|
||||
Add a function that returns the outbox relay URLs from the most recent kind-10002 event for a pubkey.
|
||||
|
||||
```c
|
||||
// Returns a cJSON array of relay URL strings: ["wss://relay1", "wss://relay2", ...]
|
||||
// Returns NULL if no kind-10002 event exists.
|
||||
cJSON* db_get_outbox_relays(const char* pubkey);
|
||||
```
|
||||
|
||||
**SQL:**
|
||||
```sql
|
||||
SELECT tags FROM events
|
||||
WHERE pubkey = $1 AND kind = 10002
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
```
|
||||
|
||||
The `tags` column is JSONB. Extract all `["r", "url"]` tag entries. In PostgreSQL this can be done in SQL:
|
||||
```sql
|
||||
SELECT tag->>1 AS relay_url
|
||||
FROM events e,
|
||||
jsonb_array_elements(e.tags) AS tag
|
||||
WHERE e.pubkey = $1 AND e.kind = 10002
|
||||
AND jsonb_typeof(tag) = 'array'
|
||||
AND jsonb_array_length(tag) >= 2
|
||||
AND tag->>0 = 'r'
|
||||
ORDER BY e.created_at DESC
|
||||
LIMIT 1
|
||||
```
|
||||
Actually, simpler: fetch the tags JSONB for the latest kind-10002 and parse in C with cJSON (consistent with how the rest of the codebase works).
|
||||
|
||||
### Step 3: New Admin Command `caching_follows_status`
|
||||
|
||||
**File:** `src/config.c` (in `handle_system_command_unified()`, near the existing `caching_status` handler at line 4128)
|
||||
|
||||
Add a new command `caching_follows_status` that:
|
||||
|
||||
1. Queries `caching_followed_pubkeys` for all followed pubkeys
|
||||
2. For each pubkey, calls `db_get_profile_metadata()` to get the username
|
||||
3. For each pubkey, queries event counts by kind: `SELECT kind, count(*) FROM events WHERE pubkey = $1 GROUP BY kind ORDER BY kind`
|
||||
4. For each pubkey, calls `db_get_outbox_relays()` to get outbox relay list
|
||||
5. For each pubkey, queries `caching_backfill_relay_progress` for per-relay status
|
||||
6. Builds a JSON response:
|
||||
|
||||
```json
|
||||
{
|
||||
"command": "caching_follows_status",
|
||||
"status": "success",
|
||||
"timestamp": 1785234000,
|
||||
"follows": [
|
||||
{
|
||||
"pubkey": "4d7842051782...",
|
||||
"npub": "npub1f4uyypgh...",
|
||||
"name": "username_or_null",
|
||||
"picture": "url_or_null",
|
||||
"is_root": false,
|
||||
"backfill_complete": false,
|
||||
"events_fetched": 6965,
|
||||
"first_seen": 1785233973,
|
||||
"last_seen": 1785233973,
|
||||
"event_counts": [
|
||||
{"kind": 0, "count": 1},
|
||||
{"kind": 1, "count": 658},
|
||||
{"kind": 6, "count": 1}
|
||||
],
|
||||
"total_events_in_db": 660,
|
||||
"outbox_relays": [
|
||||
"wss://nostr-01.yakihonne.com",
|
||||
"wss://nostr.land",
|
||||
"wss://relay.snort.social"
|
||||
],
|
||||
"relay_progress": [
|
||||
{"relay_url": "wss://nos.lol", "until_cursor": 1782686796, "complete": false, "events_fetched": 1000},
|
||||
{"relay_url": "wss://relay.damus.io", "until_cursor": 1785234229, "complete": true, "events_fetched": 0}
|
||||
]
|
||||
}
|
||||
],
|
||||
"total_follows": 8,
|
||||
"total_events_in_db": 9843
|
||||
}
|
||||
```
|
||||
|
||||
**Performance consideration:** With up to 5000 followed pubkeys (`caching_max_followed_pubkeys`), running per-pubkey queries in a loop could be slow. Two approaches:
|
||||
|
||||
- **Option A (simple, fine for <100 follows):** Loop in C, call the helper functions per pubkey. With 8-50 follows this is fast enough.
|
||||
- **Option B (scalable, for large follow sets):** Use a single SQL query with CTEs and LEFT JOINs to get everything in one round-trip. This is more complex but handles 5000 follows.
|
||||
|
||||
**Recommendation:** Start with Option A. If follow counts grow large, optimize to Option B later. The per-pubkey queries are all indexed (`idx_events_pubkey_kind`, `idx_events_pubkey`).
|
||||
|
||||
### Step 4: Frontend — Caching Follows Table
|
||||
|
||||
**File:** `api/index.html`
|
||||
|
||||
Add a new section to the caching page (below the existing config form and service/inbox status), containing a table:
|
||||
|
||||
```html
|
||||
<div id="cachingFollowsSection">
|
||||
<h3>Followed Pubkeys</h3>
|
||||
<table id="caching-follows-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>npub</th>
|
||||
<th>Root?</th>
|
||||
<th>Events in DB</th>
|
||||
<th>Backfill</th>
|
||||
<th>Outbox Relays</th>
|
||||
<th>Event Counts by Kind</th>
|
||||
<th>Last Seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="caching-follows-table-body">
|
||||
<tr><td colspan="8">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
```
|
||||
|
||||
**File:** `api/index.js`
|
||||
|
||||
1. Add a `fetchCachingFollowsStatus()` function that sends `['system_command', 'caching_follows_status']` via the admin WebSocket.
|
||||
|
||||
2. Add a `handleCachingFollowsResponse(responseData)` function that renders the table:
|
||||
- Each row shows: name (or "unknown" with a muted style), npub (as a clickable njump.me link), root badge, total events in DB, backfill status (complete/pending with relay count), outbox relays (comma-separated or expandable), event counts by kind (e.g. "k0:1, k1:658, k6:1"), last seen timestamp.
|
||||
- Sort by total_events_in_db descending (most active first).
|
||||
|
||||
3. Hook into the existing caching page auto-refresh: in the `if (pageName === 'caching')` block at line 5445, also call `fetchCachingFollowsStatus()` and add it to the `cachingRefreshInterval` polling.
|
||||
|
||||
4. Add response routing in `handleSystemCommandResponse()` near line 2082:
|
||||
```js
|
||||
if (responseData.command === 'caching_follows_status') {
|
||||
handleCachingFollowsResponse(responseData);
|
||||
}
|
||||
```
|
||||
|
||||
### Step 5: Frontend — Username in Stats Top-Pubkeys Table
|
||||
|
||||
**File:** `api/index.js`
|
||||
|
||||
Modify `populateStatsPubkeysFromMonitoring()` (line 4882) and `populateStatsPubkeys()` (line 4844) to add a "Name" column.
|
||||
|
||||
**Challenge:** The top-pubkeys monitoring data (`src/api.c:470` `query_top_pubkeys()`) currently returns only `{pubkey, event_count}`. To add usernames, we have two options:
|
||||
|
||||
- **Option A (backend enrichment):** Modify `query_top_pubkeys()` in `src/api.c` to also call `db_get_profile_metadata()` for each of the top 10 pubkeys and include `name` in the response. This is the cleanest approach — 10 extra queries on a 30-second polling cycle is negligible.
|
||||
|
||||
- **Option B (frontend enrichment):** The frontend separately queries kind-0 metadata for each pubkey. This is messier and adds latency.
|
||||
|
||||
**Recommendation:** Option A. Modify `query_top_pubkeys()` to include `name` and `picture` fields.
|
||||
|
||||
**Changes:**
|
||||
1. `src/api.c:470` — in `query_top_pubkeys()`, after getting each pubkey row, call `db_get_profile_metadata(pubkey)` and add `name` to the pubkey object.
|
||||
2. `api/index.js:4882` — in `populateStatsPubkeysFromMonitoring()`, add a "Name" column before the npub column. Display `pubkey.name || 'unknown'`.
|
||||
3. `api/index.js:4844` — same change in `populateStatsPubkeys()`.
|
||||
4. `api/index.html` — add a `<th>Name</th>` column to the top-pubkeys table header.
|
||||
|
||||
### Step 6: npub Conversion
|
||||
|
||||
The frontend already has `window.NostrTools.nip19.npubEncode()` available (used at `api/index.js:4902`). The backend `caching_follows_status` response should include both the hex pubkey and the npub for convenience, but the frontend can also convert. Including npub in the backend response is preferred since the backend has access to the same nostr library.
|
||||
|
||||
**File:** `src/config.c` — use `nostr_bytes_to_hex` / `nostr_decode_npub` from nostr_core_lib, or simply return the hex pubkey and let the frontend convert (it already does this in the stats page). **Recommendation: let the frontend convert** — it already has the pattern at line 4902, and it avoids adding nostr_core_lib dependency to the config.c command handler.
|
||||
|
||||
## File Change Summary
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/db_ops.h` | Declare `db_get_profile_metadata()` and `db_get_outbox_relays()` |
|
||||
| `src/db_ops_postgres.c` | Implement both functions (PostgreSQL) |
|
||||
| `src/db_ops_sqlite.c` | Implement both functions (SQLite, for parity) |
|
||||
| `src/db_ops.c` | Wire up dispatch for both functions |
|
||||
| `src/config.c` | Add `caching_follows_status` command handler in `handle_system_command_unified()` |
|
||||
| `src/api.c` | Enrich `query_top_pubkeys()` with `name` field |
|
||||
| `api/index.html` | Add follows table to caching page; add Name column to stats top-pubkeys table |
|
||||
| `api/index.js` | Add `fetchCachingFollowsStatus()`, `handleCachingFollowsResponse()`, route the response, add Name column to pubkey tables, hook into caching page auto-refresh |
|
||||
|
||||
## Mermaid: Data Flow
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant UI as Admin Web UI
|
||||
participant Relay as relay config.c
|
||||
participant DB as PostgreSQL
|
||||
participant Cache as caching_relay process
|
||||
|
||||
UI->>Relay: system_command: caching_follows_status
|
||||
Relay->>DB: SELECT * FROM caching_followed_pubkeys
|
||||
DB-->>Relay: 8 rows
|
||||
loop each pubkey
|
||||
Relay->>DB: SELECT content FROM events WHERE kind=0 AND pubkey=$1
|
||||
DB-->>Relay: metadata JSON
|
||||
Relay->>DB: SELECT kind,count(*) FROM events WHERE pubkey=$1 GROUP BY kind
|
||||
DB-->>Relay: event counts
|
||||
Relay->>DB: SELECT tags FROM events WHERE kind=10002 AND pubkey=$1
|
||||
DB-->>Relay: outbox relay tags
|
||||
Relay->>DB: SELECT * FROM caching_backfill_relay_progress WHERE author_pubkey=$1
|
||||
DB-->>Relay: per-relay status
|
||||
end
|
||||
Relay-->>UI: JSON response with follows array
|
||||
UI->>UI: render table with usernames, event counts, relays
|
||||
```
|
||||
|
||||
## Edge Cases
|
||||
|
||||
- **No kind-0 metadata for a pubkey:** Show "unknown" in the name column with muted styling. The pubkey is still followed and being cached; we just don't have their profile yet. The backfill should eventually fetch their kind-0.
|
||||
- **No kind-10002 for a pubkey:** Show "none" in the outbox relays column. The caching service falls back to bootstrap relays for these authors.
|
||||
- **Large follow sets (5000):** The per-pubkey loop could be slow. Add a `LIMIT` parameter to the command (default 100, configurable). The UI can paginate or show "showing first 100 of 5000".
|
||||
- **Kind-0 content is not valid JSON:** Parse defensively — if cJSON parsing fails, treat as no metadata.
|
||||
- **Replaceable events:** Kind 0 is replaceable, so `ORDER BY created_at DESC LIMIT 1` gets the latest. The unique index `uq_events_replaceable_pubkey_kind` ensures only one kind-0 per pubkey is stored, so this is already handled at the DB level.
|
||||
|
||||
## Future Enhancements (not in this plan)
|
||||
|
||||
- Add/remove follows manually via admin commands (currently follows come from root npub's kind-3)
|
||||
- Delete individual events via admin command
|
||||
- Real-time backfill tick log streaming
|
||||
- Per-follow detail view (click a row to see all events for that author)
|
||||
- Search/filter the follows table by name or npub
|
||||
@@ -0,0 +1,503 @@
|
||||
# Caching Relay Daemon - Architecture Plan
|
||||
|
||||
> **Status:** Draft v3 - adds NIP-65 outbox model: bootstrap relays for
|
||||
> discovery, kind-10002 per-pubkey relay lists, minimum covering set relay
|
||||
> selection, local-relay-first on subsequent startups.
|
||||
|
||||
## Goal
|
||||
|
||||
A standalone C99 daemon that acts as a "caching relay feeder". It:
|
||||
|
||||
1. Reads a `.jsonc` config file listing one or more **root npubs** (e.g. your own npub), a set of upstream relays, a local relay URL, and configurable event kinds.
|
||||
2. For each root npub, fetches its kind-3 contact list to discover **followed pubkeys**.
|
||||
3. Subscribes **live** to events of the configured kinds from the union of followed pubkeys (root npubs + their follows).
|
||||
4. Performs a **throttled backfill** of historical events per followed pubkey, spread out over time so upstream relays are not hammered.
|
||||
5. Re-publishes every fetched event to the **local relay** via a plain WebSocket `EVENT` client connection (relay-agnostic - works with c-relay, c-relay-pg, or any Nostr relay).
|
||||
6. Builds as a **statically linked C99 binary** following the c-relay model, linking against `nostr_core_lib` and `c_utils_lib`.
|
||||
|
||||
The user's Nostr client then points only at the local relay and gets a fast, pre-populated feed without doing any fan-out itself.
|
||||
|
||||
## Design Principles
|
||||
|
||||
- **Relay-agnostic on the sink side.** The daemon is just a Nostr client publishing `EVENT` messages over WebSocket. It does not touch the local relay's database directly. This keeps it decoupled and safe.
|
||||
- **Reuse `nostr_core_lib`.** All Nostr protocol concerns (event validation, kind-3 parsing, relay pool, WebSocket client, NIP-19 npub decoding) come from `nostr_core_lib`. The daemon is orchestration logic only.
|
||||
- **Single statically linked binary.** Built with the same Makefile pattern as c-relay: link `libnostr_core_x64.a` (or arm64) + `libc_utils.a` + system libs (`-lwebsockets -lsqlite3 -lssl -lcrypto -lsecp256k1 -lcurl -lz -ldl -lpthread -lm`).
|
||||
- **C99, `-Wall -Wextra -std=c99 -g -O2`** matching c-relay's `CFLAGS`.
|
||||
- **No config-file framework.** Hand-rolled `.jsonc` parser using cJSON (strip `//` and `/* */` comments before parsing, since cJSON does not natively support JSONC).
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph Config
|
||||
CFG[caching_relay.jsonc]
|
||||
end
|
||||
|
||||
subgraph Daemon
|
||||
MAIN[main.c - lifecycle / signals]
|
||||
CFGP[config.c - parse jsonc]
|
||||
FOLLOW[follow_graph.c - root npubs + kind-3 resolution]
|
||||
BACKFILL[backfill.c - throttled historical pull]
|
||||
LIVE[live_subscriber.c - open subscriptions]
|
||||
SINK[relay_sink.c - publish EVENT to local relay]
|
||||
STATE[state.c - in-memory follow set + seen cache]
|
||||
end
|
||||
|
||||
subgraph nostr_core_lib
|
||||
UPOOL[core_relay_pool.c - upstream pool - query/subscribe]
|
||||
SPOOL[core_relay_pool.c - sink pool - publish only]
|
||||
NIP01[nip001.c - validate / parse]
|
||||
NIP19[nip019.c - npub decode]
|
||||
WS[nostr_websocket - client WS]
|
||||
end
|
||||
|
||||
subgraph Upstream
|
||||
R1[wss://relay.damus.io]
|
||||
R2[wss://nos.lol]
|
||||
R3[wss://...]
|
||||
end
|
||||
|
||||
subgraph Local
|
||||
LOCAL[wss://127.0.0.1:8888 - c-relay / c-relay-pg / any]
|
||||
end
|
||||
|
||||
CFG --> CFGP
|
||||
CFGP --> MAIN
|
||||
MAIN --> FOLLOW
|
||||
FOLLOW -->|query kind 3| UPOOL
|
||||
UPOOL --> R1
|
||||
UPOOL --> R2
|
||||
UPOOL --> R3
|
||||
FOLLOW --> STATE
|
||||
MAIN --> LIVE
|
||||
LIVE -->|subscribe authors + kinds| UPOOL
|
||||
LIVE -->|on_event| SINK
|
||||
MAIN --> BACKFILL
|
||||
BACKFILL -->|query per pubkey since T| UPOOL
|
||||
BACKFILL -->|on_event| SINK
|
||||
SINK -->|publish_async| SPOOL
|
||||
SPOOL -->|EVENT json| LOCAL
|
||||
STATE --> LIVE
|
||||
STATE --> BACKFILL
|
||||
CFGP -->|read/write state| CFG
|
||||
```
|
||||
|
||||
## Data Flow
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant D as Daemon
|
||||
participant UP as upstream_pool
|
||||
participant SP as sink_pool
|
||||
participant U as Upstream Relays
|
||||
participant L as Local Relay
|
||||
|
||||
D->>D: parse caching_relay.jsonc + state
|
||||
D->>UP: add_relay x N upstreams
|
||||
D->>SP: add_relay local only
|
||||
D->>UP: query_sync kind=3 authors=root_npubs
|
||||
U-->>UP: kind-3 events
|
||||
UP-->>D: followed pubkeys set
|
||||
D->>D: merge root + follows into author set
|
||||
D->>UP: subscribe live authors=author_set kinds=configured since=now
|
||||
loop live loop - pump upstream_pool_run
|
||||
U-->>UP: EVENT
|
||||
UP-->>D: on_event callback
|
||||
D->>SP: publish_async EVENT
|
||||
SP->>L: EVENT json
|
||||
D->>SP: pump sink_pool_run to flush callbacks
|
||||
end
|
||||
loop backfill - progressive window expansion
|
||||
D->>UP: query_sync authors=pubkey_i kinds since=now-window limit=K
|
||||
U-->>UP: historical events
|
||||
UP-->>D: events
|
||||
D->>SP: publish_async each EVENT
|
||||
D->>D: sleep tick_interval_seconds
|
||||
D->>D: on full round-robin pass - advance window, write state to jsonc
|
||||
end
|
||||
```
|
||||
|
||||
## Config File Format
|
||||
|
||||
`caching_relay.jsonc` (JSONC = JSON with comments). The config file is the
|
||||
**single source of truth** and is also the daemon's persistent state store:
|
||||
the daemon rewrites it to disk whenever the backfill window advances, so a
|
||||
restart resumes exactly where it left off.
|
||||
|
||||
```jsonc
|
||||
{
|
||||
// Root npubs whose follows list we crawl
|
||||
"root_npubs": [
|
||||
"npub1...",
|
||||
"npub1..."
|
||||
],
|
||||
// Upstream relays to pull events from
|
||||
"upstream_relays": [
|
||||
"wss://relay.damus.io",
|
||||
"wss://nos.lol",
|
||||
"wss://relay.nostr.band"
|
||||
],
|
||||
// Local relay to feed events into (publish-only, never queried)
|
||||
"local_relay": "ws://127.0.0.1:8888",
|
||||
// Event kinds to cache
|
||||
"kinds": [1, 3, 6, 10000, 30023],
|
||||
|
||||
// ---- Backfill: progressive window expansion ----
|
||||
"backfill": {
|
||||
"enabled": true,
|
||||
// Window schedule in seconds-from-now, applied in order.
|
||||
// The daemon first pulls everything newer than (now - 24h).
|
||||
// Once complete, it expands to (now - 7d), then (now - 30d), etc.
|
||||
"window_schedule_seconds": [86400, 604800, 2592000, 7776000, 31536000],
|
||||
// Per-pubkey query limit per tick (keeps individual queries light)
|
||||
"events_per_tick": 50,
|
||||
// Delay between pubkey backfill ticks (throttle to be polite)
|
||||
"tick_interval_seconds": 5,
|
||||
// Delay between completing one window and starting the next
|
||||
"window_cooldown_seconds": 60
|
||||
},
|
||||
|
||||
// ---- Live subscription ----
|
||||
"live": {
|
||||
"enabled": true,
|
||||
"resubscribe_interval_seconds": 300
|
||||
},
|
||||
|
||||
// Refresh the follow graph periodically
|
||||
"follow_graph_refresh_seconds": 600,
|
||||
|
||||
// ---- Persistent state (managed by the daemon; do not hand-edit) ----
|
||||
// The daemon writes these back to this file as backfill progresses.
|
||||
// On restart it reads them to avoid re-pulling already-cached history.
|
||||
"state": {
|
||||
// How far back we have fully backfilled, as a unix timestamp.
|
||||
// Starts at 0 / absent on first run. Advances as each window completes.
|
||||
"backfilled_until": 0,
|
||||
// Index into window_schedule_seconds we are currently working on.
|
||||
"current_window_index": 0,
|
||||
// Round-robin cursor: which followed pubkey we backfill next.
|
||||
"backfill_cursor": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**State write-back rules:**
|
||||
- The daemon rewrites the `.jsonc` file (preserving comments is *not* required
|
||||
on rewrite - it may emit plain JSON once it has been modified at runtime;
|
||||
the comments are only for the user's initial authoring convenience).
|
||||
- Write-back happens: (a) when a window completes and `backfilled_until`
|
||||
advances, (b) on graceful shutdown, (c) periodically (e.g. every 60s) so a
|
||||
crash loses at most one minute of cursor progress.
|
||||
- A write-ahead temp file + rename is used so the config is never left
|
||||
half-written.
|
||||
|
||||
## Progressive Window-Expansion Backfill
|
||||
|
||||
Instead of a fixed per-pubkey cursor, the daemon uses a **global window** that
|
||||
expands backwards in time in discrete steps. This is simpler, gives a natural
|
||||
"recent first" experience, and is trivially resumable from the config file.
|
||||
|
||||
### Window schedule
|
||||
|
||||
`backfill.window_schedule_seconds` is an ordered list, e.g.
|
||||
`[86400, 604800, 2592000, 7776000, 31536000]`
|
||||
= 1 day, 1 week, 1 month, 3 months, 1 year.
|
||||
|
||||
### Algorithm
|
||||
|
||||
1. On startup, read `state.backfilled_until` (unix ts) and
|
||||
`state.current_window_index` from the config.
|
||||
2. If `backfilled_until == 0` (first run), set the current target window to
|
||||
`window_schedule[0]` (e.g. 24h). The backfill `since` cutoff is
|
||||
`now - window_schedule[0]`.
|
||||
3. Round-robin through every followed pubkey. For each pubkey, issue
|
||||
`nostr_relay_pool_query_sync` with:
|
||||
- `authors = [pubkey]`
|
||||
- `kinds = configured kinds`
|
||||
- `since = now - current_window_seconds` (i.e. the *full* current window,
|
||||
not an incremental slice - the local relay dedups, so re-overlap is free
|
||||
and simpler than tracking per-pubkey cursors)
|
||||
- `limit = events_per_tick`
|
||||
4. Sleep `tick_interval_seconds` between pubkeys (throttle).
|
||||
5. When one full round-robin pass over all followed pubkeys completes for the
|
||||
current window:
|
||||
- Set `state.backfilled_until = now - current_window_seconds`.
|
||||
- Advance `state.current_window_index += 1`.
|
||||
- Persist the config file (temp + rename).
|
||||
- Sleep `window_cooldown_seconds` before starting the next (wider) window.
|
||||
6. Repeat with the next (wider) window. The `since` cutoff moves further back
|
||||
in time, so each window pulls the *additional* older slice. Because the
|
||||
local relay dedups inserts, events that fall in the overlap with the
|
||||
previous window are simply ignored on insert - no correctness issue.
|
||||
7. After the last (widest) window completes, the daemon switches to
|
||||
**steady-state**: it only keeps the live subscription running and
|
||||
periodically re-runs the widest window to catch any events that migrated
|
||||
into scope (e.g. a followed user backfilling their own old notes to a
|
||||
different relay). `backfilled_until` stays pinned at the oldest window.
|
||||
|
||||
### Restart behavior
|
||||
|
||||
- On restart, the daemon reads `backfilled_until` and `current_window_index`.
|
||||
- It resumes at the *current* window (the one that was in progress when it
|
||||
stopped). Because each window re-pulls `since = now - window_seconds`, a
|
||||
partial window just re-runs from the start of the round-robin - cheap and
|
||||
correct.
|
||||
- `state.backfill_cursor` records which pubkey in the round-robin was next;
|
||||
this is a minor optimization and may be reset to 0 on restart without harm.
|
||||
|
||||
### Throttling summary
|
||||
|
||||
- `tick_interval_seconds` paces individual pubkey queries (e.g. 5s).
|
||||
- `window_cooldown_seconds` paces window-to-window transitions (e.g. 60s).
|
||||
- `events_per_tick` caps each query's result size (e.g. 50).
|
||||
- Per-relay query latency from `nostr_relay_pool_get_relay_query_latency`
|
||||
can be used to prefer fast relays for backfill; slow relays are still used
|
||||
for the live subscription (where completeness matters more than speed).
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
caching_relay/
|
||||
plans/plan.md (this file)
|
||||
Makefile (c-relay-style, static link)
|
||||
build_static.sh (Alpine MUSL static builder, adapted from c-relay)
|
||||
Dockerfile.alpine-musl (adapted from c-relay)
|
||||
src/
|
||||
main.c (lifecycle, signal handling, main loop)
|
||||
main.h
|
||||
config.c / config.h (jsonc parse + validation)
|
||||
follow_graph.c / .h (resolve root npubs -> kind-3 -> followed set)
|
||||
live_subscriber.c / .h (open-ended live subscription on the pool)
|
||||
backfill.c / .h (throttled historical pull worker)
|
||||
relay_sink.c / .h (publish EVENT to local relay via dedicated sink pool)
|
||||
state.c / .h (in-memory author set + seen-event ring buffer; config state read/write)
|
||||
jsonc_strip.c / .h (strip // and /* */ comments before cJSON_Parse)
|
||||
log.c / log.h (colored stderr logging, c-relay style)
|
||||
examples/
|
||||
caching_relay.jsonc (sample config)
|
||||
README.md
|
||||
```
|
||||
|
||||
## Build Model (mirrors c-relay)
|
||||
|
||||
- `Makefile` with:
|
||||
- `CC = gcc`, `CFLAGS = -Wall -Wextra -std=c99 -g -O2`
|
||||
- `INCLUDES = -I. -Isrc -I../nostr_core_lib -I../nostr_core_lib/nostr_core -I../nostr_core_lib/cjson -I../nostr_core_lib/nostr_websocket -I../c_utils_lib/src`
|
||||
- `LIBS = -lwebsockets -lssl -lcrypto -lsecp256k1 -lcurl -lz -ldl -lpthread -lm -L../c_utils_lib -lc_utils`
|
||||
- Note: no `-lsqlite3` - the daemon itself does not use SQLite. (c-relay uses
|
||||
SQLite for its event store, but that is the local relay's concern, not the
|
||||
daemon's.)
|
||||
- `NOSTR_CORE_LIB = ../nostr_core_lib/libnostr_core_x64.a` (or arm64)
|
||||
- Build `nostr_core_lib` with `--nips=1,6,19` (1 basic, 6 keys, 19 npub bech32).
|
||||
NIP-42 is deferred to a later phase (public relays only for now). Kind-3
|
||||
parsing is plain cJSON tag walking in NIP-01, no special NIP-02 module
|
||||
needed.
|
||||
- Single static link line: `$(CC) $(CFLAGS) $(INCLUDES) $(MAIN_SRC) -o $(TARGET) $(NOSTR_CORE_LIB) $(C_UTILS_LIB) $(LIBS)`
|
||||
- `build_static.sh` adapted from c-relay's Alpine MUSL Docker builder to produce a truly static binary.
|
||||
|
||||
## Key nostr_core_lib APIs Used
|
||||
|
||||
- `nostr_relay_pool_create()` / `nostr_relay_pool_add_relay()` / `nostr_relay_pool_destroy()`
|
||||
- `nostr_relay_pool_query_sync()` - one-shot historical/backfill queries (kind-3 fetch, per-pubkey backfill)
|
||||
- `nostr_relay_pool_subscribe()` - long-lived live subscription with `on_event` / `on_eose` callbacks
|
||||
- `nostr_relay_pool_run()` / `nostr_relay_pool_poll()` - event loop driving both live sub and backfill
|
||||
- `nostr_relay_pool_publish_async()` - publish fetched events to the local relay
|
||||
via a **dedicated single-relay sink pool** (separate from the upstream query
|
||||
pool, so the local relay is never included in `query_sync` fan-out)
|
||||
- `nostr_validate_event()` - validate before republishing (defensive; upstream events should already be valid)
|
||||
- NIP-19: `nostr_nip19_decode` (or equivalent) to convert `npub1...` -> hex pubkey for filters
|
||||
- cJSON for filter construction and kind-3 tag parsing (`["p", "<hex>", "<relay>", "<petname>"]`)
|
||||
|
||||
## Concurrency Model
|
||||
|
||||
Keep it simple and single-threaded with a cooperative event loop, matching c-relay's spirit:
|
||||
|
||||
- **Two `nostr_relay_pool_t` instances**, both driven from the main thread:
|
||||
1. `upstream_pool` - holds all `upstream_relays`. Used for `query_sync`
|
||||
(kind-3 fetch, backfill) and the long-lived live `subscribe`.
|
||||
2. `sink_pool` - holds only `local_relay`. Used exclusively for
|
||||
`publish_async` of fetched events. Never queried.
|
||||
- The main loop alternates between:
|
||||
- `nostr_relay_pool_run(upstream_pool, timeout_ms)` to pump live sub events,
|
||||
- a time-sliced backfill step (one pubkey `query_sync` per
|
||||
`tick_interval_seconds`),
|
||||
- `nostr_relay_pool_run(sink_pool, 0)` to flush pending publish callbacks.
|
||||
- Live subscription's `on_event` callback calls `relay_sink_publish()` which
|
||||
enqueues an `publish_async` on the sink pool.
|
||||
- Backfill `query_sync` is synchronous and blocks the upstream loop briefly -
|
||||
acceptable since `events_per_tick` is small and `tick_interval_seconds`
|
||||
provides pacing.
|
||||
- If profiling later shows backfill blocking the live sub too much, backfill
|
||||
can be moved to a second thread with its own upstream pool. Start without
|
||||
that.
|
||||
|
||||
## State Persistence
|
||||
|
||||
- **No SQLite in the daemon.** The daemon's only persistent state is the
|
||||
backfill window cursor, and that lives in the `.jsonc` config file itself
|
||||
under the `state` object (see Config File Format above).
|
||||
- The followed-pubkey set is re-derived from kind-3 on every
|
||||
`follow_graph_refresh_seconds` tick and on startup - it is not persisted.
|
||||
- Event dedup is delegated entirely to the local relay (c-relay's
|
||||
`INSERT OR IGNORE` on event id). The daemon keeps a small in-memory ring
|
||||
buffer of recently-published event ids only to avoid redundant publish
|
||||
*attempts* within a single run; this is not persisted.
|
||||
- Config write-back uses a temp file + atomic rename so the config is never
|
||||
left half-written, even on crash.
|
||||
|
||||
## Signals & Lifecycle
|
||||
|
||||
- `SIGINT` / `SIGTERM` -> graceful shutdown: close subscriptions, destroy pool, close sink WS.
|
||||
- `SIGHUP` -> reload config (re-read jsonc, refresh follow graph, adjust subscriptions).
|
||||
- Logs to stderr with c-relay-style color prefixes.
|
||||
|
||||
## Decisions Resolved
|
||||
|
||||
1. **Sink pool.** Use a *dedicated* single-relay `nostr_relay_pool_t` for the
|
||||
local sink, separate from the upstream pool. The local relay is never
|
||||
queried, only published to. **Confirmed.**
|
||||
- **Update (v3):** On subsequent startups, the local relay IS queried for
|
||||
kind-3 and kind-10002 events (fast local cache lookup). It is still never
|
||||
queried for general event backfill -- only for discovery metadata.
|
||||
2. **NIP-42 auth.** Deferred to a later phase. Phase 1 targets public relays
|
||||
only. **Confirmed.**
|
||||
3. **Kind-3 freshness.** Use the most recent kind-3 per root npub
|
||||
(`nostr_relay_pool_get_event` with `authors=[npub], kinds=[3]`). Re-resolve
|
||||
on `follow_graph_refresh_seconds` interval. **Confirmed.**
|
||||
4. **State persistence.** No SQLite in the daemon. The `.jsonc` config file is
|
||||
the state store; the daemon writes back `state.backfilled_until`,
|
||||
`state.current_window_index`, and `state.backfill_cursor` as backfill
|
||||
progresses. **Confirmed.**
|
||||
5. **Backfill strategy.** Progressive window expansion
|
||||
(24h -> 7d -> 30d -> 90d -> 365d), round-robin per pubkey within each
|
||||
window, full-window re-pull (local relay dedups). **Confirmed.**
|
||||
6. **NIP-65 outbox model.** `upstream_relays` in config are **bootstrap
|
||||
relays** only. The daemon discovers each followed pubkey's outbox relays
|
||||
via kind 10002, then computes the **minimum covering set** of relays
|
||||
(greedy set cover) that covers all followed pubkeys. Bootstrap relays are
|
||||
always included in the final set. Pubkeys with no kind 10002 fall back to
|
||||
bootstrap relays. **Confirmed.**
|
||||
7. **Local-relay-first on subsequent startups.** On startup, if
|
||||
`state.backfilled_until > 0`, the daemon queries the local relay first for
|
||||
kind-3 and kind-10002 events (fast, no network). Falls back to bootstrap
|
||||
relays for any pubkey not found locally. **Confirmed.**
|
||||
8. **Admin kinds.** Root (admin) npubs get a separate kind list (`admin_kinds`)
|
||||
with `["*"]` support for all kinds. Two live subscriptions: follows_sub
|
||||
(non-admin, regular kinds) and admin_sub (admin, admin_kinds). **Confirmed.**
|
||||
|
||||
## NIP-65 Outbox Model Design (Phase 2)
|
||||
|
||||
### New module: `relay_discovery.c`
|
||||
|
||||
Responsibilities:
|
||||
- For each followed pubkey, fetch their most recent kind 10002 (relay list).
|
||||
- First-time: query from bootstrap relays.
|
||||
- Subsequent: query from local relay first, bootstrap fallback.
|
||||
- Parse `r` tags from kind 10002 events: `["r", "<url>"]` or
|
||||
`["r", "<url>", "read"]` / `["r", "<url>", "write"]`.
|
||||
- We care about "read" relays (we are reading events FROM them).
|
||||
- If no read/write marker, assume both.
|
||||
- Build a map: `pubkey -> list of outbox relay URLs`.
|
||||
- Publish all kind-10002 events to the local relay (cache them for next startup).
|
||||
|
||||
### Greedy set cover algorithm
|
||||
|
||||
```
|
||||
Input: pubkey_to_relays map {pubkey -> [relay1, relay2, ...]}
|
||||
Output: minimal set of relay URLs covering all pubkeys
|
||||
|
||||
1. uncovered = set of all pubkeys
|
||||
2. selected = empty set
|
||||
3. Add all bootstrap relays to selected (always included)
|
||||
4. For each bootstrap relay, remove its known pubkeys from uncovered
|
||||
(bootstrap relays cover pubkeys with no 10002)
|
||||
5. While uncovered is not empty:
|
||||
a. Find relay R that covers the most pubkeys in uncovered
|
||||
b. If no relay covers any uncovered pubkey, break (orphaned pubkeys)
|
||||
c. Add R to selected
|
||||
d. Remove all pubkeys covered by R from uncovered
|
||||
6. Return selected
|
||||
```
|
||||
|
||||
### Data structures
|
||||
|
||||
```c
|
||||
/* Per-pubkey outbox relay list */
|
||||
typedef struct {
|
||||
char pubkey[CR_HEX_LEN];
|
||||
char relays[CR_MAX_RELAYS_PER_PUBKEY][CR_URL_LEN];
|
||||
int relay_count;
|
||||
} cr_outbox_entry_t;
|
||||
|
||||
/* Relay-to-pubkeys coverage map (for set cover) */
|
||||
typedef struct {
|
||||
char url[CR_URL_LEN];
|
||||
char pubkeys[CR_MAX_PUBKEYS_PER_RELAY][CR_HEX_LEN];
|
||||
int pubkey_count;
|
||||
} cr_relay_coverage_t;
|
||||
|
||||
/* Result of relay discovery */
|
||||
typedef struct {
|
||||
cr_outbox_entry_t *outboxes; /* per-pubkey relay lists */
|
||||
int outbox_count;
|
||||
char selected_relays[CR_MAX_UPSTREAM][CR_URL_LEN];
|
||||
int selected_count;
|
||||
} cr_relay_map_t;
|
||||
```
|
||||
|
||||
### Startup sequence change
|
||||
|
||||
```
|
||||
OLD:
|
||||
load config -> create pools -> resolve follow graph -> open live sub -> backfill
|
||||
|
||||
NEW:
|
||||
load config -> create pools ->
|
||||
resolve follow graph (local-first on subsequent) ->
|
||||
discover outbox relays (kind 10002, local-first on subsequent) ->
|
||||
compute minimum covering set ->
|
||||
add selected relays to upstream_pool ->
|
||||
log selected relays + coverage ->
|
||||
open live sub -> backfill (per-pubkey from their outbox relays)
|
||||
```
|
||||
|
||||
### Backfill change
|
||||
|
||||
Instead of fanning out each backfill query to ALL upstream relays, backfill
|
||||
queries each pubkey from **that pubkey's specific outbox relays** (or bootstrap
|
||||
relays as fallback). This is more efficient and more polite to relays that
|
||||
don't have that pubkey's events.
|
||||
|
||||
### New config fields
|
||||
|
||||
None required. `upstream_relays` is reinterpreted as bootstrap relays.
|
||||
Optionally a `max_outbox_relays` cap could be added later if the covering set
|
||||
grows too large.
|
||||
|
||||
## Implementation Todo List
|
||||
|
||||
### Phase 0 - Local relay sanity check (DONE)
|
||||
|
||||
0. **Start a local c-relay and verify read/write with `nak`.** DONE.
|
||||
|
||||
### Phase 1 - Daemon implementation (DONE)
|
||||
|
||||
1-12. All implemented, built, and tested. See git history. DONE.
|
||||
|
||||
### Phase 2 - NIP-65 outbox model
|
||||
|
||||
1. Implement `relay_discovery.c/.h`: fetch kind-10002 per pubkey, parse `r`
|
||||
tags, build `pubkey -> relays` map, publish 10002 events to local relay.
|
||||
2. Implement greedy set cover: compute minimum covering relay set from the
|
||||
outbox map, always include bootstrap relays.
|
||||
3. Update `follow_graph.c`: query local relay first for kind-3 on subsequent
|
||||
startups, fall back to bootstrap relays.
|
||||
4. Update `backfill.c`: query each pubkey from their specific outbox relays
|
||||
instead of all upstream relays.
|
||||
5. Update `main.c`: insert relay discovery phase between follow graph
|
||||
resolution and live subscription. Add discovered relays to upstream_pool.
|
||||
Log selected relays and coverage.
|
||||
6. Update `caching_relay_config.jsonc`: update comments to say "bootstrap
|
||||
relays" instead of "upstream relays".
|
||||
7. Build and test NIP-65 outbox model end-to-end with a real npub.
|
||||
8. Update `README.md` if any flowchart details change during implementation.
|
||||
@@ -0,0 +1,543 @@
|
||||
# Caching Relay PostgreSQL Inbox Integration Plan
|
||||
|
||||
> **Status:** Revised and simplified architecture.
|
||||
>
|
||||
> **Decision:** Keep caching as a separate application. The caching application
|
||||
> focuses on polite upstream collection and writes raw event JSON to a minimal
|
||||
> PostgreSQL inbox. c-relay-pg removes small priority-ordered batches and handles
|
||||
> them through relay-owned event processing. The design deliberately accepts a
|
||||
> small crash-loss window between dequeue and canonical storage; live overlap and
|
||||
> backfill retries recover those events naturally.
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Connect the standalone caching application to c-relay-pg without publishing
|
||||
cached events through a loopback WebSocket and without allowing the caching
|
||||
application to write directly to the canonical [`events`](../src/pg_schema.sql:12)
|
||||
table.
|
||||
|
||||
The responsibilities are intentionally narrow:
|
||||
|
||||
- **Caching application:** sensibly and politely acquire the desired events.
|
||||
- **PostgreSQL inbox:** temporarily hold structurally plausible event JSON.
|
||||
- **c-relay-pg:** remain the sole authority for accepting, storing, and
|
||||
broadcasting events.
|
||||
|
||||
This integration is PostgreSQL-only. The relay's normal SQLite operation remains
|
||||
unchanged, but the external caching service is unavailable with that backend.
|
||||
|
||||
## 2. Simplified Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
ROOT[Configured root npubs] --> CACHE[Caching application]
|
||||
UP[Upstream relays] --> CACHE
|
||||
CACHE --> INBOX[PostgreSQL caching event inbox]
|
||||
INBOX --> POLLER[c-relay-pg bounded poller]
|
||||
POLLER --> INGEST[Relay-owned event ingestion]
|
||||
INGEST --> EVENTS[Canonical events table]
|
||||
INGEST --> CLIENTS[Active subscriptions]
|
||||
|
||||
UI[Caching admin page] --> CFG[Shared caching configuration]
|
||||
CFG --> CACHE
|
||||
CACHE --> STATUS[Service heartbeat and progress]
|
||||
STATUS --> UI
|
||||
```
|
||||
|
||||
No loopback sink relay pool is used. No direct insert into the canonical event
|
||||
store is allowed from the caching application.
|
||||
|
||||
## 3. Deliberate Simplicity Rules
|
||||
|
||||
The first implementation will not include:
|
||||
|
||||
- multiple caching workers;
|
||||
- multiple inbox consumers;
|
||||
- row leases or claim expiration;
|
||||
- a queue state machine;
|
||||
- retry rows;
|
||||
- a dead-letter queue;
|
||||
- PostgreSQL `LISTEN`/`NOTIFY`;
|
||||
- persistent relay-health history;
|
||||
- exactly-once delivery;
|
||||
- direct process start/stop through the web page;
|
||||
- upstream NIP-42 authentication;
|
||||
- support for the SQLite backend.
|
||||
|
||||
The design prefers harmless duplicate acquisition over complicated delivery
|
||||
coordination. Event IDs already provide natural deduplication in both the inbox
|
||||
and canonical store.
|
||||
|
||||
## 4. Responsibility Boundaries
|
||||
|
||||
### 4.1 Caching application
|
||||
|
||||
The caching application is responsible for:
|
||||
|
||||
1. Reading caching configuration from PostgreSQL.
|
||||
2. Decoding configured root npubs.
|
||||
3. Resolving the roots' newest kind-3 follow lists.
|
||||
4. Discovering useful read relays from kind 10002.
|
||||
5. Computing a reasonable covering relay set.
|
||||
6. Maintaining live subscriptions for roots and follows.
|
||||
7. Performing polite, resumable historical backfill.
|
||||
8. Inserting received event JSON into the inbox.
|
||||
9. Persisting only the progress needed to resume discovery and backfill.
|
||||
10. Publishing a lightweight heartbeat and status snapshot.
|
||||
|
||||
It is not responsible for:
|
||||
|
||||
- deciding whether an event belongs in the canonical relay database;
|
||||
- applying c-relay-pg publication authorization;
|
||||
- processing NIP-09 deletions;
|
||||
- implementing replacement semantics;
|
||||
- executing relay administrator commands;
|
||||
- broadcasting to connected relay clients.
|
||||
|
||||
The caching callback may perform minimal application-side checks before insert:
|
||||
|
||||
- the callback supplied a JSON object;
|
||||
- an event ID string exists;
|
||||
- the serialized event is below a configured maximum size.
|
||||
|
||||
Cryptographic verification is optional in the fetcher and is not trusted by
|
||||
c-relay-pg. The initial version should omit it unless the existing standalone
|
||||
code already provides it at negligible integration cost.
|
||||
|
||||
### 4.2 PostgreSQL inbox
|
||||
|
||||
The inbox is deliberately dumb. It rejects obvious malformed data, deduplicates
|
||||
events currently waiting in the inbox, and establishes live-over-backfill
|
||||
priority. It does not attempt to verify a Nostr hash or Schnorr signature.
|
||||
|
||||
### 4.3 c-relay-pg
|
||||
|
||||
c-relay-pg is responsible for:
|
||||
|
||||
1. Removing a bounded batch from the inbox.
|
||||
2. Parsing each event JSON object.
|
||||
3. Applying authoritative structural, ID, and signature validation.
|
||||
4. Applying relay-wide event rules such as expiration and PoW where applicable.
|
||||
5. Bypassing client-session NIP-42 requirements because the source is internal.
|
||||
6. Preventing imported kind 23456 events from executing administrator commands.
|
||||
7. Applying duplicate, ephemeral, replaceable, addressable, and NIP-09 behavior.
|
||||
8. Storing accepted events through its normal database abstraction.
|
||||
9. Running post-store work and broadcasting newly accepted events from the main
|
||||
libwebsockets thread.
|
||||
10. Recording simple in-memory and cumulative import counters.
|
||||
|
||||
## 5. Minimal PostgreSQL Schema
|
||||
|
||||
Add the following objects to [`src/pg_schema.sql`](../src/pg_schema.sql) and the
|
||||
embedded PostgreSQL schema header.
|
||||
|
||||
### 5.1 `caching_event_inbox`
|
||||
|
||||
Suggested logical schema:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS caching_event_inbox (
|
||||
queue_id BIGSERIAL PRIMARY KEY,
|
||||
event_id TEXT NOT NULL UNIQUE,
|
||||
event_json JSONB NOT NULL,
|
||||
source_relay TEXT,
|
||||
source_class TEXT NOT NULL DEFAULT 'backfill',
|
||||
priority SMALLINT NOT NULL DEFAULT 1,
|
||||
received_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,
|
||||
|
||||
CHECK (jsonb_typeof(event_json) = 'object'),
|
||||
CHECK (jsonb_typeof(event_json->'id') = 'string'),
|
||||
CHECK (length(event_json->>'id') = 64),
|
||||
CHECK (event_id = event_json->>'id'),
|
||||
CHECK (jsonb_typeof(event_json->'pubkey') = 'string'),
|
||||
CHECK (length(event_json->>'pubkey') = 64),
|
||||
CHECK (jsonb_typeof(event_json->'sig') = 'string'),
|
||||
CHECK (length(event_json->>'sig') = 128),
|
||||
CHECK (jsonb_typeof(event_json->'created_at') = 'number'),
|
||||
CHECK (jsonb_typeof(event_json->'kind') = 'number'),
|
||||
CHECK (jsonb_typeof(event_json->'tags') = 'array'),
|
||||
CHECK (jsonb_typeof(event_json->'content') = 'string'),
|
||||
CHECK (source_class IN ('live', 'discovery', 'backfill')),
|
||||
CHECK (priority IN (0, 1))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_caching_inbox_dequeue
|
||||
ON caching_event_inbox(priority, received_at, queue_id);
|
||||
```
|
||||
|
||||
Priority meanings:
|
||||
|
||||
- `0`: live and discovery metadata;
|
||||
- `1`: historical backfill.
|
||||
|
||||
The caching application uses a parameterized insert with
|
||||
`ON CONFLICT (event_id) DO NOTHING`. It must impose a serialized event-size
|
||||
limit before sending the row. PostgreSQL schema constraints are structural
|
||||
protection, not authoritative Nostr validation.
|
||||
|
||||
### 5.2 `caching_service_state`
|
||||
|
||||
Use one singleton row for inexpensive UI status:
|
||||
|
||||
- service version;
|
||||
- service state: `starting`, `running`, `degraded`, or `stopped`;
|
||||
- applied config generation;
|
||||
- heartbeat timestamp;
|
||||
- followed author count;
|
||||
- selected and connected relay counts;
|
||||
- current backfill window and author cursor;
|
||||
- events fetched and inbox inserts;
|
||||
- last error text and timestamp.
|
||||
|
||||
This is a status snapshot, not an audit log. The caching process overwrites the
|
||||
same row periodically.
|
||||
|
||||
### 5.3 `caching_backfill_progress`
|
||||
|
||||
Persist only enough state for the caching application to resume politely:
|
||||
|
||||
- author pubkey;
|
||||
- window index;
|
||||
- immutable window anchor;
|
||||
- current inclusive `until` cursor;
|
||||
- completion flag;
|
||||
- updated timestamp.
|
||||
|
||||
Use a composite primary key on author and window. Do not store individual fetched
|
||||
event IDs here; inbox and canonical event IDs handle deduplication.
|
||||
|
||||
Timestamp-only Nostr pagination can saturate when many events share one second.
|
||||
For the first version, increase the query limit up to a fixed safety ceiling when
|
||||
a full page ends at one timestamp. If that ceiling remains saturated, leave the
|
||||
author incomplete, log it, and retry later rather than falsely marking it done.
|
||||
|
||||
## 6. Simple Destructive Dequeue
|
||||
|
||||
c-relay-pg uses one PostgreSQL transaction to remove and return a small batch:
|
||||
|
||||
```sql
|
||||
WITH selected AS (
|
||||
SELECT queue_id
|
||||
FROM caching_event_inbox
|
||||
ORDER BY priority ASC, received_at ASC, queue_id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE
|
||||
)
|
||||
DELETE FROM caching_event_inbox AS inbox
|
||||
USING selected
|
||||
WHERE inbox.queue_id = selected.queue_id
|
||||
RETURNING inbox.event_id,
|
||||
inbox.event_json,
|
||||
inbox.source_relay,
|
||||
inbox.source_class,
|
||||
inbox.received_at;
|
||||
```
|
||||
|
||||
Only one c-relay-pg inbox consumer is supported. `SKIP LOCKED`, leases, and claim
|
||||
owners are therefore unnecessary.
|
||||
|
||||
### 6.1 Accepted trade-off
|
||||
|
||||
There is a small loss window if c-relay-pg commits the delete and crashes before
|
||||
storing the returned events. This is accepted deliberately:
|
||||
|
||||
- live subscriptions reconnect with overlap;
|
||||
- backfill revisits incomplete and steady-state windows;
|
||||
- duplicate reacquisition is safe;
|
||||
- the reduced implementation complexity is worth the rare temporary loss.
|
||||
|
||||
The poller should keep batches small so the loss window and memory footprint are
|
||||
small.
|
||||
|
||||
## 7. c-relay-pg Inbox Consumer
|
||||
|
||||
### 7.1 Polling
|
||||
|
||||
- Compile the consumer only for `DB_BACKEND_POSTGRES`.
|
||||
- Poll a small batch at a configurable interval.
|
||||
- Poll quickly while rows are found and back off to a slower interval when empty.
|
||||
- Always order live/discovery rows before backfill rows.
|
||||
- Allow only one batch in memory at a time.
|
||||
- Stop polling immediately during relay shutdown.
|
||||
|
||||
Initial conservative defaults should be small, for example a few dozen rows per
|
||||
batch and subsecond-to-multisecond active polling with a longer idle interval.
|
||||
Exact defaults should be selected during integration testing rather than encoded
|
||||
as architectural requirements.
|
||||
|
||||
### 7.2 Ingestion behavior
|
||||
|
||||
Do not send dequeued events through a loopback WebSocket. Refactor the existing
|
||||
inbound event handling into a reusable internal ingestion entry point with an
|
||||
explicit source mode:
|
||||
|
||||
- `CLIENT_EVENT`: normal client authentication and response behavior;
|
||||
- `CACHING_INBOX_EVENT`: no client session or NIP-42 requirement, no OK response,
|
||||
and no administrator command execution.
|
||||
|
||||
The shared path should preserve:
|
||||
|
||||
- event ID and signature verification;
|
||||
- event limits;
|
||||
- expiration and PoW behavior;
|
||||
- NIP-09 authorization and deletion behavior;
|
||||
- ephemeral handling;
|
||||
- PostgreSQL replacement semantics in
|
||||
[`postgres_db_insert_event_with_json()`](../src/db_ops_postgres.c:1596);
|
||||
- duplicate detection;
|
||||
- post-store monitoring;
|
||||
- main-thread broadcast.
|
||||
|
||||
Avoid creating a second implementation of these rules solely for inbox events.
|
||||
|
||||
### 7.3 Main-thread broadcast
|
||||
|
||||
Database and cryptographic work may run off the libwebsockets thread, but
|
||||
[`store_event_post_actions()`](../src/main.c:1161) and active subscription
|
||||
broadcast must be queued to the main thread, following the existing asynchronous
|
||||
completion pattern in
|
||||
[`process_async_event_completions()`](../src/websockets.c:725).
|
||||
|
||||
Newly stored events are broadcast. Duplicates and stale replacements are not.
|
||||
Validated ephemeral events are broadcast but not stored.
|
||||
|
||||
### 7.4 Failure behavior
|
||||
|
||||
Because rows have already been removed, malformed or invalid events are simply
|
||||
counted and logged at a rate-limited level. They are not requeued.
|
||||
|
||||
A temporary canonical database failure should stop further inbox polling until
|
||||
the relay database is healthy. The current in-memory batch may be lost under the
|
||||
accepted simple-delivery model.
|
||||
|
||||
## 8. Caching Application Acquisition Strategy
|
||||
|
||||
The external application should spend most of its design effort here.
|
||||
|
||||
### 8.1 Follow graph
|
||||
|
||||
- Decode configured root npubs.
|
||||
- Fetch the newest kind 3 for each root from bootstrap relays.
|
||||
- Include roots themselves in the author set.
|
||||
- Parse valid `p` tags, deduplicate authors, and enforce a configured author cap.
|
||||
- Refresh periodically and replace live subscriptions when the set changes.
|
||||
|
||||
Local-first querying is optional in this architecture. PostgreSQL already holds
|
||||
canonical kind-3 events, so the caching application may query the canonical event
|
||||
store read-only for discovery metadata before contacting upstream relays. It must
|
||||
not write canonical rows.
|
||||
|
||||
### 8.2 NIP-65 relay discovery
|
||||
|
||||
- Fetch newest kind 10002 events in bounded author batches.
|
||||
- Use unmarked or `read` relay tags; ignore `write`-only tags.
|
||||
- Normalize and deduplicate `wss://` URLs.
|
||||
- Enforce per-author and global relay caps.
|
||||
- Use the greedy covering-set logic from the standalone implementation.
|
||||
- Retain configured bootstrap relays as fallback.
|
||||
|
||||
### 8.3 Friendly live subscriptions
|
||||
|
||||
- Separate root and followed-author subscriptions when their kind lists differ.
|
||||
- Batch author lists into reasonable filter sizes.
|
||||
- Reconnect with bounded exponential backoff and jitter.
|
||||
- Use a small timestamp overlap when resubscribing.
|
||||
- Insert callback event JSON into the inbox and continue; do not wait for
|
||||
c-relay-pg processing.
|
||||
|
||||
### 8.4 Friendly backfill
|
||||
|
||||
- Use recent-first progressive windows.
|
||||
- Persist an immutable anchor for each active window.
|
||||
- Query one author at a time from that author's declared outbox relays, with
|
||||
bootstrap fallback.
|
||||
- Use configurable page size and delay between queries.
|
||||
- Apply relay-specific backoff after errors or timeouts.
|
||||
- Keep global and per-relay request rates low.
|
||||
- Advance progress only after the returned page has been inserted into the
|
||||
inbox or identified as already queued.
|
||||
- Periodically repeat a bounded widest-window sweep after initial completion.
|
||||
|
||||
## 9. Shared Configuration
|
||||
|
||||
Keep administrator intent in the existing [`config`](../src/pg_schema.sql:192)
|
||||
table with category `caching` and `requires_restart = 0`.
|
||||
|
||||
Suggested keys:
|
||||
|
||||
- `caching_enabled`;
|
||||
- `caching_config_generation`;
|
||||
- `caching_root_npubs`;
|
||||
- `caching_bootstrap_relays`;
|
||||
- `caching_kinds`;
|
||||
- `caching_admin_kinds`;
|
||||
- `caching_live_enabled`;
|
||||
- `caching_live_resubscribe_seconds`;
|
||||
- `caching_backfill_enabled`;
|
||||
- `caching_backfill_windows`;
|
||||
- `caching_backfill_page_size`;
|
||||
- `caching_backfill_tick_interval_ms`;
|
||||
- `caching_backfill_window_cooldown_seconds`;
|
||||
- `caching_follow_graph_refresh_seconds`;
|
||||
- `caching_relay_discovery_refresh_seconds`;
|
||||
- `caching_max_followed_pubkeys`;
|
||||
- `caching_max_upstream_relays`;
|
||||
- `caching_max_relays_per_pubkey`;
|
||||
- `caching_query_timeout_ms`;
|
||||
- `caching_inbox_batch_size`;
|
||||
- `caching_inbox_active_poll_ms`;
|
||||
- `caching_inbox_idle_poll_ms`;
|
||||
- `caching_max_event_json_bytes`.
|
||||
|
||||
The web UI updates these through the existing encrypted administrator API. After
|
||||
a successful update, increment `caching_config_generation`. The caching
|
||||
application polls the generation and reloads valid changes. No cross-process
|
||||
command queue is needed.
|
||||
|
||||
`caching_enabled = false` tells the external application to close upstream
|
||||
activity. systemd or the container runtime, not the web page, owns the process
|
||||
lifecycle.
|
||||
|
||||
## 10. Admin UI
|
||||
|
||||
### 10.1 Sidenav placement
|
||||
|
||||
In [`api/index.html`](../api/index.html:14), insert **Caching** immediately after
|
||||
**Relay Events** and before **DM**:
|
||||
|
||||
1. Statistics
|
||||
2. Subscriptions
|
||||
3. Configuration
|
||||
4. Authorization
|
||||
5. IP BAN
|
||||
6. Relay Events
|
||||
7. **Caching**
|
||||
8. DM
|
||||
9. Database Query
|
||||
|
||||
Register `cachingSection` in [`switchPage()`](../api/index.js:5349).
|
||||
|
||||
### 10.2 Page layout
|
||||
|
||||
Reuse the existing admin UI classes and divide the page into three blocks.
|
||||
|
||||
**Service status**
|
||||
|
||||
- enabled configuration;
|
||||
- service heartbeat and state;
|
||||
- desired and applied config generation;
|
||||
- followed author count;
|
||||
- selected and connected relay counts;
|
||||
- current backfill window and cursor;
|
||||
- fetched and inbox-inserted counters;
|
||||
- last service error.
|
||||
|
||||
**Relay inbox status**
|
||||
|
||||
- pending live/discovery count;
|
||||
- pending backfill count;
|
||||
- oldest inbox row age;
|
||||
- relay-consumed, accepted, duplicate, invalid, and failed counters;
|
||||
- last successful inbox poll.
|
||||
|
||||
**Configuration**
|
||||
|
||||
- root npubs and bootstrap relays;
|
||||
- normal and root kind lists;
|
||||
- live and backfill toggles;
|
||||
- backfill windows and pacing;
|
||||
- discovery refresh intervals;
|
||||
- author and relay safety caps;
|
||||
- inbox batch and polling settings;
|
||||
- **Apply Configuration** button;
|
||||
- **Reset Backfill Progress** button with confirmation.
|
||||
|
||||
The UI reads service state from the singleton status row and lightweight inbox
|
||||
aggregates. Poll only while the Caching page is visible and use a modest refresh
|
||||
interval.
|
||||
|
||||
## 11. Process and Database Security
|
||||
|
||||
Use separate PostgreSQL roles:
|
||||
|
||||
- **Caching service role:** read caching configuration and permitted canonical
|
||||
discovery metadata; insert/select its own status and progress; insert into the
|
||||
inbox; no insert/update/delete permission on canonical events.
|
||||
- **c-relay-pg role:** normal relay permissions plus dequeue permission on the
|
||||
inbox and read access to caching status.
|
||||
|
||||
Use parameterized SQL throughout. Restrict public upstreams to normalized
|
||||
`wss://` URLs. Apply a database statement timeout to caching queries so the
|
||||
fetcher cannot hold resources indefinitely.
|
||||
|
||||
## 12. Lifecycle
|
||||
|
||||
### 12.1 Caching service
|
||||
|
||||
- Starts and stops independently under systemd or a container runtime.
|
||||
- On startup, loads configuration and progress, updates heartbeat state, then
|
||||
begins discovery/live/backfill.
|
||||
- On shutdown, closes subscriptions, saves current progress, marks status
|
||||
stopped, and disconnects from PostgreSQL.
|
||||
- Its failure does not stop c-relay-pg; cached acquisition merely pauses.
|
||||
|
||||
### 12.2 c-relay-pg
|
||||
|
||||
- Starts the inbox poller only for PostgreSQL builds after database, writer pool,
|
||||
and WebSocket systems are ready.
|
||||
- Stops new polls before shutting down the writer pool or WebSocket context.
|
||||
- Drains already-created main-thread completions before destroying those
|
||||
dependencies.
|
||||
- A missing inbox table should be logged as caching unavailable, not terminate
|
||||
the relay, unless schema migration policy requires otherwise.
|
||||
|
||||
## 13. Implementation Sequence
|
||||
|
||||
1. Add `caching_event_inbox`, `caching_service_state`, and
|
||||
`caching_backfill_progress` to the PostgreSQL schema and embedded schema.
|
||||
2. Add PostgreSQL database abstraction functions for bounded destructive dequeue
|
||||
and lightweight inbox counts.
|
||||
3. Refactor c-relay-pg inbound event processing into a shared source-aware
|
||||
ingestion entry point without changing normal client behavior.
|
||||
4. Add the PostgreSQL-only inbox poller, bounded in-memory batch, and main-thread
|
||||
post-action/broadcast completion path.
|
||||
5. Add caching configuration defaults and validation to c-relay-pg.
|
||||
6. Adapt the standalone caching application to read PostgreSQL configuration,
|
||||
insert raw event JSON into the inbox, and update heartbeat/status.
|
||||
7. Correct the standalone backfill implementation so limited queries paginate
|
||||
and do not falsely complete an author/window.
|
||||
8. Persist simple per-author/window progress and implement polite retry/backoff.
|
||||
9. Add the Caching page after Relay Events and before DM, including service,
|
||||
inbox, configuration, and reset-progress controls.
|
||||
10. Add PostgreSQL role/grant documentation and systemd/container deployment
|
||||
examples for the separate caching process.
|
||||
11. Test live priority, duplicate acquisition, invalid inbox rows, replacement,
|
||||
deletion, ephemeral broadcast, process restarts, destructive-dequeue crash
|
||||
behavior, upstream outages, and graceful shutdown.
|
||||
12. Build and validate c-relay-pg only through
|
||||
[`make_and_restart_relay.sh`](../make_and_restart_relay.sh), using
|
||||
`--preserve-database` where test state must survive.
|
||||
|
||||
## 14. Acceptance Criteria
|
||||
|
||||
- The caching application cannot write canonical event rows.
|
||||
- Received upstream events require only minimal fetcher checks before inbox
|
||||
insertion.
|
||||
- Inbox constraints reject structurally implausible rows and deduplicate pending
|
||||
event IDs.
|
||||
- c-relay-pg removes small batches ordered live/discovery before backfill.
|
||||
- c-relay-pg remains the authoritative ID/signature validator and storage owner.
|
||||
- Imported events never execute relay administrator commands or require a client
|
||||
NIP-42 session.
|
||||
- Newly accepted events are broadcast to active subscriptions from the main
|
||||
libwebsockets thread.
|
||||
- Duplicate and stale replacement events are not rebroadcast.
|
||||
- The caching service behaves politely through bounded filters, paced backfill,
|
||||
targeted outbox queries, and retry backoff.
|
||||
- Backfill progress survives caching-service restarts and does not falsely mark
|
||||
saturated timestamps complete.
|
||||
- A caching-service failure does not stop the relay.
|
||||
- The accepted destructive-dequeue crash window is documented and recoverable
|
||||
through live overlap and repeated backfill.
|
||||
- The Caching UI appears after Relay Events and before DM and distinguishes
|
||||
external service health from relay inbox consumption.
|
||||
@@ -0,0 +1,113 @@
|
||||
# Caching Status "Not Implemented" Fix Plan
|
||||
|
||||
## Root Cause
|
||||
|
||||
The caching page shows "Service status unavailable (caching_status command not
|
||||
implemented on relay)" — but the command **is** implemented at
|
||||
[`src/config.c:4128`](../src/config.c:4128). The real problem is **admin
|
||||
authorization failure**: every admin command from the browser is rejected at
|
||||
publish time with:
|
||||
|
||||
```
|
||||
Unauthorized admin event attempt: invalid admin pubkey
|
||||
```
|
||||
|
||||
This rejection happens at [`src/main.c:2249`](../src/main.c:2249) because the
|
||||
browser's pubkey is not in the relay's `admin_pubkey` config list. The kind
|
||||
23456 event never reaches the command handler, so `caching_status` never
|
||||
executes, and the placeholder text persists.
|
||||
|
||||
### Confirmed admin key
|
||||
|
||||
- Relay admin pubkey (hex): `6a04ab98d9e4774ad806e302dddeb63bea16b5cb5f223ee77478e861bb583eb3`
|
||||
- Relay admin npub: `npub13lm5wf8dvsdnc2894pkhch9uf8phvw9varrv8zf4sc885hhdmc8q6lx7ks`
|
||||
- Source: [`.relay.laantungir.net.keys`](../.relay.laantungir.net.keys:1)
|
||||
|
||||
The browser extension (nos2x) was using pubkey `8ff74724...`, which is not an
|
||||
admin on the port 7777 relay.
|
||||
|
||||
## Secondary Issue (latent)
|
||||
|
||||
Even after auth is fixed, the caching status UI would render **empty blocks**
|
||||
due to a schema mismatch between backend and frontend:
|
||||
|
||||
| Frontend expects ([`api/index.js:7046`](../api/index.js:7046)) | Backend emits ([`src/config.c:4135`](../src/config.c:4135)) |
|
||||
|---|---|
|
||||
| `data.service.enabled` | `data.caching_enabled` |
|
||||
| `data.service.running` | *(not emitted)* |
|
||||
| `data.inbox.enabled` | `data.caching_inbox_enabled` |
|
||||
| `data.inbox.queue_depth` | `data.inbox_pending_live` + `data.inbox_pending_backfill` |
|
||||
| *(not expected)* | `data.inbox_total_dequeued`, `data.inbox_total_accepted`, `data.inbox_total_rejected`, `data.inbox_oldest_age_seconds` |
|
||||
|
||||
## Fix Steps
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Step 1: Fix admin auth] --> B[Step 2: Align response schema]
|
||||
B --> C[Step 3: Fix frontend handler]
|
||||
C --> D[Step 4: Improve error messaging]
|
||||
D --> E[Step 5: Test end-to-end]
|
||||
```
|
||||
|
||||
### Step 1 — Fix admin authorization (config, user action)
|
||||
|
||||
Load the admin private key (corresponding to `6a04ab98...` /
|
||||
`npub13lm5wf8dvsdnc2894pkhch9uf8phvw9varrv8zf4sc885hhdmc8q6lx7ks`) into the
|
||||
nos2x browser extension so admin commands authenticate. Use `nak` on the
|
||||
command line to convert/derive the nsec if needed.
|
||||
|
||||
This unblocks **all** admin commands, not just caching.
|
||||
|
||||
### Step 2 — Align backend response schema (code, [`src/config.c:4128`](../src/config.c:4128))
|
||||
|
||||
Restructure the `data` object in the `caching_status` handler to emit nested
|
||||
`service` and `inbox` objects matching the frontend handler:
|
||||
|
||||
```json
|
||||
{
|
||||
"command": "caching_status",
|
||||
"status": "success",
|
||||
"data": {
|
||||
"service": {
|
||||
"enabled": false,
|
||||
"running": false,
|
||||
"connected_relays": 0,
|
||||
"events_cached": 0
|
||||
},
|
||||
"inbox": {
|
||||
"enabled": false,
|
||||
"running": true,
|
||||
"queue_depth": 0,
|
||||
"last_poll": 0,
|
||||
"total_dequeued": 0,
|
||||
"total_accepted": 0,
|
||||
"total_rejected": 0,
|
||||
"total_duplicates": 0,
|
||||
"pending_live": 0,
|
||||
"pending_backfill": 0,
|
||||
"oldest_age_seconds": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Populate `service.running` / `connected_relays` / `events_cached` from the
|
||||
caching service launcher state if available; otherwise emit zeros/defaults.
|
||||
|
||||
### Step 3 — Update frontend handler (code, [`api/index.js:7027`](../api/index.js:7027))
|
||||
|
||||
Update `handleCachingStatusResponse()` to render the inbox poller stats the
|
||||
backend actually produces (dequeued/accepted/rejected/pending/oldest_age), and
|
||||
map `data.inbox.pending_live + pending_backfill` to queue depth.
|
||||
|
||||
### Step 4 — Replace misleading placeholder (code, [`api/index.js:6889`](../api/index.js:6889))
|
||||
|
||||
- Change the pre-send placeholder from "caching_status command not implemented"
|
||||
to a neutral "Loading..." message.
|
||||
- On auth failure, show the actual error (e.g., "Not authorized: pubkey not
|
||||
registered as admin") instead of the generic "unavailable" text.
|
||||
|
||||
### Step 5 — Test end-to-end on port 7777
|
||||
|
||||
Verify the caching page shows real service/inbox status after auth is fixed
|
||||
and the schema is aligned.
|
||||
@@ -0,0 +1,306 @@
|
||||
# Forward Catch-Up Plan: Bridging the Gap When Caching Resumes
|
||||
|
||||
## Problem
|
||||
|
||||
When caching is turned off (or the caching service stops), events posted by
|
||||
followed authors during the downtime are missed. The current backfill walks
|
||||
**backward** from `until_cursor` toward the beginning of time — it does not
|
||||
cover events **newer** than the cursor. The live subscriber uses
|
||||
`since = time(NULL)`, so it only catches events from the moment it connects.
|
||||
Events in the gap between "last event we know about" and "caching resumed"
|
||||
are lost.
|
||||
|
||||
## Schema Change: `last_event_at` column
|
||||
|
||||
Add a `last_event_at BIGINT NOT NULL DEFAULT 0` column to
|
||||
`caching_followed_pubkeys`. This records the `created_at` of the most
|
||||
recent event we've ever seen for this author — not a proxy like
|
||||
`updated_at` (which records when we last *touched* the row, which could
|
||||
be a zero-event progress write or a follow-graph refresh).
|
||||
|
||||
`since = last_event_at + 1, until = now` is exact: it catches every
|
||||
event the relay doesn't yet have, with no assumptions about when the
|
||||
last event occurred relative to when we last checked.
|
||||
|
||||
### Population
|
||||
|
||||
- **Backfill**: when publishing a page of events, compute
|
||||
`max(created_at)` across the page (we already compute
|
||||
`min(created_at)` for the cursor advance — add a parallel max).
|
||||
Update `last_event_at = GREATEST(last_event_at, page_max_created_at)`.
|
||||
- **Live subscriber**: when an event is received and published, update
|
||||
`last_event_at = GREATEST(last_event_at, event.created_at)`.
|
||||
- **One-time seed from `events` table**: on schema upgrade, set
|
||||
`last_event_at = COALESCE((SELECT MAX(created_at) FROM events WHERE
|
||||
pubkey = caching_followed_pubkeys.pubkey), 0)` for all existing rows.
|
||||
This seeds the column from the relay's own data.
|
||||
|
||||
### Migration
|
||||
|
||||
Added to `src/pg_schema.sql` and `src/pg_schema.h`:
|
||||
|
||||
```sql
|
||||
ALTER TABLE caching_followed_pubkeys
|
||||
ADD COLUMN IF NOT EXISTS last_event_at BIGINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- One-time seed from the events table.
|
||||
UPDATE caching_followed_pubkeys fp
|
||||
SET last_event_at = COALESCE(
|
||||
(SELECT MAX(e.created_at) FROM events e WHERE e.pubkey = fp.pubkey),
|
||||
0
|
||||
)
|
||||
WHERE fp.last_event_at = 0;
|
||||
```
|
||||
|
||||
No schema version bump needed — `ALTER TABLE ADD COLUMN IF NOT EXISTS`
|
||||
is idempotent and the seed `UPDATE` is guarded by `WHERE last_event_at = 0`.
|
||||
|
||||
## Current Architecture
|
||||
|
||||
### Tables
|
||||
|
||||
**`caching_followed_pubkeys`** (per-author state):
|
||||
| Column | Purpose |
|
||||
|---|---|
|
||||
| `pubkey` | PK |
|
||||
| `until_cursor` | Unix timestamp; backfill queries `until = this`, walks backward |
|
||||
| `backfill_complete` | TRUE when drained to the beginning of time |
|
||||
| `events_fetched` | Cumulative count |
|
||||
| `last_seen` | Updated on follow-graph refresh |
|
||||
| `updated_at` | Updated on every backfill progress write and completion mark |
|
||||
| `last_event_at` | **NEW**: `created_at` of the most recent event we've seen for this author |
|
||||
|
||||
**`caching_backfill_relay_progress`** (per-author-per-relay state):
|
||||
| Column | Purpose |
|
||||
|---|---|
|
||||
| `author_pubkey, relay_url` | Composite PK |
|
||||
| `until_cursor` | Per-relay backward-walk cursor |
|
||||
| `complete` | TRUE when this relay is drained for this author |
|
||||
| `updated_at` | Updated on every relay progress write |
|
||||
|
||||
### Backfill flow ([`caching/src/backfill.c`](../caching/src/backfill.c:256))
|
||||
|
||||
1. Pick next incomplete author (round-robin)
|
||||
2. For each incomplete relay for that author:
|
||||
- Query `authors=[pk], until=until_cursor, limit=page_size`
|
||||
- Publish events to the relay via the sink
|
||||
- Advance `until_cursor` to `oldest_event_created_at - 1`
|
||||
- If < page_size events + EOSE: mark relay `complete = TRUE`
|
||||
3. If all relays complete: mark author `backfill_complete = TRUE`
|
||||
|
||||
### Restart behavior
|
||||
|
||||
- **Normal restart** (no `--restart`): cursor and completion state preserved.
|
||||
Backfill resumes the backward walk from stored cursors. **Gap not covered.**
|
||||
- **`--restart`**: all cursors reset to 0, all completion flags cleared. Full
|
||||
re-drain from `now` backward. **Re-fetches everything** but covers the gap
|
||||
incidentally (since it starts from `now`).
|
||||
|
||||
### Live subscriber ([`caching/src/live_subscriber.c`](../caching/src/live_subscriber.c:86))
|
||||
|
||||
Uses `since = time(NULL)` — only catches events from the moment it connects.
|
||||
No gap-bridging.
|
||||
|
||||
## The Gap
|
||||
|
||||
```
|
||||
Time ──────────────────────────────────────────────────────►
|
||||
│ │ │
|
||||
last_event_at caching now
|
||||
(most recent event turned (caching
|
||||
we know about) off resumed)
|
||||
│
|
||||
└── events posted here are missed
|
||||
```
|
||||
|
||||
For **completed authors** (`backfill_complete = TRUE`): the backward walk is
|
||||
done. `last_event_at` tells us the most recent event we have. Events posted
|
||||
after `last_event_at` are in the gap.
|
||||
|
||||
For **incomplete authors**: the backward walk is still in progress. The cursor
|
||||
is somewhere in the past, walking backward. Events newer than the cursor are
|
||||
not fetched by backfill. The live subscriber covers events from `now` forward.
|
||||
The gap is between `last_event_at` and `now`.
|
||||
|
||||
## Solution: Forward Catch-Up Phase
|
||||
|
||||
Add a **forward catch-up** phase that runs once when the caching service
|
||||
starts (or when backfill is re-enabled), before the normal backward-drain
|
||||
backfill loop begins.
|
||||
|
||||
### Logic
|
||||
|
||||
For **every** followed author (both complete and incomplete):
|
||||
1. Read `last_event_at` from `caching_followed_pubkeys`.
|
||||
2. If `last_event_at = 0`, skip (no events known yet — the backward drain
|
||||
will handle it).
|
||||
3. Query one or two outbox relays:
|
||||
`authors=[pk], since=last_event_at + 1, until=now, limit=page_size`
|
||||
4. Publish all returned events to the relay via the sink.
|
||||
5. Update `last_event_at` to the max `created_at` seen (or `now` if no
|
||||
events were returned, to avoid re-querying the same empty window next
|
||||
time).
|
||||
|
||||
This is safe for both complete and incomplete authors:
|
||||
- **Complete authors**: the backward drain is done, so the forward catch-up
|
||||
is the only thing needed.
|
||||
- **Incomplete authors**: the backward drain walks *below* `until_cursor`,
|
||||
so events above `until_cursor` up to `last_event_at` were already fetched
|
||||
during the initial drain (when `until_cursor` started at `now`). The
|
||||
forward catch-up fills from `last_event_at + 1` to `now` — the gap that
|
||||
formed while caching was off.
|
||||
|
||||
### When to run
|
||||
|
||||
- **On caching service startup** (not `--restart`, which does a full reset).
|
||||
- The caching service process starts when `caching_enabled` is turned on,
|
||||
so this covers the "caching was off, now it's on" case.
|
||||
|
||||
### Implementation
|
||||
|
||||
#### 1. Schema: add `last_event_at` column
|
||||
|
||||
In `src/pg_schema.sql` and `src/pg_schema.h`:
|
||||
|
||||
```sql
|
||||
ALTER TABLE caching_followed_pubkeys
|
||||
ADD COLUMN IF NOT EXISTS last_event_at BIGINT NOT NULL DEFAULT 0;
|
||||
|
||||
UPDATE caching_followed_pubkeys fp
|
||||
SET last_event_at = COALESCE(
|
||||
(SELECT MAX(e.created_at) FROM events e WHERE e.pubkey = fp.pubkey),
|
||||
0
|
||||
)
|
||||
WHERE fp.last_event_at = 0;
|
||||
```
|
||||
|
||||
#### 2. New function: `pg_inbox_get_authors_for_catchup()`
|
||||
|
||||
In `caching/src/pg_inbox.c`:
|
||||
|
||||
```c
|
||||
/* Returns a cJSON array of {pubkey, last_event_at} objects for all
|
||||
* followed authors where last_event_at > 0. Caller must cJSON_Delete().
|
||||
* Returns NULL on error. */
|
||||
cJSON* pg_inbox_get_authors_for_catchup(void);
|
||||
```
|
||||
|
||||
SQL:
|
||||
```sql
|
||||
SELECT pubkey, last_event_at
|
||||
FROM caching_followed_pubkeys
|
||||
WHERE last_event_at > 0
|
||||
ORDER BY last_event_at ASC
|
||||
```
|
||||
|
||||
#### 3. New function: `pg_inbox_update_last_event_at()`
|
||||
|
||||
```c
|
||||
/* Update last_event_at for a pubkey to the max of current and new value. */
|
||||
int pg_inbox_update_last_event_at(const char *pk, long event_created_at);
|
||||
```
|
||||
|
||||
SQL:
|
||||
```sql
|
||||
UPDATE caching_followed_pubkeys
|
||||
SET last_event_at = GREATEST(last_event_at, $2::BIGINT)
|
||||
WHERE pubkey = $1
|
||||
```
|
||||
|
||||
#### 4. New function: `cr_forward_catchup()`
|
||||
|
||||
In a new file `caching/src/forward_catchup.c`:
|
||||
|
||||
```c
|
||||
/* Run forward catch-up for all followed authors.
|
||||
* For each author with last_event_at > 0, query events from
|
||||
* last_event_at + 1 to now and publish them to the sink.
|
||||
* Returns 0 on success, -1 on error. */
|
||||
int cr_forward_catchup(cr_config_t *cfg,
|
||||
nostr_relay_pool_t *upstream,
|
||||
cr_sink_t *sink);
|
||||
```
|
||||
|
||||
Flow:
|
||||
1. Call `pg_inbox_get_authors_for_catchup()` to get the list.
|
||||
2. For each author:
|
||||
a. Get the author's outbox relays from `caching_backfill_relay_progress`
|
||||
(any relay, since we just need one good source).
|
||||
b. Query `authors=[pk], since=last_event_at + 1, until=now,
|
||||
limit=page_size` on one relay.
|
||||
c. Publish all returned events to the sink.
|
||||
d. If events were returned, update `last_event_at` to the max
|
||||
`created_at` in the batch. If no events, update `last_event_at`
|
||||
to `now` (so we don't re-query the same empty window).
|
||||
3. Log: "forward catch-up: N authors checked, M events published".
|
||||
|
||||
#### 5. Update backfill to maintain `last_event_at`
|
||||
|
||||
In `caching/src/backfill.c`, in the page-publishing loop (around line 375):
|
||||
- Add a `find_newest_created_at()` helper (parallel to the existing
|
||||
`find_oldest_created_at()`).
|
||||
- After publishing a page, call
|
||||
`pg_inbox_update_last_event_at(pk, newest_created_at)`.
|
||||
|
||||
#### 6. Update live subscriber to maintain `last_event_at`
|
||||
|
||||
In `caching/src/live_subscriber.c`, in the event-received callback:
|
||||
- Extract `created_at` from the event.
|
||||
- Call `pg_inbox_update_last_event_at(pubkey, created_at)`.
|
||||
|
||||
#### 7. Call from `main.c`
|
||||
|
||||
In `caching/src/main.c`, after relay discovery and followed-set sync,
|
||||
before the main loop:
|
||||
|
||||
```c
|
||||
/* Forward catch-up: bridge the gap for all followed authors. */
|
||||
if (cfg->backfill.enabled && pg_conn && !restart) {
|
||||
DEBUG_INFO("forward catch-up: checking for missed events");
|
||||
cr_forward_catchup(&cfg, upstream, &sink);
|
||||
}
|
||||
```
|
||||
|
||||
This runs once at startup. It's not throttled — it's a one-time pass.
|
||||
|
||||
### Edge cases
|
||||
|
||||
- **`--restart` flag**: full reset already starts from `now`, so forward
|
||||
catch-up is skipped. The `last_event_at` seed from the `events` table
|
||||
will set it to the most recent known event, and the backward drain from
|
||||
`now` will cover everything.
|
||||
- **`last_event_at = 0`**: author has no known events. Skip — the backward
|
||||
drain handles it.
|
||||
- **Very large gap** (caching off for months): the forward catch-up query
|
||||
may return many events. Use `limit = page_size` and paginate if needed.
|
||||
The relay's own dedup (unique index on event ID) handles duplicates.
|
||||
- **Relay doesn't support `since`**: most Nostr relays support `since`/
|
||||
`until` (NIP-01). If ignored, the relay returns all events — dedup
|
||||
handles it.
|
||||
|
||||
### Files to change
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/pg_schema.sql` | `ALTER TABLE` add `last_event_at` + seed from `events` |
|
||||
| `src/pg_schema.h` | Mirror the above |
|
||||
| `caching/src/forward_catchup.c` | New: `cr_forward_catchup()` |
|
||||
| `caching/src/forward_catchup.h` | New: declaration |
|
||||
| `caching/src/pg_inbox.c` | New: `pg_inbox_get_authors_for_catchup()`, `pg_inbox_update_last_event_at()` |
|
||||
| `caching/src/pg_inbox.h` | New: declarations |
|
||||
| `caching/src/backfill.c` | Add `find_newest_created_at()`, call `pg_inbox_update_last_event_at()` after each page |
|
||||
| `caching/src/live_subscriber.c` | Call `pg_inbox_update_last_event_at()` on event receipt |
|
||||
| `caching/src/main.c` | Call `cr_forward_catchup()` at startup |
|
||||
| `caching/Makefile` | Add `forward_catchup.c` to sources |
|
||||
|
||||
### Sequencing
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[Caching service starts] --> B{Is --restart?}
|
||||
B -- Yes --> C[Reset all progress, full re-drain from now]
|
||||
B -- No --> D[Forward catch-up: all authors with last_event_at > 0]
|
||||
D --> E[Normal backward-drain backfill loop]
|
||||
E --> F[Live subscriber: since = now, ongoing]
|
||||
F --> G[Live subscriber updates last_event_at on each event]
|
||||
E --> H[Backfill updates last_event_at on each page]
|
||||
@@ -0,0 +1,201 @@
|
||||
# Multi-Admin Support Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Enable multiple Nostr pubkeys to act as relay administrators. Currently the relay stores a single `admin_pubkey` string in the `config` table and uses `strcmp()` equality checks throughout the codebase. This plan converts that to a comma-separated list of pubkeys stored in the same config key, with a centralized `is_admin_pubkey()` helper that all authorization gates call.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. **Minimal schema change** — keep using the existing `config` table with key `admin_pubkey`. The value changes from a single 64-char hex string to a comma-separated list of 64-char hex strings.
|
||||
2. **Primary admin concept** — the first pubkey in the list is the "primary admin" (the original one). Only the primary admin can add/remove other admins. This prevents privilege escalation.
|
||||
3. **Centralized check** — introduce one function `is_admin_pubkey(const char* pubkey)` that all 8 authorization gates call instead of inline `strcmp()`.
|
||||
4. **In-memory cache** — parse the comma-separated list once on startup/config-change into a static array for O(1)-ish lookups without repeated string parsing.
|
||||
5. **Backward compatible** — a single pubkey with no commas works identically to today.
|
||||
|
||||
## Data Model
|
||||
|
||||
```
|
||||
config table:
|
||||
key = "admin_pubkey"
|
||||
value = "hex1,hex2,hex3" (comma-separated, no spaces)
|
||||
data_type = "string"
|
||||
```
|
||||
|
||||
Maximum admins: 10 (compile-time constant `MAX_ADMIN_PUBKEYS`).
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Config Table: admin_pubkey = hex1,hex2,hex3] --> B[reload_admin_pubkeys_cache]
|
||||
B --> C[Static array: g_admin_pubkeys with count]
|
||||
D[Any authorization check] --> E[is_admin_pubkey - pubkey]
|
||||
E --> C
|
||||
F[CLI: -a flag] --> G[first_time_startup_sequence]
|
||||
G --> A
|
||||
H[Admin command: add_admin / remove_admin] --> I[modify_admin_list]
|
||||
I --> A
|
||||
I --> B
|
||||
```
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
### Step 1: Add centralized admin check infrastructure in config.c / config.h
|
||||
|
||||
**New constants and types:**
|
||||
```c
|
||||
#define MAX_ADMIN_PUBKEYS 10
|
||||
```
|
||||
|
||||
**New functions in config.h:**
|
||||
```c
|
||||
int is_admin_pubkey(const char* pubkey); // Returns 1 if pubkey is in admin list
|
||||
int get_admin_pubkey_count(void); // Returns number of admins
|
||||
const char* get_primary_admin_pubkey(void); // Returns first admin pubkey
|
||||
int reload_admin_pubkeys_cache(void); // Parse config value into cache
|
||||
int add_admin_pubkey(const char* pubkey); // Add pubkey to admin list (primary admin only)
|
||||
int remove_admin_pubkey(const char* pubkey); // Remove pubkey from admin list (primary admin only)
|
||||
```
|
||||
|
||||
**Implementation in config.c:**
|
||||
- Static array `g_admin_pubkeys[MAX_ADMIN_PUBKEYS][65]` and `g_admin_pubkey_count`
|
||||
- `reload_admin_pubkeys_cache()` reads `admin_pubkey` from config table, splits on commas, validates each is 64 hex chars, populates array
|
||||
- `is_admin_pubkey()` iterates the cached array with `strcmp()`
|
||||
- `get_primary_admin_pubkey()` returns `g_admin_pubkeys[0]` — used where only the primary admin matters (WoT sync, etc.)
|
||||
- `add_admin_pubkey()` / `remove_admin_pubkey()` modify the comma-separated string in the config table and call `reload_admin_pubkeys_cache()`
|
||||
|
||||
### Step 2: Replace all 8 authorization strcmp gates
|
||||
|
||||
Each of these currently does:
|
||||
```c
|
||||
const char* admin_pubkey = get_config_value("admin_pubkey");
|
||||
if (strcmp(sender_pubkey, admin_pubkey) == 0) { /* authorized */ }
|
||||
```
|
||||
|
||||
Replace with:
|
||||
```c
|
||||
if (is_admin_pubkey(sender_pubkey)) { /* authorized */ }
|
||||
```
|
||||
|
||||
**Files and locations:**
|
||||
|
||||
| File | Function | Line | Change |
|
||||
|------|----------|------|--------|
|
||||
| src/config.c | `process_admin_config_event()` | ~1441 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/config.c | `handle_kind_23456_unified()` | ~2727 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/main.c | `verify_admin_event()` | ~1905 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/main.c | Event storage / WoT trigger | ~1174 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/websockets.c | NIP-42 auth bypass | ~2324 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/websockets.c | NIP-17 DM admin check | ~3560 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/dm_admin.c | `process_admin_dm()` | ~503 | Replace strcmp with `is_admin_pubkey()` |
|
||||
| src/request_validator.c | Event validation bypass | ~307 | Replace strcmp with `is_admin_pubkey()` |
|
||||
|
||||
Each replacement also eliminates the `get_config_value("admin_pubkey")` call and its associated `free()`, simplifying the code.
|
||||
|
||||
### Step 3: Update startup / initialization pipeline
|
||||
|
||||
**config.c `first_time_startup_sequence()`:**
|
||||
- No change needed — it already stores a single pubkey. The comma-separated format with one entry is identical to the current format.
|
||||
|
||||
**config.c `populate_all_config_values_atomic()`:**
|
||||
- No change needed — inserts the initial admin_pubkey as a single value.
|
||||
|
||||
**config.c `add_pubkeys_to_config_table()`:**
|
||||
- No change needed — stores single pubkey on migration.
|
||||
|
||||
**config.c `apply_cli_overrides_atomic()`:**
|
||||
- No change needed — CLI override replaces the entire admin_pubkey value.
|
||||
|
||||
**main.c startup validation:**
|
||||
- Update the pre-warm validation at ~line 2438 to call `reload_admin_pubkeys_cache()` and validate that at least one admin pubkey exists, rather than checking for exactly 64 chars.
|
||||
|
||||
**Call `reload_admin_pubkeys_cache()`** after:
|
||||
- `populate_all_config_values_atomic()` completes
|
||||
- `reload_config_from_table()` completes
|
||||
- Any admin add/remove operation
|
||||
|
||||
### Step 4: Add admin management commands to kind 23456 API
|
||||
|
||||
Add two new admin commands that can be sent as encrypted kind 23456 events:
|
||||
|
||||
```json
|
||||
["add_admin", "pubkey_hex"]
|
||||
["remove_admin", "pubkey_hex"]
|
||||
["list_admins"]
|
||||
```
|
||||
|
||||
**Authorization rule:** Only the primary admin (first in list) can add/remove other admins. This is enforced in the command handler, not in the general `is_admin_pubkey()` check.
|
||||
|
||||
**Implementation location:** `handle_system_command_unified()` in config.c, alongside existing commands like `system_status`, `graceful_shutdown`, etc.
|
||||
|
||||
### Step 5: Update WoT sync to handle multiple admins
|
||||
|
||||
In `wot_sync_from_admin_kind3()`:
|
||||
- Currently queries kind 3 events from the single admin pubkey
|
||||
- Change to use `get_primary_admin_pubkey()` — WoT trust anchor should remain the primary admin
|
||||
- This is a deliberate design choice: secondary admins can manage the relay but the WoT trust graph is rooted in the primary admin's social graph
|
||||
|
||||
### Step 6: Update CLI to accept multiple admin pubkeys
|
||||
|
||||
**Option A - Comma-separated single flag (recommended):**
|
||||
```bash
|
||||
./c_relay_pg -a npub1...,npub2...,npub3...
|
||||
```
|
||||
|
||||
**Changes:**
|
||||
- `cli_options_t.admin_pubkey_override` becomes `char admin_pubkeys_override[MAX_ADMIN_PUBKEYS * 65]` (larger buffer for comma-separated list)
|
||||
- CLI parsing in main.c splits on commas, decodes each npub/hex, reassembles as comma-separated hex
|
||||
- `make_and_restart_relay.sh` `-a` flag documentation updated
|
||||
|
||||
### Step 7: Update config validation in api.c
|
||||
|
||||
In the config validation table at line ~912:
|
||||
```c
|
||||
{"admin_pubkey", "string", 0, 65},
|
||||
```
|
||||
Change max length to accommodate multiple pubkeys:
|
||||
```c
|
||||
{"admin_pubkey", "string", 0, MAX_ADMIN_PUBKEYS * 65},
|
||||
```
|
||||
|
||||
### Step 8: Update get_admin_pubkey_cached
|
||||
|
||||
`get_admin_pubkey_cached()` currently returns the raw config value. Update to return only the primary admin pubkey for backward compatibility with any callers that expect a single 64-char string.
|
||||
|
||||
### Step 9: Update test scripts
|
||||
|
||||
Update test scripts that hardcode admin keys to work with the multi-admin system:
|
||||
- `tests/sql_test.sh`
|
||||
- `tests/white_black_test.sh`
|
||||
- `tests/17_nip_test.sh`
|
||||
- `tests/run_all_tests.sh`
|
||||
- `tests/.test_keys.txt`
|
||||
|
||||
Add new test cases for:
|
||||
- Adding a second admin via kind 23456 command
|
||||
- Verifying second admin can send admin commands
|
||||
- Verifying non-admin cannot send admin commands
|
||||
- Removing a secondary admin
|
||||
- Verifying primary admin cannot be removed
|
||||
- Verifying secondary admin cannot add/remove admins
|
||||
|
||||
### Step 10: Update documentation
|
||||
|
||||
- `AGENTS.md` — update admin API event structure section
|
||||
- `docs/configuration_guide.md` — document multi-admin setup
|
||||
- `docs/user_guide.md` — document admin management commands
|
||||
- `API.md` — document new add_admin/remove_admin/list_admins commands
|
||||
- `README.md` — mention multi-admin capability
|
||||
|
||||
## Migration / Backward Compatibility
|
||||
|
||||
- Existing databases with a single `admin_pubkey` value work without any migration — a string with no commas is treated as a single-admin list
|
||||
- The `is_admin_pubkey()` function handles both formats transparently
|
||||
- No schema changes needed — the config table already stores strings of arbitrary length
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Primary admin privilege** — only the first pubkey in the list can add/remove admins, preventing secondary admins from locking out the primary
|
||||
2. **No self-removal** — the primary admin cannot remove themselves (prevents accidental lockout)
|
||||
3. **Max admin limit** — capped at 10 to prevent abuse and keep the linear scan fast
|
||||
4. **Validation** — each pubkey in the list is validated as 64 hex characters before storage
|
||||
@@ -0,0 +1,305 @@
|
||||
# PHP Admin Page for Caching Service
|
||||
|
||||
## Problem
|
||||
|
||||
The existing Nostr-based admin API (`caching_follows_status` kind-23456
|
||||
command) encrypts its JSON response with NIP-44, which has a hard 64KB
|
||||
plaintext limit. With 679+ followed pubkeys, the response exceeds this
|
||||
limit by ~2-4×, causing `Encryption result code: -15`
|
||||
(`NOSTR_ERROR_NIP44_BUFFER_TOO_SMALL`) and the UI shows
|
||||
"Failed to load".
|
||||
|
||||
Rather than paginating the NIP-44 admin command (complex, still limited),
|
||||
we build a **separate traditional admin page** using PHP + nginx that
|
||||
queries PostgreSQL directly. This bypasses the NIP-44 limit entirely,
|
||||
scales to thousands of follows, and runs in parallel with the existing
|
||||
Nostr admin API (no changes to the relay binary needed).
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Browser[Admin Browser] -->|HTTPS| Nginx
|
||||
Nginx -->|/admin/ *.php| PHP_FPM[PHP-FPM]
|
||||
Nginx -->|/ and /api| Relay[C-Relay-PG :8888]
|
||||
PHP_FPM -->|PDO| PostgreSQL[(PostgreSQL crelay)]
|
||||
Relay -->|libpq| PostgreSQL
|
||||
```
|
||||
|
||||
- **nginx** fronts everything on 443 (existing SSL setup).
|
||||
- `/` and `/api` → proxy to C-Relay-PG (existing, unchanged)
|
||||
- `/admin/` → PHP-FPM (new)
|
||||
- **PHP-FPM** runs as a pool (e.g. `www` or a dedicated `relay-admin`
|
||||
pool). PHP files live in `/opt/c-relay-pg/admin/` (or similar).
|
||||
- **PDO** connects to PostgreSQL using the same `crelay` credentials
|
||||
the relay uses. Read-only queries for display; write queries only for
|
||||
config updates (with auth guard).
|
||||
- **No relay restart needed** — this is purely nginx + PHP-FPM
|
||||
configuration + static files.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **PHP 8.x** with PDO PostgreSQL extension (`php-pgsql`)
|
||||
- **PHP-FPM** managed by systemd (or the existing nginx PHP-FPM pool)
|
||||
- **nginx** `location /admin/` block with `fastcgi_pass` to PHP-FPM
|
||||
- **Frontend**: vanilla HTML + JS (no build step), same visual style as
|
||||
the existing `api/index.html` admin page. Optionally use HTMX or
|
||||
Alpine.js for lightweight interactivity, but vanilla JS is sufficient.
|
||||
|
||||
## Authentication
|
||||
|
||||
The PHP admin page needs its own auth since it's not going through the
|
||||
Nostr admin command path. Options (in order of simplicity):
|
||||
|
||||
1. **HTTP Basic Auth** (nginx-level) — simplest, sufficient for a
|
||||
single-admin relay. Add `auth_basic` to the `/admin/` location block
|
||||
with a `.htpasswd` file.
|
||||
2. **Session-based login** — PHP login form, password hash in a config
|
||||
file, session cookie. More flexible but more code.
|
||||
3. **Nostr login (NIP-07 extension)** — the page could verify a signed
|
||||
auth event from the admin's browser extension, matching against the
|
||||
`admin_pubkey` in the config table. Most aligned with the existing
|
||||
system but most complex.
|
||||
|
||||
**Recommendation:** Start with HTTP Basic Auth (option 1) for the MVP,
|
||||
upgrade to Nostr login later if desired.
|
||||
|
||||
## Database Access
|
||||
|
||||
PHP connects via PDO with the same credentials the relay uses:
|
||||
```
|
||||
host=localhost port=5432 dbname=crelay user=crelay password=crelay
|
||||
```
|
||||
|
||||
Store the connection string in a PHP config file
|
||||
(`/opt/c-relay-pg/admin/config.php`) that is NOT web-accessible (outside
|
||||
the document root or protected by nginx).
|
||||
|
||||
## Pages / Endpoints
|
||||
|
||||
### 1. Dashboard (`/admin/index.php`)
|
||||
Overview cards:
|
||||
- Service state, heartbeat age, config generation (from
|
||||
`caching_service_state`)
|
||||
- Followed author count, selected relay count, connected relay count
|
||||
- Events fetched, inbox inserts, inbox pending count
|
||||
- Backfill progress: X/Y authors complete
|
||||
- Active backfill target (pubkey + relay) from `caching_backfill_active`
|
||||
- Error relays summary (count of rows with `consecutive_errors >= 3`)
|
||||
|
||||
### 2. Follows (`/admin/follows.php`)
|
||||
The main table that was failing via NIP-44. Paginated server-side:
|
||||
- Page size: 50 or 100 follows per page
|
||||
- Columns: Name (from kind-0), npub, Root?, Events in DB, Backfill
|
||||
Complete, Relays (expandable), Last Seen
|
||||
- JOIN with `events` table for kind-0 profile metadata (name,
|
||||
display_name, picture, nip05)
|
||||
- JOIN with `caching_backfill_relay_progress` for per-relay status
|
||||
- Filter: all / incomplete only / root only / error relays only
|
||||
- Search by name or pubkey prefix
|
||||
- Sort by: events_fetched desc, name, last_seen, backfill_complete
|
||||
|
||||
SQL (paginated):
|
||||
```sql
|
||||
SELECT fp.pubkey, fp.is_root, fp.backfill_complete, fp.events_fetched,
|
||||
fp.first_seen, fp.last_seen,
|
||||
e.content::json->>'name' AS name,
|
||||
e.content::json->>'display_name' AS display_name,
|
||||
e.content::json->>'picture' AS picture,
|
||||
e.content::json->>'nip05' AS nip05,
|
||||
(SELECT COUNT(*) FROM events WHERE pubkey = fp.pubkey) AS total_events
|
||||
FROM caching_followed_pubkeys fp
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT content FROM events
|
||||
WHERE pubkey = fp.pubkey AND kind = 0
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
) e ON true
|
||||
ORDER BY fp.is_root DESC, fp.events_fetched DESC
|
||||
LIMIT $page_size OFFSET $offset;
|
||||
```
|
||||
|
||||
### 3. Relay Progress (`/admin/relays.php`)
|
||||
Per (author, relay) backfill status from
|
||||
`caching_backfill_relay_progress`:
|
||||
- Columns: Author (name + npub), Relay URL, Until Cursor, Complete,
|
||||
Events Fetched, Consecutive Errors, Last Status, Updated At
|
||||
- Filter: incomplete only / error relays / complete / all
|
||||
- Highlight rows with `consecutive_errors >= 3`
|
||||
- Show `last_status` (now descriptive: `error: tls/ws handshake failed`,
|
||||
`eose`, `timeout`, etc.)
|
||||
|
||||
### 4. Config (`/admin/config.php`)
|
||||
Read and edit caching config values from the `config` table:
|
||||
- Read: `caching_root_npubs`, `caching_bootstrap_relays`,
|
||||
`caching_kinds`, `caching_admin_kinds`, all `caching_*` keys
|
||||
- Edit: form to update values, bumps `caching_config_generation` on
|
||||
save (so the caching service hot-reloads)
|
||||
- This replaces the "Apply Configuration" button for caching settings
|
||||
|
||||
### 5. Inbox (`/admin/inbox.php`)
|
||||
Monitor the `caching_event_inbox` queue:
|
||||
- Pending count by source_class (live / backfill / discovery)
|
||||
- Oldest pending age
|
||||
- Recent events preview (last 20, with kind + pubkey + source)
|
||||
- This helps diagnose if the poller is keeping up
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
/opt/c-relay-pg/admin/ # PHP document root (or symlink)
|
||||
├── config.php # DB connection config (NOT web-served)
|
||||
├── index.php # Dashboard
|
||||
├── follows.php # Followed pubkeys table (paginated)
|
||||
├── relays.php # Per-relay backfill progress
|
||||
├── config.php # Caching config editor
|
||||
├── inbox.php # Inbox queue monitor
|
||||
├── api/
|
||||
│ ├── follows.php # JSON endpoint for follows (AJAX)
|
||||
│ ├── relays.php # JSON endpoint for relay progress
|
||||
│ ├── status.php # JSON endpoint for dashboard stats
|
||||
│ └── config.php # JSON endpoint for config get/set
|
||||
├── assets/
|
||||
│ ├── style.css # Shared CSS (match existing admin theme)
|
||||
│ └── app.js # Shared JS (pagination, search, refresh)
|
||||
└── .htpasswd # HTTP Basic Auth passwords
|
||||
```
|
||||
|
||||
In the repo, these live under `admin/` (new top-level directory,
|
||||
parallel to `api/`).
|
||||
|
||||
## nginx Configuration
|
||||
|
||||
Add to the existing `server` block in
|
||||
[`examples/deployment/nginx-proxy/nginx.conf`](../examples/deployment/nginx-proxy/nginx.conf):
|
||||
|
||||
```nginx
|
||||
# PHP admin page for caching service (direct PostgreSQL access)
|
||||
location /admin/ {
|
||||
root /opt/c-relay-pg;
|
||||
index index.php;
|
||||
|
||||
# HTTP Basic Auth
|
||||
auth_basic "Relay Admin";
|
||||
auth_basic_user_file /opt/c-relay-pg/admin/.htpasswd;
|
||||
|
||||
# Pass .php files to PHP-FPM
|
||||
location ~ \.php$ {
|
||||
fastcgi_pass unix:/run/php/php8.2-fpm.sock;
|
||||
fastcgi_index index.php;
|
||||
include fastcgi_params;
|
||||
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
|
||||
}
|
||||
|
||||
# Deny access to config.php (contains DB credentials)
|
||||
location = /admin/config.php {
|
||||
deny all;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Step 1 — Server setup (one-time)
|
||||
- Install PHP 8.x + PHP-FPM + `php-pgsql` on the relay server
|
||||
- Create `/opt/c-relay-pg/admin/` directory
|
||||
- Configure PHP-FPM pool (or use default `www` pool)
|
||||
- Add the nginx `location /admin/` block
|
||||
- Create `.htpasswd` with `htpasswd` / `openssl passwd`
|
||||
- Reload nginx
|
||||
|
||||
### Step 2 — PHP config + DB connection helper
|
||||
- `admin/lib/db.php` — PDO connection helper, read-only by default
|
||||
- `admin/config.php` — connection string (protected from web access)
|
||||
- Test: `php -r "echo 'ok';"` + simple `SELECT 1` query
|
||||
|
||||
### Step 3 — Dashboard page (`index.php`)
|
||||
- Query `caching_service_state` for overview stats
|
||||
- Query `caching_backfill_active` for active target
|
||||
- Query `caching_backfill_relay_progress` for error summary
|
||||
- Display as cards + summary table
|
||||
- Auto-refresh every 15s via JS `setInterval` + `fetch()`
|
||||
|
||||
### Step 4 — Follows page (`follows.php`)
|
||||
- Server-side pagination (page param in URL)
|
||||
- JOIN with `events` for kind-0 profile metadata
|
||||
- Sub-query for per-pubkey event count
|
||||
- Expandable relay progress per follow (AJAX load on click)
|
||||
- Search + filter controls
|
||||
- This is the page that replaces the broken NIP-44 `caching_follows_status`
|
||||
|
||||
### Step 5 — Relay progress page (`relays.php`)
|
||||
- Full `caching_backfill_relay_progress` table with pagination
|
||||
- Filter by complete/incomplete/error
|
||||
- Show `last_status` and `consecutive_errors` columns
|
||||
- Color-code: green=eose, yellow=timeout, red=error, gray=complete
|
||||
|
||||
### Step 6 — Config editor page (`config.php` → `admin/config-edit.php`)
|
||||
- Read all `caching_*` config keys
|
||||
- Form to edit `caching_root_npubs`, `caching_bootstrap_relays`,
|
||||
`caching_kinds`, etc.
|
||||
- On save: `UPDATE config SET value = $1 WHERE key = $2` + bump
|
||||
`caching_config_generation`
|
||||
- Confirmation dialog before applying
|
||||
|
||||
### Step 7 — Inbox monitor page (`inbox.php`)
|
||||
- Pending counts by source_class
|
||||
- Oldest pending age
|
||||
- Recent 20 events preview
|
||||
- Poller stats (from relay log or a new stats table)
|
||||
|
||||
### Step 8 — Styling + polish
|
||||
- Match the existing `api/index.css` dark theme
|
||||
- Responsive layout (works on mobile)
|
||||
- Auto-refresh for dashboard and follows pages
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **HTTP Basic Auth** on `/admin/` — minimum barrier
|
||||
2. **`config.php` denied** via nginx `location` block (contains DB password)
|
||||
3. **PDO prepared statements** everywhere — no SQL injection
|
||||
4. **Read-only DB user** for display pages (optional: create a
|
||||
`crelay_readonly` role with SELECT-only on caching tables; use the
|
||||
full `crelay` user only for the config editor)
|
||||
5. **HTTPS only** — the existing nginx SSL config covers this
|
||||
6. **No CORS needed** — same origin as the relay
|
||||
|
||||
## Out of Scope (for MVP)
|
||||
|
||||
- Nostr NIP-07 extension login (use HTTP Basic Auth first)
|
||||
- Write operations beyond config editing (reset backfill, etc. — those
|
||||
stay on the Nostr admin API)
|
||||
- Real-time WebSocket updates (use polling instead — 15s interval is
|
||||
sufficient for a monitoring dashboard)
|
||||
- Multi-user / role-based access
|
||||
|
||||
## Files to Create
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `admin/config.php` | DB connection config (protected) |
|
||||
| `admin/lib/db.php` | PDO connection helper |
|
||||
| `admin/lib/helpers.php` | Shared helpers (pagination, formatting) |
|
||||
| `admin/index.php` | Dashboard |
|
||||
| `admin/follows.php` | Followed pubkeys table (paginated) |
|
||||
| `admin/relays.php` | Per-relay backfill progress |
|
||||
| `admin/config-edit.php` | Caching config editor |
|
||||
| `admin/inbox.php` | Inbox queue monitor |
|
||||
| `admin/api/status.php` | JSON: dashboard stats |
|
||||
| `admin/api/follows.php` | JSON: paginated follows |
|
||||
| `admin/api/relays.php` | JSON: relay progress |
|
||||
| `admin/assets/style.css` | Shared CSS |
|
||||
| `admin/assets/app.js` | Shared JS |
|
||||
| `admin/.htpasswd` | HTTP Basic Auth passwords |
|
||||
| `examples/deployment/nginx-proxy/nginx.conf` | Add `/admin/` location block |
|
||||
|
||||
## Deployment
|
||||
|
||||
1. Copy `admin/` to `/opt/c-relay-pg/admin/` on the server
|
||||
2. Install `php8.2-fpm php8.2-pgsql` (or distro-equivalent)
|
||||
3. Create `.htpasswd`: `htpasswd -c /opt/c-relay-pg/admin/.htpasswd admin`
|
||||
4. Edit `admin/config.php` with the PostgreSQL connection string
|
||||
5. Add the nginx `/admin/` location block, reload nginx
|
||||
6. Visit `https://relay.yourdomain.com/admin/`
|
||||
|
||||
No relay restart required. The PHP admin page runs entirely in parallel
|
||||
with the existing relay and Nostr admin API.
|
||||
@@ -0,0 +1,708 @@
|
||||
# Profile (kind-0) Cache Plan
|
||||
|
||||
Make username/profile resolution a first-class c-relay-pg feature backed by a
|
||||
dedicated `profiles` table, replacing the three independent ad-hoc
|
||||
implementations that exist today. Phase 1 (name/metadata cache) is scoped for
|
||||
implementation now; Phase 2 (image caching) is designed here but deferred.
|
||||
|
||||
## 0. Scope Boundary
|
||||
|
||||
**In scope:** `src/` (schema + C data layer) and `admin/` (the PHP admin).
|
||||
|
||||
**Out of scope:** the top-level [`api/`](../api/index.js) directory — the legacy
|
||||
embedded JS admin compiled into the binary via
|
||||
[`src/embedded_web_content.h`](../src/embedded_web_content.h) and served by
|
||||
[`handle_embedded_file_request()`](../src/api.c:1006). It is being replaced by
|
||||
`admin/` and is deliberately left untouched.
|
||||
|
||||
Two consequences worth being explicit about:
|
||||
|
||||
1. **Do not "fix" the duplicated logic in `api/index.js`.** It carries its own
|
||||
copies of the profile-name preference ([`api/index.js:962`](../api/index.js:962))
|
||||
and profile-picture handling ([`api/index.js:986`](../api/index.js:986)), plus
|
||||
the same class of unescaped-`innerHTML` issue described in §2A.2. These are
|
||||
knowingly left as-is because the whole tree is slated for removal. Note this
|
||||
means the XSS exposure persists for as long as the embedded UI remains
|
||||
reachable on the relay's HTTP port — a reason to prioritize retiring `api/`,
|
||||
tracked separately from this plan.
|
||||
2. **The C-side changes still matter to both.** [`src/api.c`](../src/api.c) serves
|
||||
the JSON that the embedded UI consumes, so the batching work in §2.5 benefits
|
||||
`api/` incidentally. The C API must therefore stay backward-compatible: keep
|
||||
emitting the existing `name` field (now the resolved value) alongside the new
|
||||
`display_name` / `best_name` fields, so the legacy frontend keeps working
|
||||
unchanged until it is deleted.
|
||||
|
||||
---
|
||||
|
||||
## 1. Current State
|
||||
|
||||
Profile display-name resolution was introduced alongside the caching service and
|
||||
never generalized. There are **three separate implementations**, none cached:
|
||||
|
||||
### 1.1 C backend — per-pubkey query inside a loop
|
||||
|
||||
[`postgres_db_get_profile_metadata()`](../src/db_ops_postgres.c:2523) runs:
|
||||
|
||||
```sql
|
||||
SELECT content FROM events WHERE pubkey = $1 AND kind = 0
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
```
|
||||
|
||||
then `cJSON_Parse`es `content` and copies out eight known fields
|
||||
(`name`, `display_name`, `picture`, `about`, `nip05`, `website`, `lud16`, `lud06`).
|
||||
|
||||
Dispatched through [`db_get_profile_metadata()`](../src/db_ops.c:189); the SQLite
|
||||
backend is a `NULL` stub ([`src/db_ops.c:389`](../src/db_ops.c:389)).
|
||||
|
||||
Called from three places, **always inside a row loop** — a classic N+1:
|
||||
|
||||
| Call site | Loop over | Queries per response |
|
||||
|---|---|---|
|
||||
| [`query_top_pubkeys()`](../src/api.c:503) | top 10 pubkeys | 10 |
|
||||
| [`src/api.c:1646`](../src/api.c:1646) | top pubkeys (2nd copy) | 10 |
|
||||
| caching follows status [`src/config.c:4298`](../src/config.c:4298) | every followed pubkey | 1 per follow (unbounded) |
|
||||
|
||||
The `config.c` loop additionally issues a per-pubkey kind-count query
|
||||
([`src/config.c:4326`](../src/config.c:4326)), an outbox lookup
|
||||
([`src/config.c:4346`](../src/config.c:4346)) and a relay-progress query
|
||||
([`src/config.c:4356`](../src/config.c:4356)) — so a relay following 500 authors
|
||||
performs ~2000 queries to render one admin panel.
|
||||
|
||||
The `display_name || name` preference logic is **duplicated verbatim** at all
|
||||
three call sites.
|
||||
|
||||
### 1.2 PHP admin — repeated LATERAL joins
|
||||
|
||||
[`admin/api/stats.php:90`](../admin/api/stats.php:90) and
|
||||
[`admin/api/caching.php:23`](../admin/api/caching.php:23) each hand-roll:
|
||||
|
||||
```sql
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT content FROM events WHERE pubkey = e.pubkey AND kind = 0
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
) p ON true
|
||||
... p.content::json->>'name', p.content::json->>'display_name'
|
||||
```
|
||||
|
||||
with the same `$display_name ?: $name` fallback repeated in PHP. `content::json`
|
||||
re-parses the JSON text on every single admin page poll. The cast is also
|
||||
fragile: a malformed kind-0 `content` raises a PostgreSQL error that aborts the
|
||||
whole query (the `try/catch` then silently returns an empty result set).
|
||||
|
||||
### 1.3 Browser JS — fetches from public relays
|
||||
|
||||
[`loadUserProfile()`](../admin/assets/app.js:626) opens WebSocket connections to
|
||||
**third-party public relays** to fetch the logged-in admin's own kind-0, even
|
||||
though the relay's own database very likely has it. A third variant of the
|
||||
name-preference logic lives at [`admin/assets/app.js:650`](../admin/assets/app.js:650)
|
||||
(`profile.name || profile.display_name || profile.displayName`) — note this one
|
||||
prefers `name` over `display_name`, the **opposite** of the C and PHP versions,
|
||||
so the same user can render under two different names in one UI.
|
||||
|
||||
Profile images are hotlinked straight to whatever URL the kind-0 contains
|
||||
([`admin/assets/app.js:653`](../admin/assets/app.js:653)), which leaks the admin's
|
||||
IP to arbitrary hosts and breaks silently on dead links.
|
||||
|
||||
### 1.4 Conclusion
|
||||
|
||||
A cache table is clearly warranted:
|
||||
|
||||
- Kind 0 is **replaceable** — the unique index
|
||||
[`uq_events_replaceable_pubkey_kind`](../src/pg_schema.sql:82) guarantees at
|
||||
most one kind-0 row per pubkey. A `profiles` table is therefore a strict 1:1
|
||||
projection of existing data and can be rebuilt from scratch at any time. No
|
||||
risk of divergence-by-design.
|
||||
- Profiles change rarely but are read constantly.
|
||||
- Parsing JSON at write time (once per profile update) instead of read time
|
||||
(every page poll × every row) is a large, cheap win.
|
||||
- One canonical name-preference rule fixes the inconsistency across the three
|
||||
layers.
|
||||
|
||||
---
|
||||
|
||||
## 2. Phase 1 — `profiles` Table
|
||||
|
||||
### 2.1 Schema
|
||||
|
||||
Added to [`src/pg_schema.sql`](../src/pg_schema.sql) before the `COMMIT;` at
|
||||
[line 405](../src/pg_schema.sql:405), and mirrored into
|
||||
[`src/pg_schema.h`](../src/pg_schema.h) as escaped C string literals.
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS profiles (
|
||||
pubkey TEXT PRIMARY KEY,
|
||||
event_id TEXT NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
display_name TEXT NOT NULL DEFAULT '',
|
||||
about TEXT NOT NULL DEFAULT '',
|
||||
picture TEXT NOT NULL DEFAULT '',
|
||||
banner TEXT NOT NULL DEFAULT '',
|
||||
nip05 TEXT NOT NULL DEFAULT '',
|
||||
website TEXT NOT NULL DEFAULT '',
|
||||
lud16 TEXT NOT NULL DEFAULT '',
|
||||
lud06 TEXT NOT NULL DEFAULT '',
|
||||
raw_content TEXT NOT NULL DEFAULT '',
|
||||
parse_ok BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
updated_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_profiles_name ON profiles(name)
|
||||
WHERE name <> '';
|
||||
CREATE INDEX IF NOT EXISTS idx_profiles_display_name ON profiles(display_name)
|
||||
WHERE display_name <> '';
|
||||
CREATE INDEX IF NOT EXISTS idx_profiles_nip05 ON profiles(nip05)
|
||||
WHERE nip05 <> '';
|
||||
```
|
||||
|
||||
Notes:
|
||||
- **`name` and `display_name` are both stored verbatim, always.** Storing both
|
||||
is free (they are short strings on a table with one row per pubkey), and it
|
||||
means the question "which field do Nostr clients actually populate?" can be
|
||||
answered from real data later rather than guessed at now — see
|
||||
[§2.9](#29-which-field-do-people-actually-use). Neither field is ever
|
||||
discarded, overwritten by the other, or collapsed into a single value at write
|
||||
time.
|
||||
- There is deliberately **no generated `best_name` column.** An earlier draft
|
||||
had one; it was wrong. A `STORED` generated column freezes the preference rule
|
||||
into the schema, so changing which field is displayed would require a schema
|
||||
migration and a full-table rewrite. Display preference is a presentation
|
||||
decision and belongs at read time.
|
||||
- `raw_content` keeps the original JSON so non-standard fields (including
|
||||
`displayName`, the camelCase variant some clients emit — see
|
||||
[`admin/assets/app.js:650`](../admin/assets/app.js:650)) remain reachable
|
||||
without re-querying `events`.
|
||||
- `parse_ok = FALSE` records "we saw a kind-0 but its content was not valid
|
||||
JSON" — distinct from "no profile at all" (row absent). This makes the
|
||||
malformed-JSON case explicit instead of an aborted query.
|
||||
- Empty-string defaults rather than `NULL` keep the C accessors branch-free.
|
||||
|
||||
### 2.1.1 Display preference as configuration
|
||||
|
||||
The preference rule lives in the existing `config` table
|
||||
([`src/pg_schema.sql:192`](../src/pg_schema.sql:192)) so it can be changed at
|
||||
runtime through the normal admin config path, with no migration:
|
||||
|
||||
```sql
|
||||
INSERT INTO config (key, value, data_type, description, category, requires_restart)
|
||||
VALUES ('profile_name_preference', 'display_name',
|
||||
'string', 'Which kind-0 field to prefer for display: display_name or name',
|
||||
'display', 0)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
```
|
||||
|
||||
Valid values: `display_name` (prefer `display_name`, fall back to `name`) or
|
||||
`name` (the reverse). Default `display_name`, matching the current C and PHP
|
||||
behaviour so nothing visibly changes on upgrade.
|
||||
|
||||
Each layer gets **one** resolver that reads this key — replacing the four
|
||||
scattered inline copies with one function per layer, while keeping the choice
|
||||
adjustable:
|
||||
|
||||
```c
|
||||
// Applies profile_name_preference; falls back to the other field when the
|
||||
// preferred one is empty. Returns "" when neither is set (never NULL).
|
||||
const char* profile_display_name(const cJSON* profile);
|
||||
```
|
||||
|
||||
Every profile object returned to a UI carries `name`, `display_name`, **and** the
|
||||
resolved `best_name`, so a consumer can render the resolved label while still
|
||||
having both raw values available.
|
||||
|
||||
### 2.2 Population — PostgreSQL trigger
|
||||
|
||||
A trigger keeps the table correct regardless of which process writes the event
|
||||
(relay ingest, the caching inbox poller, or a manual `psql` insert), so no
|
||||
writer can bypass it.
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE FUNCTION sync_profile_from_event() RETURNS TRIGGER AS $$
|
||||
DECLARE
|
||||
j JSONB;
|
||||
BEGIN
|
||||
IF NEW.kind <> 0 THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
j := NEW.content::jsonb;
|
||||
IF jsonb_typeof(j) <> 'object' THEN
|
||||
j := NULL;
|
||||
END IF;
|
||||
EXCEPTION WHEN others THEN
|
||||
j := NULL;
|
||||
END;
|
||||
|
||||
INSERT INTO profiles (
|
||||
pubkey, event_id, created_at,
|
||||
name, display_name, about, picture, banner,
|
||||
nip05, website, lud16, lud06,
|
||||
raw_content, parse_ok, updated_at
|
||||
) VALUES (
|
||||
NEW.pubkey, NEW.id, NEW.created_at,
|
||||
COALESCE(j->>'name',''),
|
||||
COALESCE(j->>'display_name',''),
|
||||
COALESCE(j->>'about',''),
|
||||
COALESCE(j->>'picture',''),
|
||||
COALESCE(j->>'banner',''),
|
||||
COALESCE(j->>'nip05',''),
|
||||
COALESCE(j->>'website',''),
|
||||
COALESCE(j->>'lud16',''),
|
||||
COALESCE(j->>'lud06',''),
|
||||
NEW.content, (j IS NOT NULL),
|
||||
EXTRACT(EPOCH FROM NOW())::BIGINT
|
||||
)
|
||||
ON CONFLICT (pubkey) DO UPDATE SET
|
||||
event_id = EXCLUDED.event_id,
|
||||
created_at = EXCLUDED.created_at,
|
||||
name = EXCLUDED.name,
|
||||
display_name = EXCLUDED.display_name,
|
||||
about = EXCLUDED.about,
|
||||
picture = EXCLUDED.picture,
|
||||
banner = EXCLUDED.banner,
|
||||
nip05 = EXCLUDED.nip05,
|
||||
website = EXCLUDED.website,
|
||||
lud16 = EXCLUDED.lud16,
|
||||
lud06 = EXCLUDED.lud06,
|
||||
raw_content = EXCLUDED.raw_content,
|
||||
parse_ok = EXCLUDED.parse_ok,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
-- Never let an older kind-0 overwrite a newer one.
|
||||
WHERE EXCLUDED.created_at >= profiles.created_at;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_events_sync_profile ON events;
|
||||
CREATE TRIGGER trg_events_sync_profile
|
||||
AFTER INSERT OR UPDATE OF content ON events
|
||||
FOR EACH ROW EXECUTE FUNCTION sync_profile_from_event();
|
||||
```
|
||||
|
||||
The `NEW.kind <> 0` early return means the cost for the 99.9% of events that are
|
||||
not profiles is one integer comparison — negligible next to the two triggers
|
||||
already firing on every insert
|
||||
([`trg_events_set_derived_fields`](../src/pg_schema.sql:160),
|
||||
[`trg_events_sync_event_tags`](../src/pg_schema.sql:166),
|
||||
[`trg_notify_event_stored`](../src/pg_schema.sql:283)).
|
||||
|
||||
**Deletion:** add a companion `AFTER DELETE` trigger removing the `profiles` row
|
||||
when its backing kind-0 is deleted (NIP-09 via [`src/nip009.c`](../src/nip009.c)),
|
||||
guarded on `OLD.kind = 0 AND profiles.event_id = OLD.id` so a delete of a
|
||||
superseded event does not drop a current profile.
|
||||
|
||||
### 2.3 One-time backfill
|
||||
|
||||
Existing databases already hold kind-0 events. Following the established
|
||||
guarded-migration pattern used for `d_tag_value`
|
||||
([`src/pg_schema.sql:57`](../src/pg_schema.sql:57)), the backfill runs only on
|
||||
the version transition, not on every boot:
|
||||
|
||||
```sql
|
||||
DO $$
|
||||
BEGIN
|
||||
IF COALESCE((SELECT value FROM schema_info WHERE key = 'version'), '0') < '6' THEN
|
||||
INSERT INTO profiles (pubkey, event_id, created_at, name, display_name,
|
||||
about, picture, banner, nip05, website, lud16, lud06,
|
||||
raw_content, parse_ok)
|
||||
SELECT e.pubkey, e.id, e.created_at,
|
||||
COALESCE(c.j->>'name',''), COALESCE(c.j->>'display_name',''),
|
||||
COALESCE(c.j->>'about',''), COALESCE(c.j->>'picture',''),
|
||||
COALESCE(c.j->>'banner',''), COALESCE(c.j->>'nip05',''),
|
||||
COALESCE(c.j->>'website',''), COALESCE(c.j->>'lud16',''),
|
||||
COALESCE(c.j->>'lud06',''),
|
||||
e.content, (c.j IS NOT NULL)
|
||||
FROM events e
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT CASE WHEN e.content ~ '^\s*\{' THEN
|
||||
(SELECT x FROM jsonb(e.content::jsonb) AS x)
|
||||
END AS j
|
||||
) c ON true
|
||||
WHERE e.kind = 0
|
||||
ON CONFLICT (pubkey) DO NOTHING;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
```
|
||||
|
||||
The `::jsonb` cast can still raise on malformed content. Implementation should
|
||||
use a small `PL/pgSQL` loop with a per-row `EXCEPTION` block, or a
|
||||
`safe_jsonb(text)` helper function marked `IMMUTABLE` that returns `NULL` on
|
||||
parse failure — cleaner and reusable by the trigger too. Prefer the
|
||||
`safe_jsonb()` helper and use it in both the trigger and the backfill.
|
||||
|
||||
Bump `EMBEDDED_PG_SCHEMA_VERSION` from `"5"` to `"6"` in
|
||||
[`src/pg_schema.h:4`](../src/pg_schema.h:4) and the `schema_info` insert at
|
||||
[`src/pg_schema.sql:260`](../src/pg_schema.sql:260).
|
||||
|
||||
`postgres_db_apply_schema()` ([`src/db_ops_postgres.c:138`](../src/db_ops_postgres.c:138))
|
||||
runs the whole embedded script at startup, so existing deployments upgrade
|
||||
automatically. There is no generator script for `pg_schema.h` — it is a
|
||||
hand-maintained mirror, so **both files must be edited and kept identical**.
|
||||
|
||||
### 2.4 C API
|
||||
|
||||
Replace the single-row helper with a batch-capable pair in
|
||||
[`src/db_ops.h`](../src/db_ops.h:144):
|
||||
|
||||
```c
|
||||
// Single profile from the profiles cache. NULL if no profile is cached.
|
||||
// Result object always contains "name", "display_name" (each possibly "")
|
||||
// and the resolved "best_name". Caller must cJSON_Delete().
|
||||
cJSON* db_get_profile(const char* pubkey);
|
||||
|
||||
// Batch lookup: one query for many pubkeys. Returns an object keyed by
|
||||
// pubkey hex -> profile object. Pubkeys with no cached profile are absent.
|
||||
// Caller must cJSON_Delete().
|
||||
cJSON* db_get_profiles(const char** pubkeys, int count);
|
||||
```
|
||||
|
||||
`db_get_profiles()` issues a single `WHERE pubkey = ANY($1::text[])` query,
|
||||
collapsing the N+1 loops into one round trip.
|
||||
|
||||
`db_get_profile_metadata()` is retained as a deprecated thin wrapper over
|
||||
`db_get_profile()` so nothing breaks mid-refactor, then removed once all call
|
||||
sites are migrated.
|
||||
|
||||
SQLite stubs in [`src/db_ops.c:389`](../src/db_ops.c:389) continue returning
|
||||
`NULL` — the `profiles` table is PostgreSQL-only, consistent with how the
|
||||
caching tables are handled.
|
||||
|
||||
### 2.5 C call-site migration
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| [`src/api.c:502`](../src/api.c:502) | Collect the 10 pubkeys, one `db_get_profiles()` call, then attach `name`/`display_name`/`best_name`/`picture` from the map. Replace the inline preference logic with `profile_display_name()`. |
|
||||
| [`src/api.c:1645`](../src/api.c:1645) | Same. Consider factoring the two near-identical blocks into one shared `api_attach_profile_fields()` helper. |
|
||||
| [`src/config.c:4297`](../src/config.c:4297) | Batch all followed pubkeys up front (they are already fully enumerated by the outer query) and look them up from the returned map inside the loop. |
|
||||
|
||||
The `config.c` loop's other per-row queries (kind counts, relay progress) are
|
||||
out of scope here but are noted as the next optimization target — they can
|
||||
become two `GROUP BY` queries executed once.
|
||||
|
||||
### 2.6 PHP migration
|
||||
|
||||
Add one helper to [`admin/lib/helpers.php`](../admin/lib/helpers.php):
|
||||
|
||||
```php
|
||||
/**
|
||||
* Batch-resolve profiles from the cache.
|
||||
* Returns [pubkey_hex => ['name'=>..., 'display_name'=>...,
|
||||
* 'best_name'=>..., 'picture'=>..., 'nip05'=>...]].
|
||||
*/
|
||||
function profile_map(array $pubkeys): array
|
||||
|
||||
/** Applies the profile_name_preference config key. Never returns null. */
|
||||
function profile_display_name(array $profile): string
|
||||
```
|
||||
|
||||
`profile_map()` is a single parameterized `WHERE pubkey = ANY(...)` query against
|
||||
`profiles`, returning both raw name fields plus the resolved label. Then:
|
||||
|
||||
- [`admin/api/stats.php:90`](../admin/api/stats.php:90) — drop the
|
||||
`LEFT JOIN LATERAL` and the `content::json` casts; the top-pubkeys query
|
||||
becomes a plain `GROUP BY e.pubkey`, and names come from `profile_map()`.
|
||||
This also removes the `GROUP BY e.pubkey, p.content` grouping-by-a-JSON-blob
|
||||
wart.
|
||||
- [`admin/api/caching.php:23`](../admin/api/caching.php:23) — same; or simply
|
||||
`LEFT JOIN profiles p ON p.pubkey = fp.pubkey` and select `p.name,
|
||||
p.display_name`, which is a cheap indexed join now that no subquery or parsing
|
||||
is involved.
|
||||
- Replace both copies of the `$display_name ?: $name` fallback with
|
||||
`profile_display_name()`.
|
||||
|
||||
### 2.7 JS migration
|
||||
|
||||
- Add a read-only admin endpoint (`admin/api/profile.php?pubkey=...`) returning
|
||||
the cached profile.
|
||||
- [`loadUserProfile()`](../admin/assets/app.js:626) tries that endpoint first and
|
||||
only falls back to public relays if the relay has no cached kind-0 for the
|
||||
logged-in admin (a real possibility for a fresh relay), then keeps the existing
|
||||
render path.
|
||||
- Consume the server-provided `best_name` instead of re-deriving a preference in
|
||||
the browser, so all three layers finally agree and the JS copy at
|
||||
[`admin/assets/app.js:650`](../admin/assets/app.js:650) — which currently
|
||||
prefers `name`, the opposite of C and PHP — stops disagreeing. Keep
|
||||
`displayName` (camelCase) handling only in the public-relay fallback path,
|
||||
where raw client JSON is parsed directly.
|
||||
|
||||
---
|
||||
|
||||
## 2A. Hostile Characters in Names
|
||||
|
||||
Nostr names are attacker-controlled free-form UTF-8. The guiding principle:
|
||||
|
||||
> **Store bytes verbatim. Neutralize at the point of rendering.**
|
||||
|
||||
Sanitizing at write time would be wrong — it is lossy, irreversible, and the
|
||||
"correct" transformation differs per output context (HTML body vs. attribute vs.
|
||||
JSON vs. CSV vs. terminal log). A name mangled on the way into the cache can
|
||||
never be recovered, and the cache would no longer faithfully mirror the kind-0
|
||||
event. So the cache table stores exactly what the user published.
|
||||
|
||||
**But "it's a frontend issue" is only ~90% true.** There is one true storage-layer
|
||||
concern, and one place where the current frontend is actively unsafe.
|
||||
|
||||
### 2A.1 Storage-layer concern: NUL bytes (must handle at write time)
|
||||
|
||||
PostgreSQL `TEXT` **cannot** store `U+0000`. A kind-0 containing `\u0000` in its
|
||||
JSON string makes `->>` yield a value that PostgreSQL refuses to store, raising
|
||||
`ERROR: unsupported Unicode escape sequence` — which would abort the trigger and
|
||||
therefore **reject the whole event insert**. That turns a cosmetic nuisance into
|
||||
a denial-of-service on event ingestion.
|
||||
|
||||
This is not a presentation problem and must be handled in the trigger:
|
||||
|
||||
```sql
|
||||
-- Strip NUL only; everything else is preserved byte-for-byte.
|
||||
replace(COALESCE(j->>'name',''), E'\\u0000', '')
|
||||
```
|
||||
|
||||
Implement as a small `sanitize_pg_text(text)` helper used for every extracted
|
||||
string column. It removes **only** characters PostgreSQL structurally cannot
|
||||
store — not "weird" characters generally. Invalid UTF-8 byte sequences are
|
||||
already rejected earlier by `cJSON` parsing and by the `safe_jsonb()` helper
|
||||
(the row lands with `parse_ok = FALSE`), so no additional handling is needed.
|
||||
|
||||
A defensive `byte_size` guard is also worth adding: cap stored name fields at a
|
||||
sane length (e.g. 1 KB) so a megabyte-long "name" cannot bloat the table or the
|
||||
admin JSON payloads. Truncation is recorded in `raw_content`, which keeps the
|
||||
full original.
|
||||
|
||||
### 2A.2 Live vulnerability: stored XSS in the admin UI
|
||||
|
||||
This must be fixed as part of this work, because the whole point of the change is
|
||||
to route more user-controlled names into more admin pages.
|
||||
|
||||
[`admin/assets/app.js:142`](../admin/assets/app.js:142) interpolates the name
|
||||
directly into `innerHTML`:
|
||||
|
||||
```js
|
||||
tbody.innerHTML = d.top_pubkeys.map((p, i) =>
|
||||
`<tr><td>${i+1}</td><td>${p.name || '<i>unknown</i>'}</td>...`
|
||||
```
|
||||
|
||||
and [`admin/assets/app.js:411`](../admin/assets/app.js:411) does the same for the
|
||||
caching-follows table. A user who sets their kind-0 `name` to
|
||||
`<img src=x onerror="...">` achieves **script execution in the relay
|
||||
administrator's authenticated browser session** merely by posting enough events
|
||||
to appear in the top-pubkeys list. No privileged access is required.
|
||||
|
||||
The codebase is already inconsistent about this: the header name at
|
||||
[`app.js:652`](../admin/assets/app.js:652) correctly uses `textContent` and is
|
||||
safe. The table renderers are not.
|
||||
|
||||
**Fix:** add an escaping helper and apply it to every interpolated
|
||||
user-controlled value in `innerHTML` template strings:
|
||||
|
||||
```js
|
||||
const esc = (s) => String(s ?? '').replace(/[&<>"']/g,
|
||||
c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
```
|
||||
|
||||
Auditing the surrounding rows shows the same pattern applied to other
|
||||
user-controlled fields — event `content` ([`app.js:380`](../admin/assets/app.js:380)),
|
||||
DM content ([`app.js:441`](../admin/assets/app.js:441)), config values
|
||||
([`app.js:219`](../admin/assets/app.js:219)), and auth-rule `pattern_value`
|
||||
([`app.js:251`](../admin/assets/app.js:251)) — so the sweep should cover all of
|
||||
them, not just names. Preferring `textContent` / `createElement` over `innerHTML`
|
||||
in these renderers is the more durable fix where it is not too invasive.
|
||||
|
||||
Note the PHP side is already correct: [`e()`](../admin/lib/helpers.php:9) wraps
|
||||
`htmlspecialchars(..., ENT_QUOTES, 'UTF-8')` and is used for server-rendered
|
||||
output. The gap is purely in the JS-built tables.
|
||||
|
||||
### 2A.3 Presentation-layer nuisances (frontend, cosmetic)
|
||||
|
||||
These stay unsanitized in the database and are handled with CSS/formatting:
|
||||
|
||||
| Issue | Effect | Mitigation |
|
||||
|---|---|---|
|
||||
| Bidi overrides (`U+202E` RTL) | Reverses surrounding text, spoofs other names | Render names in a `<bdi>` element — purpose-built for exactly this, isolates bidi without altering the value |
|
||||
| Zalgo / stacked combining marks | Vertical overflow past row bounds | `overflow: hidden` + fixed line-height on the name cell |
|
||||
| Zero-width chars (`U+200B`, `U+FEFF`) | Invisible; two names look identical | Optional: reveal-on-hover indicator; do not strip |
|
||||
| Newlines / tabs | Break single-line table layout | CSS `white-space: nowrap` + `text-overflow: ellipsis` |
|
||||
| Very long names | Blow out column width | CSS `max-width` + ellipsis (value stays intact in a `title` tooltip) |
|
||||
| Emoji / astral-plane chars | None — legitimate usage | Nothing; ensure JS length math uses code points, not UTF-16 units, when truncating |
|
||||
|
||||
Truncation in JS deserves care: `substring()` on a UTF-16 string can split a
|
||||
surrogate pair and emit a replacement glyph. Use `Array.from(str).slice(0, n)` or
|
||||
CSS-based ellipsis (preferred — no string surgery at all).
|
||||
|
||||
### 2A.4 Terminal/log safety
|
||||
|
||||
Names flow into `DEBUG_*` output. ANSI escape sequences in a name can manipulate
|
||||
a maintainer's terminal. Log rendering should escape non-printable bytes, or
|
||||
simply avoid logging profile names at all — the pubkey is the useful identifier
|
||||
in logs anyway.
|
||||
|
||||
### 2.9 Which field do people actually use?
|
||||
|
||||
Storing both fields turns this into an empirical question rather than a guess.
|
||||
Once the table is populated, one aggregate query answers it against real data
|
||||
from your relay's own corpus:
|
||||
|
||||
```sql
|
||||
SELECT count(*) FILTER (WHERE name <> '' AND display_name <> '') AS both,
|
||||
count(*) FILTER (WHERE name <> '' AND display_name = '') AS name_only,
|
||||
count(*) FILTER (WHERE name = '' AND display_name <> '') AS display_only,
|
||||
count(*) FILTER (WHERE name = '' AND display_name = '') AS neither,
|
||||
count(*) FILTER (WHERE name <> '' AND display_name <> ''
|
||||
AND name <> display_name) AS both_differ,
|
||||
count(*) AS total
|
||||
FROM profiles;
|
||||
```
|
||||
|
||||
`both_differ` is the number that matters: it counts profiles where the preference
|
||||
setting actually changes what gets rendered. If it is near zero, the setting is
|
||||
academic and either default is fine. If it is large, the setting earns its keep.
|
||||
|
||||
Worth surfacing as a small panel on the admin stats page — it is one cheap
|
||||
aggregate over a table with one row per pubkey, and it makes
|
||||
`profile_name_preference` self-documenting: you can see the impact of the choice
|
||||
before making it. Add it once the table has accumulated real data.
|
||||
|
||||
### 2.8 Verification
|
||||
|
||||
- Fresh database: relay starts, `profiles` exists, posting a kind-0 populates
|
||||
exactly one row with `name` and `display_name` both preserved verbatim.
|
||||
- Upgrade path: start against a database with pre-existing kind-0 events, confirm
|
||||
the backfill fills every row once and does **not** re-run on the next restart.
|
||||
- Replaceable-update: publish a newer kind-0, confirm the row updates; replay an
|
||||
older one, confirm the row does **not** regress.
|
||||
- Malformed content: store a kind-0 whose content is not JSON; confirm the insert
|
||||
still succeeds, `parse_ok = FALSE`, and the admin pages render without error.
|
||||
- **NUL byte:** publish a kind-0 whose `name` contains `\u0000`; confirm the
|
||||
event is still accepted, the profile row is created, and the relay does not
|
||||
error. This is the regression test for the ingest-DoS path in §2A.1.
|
||||
- **XSS:** publish a kind-0 with `name` set to
|
||||
`<img src=x onerror="window.__xss=1">`, load the stats and caching pages, and
|
||||
confirm the markup is rendered as visible text and `window.__xss` is
|
||||
undefined.
|
||||
- **Bidi/Zalgo:** publish names containing `U+202E` and stacked combining marks;
|
||||
confirm table layout and neighbouring rows are unaffected.
|
||||
- Consistency: the same pubkey shows an identical name in the stats table, the
|
||||
caching follows table, and the header.
|
||||
- Query-count check: confirm the top-pubkeys API response issues one profile
|
||||
query rather than ten.
|
||||
- Both-fields check: query `profiles` for a pubkey whose kind-0 sets `name` and
|
||||
`display_name` to different values; confirm both are stored distinctly.
|
||||
|
||||
---
|
||||
|
||||
## 3. Phase 2 — Image Caching (design only, deferred)
|
||||
|
||||
Recorded here so Phase 1's schema does not need reworking later.
|
||||
|
||||
### 3.1 Motivation
|
||||
|
||||
Today the admin UI hotlinks `picture` URLs directly
|
||||
([`admin/assets/app.js:653`](../admin/assets/app.js:653)). Problems: the admin's
|
||||
browser reveals its IP to arbitrary third-party hosts on every page load; dead
|
||||
or slow hosts degrade the UI; images can be arbitrarily large; there is no way to
|
||||
show avatars offline.
|
||||
|
||||
### 3.2 Proposed schema
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS profile_images (
|
||||
pubkey TEXT PRIMARY KEY,
|
||||
source_url TEXT NOT NULL,
|
||||
mime_type TEXT NOT NULL DEFAULT '',
|
||||
byte_size INTEGER NOT NULL DEFAULT 0,
|
||||
sha256 TEXT NOT NULL DEFAULT '',
|
||||
etag TEXT NOT NULL DEFAULT '',
|
||||
image_data BYTEA,
|
||||
fetch_state TEXT NOT NULL DEFAULT 'pending',
|
||||
fetch_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
fetched_at BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,
|
||||
CHECK (fetch_state IN ('pending','ok','failed','skipped','too_large'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_profile_images_pending
|
||||
ON profile_images(fetch_state, fetch_attempts) WHERE fetch_state = 'pending';
|
||||
```
|
||||
|
||||
`BYTEA` in PostgreSQL rather than the filesystem keeps backup/restore and the
|
||||
container story single-artifact, matching how everything else in this project is
|
||||
stored. Avatars are small; a cap keeps total size bounded.
|
||||
|
||||
### 3.3 Fetch worker
|
||||
|
||||
`libcurl` is already linked ([`Makefile:6`](../Makefile:6)) but currently unused
|
||||
in `src/`. A worker modeled on
|
||||
[`caching_inbox_poller.c`](../src/caching_inbox_poller.c) — two-state
|
||||
idle/active polling, config-gated, off the main libwebsockets thread — would:
|
||||
|
||||
1. Enqueue `pending` rows when `profiles.picture` changes (trigger or poll).
|
||||
2. Fetch with a hard timeout, a max-bytes ceiling (~256 KB), redirect limit,
|
||||
and `Content-Type` allow-list (`image/png|jpeg|webp|gif`).
|
||||
3. Send `If-None-Match` on refresh, honour `304`.
|
||||
4. Exponential backoff, capped `fetch_attempts`, terminal `failed`.
|
||||
|
||||
New config keys following existing naming: `profile_image_cache_enabled`
|
||||
(default **off**), `profile_image_max_bytes`, `profile_image_refresh_days`,
|
||||
`profile_image_fetch_concurrency`.
|
||||
|
||||
### 3.4 Serving
|
||||
|
||||
A relay HTTP route `/avatar/<pubkey>` handled in
|
||||
[`handle_embedded_file_request()`](../src/api.c:1006) (called from
|
||||
[`src/websockets.c:1261`](../src/websockets.c:1261)), returning the bytes with a
|
||||
long `Cache-Control` and an `ETag`, falling back to a generated identicon or
|
||||
`404` when uncached. The UI then only ever loads images from the relay's own
|
||||
origin.
|
||||
|
||||
### 3.5 Risks to weigh before committing
|
||||
|
||||
- **Outbound HTTP from the relay** is a new capability and a real SSRF surface —
|
||||
needs a private-IP/localhost block-list and scheme restriction. This is the
|
||||
main reason to keep it default-off and deferred.
|
||||
- Database growth: bounded by `max_bytes × profile count`; needs a documented
|
||||
ceiling and a prune path.
|
||||
- Content risk: the relay would be re-serving arbitrary third-party bytes under
|
||||
its own origin. Strict `Content-Type` enforcement plus
|
||||
`Content-Security-Policy` / `X-Content-Type-Options: nosniff` on the route.
|
||||
|
||||
---
|
||||
|
||||
## 4. Files Touched (Phase 1)
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| [`src/pg_schema.sql`](../src/pg_schema.sql) | `profiles` table (both name fields, no generated column), `safe_jsonb()`, `sanitize_pg_text()`, sync + delete triggers, guarded backfill, `profile_name_preference` config default, version → 6 |
|
||||
| [`src/pg_schema.h`](../src/pg_schema.h) | Mirror the above as C string literals; bump `EMBEDDED_PG_SCHEMA_VERSION` |
|
||||
| [`src/db_ops.h`](../src/db_ops.h) | Declare `db_get_profile()` / `db_get_profiles()` |
|
||||
| [`src/db_ops_postgres.h`](../src/db_ops_postgres.h) | Declare the postgres implementations |
|
||||
| [`src/db_ops_postgres.c`](../src/db_ops_postgres.c) | Implement both against `profiles`; retire the events-table query |
|
||||
| [`src/db_ops.c`](../src/db_ops.c) | Dispatch entries + SQLite stubs |
|
||||
| [`src/config.h`](../src/config.h) / [`src/config.c`](../src/config.c) | `profile_display_name()` resolver; batch profile lookup in the caching follows loop |
|
||||
| [`src/api.c`](../src/api.c) | Batch both top-pubkeys loops; shared attach helper emitting `name` + `display_name` + `best_name` |
|
||||
| [`admin/lib/helpers.php`](../admin/lib/helpers.php) | `profile_map()` + `profile_display_name()` |
|
||||
| [`admin/api/stats.php`](../admin/api/stats.php) | Use `profile_map()`; drop LATERAL + JSON casts |
|
||||
| [`admin/api/caching.php`](../admin/api/caching.php) | Join `profiles`; drop LATERAL + JSON casts |
|
||||
| `admin/api/profile.php` | New: single-profile lookup endpoint |
|
||||
| [`admin/assets/app.js`](../admin/assets/app.js) | **`esc()` helper + XSS sweep of all `innerHTML` renderers (§2A.2)**; local-first profile load; consume server `best_name` |
|
||||
| [`admin/assets/index.css`](../admin/assets/index.css) | `nowrap` / `overflow` / `max-width` + ellipsis on name cells (§2A.3) |
|
||||
| `tests/` | New script covering populate / upgrade / replace / malformed / NUL / XSS / bidi cases |
|
||||
|
||||
---
|
||||
|
||||
## 5. Sequencing
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[Add profiles table + safe_jsonb + sanitize_pg_text + triggers to pg_schema.sql] --> B[Mirror into pg_schema.h and bump version to 6]
|
||||
B --> C[Guarded one-time backfill + profile_name_preference config default]
|
||||
C --> D[Implement db_get_profile and db_get_profiles]
|
||||
D --> E[Migrate api.c and config.c to batch lookups]
|
||||
E --> F[Add profile_map helper and migrate PHP endpoints]
|
||||
F --> X[Fix stored XSS: esc helper and innerHTML sweep in app.js]
|
||||
X --> G[Add profile.php endpoint and update app.js profile load]
|
||||
G --> Y[CSS hardening for hostile name rendering]
|
||||
Y --> Z[Add name-field usage panel to stats page]
|
||||
G --> H[Tests: populate, upgrade, replace, malformed, consistency]
|
||||
H --> I[Phase 2 image caching - deferred]
|
||||
```
|
||||
@@ -0,0 +1,297 @@
|
||||
# Relay Admin Page Migration Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Migrate the relay admin functionality from the internally-served `api/index.html` + `api/index.js` (served by the C relay binary) to an externally-hosted `relay-admin.html` page in the `client-ndk` project. This page uses the NDK SharedWorker architecture (template.html pattern) and communicates with the relay via kind 23456/23457/24567 Nostr events.
|
||||
|
||||
## Current State
|
||||
|
||||
### Working Reference: [`c-relay-pg.html`](/home/user/lt/client-ndk/www/c-relay-pg.html)
|
||||
- Successfully receives kind 24567 monitoring events
|
||||
- Has `SUBSCRIPTION_ONLY_MODE = true` — admin UI actions are disabled
|
||||
- Contains all admin section HTML (Statistics, Subscriptions, Configuration, Authorization, IP Bans, Relay Events, DM, Database Query)
|
||||
- Has sidenav with admin page links
|
||||
- Has `sendAdminCommand()`, `encryptForRelay()`, `decryptFromRelay()`, `fetchRelayInfo()` functions
|
||||
- Has `handleMonitoringEvent()` and `handleAdminResponse()` event handlers
|
||||
|
||||
### Target File: [`relay-admin.html`](/home/user/lt/client-ndk/www/relay-admin.html)
|
||||
- Currently an exact copy of `template.html`
|
||||
- Has full template infrastructure: hamburger menu, sidenav, footer, auth system, relay-ui, blossom-ui, ai-ui
|
||||
- Has the newer `injectHeaderLoginButton` import (slightly newer than c-relay-pg.html)
|
||||
- Empty `divBody` — ready for admin content
|
||||
|
||||
### Legacy Internal Pages: [`api/index.html`](api/index.html) + [`api/index.js`](api/index.js)
|
||||
- 6,779 lines of JavaScript — the full admin UI
|
||||
- Uses `nostr-tools SimplePool` directly (not NDK)
|
||||
- Uses `NOSTR_LOGIN_LITE` for auth (not NDK SharedWorker)
|
||||
- Has rich UI: config tables with inline editing, IP ban management with filters, SQL query console with history, WoT management, relay event forms
|
||||
- All admin commands go through `sendAdminCommand()` → kind 23456 with NIP-44 encryption
|
||||
- Responses come back as kind 23457 events, decrypted and routed to handlers
|
||||
|
||||
## Architecture Comparison
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Legacy - api/index.html
|
||||
A1[NOSTR_LOGIN_LITE auth] --> A2[SimplePool direct WS]
|
||||
A2 --> A3[Kind 23456 admin commands]
|
||||
A2 --> A4[Kind 24567 monitoring sub]
|
||||
A3 --> A5[Kind 23457 responses]
|
||||
end
|
||||
|
||||
subgraph New - relay-admin.html
|
||||
B1[NDK SharedWorker auth] --> B2[NDK subscribe/publishEvent]
|
||||
B2 --> B3[Kind 23456 admin commands]
|
||||
B2 --> B4[Kind 24567 monitoring sub]
|
||||
B3 --> B5[Kind 23457 responses]
|
||||
end
|
||||
```
|
||||
|
||||
### Key Difference: NDK vs SimplePool
|
||||
|
||||
| Aspect | Legacy api/index.js | New relay-admin.html |
|
||||
|--------|-------------------|---------------------|
|
||||
| Auth | `NOSTR_LOGIN_LITE` direct | `initNDKPage()` via SharedWorker |
|
||||
| Subscriptions | `SimplePool.subscribeMany()` | `subscribe()` from init-ndk.mjs |
|
||||
| Publishing | `SimplePool.publish()` + manual signing | `publishEvent()` — auto-signed by worker |
|
||||
| Encryption | `window.nostr.nip44.encrypt/decrypt` | Same — `window.nostr.nip44.encrypt/decrypt` |
|
||||
| Event listening | SimplePool callbacks | `window.addEventListener('ndkEvent', ...)` |
|
||||
| Relay URL | Auto-derived from page URL | Configurable constant `ADMIN_RELAY_WS_URL` |
|
||||
|
||||
## Migration Strategy
|
||||
|
||||
The approach is to merge the admin functionality from `c-relay-pg.html` into `relay-admin.html`, removing the `SUBSCRIPTION_ONLY_MODE` restriction and enabling full admin UI. We use the NDK `publishEvent()` and `subscribe()` APIs instead of SimplePool.
|
||||
|
||||
### What stays from template (relay-admin.html already has)
|
||||
- Full hamburger menu + sidenav infrastructure
|
||||
- Footer with relay status animations
|
||||
- Auth system with required/optional/none modes
|
||||
- Relay-ui, blossom-ui, ai-ui sidenav sections
|
||||
- Theme toggle, logout, version display
|
||||
|
||||
### What gets added from c-relay-pg.html
|
||||
- Admin nav items in sidenav
|
||||
- Admin section HTML panels in divBody
|
||||
- Admin-specific CSS styles
|
||||
- Relay pubkey fetch via NIP-11
|
||||
- NIP-44 encrypt/decrypt helpers
|
||||
- `sendAdminCommand()` for kind 23456
|
||||
- Kind 24567 monitoring subscription
|
||||
- Kind 23457 response handling with per-section routing
|
||||
- All admin UI button handlers
|
||||
- Boot diagnostics panel
|
||||
- System info panel
|
||||
|
||||
## Detailed Implementation Steps
|
||||
|
||||
### Step 1: Add Admin Nav Links to Sidenav
|
||||
|
||||
The admin links go directly inside `divSideNavBody` as simple styled items — **not** inside a collapsible `.sidenavSection` container. They should be always-visible, non-collapsible links that match the font/styling of the sidenav sections but remain in the open body area. Place them before `divFiles`:
|
||||
|
||||
```html
|
||||
<div id="divSideNavBody">
|
||||
<!-- Admin nav links - always visible, not collapsible -->
|
||||
<div id="divAdminNav">
|
||||
<div class="adminNavItem active" data-section="statistics">Statistics</div>
|
||||
<div class="adminNavItem" data-section="subscriptions">Subscriptions</div>
|
||||
<div class="adminNavItem" data-section="configuration">Configuration</div>
|
||||
<div class="adminNavItem" data-section="authorization">Authorization</div>
|
||||
<div class="adminNavItem" data-section="ip-bans">IP Bans</div>
|
||||
<div class="adminNavItem" data-section="relay-events">Relay Events</div>
|
||||
<div class="adminNavItem" data-section="dm">DM</div>
|
||||
<div class="adminNavItem" data-section="database">Database Query</div>
|
||||
</div>
|
||||
<div id="divFiles"></div>
|
||||
</div>
|
||||
```
|
||||
|
||||
The `.adminNavItem` styling uses the same font-family and sizing as sidenav section content (`font-size: 70%`, `font-family: var(--font-family)`) with a simple border-bottom separator, hover accent color, and bold+background for the active state. No section title header, no collapse toggle icon.
|
||||
|
||||
### Step 2: Add Admin Section HTML Panels
|
||||
|
||||
Port the section HTML from `c-relay-pg.html` into `divBody`. Each section follows the pattern:
|
||||
|
||||
```html
|
||||
<div id="section-{name}" class="adminSection">
|
||||
<div class="adminSectionHeader">{Title}</div>
|
||||
<!-- Section-specific content -->
|
||||
</div>
|
||||
```
|
||||
|
||||
Sections to port:
|
||||
1. **Statistics** — CPU metrics, time stats, top pubkeys, event kinds cards
|
||||
2. **Subscriptions** — Subscription details log
|
||||
3. **Configuration** — Config key/value inputs, fetch/set buttons, log
|
||||
4. **Authorization** — Pubkey input, whitelist/blacklist/query buttons, log
|
||||
5. **IP Bans** — IP input, duration, ban/query buttons, log
|
||||
6. **Relay Events** — Kind 0 metadata form, live event feed
|
||||
7. **DM** — Message textarea, send button, log
|
||||
8. **Database Query** — SQL textarea, run button, log
|
||||
9. **System** (always visible) — Relay WS/HTTP URLs, relay/user pubkeys
|
||||
10. **Boot Diagnostics** (always visible) — Boot log
|
||||
|
||||
### Step 3: Add Admin CSS
|
||||
|
||||
Port the admin-specific styles from `c-relay-pg.html` `<style>` block. These use CSS variables from `client.css` so they integrate with the theme system:
|
||||
|
||||
- `#divBody` flex layout overrides
|
||||
- `.adminNavItem` styles with hover/active states
|
||||
- `.adminSection` show/hide with `.active` class
|
||||
- `.adminGrid`, `.adminCard`, `.adminTable` layout
|
||||
- `.adminInput`, `.adminTextArea`, `.adminBtn` form elements
|
||||
- `.adminLog` scrollable log panels
|
||||
|
||||
### Step 4: Add Admin State Variables
|
||||
|
||||
```javascript
|
||||
const ADMIN_RELAY_WS_URL = 'wss://relay.laantungir.net';
|
||||
const ADMIN_RELAY_HTTP_URL = 'https://relay.laantungir.net';
|
||||
|
||||
let relayPubkey = '';
|
||||
let adminInitialized = false;
|
||||
let adminUiBound = false;
|
||||
const adminSubscriptions = [];
|
||||
const MAX_LOG_LINES = 200;
|
||||
```
|
||||
|
||||
### Step 5: Add Core Admin Functions
|
||||
|
||||
Port from `c-relay-pg.html`:
|
||||
|
||||
1. **`appendLog(id, msg)`** — Prepend timestamped messages to log panels
|
||||
2. **`setSystemInfo()`** — Populate system info table
|
||||
3. **`fetchRelayInfo()`** — NIP-11 fetch to get relay pubkey
|
||||
4. **`encryptForRelay(content)`** — NIP-44 encrypt using `window.nostr.nip44.encrypt`
|
||||
5. **`decryptFromRelay(content)`** — NIP-44 decrypt using `window.nostr.nip44.decrypt`
|
||||
6. **`sendAdminCommand(commandArray, sink)`** — Encrypt + publish kind 23456 event via `publishEvent()`
|
||||
|
||||
### Step 6: Add Subscription Setup
|
||||
|
||||
```javascript
|
||||
async function setupAdminSubscriptions() {
|
||||
// Subscribe to kind 24567 monitoring events
|
||||
const monitoringSub = subscribe(
|
||||
{ kinds: [24567], since: Math.floor(Date.now() / 1000) - 60 },
|
||||
{ closeOnEose: false, cacheUsage: 'ONLY_RELAY' }
|
||||
);
|
||||
|
||||
// Subscribe to kind 23457 admin responses
|
||||
const responseSub = subscribe(
|
||||
{ kinds: [23457], '#p': [currentPubkey], since: Math.floor(Date.now() / 1000) - 60 },
|
||||
{ closeOnEose: false, cacheUsage: 'ONLY_RELAY' }
|
||||
);
|
||||
|
||||
// Listen for events via ndkEvent window event
|
||||
window.addEventListener('ndkEvent', async (evt) => {
|
||||
const event = evt.detail;
|
||||
if (event.kind === 24567) handleMonitoringEvent(event);
|
||||
else if (event.kind === 23457) await handleAdminResponse(event);
|
||||
else handleLiveEvent(event);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### Step 7: Add Event Handlers
|
||||
|
||||
Port from `c-relay-pg.html`:
|
||||
|
||||
1. **`handleMonitoringEvent(event)`** — Route d-tag values to stat cards
|
||||
2. **`handleAdminResponse(event)`** — Decrypt kind 23457 and route to logs
|
||||
3. **`handleLiveEvent(event)`** — Sample non-admin events to live feed
|
||||
4. **`activateSection(sectionKey)`** — Toggle active section visibility
|
||||
5. **`bindAdminUi()`** — Wire up all button click handlers
|
||||
|
||||
### Step 8: Add Button Handlers
|
||||
|
||||
All handlers follow the same pattern: gather input → `sendAdminCommand(array, logId)`:
|
||||
|
||||
- **btnRefreshStats** → `['system_command', 'system_status']`
|
||||
- **btnConfigRefresh** → `['config_query', 'all']`
|
||||
- **btnConfigSet** → `[key, value]`
|
||||
- **btnWhitelist** → `['whitelist', 'pubkey', pk]`
|
||||
- **btnBlacklist** → `['blacklist', 'pubkey', pk]`
|
||||
- **btnAuthQuery** → `['auth_query', 'all']`
|
||||
- **btnBanIp** → `['ip_ban', 'add', ip, duration]`
|
||||
- **btnIpQuery** → `['ip_ban', 'query', 'all']`
|
||||
- **btnPublishKind0** → Direct `publishEvent()` with kind 0
|
||||
- **btnSendDm** → `['dm', message]`
|
||||
- **btnRunSql** → `['sql_query', query]`
|
||||
|
||||
### Step 9: Integration into main() Flow
|
||||
|
||||
```javascript
|
||||
(async function main() {
|
||||
// 1. Template init (hamburger, theme, auth)
|
||||
initHamburgerMenu();
|
||||
// ... existing template setup ...
|
||||
|
||||
// 2. Auth
|
||||
await initializeAuthentication(authMode);
|
||||
await initializeAuthenticatedPageFeatures();
|
||||
|
||||
// 3. Admin init (NEW)
|
||||
await initializeAdminFeatures();
|
||||
|
||||
// 4. Template finalization
|
||||
await updateVersionDisplay();
|
||||
})();
|
||||
```
|
||||
|
||||
Where `initializeAdminFeatures()`:
|
||||
```javascript
|
||||
async function initializeAdminFeatures() {
|
||||
if (!isAuthenticated || adminInitialized) return;
|
||||
bindAdminUi();
|
||||
await fetchRelayInfo();
|
||||
await setupAdminSubscriptions();
|
||||
setSystemInfo();
|
||||
adminInitialized = true;
|
||||
}
|
||||
```
|
||||
|
||||
## Response Routing Enhancement
|
||||
|
||||
The current `c-relay-pg.html` broadcasts all responses to all logs. For a better UX, we should parse the decrypted response and route to the appropriate log:
|
||||
|
||||
```javascript
|
||||
async function handleAdminResponse(event) {
|
||||
const plain = await decryptFromRelay(event.content);
|
||||
const parsed = JSON.parse(plain);
|
||||
|
||||
// Route based on query_type
|
||||
if (parsed.query_type === 'config_all' || parsed.query_type === 'config_update') {
|
||||
appendLog('configLog', plain);
|
||||
} else if (parsed.query_type === 'auth_query' || parsed.query_type === 'auth_rule') {
|
||||
appendLog('authLog', plain);
|
||||
} else if (parsed.query_type === 'sql_query') {
|
||||
appendLog('sqlLog', plain);
|
||||
} else if (parsed.query_type === 'ip_ban') {
|
||||
appendLog('ipBanLog', plain);
|
||||
} else {
|
||||
// Fallback: broadcast to all
|
||||
appendLog('configLog', `RESP <- ${plain}`);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Files Modified
|
||||
|
||||
| File | Location | Change |
|
||||
|------|----------|--------|
|
||||
| `relay-admin.html` | `~/lt/client-ndk/www/` | Add admin HTML, CSS, and JavaScript |
|
||||
|
||||
## Files NOT Modified
|
||||
|
||||
- `template.html` — Stays as the clean template
|
||||
- `c-relay-pg.html` — Stays as the debug/test reference
|
||||
- `api/index.html` / `api/index.js` — Legacy, stays for now (still served by relay binary)
|
||||
- No C code changes needed — the relay already handles kind 23456/23457/24567
|
||||
|
||||
## Future Enhancements (Out of Scope)
|
||||
|
||||
- Rich config table with inline editing (like api/index.html has)
|
||||
- IP ban table with filter tabs
|
||||
- SQL query history dropdown
|
||||
- WoT management section
|
||||
- Event rate chart visualization
|
||||
- Admin verification handshake
|
||||
@@ -0,0 +1,233 @@
|
||||
# Server-Side ASCII Chart Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Replace the client-side `text_graph.js` ASCII chart with a **server-side PHP renderer** that produces the ASCII X-bar chart string. The chart is served from a **dedicated plain-text endpoint** (`api/chart.php`) that works both in the browser (injected into a `<div>`) and in the terminal via `curl` — the ASCII art renders correctly either way.
|
||||
|
||||
Four time ranges, each with its own bin size and refresh/caching strategy:
|
||||
|
||||
| Range | Span | Bin size | Bins | Refresh | Cache TTL |
|
||||
|--------|-------------|------------|------|------------------|------------|
|
||||
| Hour | last 1h | 10 seconds | 360 | every 10s (live) | none |
|
||||
| Day | last 24h | 5 minutes | 288 | every 10s* | 1 hour |
|
||||
| Month | last 30d | 1 hour | 720 | every 10s* | 1 day |
|
||||
| Year | last 365d | 1 day | 365 | every 10s* | 1 month |
|
||||
|
||||
\* The client polls every 10s, but the server only re-runs the expensive query when the cache expires. Between cache expirations, the cached ASCII string is returned instantly.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph Clients
|
||||
T[Terminal — curl]
|
||||
W[Web UI — app.js]
|
||||
end
|
||||
|
||||
T -->|GET chart.php?range=hour| EP[chart.php]
|
||||
W -->|GET chart.php?range=hour| EP
|
||||
T -->|GET chart.php?range=day| EP
|
||||
W -->|GET chart.php?range=day| EP
|
||||
|
||||
EP --> C{Cache valid?}
|
||||
C -->|Yes| Return[Return cached ASCII string]
|
||||
C -->|No| Query[Run GROUP BY binning query]
|
||||
Query --> Render[render_ascii_chart in lib/ascii_chart.php]
|
||||
Render --> CacheWrite[Write to cache file]
|
||||
CacheWrite --> Return
|
||||
|
||||
Return -->|text/plain; charset=utf-8| T
|
||||
Return -->|text/plain; charset=utf-8| W
|
||||
|
||||
subgraph "Cache files (admin2/cache/)"
|
||||
H[chart_hour.txt — never cached]
|
||||
D[chart_day.txt — TTL 1h]
|
||||
M[chart_month.txt — TTL 1d]
|
||||
Y[chart_year.txt — TTL 1mo]
|
||||
end
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
### 1. PHP ASCII Chart Renderer — `admin2/lib/ascii_chart.php`
|
||||
|
||||
A pure function that takes an array of bin counts and produces the ASCII X-bar chart string. Mirrors the layout of the original `text_graph.js`:
|
||||
|
||||
```
|
||||
New Events
|
||||
|
||||
11 | X
|
||||
10 | X X
|
||||
9 | X X X X
|
||||
8 | X X X X X X X
|
||||
7 | X X X X X X X X X X
|
||||
6 | X X X X X X X X X X X X X
|
||||
5 | X X X X X X X X X X X X X X X
|
||||
4 | X X X X X X X X X X X X X X X X X
|
||||
3 |X X X X X X X X X X X X X X X X X X X
|
||||
2 |X X X X X X X X X X X X X X X X X X X
|
||||
1 |X X X X X X X X X X X X X X X X X X X
|
||||
+----------------------------------------
|
||||
0s 50s 100s 150s 200s 250s 300s
|
||||
```
|
||||
|
||||
**Function signature:**
|
||||
```php
|
||||
function render_ascii_chart(array $bins, array $options = []): string
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `title` (string, default `'New Events'`)
|
||||
- `max_height` (int, default `11`) — chart height in rows
|
||||
- `x_axis_label` (string, default `''`)
|
||||
- `bin_duration` (int, seconds) — for X-axis elapsed-time labels
|
||||
- `label_interval` (int, default `5`) — label every N bins
|
||||
|
||||
**Algorithm** (same as `text_graph.js` render method):
|
||||
1. `max_count = max($bins)`; `scale_factor = max(1, ceil(max_count / max_height))`
|
||||
2. For each row from `max_height` down to `1`:
|
||||
- Y-axis label = `(row - 1) * scale_factor + 1`, right-padded to 3 chars
|
||||
- For each bin: if `ceil(count / scale_factor) >= row` → `X`, else space
|
||||
3. X-axis: `+` followed by dashes (one per bin)
|
||||
4. X-axis labels: elapsed time every `label_interval` bins, formatted as `Ns` / `Nm` / `Hh` / `Dd` depending on magnitude
|
||||
|
||||
### 2. Binning SQL Queries — in `stats.php`
|
||||
|
||||
Each range uses a `FLOOR((created_at - epoch) / bin_size)` GROUP BY query. The `idx_events_created_at` index makes these fast.
|
||||
|
||||
**Hour (live, no cache):**
|
||||
```sql
|
||||
SELECT FLOOR((created_at - :epoch) / 10)::INT AS bin, COUNT(*) AS cnt
|
||||
FROM events
|
||||
WHERE created_at >= :epoch
|
||||
GROUP BY bin
|
||||
ORDER BY bin;
|
||||
```
|
||||
- `epoch = now - 3600`, `bin_size = 10s`, produces up to 360 bins
|
||||
- Runs on every 10s poll (cheap: only scans last hour, indexed)
|
||||
|
||||
**Day (cache TTL 1h):**
|
||||
```sql
|
||||
SELECT FLOOR((created_at - :epoch) / 300)::INT AS bin, COUNT(*) AS cnt
|
||||
FROM events
|
||||
WHERE created_at >= :epoch
|
||||
GROUP BY bin
|
||||
ORDER BY bin;
|
||||
```
|
||||
- `epoch = now - 86400`, `bin_size = 300s` (5 min), produces up to 288 bins
|
||||
|
||||
**Month (cache TTL 1d):**
|
||||
```sql
|
||||
SELECT FLOOR((created_at - :epoch) / 3600)::INT AS bin, COUNT(*) AS cnt
|
||||
FROM events
|
||||
WHERE created_at >= :epoch
|
||||
GROUP BY bin
|
||||
ORDER BY bin;
|
||||
```
|
||||
- `epoch = now - 2592000`, `bin_size = 3600s` (1h), produces up to 720 bins
|
||||
|
||||
**Year (cache TTL 1 month):**
|
||||
```sql
|
||||
SELECT FLOOR((created_at - :epoch) / 86400)::INT AS bin, COUNT(*) AS cnt
|
||||
FROM events
|
||||
WHERE created_at >= :epoch
|
||||
GROUP BY bin
|
||||
ORDER BY bin;
|
||||
```
|
||||
- `epoch = now - 31536000`, `bin_size = 86400s` (1 day), produces up to 365 bins
|
||||
|
||||
**Bin array assembly:** Query returns only non-empty bins. PHP fills a fixed-length array (all zeros) and overlays the counts at the correct positions, so empty time slots show as blank columns — the chart always advances in time.
|
||||
|
||||
### 3. File-Based Cache — `admin2/cache/`
|
||||
|
||||
Simple file cache with TTL. No APCu/Redis dependency.
|
||||
|
||||
```php
|
||||
function get_cached_chart(string $range): ?string
|
||||
function set_cached_chart(string $range, string $ascii): void
|
||||
```
|
||||
|
||||
- Cache files: `admin2/cache/chart_{range}.txt`
|
||||
- TTLs: `hour` = 0 (never cache), `day` = 3600, `month` = 86400, `year` = 2592000
|
||||
- Check: `filemtime($file) > time() - $ttl`
|
||||
- Directory `admin2/cache/` created automatically with `mkdir(..., 0775, true)`
|
||||
|
||||
### 4. Standalone Chart Endpoint — `admin2/api/chart.php`
|
||||
|
||||
A dedicated plain-text endpoint that returns the raw ASCII chart string. Works in both the browser and the terminal.
|
||||
|
||||
**Request:** `GET api/chart.php?range=hour|day|month|year`
|
||||
|
||||
**Response:** `Content-Type: text/plain; charset=utf-8` — just the ASCII chart string, no JSON wrapper.
|
||||
|
||||
**Terminal usage:**
|
||||
```bash
|
||||
curl http://localhost:8088/api/chart.php?range=hour
|
||||
curl http://localhost:8088/api/chart.php?range=day
|
||||
curl http://localhost:8088/api/chart.php?range=month
|
||||
curl http://localhost:8088/api/chart.php?range=year
|
||||
```
|
||||
|
||||
**Logic:**
|
||||
1. Read `range` query param (default: `hour`)
|
||||
2. Validate against allowed ranges (`hour`, `day`, `month`, `year`)
|
||||
3. Check cache: if valid, return cached string immediately
|
||||
4. If cache miss/expired: run the binning SQL query, fill the bin array, call `render_ascii_chart()`, write to cache, return the string
|
||||
5. Set `Content-Type: text/plain; charset=utf-8` header
|
||||
|
||||
**`stats.php` is unchanged** — it continues to return the existing JSON stats (numbers only). The chart is a completely separate endpoint, keeping concerns cleanly separated.
|
||||
|
||||
### 5. Frontend Changes
|
||||
|
||||
#### `admin2/index.php`
|
||||
- Replace the single chart div with a chart container + range selector tabs:
|
||||
```html
|
||||
<div class="chart-range-tabs">
|
||||
<button class="chart-tab active" data-range="hour">1H</button>
|
||||
<button class="chart-tab" data-range="day">1D</button>
|
||||
<button class="chart-tab" data-range="month">1M</button>
|
||||
<button class="chart-tab" data-range="year">1Y</button>
|
||||
</div>
|
||||
<div id="event-rate-chart" class="event-rate-chart-container">Loading chart...</div>
|
||||
```
|
||||
- Remove `<script src="assets/text_graph.js">` (no longer needed)
|
||||
|
||||
#### `admin2/assets/app.js`
|
||||
- Remove: `eventRateChart`, `previousTotalEvents`, `initializeEventRateChart`, `createChartStubElements`, the `addValue` call in `loadStats`
|
||||
- Add: `currentChartRange = 'hour'` state variable
|
||||
- Add: `loadChart(range)` function — fetches `api/chart.php?range=${range}` as plain text, injects the response into `#event-rate-chart` via `textContent`
|
||||
- Add: tab click handlers that set `currentChartRange` and call `loadChart`
|
||||
- In `loadStats`: call `loadChart(currentChartRange)` at the end (so the hour chart refreshes every 10s)
|
||||
- On DOMContentLoaded: call `loadChart('hour')` to load the initial chart
|
||||
|
||||
#### `admin2/assets/index.css`
|
||||
- Add `.chart-range-tabs` styles (small monospace buttons, active state with accent border)
|
||||
|
||||
### 6. Cache Directory
|
||||
|
||||
- `admin2/cache/` — add to `.gitignore` (runtime artifacts)
|
||||
- Created at runtime by `mkdir` if missing
|
||||
|
||||
## File Summary
|
||||
|
||||
| File | Action |
|
||||
|------|--------|
|
||||
| `admin2/lib/ascii_chart.php` | **New** — PHP ASCII chart renderer function |
|
||||
| `admin2/api/chart.php` | **New** — standalone plain-text chart endpoint with 4 binning queries + cache |
|
||||
| `admin2/api/stats.php` | **Unchanged** — continues returning JSON stats as before |
|
||||
| `admin2/index.php` | **Modify** — add range tabs, remove text_graph.js script |
|
||||
| `admin2/assets/app.js` | **Modify** — replace client chart with `loadChart()` fetch+inject |
|
||||
| `admin2/assets/index.css` | **Modify** — add `.chart-range-tabs` styles |
|
||||
| `admin2/cache/` | **New dir** — runtime cache files (gitignored) |
|
||||
| `admin2/assets/text_graph.js` | **Delete** — no longer needed |
|
||||
|
||||
## Execution Order
|
||||
|
||||
1. Create `admin2/lib/ascii_chart.php` (the renderer)
|
||||
2. Create `admin2/api/chart.php` (standalone chart endpoint + cache logic)
|
||||
3. Modify `admin2/index.php` (add range tabs, remove text_graph.js)
|
||||
4. Modify `admin2/assets/app.js` (replace chart logic with `loadChart`)
|
||||
5. Modify `admin2/assets/index.css` (add tab styles)
|
||||
6. Add `admin2/cache/` to `.gitignore`
|
||||
7. Delete `admin2/assets/text_graph.js`
|
||||
8. Test: verify all 4 ranges load, cache files appear, hour chart refreshes live
|
||||
@@ -1,10 +0,0 @@
|
||||
https://github.com/rushmi0/Fenrir-s
|
||||
https://github.com/barkyq/gnost-relay
|
||||
https://github.com/lpicanco/knostr
|
||||
https://github.com/bezysoftware/netstr
|
||||
https://github.com/lebrunel/nex
|
||||
https://github.com/CodyTseng/nostr-relay-nestjs
|
||||
https://github.com/mattn/nostr-relay
|
||||
https://github.com/Cameri/nostream
|
||||
https://github.com/Giszmo/NostrPostr/tree/master/NostrRelay
|
||||
https://github.com/fiatjaf/relayer/tree/master/examples/basic
|
||||
@@ -16,7 +16,9 @@ extern void log_query_execution(const char* query_type, const char* sub_id,
|
||||
int get_active_connection_count(void);
|
||||
#include <time.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/select.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <strings.h>
|
||||
#include <stdbool.h>
|
||||
#include "api.h"
|
||||
@@ -28,9 +30,11 @@ int get_active_connection_count(void);
|
||||
#include "../nostr_core_lib/nostr_core/nip044.h"
|
||||
#include "subscriptions.h"
|
||||
#include "db_ops.h"
|
||||
#include "caching_service_launcher.h"
|
||||
|
||||
// External subscription manager (from main.c via subscriptions.c)
|
||||
extern subscription_manager_t g_subscription_manager;
|
||||
extern struct lws_context *ws_context;
|
||||
|
||||
// Global variables for config change system
|
||||
static pending_config_change_t* pending_changes_head = NULL;
|
||||
@@ -67,16 +71,42 @@ int get_monitoring_throttle_seconds(void) {
|
||||
return get_config_int("kind_24567_reporting_throttle_sec", 5);
|
||||
}
|
||||
|
||||
typedef enum {
|
||||
API_WORK_JOB_EVENT_STORED = 1,
|
||||
API_WORK_JOB_SUBSCRIPTION_CHANGE,
|
||||
API_WORK_JOB_STATUS_POST
|
||||
} api_work_job_type_t;
|
||||
// Forward declaration for the round-robin monitoring generator (defined below).
|
||||
static int generate_round_robin_monitoring_event(void);
|
||||
|
||||
typedef struct api_work_job {
|
||||
api_work_job_type_t type;
|
||||
struct api_work_job* next;
|
||||
} api_work_job_t;
|
||||
// Forward declaration for has_any_subscription_for_kind (subscriptions.c).
|
||||
// Returns 1 if any active subscription would receive the given event kind
|
||||
// (explicit kind filter match OR no kind filter at all).
|
||||
int has_any_subscription_for_kind(int event_kind);
|
||||
|
||||
// Forward declaration for the periodic kind-1 status post generator.
|
||||
int generate_and_post_status_event(void);
|
||||
|
||||
// =====================================================================
|
||||
// api-worker thread (Phase 1 + 2 + 5 of api_upgrade_plan.md)
|
||||
//
|
||||
// The worker owns ALL monitoring event generation on a dedicated thread
|
||||
// with its own PG connection. It is NOT triggered by event storage or
|
||||
// subscription changes (Phase 2 removed those hooks). Instead it runs on
|
||||
// a timer (throttle_sec) and, on PostgreSQL, wakes reactively via
|
||||
// LISTEN/NOTIFY on the 'event_stored' channel (Phase 5).
|
||||
//
|
||||
// Loop:
|
||||
// - If no one is subscribed to kind 24567: condvar-timedwait for
|
||||
// throttle_sec (zero DB work, zero notify polling). Wakeable for
|
||||
// STATUS_POST jobs and shutdown via the self-pipe.
|
||||
// - If subscribers exist: select() on {PG socket, self-pipe} with
|
||||
// timeout = throttle_sec. On NOTIFY or timeout, run one round-robin
|
||||
// monitoring tick. On self-pipe wake, process STATUS_POST.
|
||||
//
|
||||
// A self-pipe is used so api_worker_enqueue_status_post() and shutdown
|
||||
// can wake the select() / condvar wait from another thread safely.
|
||||
// =====================================================================
|
||||
|
||||
typedef enum {
|
||||
API_WORK_JOB_STATUS_POST = 1,
|
||||
API_WORK_JOB_CACHING_TOGGLE = 2
|
||||
} api_work_job_type_t;
|
||||
|
||||
typedef struct api_work_completion {
|
||||
api_work_job_type_t type;
|
||||
@@ -86,18 +116,34 @@ typedef struct api_work_completion {
|
||||
|
||||
static pthread_mutex_t g_api_work_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
static pthread_cond_t g_api_work_cond = PTHREAD_COND_INITIALIZER;
|
||||
static api_work_job_t* g_api_work_head = NULL;
|
||||
static api_work_job_t* g_api_work_tail = NULL;
|
||||
// Pending STATUS_POST request flag (protected by g_api_work_mutex).
|
||||
static int g_api_status_post_pending = 0;
|
||||
|
||||
static pthread_mutex_t g_api_completion_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
static api_work_completion_t* g_api_completion_head = NULL;
|
||||
static api_work_completion_t* g_api_completion_tail = NULL;
|
||||
|
||||
static pthread_t g_api_worker_thread;
|
||||
static int g_api_worker_running = 0;
|
||||
static volatile int g_api_worker_running = 0;
|
||||
|
||||
static int monitoring_on_event_stored_sync(void);
|
||||
static int monitoring_on_subscription_change_sync(void);
|
||||
// Self-pipe fds for waking the worker's select()/condvar. [0]=read, [1]=write.
|
||||
static int g_api_wake_pipe[2] = { -1, -1 };
|
||||
|
||||
static void api_worker_wake(void) {
|
||||
if (g_api_wake_pipe[1] >= 0) {
|
||||
char byte = 1;
|
||||
ssize_t rc = write(g_api_wake_pipe[1], &byte, 1);
|
||||
(void)rc;
|
||||
}
|
||||
}
|
||||
|
||||
static void api_worker_drain_wake_pipe(void) {
|
||||
if (g_api_wake_pipe[0] < 0) return;
|
||||
char buf[64];
|
||||
while (read(g_api_wake_pipe[0], buf, sizeof(buf)) > 0) {
|
||||
// drain
|
||||
}
|
||||
}
|
||||
|
||||
static void api_worker_push_completion(api_work_completion_t* completion) {
|
||||
if (!completion) return;
|
||||
@@ -126,91 +172,200 @@ static api_work_completion_t* api_worker_pop_completion(void) {
|
||||
return completion;
|
||||
}
|
||||
|
||||
static int api_worker_enqueue_job(api_work_job_type_t type) {
|
||||
// Dequeue a pending STATUS_POST request (returns 1 if one was pending).
|
||||
static int api_worker_take_status_post(void) {
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
int pending = g_api_status_post_pending;
|
||||
g_api_status_post_pending = 0;
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
return pending;
|
||||
}
|
||||
|
||||
int api_worker_enqueue_status_post(void) {
|
||||
if (!g_api_worker_running) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
api_work_job_t* job = calloc(1, sizeof(*job));
|
||||
if (!job) {
|
||||
return -1;
|
||||
}
|
||||
job->type = type;
|
||||
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
job->next = NULL;
|
||||
if (!g_api_work_tail) {
|
||||
g_api_work_head = job;
|
||||
g_api_work_tail = job;
|
||||
} else {
|
||||
g_api_work_tail->next = job;
|
||||
g_api_work_tail = job;
|
||||
}
|
||||
pthread_cond_signal(&g_api_work_cond);
|
||||
g_api_status_post_pending = 1;
|
||||
pthread_cond_broadcast(&g_api_work_cond);
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
api_worker_wake();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static api_work_job_t* api_worker_pop_job_blocking(void) {
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
while (g_api_worker_running && !g_api_work_head) {
|
||||
pthread_cond_wait(&g_api_work_cond, &g_api_work_mutex);
|
||||
// Poll the config_changed LISTEN channel on the worker DB connection for up
|
||||
// to timeout_ms. If a config_changed notification is received, enqueue an
|
||||
// API_WORK_JOB_CACHING_TOGGLE completion so the main thread re-reads
|
||||
// caching_enabled and starts/stops the caching service. Any other
|
||||
// notifications (e.g. event_stored) are simply drained. Returns 1 if a
|
||||
// config_changed notification was processed, 0 otherwise.
|
||||
static int api_worker_poll_config_changed(void* worker_db, int timeout_ms) {
|
||||
if (!worker_db) return 0;
|
||||
char* notify_channel = NULL;
|
||||
char* notify_payload = NULL;
|
||||
int prc = db_worker_poll_notify_with_payload(worker_db, timeout_ms,
|
||||
¬ify_channel, ¬ify_payload);
|
||||
if (prc != 1) return 0;
|
||||
int found_config = 0;
|
||||
if (notify_channel && strcmp(notify_channel, "config_changed") == 0) {
|
||||
found_config = 1;
|
||||
}
|
||||
|
||||
api_work_job_t* job = g_api_work_head;
|
||||
if (job) {
|
||||
g_api_work_head = job->next;
|
||||
if (!g_api_work_head) {
|
||||
g_api_work_tail = NULL;
|
||||
if (notify_channel) free(notify_channel);
|
||||
if (notify_payload) free(notify_payload);
|
||||
// Drain any additional queued notifications.
|
||||
char* extra_ch = NULL;
|
||||
char* extra_pl = NULL;
|
||||
while (db_worker_poll_notify_with_payload(worker_db, 0,
|
||||
&extra_ch, &extra_pl) == 1) {
|
||||
if (extra_ch && strcmp(extra_ch, "config_changed") == 0) {
|
||||
found_config = 1;
|
||||
}
|
||||
if (extra_ch) free(extra_ch);
|
||||
if (extra_pl) free(extra_pl);
|
||||
}
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
return job;
|
||||
if (found_config) {
|
||||
api_work_completion_t* c = calloc(1, sizeof(*c));
|
||||
if (c) {
|
||||
c->type = API_WORK_JOB_CACHING_TOGGLE;
|
||||
c->result = 0;
|
||||
api_worker_push_completion(c);
|
||||
}
|
||||
DEBUG_LOG("api-worker: config_changed notify received, enqueued CACHING_TOGGLE");
|
||||
}
|
||||
return found_config;
|
||||
}
|
||||
|
||||
static void* api_worker_main(void* arg) {
|
||||
(void)arg;
|
||||
pthread_setname_np(pthread_self(), "api-worker");
|
||||
|
||||
// Hold a dedicated DB worker connection for the full api-worker lifetime
|
||||
// to avoid per-query open/close churn in monitoring/stats paths.
|
||||
// Open a dedicated DB worker connection for the full api-worker lifetime.
|
||||
void* worker_db = NULL;
|
||||
if (api_open_temp_db_connection(&worker_db) != 0) {
|
||||
DEBUG_WARN("api-worker: failed to open dedicated DB connection, using fallback behavior");
|
||||
}
|
||||
|
||||
// CRITICAL: bind the worker connection to THIS thread. The __thread
|
||||
// pointer g_thread_pg_conn is set by db_open_worker_connection() in the
|
||||
// caller's thread (lws-main), not here. Without this rebind, monitoring
|
||||
// query functions fall back to the shared g_pg_conn and race with the
|
||||
// main thread — which is why the dashboard stopped receiving updates.
|
||||
if (worker_db) {
|
||||
db_set_thread_connection(worker_db);
|
||||
}
|
||||
|
||||
// Phase 5: register LISTEN event_stored on the worker connection so PG
|
||||
// can push notifications when new events are inserted. Returns -1 on
|
||||
// SQLite / unsupported backends; the worker then falls back to pure
|
||||
// timer-based polling.
|
||||
int listen_active = 0;
|
||||
if (worker_db) {
|
||||
if (db_worker_listen(worker_db, "event_stored") == 0) {
|
||||
listen_active = 1;
|
||||
DEBUG_LOG("api-worker: LISTEN event_stored registered");
|
||||
// Also LISTEN on the config_changed channel so the worker wakes
|
||||
// reactively when the PHP admin page toggles caching_enabled /
|
||||
// caching_inbox_enabled (or any other config key). The trigger
|
||||
// notify_config_changed() in pg_schema.sql fires
|
||||
// pg_notify('config_changed', NEW.key) on every config UPDATE.
|
||||
if (db_worker_listen(worker_db, "config_changed") == 0) {
|
||||
DEBUG_LOG("api-worker: LISTEN config_changed registered");
|
||||
} else {
|
||||
DEBUG_WARN("api-worker: LISTEN config_changed failed; config changes will not be reactive");
|
||||
}
|
||||
} else {
|
||||
DEBUG_LOG("api-worker: LISTEN not supported on this backend; using timer-only mode");
|
||||
}
|
||||
}
|
||||
|
||||
while (g_api_worker_running) {
|
||||
api_work_job_t* job = api_worker_pop_job_blocking();
|
||||
if (!job) {
|
||||
int throttle_sec = get_monitoring_throttle_seconds();
|
||||
if (throttle_sec <= 0) {
|
||||
// Monitoring disabled. Sleep briefly and re-check config, but
|
||||
// still poll for config_changed notifications so caching toggle
|
||||
// works even when monitoring is off.
|
||||
if (listen_active && worker_db) {
|
||||
(void)api_worker_poll_config_changed(worker_db, 1000);
|
||||
} else {
|
||||
struct timespec ts;
|
||||
ts.tv_sec = 1;
|
||||
ts.tv_nsec = 0;
|
||||
nanosleep(&ts, NULL);
|
||||
}
|
||||
// Still honor a STATUS_POST request if one arrived.
|
||||
if (api_worker_take_status_post()) {
|
||||
int rc = generate_and_post_status_event();
|
||||
api_work_completion_t* c = calloc(1, sizeof(*c));
|
||||
if (c) { c->type = API_WORK_JOB_STATUS_POST; c->result = rc; api_worker_push_completion(c); }
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
int result = 0;
|
||||
switch (job->type) {
|
||||
case API_WORK_JOB_EVENT_STORED:
|
||||
result = monitoring_on_event_stored_sync();
|
||||
break;
|
||||
case API_WORK_JOB_SUBSCRIPTION_CHANGE:
|
||||
result = monitoring_on_subscription_change_sync();
|
||||
break;
|
||||
case API_WORK_JOB_STATUS_POST:
|
||||
result = generate_and_post_status_event();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
// Is anyone listening to kind 24567 monitoring events?
|
||||
int has_subs = has_any_subscription_for_kind(24567);
|
||||
|
||||
if (!has_subs) {
|
||||
// No subscribers. Use the notify poll as the wait mechanism so we
|
||||
// still wake reactively for config_changed notifications. On
|
||||
// non-LISTEN backends, fall back to condvar wait.
|
||||
if (listen_active && worker_db) {
|
||||
(void)api_worker_poll_config_changed(worker_db, throttle_sec * 1000);
|
||||
} else {
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
if (g_api_worker_running && !g_api_status_post_pending) {
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += throttle_sec;
|
||||
pthread_cond_timedwait(&g_api_work_cond, &g_api_work_mutex, &deadline);
|
||||
}
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
}
|
||||
api_worker_drain_wake_pipe();
|
||||
if (api_worker_take_status_post()) {
|
||||
int rc = generate_and_post_status_event();
|
||||
api_work_completion_t* c = calloc(1, sizeof(*c));
|
||||
if (c) { c->type = API_WORK_JOB_STATUS_POST; c->result = rc; api_worker_push_completion(c); }
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
api_work_completion_t* completion = calloc(1, sizeof(*completion));
|
||||
if (completion) {
|
||||
completion->type = job->type;
|
||||
completion->result = result;
|
||||
api_worker_push_completion(completion);
|
||||
// Subscribers exist. Wait for either a NOTIFY, the throttle timeout,
|
||||
// or a self-pipe wake (STATUS_POST / shutdown) — whichever comes first.
|
||||
if (listen_active && worker_db) {
|
||||
(void)api_worker_poll_config_changed(worker_db, throttle_sec * 1000);
|
||||
} else {
|
||||
// No LISTEN support: sleep for throttle_sec on the condvar.
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
if (g_api_worker_running && !g_api_status_post_pending) {
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += throttle_sec;
|
||||
pthread_cond_timedwait(&g_api_work_cond, &g_api_work_mutex, &deadline);
|
||||
}
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
}
|
||||
|
||||
free(job);
|
||||
if (!g_api_worker_running) {
|
||||
break;
|
||||
}
|
||||
|
||||
api_worker_drain_wake_pipe();
|
||||
|
||||
// Run one round-robin monitoring tick (builds + signs + broadcasts
|
||||
// one kind 24567 event with the next d-tag in the schedule).
|
||||
(void)generate_round_robin_monitoring_event();
|
||||
|
||||
// Honor any STATUS_POST request that arrived during the wait.
|
||||
if (api_worker_take_status_post()) {
|
||||
int rc = generate_and_post_status_event();
|
||||
api_work_completion_t* c = calloc(1, sizeof(*c));
|
||||
if (c) { c->type = API_WORK_JOB_STATUS_POST; c->result = rc; api_worker_push_completion(c); }
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup: unbind the thread connection before closing it.
|
||||
if (worker_db) {
|
||||
db_clear_thread_connection();
|
||||
api_close_temp_db_connection(worker_db);
|
||||
}
|
||||
|
||||
@@ -222,6 +377,21 @@ int start_api_worker(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Create the self-pipe used to wake select()/condvar waits.
|
||||
if (g_api_wake_pipe[0] < 0) {
|
||||
if (pipe(g_api_wake_pipe) != 0) {
|
||||
g_api_wake_pipe[0] = g_api_wake_pipe[1] = -1;
|
||||
DEBUG_WARN("api-worker: pipe() failed; wakeups will rely on condvar only");
|
||||
} else {
|
||||
// Non-blocking both ends so wake() never blocks and drain is safe.
|
||||
int flags = fcntl(g_api_wake_pipe[1], F_GETFL, 0);
|
||||
if (flags >= 0) fcntl(g_api_wake_pipe[1], F_SETFL, flags | O_NONBLOCK);
|
||||
flags = fcntl(g_api_wake_pipe[0], F_GETFL, 0);
|
||||
if (flags >= 0) fcntl(g_api_wake_pipe[0], F_SETFL, flags | O_NONBLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
g_api_status_post_pending = 0;
|
||||
g_api_worker_running = 1;
|
||||
if (pthread_create(&g_api_worker_thread, NULL, api_worker_main, NULL) != 0) {
|
||||
g_api_worker_running = 0;
|
||||
@@ -238,41 +408,60 @@ void stop_api_worker(void) {
|
||||
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
g_api_worker_running = 0;
|
||||
g_api_status_post_pending = 0;
|
||||
pthread_cond_broadcast(&g_api_work_cond);
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
api_worker_wake();
|
||||
|
||||
pthread_join(g_api_worker_thread, NULL);
|
||||
|
||||
pthread_mutex_lock(&g_api_work_mutex);
|
||||
api_work_job_t* job = g_api_work_head;
|
||||
while (job) {
|
||||
api_work_job_t* next = job->next;
|
||||
free(job);
|
||||
job = next;
|
||||
}
|
||||
g_api_work_head = g_api_work_tail = NULL;
|
||||
pthread_mutex_unlock(&g_api_work_mutex);
|
||||
|
||||
api_work_completion_t* completion = NULL;
|
||||
while ((completion = api_worker_pop_completion()) != NULL) {
|
||||
free(completion);
|
||||
}
|
||||
|
||||
if (g_api_wake_pipe[0] >= 0) { close(g_api_wake_pipe[0]); g_api_wake_pipe[0] = -1; }
|
||||
if (g_api_wake_pipe[1] >= 0) { close(g_api_wake_pipe[1]); g_api_wake_pipe[1] = -1; }
|
||||
}
|
||||
|
||||
void api_worker_process_completions(void) {
|
||||
api_work_completion_t* completion = NULL;
|
||||
while ((completion = api_worker_pop_completion()) != NULL) {
|
||||
if (completion->result != 0) {
|
||||
if (completion->type == API_WORK_JOB_CACHING_TOGGLE) {
|
||||
// A config_changed NOTIFY was received by the worker. Re-read
|
||||
// caching_enabled from the config table and start/stop the
|
||||
// caching service process accordingly. This runs on the main
|
||||
// lws thread (the caller), which is the correct place to fork
|
||||
// — forking on the worker thread would let the child inherit
|
||||
// the worker's dedicated PG connection.
|
||||
//
|
||||
// Invalidate the in-memory config cache first — the PHP admin
|
||||
// page updates the config table directly via SQL, which fires
|
||||
// the NOTIFY trigger but does NOT invalidate our cache. Without
|
||||
// this, get_config_bool() would return the stale cached value.
|
||||
invalidate_config_cache();
|
||||
int caching_on = get_config_bool("caching_enabled", 0);
|
||||
int running = caching_service_is_running();
|
||||
DEBUG_LOG("api-worker: CACHING_TOGGLE — caching_enabled=%d, running=%d",
|
||||
caching_on, running);
|
||||
if (caching_on && !running) {
|
||||
int rc = caching_service_start();
|
||||
if (rc != 0) {
|
||||
DEBUG_ERROR("api-worker: caching_service_start() failed (rc=%d)", rc);
|
||||
}
|
||||
} else if (!caching_on && running) {
|
||||
int rc = caching_service_stop();
|
||||
if (rc != 0) {
|
||||
DEBUG_ERROR("api-worker: caching_service_stop() failed (rc=%d)", rc);
|
||||
}
|
||||
}
|
||||
} else if (completion->result != 0) {
|
||||
DEBUG_TRACE("api-worker job %d completed with rc=%d", (int)completion->type, completion->result);
|
||||
}
|
||||
free(completion);
|
||||
}
|
||||
}
|
||||
|
||||
int api_worker_enqueue_status_post(void) {
|
||||
return api_worker_enqueue_job(API_WORK_JOB_STATUS_POST);
|
||||
}
|
||||
|
||||
// Query event kind distribution from database
|
||||
cJSON* query_event_kind_distribution(void) {
|
||||
void* temp_db = NULL;
|
||||
@@ -366,6 +555,62 @@ cJSON* query_time_based_statistics(void) {
|
||||
return time_stats;
|
||||
}
|
||||
|
||||
// Attach profile fields (name, display_name, best_name, picture) from the
|
||||
// profiles cache to a pubkey_obj. The profile_map is a cJSON object keyed by
|
||||
// pubkey hex (from db_get_profiles). If the pubkey is not in the map, empty
|
||||
// defaults are added. The "name" field is set to best_name for backward
|
||||
// compatibility with the legacy api/ frontend.
|
||||
static void api_attach_profile_fields(cJSON* pubkey_obj, const char* pk_str,
|
||||
cJSON* profile_map) {
|
||||
if (!pubkey_obj || !pk_str) return;
|
||||
|
||||
if (profile_map && pk_str[0] != '\0') {
|
||||
cJSON* profile = cJSON_GetObjectItemCaseSensitive(profile_map, pk_str);
|
||||
if (profile) {
|
||||
cJSON* best = cJSON_GetObjectItemCaseSensitive(profile, "best_name");
|
||||
cJSON* name = cJSON_GetObjectItemCaseSensitive(profile, "name");
|
||||
cJSON* dn = cJSON_GetObjectItemCaseSensitive(profile, "display_name");
|
||||
cJSON* pic = cJSON_GetObjectItemCaseSensitive(profile, "picture");
|
||||
|
||||
/* "name" = best_name for backward compat with legacy api/ UI. */
|
||||
const char* best_str = (best && cJSON_IsString(best)) ? best->valuestring : "";
|
||||
cJSON_AddStringToObject(pubkey_obj, "name", best_str);
|
||||
if (name && cJSON_IsString(name))
|
||||
cJSON_AddStringToObject(pubkey_obj, "display_name_raw", name->valuestring);
|
||||
if (dn && cJSON_IsString(dn))
|
||||
cJSON_AddStringToObject(pubkey_obj, "display_name", dn->valuestring);
|
||||
if (pic && cJSON_IsString(pic) && pic->valuestring[0])
|
||||
cJSON_AddStringToObject(pubkey_obj, "picture", pic->valuestring);
|
||||
return;
|
||||
}
|
||||
}
|
||||
/* No profile cached — add empty defaults. */
|
||||
cJSON_AddStringToObject(pubkey_obj, "name", "");
|
||||
}
|
||||
|
||||
// Collect pubkeys from a rows array (each row has a "pubkey" string field)
|
||||
// into a NULL-terminated array suitable for db_get_profiles(). Returns a
|
||||
// malloc'd array; caller must free. Sets *out_count.
|
||||
static const char** collect_pubkeys_from_rows(cJSON* rows, int* out_count) {
|
||||
if (!rows || !out_count) return NULL;
|
||||
int n = cJSON_GetArraySize(rows);
|
||||
if (n <= 0) { *out_count = 0; return NULL; }
|
||||
|
||||
const char** pks = (const char**)malloc(sizeof(char*) * n);
|
||||
if (!pks) { *out_count = 0; return NULL; }
|
||||
|
||||
int count = 0;
|
||||
cJSON* row = NULL;
|
||||
cJSON_ArrayForEach(row, rows) {
|
||||
cJSON* pk_item = cJSON_GetObjectItemCaseSensitive(row, "pubkey");
|
||||
if (pk_item && cJSON_IsString(pk_item) && pk_item->valuestring[0] != '\0') {
|
||||
pks[count++] = pk_item->valuestring;
|
||||
}
|
||||
}
|
||||
*out_count = count;
|
||||
return pks;
|
||||
}
|
||||
|
||||
// Query top pubkeys by event count from database
|
||||
cJSON* query_top_pubkeys(void) {
|
||||
void* temp_db = NULL;
|
||||
@@ -386,6 +631,15 @@ cJSON* query_top_pubkeys(void) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Batch profile lookup: one query for all 10 pubkeys. */
|
||||
int pk_count = 0;
|
||||
const char** pks = collect_pubkeys_from_rows(rows, &pk_count);
|
||||
cJSON* profile_map = NULL;
|
||||
if (pks && pk_count > 0) {
|
||||
profile_map = db_get_profiles(pks, pk_count);
|
||||
}
|
||||
free(pks);
|
||||
|
||||
cJSON* pubkeys_array = cJSON_CreateArray();
|
||||
cJSON* row = NULL;
|
||||
cJSON_ArrayForEach(row, rows) {
|
||||
@@ -393,12 +647,16 @@ cJSON* query_top_pubkeys(void) {
|
||||
cJSON* pubkey_item = cJSON_GetObjectItemCaseSensitive(row, "pubkey");
|
||||
cJSON* count_item = cJSON_GetObjectItemCaseSensitive(row, "count");
|
||||
|
||||
cJSON_AddStringToObject(pubkey_obj, "pubkey",
|
||||
cJSON_IsString(pubkey_item) ? pubkey_item->valuestring : "");
|
||||
const char* pk_str = cJSON_IsString(pubkey_item) ? pubkey_item->valuestring : "";
|
||||
cJSON_AddStringToObject(pubkey_obj, "pubkey", pk_str);
|
||||
cJSON_AddNumberToObject(pubkey_obj, "event_count",
|
||||
cJSON_IsNumber(count_item) ? count_item->valuedouble : 0.0);
|
||||
|
||||
api_attach_profile_fields(pubkey_obj, pk_str, profile_map);
|
||||
|
||||
cJSON_AddItemToArray(pubkeys_array, pubkey_obj);
|
||||
}
|
||||
cJSON_Delete(profile_map);
|
||||
cJSON_Delete(rows);
|
||||
|
||||
long long total_events = 0;
|
||||
@@ -567,52 +825,48 @@ cJSON* query_subscription_details(void) {
|
||||
return subscriptions_data;
|
||||
}
|
||||
|
||||
// Generate event-driven monitoring events (triggered by event storage)
|
||||
int generate_event_driven_monitoring(void) {
|
||||
// Generate event_kinds monitoring event
|
||||
if (generate_monitoring_event_for_type("event_kinds", query_event_kind_distribution) != 0) {
|
||||
DEBUG_ERROR("Failed to generate event_kinds monitoring event");
|
||||
return -1;
|
||||
}
|
||||
static int generate_round_robin_monitoring_event(void) {
|
||||
static int rr_index = 0;
|
||||
|
||||
// Generate time_stats monitoring event
|
||||
if (generate_monitoring_event_for_type("time_stats", query_time_based_statistics) != 0) {
|
||||
DEBUG_ERROR("Failed to generate time_stats monitoring event");
|
||||
return -1;
|
||||
}
|
||||
struct monitor_event_def {
|
||||
const char* d_tag;
|
||||
cJSON* (*query_func)(void);
|
||||
};
|
||||
|
||||
// Generate top_pubkeys monitoring event
|
||||
if (generate_monitoring_event_for_type("top_pubkeys", query_top_pubkeys) != 0) {
|
||||
DEBUG_ERROR("Failed to generate top_pubkeys monitoring event");
|
||||
return -1;
|
||||
}
|
||||
// Weighted schedule: event_kinds is emitted most frequently since it drives
|
||||
// the New Events graph and Total Events counter in the admin UI.
|
||||
// Sequence cadence at throttle=5s: event_kinds(0s), cpu(5s), event_kinds(10s),
|
||||
// time_stats(15s), event_kinds(20s), top_pubkeys(25s)
|
||||
// => event_kinds every ~15s, cpu every ~30s, others every ~30s.
|
||||
static const struct monitor_event_def events[] = {
|
||||
{"event_kinds", query_event_kind_distribution},
|
||||
{"cpu_metrics", query_cpu_metrics},
|
||||
{"event_kinds", query_event_kind_distribution},
|
||||
{"time_stats", query_time_based_statistics},
|
||||
{"event_kinds", query_event_kind_distribution},
|
||||
{"top_pubkeys", query_top_pubkeys}
|
||||
};
|
||||
|
||||
const int event_count = (int)(sizeof(events) / sizeof(events[0]));
|
||||
const struct monitor_event_def* selected = &events[rr_index];
|
||||
rr_index = (rr_index + 1) % event_count;
|
||||
|
||||
// Generate CPU metrics monitoring event (also triggered by event storage)
|
||||
if (generate_monitoring_event_for_type("cpu_metrics", query_cpu_metrics) != 0) {
|
||||
DEBUG_ERROR("Failed to generate cpu_metrics monitoring event");
|
||||
if (generate_monitoring_event_for_type(selected->d_tag, selected->query_func) != 0) {
|
||||
DEBUG_ERROR("Failed to generate %s monitoring event", selected->d_tag);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Generate event-driven monitoring events (triggered by event storage)
|
||||
int generate_event_driven_monitoring(void) {
|
||||
return generate_round_robin_monitoring_event();
|
||||
}
|
||||
|
||||
// Generate subscription-driven monitoring events (triggered by subscription changes)
|
||||
int generate_subscription_driven_monitoring(void) {
|
||||
|
||||
// Generate subscription_details monitoring event (admin-only)
|
||||
if (generate_monitoring_event_for_type("subscription_details", query_subscription_details) != 0) {
|
||||
DEBUG_ERROR("Failed to generate subscription_details monitoring event");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Generate CPU metrics monitoring event (also triggered by subscription changes)
|
||||
if (generate_monitoring_event_for_type("cpu_metrics", query_cpu_metrics) != 0) {
|
||||
DEBUG_ERROR("Failed to generate cpu_metrics monitoring event");
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
return generate_round_robin_monitoring_event();
|
||||
}
|
||||
|
||||
// Generate and broadcast monitoring event (legacy function - now calls event-driven version)
|
||||
@@ -699,6 +953,11 @@ int generate_monitoring_event_for_type(const char* d_tag_value, cJSON* (*query_f
|
||||
// Broadcast the ephemeral event to active subscriptions (no database storage)
|
||||
broadcast_event_to_subscriptions(monitoring_event);
|
||||
|
||||
// If running on api-worker thread, wake the lws service loop so queued frames flush promptly.
|
||||
if (ws_context) {
|
||||
lws_cancel_service(ws_context);
|
||||
}
|
||||
|
||||
cJSON_Delete(monitoring_event);
|
||||
free(content_json);
|
||||
|
||||
@@ -706,68 +965,6 @@ int generate_monitoring_event_for_type(const char* d_tag_value, cJSON* (*query_f
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int monitoring_on_event_stored_sync(void) {
|
||||
// Check if monitoring is disabled (throttle = 0)
|
||||
int throttle_seconds = get_monitoring_throttle_seconds();
|
||||
if (throttle_seconds == 0) {
|
||||
return 0; // Monitoring disabled
|
||||
}
|
||||
|
||||
// Check throttling
|
||||
static time_t last_monitoring_time = 0;
|
||||
time_t current_time = time(NULL);
|
||||
|
||||
if (current_time - last_monitoring_time < throttle_seconds) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Check if anyone is subscribed to monitoring events (kind 24567)
|
||||
if (!has_subscriptions_for_kind(24567)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
last_monitoring_time = current_time;
|
||||
return generate_event_driven_monitoring();
|
||||
}
|
||||
|
||||
static int monitoring_on_subscription_change_sync(void) {
|
||||
// Check if monitoring is disabled (throttle = 0)
|
||||
int throttle_seconds = get_monitoring_throttle_seconds();
|
||||
if (throttle_seconds == 0) {
|
||||
return 0; // Monitoring disabled
|
||||
}
|
||||
|
||||
// Check throttling
|
||||
static time_t last_monitoring_time = 0;
|
||||
time_t current_time = time(NULL);
|
||||
|
||||
if (current_time - last_monitoring_time < throttle_seconds) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Check if anyone is subscribed to monitoring events (kind 24567)
|
||||
if (!has_subscriptions_for_kind(24567)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
last_monitoring_time = current_time;
|
||||
return generate_subscription_driven_monitoring();
|
||||
}
|
||||
|
||||
// Monitoring hook called when an event is stored
|
||||
void monitoring_on_event_stored(void) {
|
||||
if (api_worker_enqueue_job(API_WORK_JOB_EVENT_STORED) != 0) {
|
||||
(void)monitoring_on_event_stored_sync();
|
||||
}
|
||||
}
|
||||
|
||||
// Monitoring hook called when subscriptions change (create/close)
|
||||
void monitoring_on_subscription_change(void) {
|
||||
if (api_worker_enqueue_job(API_WORK_JOB_SUBSCRIPTION_CHANGE) != 0) {
|
||||
(void)monitoring_on_subscription_change_sync();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate and post a kind 1 status event with relay statistics
|
||||
* Called periodically from main event loop based on configuration
|
||||
@@ -909,7 +1106,7 @@ static const config_definition_t known_configs[] = {
|
||||
// Relay keys
|
||||
{"relay_pubkey", "string", 0, 65},
|
||||
{"relay_privkey", "string", 0, 65},
|
||||
{"admin_pubkey", "string", 0, 65},
|
||||
{"admin_pubkey", "string", 0, ADMIN_PUBKEY_LIST_MAX_LENGTH},
|
||||
|
||||
// Kind 1 Status Posts
|
||||
{"kind_1_status_posts_hours", "int", 0, 8760},
|
||||
@@ -1316,6 +1513,89 @@ int handle_embedded_file_writeable(struct lws* wsi) {
|
||||
|
||||
return 0;
|
||||
}
|
||||
static void add_relay_thread_snapshot(cJSON* cpu_stats, pid_t relay_pid) {
|
||||
if (!cpu_stats || relay_pid <= 0) return;
|
||||
|
||||
cJSON* threads = cJSON_CreateArray();
|
||||
if (!threads) return;
|
||||
|
||||
char cmd[256];
|
||||
snprintf(cmd, sizeof(cmd), "ps -T -p %d -o tid=,comm=,pcpu=,pmem=,etime=", relay_pid);
|
||||
FILE* fp = popen(cmd, "r");
|
||||
if (!fp) {
|
||||
cJSON_Delete(threads);
|
||||
return;
|
||||
}
|
||||
|
||||
char line[512];
|
||||
int row_count = 0;
|
||||
while (fgets(line, sizeof(line), fp) && row_count < 32) {
|
||||
int tid = 0;
|
||||
char comm[64] = {0};
|
||||
double pcpu = 0.0, pmem = 0.0;
|
||||
char elapsed[64] = {0};
|
||||
if (sscanf(line, "%d %63s %lf %lf %63s", &tid, comm, &pcpu, &pmem, elapsed) == 5) {
|
||||
cJSON* row = cJSON_CreateObject();
|
||||
if (row) {
|
||||
cJSON_AddNumberToObject(row, "tid", tid);
|
||||
cJSON_AddStringToObject(row, "thread", comm);
|
||||
cJSON_AddNumberToObject(row, "cpu_percent", pcpu);
|
||||
cJSON_AddNumberToObject(row, "mem_percent", pmem);
|
||||
cJSON_AddStringToObject(row, "elapsed", elapsed);
|
||||
cJSON_AddItemToArray(threads, row);
|
||||
row_count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
pclose(fp);
|
||||
|
||||
cJSON_AddItemToObject(cpu_stats, "relay_threads", threads);
|
||||
}
|
||||
|
||||
static void add_postgres_backend_snapshot(cJSON* cpu_stats) {
|
||||
if (!cpu_stats) return;
|
||||
|
||||
cJSON* backends = cJSON_CreateArray();
|
||||
if (!backends) return;
|
||||
|
||||
FILE* fp = popen("ps -C postgres -o pid=,comm=,pcpu=,pmem=,etime=,args=", "r");
|
||||
if (!fp) {
|
||||
cJSON_Delete(backends);
|
||||
return;
|
||||
}
|
||||
|
||||
char line[1024];
|
||||
int row_count = 0;
|
||||
while (fgets(line, sizeof(line), fp) && row_count < 48) {
|
||||
int pid = 0;
|
||||
char comm[64] = {0};
|
||||
double pcpu = 0.0, pmem = 0.0;
|
||||
char elapsed[64] = {0};
|
||||
char args[768] = {0};
|
||||
if (sscanf(line, "%d %63s %lf %lf %63s %767[^\n]", &pid, comm, &pcpu, &pmem, elapsed, args) >= 5) {
|
||||
cJSON* row = cJSON_CreateObject();
|
||||
if (row) {
|
||||
cJSON_AddNumberToObject(row, "pid", pid);
|
||||
cJSON_AddStringToObject(row, "command", comm);
|
||||
cJSON_AddNumberToObject(row, "cpu_percent", pcpu);
|
||||
cJSON_AddNumberToObject(row, "mem_percent", pmem);
|
||||
cJSON_AddStringToObject(row, "elapsed", elapsed);
|
||||
if (args[0] != '\0') {
|
||||
cJSON_AddStringToObject(row, "args", args);
|
||||
if (strstr(args, " c-relay-pg ") || strstr(args, " c-relay-pg") || strstr(args, " crelay ")) {
|
||||
cJSON_AddBoolToObject(row, "is_crelay_connection", 1);
|
||||
}
|
||||
}
|
||||
cJSON_AddItemToArray(backends, row);
|
||||
row_count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
pclose(fp);
|
||||
|
||||
cJSON_AddItemToObject(cpu_stats, "postgres_backends", backends);
|
||||
}
|
||||
|
||||
// Query CPU usage metrics
|
||||
cJSON* query_cpu_metrics(void) {
|
||||
cJSON* cpu_stats = cJSON_CreateObject();
|
||||
@@ -1358,6 +1638,10 @@ cJSON* query_cpu_metrics(void) {
|
||||
pid_t pid = getpid();
|
||||
cJSON_AddNumberToObject(cpu_stats, "process_id", (double)pid);
|
||||
|
||||
// Include relay thread/process snapshots (mirrors c-relay-pg-threads.sh style data)
|
||||
add_relay_thread_snapshot(cpu_stats, pid);
|
||||
add_postgres_backend_snapshot(cpu_stats);
|
||||
|
||||
// Get active WebSocket connection count
|
||||
cJSON_AddNumberToObject(cpu_stats, "active_connections", (double)get_active_connection_count());
|
||||
|
||||
@@ -1413,13 +1697,17 @@ char* generate_stats_json(void) {
|
||||
cJSON_AddStringToObject(response, "query_type", "stats_query");
|
||||
cJSON_AddNumberToObject(response, "timestamp", (double)time(NULL));
|
||||
|
||||
// Get database file size
|
||||
const char* db_path = db_get_database_path();
|
||||
struct stat db_stat;
|
||||
// Get database storage size (PostgreSQL only)
|
||||
long long db_size = 0;
|
||||
if (db_path && db_path[0] != '\0' && stat(db_path, &db_stat) == 0) {
|
||||
db_size = db_stat.st_size;
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
db_stmt_t* db_size_stmt = NULL;
|
||||
if (db_prepare("SELECT pg_database_size(current_database())", &db_size_stmt) == DB_OK) {
|
||||
if (db_step_stmt(db_size_stmt) == DB_ROW) {
|
||||
db_size = db_column_int64_value(db_size_stmt, 0);
|
||||
}
|
||||
db_finalize_stmt(db_size_stmt);
|
||||
}
|
||||
#endif
|
||||
cJSON_AddNumberToObject(response, "database_size_bytes", db_size);
|
||||
|
||||
// Get active subscriptions count from in-memory manager
|
||||
@@ -1475,6 +1763,15 @@ char* generate_stats_json(void) {
|
||||
cJSON* top_pubkeys = cJSON_CreateArray();
|
||||
cJSON* pubkey_rows = db_get_top_pubkeys_rows(10);
|
||||
if (pubkey_rows) {
|
||||
/* Batch profile lookup: one query for all pubkeys. */
|
||||
int pk_count = 0;
|
||||
const char** pks = collect_pubkeys_from_rows(pubkey_rows, &pk_count);
|
||||
cJSON* profile_map = NULL;
|
||||
if (pks && pk_count > 0) {
|
||||
profile_map = db_get_profiles(pks, pk_count);
|
||||
}
|
||||
free(pks);
|
||||
|
||||
cJSON* row = NULL;
|
||||
cJSON_ArrayForEach(row, pubkey_rows) {
|
||||
cJSON* pubkey_obj = cJSON_CreateObject();
|
||||
@@ -1483,11 +1780,14 @@ char* generate_stats_json(void) {
|
||||
double count_val = cJSON_IsNumber(count_item) ? count_item->valuedouble : 0.0;
|
||||
double pct = (total_events > 0) ? ((count_val * 100.0) / (double)total_events) : 0.0;
|
||||
|
||||
cJSON_AddStringToObject(pubkey_obj, "pubkey", cJSON_IsString(pubkey_item) ? pubkey_item->valuestring : "");
|
||||
const char* pk_str = cJSON_IsString(pubkey_item) ? pubkey_item->valuestring : "";
|
||||
cJSON_AddStringToObject(pubkey_obj, "pubkey", pk_str);
|
||||
cJSON_AddNumberToObject(pubkey_obj, "event_count", count_val);
|
||||
cJSON_AddNumberToObject(pubkey_obj, "percentage", pct);
|
||||
api_attach_profile_fields(pubkey_obj, pk_str, profile_map);
|
||||
cJSON_AddItemToArray(top_pubkeys, pubkey_obj);
|
||||
}
|
||||
cJSON_Delete(profile_map);
|
||||
cJSON_Delete(pubkey_rows);
|
||||
}
|
||||
cJSON_AddItemToObject(response, "top_pubkeys", top_pubkeys);
|
||||
@@ -1775,13 +2075,13 @@ char* generate_stats_text(void) {
|
||||
// Database Overview section
|
||||
offset += snprintf(stats_text + offset, 16384 - offset,
|
||||
|
||||
"Database Size %.2f MB (%lld bytes)\n"
|
||||
"Database Size %.2f MB\n"
|
||||
"Total Events %lld\n"
|
||||
"Active Subscriptions %d\n"
|
||||
"Oldest Event %s\n"
|
||||
"Newest Event %s\n"
|
||||
"\n",
|
||||
db_mb, db_bytes, total, active_subs, oldest_str, newest_str);
|
||||
db_mb, total, active_subs, oldest_str, newest_str);
|
||||
|
||||
// Event Kind Distribution section
|
||||
offset += snprintf(stats_text + offset, 16384 - offset,
|
||||
|
||||
@@ -60,8 +60,6 @@ char* execute_sql_query(const char* query, const char* request_id, char* error_m
|
||||
int handle_sql_query_unified(cJSON* event, const char* query, char* error_message, size_t error_size, struct lws* wsi);
|
||||
|
||||
// Monitoring system functions
|
||||
void monitoring_on_event_stored(void);
|
||||
void monitoring_on_subscription_change(void);
|
||||
int get_monitoring_throttle_seconds(void);
|
||||
|
||||
// Dedicated API worker thread lifecycle
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* Caching Inbox Poller
|
||||
*
|
||||
* Periodically dequeues events from the PostgreSQL `caching_event_inbox` table
|
||||
* and feeds them through the shared ingest_event() pipeline with
|
||||
* EVENT_SOURCE_CACHING_INBOX. The poller runs on the main libwebsockets
|
||||
* service thread (no separate thread) and is compiled only for the PostgreSQL
|
||||
* backend. For SQLite builds every entry point is a no-op.
|
||||
*
|
||||
* The poller maintains a simple two-state machine:
|
||||
* POLLER_IDLE - slow poll interval (no rows were found last cycle)
|
||||
* POLLER_ACTIVE - fast poll interval (a full batch was dequeued last cycle)
|
||||
*/
|
||||
|
||||
#include "caching_inbox_poller.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "debug.h"
|
||||
#include "config.h"
|
||||
#include "db_ops.h"
|
||||
#include "main.h"
|
||||
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
#include <cjson/cJSON.h>
|
||||
#endif
|
||||
|
||||
// ---- Configuration keys (read each tick) -------------------------------------
|
||||
|
||||
#define CFG_KEY_ENABLED "caching_inbox_enabled"
|
||||
#define CFG_KEY_BATCH_SIZE "caching_inbox_batch_size"
|
||||
#define CFG_KEY_ACTIVE_POLL_MS "caching_inbox_active_poll_ms"
|
||||
#define CFG_KEY_IDLE_POLL_MS "caching_inbox_idle_poll_ms"
|
||||
|
||||
// ---- Defaults ----------------------------------------------------------------
|
||||
|
||||
#define DEFAULT_BATCH_SIZE 150
|
||||
#define DEFAULT_ACTIVE_POLL_MS 200
|
||||
#define DEFAULT_IDLE_POLL_MS 5000
|
||||
|
||||
// ---- State machine -----------------------------------------------------------
|
||||
|
||||
typedef enum {
|
||||
POLLER_IDLE = 0,
|
||||
POLLER_ACTIVE = 1
|
||||
} poller_state_t;
|
||||
|
||||
// ---- Module state (PostgreSQL only; trivial for SQLite) ----------------------
|
||||
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
static poller_state_t g_state = POLLER_IDLE;
|
||||
static long long g_last_poll_ms = 0; // monotonic-ish ms timestamp of last poll
|
||||
static int g_initialized = 0;
|
||||
|
||||
// Statistics counters
|
||||
static int g_total_dequeued = 0;
|
||||
static int g_total_accepted = 0;
|
||||
static int g_total_rejected = 0;
|
||||
static int g_total_duplicates = 0;
|
||||
static int g_last_batch_size = 0;
|
||||
#endif
|
||||
|
||||
// ---- Helpers -----------------------------------------------------------------
|
||||
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
// Return a coarse millisecond timestamp using time(NULL) scaled to ms.
|
||||
// We avoid clock_gettime(CLOCK_MONOTONIC) portability concerns and keep the
|
||||
// resolution sufficient for the configured poll intervals (>= 200ms).
|
||||
static long long now_ms(void) {
|
||||
return (long long)time(NULL) * 1000LL;
|
||||
}
|
||||
#endif
|
||||
|
||||
// ---- Public API --------------------------------------------------------------
|
||||
|
||||
int caching_inbox_poller_init(void) {
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
g_state = POLLER_IDLE;
|
||||
g_last_poll_ms = 0;
|
||||
g_total_dequeued = 0;
|
||||
g_total_accepted = 0;
|
||||
g_total_rejected = 0;
|
||||
g_total_duplicates = 0;
|
||||
g_last_batch_size = 0;
|
||||
g_initialized = 1;
|
||||
DEBUG_LOG("caching_inbox_poller: initialized (PostgreSQL backend)");
|
||||
#else
|
||||
// SQLite build: no-op. The inbox feature is PostgreSQL-only.
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
int caching_inbox_poller_tick(void) {
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
if (!g_initialized) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Read configuration each tick (kept simple; values are cheap to read).
|
||||
int enabled = get_config_bool(CFG_KEY_ENABLED, 0);
|
||||
if (!enabled) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int batch_size = get_config_int(CFG_KEY_BATCH_SIZE, DEFAULT_BATCH_SIZE);
|
||||
if (batch_size <= 0) {
|
||||
batch_size = DEFAULT_BATCH_SIZE;
|
||||
}
|
||||
int active_poll_ms = get_config_int(CFG_KEY_ACTIVE_POLL_MS, DEFAULT_ACTIVE_POLL_MS);
|
||||
if (active_poll_ms < 0) {
|
||||
active_poll_ms = DEFAULT_ACTIVE_POLL_MS;
|
||||
}
|
||||
int idle_poll_ms = get_config_int(CFG_KEY_IDLE_POLL_MS, DEFAULT_IDLE_POLL_MS);
|
||||
if (idle_poll_ms < 0) {
|
||||
idle_poll_ms = DEFAULT_IDLE_POLL_MS;
|
||||
}
|
||||
|
||||
// Throttle: only poll when the configured interval has elapsed.
|
||||
long long now = now_ms();
|
||||
long long interval_ms = (g_state == POLLER_ACTIVE) ? active_poll_ms : idle_poll_ms;
|
||||
if (g_last_poll_ms != 0 && (now - g_last_poll_ms) < interval_ms) {
|
||||
return 0;
|
||||
}
|
||||
g_last_poll_ms = now;
|
||||
|
||||
// Dequeue a bounded batch in a single SQL transaction.
|
||||
int count = 0;
|
||||
cJSON* rows = db_caching_inbox_dequeue_batch(batch_size, &count);
|
||||
if (count <= 0 || rows == NULL) {
|
||||
// Empty queue (or dequeue failure). Back off to idle mode.
|
||||
if (rows != NULL) {
|
||||
// Defensive: count==0 but a non-null array was returned.
|
||||
cJSON_Delete(rows);
|
||||
}
|
||||
if (g_state == POLLER_ACTIVE) {
|
||||
DEBUG_TRACE("caching_inbox_poller: queue empty, switching to IDLE");
|
||||
}
|
||||
g_state = POLLER_IDLE;
|
||||
g_last_batch_size = 0;
|
||||
// Note: db_caching_inbox_dequeue_batch logs its own errors on failure;
|
||||
// a NULL return with count==0 is the normal "empty queue" case.
|
||||
return 0;
|
||||
}
|
||||
|
||||
DEBUG_LOG("caching_inbox_poller: dequeued %d event(s) (batch_size=%d, state=%s)",
|
||||
count, batch_size, g_state == POLLER_ACTIVE ? "ACTIVE" : "IDLE");
|
||||
|
||||
g_last_batch_size = count;
|
||||
g_total_dequeued += count;
|
||||
|
||||
// Feed each dequeued event through the shared ingestion pipeline.
|
||||
cJSON* row = NULL;
|
||||
cJSON_ArrayForEach(row, rows) {
|
||||
cJSON* event_json_obj = cJSON_GetObjectItemCaseSensitive(row, "event_json");
|
||||
if (!event_json_obj || !cJSON_IsString(event_json_obj)) {
|
||||
DEBUG_WARN("caching_inbox_poller: row missing event_json string; skipping");
|
||||
g_total_rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const char* event_json = cJSON_GetStringValue(event_json_obj);
|
||||
size_t event_json_len = strlen(event_json);
|
||||
|
||||
int rc = ingest_event(event_json, event_json_len,
|
||||
EVENT_SOURCE_CACHING_INBOX, NULL, NULL);
|
||||
if (rc == 0) {
|
||||
g_total_accepted++;
|
||||
} else {
|
||||
g_total_rejected++;
|
||||
DEBUG_WARN("caching_inbox_poller: ingest_event rejected an event (rc=%d)", rc);
|
||||
}
|
||||
}
|
||||
|
||||
cJSON_Delete(rows);
|
||||
|
||||
// State transition: a full batch suggests more work is waiting.
|
||||
if (count >= batch_size) {
|
||||
if (g_state != POLLER_ACTIVE) {
|
||||
DEBUG_TRACE("caching_inbox_poller: full batch, switching to ACTIVE");
|
||||
}
|
||||
g_state = POLLER_ACTIVE;
|
||||
} else {
|
||||
if (g_state != POLLER_IDLE) {
|
||||
DEBUG_TRACE("caching_inbox_poller: partial batch, switching to IDLE");
|
||||
}
|
||||
g_state = POLLER_IDLE;
|
||||
}
|
||||
|
||||
return 0;
|
||||
#else
|
||||
// SQLite build: no-op.
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void caching_inbox_poller_shutdown(void) {
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
DEBUG_LOG("caching_inbox_poller: shutdown (dequeued=%d accepted=%d rejected=%d)",
|
||||
g_total_dequeued, g_total_accepted, g_total_rejected);
|
||||
g_initialized = 0;
|
||||
g_state = POLLER_IDLE;
|
||||
g_last_poll_ms = 0;
|
||||
g_last_batch_size = 0;
|
||||
#else
|
||||
// SQLite build: no-op.
|
||||
#endif
|
||||
}
|
||||
|
||||
void caching_inbox_poller_get_stats(int* out_total_dequeued,
|
||||
int* out_total_accepted,
|
||||
int* out_total_rejected,
|
||||
int* out_total_duplicates,
|
||||
int* out_last_batch_size,
|
||||
int* out_pending_live,
|
||||
int* out_pending_backfill,
|
||||
int* out_oldest_age_seconds) {
|
||||
#ifdef DB_BACKEND_POSTGRES
|
||||
if (out_total_dequeued) *out_total_dequeued = g_total_dequeued;
|
||||
if (out_total_accepted) *out_total_accepted = g_total_accepted;
|
||||
if (out_total_rejected) *out_total_rejected = g_total_rejected;
|
||||
if (out_total_duplicates) *out_total_duplicates = g_total_duplicates;
|
||||
if (out_last_batch_size) *out_last_batch_size = g_last_batch_size;
|
||||
|
||||
// Pending counts and oldest age come from the database; query on demand.
|
||||
int live = 0, backfill = 0, oldest = 0;
|
||||
if (out_pending_live || out_pending_backfill) {
|
||||
if (db_caching_inbox_pending_counts(&live, &backfill) != 0) {
|
||||
live = 0;
|
||||
backfill = 0;
|
||||
}
|
||||
}
|
||||
if (out_pending_live) *out_pending_live = live;
|
||||
if (out_pending_backfill) *out_pending_backfill = backfill;
|
||||
|
||||
if (out_oldest_age_seconds) {
|
||||
if (db_caching_inbox_oldest_age(&oldest) != 0) {
|
||||
oldest = 0;
|
||||
}
|
||||
*out_oldest_age_seconds = oldest;
|
||||
}
|
||||
#else
|
||||
if (out_total_dequeued) *out_total_dequeued = 0;
|
||||
if (out_total_accepted) *out_total_accepted = 0;
|
||||
if (out_total_rejected) *out_total_rejected = 0;
|
||||
if (out_total_duplicates) *out_total_duplicates = 0;
|
||||
if (out_last_batch_size) *out_last_batch_size = 0;
|
||||
if (out_pending_live) *out_pending_live = 0;
|
||||
if (out_pending_backfill) *out_pending_backfill = 0;
|
||||
if (out_oldest_age_seconds) *out_oldest_age_seconds = 0;
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#ifndef CACHING_INBOX_POLLER_H
|
||||
#define CACHING_INBOX_POLLER_H
|
||||
|
||||
// Initialize the inbox poller. Called once at startup (PostgreSQL only).
|
||||
// Returns 0 on success, -1 on error.
|
||||
int caching_inbox_poller_init(void);
|
||||
|
||||
// Perform one polling cycle. Called from the main lws service loop.
|
||||
// Dequeues a bounded batch and feeds events through ingest_event().
|
||||
// Returns 0 on success, -1 on error.
|
||||
int caching_inbox_poller_tick(void);
|
||||
|
||||
// Shut down the poller and free resources.
|
||||
void caching_inbox_poller_shutdown(void);
|
||||
|
||||
// Get current poller statistics for status display.
|
||||
// All out-parameters are optional (may be NULL).
|
||||
void caching_inbox_poller_get_stats(int* out_total_dequeued,
|
||||
int* out_total_accepted,
|
||||
int* out_total_rejected,
|
||||
int* out_total_duplicates,
|
||||
int* out_last_batch_size,
|
||||
int* out_pending_live,
|
||||
int* out_pending_backfill,
|
||||
int* out_oldest_age_seconds);
|
||||
|
||||
#endif // CACHING_INBOX_POLLER_H
|
||||
@@ -0,0 +1,277 @@
|
||||
#define _GNU_SOURCE
|
||||
#include "caching_service_launcher.h"
|
||||
#include "config.h"
|
||||
#include "debug.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <signal.h>
|
||||
#include <sys/wait.h>
|
||||
#include <sys/types.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
// Tracked child PID for fork mode.
|
||||
// -1 means not running (or not launched by us).
|
||||
static pid_t g_caching_child_pid = -1;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fork/exec implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static int caching_service_start_fork(const char* binary_path, const char* pg_conn) {
|
||||
if (!binary_path || binary_path[0] == '\0') {
|
||||
DEBUG_ERROR("caching_service_start: caching_service_binary_path is not set");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!pg_conn || pg_conn[0] == '\0') {
|
||||
DEBUG_ERROR("caching_service_start: caching_service_pg_conn is not set");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Check that the binary exists and is executable
|
||||
if (access(binary_path, X_OK) != 0) {
|
||||
DEBUG_ERROR("caching_service_start: binary '%s' not found or not executable: %s",
|
||||
binary_path, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
// If already running, don't start again
|
||||
if (g_caching_child_pid > 0) {
|
||||
// Check if the process is still alive
|
||||
int status;
|
||||
pid_t result = waitpid(g_caching_child_pid, &status, WNOHANG);
|
||||
if (result == 0) {
|
||||
// Still running
|
||||
DEBUG_WARN("caching_service_start: caching service already running (PID %d)",
|
||||
g_caching_child_pid);
|
||||
return 0;
|
||||
}
|
||||
// Process exited — reset PID
|
||||
g_caching_child_pid = -1;
|
||||
}
|
||||
|
||||
DEBUG_INFO("caching_service_start: forking '%s' with -p (PG mode)", binary_path);
|
||||
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
DEBUG_ERROR("caching_service_start: fork failed: %s", strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pid == 0) {
|
||||
// Child process
|
||||
// Detach from parent's process group so signals to the relay
|
||||
// don't automatically propagate to the caching service.
|
||||
setsid();
|
||||
|
||||
// Redirect stdin from /dev/null
|
||||
int devnull = open("/dev/null", O_RDONLY);
|
||||
if (devnull >= 0) {
|
||||
dup2(devnull, STDIN_FILENO);
|
||||
close(devnull);
|
||||
}
|
||||
|
||||
// Redirect stdout/stderr to a log file for debugging.
|
||||
// The log file is created in the relay's working directory (build/).
|
||||
int logfile = open("caching_relay.log", O_WRONLY | O_CREAT | O_APPEND, 0644);
|
||||
if (logfile >= 0) {
|
||||
dup2(logfile, STDOUT_FILENO);
|
||||
dup2(logfile, STDERR_FILENO);
|
||||
close(logfile);
|
||||
} else {
|
||||
// Fallback to /dev/null if log file can't be opened
|
||||
devnull = open("/dev/null", O_WRONLY);
|
||||
if (devnull >= 0) {
|
||||
dup2(devnull, STDOUT_FILENO);
|
||||
dup2(devnull, STDERR_FILENO);
|
||||
close(devnull);
|
||||
}
|
||||
}
|
||||
|
||||
// Build argv for the caching_relay binary.
|
||||
// The caching_relay binary supports PostgreSQL mode via -p <pg-conn>,
|
||||
// which reads all config from the c-relay-pg config table and writes
|
||||
// fetched events to the caching_event_inbox table. No JSON config file
|
||||
// is needed in this mode.
|
||||
char* argv[5];
|
||||
int argc = 0;
|
||||
argv[argc++] = (char*)binary_path;
|
||||
argv[argc++] = (char*)"-p";
|
||||
argv[argc++] = (char*)pg_conn;
|
||||
argv[argc] = NULL;
|
||||
|
||||
execv(binary_path, argv);
|
||||
|
||||
// If we get here, exec failed
|
||||
// Use _exit to avoid flushing parent's stdio buffers
|
||||
_exit(127);
|
||||
}
|
||||
|
||||
// Parent process
|
||||
g_caching_child_pid = pid;
|
||||
DEBUG_INFO("caching_service_start: caching service started (PID %d)", pid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int caching_service_stop_fork(void) {
|
||||
if (g_caching_child_pid <= 0) {
|
||||
DEBUG_LOG("caching_service_stop: caching service not running");
|
||||
return 0;
|
||||
}
|
||||
|
||||
DEBUG_INFO("caching_service_stop: sending SIGTERM to PID %d", g_caching_child_pid);
|
||||
|
||||
if (kill(g_caching_child_pid, SIGTERM) != 0) {
|
||||
if (errno == ESRCH) {
|
||||
// Process already exited
|
||||
DEBUG_LOG("caching_service_stop: process already exited");
|
||||
g_caching_child_pid = -1;
|
||||
return 0;
|
||||
}
|
||||
DEBUG_ERROR("caching_service_stop: kill failed: %s", strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Wait briefly for graceful shutdown
|
||||
int status;
|
||||
int wait_ms = 0;
|
||||
while (wait_ms < 5000) {
|
||||
pid_t result = waitpid(g_caching_child_pid, &status, WNOHANG);
|
||||
if (result == g_caching_child_pid) {
|
||||
// Child exited
|
||||
if (WIFEXITED(status)) {
|
||||
DEBUG_INFO("caching_service_stop: process exited with status %d",
|
||||
WEXITSTATUS(status));
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
DEBUG_INFO("caching_service_stop: process killed by signal %d",
|
||||
WTERMSIG(status));
|
||||
}
|
||||
g_caching_child_pid = -1;
|
||||
return 0;
|
||||
}
|
||||
usleep(100000); // 100ms
|
||||
wait_ms += 100;
|
||||
}
|
||||
|
||||
// Force kill if still running after 5 seconds
|
||||
DEBUG_WARN("caching_service_stop: process did not exit gracefully, sending SIGKILL");
|
||||
kill(g_caching_child_pid, SIGKILL);
|
||||
waitpid(g_caching_child_pid, &status, 0);
|
||||
g_caching_child_pid = -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int caching_service_is_running_fork(void) {
|
||||
if (g_caching_child_pid <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int status;
|
||||
pid_t result = waitpid(g_caching_child_pid, &status, WNOHANG);
|
||||
if (result == 0) {
|
||||
return 1; // Still running
|
||||
}
|
||||
|
||||
// Process exited
|
||||
g_caching_child_pid = -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Systemd implementation (stub — not yet implemented)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static int caching_service_start_systemd(void) {
|
||||
DEBUG_ERROR("caching_service_start: systemd launch mode not yet implemented");
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int caching_service_stop_systemd(void) {
|
||||
DEBUG_ERROR("caching_service_stop: systemd launch mode not yet implemented");
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int caching_service_is_running_systemd(void) {
|
||||
// Could check systemctl is-active caching-relay
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
int caching_service_start(void) {
|
||||
const char* launch_mode = get_config_value("caching_service_launch_mode");
|
||||
if (!launch_mode || strcmp(launch_mode, "fork") == 0) {
|
||||
const char* binary_path = get_config_value("caching_service_binary_path");
|
||||
// The caching_relay binary uses PostgreSQL mode (-p <pg-conn>) to read
|
||||
// all config from the c-relay-pg config table and write events to the
|
||||
// caching_event_inbox table. No JSON config file is needed.
|
||||
const char* pg_conn = get_config_value("caching_service_pg_conn");
|
||||
int rc = caching_service_start_fork(binary_path, pg_conn);
|
||||
if (binary_path) free((char*)binary_path);
|
||||
if (pg_conn) free((char*)pg_conn);
|
||||
if (launch_mode) free((char*)launch_mode);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (strcmp(launch_mode, "systemd") == 0) {
|
||||
free((char*)launch_mode);
|
||||
return caching_service_start_systemd();
|
||||
}
|
||||
|
||||
DEBUG_ERROR("caching_service_start: unknown launch mode '%s'", launch_mode);
|
||||
free((char*)launch_mode);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int caching_service_stop(void) {
|
||||
const char* launch_mode = get_config_value("caching_service_launch_mode");
|
||||
if (!launch_mode || strcmp(launch_mode, "fork") == 0) {
|
||||
if (launch_mode) free((char*)launch_mode);
|
||||
return caching_service_stop_fork();
|
||||
}
|
||||
|
||||
if (strcmp(launch_mode, "systemd") == 0) {
|
||||
free((char*)launch_mode);
|
||||
return caching_service_stop_systemd();
|
||||
}
|
||||
|
||||
free((char*)launch_mode);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int caching_service_is_running(void) {
|
||||
const char* launch_mode = get_config_value("caching_service_launch_mode");
|
||||
int rc;
|
||||
if (!launch_mode || strcmp(launch_mode, "fork") == 0) {
|
||||
rc = caching_service_is_running_fork();
|
||||
} else if (strcmp(launch_mode, "systemd") == 0) {
|
||||
rc = caching_service_is_running_systemd();
|
||||
} else {
|
||||
rc = 0;
|
||||
}
|
||||
if (launch_mode) free((char*)launch_mode);
|
||||
return rc;
|
||||
}
|
||||
|
||||
int caching_service_get_pid(void) {
|
||||
if (caching_service_is_running_fork()) {
|
||||
return (int)g_caching_child_pid;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int caching_service_shutdown(void) {
|
||||
// Only stop if we launched it (fork mode with a tracked PID)
|
||||
if (g_caching_child_pid > 0) {
|
||||
DEBUG_INFO("caching_service_shutdown: stopping caching service (PID %d)",
|
||||
g_caching_child_pid);
|
||||
return caching_service_stop_fork();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef CACHING_SERVICE_LAUNCHER_H
|
||||
#define CACHING_SERVICE_LAUNCHER_H
|
||||
|
||||
// Caching service process launcher.
|
||||
//
|
||||
// Supports two launch modes controlled by the "caching_service_launch_mode" config key:
|
||||
// "fork" — c-relay-pg forks/execs the caching_relay binary directly (default)
|
||||
// "systemd" — delegates to systemctl start/stop (not yet implemented, returns error)
|
||||
//
|
||||
// The binary path is read from "caching_service_binary_path".
|
||||
// The PostgreSQL connection string is read from "caching_service_pg_conn"
|
||||
// and passed to the binary via the --pg-conn argument.
|
||||
|
||||
// Start the caching service.
|
||||
// Returns 0 on success, -1 on error.
|
||||
// On success in fork mode, the child PID is tracked internally.
|
||||
int caching_service_start(void);
|
||||
|
||||
// Stop the caching service.
|
||||
// Returns 0 on success (or if not running), -1 on error.
|
||||
// In fork mode, sends SIGTERM to the tracked child process.
|
||||
int caching_service_stop(void);
|
||||
|
||||
// Check if the caching service is currently running.
|
||||
// Returns 1 if running, 0 if not running, -1 on error.
|
||||
int caching_service_is_running(void);
|
||||
|
||||
// Get the current child PID (fork mode only).
|
||||
// Returns the PID, or -1 if not running or not in fork mode.
|
||||
int caching_service_get_pid(void);
|
||||
|
||||
// Shut down the caching service if it was launched by us.
|
||||
// Called during c-relay-pg graceful shutdown.
|
||||
// Returns 0 on success, -1 on error.
|
||||
int caching_service_shutdown(void);
|
||||
|
||||
#endif // CACHING_SERVICE_LAUNCHER_H
|
||||
+1290
-102
File diff suppressed because it is too large
Load Diff
+15
-2
@@ -15,6 +15,9 @@ struct lws;
|
||||
#define RELAY_URL_MAX_LENGTH 512
|
||||
#define RELAY_PUBKEY_MAX_LENGTH 65
|
||||
#define RELAY_CONTACT_MAX_LENGTH 256
|
||||
#define MAX_ADMIN_PUBKEYS 10
|
||||
#define ADMIN_PUBKEY_HEX_LEN 64
|
||||
#define ADMIN_PUBKEY_LIST_MAX_LENGTH ((MAX_ADMIN_PUBKEYS * ADMIN_PUBKEY_HEX_LEN) + (MAX_ADMIN_PUBKEYS - 1) + 1)
|
||||
#define SUBSCRIPTION_ID_MAX_LENGTH 64
|
||||
#define CLIENT_IP_MAX_LENGTH 46
|
||||
#define MAX_SUBSCRIPTIONS_PER_CLIENT 25
|
||||
@@ -29,10 +32,12 @@ extern char g_database_path[512];
|
||||
// Command line options structure for first-time startup
|
||||
typedef struct {
|
||||
int port_override; // -1 = not set, >0 = port value
|
||||
char admin_pubkey_override[65]; // Empty string = not set, 64-char hex = override
|
||||
char admin_pubkey_override[ADMIN_PUBKEY_LIST_MAX_LENGTH]; // Empty string = not set, comma-separated 64-char hex pubkeys
|
||||
char relay_privkey_override[65]; // Empty string = not set, 64-char hex = override
|
||||
int strict_port; // 0 = allow port increment, 1 = fail if exact port unavailable
|
||||
int debug_level; // 0-5, default 0 (no debug output)
|
||||
int start_caching; // 0 = don't auto-start, 1 = start caching service on startup
|
||||
int reset_backfill; // 0 = no, 1 = clear caching_backfill_progress before starting
|
||||
char db_connstring_override[1024];
|
||||
char db_host[128];
|
||||
int db_port; // 0 = not set
|
||||
@@ -67,7 +72,7 @@ int get_config_int(const char* key, int default_value);
|
||||
int get_config_bool(const char* key, int default_value);
|
||||
|
||||
// First-time startup functions
|
||||
int is_first_time_startup(void);
|
||||
int is_first_time_startup(const char* db_connection_target);
|
||||
int first_time_startup_sequence(const cli_options_t* cli_options, char* admin_pubkey_out, char* relay_pubkey_out, char* relay_privkey_out);
|
||||
int startup_existing_relay(const char* relay_pubkey, const cli_options_t* cli_options);
|
||||
|
||||
@@ -134,6 +139,14 @@ const char* get_relay_pubkey_cached(void);
|
||||
void invalidate_config_cache(void);
|
||||
int reload_config_from_table(void);
|
||||
|
||||
// Multi-admin helpers
|
||||
int reload_admin_pubkeys_cache(void);
|
||||
int is_admin_pubkey(const char* pubkey);
|
||||
int get_admin_pubkey_count(void);
|
||||
const char* get_primary_admin_pubkey(void);
|
||||
int add_admin_pubkey(const char* pubkey);
|
||||
int remove_admin_pubkey(const char* pubkey);
|
||||
|
||||
// Hybrid config access functions
|
||||
const char* get_config_value_hybrid(const char* key);
|
||||
int is_config_table_ready(void);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user