mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 23:18:24 +00:00
fix(relay): preserve completed ID query delivery for clients
The SDK sends CLOSED immediately after EOSE for completed ID filters. That triggers a cancellation race in nak 0.20.6: 39 of 45 production lookups returned no event although raw queries received every response. Pin the upstream SDK fixes and disable its automatic completed-ID closure. Clients retain ordinary CLOSE ownership. The SDK also fixes its aggregate query cap, so configure that cap as 20 per-filter pages to preserve our existing documented multi-filter limit instead of truncating history. Keep shared Nostr types on one Git source and update all flake packages and the NixOS module with the verified source hash. This relies on clients closing finished subscriptions; existing connection and subscription bounds still apply. It does not change the configured per-filter limit or fix cancellation in nak when using other servers. Validation: the real-server nak regression failed before and passes 20 consecutive lookups after the fix. Multi-filter overlap/EOSE regression, 32-reader LMDB benchmark and delayed-ACK benchmark pass. Module rendering preserves the default filter limit; full workspace and Nix builds follow.
This commit is contained in:
DanConwayDev
Generated
+12
-17
@@ -1359,9 +1359,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nostr"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7279feef28d43a04c0b3529acd4fea923ac816729bd643191b292296f1f6cc13"
|
||||
version = "0.45.3"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bech32",
|
||||
@@ -1379,13 +1378,13 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
"universal-time",
|
||||
"url",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nostr-database"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81c7ffd33114d62c737dd2b8f77ffec4693c64b7a1c6090cab0d0009f9986e22"
|
||||
version = "0.45.1"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"nostr",
|
||||
"opaquerr",
|
||||
@@ -1394,8 +1393,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "nostr-gossip"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa07539e52a71cb91fe0d693facaa298f03fcf9edcd66a521094e18e286e2336"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"nostr",
|
||||
"opaquerr",
|
||||
@@ -1403,9 +1401,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nostr-lmdb"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7a313418ff58bfa444ba4b71200ff8fe2d3eb31f2b5d0d8fdb7f5e60a703d31"
|
||||
version = "0.45.2"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"async-utility",
|
||||
"flatbuffers",
|
||||
@@ -1419,9 +1416,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nostr-memory"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a827de4b11c4f2b235eabe37f1a67460db68e453606f4755e1ac5a522347a86"
|
||||
version = "0.45.1"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"btreecap",
|
||||
"nostr",
|
||||
@@ -1431,9 +1427,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nostr-sdk"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "359881fbaded2d19a5003609673744b8b68c4973817598eb0e262d049ec9c3f3"
|
||||
version = "0.45.2"
|
||||
source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6"
|
||||
dependencies = [
|
||||
"async-utility",
|
||||
"async-wsocket",
|
||||
|
||||
+10
@@ -163,3 +163,13 @@ path = "tests/lifecycle/replaceable_history.rs"
|
||||
|
||||
[workspace]
|
||||
members = [".", "grasp-audit"]
|
||||
|
||||
# Pending upstream local-relay query completion fixes; pin every shared Nostr
|
||||
# type to the same source as the embedded SDK.
|
||||
[patch.crates-io]
|
||||
nostr = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
nostr-sdk = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
nostr-database = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
nostr-lmdb = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
nostr-memory = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
nostr-gossip = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" }
|
||||
|
||||
@@ -81,6 +81,11 @@ runtime:
|
||||
- Render metrics on a blocking worker, serializing concurrent scrapes before
|
||||
spawning filesystem scans so monitoring cannot occupy the network executor
|
||||
- Initialize Nostr relay builder with custom [`Nip34WritePolicy`](src/nostr/builder.rs:51)
|
||||
- Retain successful ID subscriptions after EOSE until the client sends CLOSE;
|
||||
immediate automatic CLOSED can make clients discard already queued events
|
||||
- Apply `relay_filter_limit` to each filter and allow the deduplicated union of
|
||||
up to 20 such pages. The SDK aggregate cap is set to 20 times that limit,
|
||||
preserving multi-filter history rather than silently truncating it
|
||||
- Set up shared storage (LMDB or Memory), purgatory, sync manager, and
|
||||
background maintenance tasks
|
||||
- Seed the relay owner's minimal kind-0 bot profile and single-relay
|
||||
|
||||
@@ -56,6 +56,9 @@
|
||||
NGIT_BUILD_REVISION = sourceRevision;
|
||||
cargoLock = {
|
||||
lockFile = ./Cargo.lock;
|
||||
outputHashes = {
|
||||
"nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8=";
|
||||
};
|
||||
};
|
||||
|
||||
# Only build/test the ngit-grasp package, not the whole workspace.
|
||||
@@ -77,6 +80,9 @@
|
||||
src = ./.;
|
||||
cargoLock = {
|
||||
lockFile = ./Cargo.lock;
|
||||
outputHashes = {
|
||||
"nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8=";
|
||||
};
|
||||
};
|
||||
|
||||
cargoBuildFlags = [ "-p" "grasp-audit" ];
|
||||
@@ -97,6 +103,9 @@
|
||||
NGIT_BUILD_REVISION = sourceRevision;
|
||||
cargoLock = {
|
||||
lockFile = ./Cargo.lock;
|
||||
outputHashes = {
|
||||
"nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8=";
|
||||
};
|
||||
};
|
||||
|
||||
cargoBuildFlags = [ "-p" "ngit-grasp" ];
|
||||
|
||||
@@ -11,6 +11,9 @@ let
|
||||
NGIT_BUILD_REVISION = ngitGraspSourceRevision;
|
||||
cargoLock = {
|
||||
lockFile = ../Cargo.lock;
|
||||
outputHashes = {
|
||||
"nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8=";
|
||||
};
|
||||
};
|
||||
|
||||
nativeBuildInputs = with pkgs; [ pkg-config ];
|
||||
|
||||
+11
-2
@@ -1138,6 +1138,7 @@ pub async fn create_relay(
|
||||
private_access,
|
||||
);
|
||||
|
||||
const MAX_FILTERS_PER_REQ: usize = 20;
|
||||
let mut builder = LocalRelayBuilder::default()
|
||||
.database(database.clone())
|
||||
.write_policy(write_policy.clone())
|
||||
@@ -1152,12 +1153,20 @@ pub async fn create_relay(
|
||||
.max_event_size(config.relay_max_event_size_bytes)
|
||||
.websocket_handshake_timeout(Duration::from_secs(10))
|
||||
.max_subid_length(250)
|
||||
.max_filters_per_req(20)
|
||||
.max_filters_per_req(MAX_FILTERS_PER_REQ)
|
||||
.max_subscription_bytes(MAX_SUBSCRIPTION_STATE_BYTES)
|
||||
.max_negentropy_subscriptions(10)
|
||||
.max_negentropy_items(50_000)
|
||||
.max_filter_limit(config.relay_filter_limit)
|
||||
.max_query_results(config.relay_filter_limit)
|
||||
// The configured limit is per filter, including multi-filter REQs.
|
||||
.max_query_results(
|
||||
config
|
||||
.relay_filter_limit
|
||||
.saturating_mul(MAX_FILTERS_PER_REQ),
|
||||
)
|
||||
// Some clients cancel queued EVENT delivery when CLOSED follows EOSE.
|
||||
// Keep the subscription until their ordinary CLOSE instead.
|
||||
.auto_close_completed_id_queries(false)
|
||||
.default_filter_limit(config.relay_filter_limit);
|
||||
|
||||
// `LocalRelayBuilder` otherwise applies rust-nostr's own finite default.
|
||||
|
||||
@@ -96,6 +96,7 @@ struct RelayOptions {
|
||||
/// ordering barrier. Other fixtures retain the production `info` default.
|
||||
log_level: Option<String>,
|
||||
relay_max_subscriptions: Option<usize>,
|
||||
relay_filter_limit: Option<usize>,
|
||||
sync_recursive_descendant_limit: Option<usize>,
|
||||
private_members: Option<String>,
|
||||
/// Explicit NGIT_USER_INDEX_RELAYS value (comma-separated), overriding
|
||||
@@ -526,6 +527,18 @@ impl TestRelay {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start a relay with an explicit per-filter history limit.
|
||||
pub async fn start_with_relay_filter_limit(limit: usize) -> Self {
|
||||
Self::start_internal(
|
||||
port::reserve_port(),
|
||||
RelayOptions {
|
||||
relay_filter_limit: Some(limit),
|
||||
..RelayOptions::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start a relay with LMDB backend (persistent side DBs on a temp dir).
|
||||
pub async fn start_with_lmdb() -> Self {
|
||||
Self::start_internal(
|
||||
@@ -901,6 +914,10 @@ impl TestRelay {
|
||||
cmd.env("NGIT_LOG_LEVEL", log_level);
|
||||
}
|
||||
|
||||
if let Some(limit) = options.relay_filter_limit {
|
||||
cmd.env("NGIT_RELAY_FILTER_LIMIT", limit.to_string());
|
||||
}
|
||||
|
||||
if let Some(limit) = options.relay_max_subscriptions {
|
||||
cmd.env("NGIT_RELAY_MAX_SUBSCRIPTIONS", limit.to_string());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
//! End-to-end query completion and per-filter limit compatibility.
|
||||
mod common;
|
||||
|
||||
use common::{TestClient, TestRelay};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use nostr_sdk::prelude::*;
|
||||
use std::time::Duration;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
#[tokio::test]
|
||||
async fn merged_history_preserves_per_filter_limits_and_id_queries_stay_open() {
|
||||
let relay = TestRelay::start_with_relay_filter_limit(2).await;
|
||||
let keys = relay.owner_keys().clone();
|
||||
let publisher = TestClient::new(relay.url(), keys.clone()).await.unwrap();
|
||||
let mut events = Vec::new();
|
||||
for kind in [Kind::TextNote, Kind::Repost] {
|
||||
for index in 0..2 {
|
||||
let event = EventBuilder::new(kind, format!("history {index}"))
|
||||
.finalize(&keys)
|
||||
.unwrap();
|
||||
publisher.send_event(&event).await.unwrap();
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
let (mut ws, _) = tokio_tungstenite::connect_async(relay.url()).await.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
// The aggregate must permit both complete per-filter pages, while
|
||||
// deduplicating the overlapping ID filter.
|
||||
for (id, filters, expected) in [
|
||||
(
|
||||
"merged",
|
||||
vec![
|
||||
serde_json::json!({"kinds":[1]}),
|
||||
serde_json::json!({"kinds":[6]}),
|
||||
serde_json::json!({"ids":[events[0].id]}),
|
||||
],
|
||||
4,
|
||||
),
|
||||
("id", vec![serde_json::json!({"ids":[events[0].id]})], 1),
|
||||
(
|
||||
"empty",
|
||||
vec![serde_json::json!({"ids":["0".repeat(64)]})],
|
||||
0,
|
||||
),
|
||||
] {
|
||||
let mut request = vec![serde_json::json!("REQ"), serde_json::json!(id)];
|
||||
request.extend(filters);
|
||||
ws.send(Message::Text(
|
||||
serde_json::to_string(&request).unwrap().into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut received = std::collections::HashSet::new();
|
||||
loop {
|
||||
let frame = ws.next().await.unwrap().unwrap();
|
||||
if !frame.is_text() {
|
||||
continue;
|
||||
}
|
||||
let message: serde_json::Value =
|
||||
serde_json::from_str(frame.to_text().unwrap()).unwrap();
|
||||
assert_ne!(
|
||||
message[0], "CLOSED",
|
||||
"completed queries must remain open: {message}"
|
||||
);
|
||||
assert_eq!(message[1], id);
|
||||
match message[0].as_str() {
|
||||
Some("EVENT") => {
|
||||
received.insert(message[2]["id"].clone().to_string());
|
||||
}
|
||||
Some("EOSE") => break,
|
||||
other => panic!("unexpected response: {other:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(received.len(), expected);
|
||||
ws.send(Message::Text(
|
||||
serde_json::json!(["CLOSE", id]).to_string().into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("every history response must reach EOSE");
|
||||
ws.close(None).await.unwrap();
|
||||
publisher.disconnect().await;
|
||||
relay.stop().await;
|
||||
}
|
||||
@@ -144,3 +144,47 @@ async fn small_event_batches_do_not_wait_for_delayed_ack() {
|
||||
client.disconnect().await;
|
||||
relay.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires nak; explicit completed-ID interoperability regression"]
|
||||
async fn nak_receives_completed_id_queries() {
|
||||
use common::{TestClient, TestRelay};
|
||||
use nostr_sdk::prelude::*;
|
||||
use std::time::Duration;
|
||||
|
||||
let relay = TestRelay::start().await;
|
||||
let keys = relay.owner_keys().clone();
|
||||
let client = TestClient::new(relay.url(), keys.clone()).await.unwrap();
|
||||
let event = EventBuilder::new(Kind::TextNote, "completed id lookup")
|
||||
.finalize(&keys)
|
||||
.unwrap();
|
||||
client.send_event(&event).await.unwrap();
|
||||
for attempt in 0..20 {
|
||||
let mut command = tokio::process::Command::new("nak");
|
||||
command
|
||||
.args(["req", "-i", &event.id.to_hex(), relay.url()])
|
||||
.kill_on_drop(true);
|
||||
let result = tokio::time::timeout(Duration::from_secs(10), command.output())
|
||||
.await
|
||||
.expect("nak query must finish")
|
||||
.expect("nak must be installed for this explicit regression");
|
||||
assert!(
|
||||
result.status.success(),
|
||||
"nak query failed: {:?}",
|
||||
result.stderr
|
||||
);
|
||||
let events: Vec<Event> = String::from_utf8(result.stdout)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| Event::from_json(line).unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
events.len(),
|
||||
1,
|
||||
"completed-ID result lost on attempt {attempt}"
|
||||
);
|
||||
assert_eq!(events[0].id, event.id);
|
||||
}
|
||||
client.disconnect().await;
|
||||
relay.stop().await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user