diff --git a/Cargo.lock b/Cargo.lock index f979768..70282c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1359,9 +1359,8 @@ dependencies = [ [[package]] name = "nostr" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7279feef28d43a04c0b3529acd4fea923ac816729bd643191b292296f1f6cc13" +version = "0.45.3" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "base64", "bech32", @@ -1379,13 +1378,13 @@ dependencies = [ "unicode-normalization", "universal-time", "url", + "zeroize", ] [[package]] name = "nostr-database" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c7ffd33114d62c737dd2b8f77ffec4693c64b7a1c6090cab0d0009f9986e22" +version = "0.45.1" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "nostr", "opaquerr", @@ -1394,8 +1393,7 @@ dependencies = [ [[package]] name = "nostr-gossip" version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa07539e52a71cb91fe0d693facaa298f03fcf9edcd66a521094e18e286e2336" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "nostr", "opaquerr", @@ -1403,9 +1401,8 @@ dependencies = [ [[package]] name = "nostr-lmdb" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a313418ff58bfa444ba4b71200ff8fe2d3eb31f2b5d0d8fdb7f5e60a703d31" +version = "0.45.2" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "async-utility", "flatbuffers", @@ -1419,9 +1416,8 @@ dependencies = [ [[package]] name = "nostr-memory" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a827de4b11c4f2b235eabe37f1a67460db68e453606f4755e1ac5a522347a86" +version = "0.45.1" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "btreecap", "nostr", @@ -1431,9 +1427,8 @@ dependencies = [ [[package]] name = "nostr-sdk" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "359881fbaded2d19a5003609673744b8b68c4973817598eb0e262d049ec9c3f3" +version = "0.45.2" +source = "git+https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git?rev=refs%2Fnostr%2Ff614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97#e145fa158b2f924006a2d5b2b9341a297f76cbe6" dependencies = [ "async-utility", "async-wsocket", diff --git a/Cargo.toml b/Cargo.toml index c9aaf10..be0d8e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,3 +163,13 @@ path = "tests/lifecycle/replaceable_history.rs" [workspace] members = [".", "grasp-audit"] + +# Pending upstream local-relay query completion fixes; pin every shared Nostr +# type to the same source as the embedded SDK. +[patch.crates-io] +nostr = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } +nostr-sdk = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } +nostr-database = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } +nostr-lmdb = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } +nostr-memory = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } +nostr-gossip = { git = "https://relay.ngit.dev/npub1drvpzev3syqt0kjrls50050uzf25gehpz9vgdw08hvex7e0vgfeq0eseet/nostr.git", rev = "refs/nostr/f614f8f875b978c760a587f0be403c2d8405f56f30b32d6f91449bf5186c3f97" } diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index d49fd77..2fb3a78 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -81,6 +81,11 @@ runtime: - Render metrics on a blocking worker, serializing concurrent scrapes before spawning filesystem scans so monitoring cannot occupy the network executor - Initialize Nostr relay builder with custom [`Nip34WritePolicy`](src/nostr/builder.rs:51) +- Retain successful ID subscriptions after EOSE until the client sends CLOSE; + immediate automatic CLOSED can make clients discard already queued events +- Apply `relay_filter_limit` to each filter and allow the deduplicated union of + up to 20 such pages. The SDK aggregate cap is set to 20 times that limit, + preserving multi-filter history rather than silently truncating it - Set up shared storage (LMDB or Memory), purgatory, sync manager, and background maintenance tasks - Seed the relay owner's minimal kind-0 bot profile and single-relay diff --git a/flake.nix b/flake.nix index a858d6f..97ff879 100644 --- a/flake.nix +++ b/flake.nix @@ -56,6 +56,9 @@ NGIT_BUILD_REVISION = sourceRevision; cargoLock = { lockFile = ./Cargo.lock; + outputHashes = { + "nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8="; + }; }; # Only build/test the ngit-grasp package, not the whole workspace. @@ -77,6 +80,9 @@ src = ./.; cargoLock = { lockFile = ./Cargo.lock; + outputHashes = { + "nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8="; + }; }; cargoBuildFlags = [ "-p" "grasp-audit" ]; @@ -97,6 +103,9 @@ NGIT_BUILD_REVISION = sourceRevision; cargoLock = { lockFile = ./Cargo.lock; + outputHashes = { + "nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8="; + }; }; cargoBuildFlags = [ "-p" "ngit-grasp" ]; diff --git a/nix/module.nix b/nix/module.nix index 2cf7b55..920d77e 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -11,6 +11,9 @@ let NGIT_BUILD_REVISION = ngitGraspSourceRevision; cargoLock = { lockFile = ../Cargo.lock; + outputHashes = { + "nostr-0.45.3" = "sha256-B2EDc3p6EOW/hVkSJLqFnTjiUBMQJ1Z7jPvOHPwfCW8="; + }; }; nativeBuildInputs = with pkgs; [ pkg-config ]; diff --git a/src/nostr/builder.rs b/src/nostr/builder.rs index 5b8e23f..43002a5 100644 --- a/src/nostr/builder.rs +++ b/src/nostr/builder.rs @@ -1138,6 +1138,7 @@ pub async fn create_relay( private_access, ); + const MAX_FILTERS_PER_REQ: usize = 20; let mut builder = LocalRelayBuilder::default() .database(database.clone()) .write_policy(write_policy.clone()) @@ -1152,12 +1153,20 @@ pub async fn create_relay( .max_event_size(config.relay_max_event_size_bytes) .websocket_handshake_timeout(Duration::from_secs(10)) .max_subid_length(250) - .max_filters_per_req(20) + .max_filters_per_req(MAX_FILTERS_PER_REQ) .max_subscription_bytes(MAX_SUBSCRIPTION_STATE_BYTES) .max_negentropy_subscriptions(10) .max_negentropy_items(50_000) .max_filter_limit(config.relay_filter_limit) - .max_query_results(config.relay_filter_limit) + // The configured limit is per filter, including multi-filter REQs. + .max_query_results( + config + .relay_filter_limit + .saturating_mul(MAX_FILTERS_PER_REQ), + ) + // Some clients cancel queued EVENT delivery when CLOSED follows EOSE. + // Keep the subscription until their ordinary CLOSE instead. + .auto_close_completed_id_queries(false) .default_filter_limit(config.relay_filter_limit); // `LocalRelayBuilder` otherwise applies rust-nostr's own finite default. diff --git a/tests/common/relay.rs b/tests/common/relay.rs index a097d36..43233cd 100644 --- a/tests/common/relay.rs +++ b/tests/common/relay.rs @@ -96,6 +96,7 @@ struct RelayOptions { /// ordering barrier. Other fixtures retain the production `info` default. log_level: Option, relay_max_subscriptions: Option, + relay_filter_limit: Option, sync_recursive_descendant_limit: Option, private_members: Option, /// Explicit NGIT_USER_INDEX_RELAYS value (comma-separated), overriding @@ -526,6 +527,18 @@ impl TestRelay { .await } + /// Start a relay with an explicit per-filter history limit. + pub async fn start_with_relay_filter_limit(limit: usize) -> Self { + Self::start_internal( + port::reserve_port(), + RelayOptions { + relay_filter_limit: Some(limit), + ..RelayOptions::default() + }, + ) + .await + } + /// Start a relay with LMDB backend (persistent side DBs on a temp dir). pub async fn start_with_lmdb() -> Self { Self::start_internal( @@ -901,6 +914,10 @@ impl TestRelay { cmd.env("NGIT_LOG_LEVEL", log_level); } + if let Some(limit) = options.relay_filter_limit { + cmd.env("NGIT_RELAY_FILTER_LIMIT", limit.to_string()); + } + if let Some(limit) = options.relay_max_subscriptions { cmd.env("NGIT_RELAY_MAX_SUBSCRIPTIONS", limit.to_string()); } diff --git a/tests/relay_query_completion.rs b/tests/relay_query_completion.rs new file mode 100644 index 0000000..6b807f2 --- /dev/null +++ b/tests/relay_query_completion.rs @@ -0,0 +1,87 @@ +//! End-to-end query completion and per-filter limit compatibility. +mod common; + +use common::{TestClient, TestRelay}; +use futures_util::{SinkExt, StreamExt}; +use nostr_sdk::prelude::*; +use std::time::Duration; +use tokio_tungstenite::tungstenite::Message; + +#[tokio::test] +async fn merged_history_preserves_per_filter_limits_and_id_queries_stay_open() { + let relay = TestRelay::start_with_relay_filter_limit(2).await; + let keys = relay.owner_keys().clone(); + let publisher = TestClient::new(relay.url(), keys.clone()).await.unwrap(); + let mut events = Vec::new(); + for kind in [Kind::TextNote, Kind::Repost] { + for index in 0..2 { + let event = EventBuilder::new(kind, format!("history {index}")) + .finalize(&keys) + .unwrap(); + publisher.send_event(&event).await.unwrap(); + events.push(event); + } + } + let (mut ws, _) = tokio_tungstenite::connect_async(relay.url()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + // The aggregate must permit both complete per-filter pages, while + // deduplicating the overlapping ID filter. + for (id, filters, expected) in [ + ( + "merged", + vec![ + serde_json::json!({"kinds":[1]}), + serde_json::json!({"kinds":[6]}), + serde_json::json!({"ids":[events[0].id]}), + ], + 4, + ), + ("id", vec![serde_json::json!({"ids":[events[0].id]})], 1), + ( + "empty", + vec![serde_json::json!({"ids":["0".repeat(64)]})], + 0, + ), + ] { + let mut request = vec![serde_json::json!("REQ"), serde_json::json!(id)]; + request.extend(filters); + ws.send(Message::Text( + serde_json::to_string(&request).unwrap().into(), + )) + .await + .unwrap(); + let mut received = std::collections::HashSet::new(); + loop { + let frame = ws.next().await.unwrap().unwrap(); + if !frame.is_text() { + continue; + } + let message: serde_json::Value = + serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_ne!( + message[0], "CLOSED", + "completed queries must remain open: {message}" + ); + assert_eq!(message[1], id); + match message[0].as_str() { + Some("EVENT") => { + received.insert(message[2]["id"].clone().to_string()); + } + Some("EOSE") => break, + other => panic!("unexpected response: {other:?}"), + } + } + assert_eq!(received.len(), expected); + ws.send(Message::Text( + serde_json::json!(["CLOSE", id]).to_string().into(), + )) + .await + .unwrap(); + } + }) + .await + .expect("every history response must reach EOSE"); + ws.close(None).await.unwrap(); + publisher.disconnect().await; + relay.stop().await; +} diff --git a/tests/relay_response_latency.rs b/tests/relay_response_latency.rs index 52ff71a..d6b401e 100644 --- a/tests/relay_response_latency.rs +++ b/tests/relay_response_latency.rs @@ -144,3 +144,47 @@ async fn small_event_batches_do_not_wait_for_delayed_ack() { client.disconnect().await; relay.stop().await; } + +#[tokio::test] +#[ignore = "requires nak; explicit completed-ID interoperability regression"] +async fn nak_receives_completed_id_queries() { + use common::{TestClient, TestRelay}; + use nostr_sdk::prelude::*; + use std::time::Duration; + + let relay = TestRelay::start().await; + let keys = relay.owner_keys().clone(); + let client = TestClient::new(relay.url(), keys.clone()).await.unwrap(); + let event = EventBuilder::new(Kind::TextNote, "completed id lookup") + .finalize(&keys) + .unwrap(); + client.send_event(&event).await.unwrap(); + for attempt in 0..20 { + let mut command = tokio::process::Command::new("nak"); + command + .args(["req", "-i", &event.id.to_hex(), relay.url()]) + .kill_on_drop(true); + let result = tokio::time::timeout(Duration::from_secs(10), command.output()) + .await + .expect("nak query must finish") + .expect("nak must be installed for this explicit regression"); + assert!( + result.status.success(), + "nak query failed: {:?}", + result.stderr + ); + let events: Vec = String::from_utf8(result.stdout) + .unwrap() + .lines() + .map(|line| Event::from_json(line).unwrap()) + .collect(); + assert_eq!( + events.len(), + 1, + "completed-ID result lost on attempt {attempt}" + ); + assert_eq!(events[0].id, event.id); + } + client.disconnect().await; + relay.stop().await; +}