Fix TLS read buffer in ws client: buffer excess SSL_read bytes to prevent loss of multiple WebSocket frames in a single TLS record

The 50ms receive timeout was too short for relays that need time to
process a REQ query before responding. The pool's query_sync already
uses 1000ms. This matches that behavior to avoid premature timeouts.

VERSION

@@ -1 +1 @@
-0.6.10
+0.6.11

nostr_core/core_relays.c

@@ -195,9 +195,10 @@ cJSON** synchronous_query_relays_with_progress(
                 continue;
             }
             
-            // Try to receive message (short timeout to keep UI responsive)
-            char buffer[8192];
-            int len = nostr_ws_receive(relay->client, buffer, sizeof(buffer)-1, 50);
+            // Try to receive message (1000ms timeout to allow relay time
+            // to respond — matches nostr_relay_pool_query_sync behavior)
+            char buffer[262144];
+            int len = nostr_ws_receive(relay->client, buffer, sizeof(buffer)-1, 1000);
             
             if (len > 0) {
                 buffer[len] = '\0';
@@ -322,6 +323,23 @@ cJSON** synchronous_query_relays_with_progress(
                             nostr_ws_close(relay->client);
                             relay->client = NULL;
                         }
+                    } else if (msg_type && (strcmp(msg_type, "NOTICE") == 0 ||
+                                             strcmp(msg_type, "CLOSED") == 0)) {
+                        /* Capture NOTICE and CLOSED messages from the relay.
+                         * These often contain auth-required or restriction
+                         * notices. Pass the message content to the callback. */
+                        const char* notice_msg = "";
+                        if (cJSON_IsArray(parsed) && cJSON_GetArraySize(parsed) >= 2) {
+                            cJSON* msg_json = cJSON_GetArrayItem(parsed, 1);
+                            if (msg_json && cJSON_IsString(msg_json)) {
+                                notice_msg = cJSON_GetStringValue(msg_json);
+                            }
+                        }
+                        if (callback) {
+                            callback(relay->url, msg_type, notice_msg,
+                                    relay->events_received, relay_count,
+                                    completed_relays, user_data);
+                        }
                     }
                     
                     if (msg_type) free(msg_type);
@@ -580,7 +598,7 @@ publish_result_t* synchronous_publish_event_with_progress(
             }
             
             // Try to receive message (short timeout to keep UI responsive)
-            char buffer[8192];
+            char buffer[262144];
             int len = nostr_ws_receive(relay->client, buffer, sizeof(buffer)-1, 50);
             
             if (len > 0) {

nostr_core/nostr_core.h

@@ -2,10 +2,10 @@
 #define NOSTR_CORE_H
 
 // Version information (auto-updated by increment_and_push.sh)
-#define VERSION "v0.6.10"
+#define VERSION "v0.6.11"
 #define VERSION_MAJOR 0
 #define VERSION_MINOR 6
-#define VERSION_PATCH 10
+#define VERSION_PATCH 11
 
 /*
  * NOSTR Core Library - Complete API Reference
@@ -304,6 +304,17 @@ cJSON** nostr_relay_pool_query_sync(
     cJSON* filter,
     int* event_count,
     int timeout_ms);
+// Same as above but also reports whether all relays sent EOSE (vs timed out).
+// out_got_eose is set to 1 if all connected relays sent EOSE, 0 if any timed
+// out. May be NULL if the caller doesn't care.
+cJSON** nostr_relay_pool_query_sync_with_eose(
+    nostr_relay_pool_t* pool,
+    const char** relay_urls,
+    int relay_count,
+    cJSON* filter,
+    int* event_count,
+    int timeout_ms,
+    int* out_got_eose);
 cJSON* nostr_relay_pool_get_event(
     nostr_relay_pool_t* pool,
     const char** relay_urls,

nostr_core/nostr_signer.c

@@ -284,6 +284,26 @@ static int signer_local_nip44_decrypt(nostr_signer_t* signer,
     return NOSTR_SUCCESS;
 }
 
+static int signer_local_derive_hmac(nostr_signer_t* signer,
+                                    const char* data,
+                                    char out_digest_hex[65]) {
+    unsigned char mac[32];
+
+    if (!signer || !data || !out_digest_hex) {
+        return NOSTR_ERROR_INVALID_INPUT;
+    }
+
+    if (nostr_hmac_sha256(signer->u.local.private_key, 32,
+                           (const unsigned char*)data, strlen(data),
+                           mac) != 0) {
+        return NOSTR_ERROR_CRYPTO_FAILED;
+    }
+
+    nostr_bytes_to_hex(mac, 32, out_digest_hex);
+    memset(mac, 0, sizeof(mac));
+    return NOSTR_SUCCESS;
+}
+
 #if defined(NOSTR_ENABLE_NSIGNER_CLIENT)
 
 static cJSON* signer_remote_params_with_selector(cJSON* params, const char* role, int has_nostr_index, int nostr_index) {
@@ -505,6 +525,74 @@ static int signer_remote_nip44_decrypt(nostr_signer_t* signer,
     return signer_remote_encrypt_decrypt(signer, "nostr_nip44_decrypt", peer_pubkey_hex, ciphertext, plaintext_out);
 }
 
+static int signer_remote_derive_hmac(nostr_signer_t* signer,
+                                     const char* data,
+                                     char out_digest_hex[65]) {
+    cJSON* params;
+    cJSON* opts = NULL;
+    cJSON* result = NULL;
+    cJSON* parsed = NULL;
+    cJSON* digest_item = NULL;
+    const char* digest_str = NULL;
+    int rc;
+
+    if (!signer || !data || !out_digest_hex) {
+        return NOSTR_ERROR_INVALID_INPUT;
+    }
+
+    params = cJSON_CreateArray();
+    if (params == NULL) {
+        return NOSTR_ERROR_MEMORY_FAILED;
+    }
+    cJSON_AddItemToArray(params, cJSON_CreateString(data));
+
+    /* Build the options object with algorithm:"secp256k1" and the selector
+     * (role or nostr_index). The derive verb requires index, so when
+     * has_nostr_index is set we include it; otherwise we include role (the
+     * nsigner derive handler requires index, so callers using the remote
+     * backend must set nostr_index via nostr_signer_nsigner_set_nostr_index
+     * before calling derive_hmac). */
+    opts = cJSON_CreateObject();
+    if (opts == NULL) {
+        cJSON_Delete(params);
+        return NOSTR_ERROR_MEMORY_FAILED;
+    }
+    cJSON_AddStringToObject(opts, "algorithm", "secp256k1");
+    if (signer->u.remote.has_nostr_index) {
+        cJSON_AddNumberToObject(opts, "index", signer->u.remote.nostr_index);
+    } else if (signer->u.remote.role[0] != '\0') {
+        cJSON_AddStringToObject(opts, "role", signer->u.remote.role);
+    }
+    cJSON_AddItemToArray(params, opts);
+
+    rc = nsigner_client_call(signer->u.remote.client, "derive", params, &result);
+    if (rc != NOSTR_SUCCESS) {
+        return rc;
+    }
+
+    /* result is a JSON string: {"algorithm":"secp256k1","key_id":"...","digest":"<64hex>"} */
+    if (!cJSON_IsString(result) || result->valuestring == NULL) {
+        cJSON_Delete(result);
+        return NOSTR_ERROR_NIP46_INVALID_RESPONSE;
+    }
+    parsed = cJSON_Parse(result->valuestring);
+    cJSON_Delete(result);
+    if (parsed == NULL) {
+        return NOSTR_ERROR_NIP46_INVALID_RESPONSE;
+    }
+    digest_item = cJSON_GetObjectItemCaseSensitive(parsed, "digest");
+    if (!cJSON_IsString(digest_item) || digest_item->valuestring == NULL ||
+        strlen(digest_item->valuestring) != 64) {
+        cJSON_Delete(parsed);
+        return NOSTR_ERROR_NIP46_INVALID_RESPONSE;
+    }
+    digest_str = digest_item->valuestring;
+    memcpy(out_digest_hex, digest_str, 64);
+    out_digest_hex[64] = '\0';
+    cJSON_Delete(parsed);
+    return NOSTR_SUCCESS;
+}
+
 #endif /* NOSTR_ENABLE_NSIGNER_CLIENT */
 
 nostr_signer_t* nostr_signer_local(const unsigned char private_key[32]) {
@@ -659,6 +747,25 @@ int nostr_signer_nip44_decrypt(nostr_signer_t* signer,
     return NOSTR_ERROR_INVALID_INPUT;
 }
 
+int nostr_signer_derive_hmac(nostr_signer_t* signer,
+                             const char* data,
+                             char out_digest_hex[65]) {
+    if (!signer || !data || !out_digest_hex) {
+        return NOSTR_ERROR_INVALID_INPUT;
+    }
+
+    if (signer->backend == NOSTR_SIGNER_BACKEND_LOCAL) {
+        return signer_local_derive_hmac(signer, data, out_digest_hex);
+    }
+#if defined(NOSTR_ENABLE_NSIGNER_CLIENT)
+    if (signer->backend == NOSTR_SIGNER_BACKEND_NSIGNER_REMOTE) {
+        return signer_remote_derive_hmac(signer, data, out_digest_hex);
+    }
+#endif
+
+    return NOSTR_ERROR_INVALID_INPUT;
+}
+
 #if defined(NOSTR_ENABLE_NSIGNER_CLIENT)
 static nostr_signer_t* nostr_signer_nsigner_from_transport(nsigner_transport_t* transport, const char* role) {
     nsigner_client_t* client = NULL;

nostr_core/nostr_signer.h

@@ -19,6 +19,16 @@ void nostr_signer_free(nostr_signer_t* signer);
 int nostr_signer_get_public_key(nostr_signer_t* signer, char out_pubkey_hex[65]);
 int nostr_signer_sign_event(nostr_signer_t* signer, const cJSON* unsigned_event, cJSON** signed_event_out);
 
+/* Derive a deterministic MAC over caller-supplied data using the signer's
+ * secp256k1 private key: out = HMAC-SHA256(privkey, data).
+ * data is a NUL-terminated UTF-8 string. out_digest_hex receives 64 lowercase
+ * hex chars + NUL. For the local backend this computes directly; for the
+ * nsigner remote backend it calls the "derive" verb (algorithm:"secp256k1").
+ * Returns NOSTR_SUCCESS or an error code. */
+int nostr_signer_derive_hmac(nostr_signer_t* signer,
+                             const char* data,
+                             char out_digest_hex[65]);
+
 /* Encryption verbs (for parity with upcoming remote backends) */
 int nostr_signer_nip04_encrypt(nostr_signer_t* signer,
                                const char* peer_pubkey_hex,

nostr_core/nsigner_transport.c

@@ -1021,8 +1021,20 @@ int nsigner_transport_list_serial(char paths[][64], int max_paths) {
 
     while ((ent = readdir(dir)) != NULL) {
         int n;
+        int is_serial = 0;
 
-        if (strncmp(ent->d_name, "ttyACM", 6) != 0) {
+        /* USB CDC-ACM devices (e.g. n_signer firmware using TinyUSB CDC). */
+        if (strncmp(ent->d_name, "ttyACM", 6) == 0) {
+            is_serial = 1;
+        }
+        /* USB-serial adapters (FTDI / CH340 / CP210x / PL2303). These show up
+         * as /dev/ttyUSB* and speak the same 115200 8N1 framing as CDC-ACM,
+         * so they are valid n_signer serial transports too. */
+        if (strncmp(ent->d_name, "ttyUSB", 6) == 0) {
+            is_serial = 1;
+        }
+
+        if (!is_serial) {
             continue;
         }
 

nostr_websocket/nostr_websocket_openssl.c

@@ -57,6 +57,15 @@ typedef struct {
     SSL_CTX* ssl_ctx;
     SSL* ssl;
     int ssl_connected;
+    /* Internal read buffer: SSL_read may return more bytes than the
+     * caller requested (a full TLS record at once). Without buffering
+     * the excess, those bytes are lost — which breaks ws_receive_frame
+     * when it asks for a 2-byte header but SSL_read returns a full
+     * event message. This caused the caching backfill to only receive
+     * 1 event per query from TLS relays. */
+    char read_buf[65536];
+    size_t read_buf_len;   /* total bytes in read_buf */
+    size_t read_buf_pos;   /* offset of next byte to consume */
 } tls_transport_t;
 
 // WebSocket client structure
@@ -708,10 +717,27 @@ static int tls_recv(void* ctx, void* data, size_t len, int timeout_ms) {
     if (!tls->ssl_connected || !tls->ssl) {
         return -1;
     }
-    
-    // Always use select() to ensure proper blocking behavior
-    // If SSL has pending data, use zero timeout to return immediately
-    // Otherwise use the full timeout to block until data arrives
+
+    /* Serve from the internal read buffer first. SSL_read may have
+     * returned more bytes than the caller requested on a previous call
+     * (a full TLS record), and we buffered the excess. Without this,
+     * ws_receive_frame loses the extra bytes when it asks for a 2-byte
+     * header but SSL_read returns a full event message. */
+    if (tls->read_buf_pos < tls->read_buf_len) {
+        size_t avail = tls->read_buf_len - tls->read_buf_pos;
+        size_t to_copy = (avail < len) ? avail : len;
+        memcpy(data, tls->read_buf + tls->read_buf_pos, to_copy);
+        tls->read_buf_pos += to_copy;
+        if (tls->read_buf_pos >= tls->read_buf_len) {
+            tls->read_buf_len = 0;
+            tls->read_buf_pos = 0;
+        }
+        return (int)to_copy;
+    }
+
+    /* Buffer is empty — refill it with a single SSL_read, then serve
+     * the requested bytes from it. This ensures excess bytes are
+     * preserved for subsequent tls_recv calls. */
     if (timeout_ms > 0) {
         fd_set readfds;
         struct timeval tv;
@@ -719,7 +745,6 @@ static int tls_recv(void* ctx, void* data, size_t len, int timeout_ms) {
         FD_ZERO(&readfds);
         FD_SET(tls->socket_fd, &readfds);
         
-        // If SSL has buffered data, use zero timeout; otherwise use full timeout
         if (SSL_pending(tls->ssl) > 0) {
             tv.tv_sec = 0;
             tv.tv_usec = 0;
@@ -741,16 +766,26 @@ static int tls_recv(void* ctx, void* data, size_t len, int timeout_ms) {
     const int max_attempts = 10;
     
     while (attempts < max_attempts) {
-        int ret = SSL_read(tls->ssl, data, len);
+        /* Read into the internal buffer (up to its capacity), then copy
+         * the requested number of bytes to the caller's output. */
+        int ret = SSL_read(tls->ssl, tls->read_buf, sizeof(tls->read_buf));
         
         if (ret > 0) {
-            return ret; // Success
+            tls->read_buf_len = (size_t)ret;
+            tls->read_buf_pos = 0;
+            size_t to_copy = (tls->read_buf_len < len) ? tls->read_buf_len : len;
+            memcpy(data, tls->read_buf, to_copy);
+            tls->read_buf_pos = to_copy;
+            if (tls->read_buf_pos >= tls->read_buf_len) {
+                tls->read_buf_len = 0;
+                tls->read_buf_pos = 0;
+            }
+            return (int)to_copy;
         }
         
         int err = SSL_get_error(tls->ssl, ret);
         
         if (err == SSL_ERROR_WANT_READ) {
-            // Check if more data is available on socket
             fd_set readfds;
             struct timeval tv = {0, 100000}; // 100ms timeout
             
@@ -763,7 +798,7 @@ static int tls_recv(void* ctx, void* data, size_t len, int timeout_ms) {
             }
             
             attempts++;
-            continue; // Retry the SSL read
+            continue;
             
         } else if (err == SSL_ERROR_WANT_WRITE) {
             return -1;

tests/nsigner_client_test.c

@@ -11,6 +11,7 @@
 #include "../nostr_core/nsigner_client.h"
 #include "../nostr_core/utils.h"
 #include "../cjson/cJSON.h"
+#include "../nostr_core/nostr_signer.h"
 
 static int test_count = 0;
 static int passed_count = 0;
@@ -332,6 +333,66 @@ cleanup:
     return ok;
 }
 
+/* Local-signer derive_hmac: HMAC-SHA256(privkey, data) via the high-level API
+ * must match nostr_hmac_sha256 computed directly with the same privkey. */
+static int test_local_derive_hmac_matches_reference(void) {
+    unsigned char privkey[32];
+    nostr_signer_t* signer = NULL;
+    char got_hex[65];
+    unsigned char ref_mac[32];
+    char ref_hex[65];
+    int ok = 0;
+    int i;
+
+    for (i = 0; i < 32; i++) {
+        privkey[i] = (unsigned char)(i + 1);
+    }
+
+    signer = nostr_signer_local(privkey);
+    if (signer == NULL) {
+        return 0;
+    }
+
+    if (nostr_signer_derive_hmac(signer, "sovereign-browser/bookmarks-folder-id-v1:Work/Projects", got_hex) != NOSTR_SUCCESS) {
+        nostr_signer_free(signer);
+        return 0;
+    }
+
+    if (nostr_hmac_sha256(privkey, 32,
+            (const unsigned char*)"sovereign-browser/bookmarks-folder-id-v1:Work/Projects",
+            strlen("sovereign-browser/bookmarks-folder-id-v1:Work/Projects"),
+            ref_mac) != 0) {
+        nostr_signer_free(signer);
+        return 0;
+    }
+    nostr_bytes_to_hex(ref_mac, 32, ref_hex);
+
+    ok = (strlen(got_hex) == 64 && strcmp(got_hex, ref_hex) == 0);
+
+    /* Determinism: same input -> same digest. */
+    {
+        char got2_hex[65];
+        if (nostr_signer_derive_hmac(signer, "sovereign-browser/bookmarks-folder-id-v1:Work/Projects", got2_hex) == NOSTR_SUCCESS) {
+            ok = ok && (strcmp(got_hex, got2_hex) == 0);
+        } else {
+            ok = 0;
+        }
+    }
+
+    /* Opaqueness: different input -> different digest. */
+    {
+        char other_hex[65];
+        if (nostr_signer_derive_hmac(signer, "sovereign-browser/bookmarks-folder-id-v1:Personal", other_hex) == NOSTR_SUCCESS) {
+            ok = ok && (strcmp(got_hex, other_hex) != 0);
+        } else {
+            ok = 0;
+        }
+    }
+
+    nostr_signer_free(signer);
+    return ok;
+}
+
 int main(void) {
 #if !defined(NOSTR_ENABLE_NSIGNER_CLIENT)
     printf("nsigner client disabled in this build; skipping\n");
@@ -342,6 +403,7 @@ int main(void) {
     run_result("auth body hash matches sha256(compact params)", test_body_hash_matches_sha256_compact_params());
     run_result("/proc/net/unix discovery parser no-crash", test_unix_discovery_no_crash());
     run_result("fds transport client round-trip over framed fd pair", test_fds_transport_round_trip_via_client());
+    run_result("local signer derive_hmac matches reference HMAC-SHA256", test_local_derive_hmac_matches_reference());
 
     printf("\nTotal: %d  Passed: %d  Failed: %d\n", test_count, passed_count, test_count - passed_count);
     return (test_count == passed_count) ? 0 : 1;