From 5e6610823feb0d31ba480caf73fc24d985c06b61 Mon Sep 17 00:00:00 2001 From: Didactyl User Date: Sat, 11 Apr 2026 15:53:52 -0400 Subject: [PATCH] v0.2.38 - Resolve NIP-17 gift wrap send failure investigation and docs --- README.md | 4 +- plans/nip17_gift_wrap_investigation.md | 127 +++++++++++++++++++++++++ src/agent.c | 62 ++++++++---- src/config.c | 8 ++ src/config.h | 1 + src/main.c | 66 +++++++++++++ src/main.h | 4 +- src/setup_wizard.c | 5 +- 8 files changed, 255 insertions(+), 22 deletions(-) create mode 100644 plans/nip17_gift_wrap_investigation.md diff --git a/README.md b/README.md index 75d1c43..961e775 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ Skills compose by adoption-list order (`10123`) and trigger tags carry runtime e Didactyl will support local inference, which is very privacy preserving. Remote inference does however have it's advantages, and in those cases Didactyl supports using Bitcoin Lightning and eCash inference providers. -## Current Status — v0.2.37 +## Current Status — v0.2.38 **Active build — this project is barely working. Experiment at your own risk.** -> Last release update: v0.2.37 — Add nostr-core relay reconnect cooldown with stability-based reset to prevent flapping loops +> Last release update: v0.2.38 — Resolve NIP-17 gift wrap send failure investigation and docs - Connects to configured relays with auto-reconnect and relay state transition logging - Publishes configured startup events per relay as each relay becomes connected diff --git a/plans/nip17_gift_wrap_investigation.md b/plans/nip17_gift_wrap_investigation.md new file mode 100644 index 0000000..8a95579 --- /dev/null +++ b/plans/nip17_gift_wrap_investigation.md @@ -0,0 +1,127 @@ +# NIP-17 Gift Wrap Send Failure — Investigation + +## Status: Resolved — Fixed in code (2026-04-11) + +## Date Discovered: 2026-04-11 + +## Symptom + +Every outbound NIP-17 DM send from the Anvil agent fails with: +``` +NIP-17 send aborted: gift_wrap creation failed for 1ec454734dcbf6fe... +auto DM fallback to NIP-04 for 1ec454734dcbf6fe... after NIP-17 send failure +``` + +The agent successfully falls back to NIP-04 (kind 4), so DMs are still delivered. However, NIP-17 outbound is completely broken. + +**Inbound NIP-17 works fine** — the agent successfully receives and decrypts kind 1059 gift wraps from the admin. + +## Affected Code Path + +### Send path (broken): +1. `nostr_handler_send_dm_auto_with_role()` in `src/nostr_handler.c:3193` — routes to NIP-17 when `dm_protocol` is `nip17` or `both` +2. `nostr_handler_send_dm_nip17_with_role()` in `src/nostr_handler.c:4050` — creates chat event, calls `nostr_nip17_send_dm()` +3. `nostr_nip17_send_dm()` in `nostr_core_lib/nostr_core/nip017.c:260` — iterates recipients, creates seal + gift wrap +4. `nostr_nip59_create_seal()` in `nostr_core_lib/nostr_core/nip059.c:150` — **encrypts rumor JSON with NIP-44** +5. `nostr_nip59_create_gift_wrap()` in `nostr_core_lib/nostr_core/nip059.c:226` — wraps seal with random key + +### Receive path (working): +1. `on_event()` receives kind 1059 +2. `nostr_nip17_receive_dm()` in `nostr_core_lib/nostr_core/nip017.c:326` — unwraps gift, unseals rumor +3. Successfully extracts kind 14 content + +## Probable Root Cause: Fixed Buffer Overflow in Seal Encryption + +In `nostr_core_lib/nostr_core/nip059.c:163-165`: + +```c +char encrypted_content[4096]; // Should be large enough for most events +int encrypt_result = nostr_nip44_encrypt(sender_private_key, recipient_public_key, + rumor_json, encrypted_content, sizeof(encrypted_content)); +``` + +The seal creation uses a **fixed 4096-byte stack buffer** for the NIP-44 encrypted output. The rumor JSON includes the full DM message content. When the agent sends long responses (the FIPS mesh conversation had multi-KB markdown tables), the rumor JSON easily exceeds 4096 bytes after NIP-44 encryption overhead (base64 encoding roughly 4/3x expansion + padding + nonce). + +**Evidence:** +- The agent's DM responses in the log were 2-5KB of markdown content +- After JSON serialization of the kind 14 rumor (adding pubkey, tags, created_at, kind fields), the total is larger +- NIP-44 encryption adds: 32-byte nonce + padding + base64 encoding → roughly 1.5-2x expansion +- A 3KB message → ~4.5KB rumor JSON → ~6-9KB encrypted → overflows 4096 buffer +- `nostr_nip44_encrypt()` returns error, `nostr_nip59_create_seal()` returns NULL +- `nostr_nip17_send_dm()` skips the recipient, returns 0 wraps +- `nostr_handler_send_dm_nip17_with_role()` logs "gift_wrap creation failed" + +**Why receive works:** The receive path uses `nostr_nip44_decrypt()` which likely allocates dynamically or has a larger buffer. + +## Secondary Suspect: NIP-44 Encryption Implementation + +The NIP-44 encrypt function itself may have issues with large payloads. Need to check: +- `nostr_nip44_encrypt()` in `nostr_core_lib/nostr_core/nip044.c` +- Whether it handles the output buffer size parameter correctly +- Whether it returns an error code that distinguishes "buffer too small" from other failures + +## Anvil Config Context + +The Anvil agent's genesis config (`/home/user/anvil/genesis.jsonc`) has: +```json +"dm_protocol": "nip04" +``` + +But the encrypted user-settings on Nostr (kind 30078) may have overridden this to `"both"` at runtime, which is why NIP-17 send is being attempted. + +## Recommended Fix + +### Option A: Dynamic allocation (preferred) +Replace the fixed 4096-byte buffer in `nostr_nip59_create_seal()` with dynamic allocation: +```c +size_t rumor_len = strlen(rumor_json); +size_t encrypted_buf_size = rumor_len * 2 + 256; // generous overhead for NIP-44 +char* encrypted_content = malloc(encrypted_buf_size); +``` + +### Option B: Larger fixed buffer (quick fix) +Increase the buffer to 64KB or 128KB: +```c +char encrypted_content[131072]; // 128KB should handle any reasonable DM +``` +Note: This is a stack allocation, so very large buffers could cause stack overflow. + +### Also check `nostr_nip59_create_gift_wrap()` +The gift wrap function at `nip059.c:226` may have a similar fixed-buffer issue for encrypting the seal JSON. + +## Log Evidence + +From `/home/user/anvil/debug.log` (2026-04-11): + +``` +[2026-04-11 09:45:14] [WARN ] NIP-17 send aborted: gift_wrap creation failed for 1ec454734dcbf6fe... +[2026-04-11 09:45:14] [WARN ] auto DM fallback to NIP-04 for 1ec454734dcbf6fe... after NIP-17 send failure +[2026-04-11 09:50:16] [WARN ] NIP-17 send aborted: gift_wrap creation failed for 1ec454734dcbf6fe... +[2026-04-11 09:50:16] [WARN ] auto DM fallback to NIP-04 for 1ec454734dcbf6fe... after NIP-17 send failure +[2026-04-11 09:53:42] [WARN ] NIP-17 send aborted: gift_wrap creation failed for 1ec454734dcbf6fe... +[2026-04-11 09:53:42] [WARN ] auto DM fallback to NIP-04 for 1ec454734dcbf6fe... after NIP-17 send failure +``` + +This pattern is 100% consistent — every outbound NIP-17 attempt fails, every time. + +## Related Files + +- `nostr_core_lib/nostr_core/nip059.c` — Gift wrap / seal creation (probable bug location) +- `nostr_core_lib/nostr_core/nip044.c` — NIP-44 encryption (check buffer handling) +- `nostr_core_lib/nostr_core/nip017.c` — NIP-17 DM send/receive orchestration +- `src/nostr_handler.c:4050` — Agent-level NIP-17 send function +- `plans/nip17_messaging.md` — Original NIP-17 implementation plan + +## Related Existing Plan + +The `plans/nip17_messaging.md` document covers the original NIP-17 implementation. This investigation is specifically about the send-side gift wrap failure that emerged in production. + +## Resolution Summary + +Implemented in `nostr_core_lib/nostr_core/nip059.c`: + +- Replaced fixed encryption buffers in `nostr_nip59_create_seal()` and `nostr_nip59_create_gift_wrap()` with dynamically sized heap buffers calculated from NIP-44 padding/base64 overhead. +- Replaced fixed decrypt buffers in `nostr_nip59_unwrap_gift()` and `nostr_nip59_unseal_rumor()` with dynamic buffers sized from ciphertext length. +- Added missing cleanup path in `nostr_nip59_create_seal()` for failure after encryption allocation. + +Expected outcome: outbound NIP-17 gift wrap creation no longer fails for multi-KB DM payloads due to fixed buffer limits. diff --git a/src/agent.c b/src/agent.c index c0c265d..150115a 100644 --- a/src/agent.c +++ b/src/agent.c @@ -2182,9 +2182,11 @@ void agent_on_message(const char* sender_pubkey_hex, int max_turns = g_cfg->tools.max_turns > 0 ? g_cfg->tools.max_turns : 40; int stall_repeat_threshold = g_cfg->tools.stall_repeat_threshold > 1 ? g_cfg->tools.stall_repeat_threshold : 3; + int max_context_bytes = g_cfg->tools.max_context_bytes >= 4096 ? g_cfg->tools.max_context_bytes : (128 * 1024); uint64_t last_tool_fp = 0; int repeated_tool_turns = 0; int exited_on_stall = 0; + int exited_on_context_limit = 0; int stall_due_to_length = 0; char* final_answer_owned = NULL; int turns_run = 0; @@ -2196,6 +2198,17 @@ void agent_on_message(const char* sender_pubkey_hex, break; } + size_t messages_json_len = strlen(messages_json); + if ((int)messages_json_len > max_context_bytes) { + DEBUG_WARN("[didactyl] DM tool loop context limit exceeded: bytes=%zu max_context_bytes=%d turn=%d", + messages_json_len, + max_context_bytes, + turns_run); + exited_on_context_limit = 1; + free(messages_json); + break; + } + llm_response_t resp; int rc = llm_chat_with_tools_messages(messages_json, tools_json, "auto", &resp); free(messages_json); @@ -2275,28 +2288,35 @@ void agent_on_message(const char* sender_pubkey_hex, llm_response_free(&resp); } - if (!final_answer_owned) { + if (!final_answer_owned && !exited_on_context_limit) { char* messages_json = cJSON_PrintUnformatted(messages); if (messages_json) { - llm_response_t final_resp; - int final_rc = llm_chat_with_tools_messages(messages_json, tools_json, "none", &final_resp); - free(messages_json); - if (final_rc == 0) { - const char* forced_answer = final_resp.content ? final_resp.content : ""; - if (forced_answer[0] != '\0') { - final_answer_owned = strdup(forced_answer); + size_t messages_json_len = strlen(messages_json); + if ((int)messages_json_len <= max_context_bytes) { + llm_response_t final_resp; + int final_rc = llm_chat_with_tools_messages(messages_json, tools_json, "none", &final_resp); + if (final_rc == 0) { + const char* forced_answer = final_resp.content ? final_resp.content : ""; + if (forced_answer[0] != '\0') { + final_answer_owned = strdup(forced_answer); + } + llm_response_free(&final_resp); } - llm_response_free(&final_resp); + } else { + exited_on_context_limit = 1; } + free(messages_json); } } - int exhausted_on_max_turns = (!final_answer_owned && !exited_on_stall && turns_run >= max_turns); + int exhausted_on_max_turns = (!final_answer_owned && !exited_on_stall && !exited_on_context_limit && turns_run >= max_turns); if (!final_answer_owned) { - final_answer_owned = strdup(exited_on_stall - ? "I stopped repeated tool calls after detecting a loop and could not produce a final summary." - : "I reached the tool-turn limit before getting a final response from the model."); + final_answer_owned = strdup(exited_on_context_limit + ? "I paused because the working context became too large while using tools. Please narrow the scope and ask me to continue from a smaller context." + : (exited_on_stall + ? "I stopped repeated tool calls after detecting a loop and could not produce a final summary." + : "I reached the tool-turn limit before getting a final response from the model.")); } if (exhausted_on_max_turns) { @@ -2313,17 +2333,25 @@ void agent_on_message(const char* sender_pubkey_hex, const char* final_answer = final_answer_owned ? final_answer_owned : "I reached the tool-turn limit before getting a final response from the model."; - if (exited_on_stall || exhausted_on_max_turns) { + if (exited_on_stall || exhausted_on_max_turns || exited_on_context_limit) { notify_admin_limit_diagnostic("dm_agent_loop", - exited_on_stall ? "stall_repetition_detected" : "max_turns_exhausted", + exited_on_context_limit + ? "max_context_bytes_exceeded" + : (exited_on_stall ? "stall_repetition_detected" : "max_turns_exhausted"), sender_pubkey_hex, max_turns, turns_run, stall_repeat_threshold, repeated_tool_turns, g_cfg->llm.max_tokens, - (exited_on_stall && stall_due_to_length) ? "max_tokens_truncation" : NULL, - (exited_on_stall && stall_due_to_length) ? "Increase max_tokens (example: /model_set {\"max_tokens\": 4096})" : NULL, + exited_on_context_limit + ? "tool_context_growth" + : ((exited_on_stall && stall_due_to_length) ? "max_tokens_truncation" : NULL), + exited_on_context_limit + ? "Reduce tool-loop breadth or lower max_turns; context exceeded max_context_bytes." + : ((exited_on_stall && stall_due_to_length) + ? "Increase max_tokens (example: /model_set {\"max_tokens\": 4096})" + : NULL), final_answer); } fprintf(stdout, "[didactyl] final response: %.240s%s\n", diff --git a/src/config.c b/src/config.c index 3333338..5dadc2d 100644 --- a/src/config.c +++ b/src/config.c @@ -226,6 +226,7 @@ static int parse_tools_config(cJSON* root, didactyl_config_t* config) { cJSON* api_default_max_turns = cJSON_GetObjectItemCaseSensitive(tools, "api_default_max_turns"); cJSON* api_max_turns_ceiling = cJSON_GetObjectItemCaseSensitive(tools, "api_max_turns_ceiling"); cJSON* stall_repeat_threshold = cJSON_GetObjectItemCaseSensitive(tools, "stall_repeat_threshold"); + cJSON* max_context_bytes = cJSON_GetObjectItemCaseSensitive(tools, "max_context_bytes"); cJSON* local_http_fetch_default_timeout_seconds = cJSON_GetObjectItemCaseSensitive(tools, "local_http_fetch_default_timeout_seconds"); cJSON* local_http_fetch_max_timeout_seconds = @@ -252,6 +253,9 @@ static int parse_tools_config(cJSON* root, didactyl_config_t* config) { if (stall_repeat_threshold && cJSON_IsNumber(stall_repeat_threshold)) { config->tools.stall_repeat_threshold = (int)stall_repeat_threshold->valuedouble; } + if (max_context_bytes && cJSON_IsNumber(max_context_bytes)) { + config->tools.max_context_bytes = (int)max_context_bytes->valuedouble; + } if (local_http_fetch_default_timeout_seconds && cJSON_IsNumber(local_http_fetch_default_timeout_seconds)) { config->tools.local_http_fetch_default_timeout_seconds = (int)local_http_fetch_default_timeout_seconds->valuedouble; @@ -311,6 +315,9 @@ static int parse_tools_config(cJSON* root, didactyl_config_t* config) { if (config->tools.stall_repeat_threshold < 2) { config->tools.stall_repeat_threshold = 3; } + if (config->tools.max_context_bytes < 4096) { + config->tools.max_context_bytes = 128 * 1024; + } if (config->tools.local_http_fetch_default_timeout_seconds < 1) { config->tools.local_http_fetch_default_timeout_seconds = 20; } @@ -1441,6 +1448,7 @@ int config_load(const char* path, didactyl_config_t* config) { config->tools.api_default_max_turns = 8; config->tools.api_max_turns_ceiling = 32; config->tools.stall_repeat_threshold = 3; + config->tools.max_context_bytes = 128 * 1024; config->tools.local_http_fetch_default_timeout_seconds = 20; config->tools.local_http_fetch_max_timeout_seconds = 120; config->tools.blossom_max_upload_bytes = 16 * 1024 * 1024; diff --git a/src/config.h b/src/config.h index c17fe89..caa5f6e 100644 --- a/src/config.h +++ b/src/config.h @@ -51,6 +51,7 @@ typedef struct { int api_default_max_turns; int api_max_turns_ceiling; int stall_repeat_threshold; + int max_context_bytes; int local_http_fetch_default_timeout_seconds; int local_http_fetch_max_timeout_seconds; shell_tools_config_t shell; diff --git a/src/main.c b/src/main.c index 60aad0c..c74fd42 100644 --- a/src/main.c +++ b/src/main.c @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include "../nostr_core_lib/nostr_core/nostr_core.h" #include "main.h" @@ -68,6 +71,59 @@ static void signal_handler(int signum) { g_running = 0; } +static int systemd_notify_send(const char* state) { + if (!state || state[0] == '\0') { + return 0; + } + + const char* notify_socket = getenv("NOTIFY_SOCKET"); + if (!notify_socket || notify_socket[0] == '\0') { + return 0; + } + + if (notify_socket[0] != '/' && notify_socket[0] != '@') { + return -1; + } + + int fd = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (fd < 0) { + return -1; + } + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + + size_t path_len = strlen(notify_socket); + if (path_len >= sizeof(addr.sun_path)) { + close(fd); + return -1; + } + + memcpy(addr.sun_path, notify_socket, path_len + 1); + socklen_t addr_len = (socklen_t)(sizeof(sa_family_t) + path_len + 1U); + if (notify_socket[0] == '@') { + addr.sun_path[0] = '\0'; + addr_len = (socklen_t)(sizeof(sa_family_t) + path_len); + } + + ssize_t sent = sendto(fd, + state, + strlen(state), + MSG_NOSIGNAL, + (const struct sockaddr*)&addr, + addr_len); + int saved_errno = errno; + close(fd); + errno = saved_errno; + + if (sent < 0) { + return -1; + } + + return 1; +} + static void configure_mongoose_log_level_from_debug(void) { if (g_debug_level >= DEBUG_LEVEL_TRACE) { mg_log_set(MG_LL_INFO); @@ -1612,12 +1668,22 @@ int main(int argc, char** argv) { DEBUG_INFO("[didactyl] entering main poll loop"); DEBUG_INFO("[didactyl] running with pubkey %s", cfg.keys.public_key_hex); + (void)systemd_notify_send("READY=1"); + time_t last_watchdog_ping = 0; + while (g_running) { (void)nostr_handler_poll(100); (void)trigger_manager_poll(&trigger_manager); if (http_api_started) { (void)http_api_poll(0); } + + time_t now = time(NULL); + if (now != (time_t)-1 && (last_watchdog_ping == 0 || (now - last_watchdog_ping) >= 10)) { + (void)systemd_notify_send("WATCHDOG=1"); + last_watchdog_ping = now; + } + struct timespec ts = {0, 10 * 1000 * 1000}; nanosleep(&ts, NULL); } diff --git a/src/main.h b/src/main.h index ac805ec..0bee011 100644 --- a/src/main.h +++ b/src/main.h @@ -12,8 +12,8 @@ // Using DIDACTYL_ prefix to avoid conflicts with nostr_core_lib VERSION macros #define DIDACTYL_VERSION_MAJOR 0 #define DIDACTYL_VERSION_MINOR 2 -#define DIDACTYL_VERSION_PATCH 37 -#define DIDACTYL_VERSION "v0.2.37" +#define DIDACTYL_VERSION_PATCH 38 +#define DIDACTYL_VERSION "v0.2.38" // Agent metadata #define DIDACTYL_NAME "Didactyl" diff --git a/src/setup_wizard.c b/src/setup_wizard.c index 86003c5..907a42f 100644 --- a/src/setup_wizard.c +++ b/src/setup_wizard.c @@ -276,6 +276,7 @@ static void config_set_defaults(didactyl_config_t* cfg) { cfg->tools.api_default_max_turns = 8; cfg->tools.api_max_turns_ceiling = 32; cfg->tools.stall_repeat_threshold = 3; + cfg->tools.max_context_bytes = 128 * 1024; cfg->tools.local_http_fetch_default_timeout_seconds = 20; cfg->tools.local_http_fetch_max_timeout_seconds = 120; cfg->tools.blossom_max_upload_bytes = 16 * 1024 * 1024; @@ -2014,7 +2015,9 @@ static int install_system_service_with_dedicated_user(const didactyl_config_t* c "After=network-online.target\n" "Wants=network-online.target\n\n" "[Service]\n" - "Type=simple\n" + "Type=notify\n" + "NotifyAccess=main\n" + "WatchdogSec=120\n" "User=%s\n" "Group=%s\n" "WorkingDirectory=%s\n"