v0.0.51 - Added in-memory DM ring buffer for inbound and outbound capture and switched dm_history to local cache

README.md

@@ -53,11 +53,11 @@ Skills support context modes (`inject`, `full`, `override`) and per-skill LLM fa
 
 Didactyl will support local inference, which is very privacy preserving. Remote inference does however have it's advantages, and in those cases Didactyl supports using Bitcoin Lightning and eCash inference providers.
 
-## Current Status — v0.0.50
+## Current Status — v0.0.51
 
 **Active build — this project is barely working. Experiment at your own risk.**
 
-> Last release update: v0.0.50 — Rename tools to category-consistent names, add local_* prefixes, merge context tool naming, and keep message_current template-only
+> Last release update: v0.0.51 — Added in-memory DM ring buffer for inbound and outbound capture and switched dm_history to local cache
 
 - Connects to configured relays with auto-reconnect and relay state transition logging
 - Publishes configured startup events per relay as each relay becomes connected

src/agent.c

@@ -1074,55 +1074,6 @@ void agent_append_context_log(const char* sender_pubkey_hex, const char* phase,
     append_context_log(sender_pubkey_hex, phase, context_payload);
 }
 
-typedef struct {
-    time_t created_at;
-    int role_is_user;
-    char* content;
-} agent_history_item_t;
-
-static int extract_first_p_tag_local(cJSON* tags, char out_pubkey_hex[65]) {
-    if (!tags || !cJSON_IsArray(tags) || !out_pubkey_hex) {
-        return -1;
-    }
-
-    int n = cJSON_GetArraySize(tags);
-    for (int i = 0; i < n; i++) {
-        cJSON* tag = cJSON_GetArrayItem(tags, i);
-        if (!tag || !cJSON_IsArray(tag) || cJSON_GetArraySize(tag) < 2) {
-            continue;
-        }
-
-        cJSON* key = cJSON_GetArrayItem(tag, 0);
-        cJSON* val = cJSON_GetArrayItem(tag, 1);
-        if (!key || !val || !cJSON_IsString(key) || !cJSON_IsString(val) || !val->valuestring) {
-            continue;
-        }
-
-        if (strcmp(key->valuestring, "p") == 0 && strlen(val->valuestring) == 64U) {
-            memcpy(out_pubkey_hex, val->valuestring, 64U);
-            out_pubkey_hex[64] = '\0';
-            return 0;
-        }
-    }
-
-    return -1;
-}
-
-static int history_item_cmp_created_at(const void* a, const void* b) {
-    const agent_history_item_t* ia = (const agent_history_item_t*)a;
-    const agent_history_item_t* ib = (const agent_history_item_t*)b;
-    if (ia->created_at < ib->created_at) return -1;
-    if (ia->created_at > ib->created_at) return 1;
-    return 0;
-}
-
-static void free_history_items(agent_history_item_t* items, int count) {
-    if (!items) return;
-    for (int i = 0; i < count; i++) {
-        free(items[i].content);
-    }
-    free(items);
-}
 
 static int append_startup_events_context(cJSON* messages) {
     if (!messages || !g_cfg || g_cfg->startup_event_count <= 0 || !g_cfg->startup_events) {
@@ -1389,142 +1340,65 @@ static int append_recent_admin_dm_history(cJSON* messages, const char* current_m
         return -1;
     }
 
-    unsigned char admin_pubkey[32];
-    if (nostr_hex_to_bytes(g_cfg->admin.pubkey, admin_pubkey, 32) != 0) {
-        return -1;
-    }
-
-    cJSON* filter = cJSON_CreateObject();
-    cJSON* kinds = cJSON_CreateArray();
-    if (!filter || !kinds) {
-        cJSON_Delete(filter);
-        cJSON_Delete(kinds);
-        return -1;
-    }
-
-    cJSON_AddItemToArray(kinds, cJSON_CreateNumber(4));
-    cJSON_AddItemToObject(filter, "kinds", kinds);
-    cJSON_AddNumberToObject(filter, "limit", AGENT_HISTORY_QUERY_LIMIT * 4);
-
-    char* events_json = nostr_handler_query_json(filter, 5000);
-    cJSON_Delete(filter);
-    if (!events_json) {
+    char* history_json = nostr_handler_get_dm_history_json(g_cfg->admin.pubkey, AGENT_HISTORY_TURNS);
+    if (!history_json) {
         return 0;
     }
 
-    cJSON* events = cJSON_Parse(events_json);
-    free(events_json);
-    if (!events || !cJSON_IsArray(events)) {
-        cJSON_Delete(events);
+    cJSON* items = cJSON_Parse(history_json);
+    free(history_json);
+    if (!items || !cJSON_IsArray(items)) {
+        cJSON_Delete(items);
         return 0;
     }
 
-    agent_history_item_t* items = NULL;
-    int item_count = 0;
-
-    int n = cJSON_GetArraySize(events);
-    for (int i = 0; i < n; i++) {
-        cJSON* ev = cJSON_GetArrayItem(events, i);
-        cJSON* kind = ev ? cJSON_GetObjectItemCaseSensitive(ev, "kind") : NULL;
-        cJSON* pubkey = ev ? cJSON_GetObjectItemCaseSensitive(ev, "pubkey") : NULL;
-        cJSON* content = ev ? cJSON_GetObjectItemCaseSensitive(ev, "content") : NULL;
-        cJSON* tags = ev ? cJSON_GetObjectItemCaseSensitive(ev, "tags") : NULL;
-        cJSON* created_at = ev ? cJSON_GetObjectItemCaseSensitive(ev, "created_at") : NULL;
-
-        if (!kind || !pubkey || !content || !tags || !created_at ||
-            !cJSON_IsNumber(kind) || !cJSON_IsString(pubkey) ||
-            !cJSON_IsString(content) || !cJSON_IsArray(tags) || !cJSON_IsNumber(created_at) ||
-            (int)kind->valuedouble != 4 || !pubkey->valuestring || !content->valuestring) {
-            continue;
-        }
-
-        char p_tag_pubkey[65] = {0};
-        if (extract_first_p_tag_local(tags, p_tag_pubkey) != 0) {
-            continue;
-        }
-
-        int role_is_user = 0;
-        if (strcmp(pubkey->valuestring, g_cfg->admin.pubkey) == 0 &&
-            strcmp(p_tag_pubkey, g_cfg->keys.public_key_hex) == 0) {
-            role_is_user = 1;
-        } else if (strcmp(pubkey->valuestring, g_cfg->keys.public_key_hex) == 0 &&
-                   strcmp(p_tag_pubkey, g_cfg->admin.pubkey) == 0) {
-            role_is_user = 0;
-        } else {
-            continue;
-        }
-
-        char* plaintext = (char*)malloc(NOSTR_NIP04_MAX_PLAINTEXT_SIZE);
-        if (!plaintext) {
-            continue;
-        }
-        plaintext[0] = '\0';
-
-        if (nostr_nip04_decrypt(g_cfg->keys.private_key,
-                                admin_pubkey,
-                                content->valuestring,
-                                plaintext,
-                                NOSTR_NIP04_MAX_PLAINTEXT_SIZE) != NOSTR_SUCCESS) {
-            free(plaintext);
-            continue;
-        }
-
-        agent_history_item_t* grown = (agent_history_item_t*)realloc(items, (size_t)(item_count + 1) * sizeof(agent_history_item_t));
-        if (!grown) {
-            free(plaintext);
-            free_history_items(items, item_count);
-            cJSON_Delete(events);
-            return -1;
-        }
-
-        items = grown;
-        items[item_count].created_at = (time_t)created_at->valuedouble;
-        items[item_count].role_is_user = role_is_user;
-        items[item_count].content = plaintext;
-        item_count++;
-    }
-
-    cJSON_Delete(events);
-
-    if (item_count > 1) {
-        qsort(items, (size_t)item_count, sizeof(agent_history_item_t), history_item_cmp_created_at);
-    }
-
-    int start = item_count > AGENT_HISTORY_TURNS ? item_count - AGENT_HISTORY_TURNS : 0;
     const char* last_appended_role = NULL;
     const char* last_appended_content = NULL;
-    for (int i = start; i < item_count; i++) {
-        if (i == item_count - 1 &&
-            items[i].role_is_user &&
-            current_message &&
-            strcmp(items[i].content, current_message) == 0) {
+
+    int n = cJSON_GetArraySize(items);
+    for (int i = 0; i < n; i++) {
+        cJSON* item = cJSON_GetArrayItem(items, i);
+        cJSON* role = item ? cJSON_GetObjectItemCaseSensitive(item, "role") : NULL;
+        cJSON* content = item ? cJSON_GetObjectItemCaseSensitive(item, "content") : NULL;
+        cJSON* created_at = item ? cJSON_GetObjectItemCaseSensitive(item, "created_at") : NULL;
+
+        if (!role || !content || !cJSON_IsString(role) || !cJSON_IsString(content) ||
+            !role->valuestring || !content->valuestring) {
             continue;
         }
 
-        const char* role = items[i].role_is_user ? "user" : "assistant";
-        const char* content = items[i].content ? items[i].content : "";
+        const char* role_s = role->valuestring;
+        const char* content_s = content->valuestring;
+        int role_is_user = (strcmp(role_s, "user") == 0);
+        if (!role_is_user && strcmp(role_s, "assistant") != 0) {
+            continue;
+        }
+
+        if (i == n - 1 && role_is_user && current_message && strcmp(content_s, current_message) == 0) {
+            continue;
+        }
 
         if (last_appended_role && last_appended_content &&
-            strcmp(last_appended_role, role) == 0 &&
-            strcmp(last_appended_content, content) == 0) {
+            strcmp(last_appended_role, role_s) == 0 &&
+            strcmp(last_appended_content, content_s) == 0) {
             continue;
         }
 
-        if (append_simple_message(messages, role, content) != 0) {
-            free_history_items(items, item_count);
+        if (append_simple_message(messages, role_s, content_s) != 0) {
+            cJSON_Delete(items);
             return -1;
         }
 
         cJSON* appended = cJSON_GetArrayItem(messages, cJSON_GetArraySize(messages) - 1);
-        if (appended && cJSON_IsObject(appended)) {
-            cJSON_AddNumberToObject(appended, "_ts", (double)items[i].created_at);
+        if (appended && cJSON_IsObject(appended) && created_at && cJSON_IsNumber(created_at)) {
+            cJSON_AddNumberToObject(appended, "_ts", created_at->valuedouble);
         }
 
-        last_appended_role = role;
-        last_appended_content = content;
+        last_appended_role = role_s;
+        last_appended_content = content_s;
     }
 
-    free_history_items(items, item_count);
+    cJSON_Delete(items);
     return 0;
 }
 

src/main.h

@@ -12,8 +12,8 @@
 // Using DIDACTYL_ prefix to avoid conflicts with nostr_core_lib VERSION macros
 #define DIDACTYL_VERSION_MAJOR 0
 #define DIDACTYL_VERSION_MINOR 0
-#define DIDACTYL_VERSION_PATCH 50
-#define DIDACTYL_VERSION "v0.0.50"
+#define DIDACTYL_VERSION_PATCH 51
+#define DIDACTYL_VERSION "v0.0.51"
 
 // Agent metadata
 #define DIDACTYL_NAME "Didactyl"

src/nostr_handler.c

@@ -55,6 +55,7 @@ static int g_seen_dm_next = 0;
 static pthread_mutex_t g_dm_dedup_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 #define SENDER_PROTOCOL_CACHE_SIZE 128
+#define DM_HISTORY_RING_SIZE 256
 
 typedef struct {
     char pubkey_hex[65];
@@ -62,9 +63,21 @@ typedef struct {
     time_t seen_at;
 } sender_protocol_entry_t;
 
+typedef struct {
+    time_t created_at;
+    int incoming;
+    char peer_pubkey_hex[65];
+    char* content;
+} dm_history_entry_t;
+
 static sender_protocol_entry_t g_sender_protocol_cache[SENDER_PROTOCOL_CACHE_SIZE];
 static pthread_mutex_t g_sender_protocol_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+static dm_history_entry_t g_dm_history_ring[DM_HISTORY_RING_SIZE];
+static int g_dm_history_count = 0;
+static int g_dm_history_next = 0;
+static pthread_mutex_t g_dm_history_mutex = PTHREAD_MUTEX_INITIALIZER;
+
 static int dm_id_seen_or_remember(const char* event_id_hex) {
     if (!event_id_hex || strlen(event_id_hex) != 64U) {
         return 0;
@@ -180,6 +193,49 @@ static dm_protocol_t sender_protocol_lookup(const char* sender_pubkey_hex) {
     return out;
 }
 
+static void dm_history_clear_locked(void) {
+    for (int i = 0; i < DM_HISTORY_RING_SIZE; i++) {
+        free(g_dm_history_ring[i].content);
+        g_dm_history_ring[i].content = NULL;
+        g_dm_history_ring[i].created_at = 0;
+        g_dm_history_ring[i].incoming = 0;
+        g_dm_history_ring[i].peer_pubkey_hex[0] = '\0';
+    }
+    g_dm_history_count = 0;
+    g_dm_history_next = 0;
+}
+
+static void dm_history_remember(const char* peer_pubkey_hex, const char* content, int incoming, time_t created_at) {
+    if (!peer_pubkey_hex || strlen(peer_pubkey_hex) != 64U || !content) {
+        return;
+    }
+
+    char* dup = strdup(content);
+    if (!dup) {
+        return;
+    }
+
+    pthread_mutex_lock(&g_dm_history_mutex);
+
+    int slot = 0;
+    if (g_dm_history_count < DM_HISTORY_RING_SIZE) {
+        slot = g_dm_history_count;
+        g_dm_history_count++;
+    } else {
+        slot = g_dm_history_next;
+        g_dm_history_next = (g_dm_history_next + 1) % DM_HISTORY_RING_SIZE;
+    }
+
+    free(g_dm_history_ring[slot].content);
+    g_dm_history_ring[slot].content = dup;
+    g_dm_history_ring[slot].created_at = (created_at > 0) ? created_at : time(NULL);
+    g_dm_history_ring[slot].incoming = incoming ? 1 : 0;
+    memcpy(g_dm_history_ring[slot].peer_pubkey_hex, peer_pubkey_hex, 64U);
+    g_dm_history_ring[slot].peer_pubkey_hex[64] = '\0';
+
+    pthread_mutex_unlock(&g_dm_history_mutex);
+}
+
 static const char* relay_status_str(nostr_pool_relay_status_t status) {
     switch (status) {
         case NOSTR_POOL_RELAY_DISCONNECTED:
@@ -211,6 +267,8 @@ static int startup_self_kind1_exists(void);
 static void load_startup_display_name(void);
 static void build_startup_kind1_content(char* out, size_t out_size, const char* fallback);
 static void on_self_skill_event(cJSON* event, const char* relay_url, void* user_data);
+static void dm_history_clear_locked(void);
+static void dm_history_remember(const char* peer_pubkey_hex, const char* content, int incoming, time_t created_at);
 
 static void log_relay_state_changes(void) {
     if (!g_pool || !g_cfg || !g_last_relay_statuses) {
@@ -869,6 +927,11 @@ static void on_event(cJSON* event, const char* relay_url, void* user_data) {
                (int)tier,
                received_protocol == DM_PROTOCOL_NIP17 ? "nip17" : "nip04");
 
+    dm_history_remember(sender_pubkey_hex,
+                        decrypted,
+                        1,
+                        time(NULL));
+
     g_dm_callback(sender_pubkey_hex, decrypted, tier, g_dm_user_data);
     free(decrypted);
 }
@@ -1106,6 +1169,10 @@ int nostr_handler_init(didactyl_config_t* config) {
     g_seen_dm_count = 0;
     g_seen_dm_next = 0;
 
+    pthread_mutex_lock(&g_dm_history_mutex);
+    dm_history_clear_locked();
+    pthread_mutex_unlock(&g_dm_history_mutex);
+
     DEBUG_INFO("[didactyl] initializing relay pool with %d relays", g_cfg->relay_count);
 
     nostr_pool_reconnect_config_t reconnect = *nostr_pool_reconnect_config_default();
@@ -1505,6 +1572,13 @@ int nostr_handler_send_dm(const char* recipient_pubkey_hex, const char* message)
                recipient_pubkey_hex,
                sent);
 
+    if (sent > 0) {
+        dm_history_remember(recipient_pubkey_hex,
+                            message,
+                            0,
+                            time(NULL));
+    }
+
     free(connected_relays);
     cJSON_Delete(event);
     return sent > 0 ? 0 : -1;
@@ -2178,9 +2252,80 @@ int nostr_handler_send_dm_nip17(const char* recipient_pubkey_hex, const char* me
         return -1;
     }
 
+    dm_history_remember(recipient_pubkey_hex,
+                        message,
+                        0,
+                        time(NULL));
+
     return 0;
 }
 
+char* nostr_handler_get_dm_history_json(const char* peer_pubkey_hex, int limit) {
+    if (!peer_pubkey_hex || strlen(peer_pubkey_hex) != 64U) {
+        return strdup("[]");
+    }
+
+    int max_take = limit > 0 ? limit : 64;
+    if (max_take > DM_HISTORY_RING_SIZE) {
+        max_take = DM_HISTORY_RING_SIZE;
+    }
+
+    cJSON* arr = cJSON_CreateArray();
+    if (!arr) {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&g_dm_history_mutex);
+
+    int total = g_dm_history_count;
+    int matches = 0;
+    for (int i = 0; i < total; i++) {
+        int idx = (g_dm_history_count < DM_HISTORY_RING_SIZE)
+                  ? i
+                  : (g_dm_history_next + i) % DM_HISTORY_RING_SIZE;
+        dm_history_entry_t* entry = &g_dm_history_ring[idx];
+        if (entry->peer_pubkey_hex[0] == '\0' || !entry->content) {
+            continue;
+        }
+        if (strcmp(entry->peer_pubkey_hex, peer_pubkey_hex) == 0) {
+            matches++;
+        }
+    }
+
+    int skip = matches > max_take ? (matches - max_take) : 0;
+    int seen = 0;
+    for (int i = 0; i < total; i++) {
+        int idx = (g_dm_history_count < DM_HISTORY_RING_SIZE)
+                  ? i
+                  : (g_dm_history_next + i) % DM_HISTORY_RING_SIZE;
+        dm_history_entry_t* entry = &g_dm_history_ring[idx];
+        if (entry->peer_pubkey_hex[0] == '\0' || !entry->content) {
+            continue;
+        }
+        if (strcmp(entry->peer_pubkey_hex, peer_pubkey_hex) != 0) {
+            continue;
+        }
+        if (seen++ < skip) {
+            continue;
+        }
+
+        cJSON* item = cJSON_CreateObject();
+        if (!item) {
+            continue;
+        }
+        cJSON_AddStringToObject(item, "role", entry->incoming ? "user" : "assistant");
+        cJSON_AddStringToObject(item, "content", entry->content);
+        cJSON_AddNumberToObject(item, "created_at", (double)entry->created_at);
+        cJSON_AddItemToArray(arr, item);
+    }
+
+    pthread_mutex_unlock(&g_dm_history_mutex);
+
+    char* out = cJSON_PrintUnformatted(arr);
+    cJSON_Delete(arr);
+    return out ? out : strdup("[]");
+}
+
 char* nostr_handler_get_admin_kind0_context(void) {
     if (!g_cfg || !g_cfg->admin_context.enabled || !g_cfg->admin_context.track_kind_0) {
         return NULL;
@@ -2350,6 +2495,10 @@ void nostr_handler_cleanup(void) {
     memset(g_sender_protocol_cache, 0, sizeof(g_sender_protocol_cache));
     pthread_mutex_unlock(&g_sender_protocol_mutex);
 
+    pthread_mutex_lock(&g_dm_history_mutex);
+    dm_history_clear_locked();
+    pthread_mutex_unlock(&g_dm_history_mutex);
+
     pthread_mutex_lock(&g_admin_ctx_mutex);
     free_admin_context_locked();
     pthread_mutex_unlock(&g_admin_ctx_mutex);

src/nostr_handler.h

@@ -52,6 +52,7 @@ int nostr_handler_is_wot_contact(const char* pubkey_hex);
 char* nostr_handler_relay_status_json(void);
 char* nostr_handler_relay_info_json(const char* relay_url);
 int nostr_handler_send_dm_nip17(const char* recipient_pubkey_hex, const char* message, const char* subject);
+char* nostr_handler_get_dm_history_json(const char* peer_pubkey_hex, int limit);
 void nostr_handler_cleanup(void);
 
 #endif