diff --git a/abstract-pool.ts b/abstract-pool.ts index 6fab8ea..9124e11 100644 --- a/abstract-pool.ts +++ b/abstract-pool.ts @@ -11,6 +11,7 @@ import { normalizeURL } from './utils.ts' import type { Event, EventTemplate, Nostr, VerifiedEvent } from './core.ts' import { type Filter } from './filter.ts' import { alwaysTrue } from './helpers.ts' +import { getCountManyFilter, hllDecode, hllEncode, mergeHll, type CountManyDirective } from './nip45.ts' import { Relay } from './relay.ts' export type SubCloser = { close: (reason?: string) => void } @@ -318,6 +319,63 @@ export class AbstractSimplePool { return events[0] || null } + async countMany( + relays: string[], + target: string, + directive: CountManyDirective, + params?: { id?: string | null; maxWait?: number; abort?: AbortSignal }, + ): Promise<{ count: number; hll?: string }> { + const filter = getCountManyFilter(target, directive) + const urls: string[] = [] + + for (let i = 0; i < relays.length; i++) { + const url = normalizeURL(relays[i]) + if (urls.indexOf(url) === -1) urls.push(url) + } + + const responses = await Promise.all( + urls.map(async url => { + if (this.allowConnectingToRelay?.(url, ['read', [filter]]) === false) return null + + let relay: AbstractRelay + try { + relay = await this.ensureRelay(url, { + connectionTimeout: + this.maxWaitForConnection < (params?.maxWait || 0) + ? Math.max(params!.maxWait! * 0.8, params!.maxWait! - 1000) + : this.maxWaitForConnection, + abort: params?.abort, + }) + } catch (err) { + this.onRelayConnectionFailure?.(url) + return null + } + + this.onRelayConnectionSuccess?.(url) + + return relay.countWithHLL([filter], { id: params?.id }).catch(() => null) + }), + ) + + let count = 0 + let hll: Uint8Array | undefined + + for (const response of responses) { + if (!response) continue + + if (response.count > count) count = response.count + + if (!response.hll || response.hll.length !== 512) continue + + const registers = hllDecode(response.hll) + if (!registers) continue + + hll = mergeHll(hll || new Uint8Array(0), registers) + } + + return hll ? { count, hll: hllEncode(hll) } : { count } + } + publish( relays: string[], event: Event, diff --git a/abstract-relay.ts b/abstract-relay.ts index b7fe4f9..f5612fd 100644 --- a/abstract-relay.ts +++ b/abstract-relay.ts @@ -496,7 +496,14 @@ export class AbstractRelay { case 'CLOSED': { const id: string = data[1] const so = this.openSubs.get(id) - if (!so) return + if (!so) { + const cr = this.openCountRequests.get(id) as CountResolver + if (cr) { + cr.reject(new Error(data[2] as string)) + this.openCountRequests.delete(id) + } + return + } so.closed = true so.close(data[2] as string) return diff --git a/nip45.test.ts b/nip45.test.ts new file mode 100644 index 0000000..405d80c --- /dev/null +++ b/nip45.test.ts @@ -0,0 +1,58 @@ +import { expect, test } from 'bun:test' + +import { computeOffset, estimateCount, feedPubkey, hllDecode, hllEncode, mergeHll, newHll } from './nip45.ts' + +const pubkeys = [ + '0000000000000000000000000000000000000000000000000000000000000000', + 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef', + 'abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789', + '0000000000000000010000000000000000000000000000000000000000000000', + '0000000000000000028000000000000000000000000000000000000000000000', + '0000000000000000024000000000000000000000000000000000000000000000', +] + +const goRegisters = + '39390200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001' + +test('matches nostrlib hyperloglog registers and count', () => { + let hll = newHll() + for (const pubkey of pubkeys) { + hll = feedPubkey(hll, pubkey, 8) + } + + expect(hllEncode(hll)).toBe(goRegisters) + expect(estimateCount(hll)).toBe(5) +}) + +test('matches nostrlib hyperloglog merge', () => { + let full = newHll() + let left = newHll() + let right = newHll() + + pubkeys.forEach((pubkey, i) => { + full = feedPubkey(full, pubkey, 8) + if (i % 2 === 0) left = feedPubkey(left, pubkey, 8) + else right = feedPubkey(right, pubkey, 8) + }) + + left = mergeHll(left, right) + + expect(hllEncode(left)).toBe(hllEncode(full)) + expect(hllEncode(left)).toBe(goRegisters) + expect(estimateCount(left)).toBe(5) +}) + +test('matches nostrlib MergeRegisters behavior', () => { + const registers = hllDecode(goRegisters) + expect(registers).toBeDefined() + + const hll = mergeHll(newHll(), registers!) + expect(hllEncode(hll)).toBe(goRegisters) + expect(estimateCount(hll)).toBe(5) +}) + +test('computes NIP-45 offset', () => { + expect(computeOffset('00000000000000000000000000000000f0000000000000000000000000000000')).toBe(23) + expect(computeOffset('30023:00000000000000000000000000000000a0000000000000000000000000000000:test')).toBe(18) +}) diff --git a/nip45.ts b/nip45.ts new file mode 100644 index 0000000..4b441ed --- /dev/null +++ b/nip45.ts @@ -0,0 +1,167 @@ +import { sha256 } from '@noble/hashes/sha2.js' +import { bytesToHex } from '@noble/hashes/utils.js' + +import type { Event } from './core.ts' +import type { Filter } from './filter.ts' + +const M = 256 +const HLL_HEX_LENGTH = M * 2 +const utf8Encoder = new TextEncoder() + +export type CountManyDirective = 'reactions' | 'reposts' | 'quotes' | 'replies' | 'comments' | 'followers' + +export function getCountManyFilter(target: string, directive: CountManyDirective): Filter { + switch (directive) { + case 'reactions': + return { '#e': [target], kinds: [7] } + case 'reposts': + return { '#e': [target], kinds: [6] } + case 'quotes': + return { '#q': [target], kinds: [1, 1111] } + case 'replies': + return { '#e': [target], kinds: [1] } + case 'comments': + return { '#E': [target], kinds: [1111] } + case 'followers': + return { '#p': [target], kinds: [3] } + } +} + +export function newHll(): Uint8Array { + return new Uint8Array(M) +} + +export function hllDecode(hex: string): Uint8Array | undefined { + if (hex.length !== HLL_HEX_LENGTH || !/^[0-9a-fA-F]+$/.test(hex)) return undefined + + const registers = new Uint8Array(M) + for (let i = 0; i < M; i++) { + registers[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + } + return registers +} + +export function hllEncode(registers: Uint8Array): string { + if (registers.length !== M) throw new Error(`invalid number of registers ${registers.length}`) + + let hex = '' + for (let i = 0; i < M; i++) { + hex += registers[i].toString(16).padStart(2, '0') + } + return hex +} + +export function computeOffset(filterFirstTagValue: string): number { + let hex = filterFirstTagValue + + if (!isHex64(hex)) { + const parts = hex.split(':') + if (parts.length === 3 && isHex64(parts[1])) { + hex = parts[1] + } else { + hex = bytesToHex(sha256(utf8Encoder.encode(filterFirstTagValue))) + } + } + + return parseInt(hex[32], 16) + 8 +} + +export function getFilterFirstTagValue(filter: Filter): string | undefined { + for (const key in filter) { + if (key[0] !== '#') continue + + const values = filter[key as `#${string}`] + if (Array.isArray(values) && typeof values[0] === 'string') return values[0] + } + + return undefined +} + +export function feedPubkey(hll: Uint8Array, pubkey: string, offset: number): Uint8Array { + if (offset < 0 || offset > 24) throw new Error(`invalid offset ${offset}`) + if (!isHex64(pubkey)) throw new Error('pubkey must be 32-byte hex') + + if (hll.length === 0) hll = newHll() + if (hll.length !== M) throw new Error(`invalid number of registers ${hll.length}`) + + const ri = parseInt(pubkey.slice(offset * 2, offset * 2 + 2), 16) + const value = countLeadingZeroBitsAfterOffset(pubkey, offset) + 1 + if (value > hll[ri]) hll[ri] = value + + return hll +} + +export function feedEvent(hll: Uint8Array, event: Event, offset: number): Uint8Array { + return feedPubkey(hll, event.pubkey, offset) +} + +export function mergeHll(target: Uint8Array, source: Uint8Array): Uint8Array { + if (target.length === 0) target = newHll() + if (target.length !== M) throw new Error(`invalid number of registers ${target.length}`) + if (source.length !== M) throw new Error(`invalid number of registers ${source.length}`) + + for (let i = 0; i < M; i++) { + if (source[i] > target[i]) target[i] = source[i] + } + + return target +} + +export function estimateCount(hll: Uint8Array): number { + if (hll.length === 0) return 0 + if (hll.length !== M) throw new Error(`invalid number of registers ${hll.length}`) + + const v = countZeros(hll) + if (v !== 0) { + const lc = linearCounting(M, v) + if (lc <= 220) return Math.floor(lc) + } + + const estimate = calculateEstimate(hll) + if (estimate <= M * 3 && v !== 0) return Math.floor(linearCounting(M, v)) + + return Math.floor(estimate) +} + +function isHex64(value: string): boolean { + return /^[0-9a-fA-F]{64}$/.test(value) +} + +function countLeadingZeroBitsAfterOffset(pubkey: string, offset: number): number { + let zeroBits = 0 + for (let i = offset + 1; i < offset + 8; i++) { + const byte = parseInt(pubkey.slice(i * 2, i * 2 + 2), 16) + if (byte === 0) { + zeroBits += 8 + continue + } + + let mask = 0x80 + while ((byte & mask) === 0) { + zeroBits++ + mask >>= 1 + } + break + } + return zeroBits +} + +function countZeros(registers: Uint8Array): number { + let count = 0 + for (let i = 0; i < M; i++) { + if (registers[i] === 0) count++ + } + return count +} + +function linearCounting(m: number, v: number): number { + return m * Math.log(m / v) +} + +function calculateEstimate(registers: Uint8Array): number { + let sum = 0 + for (let i = 0; i < M; i++) { + sum += 1 / 2 ** registers[i] + } + return (0.7182725932495458 * M * M) / sum +} diff --git a/package.json b/package.json index 0d8ebef..b88189a 100644 --- a/package.json +++ b/package.json @@ -198,6 +198,12 @@ "require": "./lib/cjs/nip44.js", "types": "./lib/types/nip44.d.ts" }, + "./nip45": { + "source": "./nip45.ts", + "import": "./lib/esm/nip45.js", + "require": "./lib/cjs/nip45.js", + "types": "./lib/types/nip45.d.ts" + }, "./nip46": { "source": "./nip46.ts", "import": "./lib/esm/nip46.js",