diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt index adba985212..93c0c360cb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt @@ -46,6 +46,7 @@ import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.longOrNull object MessageKSerializer : KSerializer { override val descriptor: SerialDescriptor = @@ -112,6 +113,10 @@ object MessageKSerializer : KSerializer { is NegErrMessage -> { add(JsonPrimitive(value.subId)) add(JsonPrimitive(value.reason)) + // Only written when there is one: a three-element + // NEG-ERR is what NIP-77 describes, and that is what a + // refusal with nothing to state stays. + value.cap?.let { add(JsonPrimitive(it)) } } } } @@ -184,6 +189,10 @@ object MessageKSerializer : KSerializer { NegErrMessage( subId = array[1].jsonPrimitive.content, reason = if (array.size > 2) array[2].jsonPrimitive.content else "", + // Optional, and only a number: a relay that puts something + // else there is telling us nothing rather than breaking the + // frame. + cap = if (array.size > 3) array[3].jsonPrimitive.longOrNull else null, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt index 66e1675e32..18b4a31925 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt @@ -104,7 +104,13 @@ class NegSessionRegistry( // `null` = matching set exceeds the cap (strfry-parity error). val sealedStorage = store.sealedNegentropyStorage(filters, maxEntries = settings.maxSyncEvents) if (sealedStorage == null) { - send(NegErrMessage(cmd.subId, "blocked: too many query results")) + // The cap rides along with the refusal. A client cannot discover + // this number any other way — NIP-11 has no field for it — so + // without it the only route to a window we WILL answer is guessing, + // halving, one refused NEG-OPEN at a time. Every one of those costs + // us the snapshot scan that produced this rejection, which makes + // stating it cheaper for the relay than staying quiet. + send(NegErrMessage(cmd.subId, "blocked: too many query results", settings.maxSyncEvents.toLong())) return } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt index a81e1ded1f..e191598d97 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt @@ -22,13 +22,64 @@ package com.vitorpamplona.quartz.nip77Negentropy import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message +/** + * `["NEG-ERR", , ]`, optionally followed by the relay's own + * `max_sync_events` when the refusal is about result-set size. + * + * That fourth element is not in NIP-77, but it is the only way a client learns + * the one number that decides how to ask again — no NIP-11 field carries it — + * and it is free for the relay to send, since it must know its own cap to have + * refused. strfry states it in the prose (`… too many records (2431002 > + * 1000000)`); [statedCap] reads either form. + * + * @property cap the fourth wire element, when present. + */ class NegErrMessage( val subId: String, val reason: String, + val cap: Long? = null, ) : Message { override fun label() = LABEL + /** + * The relay's negentropy cap if this refusal states one, from the wire + * field or from the prose, in that order. + * + * Only read for a refusal that is about SIZE ([isOverflow]). A quota or + * rate-limit refusal can carry numbers too, and sizing future windows + * against one of those would shrink every ask against a relay that has no + * size limit at all — while the limit that actually refused does not move + * however small the window gets. + */ + val statedCap: Long? + get() = if (!isOverflow(reason)) null else cap?.takeIf { it > 0 } ?: capInReason(reason) + companion object { const val LABEL = "NEG-ERR" + + /** `(2431002 > 1000000)` — the cap is the right-hand side. */ + private val COMPARISON = Regex("""\(\s*\d+\s*>\s*(\d+)\s*\)""") + + /** + * Does this reason mean "your query matched more than I will + * reconcile"? — as opposed to any other refusal, which no amount of + * window splitting will get past. + */ + fun isOverflow(reason: String): Boolean = + reason.contains("too many records", ignoreCase = true) || + reason.contains("too many results", ignoreCase = true) || + reason.contains("too many query results", ignoreCase = true) || + reason.contains("result set too large", ignoreCase = true) || + reason.contains("results too large", ignoreCase = true) || + reason.contains("max_sync_events", ignoreCase = true) + + /** The cap strfry writes into the refusal text, when it is there. */ + fun capInReason(reason: String): Long? = + COMPARISON + .find(reason) + ?.groupValues + ?.get(1) + ?.toLongOrNull() + ?.takeIf { it > 0 } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt index 896ec7c472..0bd1c1027d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt @@ -32,7 +32,9 @@ package com.vitorpamplona.quartz.nip77Negentropy * unlimited). * @param maxSyncEvents Hard cap on the snapshot size for a single * NEG-OPEN. Mirrors strfry's `relay__negentropy__maxSyncEvents`. - * Overflow returns NEG-ERR `"blocked: too many query results"`. + * Overflow returns NEG-ERR `"blocked: too many query results"` + * carrying this number as its fourth element, so a client can size + * its next window instead of halving its way down to one. * @param maxSessionsPerConnection Cap on concurrent NEG sessions * held by one connection. strfry shares 200 with REQ subs; we * count NEG independently. Overflow sends NOTICE diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt new file mode 100644 index 0000000000..9adafce838 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip77Negentropy + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * A stated cap is acted on — it sizes the next NEG-OPEN — so reading one out of + * a refusal that is not about size is worse than reading none at all: a quota or + * rate limit does not shrink when the window shrinks, so a client that mistook + * one for a cap would shrink its windows forever against a relay that has no + * size limit. + */ +class NegErrMessageTest { + @Test + fun capComesFromTheWireField() { + assertEquals(1_000_000L, NegErrMessage("s", "blocked: too many query results", 1_000_000L).statedCap) + } + + @Test + fun capComesFromStrfrysProseWhenTheFieldIsAbsent() { + val msg = NegErrMessage("s", "blocked: query matches too many records (2431002 > 1000000)") + assertEquals(1_000_000L, msg.statedCap) + } + + @Test + fun theWireFieldWinsOverTheProse() { + val msg = NegErrMessage("s", "blocked: too many records (5 > 10)", 1_000L) + assertEquals(1_000L, msg.statedCap) + } + + @Test + fun anOverflowWithNoNumberStatesNothing() { + assertNull(NegErrMessage("s", "blocked: too many query results").statedCap) + } + + @Test + fun aRateLimitIsNotACapHoweverManyNumbersItCarries() { + assertFalse(NegErrMessage.isOverflow("rate-limited: too many requests (30 > 10)")) + assertNull(NegErrMessage("s", "rate-limited: too many requests (30 > 10)", 10L).statedCap) + } + + @Test + fun refusalsThatAreNotAboutSizeStateNothing() { + listOf( + "auth-required: we only serve negentropy to authenticated users", + "blocked: pubkey is banned", + "error: negentropy disabled", + "closed: unknown subscription handle", + ).forEach { + assertFalse(NegErrMessage.isOverflow(it), "read as an overflow: $it") + assertNull(NegErrMessage("s", it, 42L).statedCap, "read a cap from: $it") + } + } + + @Test + fun theWordingsThatDoMeanOverflow() { + listOf( + "blocked: query matches too many records (5 > 1)", + "blocked: too many query results", + "error: result set too large", + "blocked: results too large", + "blocked: max_sync_events exceeded", + ).forEach { assertTrue(NegErrMessage.isOverflow(it), "not read as an overflow: $it") } + } + + @Test + fun aNonsensicalCapIsRefused() { + // Zero would wedge a client at a window that can never fit. + assertNull(NegErrMessage("s", "blocked: too many query results", 0L).statedCap) + assertNull(NegErrMessage("s", "blocked: too many records (5 > 0)").statedCap) + assertNull(NegErrMessage("s", "blocked: too many query results", -1L).statedCap) + } +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt index 7e45082421..a2d3e10df3 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt @@ -121,10 +121,18 @@ class MessageDeserializer : StdDeserializer(Message::class.java) { } NegErrMessage.LABEL -> { - NegErrMessage( - subId = jp.nextTextValue(), - reason = jp.nextTextValue() ?: "", - ) + val subId = jp.nextTextValue() + val reason = jp.nextTextValue() ?: "" + // The optional fourth element, the relay's own cap. Read by + // stepping one token: anything that is not a number leaves + // the loop below to drain the frame, as before. + val cap = + if (jp.nextToken() == JsonToken.VALUE_NUMBER_INT) { + jp.longValue + } else { + null + } + NegErrMessage(subId, reason, cap) } else -> { diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt index 86274bf1e6..76671a396f 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt @@ -129,6 +129,7 @@ class MessageSerializer : StdSerializer(Message::class.java) { is NegErrMessage -> { gen.writeString(msg.subId) gen.writeString(msg.reason) + msg.cap?.let { gen.writeNumber(it) } } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt index 8b3bce89ba..7a71cc9a1b 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt @@ -124,6 +124,70 @@ class Nip77SerializationTest { assertEquals(msg.reason, jacksonDeserialized.reason) } + @Test + fun serializeNegErrMessageWithCap_matchesJackson() { + val msg = NegErrMessage("neg-sub1", "blocked: too many query results", 1_000_000L) + val jacksonJson = JacksonMapper.toJson(msg) + val kotlinJson = KotlinSerializationMapper.toJson(msg) + + assertEquals(jacksonJson, kotlinJson) + assertEquals("""["NEG-ERR","neg-sub1","blocked: too many query results",1000000]""", kotlinJson) + } + + @Test + fun serializeNegErrMessageWithoutCap_staysThreeElements() { + // NIP-77 describes a three-element NEG-ERR. A refusal with no cap to + // state must stay exactly that, rather than growing a fourth element + // every existing reader then has to tolerate. + val msg = NegErrMessage("neg-sub1", "closed: timeout") + assertEquals("""["NEG-ERR","neg-sub1","closed: timeout"]""", KotlinSerializationMapper.toJson(msg)) + assertEquals("""["NEG-ERR","neg-sub1","closed: timeout"]""", JacksonMapper.toJson(msg)) + } + + @Test + fun deserializeNegErrMessageWithCap_bothMappers() { + val json = """["NEG-ERR","neg-sub1","blocked: too many query results",1000000]""" + + val jackson = JacksonMapper.fromJsonToMessage(json) + assertTrue(jackson is NegErrMessage) + assertEquals(1_000_000L, jackson.cap) + assertEquals(1_000_000L, jackson.statedCap) + + val kotlin = KotlinSerializationMapper.fromJsonToMessage(json) + assertTrue(kotlin is NegErrMessage) + assertEquals(1_000_000L, kotlin.cap) + } + + @Test + fun deserializeNegErrMessageWithGarbageFourthElement_bothMappers() { + // A relay that puts something else there is telling us nothing; it must + // not break the frame that carries the reason. + val json = """["NEG-ERR","neg-sub1","blocked: too many query results","soon"]""" + + val jackson = JacksonMapper.fromJsonToMessage(json) + assertTrue(jackson is NegErrMessage) + assertEquals("blocked: too many query results", jackson.reason) + assertEquals(null, jackson.cap) + + val kotlin = KotlinSerializationMapper.fromJsonToMessage(json) + assertTrue(kotlin is NegErrMessage) + assertEquals("blocked: too many query results", kotlin.reason) + assertEquals(null, kotlin.cap) + } + + @Test + fun negErrMessageWithCap_crossDeserialization() { + val msg = NegErrMessage("neg-sub1", "blocked: too many records", 500_000L) + + val kotlinDeserialized = KotlinSerializationMapper.fromJsonToMessage(JacksonMapper.toJson(msg)) + assertTrue(kotlinDeserialized is NegErrMessage) + assertEquals(500_000L, kotlinDeserialized.cap) + + val jacksonDeserialized = JacksonMapper.fromJsonToMessage(KotlinSerializationMapper.toJson(msg)) + assertTrue(jacksonDeserialized is NegErrMessage) + assertEquals(500_000L, jacksonDeserialized.cap) + } + // ========================================================================= // NEG-OPEN Command (client-to-relay) Tests // =========================================================================